diff --git a/.env.example b/.env.example index b569cf28c..3602409b5 100644 --- a/.env.example +++ b/.env.example @@ -61,6 +61,9 @@ MINIO_PORT=9000 MINIO_CONSOLE_PORT=9001 +# Embedding并发数，出现429错误时，可调小此参数 +CONCURRENCY_POOL_SIZE=5 + # 如果使用ElasticSearch作为向量存储，需要配置以下参数 # ElasticSearch地址，例如 http://localhost:9200 # ELASTICSEARCH_ADDR=your_elasticsearch_addr diff --git a/docker-compose.yml b/docker-compose.yml index 2a4ce916b..449343e3e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,6 +44,7 @@ services: - REDIS_PREFIX=${REDIS_PREFIX} - ENABLE_GRAPH_RAG=${ENABLE_GRAPH_RAG} - TENANT_AES_KEY=${TENANT_AES_KEY} + - CONCURRENCY_POOL_SIZE=${CONCURRENCY_POOL_SIZE:-5} - INIT_LLM_MODEL_NAME=${INIT_LLM_MODEL_NAME} - INIT_LLM_MODEL_BASE_URL=${INIT_LLM_MODEL_BASE_URL} - INIT_LLM_MODEL_API_KEY=${INIT_LLM_MODEL_API_KEY} diff --git a/docker/Dockerfile.docreader b/docker/Dockerfile.docreader index 690a048aa..e9c2faeed 100644 --- a/docker/Dockerfile.docreader +++ b/docker/Dockerfile.docreader @@ -91,25 +91,26 @@ RUN apt-get update && apt-get install -y \ libcups2 \ libglu1-mesa \ libsm6 \ + libreoffice \ && rm -rf /var/lib/apt/lists/* -# 下载并安装 LibreOffice（区分架构） -RUN mkdir -p /tmp/libreoffice && cd /tmp/libreoffice && \ - if [ "$(uname -m)" = "x86_64" ]; then \ - wget https://mirrors.tuna.tsinghua.edu.cn/libreoffice/libreoffice/stable/25.2.5/deb/x86_64/LibreOffice_25.2.5_Linux_x86-64_deb.tar.gz && \ - tar -xzf LibreOffice_25.2.5_Linux_x86-64_deb.tar.gz && \ - cd LibreOffice_25.2.5.2_Linux_x86-64_deb/DEBS && dpkg -i *.deb; \ - elif [ "$(uname -m)" = "aarch64" ] || [ "$(uname -m)" = "arm64" ]; then \ - wget https://mirrors.aliyun.com/libreoffice/testing/25.8.0/deb/aarch64/LibreOffice_25.8.0.3_Linux_aarch64_deb.tar.gz && \ - tar -xzf LibreOffice_25.8.0.3_Linux_aarch64_deb.tar.gz && \ - cd LibreOffice_25.8.0.3_Linux_aarch64_deb/DEBS && dpkg -i *.deb; \ - else \ - echo "Unsupported architecture: $(uname -m)" && exit 1; \ - fi && \ - cd / && rm -rf /tmp/libreoffice +# # 下载并安装 LibreOffice（区分架构） +# RUN mkdir -p /tmp/libreoffice && cd /tmp/libreoffice && \ +# if [ "$(uname -m)" = "x86_64" ]; then \ +# wget https://mirrors.tuna.tsinghua.edu.cn/libreoffice/libreoffice/stable/25.2.5/deb/x86_64/LibreOffice_25.2.5_Linux_x86-64_deb.tar.gz && \ +# tar -xzf LibreOffice_25.2.5_Linux_x86-64_deb.tar.gz && \ +# cd LibreOffice_25.2.5.2_Linux_x86-64_deb/DEBS && dpkg -i *.deb; \ +# elif [ "$(uname -m)" = "aarch64" ] || [ "$(uname -m)" = "arm64" ]; then \ +# wget https://mirrors.aliyun.com/libreoffice/testing/25.8.0/deb/aarch64/LibreOffice_25.8.0.3_Linux_aarch64_deb.tar.gz && \ +# tar -xzf LibreOffice_25.8.0.3_Linux_aarch64_deb.tar.gz && \ +# cd LibreOffice_25.8.0.3_Linux_aarch64_deb/DEBS && dpkg -i *.deb; \ +# else \ +# echo "Unsupported architecture: $(uname -m)" && exit 1; \ +# fi && \ +# cd / && rm -rf /tmp/libreoffice # 设置 LibreOffice 环境变量 -RUN echo 'export LIBREOFFICE_PATH=/opt/libreoffice25.2/program/soffice' >> /etc/environment; +# RUN echo 'export LIBREOFFICE_PATH=/opt/libreoffice25.2/program/soffice' >> /etc/environment; # 从构建阶段复制已安装的依赖和生成的代码 COPY --from=builder /usr/local/lib/python3.10/site-packages /usr/local/lib/python3.10/site-packages diff --git a/internal/container/container.go b/internal/container/container.go index 614ff7bbe..2afa8f190 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -8,6 +8,7 @@ import ( "fmt" "os" "slices" + "strconv" "strings" "time" @@ -318,10 +319,17 @@ func initRetrieveEngineRegistry(db *gorm.DB, cfg *config.Config) (interfaces.Ret // - Configured goroutine pool // - Error if initialization fails func initAntsPool(cfg *config.Config) (*ants.Pool, error) { - // Default to 50 if not specified in config - poolSize := 300 + // Default to 5 if not specified in config + poolSize := os.Getenv("CONCURRENCY_POOL_SIZE") + if poolSize == "" { + poolSize = "5" + } + poolSizeInt, err := strconv.Atoi(poolSize) + if err != nil { + return nil, err + } // Set up the pool with pre-allocation for better performance - return ants.NewPool(poolSize, ants.WithPreAlloc(true)) + return ants.NewPool(poolSizeInt, ants.WithPreAlloc(true)) } // registerPoolCleanup registers the goroutine pool for cleanup diff --git a/services/docreader/src/parser/storage.py b/services/docreader/src/parser/storage.py index 433a39306..33cb9a21c 100644 --- a/services/docreader/src/parser/storage.py +++ b/services/docreader/src/parser/storage.py @@ -203,23 +203,20 @@ class MinioStorage(Storage): prefer those values to override envs. """ try: - if self.storage_config and self.storage_config.get("endpoint") and self.storage_config.get("bucket_name"): + endpoint = os.getenv("MINIO_ENDPOINT") + use_ssl = os.getenv("MINIO_USE_SSL", "false").lower() == "true" + if self.storage_config and self.storage_config.get("bucket_name"): storage_config = self.storage_config - # Direct fields - endpoint = storage_config.get("endpoint") bucket_name = storage_config.get("bucket_name") path_prefix = storage_config.get("path_prefix").strip().strip("/") access_key = storage_config.get("access_key_id") secret_key = storage_config.get("secret_access_key") else: - endpoint = os.getenv("MINIO_ENDPOINT") access_key = os.getenv("MINIO_ACCESS_KEY_ID") secret_key = os.getenv("MINIO_SECRET_ACCESS_KEY") bucket_name = os.getenv("MINIO_BUCKET_NAME") - use_ssl = os.getenv("MINIO_USE_SSL", "false").lower() == "true" path_prefix = os.getenv("MINIO_PATH_PREFIX", "").strip().strip("/") - if not all([endpoint, access_key, secret_key, bucket_name]): logger.error("Incomplete MinIO configuration, missing required environment variables") return None, None, None, None, None