diff --git a/.env.example b/.env.example index 028d6eddb..b569cf28c 100644 --- a/.env.example +++ b/.env.example @@ -106,6 +106,8 @@ MINIO_CONSOLE_PORT=9001 # COS_ENABLE_OLD_DOMAIN=true 表示启用旧的域名格式,默认为 true COS_ENABLE_OLD_DOMAIN=true +# 如果解析网络连接使用Web代理,需要配置以下参数 +# WEB_PROXY=your_web_proxy ############################################################## diff --git a/docker-compose.yml b/docker-compose.yml index 8edce0d61..2a4ce916b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -128,6 +128,7 @@ services: - MINIO_SECRET_ACCESS_KEY=${MINIO_SECRET_ACCESS_KEY:-minioadmin} - MINIO_BUCKET_NAME=${MINIO_BUCKET_NAME} - MINIO_USE_SSL=${MINIO_USE_SSL} + - WEB_PROXY=${WEB_PROXY} networks: - WeKnora-network restart: unless-stopped diff --git a/services/docreader/src/parser/web_parser.py b/services/docreader/src/parser/web_parser.py index 44c42abf9..44c883a75 100644 --- a/services/docreader/src/parser/web_parser.py +++ b/services/docreader/src/parser/web_parser.py @@ -1,4 +1,5 @@ from typing import Any, Optional, Tuple, Dict, Union +import os from playwright.async_api import async_playwright from bs4 import BeautifulSoup @@ -14,6 +15,7 @@ class WebParser(BaseParser): def __init__(self, title: str, **kwargs): self.title = title + self.proxy = os.environ.get("WEB_PROXY", "") super().__init__(file_name=title, **kwargs) logger.info(f"Initialized WebParser with title: {title}") @@ -21,11 +23,11 @@ class WebParser(BaseParser): logger.info(f"Starting web page scraping for URL: {url}") try: async with async_playwright() as p: - args = [ - # "--single-process", - ] + kwargs = {} + if self.proxy: + kwargs["proxy"] = {"server": self.proxy} logger.info("Launching WebKit browser") - browser = await p.webkit.launch(args=args) + browser = await p.webkit.launch(**kwargs) page = await browser.new_page() logger.info(f"Navigating to URL: {url}") @@ -39,15 +41,6 @@ class WebParser(BaseParser): "", "html.parser" ) # Return empty soup on navigation error - try: - logger.info("Waiting for network idle") - await page.reload(wait_until="networkidle") - logger.info("Page reload complete") - except Exception as e: - logger.warning( - f"Network idle wait failed: {str(e)}, proceeding with current page content" - ) - logger.info("Retrieving page HTML content") content = await page.content() logger.info(f"Retrieved {len(content)} bytes of HTML content") @@ -117,6 +110,10 @@ class WebParser(BaseParser): logger.info("No title found, using default") logger.info(f"Web page title: {title}") + text = "\n".join( + (line.strip() for line in text.splitlines() if line.strip()) + ) + result = title + "\n\n" + text logger.info( f"Web page parsing complete, total content: {len(result)} characters"