fix(api): centralize remote file retrieval (#36399)

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

api/core/rag/extractor/extract_processor.py

@@ -5,7 +5,7 @@ from typing import Union
 from urllib.parse import unquote
 
 from configs import dify_config
-from core.helper import ssrf_proxy
+from core.file import remote_fetcher
 from core.rag.extractor.csv_extractor import CSVExtractor
 from core.rag.extractor.entity.datasource_type import DatasourceType
 from core.rag.extractor.entity.extract_setting import ExtractSetting
@@ -55,7 +55,7 @@ class ExtractProcessor:
 
     @classmethod
     def load_from_url(cls, url: str, return_text: bool = False) -> Union[list[Document], str]:
-        response = ssrf_proxy.get(url, headers={"User-Agent": USER_AGENT})
+        response = remote_fetcher.make_request("GET", url, headers={"User-Agent": USER_AGENT})
 
         with tempfile.TemporaryDirectory() as temp_dir:
             suffix = Path(url).suffix

api/core/rag/extractor/word_extractor.py

@@ -1,6 +1,6 @@
 """Word (.docx) document extractor used for RAG ingestion.
 
-Supports local file paths and remote URLs (downloaded via `core.helper.ssrf_proxy`).
+Supports local file paths and remote URLs downloaded through the unified remote-file fetcher.
 """
 
 import inspect
@@ -17,7 +17,7 @@ from docx.oxml.ns import qn
 from docx.text.run import Run
 
 from configs import dify_config
-from core.helper import ssrf_proxy
+from core.file import remote_fetcher
 from core.rag.extractor.extractor_base import BaseExtractor
 from core.rag.models.document import Document
 from extensions.ext_database import db
@@ -51,7 +51,7 @@ class WordExtractor(BaseExtractor):
 
         # If the file is a web path, download it to a temporary file, and use that
         if not os.path.isfile(self.file_path) and self._is_valid_url(self.file_path):
-            response = ssrf_proxy.get(self.file_path)
+            response = remote_fetcher.make_request("GET", self.file_path)
 
             if response.status_code != 200:
                 response.close()
@@ -120,7 +120,7 @@ class WordExtractor(BaseExtractor):
                     if not self._is_valid_url(url):
                         continue
                     try:
-                        response = ssrf_proxy.get(url)
+                        response = remote_fetcher.make_request("GET", url)
                     except Exception as e:
                         logger.warning("Failed to download image from URL: %s: %s", url, str(e))
                         continue

api/core/rag/index_processor/index_processor_base.py

@@ -15,7 +15,7 @@ from sqlalchemy import select
 
 from configs import dify_config
 from core.entities.knowledge_entities import PreviewDetail
-from core.helper import ssrf_proxy
+from core.file import remote_fetcher
 from core.rag.data_post_processor.data_post_processor import RerankingModelDict
 from core.rag.extractor.entity.extract_setting import ExtractSetting
 from core.rag.index_processor.constant.doc_type import DocType
@@ -243,7 +243,7 @@ class BaseIndexProcessor(ABC):
 
         try:
             # Download with timeout
-            response = ssrf_proxy.get(image_url, timeout=DOWNLOAD_TIMEOUT)
+            response = remote_fetcher.make_request("GET", image_url, timeout=DOWNLOAD_TIMEOUT)
             response.raise_for_status()
 
             # Check Content-Length header if available