fix(api): centralize remote file retrieval (#36399)

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

api/core/tools/tool_file_manager.py

@@ -13,7 +13,7 @@ from sqlalchemy import select
 
 from configs import dify_config
 from core.db.session_factory import session_factory
-from core.helper import ssrf_proxy
+from core.file import remote_fetcher
 from core.workflow.file_reference import build_file_reference
 from extensions.ext_storage import storage
 from graphon.file import File, FileTransferMethod, get_file_type_by_mime_type
@@ -60,26 +60,6 @@ class ToolFileManager:
 
         return f"{file_preview_url}?timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}"
 
-    @staticmethod
-    def verify_file(file_id: str, timestamp: str, nonce: str, sign: str) -> bool:
-        """
-        verify signature
-        """
-        data_to_sign = f"file-preview|{file_id}|{timestamp}|{nonce}"
-        recalculated_sign = hmac.new(
-            dify_config.SECRET_KEY.encode(),
-            data_to_sign.encode(),
-            hashlib.sha256,
-        ).digest()
-        recalculated_encoded_sign = base64.urlsafe_b64encode(recalculated_sign).decode()
-
-        # verify signature
-        if sign != recalculated_encoded_sign:
-            return False
-
-        current_time = int(time.time())
-        return current_time - int(timestamp) <= dify_config.FILES_ACCESS_TIMEOUT
-
     def create_file_by_raw(
         self,
         *,
@@ -129,7 +109,7 @@ class ToolFileManager:
     ) -> ToolFile:
         # try to download image
         try:
-            response = ssrf_proxy.get(file_url)
+            response = remote_fetcher.make_request("GET", file_url)
             response.raise_for_status()
             blob = response.content
         except httpx.TimeoutException:

api/core/tools/utils/web_reader_tool.py

@@ -9,7 +9,7 @@ import charset_normalizer
 import cloudscraper
 from readabilipy import simple_json_from_html_string
 
-from core.helper import ssrf_proxy
+from core.file import remote_fetcher
 from core.rag.extractor import extract_processor
 from core.rag.extractor.extract_processor import ExtractProcessor
 
@@ -38,7 +38,7 @@ def get_url(url: str, user_agent: str | None = None) -> str:
 
     main_content_type = None
     supported_content_types = extract_processor.SUPPORT_URL_CONTENT_TYPES + ["text/html"]
-    response = ssrf_proxy.head(url, headers=headers, follow_redirects=True, timeout=(5, 10))
+    response = remote_fetcher.make_request("HEAD", url, headers=headers, follow_redirects=True, timeout=(5, 10))
 
     if response.status_code == 200:
         # check content-type
@@ -60,10 +60,10 @@ def get_url(url: str, user_agent: str | None = None) -> str:
         if main_content_type in extract_processor.SUPPORT_URL_CONTENT_TYPES:
             return cast(str, ExtractProcessor.load_from_url(url, return_text=True))
 
-        response = ssrf_proxy.get(url, headers=headers, follow_redirects=True, timeout=(120, 300))
+        response = remote_fetcher.make_request("GET", url, headers=headers, follow_redirects=True, timeout=(120, 300))
     elif response.status_code == 403:
         scraper = cloudscraper.create_scraper()
-        scraper.perform_request = ssrf_proxy.make_request
+        scraper.perform_request = remote_fetcher.make_request
         response = scraper.get(url, headers=headers, timeout=(120, 300))
 
     if response.status_code != 200: