mirror of
https://github.com/langgenius/dify.git
synced 2026-09-24 23:22:26 +08:00
fix(api): centralize remote file retrieval (#36399)
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
This commit is contained in:
@@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Literal, override
|
||||
from configs import dify_config
|
||||
from core.app.file_access import DatabaseFileAccessController, FileAccessControllerProtocol
|
||||
from core.db.session_factory import session_factory
|
||||
from core.helper.ssrf_proxy import graphon_ssrf_proxy
|
||||
from core.file import remote_fetcher
|
||||
from core.tools.signature import sign_tool_file
|
||||
from core.workflow.file_reference import parse_file_reference
|
||||
from extensions.ext_storage import storage
|
||||
@@ -46,7 +46,7 @@ class DifyWorkflowFileRuntime(WorkflowFileRuntimeProtocol):
|
||||
|
||||
@override
|
||||
def http_get(self, url: str, *, follow_redirects: bool = True) -> HttpResponseProtocol:
|
||||
return graphon_ssrf_proxy.get(url, follow_redirects=follow_redirects)
|
||||
return remote_fetcher.graphon_remote_file_fetcher.get(url, follow_redirects=follow_redirects)
|
||||
|
||||
@override
|
||||
def storage_load(self, path: str, *, stream: bool = False) -> bytes | Generator:
|
||||
|
||||
@@ -12,7 +12,7 @@ from uuid import uuid4
|
||||
import httpx
|
||||
|
||||
from configs import dify_config
|
||||
from core.helper import ssrf_proxy
|
||||
from core.file import remote_fetcher
|
||||
from extensions.ext_database import db
|
||||
from extensions.ext_storage import storage
|
||||
from extensions.storage.storage_type import StorageType
|
||||
@@ -44,26 +44,6 @@ class DatasourceFileManager:
|
||||
|
||||
return f"{file_preview_url}?timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}"
|
||||
|
||||
@staticmethod
|
||||
def verify_file(datasource_file_id: str, timestamp: str, nonce: str, sign: str) -> bool:
|
||||
"""
|
||||
verify signature
|
||||
"""
|
||||
data_to_sign = f"file-preview|{datasource_file_id}|{timestamp}|{nonce}"
|
||||
recalculated_sign = hmac.new(
|
||||
dify_config.SECRET_KEY.encode(),
|
||||
data_to_sign.encode(),
|
||||
hashlib.sha256,
|
||||
).digest()
|
||||
recalculated_encoded_sign = base64.urlsafe_b64encode(recalculated_sign).decode()
|
||||
|
||||
# verify signature
|
||||
if sign != recalculated_encoded_sign:
|
||||
return False
|
||||
|
||||
current_time = int(time.time())
|
||||
return current_time - int(timestamp) <= dify_config.FILES_ACCESS_TIMEOUT
|
||||
|
||||
@staticmethod
|
||||
def create_file_by_raw(
|
||||
*,
|
||||
@@ -117,7 +97,7 @@ class DatasourceFileManager:
|
||||
) -> ToolFile:
|
||||
# try to download image
|
||||
try:
|
||||
response = ssrf_proxy.get(file_url)
|
||||
response = remote_fetcher.make_request("GET", file_url)
|
||||
response.raise_for_status()
|
||||
blob = response.content
|
||||
except httpx.TimeoutException:
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
"""File retrieval helpers shared by backend file-oriented workflows."""
|
||||
|
||||
from . import remote_fetcher
|
||||
|
||||
__all__ = ["remote_fetcher"]
|
||||
@@ -0,0 +1,345 @@
|
||||
"""Unified remote-file retrieval with Dify signed file URL resolution.
|
||||
|
||||
Use this module for backend workflows whose intent is to fetch remote file content
|
||||
or remote file metadata from a URL, even when the URL originally came from a user
|
||||
upload, a workflow variable, a tool/datasource file, or an app DSL. GET/HEAD
|
||||
requests can resolve Dify-signed file URLs locally through DB + storage before
|
||||
falling back to the SSRF-protected network client.
|
||||
|
||||
Use `core.helper.ssrf_proxy` directly only for generic outbound HTTP where the
|
||||
URL is not being treated as a remote file, such as HTTP Request nodes, external
|
||||
API integrations, auth discovery, or user-configured tool calls. Those calls must
|
||||
stay as real network requests and should not reinterpret Dify file URLs as stored
|
||||
files.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import re
|
||||
import time
|
||||
import urllib.parse
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Literal
|
||||
|
||||
import httpx
|
||||
|
||||
from configs import dify_config
|
||||
from core.app.file_access import DatabaseFileAccessController
|
||||
from core.db.session_factory import session_factory
|
||||
from core.helper import ssrf_proxy
|
||||
from core.helper.ssrf_proxy import (
|
||||
SSRF_DEFAULT_MAX_RETRIES,
|
||||
_to_graphon_http_response,
|
||||
max_retries_exceeded_error,
|
||||
request_error,
|
||||
)
|
||||
from extensions.ext_storage import storage
|
||||
from models import ToolFile, UploadFile
|
||||
|
||||
_UPLOAD_FILE_PATH_PATTERN = re.compile(
|
||||
r"^/files/(?P<file_id>[a-fA-F0-9-]+)/(?P<preview_kind>file-preview|image-preview)$"
|
||||
)
|
||||
_TOOL_FILE_PATH_PATTERN = re.compile(r"^/files/tools/(?P<file_id>[a-fA-F0-9-]+)(?P<extension>\.[^/]*)?$")
|
||||
_DATASOURCE_FILE_PATH_PATTERN = re.compile(r"^/files/datasources/(?P<file_id>[a-fA-F0-9-]+)(?P<extension>\.[^/]*)?$")
|
||||
|
||||
_file_access_controller = DatabaseFileAccessController()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _SignedFileUrl:
|
||||
file_id: str
|
||||
preview_kind: Literal["file-preview", "image-preview"]
|
||||
record_kind: Literal["upload", "tool", "datasource"]
|
||||
|
||||
|
||||
def make_request(method: str, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> httpx.Response:
|
||||
"""Fetch remote file content or metadata.
|
||||
|
||||
GET and HEAD requests for Dify-owned signed file URLs are served from local
|
||||
storage. Every other request is delegated unchanged to the SSRF proxy.
|
||||
"""
|
||||
|
||||
normalized_method = method.upper()
|
||||
if normalized_method == "GET":
|
||||
response = _resolve_dify_signed_file_url("GET", url)
|
||||
if response is not None:
|
||||
return response
|
||||
if normalized_method == "HEAD":
|
||||
response = _resolve_dify_signed_file_url("HEAD", url)
|
||||
if response is not None:
|
||||
return response
|
||||
return ssrf_proxy.make_request(method=method, url=url, max_retries=max_retries, **kwargs)
|
||||
|
||||
|
||||
class GraphonRemoteFileFetcher:
|
||||
"""Graphon HTTP-client adapter backed by the unified remote-file fetcher.
|
||||
|
||||
Graphon requires method-specific HTTP client methods, while regular Dify
|
||||
call sites should use `make_request` directly.
|
||||
"""
|
||||
|
||||
@property
|
||||
def max_retries_exceeded_error(self) -> type[Exception]:
|
||||
return max_retries_exceeded_error
|
||||
|
||||
@property
|
||||
def request_error(self) -> type[Exception]:
|
||||
return request_error
|
||||
|
||||
def get(self, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any):
|
||||
return _to_graphon_http_response(make_request("GET", url=url, max_retries=max_retries, **kwargs))
|
||||
|
||||
def head(self, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any):
|
||||
return _to_graphon_http_response(make_request("HEAD", url=url, max_retries=max_retries, **kwargs))
|
||||
|
||||
def post(self, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any):
|
||||
return _to_graphon_http_response(make_request("POST", url=url, max_retries=max_retries, **kwargs))
|
||||
|
||||
def put(self, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any):
|
||||
return _to_graphon_http_response(make_request("PUT", url=url, max_retries=max_retries, **kwargs))
|
||||
|
||||
def delete(self, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any):
|
||||
return _to_graphon_http_response(make_request("DELETE", url=url, max_retries=max_retries, **kwargs))
|
||||
|
||||
def patch(self, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any):
|
||||
return _to_graphon_http_response(make_request("PATCH", url=url, max_retries=max_retries, **kwargs))
|
||||
|
||||
|
||||
def _resolve_dify_signed_file_url(method: Literal["GET", "HEAD"], url: str) -> httpx.Response | None:
|
||||
parsed_url = urllib.parse.urlparse(url)
|
||||
if not _is_dify_file_origin(parsed_url):
|
||||
return None
|
||||
|
||||
signed_file_url = _parse_signed_file_path(parsed_url.path)
|
||||
if signed_file_url is None:
|
||||
return None
|
||||
|
||||
query = urllib.parse.parse_qs(parsed_url.query, keep_blank_values=True)
|
||||
timestamp = _single_query_value(query, "timestamp")
|
||||
nonce = _single_query_value(query, "nonce")
|
||||
sign = _single_query_value(query, "sign")
|
||||
if timestamp is None or nonce is None or sign is None:
|
||||
return None
|
||||
|
||||
if not _verify_signed_file_url(
|
||||
signed_file_url=signed_file_url,
|
||||
timestamp=timestamp,
|
||||
nonce=nonce,
|
||||
sign=sign,
|
||||
):
|
||||
return None
|
||||
|
||||
if signed_file_url.record_kind == "upload":
|
||||
return _build_upload_file_response(method=method, url=url, file_id=signed_file_url.file_id)
|
||||
if signed_file_url.record_kind == "tool":
|
||||
return _build_tool_file_response(method=method, url=url, file_id=signed_file_url.file_id)
|
||||
return _build_datasource_file_response(method=method, url=url, file_id=signed_file_url.file_id)
|
||||
|
||||
|
||||
def _parse_signed_file_path(path: str) -> _SignedFileUrl | None:
|
||||
upload_match = _UPLOAD_FILE_PATH_PATTERN.match(path)
|
||||
if upload_match:
|
||||
preview_kind: Literal["file-preview", "image-preview"]
|
||||
if upload_match.group("preview_kind") == "image-preview":
|
||||
preview_kind = "image-preview"
|
||||
else:
|
||||
preview_kind = "file-preview"
|
||||
|
||||
return _SignedFileUrl(
|
||||
file_id=upload_match.group("file_id"),
|
||||
preview_kind=preview_kind,
|
||||
record_kind="upload",
|
||||
)
|
||||
|
||||
tool_match = _TOOL_FILE_PATH_PATTERN.match(path)
|
||||
if tool_match:
|
||||
return _SignedFileUrl(
|
||||
file_id=tool_match.group("file_id"),
|
||||
preview_kind="file-preview",
|
||||
record_kind="tool",
|
||||
)
|
||||
|
||||
datasource_match = _DATASOURCE_FILE_PATH_PATTERN.match(path)
|
||||
if datasource_match:
|
||||
return _SignedFileUrl(
|
||||
file_id=datasource_match.group("file_id"),
|
||||
preview_kind="file-preview",
|
||||
record_kind="datasource",
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _is_dify_file_origin(parsed_url: urllib.parse.ParseResult) -> bool:
|
||||
if parsed_url.scheme not in {"http", "https"} or not parsed_url.hostname:
|
||||
return False
|
||||
|
||||
url_origin = _origin_parts(parsed_url)
|
||||
if url_origin is None:
|
||||
return False
|
||||
|
||||
allowed_origins = {
|
||||
origin
|
||||
for configured_url in [dify_config.FILES_URL, dify_config.INTERNAL_FILES_URL]
|
||||
if configured_url and (origin := _origin_parts(urllib.parse.urlparse(configured_url))) is not None
|
||||
}
|
||||
return url_origin in allowed_origins
|
||||
|
||||
|
||||
def _origin_parts(parsed_url: urllib.parse.ParseResult) -> tuple[str, str, int] | None:
|
||||
if parsed_url.scheme not in {"http", "https"} or not parsed_url.hostname:
|
||||
return None
|
||||
try:
|
||||
port = parsed_url.port
|
||||
except ValueError:
|
||||
return None
|
||||
return parsed_url.scheme, parsed_url.hostname.lower(), port or _default_port(parsed_url.scheme)
|
||||
|
||||
|
||||
def _default_port(scheme: str) -> int:
|
||||
return 443 if scheme == "https" else 80
|
||||
|
||||
|
||||
def _single_query_value(query: dict[str, list[str]], key: str) -> str | None:
|
||||
values = query.get(key)
|
||||
if not values or len(values) != 1:
|
||||
return None
|
||||
return values[0]
|
||||
|
||||
|
||||
def _verify_signed_file_url(
|
||||
*,
|
||||
signed_file_url: _SignedFileUrl,
|
||||
timestamp: str,
|
||||
nonce: str,
|
||||
sign: str,
|
||||
) -> bool:
|
||||
try:
|
||||
current_time = int(time.time())
|
||||
signed_at = int(timestamp)
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
if current_time - signed_at > dify_config.FILES_ACCESS_TIMEOUT:
|
||||
return False
|
||||
|
||||
payload = f"{signed_file_url.preview_kind}|{signed_file_url.file_id}|{timestamp}|{nonce}"
|
||||
recalculated = hmac.new(dify_config.SECRET_KEY.encode(), payload.encode(), hashlib.sha256).digest()
|
||||
expected = base64.urlsafe_b64encode(recalculated).decode()
|
||||
return hmac.compare_digest(sign, expected)
|
||||
|
||||
|
||||
def _build_upload_file_response(*, method: Literal["GET", "HEAD"], url: str, file_id: str) -> httpx.Response:
|
||||
with session_factory.create_session() as session:
|
||||
upload_file = _file_access_controller.get_upload_file(session=session, file_id=file_id)
|
||||
if upload_file is None:
|
||||
return _build_response(method=method, url=url, status_code=404)
|
||||
|
||||
content = b"" if method == "HEAD" else storage.load_once(upload_file.key)
|
||||
return _build_response(
|
||||
method=method,
|
||||
url=url,
|
||||
status_code=200,
|
||||
content=content,
|
||||
content_length=upload_file.size,
|
||||
content_type=upload_file.mime_type,
|
||||
filename=upload_file.name,
|
||||
)
|
||||
|
||||
|
||||
def _build_tool_file_response(*, method: Literal["GET", "HEAD"], url: str, file_id: str) -> httpx.Response:
|
||||
with session_factory.create_session() as session:
|
||||
tool_file = _file_access_controller.get_tool_file(session=session, file_id=file_id)
|
||||
if tool_file is None:
|
||||
return _build_response(method=method, url=url, status_code=404)
|
||||
|
||||
content = b"" if method == "HEAD" else storage.load_once(tool_file.file_key)
|
||||
return _build_response(
|
||||
method=method,
|
||||
url=url,
|
||||
status_code=200,
|
||||
content=content,
|
||||
content_length=tool_file.size,
|
||||
content_type=tool_file.mimetype,
|
||||
filename=tool_file.name,
|
||||
)
|
||||
|
||||
|
||||
def _build_datasource_file_response(*, method: Literal["GET", "HEAD"], url: str, file_id: str) -> httpx.Response:
|
||||
with session_factory.create_session() as session:
|
||||
upload_file = _file_access_controller.get_upload_file(session=session, file_id=file_id)
|
||||
if upload_file is not None:
|
||||
return _build_upload_file_record_response(method=method, url=url, upload_file=upload_file)
|
||||
|
||||
tool_file = _file_access_controller.get_tool_file(session=session, file_id=file_id)
|
||||
if tool_file is not None:
|
||||
return _build_tool_file_record_response(method=method, url=url, tool_file=tool_file)
|
||||
|
||||
return _build_response(method=method, url=url, status_code=404)
|
||||
|
||||
|
||||
def _build_upload_file_record_response(
|
||||
*,
|
||||
method: Literal["GET", "HEAD"],
|
||||
url: str,
|
||||
upload_file: UploadFile,
|
||||
) -> httpx.Response:
|
||||
content = b"" if method == "HEAD" else storage.load_once(upload_file.key)
|
||||
return _build_response(
|
||||
method=method,
|
||||
url=url,
|
||||
status_code=200,
|
||||
content=content,
|
||||
content_length=upload_file.size,
|
||||
content_type=upload_file.mime_type,
|
||||
filename=upload_file.name,
|
||||
)
|
||||
|
||||
|
||||
def _build_tool_file_record_response(
|
||||
*,
|
||||
method: Literal["GET", "HEAD"],
|
||||
url: str,
|
||||
tool_file: ToolFile,
|
||||
) -> httpx.Response:
|
||||
content = b"" if method == "HEAD" else storage.load_once(tool_file.file_key)
|
||||
return _build_response(
|
||||
method=method,
|
||||
url=url,
|
||||
status_code=200,
|
||||
content=content,
|
||||
content_length=tool_file.size,
|
||||
content_type=tool_file.mimetype,
|
||||
filename=tool_file.name,
|
||||
)
|
||||
|
||||
|
||||
def _build_response(
|
||||
*,
|
||||
method: Literal["GET", "HEAD"],
|
||||
url: str,
|
||||
status_code: int,
|
||||
content: bytes = b"",
|
||||
content_length: int | None = None,
|
||||
content_type: str | None = None,
|
||||
filename: str | None = None,
|
||||
) -> httpx.Response:
|
||||
headers: dict[str, str] = {}
|
||||
if content_type:
|
||||
headers["Content-Type"] = content_type
|
||||
if content_length is not None and content_length >= 0:
|
||||
headers["Content-Length"] = str(content_length)
|
||||
if filename:
|
||||
headers["Content-Disposition"] = f"attachment; filename*=UTF-8''{urllib.parse.quote(filename)}"
|
||||
return httpx.Response(
|
||||
status_code=status_code,
|
||||
headers=headers,
|
||||
content=content,
|
||||
request=httpx.Request(method, url),
|
||||
)
|
||||
|
||||
|
||||
graphon_remote_file_fetcher = GraphonRemoteFileFetcher()
|
||||
@@ -1,8 +1,7 @@
|
||||
from core.helper import ssrf_proxy
|
||||
|
||||
|
||||
def download_with_size_limit(url, max_download_size: int, **kwargs):
|
||||
response = ssrf_proxy.get(url, follow_redirects=True, **kwargs)
|
||||
from core.file import remote_fetcher
|
||||
|
||||
response = remote_fetcher.make_request("GET", url, follow_redirects=True, **kwargs)
|
||||
if response.status_code == 404:
|
||||
raise ValueError("file not found")
|
||||
|
||||
|
||||
@@ -1,5 +1,13 @@
|
||||
"""
|
||||
Proxy requests to avoid SSRF
|
||||
"""SSRF-protected HTTP client for generic outbound requests.
|
||||
|
||||
Use this module when the URL represents a normal external HTTP interaction that
|
||||
must go through network/proxy policy exactly as requested, such as HTTP Request
|
||||
nodes, provider/API integrations, auth discovery, or custom tool calls.
|
||||
|
||||
Do not use this directly for "remote file" retrieval. File downloads, probes,
|
||||
and metadata checks should use `core.file.remote_fetcher` instead so Dify-signed
|
||||
file URLs can be resolved through DB + storage before falling back to this SSRF
|
||||
client.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
@@ -5,7 +5,7 @@ from typing import Union
|
||||
from urllib.parse import unquote
|
||||
|
||||
from configs import dify_config
|
||||
from core.helper import ssrf_proxy
|
||||
from core.file import remote_fetcher
|
||||
from core.rag.extractor.csv_extractor import CSVExtractor
|
||||
from core.rag.extractor.entity.datasource_type import DatasourceType
|
||||
from core.rag.extractor.entity.extract_setting import ExtractSetting
|
||||
@@ -55,7 +55,7 @@ class ExtractProcessor:
|
||||
|
||||
@classmethod
|
||||
def load_from_url(cls, url: str, return_text: bool = False) -> Union[list[Document], str]:
|
||||
response = ssrf_proxy.get(url, headers={"User-Agent": USER_AGENT})
|
||||
response = remote_fetcher.make_request("GET", url, headers={"User-Agent": USER_AGENT})
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
suffix = Path(url).suffix
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Word (.docx) document extractor used for RAG ingestion.
|
||||
|
||||
Supports local file paths and remote URLs (downloaded via `core.helper.ssrf_proxy`).
|
||||
Supports local file paths and remote URLs downloaded through the unified remote-file fetcher.
|
||||
"""
|
||||
|
||||
import inspect
|
||||
@@ -17,7 +17,7 @@ from docx.oxml.ns import qn
|
||||
from docx.text.run import Run
|
||||
|
||||
from configs import dify_config
|
||||
from core.helper import ssrf_proxy
|
||||
from core.file import remote_fetcher
|
||||
from core.rag.extractor.extractor_base import BaseExtractor
|
||||
from core.rag.models.document import Document
|
||||
from extensions.ext_database import db
|
||||
@@ -51,7 +51,7 @@ class WordExtractor(BaseExtractor):
|
||||
|
||||
# If the file is a web path, download it to a temporary file, and use that
|
||||
if not os.path.isfile(self.file_path) and self._is_valid_url(self.file_path):
|
||||
response = ssrf_proxy.get(self.file_path)
|
||||
response = remote_fetcher.make_request("GET", self.file_path)
|
||||
|
||||
if response.status_code != 200:
|
||||
response.close()
|
||||
@@ -120,7 +120,7 @@ class WordExtractor(BaseExtractor):
|
||||
if not self._is_valid_url(url):
|
||||
continue
|
||||
try:
|
||||
response = ssrf_proxy.get(url)
|
||||
response = remote_fetcher.make_request("GET", url)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to download image from URL: %s: %s", url, str(e))
|
||||
continue
|
||||
|
||||
@@ -15,7 +15,7 @@ from sqlalchemy import select
|
||||
|
||||
from configs import dify_config
|
||||
from core.entities.knowledge_entities import PreviewDetail
|
||||
from core.helper import ssrf_proxy
|
||||
from core.file import remote_fetcher
|
||||
from core.rag.data_post_processor.data_post_processor import RerankingModelDict
|
||||
from core.rag.extractor.entity.extract_setting import ExtractSetting
|
||||
from core.rag.index_processor.constant.doc_type import DocType
|
||||
@@ -243,7 +243,7 @@ class BaseIndexProcessor(ABC):
|
||||
|
||||
try:
|
||||
# Download with timeout
|
||||
response = ssrf_proxy.get(image_url, timeout=DOWNLOAD_TIMEOUT)
|
||||
response = remote_fetcher.make_request("GET", image_url, timeout=DOWNLOAD_TIMEOUT)
|
||||
response.raise_for_status()
|
||||
|
||||
# Check Content-Length header if available
|
||||
|
||||
@@ -13,7 +13,7 @@ from sqlalchemy import select
|
||||
|
||||
from configs import dify_config
|
||||
from core.db.session_factory import session_factory
|
||||
from core.helper import ssrf_proxy
|
||||
from core.file import remote_fetcher
|
||||
from core.workflow.file_reference import build_file_reference
|
||||
from extensions.ext_storage import storage
|
||||
from graphon.file import File, FileTransferMethod, get_file_type_by_mime_type
|
||||
@@ -60,26 +60,6 @@ class ToolFileManager:
|
||||
|
||||
return f"{file_preview_url}?timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}"
|
||||
|
||||
@staticmethod
|
||||
def verify_file(file_id: str, timestamp: str, nonce: str, sign: str) -> bool:
|
||||
"""
|
||||
verify signature
|
||||
"""
|
||||
data_to_sign = f"file-preview|{file_id}|{timestamp}|{nonce}"
|
||||
recalculated_sign = hmac.new(
|
||||
dify_config.SECRET_KEY.encode(),
|
||||
data_to_sign.encode(),
|
||||
hashlib.sha256,
|
||||
).digest()
|
||||
recalculated_encoded_sign = base64.urlsafe_b64encode(recalculated_sign).decode()
|
||||
|
||||
# verify signature
|
||||
if sign != recalculated_encoded_sign:
|
||||
return False
|
||||
|
||||
current_time = int(time.time())
|
||||
return current_time - int(timestamp) <= dify_config.FILES_ACCESS_TIMEOUT
|
||||
|
||||
def create_file_by_raw(
|
||||
self,
|
||||
*,
|
||||
@@ -129,7 +109,7 @@ class ToolFileManager:
|
||||
) -> ToolFile:
|
||||
# try to download image
|
||||
try:
|
||||
response = ssrf_proxy.get(file_url)
|
||||
response = remote_fetcher.make_request("GET", file_url)
|
||||
response.raise_for_status()
|
||||
blob = response.content
|
||||
except httpx.TimeoutException:
|
||||
|
||||
@@ -9,7 +9,7 @@ import charset_normalizer
|
||||
import cloudscraper
|
||||
from readabilipy import simple_json_from_html_string
|
||||
|
||||
from core.helper import ssrf_proxy
|
||||
from core.file import remote_fetcher
|
||||
from core.rag.extractor import extract_processor
|
||||
from core.rag.extractor.extract_processor import ExtractProcessor
|
||||
|
||||
@@ -38,7 +38,7 @@ def get_url(url: str, user_agent: str | None = None) -> str:
|
||||
|
||||
main_content_type = None
|
||||
supported_content_types = extract_processor.SUPPORT_URL_CONTENT_TYPES + ["text/html"]
|
||||
response = ssrf_proxy.head(url, headers=headers, follow_redirects=True, timeout=(5, 10))
|
||||
response = remote_fetcher.make_request("HEAD", url, headers=headers, follow_redirects=True, timeout=(5, 10))
|
||||
|
||||
if response.status_code == 200:
|
||||
# check content-type
|
||||
@@ -60,10 +60,10 @@ def get_url(url: str, user_agent: str | None = None) -> str:
|
||||
if main_content_type in extract_processor.SUPPORT_URL_CONTENT_TYPES:
|
||||
return cast(str, ExtractProcessor.load_from_url(url, return_text=True))
|
||||
|
||||
response = ssrf_proxy.get(url, headers=headers, follow_redirects=True, timeout=(120, 300))
|
||||
response = remote_fetcher.make_request("GET", url, headers=headers, follow_redirects=True, timeout=(120, 300))
|
||||
elif response.status_code == 403:
|
||||
scraper = cloudscraper.create_scraper()
|
||||
scraper.perform_request = ssrf_proxy.make_request
|
||||
scraper.perform_request = remote_fetcher.make_request
|
||||
response = scraper.get(url, headers=headers, timeout=(120, 300))
|
||||
|
||||
if response.status_code != 200:
|
||||
|
||||
@@ -11,6 +11,7 @@ from configs import dify_config
|
||||
from core.app.entities.app_invoke_entities import DIFY_RUN_CONTEXT_KEY, DifyRunContext
|
||||
from core.app.llm.model_access import build_dify_model_access, fetch_model_config
|
||||
from core.db.session_factory import session_factory
|
||||
from core.file import remote_fetcher
|
||||
from core.helper.code_executor.code_executor import (
|
||||
CodeExecutionError,
|
||||
CodeExecutor,
|
||||
@@ -307,6 +308,7 @@ class DifyNodeFactory(NodeFactory):
|
||||
self._jinja2_template_renderer = CodeExecutorJinja2TemplateRenderer()
|
||||
self._template_transform_max_output_length = dify_config.TEMPLATE_TRANSFORM_MAX_LENGTH
|
||||
self._http_request_http_client = graphon_ssrf_proxy
|
||||
self._remote_file_http_client = remote_fetcher.graphon_remote_file_fetcher
|
||||
self._bound_tool_file_manager_factory = lambda: DifyToolFileManager(
|
||||
self._dify_context,
|
||||
conversation_id_getter=self._conversation_id,
|
||||
@@ -318,7 +320,7 @@ class DifyNodeFactory(NodeFactory):
|
||||
)
|
||||
self._llm_file_saver = build_dify_llm_file_saver(
|
||||
run_context=self._dify_context,
|
||||
http_client=self._http_request_http_client,
|
||||
http_client=self._remote_file_http_client,
|
||||
conversation_id_getter=self._conversation_id,
|
||||
)
|
||||
self._human_input_runtime = DifyHumanInputNodeRuntime(
|
||||
@@ -416,7 +418,7 @@ class DifyNodeFactory(NodeFactory):
|
||||
),
|
||||
BuiltinNodeTypes.DOCUMENT_EXTRACTOR: lambda: {
|
||||
"unstructured_api_config": self._document_extractor_unstructured_api_config,
|
||||
"http_client": self._http_request_http_client,
|
||||
"http_client": self._remote_file_http_client,
|
||||
},
|
||||
BuiltinNodeTypes.QUESTION_CLASSIFIER: lambda: self._build_llm_compatible_node_init_kwargs(
|
||||
node_class=node_class,
|
||||
@@ -530,7 +532,7 @@ class DifyNodeFactory(NodeFactory):
|
||||
if validated_node_data.type == BuiltinNodeTypes.QUESTION_CLASSIFIER:
|
||||
node_init_kwargs["template_renderer"] = self._jinja2_template_renderer
|
||||
if include_http_client:
|
||||
node_init_kwargs["http_client"] = self._http_request_http_client
|
||||
node_init_kwargs["http_client"] = self._remote_file_http_client
|
||||
if include_llm_file_saver:
|
||||
node_init_kwargs["llm_file_saver"] = self._llm_file_saver
|
||||
if include_prompt_message_serializer:
|
||||
|
||||
Reference in New Issue
Block a user