mirror of
https://github.com/langgenius/dify.git
synced 2026-09-24 23:22:26 +08:00
refactor(storage): remove signer, using general file storage
- Removed unused app asset download and upload endpoints, along with sandbox archive and file download endpoints. - Updated imports in the file controller to reflect the removal of these endpoints. - Simplified the generator.py file by consolidating the code context field definition. - Enhanced the storage layer with a unified presign wrapper for better handling of presigned URLs.
This commit is contained in:
Harry
@@ -70,9 +70,7 @@ class ContextGeneratePayload(BaseModel):
|
||||
model_config_data: dict[str, Any] = Field(..., alias="model_config", description="Model configuration")
|
||||
available_vars: list[AvailableVarPayload] = Field(..., description="Available variables from upstream nodes")
|
||||
parameter_info: ParameterInfoPayload = Field(..., description="Target parameter metadata from the frontend")
|
||||
code_context: CodeContextPayload = Field(
|
||||
description="Existing code node context for incremental generation"
|
||||
)
|
||||
code_context: CodeContextPayload = Field(description="Existing code node context for incremental generation")
|
||||
|
||||
|
||||
class SuggestedQuestionsPayload(BaseModel):
|
||||
|
||||
@@ -15,12 +15,8 @@ api = ExternalApi(
|
||||
files_ns = Namespace("files", description="File operations", path="/")
|
||||
|
||||
from . import (
|
||||
app_assets_download,
|
||||
app_assets_upload,
|
||||
image_preview,
|
||||
sandbox_archive,
|
||||
sandbox_file_downloads,
|
||||
storage_download,
|
||||
storage_proxy,
|
||||
tool_files,
|
||||
upload,
|
||||
)
|
||||
@@ -29,14 +25,10 @@ api.add_namespace(files_ns)
|
||||
|
||||
__all__ = [
|
||||
"api",
|
||||
"app_assets_download",
|
||||
"app_assets_upload",
|
||||
"bp",
|
||||
"files_ns",
|
||||
"image_preview",
|
||||
"sandbox_archive",
|
||||
"sandbox_file_downloads",
|
||||
"storage_download",
|
||||
"storage_proxy",
|
||||
"tool_files",
|
||||
"upload",
|
||||
]
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
from urllib.parse import quote
|
||||
|
||||
from flask import Response, request
|
||||
from flask_restx import Resource
|
||||
from pydantic import BaseModel, Field
|
||||
from werkzeug.exceptions import Forbidden, NotFound
|
||||
|
||||
from controllers.files import files_ns
|
||||
from core.app_assets.storage import AppAssetSigner, AssetPath
|
||||
from extensions.ext_storage import storage
|
||||
|
||||
DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}"
|
||||
|
||||
|
||||
class AppAssetDownloadQuery(BaseModel):
|
||||
expires_at: int = Field(..., description="Unix timestamp when the link expires")
|
||||
nonce: str = Field(..., description="Random string for signature")
|
||||
sign: str = Field(..., description="HMAC signature")
|
||||
|
||||
|
||||
files_ns.schema_model(
|
||||
AppAssetDownloadQuery.__name__,
|
||||
AppAssetDownloadQuery.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0),
|
||||
)
|
||||
|
||||
|
||||
@files_ns.route("/app-assets/<string:asset_type>/<string:tenant_id>/<string:app_id>/<string:resource_id>/download")
|
||||
@files_ns.route(
|
||||
"/app-assets/<string:asset_type>/<string:tenant_id>/<string:app_id>/<string:resource_id>/<string:sub_resource_id>/download"
|
||||
)
|
||||
class AppAssetDownloadApi(Resource):
|
||||
def get(
|
||||
self,
|
||||
asset_type: str,
|
||||
tenant_id: str,
|
||||
app_id: str,
|
||||
resource_id: str,
|
||||
sub_resource_id: str | None = None,
|
||||
):
|
||||
args = AppAssetDownloadQuery.model_validate(request.args.to_dict(flat=True))
|
||||
|
||||
try:
|
||||
asset_path = AssetPath.from_components(
|
||||
asset_type=asset_type,
|
||||
tenant_id=tenant_id,
|
||||
app_id=app_id,
|
||||
resource_id=resource_id,
|
||||
sub_resource_id=sub_resource_id,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise Forbidden(str(exc)) from exc
|
||||
|
||||
if not AppAssetSigner.verify_download_signature(
|
||||
asset_path=asset_path,
|
||||
expires_at=args.expires_at,
|
||||
nonce=args.nonce,
|
||||
sign=args.sign,
|
||||
):
|
||||
raise Forbidden("Invalid or expired download link")
|
||||
|
||||
storage_key = asset_path.get_storage_key()
|
||||
|
||||
try:
|
||||
generator = storage.load_stream(storage_key)
|
||||
except FileNotFoundError as exc:
|
||||
raise NotFound("File not found") from exc
|
||||
|
||||
encoded_filename = quote(storage_key.split("/")[-1])
|
||||
|
||||
return Response(
|
||||
generator,
|
||||
mimetype="application/octet-stream",
|
||||
direct_passthrough=True,
|
||||
headers={
|
||||
"Content-Disposition": f"attachment; filename*=UTF-8''{encoded_filename}",
|
||||
},
|
||||
)
|
||||
@@ -1,61 +0,0 @@
|
||||
from flask import Response, request
|
||||
from flask_restx import Resource
|
||||
from pydantic import BaseModel, Field
|
||||
from werkzeug.exceptions import Forbidden
|
||||
|
||||
from controllers.files import files_ns
|
||||
from core.app_assets.storage import AppAssetSigner, AssetPath
|
||||
from services.app_asset_service import AppAssetService
|
||||
|
||||
DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}"
|
||||
|
||||
|
||||
class AppAssetUploadQuery(BaseModel):
|
||||
expires_at: int = Field(..., description="Unix timestamp when the link expires")
|
||||
nonce: str = Field(..., description="Random string for signature")
|
||||
sign: str = Field(..., description="HMAC signature")
|
||||
|
||||
|
||||
files_ns.schema_model(
|
||||
AppAssetUploadQuery.__name__,
|
||||
AppAssetUploadQuery.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0),
|
||||
)
|
||||
|
||||
|
||||
@files_ns.route("/app-assets/<string:asset_type>/<string:tenant_id>/<string:app_id>/<string:resource_id>/upload")
|
||||
@files_ns.route(
|
||||
"/app-assets/<string:asset_type>/<string:tenant_id>/<string:app_id>/<string:resource_id>/<string:sub_resource_id>/upload"
|
||||
)
|
||||
class AppAssetUploadApi(Resource):
|
||||
def put(
|
||||
self,
|
||||
asset_type: str,
|
||||
tenant_id: str,
|
||||
app_id: str,
|
||||
resource_id: str,
|
||||
sub_resource_id: str | None = None,
|
||||
):
|
||||
args = AppAssetUploadQuery.model_validate(request.args.to_dict(flat=True))
|
||||
|
||||
try:
|
||||
asset_path = AssetPath.from_components(
|
||||
asset_type=asset_type,
|
||||
tenant_id=tenant_id,
|
||||
app_id=app_id,
|
||||
resource_id=resource_id,
|
||||
sub_resource_id=sub_resource_id,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise Forbidden(str(exc)) from exc
|
||||
|
||||
if not AppAssetSigner.verify_upload_signature(
|
||||
asset_path=asset_path,
|
||||
expires_at=args.expires_at,
|
||||
nonce=args.nonce,
|
||||
sign=args.sign,
|
||||
):
|
||||
raise Forbidden("Invalid or expired upload link")
|
||||
|
||||
content = request.get_data()
|
||||
AppAssetService.get_storage().save(asset_path, content)
|
||||
return Response(status=204)
|
||||
@@ -1,76 +0,0 @@
|
||||
from uuid import UUID
|
||||
|
||||
from flask import Response, request
|
||||
from flask_restx import Resource
|
||||
from pydantic import BaseModel, Field
|
||||
from werkzeug.exceptions import Forbidden, NotFound
|
||||
|
||||
from controllers.files import files_ns
|
||||
from core.sandbox.security.archive_signer import SandboxArchivePath, SandboxArchiveSigner
|
||||
from extensions.ext_storage import storage
|
||||
|
||||
DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}"
|
||||
|
||||
|
||||
class SandboxArchiveQuery(BaseModel):
|
||||
expires_at: int = Field(..., description="Unix timestamp when the link expires")
|
||||
nonce: str = Field(..., description="Random string for signature")
|
||||
sign: str = Field(..., description="HMAC signature")
|
||||
|
||||
|
||||
files_ns.schema_model(
|
||||
SandboxArchiveQuery.__name__,
|
||||
SandboxArchiveQuery.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0),
|
||||
)
|
||||
|
||||
|
||||
@files_ns.route("/sandbox-archives/<string:tenant_id>/<string:sandbox_id>/download")
|
||||
class SandboxArchiveDownloadApi(Resource):
|
||||
def get(self, tenant_id: str, sandbox_id: str):
|
||||
args = SandboxArchiveQuery.model_validate(request.args.to_dict(flat=True))
|
||||
|
||||
try:
|
||||
archive_path = SandboxArchivePath(tenant_id=UUID(tenant_id), sandbox_id=UUID(sandbox_id))
|
||||
except ValueError as exc:
|
||||
raise Forbidden(str(exc)) from exc
|
||||
|
||||
if not SandboxArchiveSigner.verify_download_signature(
|
||||
archive_path=archive_path,
|
||||
expires_at=args.expires_at,
|
||||
nonce=args.nonce,
|
||||
sign=args.sign,
|
||||
):
|
||||
raise Forbidden("Invalid or expired download link")
|
||||
|
||||
try:
|
||||
generator = storage.load_stream(archive_path.get_storage_key())
|
||||
except FileNotFoundError as exc:
|
||||
raise NotFound("Archive not found") from exc
|
||||
|
||||
return Response(
|
||||
generator,
|
||||
mimetype="application/gzip",
|
||||
direct_passthrough=True,
|
||||
)
|
||||
|
||||
|
||||
@files_ns.route("/sandbox-archives/<string:tenant_id>/<string:sandbox_id>/upload")
|
||||
class SandboxArchiveUploadApi(Resource):
|
||||
def put(self, tenant_id: str, sandbox_id: str):
|
||||
args = SandboxArchiveQuery.model_validate(request.args.to_dict(flat=True))
|
||||
|
||||
try:
|
||||
archive_path = SandboxArchivePath(tenant_id=UUID(tenant_id), sandbox_id=UUID(sandbox_id))
|
||||
except ValueError as exc:
|
||||
raise Forbidden(str(exc)) from exc
|
||||
|
||||
if not SandboxArchiveSigner.verify_upload_signature(
|
||||
archive_path=archive_path,
|
||||
expires_at=args.expires_at,
|
||||
nonce=args.nonce,
|
||||
sign=args.sign,
|
||||
):
|
||||
raise Forbidden("Invalid or expired upload link")
|
||||
|
||||
storage.save(archive_path.get_storage_key(), request.get_data())
|
||||
return Response(status=204)
|
||||
@@ -1,96 +0,0 @@
|
||||
from urllib.parse import quote
|
||||
from uuid import UUID
|
||||
|
||||
from flask import Response, request
|
||||
from flask_restx import Resource
|
||||
from pydantic import BaseModel, Field
|
||||
from werkzeug.exceptions import Forbidden, NotFound
|
||||
|
||||
from controllers.files import files_ns
|
||||
from core.sandbox.security.sandbox_file_signer import SandboxFileDownloadPath, SandboxFileSigner
|
||||
from extensions.ext_storage import storage
|
||||
|
||||
DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}"
|
||||
|
||||
|
||||
class SandboxFileDownloadQuery(BaseModel):
|
||||
expires_at: int = Field(..., description="Unix timestamp when the link expires")
|
||||
nonce: str = Field(..., description="Random string for signature")
|
||||
sign: str = Field(..., description="HMAC signature")
|
||||
|
||||
|
||||
files_ns.schema_model(
|
||||
SandboxFileDownloadQuery.__name__,
|
||||
SandboxFileDownloadQuery.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0),
|
||||
)
|
||||
|
||||
|
||||
@files_ns.route(
|
||||
"/sandbox-file-downloads/<string:tenant_id>/<string:sandbox_id>/<string:export_id>/<path:filename>/download"
|
||||
)
|
||||
class SandboxFileDownloadDownloadApi(Resource):
|
||||
def get(self, tenant_id: str, sandbox_id: str, export_id: str, filename: str):
|
||||
args = SandboxFileDownloadQuery.model_validate(request.args.to_dict(flat=True))
|
||||
|
||||
try:
|
||||
export_path = SandboxFileDownloadPath(
|
||||
tenant_id=UUID(tenant_id),
|
||||
sandbox_id=UUID(sandbox_id),
|
||||
export_id=export_id,
|
||||
filename=filename,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise Forbidden(str(exc)) from exc
|
||||
|
||||
if not SandboxFileSigner.verify_download_signature(
|
||||
export_path=export_path,
|
||||
expires_at=args.expires_at,
|
||||
nonce=args.nonce,
|
||||
sign=args.sign,
|
||||
):
|
||||
raise Forbidden("Invalid or expired download link")
|
||||
|
||||
try:
|
||||
generator = storage.load_stream(export_path.get_storage_key())
|
||||
except FileNotFoundError as exc:
|
||||
raise NotFound("File not found") from exc
|
||||
|
||||
encoded_filename = quote(filename.split("/")[-1])
|
||||
|
||||
return Response(
|
||||
generator,
|
||||
mimetype="application/octet-stream",
|
||||
direct_passthrough=True,
|
||||
headers={
|
||||
"Content-Disposition": f"attachment; filename*=UTF-8''{encoded_filename}",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@files_ns.route(
|
||||
"/sandbox-file-downloads/<string:tenant_id>/<string:sandbox_id>/<string:export_id>/<path:filename>/upload"
|
||||
)
|
||||
class SandboxFileDownloadUploadApi(Resource):
|
||||
def put(self, tenant_id: str, sandbox_id: str, export_id: str, filename: str):
|
||||
args = SandboxFileDownloadQuery.model_validate(request.args.to_dict(flat=True))
|
||||
|
||||
try:
|
||||
export_path = SandboxFileDownloadPath(
|
||||
tenant_id=UUID(tenant_id),
|
||||
sandbox_id=UUID(sandbox_id),
|
||||
export_id=export_id,
|
||||
filename=filename,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise Forbidden(str(exc)) from exc
|
||||
|
||||
if not SandboxFileSigner.verify_upload_signature(
|
||||
export_path=export_path,
|
||||
expires_at=args.expires_at,
|
||||
nonce=args.nonce,
|
||||
sign=args.sign,
|
||||
):
|
||||
raise Forbidden("Invalid or expired upload link")
|
||||
|
||||
storage.save(export_path.get_storage_key(), request.get_data())
|
||||
return Response(status=204)
|
||||
@@ -1,56 +0,0 @@
|
||||
from urllib.parse import quote, unquote
|
||||
|
||||
from flask import Response, request
|
||||
from flask_restx import Resource
|
||||
from pydantic import BaseModel, Field
|
||||
from werkzeug.exceptions import Forbidden, NotFound
|
||||
|
||||
from controllers.files import files_ns
|
||||
from extensions.ext_storage import storage
|
||||
from extensions.storage.file_presign_storage import FilePresignStorage
|
||||
|
||||
DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}"
|
||||
|
||||
|
||||
class StorageDownloadQuery(BaseModel):
|
||||
timestamp: str = Field(..., description="Unix timestamp used in the signature")
|
||||
nonce: str = Field(..., description="Random string for signature")
|
||||
sign: str = Field(..., description="HMAC signature")
|
||||
|
||||
|
||||
files_ns.schema_model(
|
||||
StorageDownloadQuery.__name__,
|
||||
StorageDownloadQuery.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0),
|
||||
)
|
||||
|
||||
|
||||
@files_ns.route("/storage/<path:filename>/download")
|
||||
class StorageFileDownloadApi(Resource):
|
||||
def get(self, filename: str):
|
||||
filename = unquote(filename)
|
||||
|
||||
args = StorageDownloadQuery.model_validate(request.args.to_dict(flat=True))
|
||||
|
||||
if not FilePresignStorage.verify_signature(
|
||||
filename=filename,
|
||||
timestamp=args.timestamp,
|
||||
nonce=args.nonce,
|
||||
sign=args.sign,
|
||||
):
|
||||
raise Forbidden("Invalid or expired download link")
|
||||
|
||||
try:
|
||||
generator = storage.load_stream(filename)
|
||||
except FileNotFoundError:
|
||||
raise NotFound("File not found")
|
||||
|
||||
encoded_filename = quote(filename.split("/")[-1])
|
||||
|
||||
return Response(
|
||||
generator,
|
||||
mimetype="application/octet-stream",
|
||||
direct_passthrough=True,
|
||||
headers={
|
||||
"Content-Disposition": f"attachment; filename*=UTF-8''{encoded_filename}",
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Unified file proxy controller for storage operations.
|
||||
|
||||
This controller handles file download and upload operations when the underlying
|
||||
storage backend doesn't support presigned URLs. It verifies signed proxy URLs
|
||||
generated by FilePresignStorage and streams files to/from storage.
|
||||
|
||||
Endpoints:
|
||||
GET /files/storage/{filename}/download - Download a file
|
||||
PUT /files/storage/{filename}/upload - Upload a file
|
||||
"""
|
||||
|
||||
from urllib.parse import quote, unquote
|
||||
|
||||
from flask import Response, request
|
||||
from flask_restx import Resource
|
||||
from pydantic import BaseModel, Field
|
||||
from werkzeug.exceptions import Forbidden, NotFound
|
||||
|
||||
from controllers.files import files_ns
|
||||
from extensions.ext_storage import storage
|
||||
from extensions.storage.file_presign_storage import FilePresignStorage
|
||||
|
||||
DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}"
|
||||
|
||||
|
||||
class StorageProxyQuery(BaseModel):
|
||||
"""Query parameters for storage proxy URLs."""
|
||||
|
||||
timestamp: str = Field(..., description="Unix timestamp used in the signature")
|
||||
nonce: str = Field(..., description="Random string for signature")
|
||||
sign: str = Field(..., description="HMAC signature")
|
||||
|
||||
|
||||
files_ns.schema_model(
|
||||
StorageProxyQuery.__name__,
|
||||
StorageProxyQuery.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0),
|
||||
)
|
||||
|
||||
|
||||
@files_ns.route("/storage/<path:filename>/download")
|
||||
class StorageFileDownloadApi(Resource):
|
||||
"""Handle file downloads through the proxy."""
|
||||
|
||||
def get(self, filename: str):
|
||||
"""Download a file from storage.
|
||||
|
||||
Verifies the signed URL and streams the file content back to the client.
|
||||
"""
|
||||
filename = unquote(filename)
|
||||
args = StorageProxyQuery.model_validate(request.args.to_dict(flat=True))
|
||||
|
||||
if not FilePresignStorage.verify_signature(
|
||||
filename=filename,
|
||||
operation="download",
|
||||
timestamp=args.timestamp,
|
||||
nonce=args.nonce,
|
||||
sign=args.sign,
|
||||
):
|
||||
raise Forbidden("Invalid or expired download link")
|
||||
|
||||
try:
|
||||
generator = storage.load_stream(filename)
|
||||
except FileNotFoundError:
|
||||
raise NotFound("File not found")
|
||||
|
||||
encoded_filename = quote(filename.split("/")[-1])
|
||||
|
||||
return Response(
|
||||
generator,
|
||||
mimetype="application/octet-stream",
|
||||
direct_passthrough=True,
|
||||
headers={
|
||||
"Content-Disposition": f"attachment; filename*=UTF-8''{encoded_filename}",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@files_ns.route("/storage/<path:filename>/upload")
|
||||
class StorageFileUploadApi(Resource):
|
||||
"""Handle file uploads through the proxy."""
|
||||
|
||||
def put(self, filename: str):
|
||||
"""Upload a file to storage.
|
||||
|
||||
Verifies the signed URL and saves the request body to storage.
|
||||
"""
|
||||
filename = unquote(filename)
|
||||
args = StorageProxyQuery.model_validate(request.args.to_dict(flat=True))
|
||||
|
||||
if not FilePresignStorage.verify_signature(
|
||||
filename=filename,
|
||||
operation="upload",
|
||||
timestamp=args.timestamp,
|
||||
nonce=args.nonce,
|
||||
sign=args.sign,
|
||||
):
|
||||
raise Forbidden("Invalid or expired upload link")
|
||||
|
||||
content = request.get_data()
|
||||
storage.save(filename, content)
|
||||
|
||||
return Response(status=204)
|
||||
Reference in New Issue
Block a user