diff --git a/api/controllers/console/app/agent_app_sandbox.py b/api/controllers/console/app/agent_app_sandbox.py index fce6f250435..c1d6c3f0f46 100644 --- a/api/controllers/console/app/agent_app_sandbox.py +++ b/api/controllers/console/app/agent_app_sandbox.py @@ -1,8 +1,8 @@ """Console routes for Agent App and workflow Agent sandbox file access. The API accepts product-facing Conversation, Build Draft, or Workflow Node -Execution locators and proxies list/read/upload to the agent backend's -``/sandbox`` contract. +Execution locators and proxies list/read/download to the agent backend's +``/execution-bindings/files`` contract. """ from __future__ import annotations @@ -13,7 +13,6 @@ from uuid import UUID from dify_agent.client import DifyAgentClientError, DifyAgentHTTPError, DifyAgentTimeoutError from flask_restx import Resource from pydantic import BaseModel, Field -from sqlalchemy.orm import Session from controllers.common.schema import ( query_params_from_model, @@ -21,9 +20,8 @@ from controllers.common.schema import ( register_response_schema_models, register_schema_models, ) -from controllers.common.session import with_session from controllers.console import console_ns -from controllers.console.agent.app_helpers import resolve_agent_runtime_app_model +from controllers.console.app.error import AppNotFoundError from controllers.console.app.wraps import get_app_model from controllers.console.wraps import ( account_initialization_required, @@ -43,11 +41,17 @@ from services.agent_app_sandbox_service import ( WorkflowAgentSandboxService, ) +_BINDING_PATH_DESCRIPTION = ( + "Binding path: relative paths start in Workspace; exact `~` and paths beginning with `~/` start in Home; " + "`~user` is an ordinary relative path from Workspace; absolute paths remain absolute; `..` and paths outside " + "Workspace are governed by backend isolation, not a Workspace-root restriction" +) + class AgentSandboxListQuery(BaseModel): caller_type: Literal["conversation", "build_draft"] caller_id: str = Field(min_length=1, description="Agent App caller ID") - path: str = Field(default=".", description="Directory path relative to the sandbox workspace") + path: str = Field(default=".", description=_BINDING_PATH_DESCRIPTION) class AgentSandboxInfoQuery(BaseModel): @@ -58,28 +62,28 @@ class AgentSandboxInfoQuery(BaseModel): class AgentSandboxFileQuery(BaseModel): caller_type: Literal["conversation", "build_draft"] caller_id: str = Field(min_length=1, description="Agent App caller ID") - path: str = Field(min_length=1, description="File path relative to the sandbox workspace") + path: str = Field(min_length=1, description=_BINDING_PATH_DESCRIPTION) -class AgentSandboxUploadPayload(BaseModel): +class AgentSandboxDownloadPayload(BaseModel): caller_type: Literal["conversation", "build_draft"] caller_id: str = Field(min_length=1, description="Agent App caller ID") - path: str = Field(min_length=1, description="File path relative to the sandbox workspace") + path: str = Field(min_length=1, description=_BINDING_PATH_DESCRIPTION) class WorkflowAgentSandboxListQuery(BaseModel): node_execution_id: str = Field(min_length=1, description="Workflow node execution ID") - path: str = Field(default=".", description="Directory path relative to the sandbox workspace") + path: str = Field(default=".", description=_BINDING_PATH_DESCRIPTION) class WorkflowAgentSandboxFileQuery(BaseModel): node_execution_id: str = Field(min_length=1, description="Workflow node execution ID") - path: str = Field(min_length=1, description="File path relative to the sandbox workspace") + path: str = Field(min_length=1, description=_BINDING_PATH_DESCRIPTION) -class WorkflowAgentSandboxUploadPayload(BaseModel): +class WorkflowAgentSandboxDownloadPayload(BaseModel): node_execution_id: str = Field(min_length=1, description="Workflow node execution ID") - path: str = Field(min_length=1, description="File path relative to the sandbox workspace") + path: str = Field(min_length=1, description=_BINDING_PATH_DESCRIPTION) class SandboxFileEntryResponse(ResponseModel): @@ -107,21 +111,21 @@ class SandboxReadResponse(ResponseModel): text: str | None = None -class SandboxUploadResponse(ResponseModel): +class SandboxDownloadResponse(ResponseModel): url: str register_schema_models( console_ns, - AgentSandboxUploadPayload, - WorkflowAgentSandboxUploadPayload, + AgentSandboxDownloadPayload, + WorkflowAgentSandboxDownloadPayload, ) register_response_schema_models( console_ns, SandboxInfoResponse, SandboxListResponse, SandboxReadResponse, - SandboxUploadResponse, + SandboxDownloadResponse, ) @@ -152,14 +156,14 @@ class AgentAppSandboxInfoResource(Resource): @account_initialization_required @with_current_tenant_id @with_current_user - @with_session(write=False) - def get(self, session: Session, current_user: Account, tenant_id: str, agent_id: UUID): - app_model = resolve_agent_runtime_app_model(session=session, tenant_id=tenant_id, agent_id=agent_id) + def get(self, current_user: Account, tenant_id: str, agent_id: UUID): + service = AgentAppSandboxService() + app_id = service.resolve_app_id(tenant_id=tenant_id, agent_id=str(agent_id)) query = query_params_from_request(AgentSandboxInfoQuery) try: - result = AgentAppSandboxService().get_info( + result = service.get_info( tenant_id=tenant_id, - app_id=app_model.id, + app_id=app_id, agent_id=str(agent_id), caller_type=query.caller_type, caller_id=query.caller_id, @@ -181,14 +185,14 @@ class AgentAppSandboxListResource(Resource): @account_initialization_required @with_current_tenant_id @with_current_user - @with_session(write=False) - def get(self, session: Session, current_user: Account, tenant_id: str, agent_id: UUID): - app_model = resolve_agent_runtime_app_model(session=session, tenant_id=tenant_id, agent_id=agent_id) + def get(self, current_user: Account, tenant_id: str, agent_id: UUID): + service = AgentAppSandboxService() + app_id = service.resolve_app_id(tenant_id=tenant_id, agent_id=str(agent_id)) query = query_params_from_request(AgentSandboxListQuery) try: - result = AgentAppSandboxService().list_files( + result = service.list_files( tenant_id=tenant_id, - app_id=app_model.id, + app_id=app_id, agent_id=str(agent_id), caller_type=query.caller_type, caller_id=query.caller_id, @@ -211,14 +215,14 @@ class AgentAppSandboxReadResource(Resource): @account_initialization_required @with_current_tenant_id @with_current_user - @with_session(write=False) - def get(self, session: Session, current_user: Account, tenant_id: str, agent_id: UUID): - app_model = resolve_agent_runtime_app_model(session=session, tenant_id=tenant_id, agent_id=agent_id) + def get(self, current_user: Account, tenant_id: str, agent_id: UUID): + service = AgentAppSandboxService() + app_id = service.resolve_app_id(tenant_id=tenant_id, agent_id=str(agent_id)) query = query_params_from_request(AgentSandboxFileQuery) try: - result = AgentAppSandboxService().read_file( + result = service.read_file( tenant_id=tenant_id, - app_id=app_model.id, + app_id=app_id, agent_id=str(agent_id), caller_type=query.caller_type, caller_id=query.caller_id, @@ -230,32 +234,31 @@ class AgentAppSandboxReadResource(Resource): return result.model_dump() -@console_ns.route("/agent//sandbox/files/upload") -class AgentAppSandboxUploadResource(Resource): - @console_ns.doc("upload_agent_app_sandbox_file") - @console_ns.doc(description="Upload one Agent App sandbox file and return a signed download URL") - @console_ns.expect(console_ns.models[AgentSandboxUploadPayload.__name__]) - @console_ns.response(200, "Uploaded", console_ns.models[SandboxUploadResponse.__name__]) +@console_ns.route("/agent//sandbox/files/download") +class AgentAppSandboxDownloadResource(Resource): + @console_ns.doc("download_agent_app_sandbox_file") + @console_ns.doc(description="Create a ToolFile from one Agent App Binding file and return its download URL") + @console_ns.expect(console_ns.models[AgentSandboxDownloadPayload.__name__]) + @console_ns.response(200, "Download URL returned", console_ns.models[SandboxDownloadResponse.__name__]) @setup_required @login_required @account_initialization_required @with_current_tenant_id @with_current_user - @with_session(write=False) - @model_validate(AgentSandboxUploadPayload) + @model_validate(AgentSandboxDownloadPayload) def post( self, - req_data: AgentSandboxUploadPayload, - session: Session, + req_data: AgentSandboxDownloadPayload, current_user: Account, tenant_id: str, agent_id: UUID, ): - app_model = resolve_agent_runtime_app_model(session=session, tenant_id=tenant_id, agent_id=agent_id) + service = AgentAppSandboxService() + app_id = service.resolve_app_id(tenant_id=tenant_id, agent_id=str(agent_id)) try: - result = AgentAppSandboxService().upload_file( + result = service.download_file( tenant_id=tenant_id, - app_id=app_model.id, + app_id=app_id, agent_id=str(agent_id), caller_type=req_data.caller_type, caller_id=req_data.caller_id, @@ -340,36 +343,41 @@ class WorkflowAgentSandboxReadResource(Resource): @console_ns.route( - "/apps//workflow-runs//agent-nodes//sandbox/files/upload" + "/apps//workflow-runs//agent-nodes//sandbox/files/download" ) -class WorkflowAgentSandboxUploadResource(Resource): - @console_ns.doc("upload_workflow_agent_sandbox_file") - @console_ns.doc(description="Upload one workflow Agent sandbox file and return a signed download URL") - @console_ns.expect(console_ns.models[WorkflowAgentSandboxUploadPayload.__name__]) - @console_ns.response(200, "Uploaded", console_ns.models[SandboxUploadResponse.__name__]) +class WorkflowAgentSandboxDownloadResource(Resource): + @console_ns.doc("download_workflow_agent_sandbox_file") + @console_ns.doc(description="Create a ToolFile from one workflow Agent Binding file and return its download URL") + @console_ns.expect(console_ns.models[WorkflowAgentSandboxDownloadPayload.__name__]) + @console_ns.response(200, "Download URL returned", console_ns.models[SandboxDownloadResponse.__name__]) @setup_required @login_required @account_initialization_required - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) + @with_current_user @with_current_tenant_id - @model_validate(WorkflowAgentSandboxUploadPayload) + @model_validate(WorkflowAgentSandboxDownloadPayload) def post( self, - req_data: WorkflowAgentSandboxUploadPayload, + req_data: WorkflowAgentSandboxDownloadPayload, tenant_id: str, - app_model: App, + current_user: Account, + app_id: UUID, workflow_run_id: UUID, node_id: str, ): + service = WorkflowAgentSandboxService() + resolved_app_id = service.resolve_app_id(tenant_id=tenant_id, app_id=str(app_id)) + if resolved_app_id is None: + raise AppNotFoundError() try: - result = WorkflowAgentSandboxService().upload_file( + result = service.download_file( tenant_id=tenant_id, - app_id=app_model.id, + app_id=resolved_app_id, workflow_run_id=str(workflow_run_id), node_id=node_id, node_execution_id=req_data.node_execution_id, + account_id=current_user.id, path=req_data.path, - session=db.session(), ) except Exception as exc: return _handle(exc) diff --git a/api/controllers/files/upload.py b/api/controllers/files/upload.py index 3de92589f6a..82a7ae4fb65 100644 --- a/api/controllers/files/upload.py +++ b/api/controllers/files/upload.py @@ -1,3 +1,5 @@ +from typing import Literal + from flask import request from flask_restx import Resource from flask_restx.api import HTTPStatus @@ -5,10 +7,12 @@ from pydantic import BaseModel, Field from werkzeug.exceptions import Forbidden import services +from core.db.session_factory import session_factory from core.tools.signature import verify_plugin_file_signature from core.tools.tool_file_manager import ToolFileManager, resolve_extension from core.workflow.file_reference import build_file_reference from fields.file_fields import FileResponse +from services.account_service import TenantService from ..common.errors import ( FileTooLargeError, @@ -26,6 +30,7 @@ class PluginUploadQuery(BaseModel): sign: str = Field(..., description="HMAC signature") tenant_id: str = Field(..., description="Tenant identifier") user_id: str | None = Field(default=None, description="User identifier") + user_from: Literal["account", "end-user"] | None = Field(default=None, description="User identity type") conversation_id: str | None = Field(default=None, description="Conversation identifier") @@ -77,8 +82,20 @@ class PluginUploadFileApi(Resource): nonce = args.nonce sign = args.sign tenant_id = args.tenant_id - user_id = args.user_id - user = get_user(tenant_id, user_id) + if args.user_from == "account": + if args.user_id is None: + raise Forbidden("Invalid request.") + with session_factory.create_session() as session: + is_tenant_member = TenantService.account_belongs_to_tenant( + args.user_id, + tenant_id, + session=session, + ) + if not is_tenant_member: + raise Forbidden("Invalid request.") + owner_id = args.user_id + else: + owner_id = get_user(tenant_id, args.user_id).id filename = file.filename mimetype = file.mimetype @@ -90,8 +107,9 @@ class PluginUploadFileApi(Resource): filename=filename, mimetype=mimetype, tenant_id=tenant_id, - user_id=user.id, + user_id=owner_id, conversation_id=args.conversation_id, + user_from=args.user_from, timestamp=timestamp, nonce=nonce, sign=sign, @@ -100,7 +118,7 @@ class PluginUploadFileApi(Resource): try: tool_file = ToolFileManager().create_file_by_raw( - user_id=user.id, + user_id=owner_id, tenant_id=tenant_id, file_binary=file.stream.read(), mimetype=mimetype, diff --git a/api/controllers/inner_api/agent/files.py b/api/controllers/inner_api/agent/files.py index 7a5cad76440..8838ba2d4fe 100644 --- a/api/controllers/inner_api/agent/files.py +++ b/api/controllers/inner_api/agent/files.py @@ -38,6 +38,7 @@ class AgentFileRequestHttpError(BaseHTTPException): class AgentFileUploadRequestPayload(RequestRequestUploadFile): tenant_id: str user_id: str + user_from: Literal["account", "end-user"] | None = None model_config = ConfigDict(extra="forbid") @@ -113,13 +114,19 @@ class AgentFileUploadRequestApi(Resource): status_code=404, ) try: - user = get_user(tenant.id, payload.user_id) + if payload.user_from == "account": + if not TenantService.account_belongs_to_tenant(payload.user_id, tenant.id, session=session): + raise ValueError("account not found") + owner_id = payload.user_id + else: + owner_id = get_user(tenant.id, payload.user_id).id upload_uri = get_signed_file_uri_for_plugin( filename=payload.filename, mimetype=payload.mimetype, tenant_id=tenant.id, - user_id=user.id, + user_id=owner_id, conversation_id=payload.conversation_id, + user_from=payload.user_from, ) except ValueError as exc: raise AgentFileRequestHttpError( diff --git a/api/core/tools/signature.py b/api/core/tools/signature.py index 02291bd2611..725160aaf8a 100644 --- a/api/core/tools/signature.py +++ b/api/core/tools/signature.py @@ -4,6 +4,7 @@ import hmac import os import time import urllib.parse +from typing import Literal from urllib.parse import urlsplit from configs import dify_config @@ -88,13 +89,27 @@ def verify_tool_file_signature(file_id: str, timestamp: str, nonce: str, sign: s def get_signed_file_uri_for_plugin( - filename: str, mimetype: str, tenant_id: str, user_id: str, conversation_id: str | None = None + filename: str, + mimetype: str, + tenant_id: str, + user_id: str, + conversation_id: str | None = None, + user_from: Literal["account", "end-user"] | None = None, ) -> str: """Build a signed plugin-upload URI without selecting a network origin.""" timestamp = str(int(time.time())) nonce = os.urandom(16).hex() - data_to_sign = f"upload|{filename}|{mimetype}|{tenant_id}|{user_id}|{conversation_id or ''}|{timestamp}|{nonce}" + data_to_sign = _plugin_upload_signature_payload( + filename=filename, + mimetype=mimetype, + tenant_id=tenant_id, + user_id=user_id, + conversation_id=conversation_id, + timestamp=timestamp, + nonce=nonce, + user_from=user_from, + ) sign = hmac.new(_secret_key(), data_to_sign.encode(), hashlib.sha256).digest() encoded_sign = base64.urlsafe_b64encode(sign).decode() query_params = { @@ -106,6 +121,8 @@ def get_signed_file_uri_for_plugin( } if conversation_id: query_params["conversation_id"] = conversation_id + if user_from is not None: + query_params["user_from"] = user_from query = urllib.parse.urlencode(query_params) return f"/files/upload/for-plugin?{query}" @@ -117,13 +134,23 @@ def verify_plugin_file_signature( tenant_id: str, user_id: str, conversation_id: str | None = None, + user_from: Literal["account", "end-user"] | None = None, timestamp: str, nonce: str, sign: str, ) -> bool: """Verify the signature used by the plugin-facing file upload endpoint.""" - data_to_sign = f"upload|{filename}|{mimetype}|{tenant_id}|{user_id}|{conversation_id or ''}|{timestamp}|{nonce}" + data_to_sign = _plugin_upload_signature_payload( + filename=filename, + mimetype=mimetype, + tenant_id=tenant_id, + user_id=user_id, + conversation_id=conversation_id, + timestamp=timestamp, + nonce=nonce, + user_from=user_from, + ) recalculated_sign = hmac.new(_secret_key(), data_to_sign.encode(), hashlib.sha256).digest() recalculated_encoded_sign = base64.urlsafe_b64encode(recalculated_sign).decode() @@ -132,3 +159,27 @@ def verify_plugin_file_signature( current_time = int(time.time()) return current_time - int(timestamp) <= dify_config.FILES_ACCESS_TIMEOUT + + +def _plugin_upload_signature_payload( + *, + filename: str, + mimetype: str, + tenant_id: str, + user_id: str, + conversation_id: str | None, + timestamp: str, + nonce: str, + user_from: Literal["account", "end-user"] | None, +) -> str: + """Build the compatible upload signature payload with optional identity ownership. + + Omitting ``user_from`` preserves the legacy payload. When present, the + identity kind is appended and HMAC-protected so account/end-user ownership + cannot be altered. + """ + + payload = f"upload|{filename}|{mimetype}|{tenant_id}|{user_id}|{conversation_id or ''}|{timestamp}|{nonce}" + if user_from is not None: + payload = f"{payload}|{user_from}" + return payload diff --git a/api/openapi/markdown/console-openapi.md b/api/openapi/markdown/console-openapi.md index 74ad92b0f7e..ef1a80c5ce2 100644 --- a/api/openapi/markdown/console-openapi.md +++ b/api/openapi/markdown/console-openapi.md @@ -1274,7 +1274,7 @@ List a directory in an Agent App conversation sandbox | agent_id | path | Agent ID | Yes | string (uuid) | | caller_id | query | Agent App caller ID | Yes | string | | caller_type | query | | Yes | string,
**Available values:** "build_draft", "conversation" | -| path | query | Directory path relative to the sandbox workspace | No | string,
**Default:** . | +| path | query | Binding path: relative paths start in Workspace; exact `~` and paths beginning with `~/` start in Home; `~user` is an ordinary relative path from Workspace; absolute paths remain absolute; `..` and paths outside Workspace are governed by backend isolation, not a Workspace-root restriction | No | string,
**Default:** . | #### Responses @@ -1282,26 +1282,8 @@ List a directory in an Agent App conversation sandbox | ---- | ----------- | ------ | | 200 | Listing returned | **application/json**: [SandboxListResponse](#sandboxlistresponse)
| -### [GET] /agent/{agent_id}/sandbox/files/read -Read a text/binary preview file in an Agent App conversation sandbox - -#### Parameters - -| Name | Located in | Description | Required | Schema | -| ---- | ---------- | ----------- | -------- | ------ | -| agent_id | path | Agent ID | Yes | string (uuid) | -| caller_id | query | Agent App caller ID | Yes | string | -| caller_type | query | | Yes | string,
**Available values:** "build_draft", "conversation" | -| path | query | File path relative to the sandbox workspace | Yes | string | - -#### Responses - -| Code | Description | Schema | -| ---- | ----------- | ------ | -| 200 | Preview returned | **application/json**: [SandboxReadResponse](#sandboxreadresponse)
| - -### [POST] /agent/{agent_id}/sandbox/files/upload -Upload one Agent App sandbox file and return a signed download URL +### [POST] /agent/{agent_id}/sandbox/files/download +Create a ToolFile from one Agent App Binding file and return its download URL #### Parameters @@ -1313,13 +1295,31 @@ Upload one Agent App sandbox file and return a signed download URL | Required | Schema | | -------- | ------ | -| Yes | **application/json**: [AgentSandboxUploadPayload](#agentsandboxuploadpayload)
| +| Yes | **application/json**: [AgentSandboxDownloadPayload](#agentsandboxdownloadpayload)
| #### Responses | Code | Description | Schema | | ---- | ----------- | ------ | -| 200 | Uploaded | **application/json**: [SandboxUploadResponse](#sandboxuploadresponse)
| +| 200 | Download URL returned | **application/json**: [SandboxDownloadResponse](#sandboxdownloadresponse)
| + +### [GET] /agent/{agent_id}/sandbox/files/read +Read a text/binary preview file in an Agent App conversation sandbox + +#### Parameters + +| Name | Located in | Description | Required | Schema | +| ---- | ---------- | ----------- | -------- | ------ | +| agent_id | path | Agent ID | Yes | string (uuid) | +| caller_id | query | Agent App caller ID | Yes | string | +| caller_type | query | | Yes | string,
**Available values:** "build_draft", "conversation" | +| path | query | Binding path: relative paths start in Workspace; exact `~` and paths beginning with `~/` start in Home; `~user` is an ordinary relative path from Workspace; absolute paths remain absolute; `..` and paths outside Workspace are governed by backend isolation, not a Workspace-root restriction | Yes | string | + +#### Responses + +| Code | Description | Schema | +| ---- | ----------- | ------ | +| 200 | Preview returned | **application/json**: [SandboxReadResponse](#sandboxreadresponse)
| ### [POST] /agent/{agent_id}/skills/upload Upload + standardize a Skill into an Agent App drive @@ -3822,7 +3822,7 @@ List a directory in a workflow Agent node sandbox | node_id | path | Workflow Agent node ID | Yes | string | | workflow_run_id | path | Workflow run ID | Yes | string (uuid) | | node_execution_id | query | Workflow node execution ID | Yes | string | -| path | query | Directory path relative to the sandbox workspace | No | string,
**Default:** . | +| path | query | Binding path: relative paths start in Workspace; exact `~` and paths beginning with `~/` start in Home; `~user` is an ordinary relative path from Workspace; absolute paths remain absolute; `..` and paths outside Workspace are governed by backend isolation, not a Workspace-root restriction | No | string,
**Default:** . | #### Responses @@ -3830,27 +3830,8 @@ List a directory in a workflow Agent node sandbox | ---- | ----------- | ------ | | 200 | Listing returned | **application/json**: [SandboxListResponse](#sandboxlistresponse)
| -### [GET] /apps/{app_id}/workflow-runs/{workflow_run_id}/agent-nodes/{node_id}/sandbox/files/read -Read a text/binary preview file in a workflow Agent node sandbox - -#### Parameters - -| Name | Located in | Description | Required | Schema | -| ---- | ---------- | ----------- | -------- | ------ | -| app_id | path | Application ID | Yes | string (uuid) | -| node_id | path | Workflow Agent node ID | Yes | string | -| workflow_run_id | path | Workflow run ID | Yes | string (uuid) | -| node_execution_id | query | Workflow node execution ID | Yes | string | -| path | query | File path relative to the sandbox workspace | Yes | string | - -#### Responses - -| Code | Description | Schema | -| ---- | ----------- | ------ | -| 200 | Preview returned | **application/json**: [SandboxReadResponse](#sandboxreadresponse)
| - -### [POST] /apps/{app_id}/workflow-runs/{workflow_run_id}/agent-nodes/{node_id}/sandbox/files/upload -Upload one workflow Agent sandbox file and return a signed download URL +### [POST] /apps/{app_id}/workflow-runs/{workflow_run_id}/agent-nodes/{node_id}/sandbox/files/download +Create a ToolFile from one workflow Agent Binding file and return its download URL #### Parameters @@ -3864,13 +3845,32 @@ Upload one workflow Agent sandbox file and return a signed download URL | Required | Schema | | -------- | ------ | -| Yes | **application/json**: [WorkflowAgentSandboxUploadPayload](#workflowagentsandboxuploadpayload)
| +| Yes | **application/json**: [WorkflowAgentSandboxDownloadPayload](#workflowagentsandboxdownloadpayload)
| #### Responses | Code | Description | Schema | | ---- | ----------- | ------ | -| 200 | Uploaded | **application/json**: [SandboxUploadResponse](#sandboxuploadresponse)
| +| 200 | Download URL returned | **application/json**: [SandboxDownloadResponse](#sandboxdownloadresponse)
| + +### [GET] /apps/{app_id}/workflow-runs/{workflow_run_id}/agent-nodes/{node_id}/sandbox/files/read +Read a text/binary preview file in a workflow Agent node sandbox + +#### Parameters + +| Name | Located in | Description | Required | Schema | +| ---- | ---------- | ----------- | -------- | ------ | +| app_id | path | Application ID | Yes | string (uuid) | +| node_id | path | Workflow Agent node ID | Yes | string | +| workflow_run_id | path | Workflow run ID | Yes | string (uuid) | +| node_execution_id | query | Workflow node execution ID | Yes | string | +| path | query | Binding path: relative paths start in Workspace; exact `~` and paths beginning with `~/` start in Home; `~user` is an ordinary relative path from Workspace; absolute paths remain absolute; `..` and paths outside Workspace are governed by backend isolation, not a Workspace-root restriction | Yes | string | + +#### Responses + +| Code | Description | Schema | +| ---- | ----------- | ------ | +| 200 | Preview returned | **application/json**: [SandboxReadResponse](#sandboxreadresponse)
| ### [GET] /apps/{app_id}/workflow/comments **Get all comments for a workflow** @@ -14703,6 +14703,14 @@ section may be empty, which is how callers express "no knowledge layer". | workflow_id | string | | No | | workflow_node_id | string | | No | +#### AgentSandboxDownloadPayload + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| caller_id | string | Agent App caller ID | Yes | +| caller_type | string,
**Available values:** "build_draft", "conversation" | *Enum:* `"build_draft"`, `"conversation"` | Yes | +| path | string | Binding path: relative paths start in Workspace; exact `~` and paths beginning with `~/` start in Home; `~user` is an ordinary relative path from Workspace; absolute paths remain absolute; `..` and paths outside Workspace are governed by backend isolation, not a Workspace-root restriction | Yes | + #### AgentSandboxProviderConfig | Name | Type | Description | Required | @@ -14712,14 +14720,6 @@ section may be empty, which is how callers express "no knowledge layer". | image | string | | No | | working_dir | string | | No | -#### AgentSandboxUploadPayload - -| Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | -| caller_id | string | Agent App caller ID | Yes | -| caller_type | string,
**Available values:** "build_draft", "conversation" | *Enum:* `"build_draft"`, `"conversation"` | Yes | -| path | string | File path relative to the sandbox workspace | Yes | - #### AgentScope Visibility and lifecycle scope of an Agent record. @@ -21522,6 +21522,12 @@ Whitelist scopes accepted by RBAC app and dataset access config APIs. | ---- | ---- | ----------- | -------- | | SSOProtocol | string | | | +#### SandboxDownloadResponse + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| url | string | | Yes | + #### SandboxFileEntryResponse | Name | Type | Description | Required | @@ -21555,12 +21561,6 @@ Whitelist scopes accepted by RBAC app and dataset access config APIs. | text | string | | No | | truncated | boolean | | Yes | -#### SandboxUploadResponse - -| Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | -| url | string | | Yes | - #### SavedMessageCreatePayload | Name | Type | Description | Required | @@ -23367,12 +23367,12 @@ How a workflow node is bound to an Agent. | variant | string | | Yes | | workflow_id | string | | No | -#### WorkflowAgentSandboxUploadPayload +#### WorkflowAgentSandboxDownloadPayload | Name | Type | Description | Required | | ---- | ---- | ----------- | -------- | | node_execution_id | string | Workflow node execution ID | Yes | -| path | string | File path relative to the sandbox workspace | Yes | +| path | string | Binding path: relative paths start in Workspace; exact `~` and paths beginning with `~/` start in Home; `~user` is an ordinary relative path from Workspace; absolute paths remain absolute; `..` and paths outside Workspace are governed by backend isolation, not a Workspace-root restriction | Yes | #### WorkflowAppLogPaginationResponse diff --git a/api/services/agent_app_sandbox_service.py b/api/services/agent_app_sandbox_service.py index 9661c30c7d3..91f16c2771c 100644 --- a/api/services/agent_app_sandbox_service.py +++ b/api/services/agent_app_sandbox_service.py @@ -4,23 +4,26 @@ from __future__ import annotations import urllib.parse from collections.abc import Callable -from typing import Any, Literal, cast +from dataclasses import dataclass +from typing import Literal, cast from dify_agent.client import Client from dify_agent.layers.execution_context import ( DifyExecutionContextAgentConfigVersionKind, DifyExecutionContextLayerConfig, ) -from dify_agent.protocol import WorkspaceListResponse, WorkspaceReadResponse, WorkspaceUploadRequest +from dify_agent.protocol import ( + BindingFileDownloadRequest, + BindingFileListResponse, + BindingFileReadResponse, +) from pydantic import BaseModel from sqlalchemy import select from sqlalchemy.orm import Session from configs import dify_config -from core.app.file_access import DatabaseFileAccessController -from core.app.workflow.file_runtime import DifyWorkflowFileRuntime from core.db.session_factory import session_factory -from factories import file_factory +from core.tools.signature import bind_file_uri from models.agent import ( Agent, AgentConfigDraft, @@ -28,10 +31,11 @@ from models.agent import ( AgentWorkspaceBinding, AgentWorkspaceOwnerType, ) -from models.model import App, Conversation +from models.model import App, AppMode, Conversation from models.workflow import WorkflowNodeExecutionModel from services.agent.roster_service import AgentRosterService from services.agent.workspace_service import AgentWorkspaceService, WorkspaceOwnerScope +from services.file_request_service import FileRequestService class AgentSandboxInspectorError(Exception): @@ -50,14 +54,37 @@ class AgentSandboxInfo(BaseModel): workspace_cwd: str -class AgentSandboxUploadDownload(BaseModel): +class AgentSandboxDownload(BaseModel): url: str +@dataclass(frozen=True, slots=True) +class _ResolvedBinding: + """Detached scalar Binding data safe to carry beyond its read transaction. + + Resolvers must end the transaction before Dify Agent network I/O; ORM and + session-bound objects never cross that boundary. + """ + + backend_binding_ref: str + agent_id: str + agent_config_version_id: str + agent_config_version_kind: str + + class AgentAppSandboxService: def __init__(self, *, client_factory: Callable[[], Client] | None = None) -> None: self._client_factory = client_factory or _default_client_factory + @staticmethod + def resolve_app_id(*, tenant_id: str, agent_id: str) -> str: + with session_factory.create_session() as session: + app = AgentRosterService(session).get_agent_runtime_app_model( + tenant_id=tenant_id, + agent_id=agent_id, + ) + return app.id + def get_info( self, *, @@ -88,7 +115,7 @@ class AgentAppSandboxService: caller_id: str, account_id: str, path: str, - ) -> WorkspaceListResponse: + ) -> BindingFileListResponse: binding = self._resolve_binding( tenant_id=tenant_id, app_id=app_id, @@ -98,7 +125,7 @@ class AgentAppSandboxService: account_id=account_id, ) with self._client_factory() as client: - return client.list_workspace_files_sync(binding.backend_binding_ref, path) + return client.list_binding_files_sync(binding.backend_binding_ref, path) def read_file( self, @@ -110,7 +137,7 @@ class AgentAppSandboxService: caller_id: str, account_id: str, path: str, - ) -> WorkspaceReadResponse: + ) -> BindingFileReadResponse: binding = self._resolve_binding( tenant_id=tenant_id, app_id=app_id, @@ -120,9 +147,9 @@ class AgentAppSandboxService: account_id=account_id, ) with self._client_factory() as client: - return client.read_workspace_file_sync(binding.backend_binding_ref, path) + return client.read_binding_file_sync(binding.backend_binding_ref, path) - def upload_file( + def download_file( self, *, tenant_id: str, @@ -132,7 +159,7 @@ class AgentAppSandboxService: caller_id: str, account_id: str, path: str, - ) -> AgentSandboxUploadDownload: + ) -> AgentSandboxDownload: binding = self._resolve_binding( tenant_id=tenant_id, app_id=app_id, @@ -142,26 +169,28 @@ class AgentAppSandboxService: account_id=account_id, ) with self._client_factory() as client: - uploaded = client.upload_workspace_file_sync( - WorkspaceUploadRequest( + downloaded = client.download_binding_file_sync( + BindingFileDownloadRequest( backend_binding_ref=binding.backend_binding_ref, path=path, execution_context=DifyExecutionContextLayerConfig( tenant_id=tenant_id, + user_id=account_id, + user_from="account", app_id=app_id, conversation_id=caller_id if caller_type == "conversation" else None, agent_id=agent_id, agent_config_version_id=binding.agent_config_version_id, agent_config_version_kind=cast( DifyExecutionContextAgentConfigVersionKind, - binding.agent_config_version_kind.value, + binding.agent_config_version_kind, ), agent_mode="agent_app", invoke_from="debugger", ), ) ) - return _upload_download_response(tenant_id=tenant_id, file_mapping=uploaded.file.model_dump(mode="python")) + return _download_response(tenant_id=tenant_id, account_id=account_id, reference=downloaded.reference) @staticmethod def _resolve_binding( @@ -172,7 +201,7 @@ class AgentAppSandboxService: caller_type: Literal["conversation", "build_draft"], caller_id: str, account_id: str, - ) -> AgentWorkspaceBinding: + ) -> _ResolvedBinding: with session_factory.create_session() as session: caller: AgentConfigDraft | Conversation | None if caller_type == "build_draft": @@ -236,14 +265,25 @@ class AgentAppSandboxService: "this caller has no active Agent Workspace Binding", status_code=404, ) - session.expunge(binding) - return binding + return _binding_value(binding) class WorkflowAgentSandboxService: def __init__(self, *, client_factory: Callable[[], Client] | None = None) -> None: self._client_factory = client_factory or _default_client_factory + @staticmethod + def resolve_app_id(*, tenant_id: str, app_id: str) -> str | None: + with session_factory.create_session() as session: + return session.scalar( + select(App.id).where( + App.id == app_id, + App.tenant_id == tenant_id, + App.status == "normal", + App.mode.in_((AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value)), + ) + ) + def list_files( self, *, @@ -254,7 +294,7 @@ class WorkflowAgentSandboxService: node_execution_id: str, path: str, session: Session, - ) -> WorkspaceListResponse: + ) -> BindingFileListResponse: binding = self._resolve_binding( tenant_id=tenant_id, app_id=app_id, @@ -264,7 +304,7 @@ class WorkflowAgentSandboxService: session=session, ) with self._client_factory() as client: - return client.list_workspace_files_sync(binding.backend_binding_ref, path) + return client.list_binding_files_sync(binding.backend_binding_ref, path) def read_file( self, @@ -276,7 +316,7 @@ class WorkflowAgentSandboxService: node_execution_id: str, path: str, session: Session, - ) -> WorkspaceReadResponse: + ) -> BindingFileReadResponse: binding = self._resolve_binding( tenant_id=tenant_id, app_id=app_id, @@ -286,9 +326,9 @@ class WorkflowAgentSandboxService: session=session, ) with self._client_factory() as client: - return client.read_workspace_file_sync(binding.backend_binding_ref, path) + return client.read_binding_file_sync(binding.backend_binding_ref, path) - def upload_file( + def download_file( self, *, tenant_id: str, @@ -296,39 +336,43 @@ class WorkflowAgentSandboxService: workflow_run_id: str, node_id: str, node_execution_id: str, + account_id: str, path: str, - session: Session, - ) -> AgentSandboxUploadDownload: - binding = self._resolve_binding( - tenant_id=tenant_id, - app_id=app_id, - workflow_run_id=workflow_run_id, - node_id=node_id, - node_execution_id=node_execution_id, - session=session, - ) + ) -> AgentSandboxDownload: + with session_factory.create_session() as session: + binding = self._resolve_binding( + tenant_id=tenant_id, + app_id=app_id, + workflow_run_id=workflow_run_id, + node_id=node_id, + node_execution_id=node_execution_id, + session=session, + ) with self._client_factory() as client: - uploaded = client.upload_workspace_file_sync( - WorkspaceUploadRequest( + downloaded = client.download_binding_file_sync( + BindingFileDownloadRequest( backend_binding_ref=binding.backend_binding_ref, path=path, execution_context=DifyExecutionContextLayerConfig( tenant_id=tenant_id, + user_id=account_id, + user_from="account", app_id=app_id, workflow_run_id=workflow_run_id, node_id=node_id, + node_execution_id=node_execution_id, agent_id=binding.agent_id, agent_config_version_id=binding.agent_config_version_id, agent_config_version_kind=cast( DifyExecutionContextAgentConfigVersionKind, - binding.agent_config_version_kind.value, + binding.agent_config_version_kind, ), agent_mode="workflow_run", invoke_from="debugger", ), ) ) - return _upload_download_response(tenant_id=tenant_id, file_mapping=uploaded.file.model_dump(mode="python")) + return _download_response(tenant_id=tenant_id, account_id=account_id, reference=downloaded.reference) @staticmethod def _resolve_binding( @@ -339,7 +383,7 @@ class WorkflowAgentSandboxService: node_id: str, node_execution_id: str, session: Session, - ) -> AgentWorkspaceBinding: + ) -> _ResolvedBinding: execution = session.scalar( select(WorkflowNodeExecutionModel).where( WorkflowNodeExecutionModel.id == node_execution_id, @@ -379,28 +423,38 @@ class WorkflowAgentSandboxService: "this Workflow Agent node execution has no active Workspace Binding", status_code=404, ) - return binding + resolved = _binding_value(binding) + # Deliberately end the read transaction before the caller performs Dify Agent I/O. + session.rollback() + return resolved -def _upload_download_response(*, tenant_id: str, file_mapping: dict[str, Any]) -> AgentSandboxUploadDownload: - controller = DatabaseFileAccessController() - runtime = DifyWorkflowFileRuntime(file_access_controller=controller) +def _binding_value(binding: AgentWorkspaceBinding) -> _ResolvedBinding: + return _ResolvedBinding( + backend_binding_ref=binding.backend_binding_ref, + agent_id=binding.agent_id, + agent_config_version_id=binding.agent_config_version_id, + agent_config_version_kind=binding.agent_config_version_kind.value, + ) + + +def _download_response(*, tenant_id: str, account_id: str, reference: str) -> AgentSandboxDownload: try: - file = file_factory.build_from_mapping(mapping=file_mapping, tenant_id=tenant_id, access_controller=controller) - url = runtime.resolve_file_url(file=file, for_external=True) + result = FileRequestService().request_download( + tenant_id=tenant_id, + user_id=account_id, + user_from="account", + invoke_from="debugger", + file_mapping={"transfer_method": "tool_file", "reference": reference}, + ) + url = bind_file_uri(result.download_uri, dify_config.FILES_URL) except ValueError as exc: raise AgentSandboxInspectorError( - "workspace_upload_download_unavailable", - "uploaded Workspace file could not be converted to a download URL", + "binding_file_download_unavailable", + "Binding file could not be converted to a download URL", status_code=502, ) from exc - if not url: - raise AgentSandboxInspectorError( - "workspace_upload_download_unavailable", - "uploaded Workspace file does not support download URL generation", - status_code=502, - ) - return AgentSandboxUploadDownload(url=_with_as_attachment(url)) + return AgentSandboxDownload(url=_with_as_attachment(url)) def _with_as_attachment(url: str) -> str: @@ -415,7 +469,7 @@ def _default_client_factory() -> Client: if not base_url: raise AgentSandboxInspectorError( "inspector_unavailable", - "the Workspace file inspector is not available (Agent backend not configured)", + "the Binding file inspector is not available (Agent backend not configured)", status_code=503, ) return Client(base_url=base_url) @@ -423,8 +477,8 @@ def _default_client_factory() -> Client: __all__ = [ "AgentAppSandboxService", + "AgentSandboxDownload", "AgentSandboxInfo", "AgentSandboxInspectorError", - "AgentSandboxUploadDownload", "WorkflowAgentSandboxService", ] diff --git a/api/tests/unit_tests/controllers/console/app/test_agent_app_sandbox.py b/api/tests/unit_tests/controllers/console/app/test_agent_app_sandbox.py index d0483a5a7ce..ed227fd09d6 100644 --- a/api/tests/unit_tests/controllers/console/app/test_agent_app_sandbox.py +++ b/api/tests/unit_tests/controllers/console/app/test_agent_app_sandbox.py @@ -2,22 +2,23 @@ from __future__ import annotations from inspect import unwrap from types import SimpleNamespace -from unittest.mock import MagicMock import pytest from dify_agent.client import DifyAgentClientError, DifyAgentHTTPError, DifyAgentTimeoutError -from dify_agent.protocol import WorkspaceListResponse, WorkspaceReadResponse -from sqlalchemy.orm import Session +from dify_agent.protocol import BindingFileListResponse, BindingFileReadResponse from controllers.console import agent_app_sandbox as module from models.model import App, AppMode, IconType -from services.agent_app_sandbox_service import AgentSandboxInfo, AgentSandboxInspectorError, AgentSandboxUploadDownload +from services.agent_app_sandbox_service import AgentSandboxDownload, AgentSandboxInfo, AgentSandboxInspectorError class _AgentAppService: def __init__(self) -> None: self.calls: list[tuple[str, str, str, str, str, str, str, str]] = [] + def resolve_app_id(self, *, tenant_id: str, agent_id: str) -> str: + return "app-1" + def get_info( self, *, @@ -41,9 +42,9 @@ class _AgentAppService: caller_id: str, account_id: str, path: str, - ) -> WorkspaceListResponse: + ) -> BindingFileListResponse: self.calls.append(("list", tenant_id, app_id, agent_id, caller_type, caller_id, account_id, path)) - return WorkspaceListResponse(path=path, entries=[], truncated=False) + return BindingFileListResponse(path=path, entries=[], truncated=False) def read_file( self, @@ -55,11 +56,11 @@ class _AgentAppService: caller_id: str, account_id: str, path: str, - ) -> WorkspaceReadResponse: + ) -> BindingFileReadResponse: self.calls.append(("read", tenant_id, app_id, agent_id, caller_type, caller_id, account_id, path)) - return WorkspaceReadResponse(path=path, size=5, truncated=False, binary=False, text="hello") + return BindingFileReadResponse(path=path, size=5, truncated=False, binary=False, text="hello") - def upload_file( + def download_file( self, *, tenant_id: str, @@ -69,14 +70,17 @@ class _AgentAppService: caller_id: str, account_id: str, path: str, - ) -> AgentSandboxUploadDownload: - self.calls.append(("upload", tenant_id, app_id, agent_id, caller_type, caller_id, account_id, path)) - return AgentSandboxUploadDownload(url="https://files.example/report.txt") + ) -> AgentSandboxDownload: + self.calls.append(("download", tenant_id, app_id, agent_id, caller_type, caller_id, account_id, path)) + return AgentSandboxDownload(url="https://files.example/report.txt") class _WorkflowService: def __init__(self) -> None: - self.calls: list[tuple[str, str, str, str, str, str]] = [] + self.calls: list[tuple[str, ...]] = [] + + def resolve_app_id(self, *, tenant_id: str, app_id: str) -> str: + return app_id def list_files( self, @@ -88,9 +92,9 @@ class _WorkflowService: node_execution_id: str, path: str, session, - ) -> WorkspaceListResponse: + ) -> BindingFileListResponse: self.calls.append(("list", tenant_id, app_id, workflow_run_id, node_id, path)) - return WorkspaceListResponse(path=path, entries=[], truncated=False) + return BindingFileListResponse(path=path, entries=[], truncated=False) def read_file( self, @@ -102,11 +106,11 @@ class _WorkflowService: node_execution_id: str, path: str, session, - ) -> WorkspaceReadResponse: + ) -> BindingFileReadResponse: self.calls.append(("read", tenant_id, app_id, workflow_run_id, node_id, path)) - return WorkspaceReadResponse(path=path, size=5, truncated=False, binary=False, text="hello") + return BindingFileReadResponse(path=path, size=5, truncated=False, binary=False, text="hello") - def upload_file( + def download_file( self, *, tenant_id: str, @@ -114,11 +118,11 @@ class _WorkflowService: workflow_run_id: str, node_id: str, node_execution_id: str, + account_id: str, path: str, - session, - ) -> AgentSandboxUploadDownload: - self.calls.append(("upload", tenant_id, app_id, workflow_run_id, node_id, path)) - return AgentSandboxUploadDownload(url="https://files.example/upload.txt") + ) -> AgentSandboxDownload: + self.calls.append(("download", tenant_id, app_id, workflow_run_id, node_id, account_id, path)) + return AgentSandboxDownload(url="https://files.example/download.txt") def _app_model(app_id: str = "app-1") -> App: @@ -160,46 +164,40 @@ def test_handle_maps_sandbox_and_agent_backend_errors() -> None: module._handle(RuntimeError("boom")) -def test_agent_app_sandbox_resources_proxy_service(monkeypatch: pytest.MonkeyPatch, unbound_session: Session) -> None: +def test_agent_app_sandbox_resources_proxy_service(monkeypatch: pytest.MonkeyPatch) -> None: service = _AgentAppService() - session = unbound_session account = SimpleNamespace(id="account-1") - resolver = MagicMock(return_value=_app_model()) monkeypatch.setattr(module, "AgentAppSandboxService", lambda: service) - monkeypatch.setattr(module, "resolve_agent_runtime_app_model", resolver) monkeypatch.setattr( module, "query_params_from_request", lambda model: SimpleNamespace(caller_type="build_draft", caller_id="build-1", path="sub/report.txt"), ) - - info = unwrap(module.AgentAppSandboxInfoResource.get)(object(), session, account, "tenant-1", "agent-1") - listing = unwrap(module.AgentAppSandboxListResource.get)(object(), session, account, "tenant-1", "agent-1") - preview = unwrap(module.AgentAppSandboxReadResource.get)(object(), session, account, "tenant-1", "agent-1") - req_data = module.AgentSandboxUploadPayload.model_validate( + info = unwrap(module.AgentAppSandboxInfoResource.get)(object(), account, "tenant-1", "agent-1") + listing = unwrap(module.AgentAppSandboxListResource.get)(object(), account, "tenant-1", "agent-1") + preview = unwrap(module.AgentAppSandboxReadResource.get)(object(), account, "tenant-1", "agent-1") + req_data = module.AgentSandboxDownloadPayload.model_validate( {"caller_type": "build_draft", "caller_id": "build-1", "path": "report.txt"} ) - upload = unwrap(module.AgentAppSandboxUploadResource.post)( - object(), req_data, session, account, "tenant-1", "agent-1" - ) + download = unwrap(module.AgentAppSandboxDownloadResource.post)(object(), req_data, account, "tenant-1", "agent-1") assert info == {"workspace_cwd": "."} assert listing["path"] == "sub/report.txt" assert preview["text"] == "hello" - assert upload == {"url": "https://files.example/report.txt"} + assert download == {"url": "https://files.example/report.txt"} assert service.calls == [ ("info", "tenant-1", "app-1", "agent-1", "build_draft", "build-1", "account-1", ""), ("list", "tenant-1", "app-1", "agent-1", "build_draft", "build-1", "account-1", "sub/report.txt"), ("read", "tenant-1", "app-1", "agent-1", "build_draft", "build-1", "account-1", "sub/report.txt"), - ("upload", "tenant-1", "app-1", "agent-1", "build_draft", "build-1", "account-1", "report.txt"), + ("download", "tenant-1", "app-1", "agent-1", "build_draft", "build-1", "account-1", "report.txt"), ] - assert all(call.kwargs["session"] is session for call in resolver.call_args_list) -def test_agent_app_sandbox_resource_returns_normalized_errors( - monkeypatch: pytest.MonkeyPatch, unbound_session: Session -) -> None: +def test_agent_app_sandbox_resource_returns_normalized_errors(monkeypatch: pytest.MonkeyPatch) -> None: class FailingService: + def resolve_app_id(self, **kwargs): + return "app-1" + def get_info(self, **kwargs): raise AgentSandboxInspectorError("no_active_binding", "no active binding", status_code=404) @@ -207,20 +205,18 @@ def test_agent_app_sandbox_resource_returns_normalized_errors( raise AgentSandboxInspectorError("no_active_binding", "no active binding", status_code=404) monkeypatch.setattr(module, "AgentAppSandboxService", FailingService) - session = unbound_session account = SimpleNamespace(id="account-1") - monkeypatch.setattr(module, "resolve_agent_runtime_app_model", MagicMock(return_value=_app_model())) monkeypatch.setattr( module, "query_params_from_request", lambda model: SimpleNamespace(caller_type="conversation", caller_id="conv-1", path="."), ) - assert unwrap(module.AgentAppSandboxInfoResource.get)(object(), session, account, "tenant-1", "agent-1") == ( + assert unwrap(module.AgentAppSandboxInfoResource.get)(object(), account, "tenant-1", "agent-1") == ( {"code": "no_active_binding", "message": "no active binding"}, 404, ) - assert unwrap(module.AgentAppSandboxListResource.get)(object(), session, account, "tenant-1", "agent-1") == ( + assert unwrap(module.AgentAppSandboxListResource.get)(object(), account, "tenant-1", "agent-1") == ( {"code": "no_active_binding", "message": "no active binding"}, 404, ) @@ -242,18 +238,19 @@ def test_workflow_agent_sandbox_resources_proxy_service(monkeypatch: pytest.Monk preview = unwrap(module.WorkflowAgentSandboxReadResource.get)( object(), "tenant-1", app_model, "run-1", "agent-node" ) - req_data = module.WorkflowAgentSandboxUploadPayload.model_validate( - {"node_execution_id": "execution-1", "path": "upload.txt"} + req_data = module.WorkflowAgentSandboxDownloadPayload.model_validate( + {"node_execution_id": "execution-1", "path": "download.txt"} ) - upload = unwrap(module.WorkflowAgentSandboxUploadResource.post)( - object(), req_data, "tenant-1", app_model, "run-1", "agent-node" + account = SimpleNamespace(id="account-1") + download = unwrap(module.WorkflowAgentSandboxDownloadResource.post)( + object(), req_data, "tenant-1", account, "app-1", "run-1", "agent-node" ) assert listing["path"] == "out.txt" assert preview["text"] == "hello" - assert upload == {"url": "https://files.example/upload.txt"} + assert download == {"url": "https://files.example/download.txt"} assert service.calls == [ ("list", "tenant-1", "app-1", "run-1", "agent-node", "out.txt"), ("read", "tenant-1", "app-1", "run-1", "agent-node", "out.txt"), - ("upload", "tenant-1", "app-1", "run-1", "agent-node", "upload.txt"), + ("download", "tenant-1", "app-1", "run-1", "agent-node", "account-1", "download.txt"), ] diff --git a/api/tests/unit_tests/controllers/files/test_upload.py b/api/tests/unit_tests/controllers/files/test_upload.py index 4f6fcd6f3e0..5e0f5273253 100644 --- a/api/tests/unit_tests/controllers/files/test_upload.py +++ b/api/tests/unit_tests/controllers/files/test_upload.py @@ -1,13 +1,17 @@ import io import types +from contextlib import contextmanager from inspect import unwrap from unittest.mock import patch import pytest +from sqlalchemy.orm import Session from werkzeug.exceptions import Forbidden import controllers.files.upload as module from core.workflow.file_reference import build_file_reference +from models import Account, TenantAccountJoin +from models.account import AccountStatus def fake_request(args: dict, file=None): @@ -17,6 +21,22 @@ def fake_request(args: dict, file=None): ) +def _persist_account_memberships(session: Session) -> None: + account = Account(name="Tenant member", email="member@example.com", status=AccountStatus.ACTIVE) + account.id = "account-1" + decoy = Account(name="Other tenant member", email="decoy@example.com", status=AccountStatus.ACTIVE) + decoy.id = "account-outside-tenant" + session.add_all( + [ + account, + decoy, + TenantAccountJoin(tenant_id="tenant-1", account_id=account.id), + TenantAccountJoin(tenant_id="tenant-other", account_id=decoy.id), + ] + ) + session.commit() + + class DummyUser: def __init__(self, user_id="user-1"): self.id = user_id @@ -33,6 +53,16 @@ class DummyFile: return self.stream.read() +class RecordingStream(io.BytesIO): + def __init__(self, content: bytes, events: list[str]): + super().__init__(content) + self.events = events + + def read(self, *args, **kwargs): + self.events.append("file-read") + return super().read(*args, **kwargs) + + class DummyToolFile: def __init__(self, name="test.txt", mimetype="text/plain"): self.id = "file-id" @@ -94,6 +124,138 @@ class TestPluginUploadFileApi: assert tool_file_manager_instance.create_file_by_raw.call_args.kwargs["conversation_id"] == "conversation-1" mock_tool_file_manager.sign_file.assert_called_once_with(tool_file_id="file-id", extension=".docx") + @patch.object(module, "get_user") + @patch.object(module, "ToolFileManager") + @pytest.mark.parametrize("sqlite_session", [(Account, TenantAccountJoin)], indirect=True) + def test_account_upload_preserves_signed_account_owner( + self, + mock_tool_file_manager, + mock_get_user, + monkeypatch: pytest.MonkeyPatch, + sqlite_session: Session, + ): + _persist_account_memberships(sqlite_session) + events: list[str] = [] + dummy_file = DummyFile(filename="report.pdf", mimetype="application/pdf", content=b"account-owned") + dummy_file.stream = RecordingStream(b"account-owned", events) + + @contextmanager + def membership_session(): + events.append("membership-session-enter") + try: + yield sqlite_session + finally: + events.append("membership-session-exit") + + monkeypatch.setattr(module.session_factory, "create_session", membership_session) + monkeypatch.setattr( + module, + "request", + fake_request( + { + "timestamp": "123", + "nonce": "abc", + "sign": "sig", + "tenant_id": "tenant-1", + "user_id": "account-1", + "user_from": "account", + }, + file=dummy_file, + ), + ) + tool_file_manager = mock_tool_file_manager.return_value + tool_file_manager.create_file_by_raw.side_effect = lambda **_kwargs: ( + events.append("storage-create-file") or DummyToolFile(name="report.pdf", mimetype="application/pdf") + ) + mock_tool_file_manager.sign_file.return_value = "signed-url" + + with patch.object( + module, + "verify_plugin_file_signature", + side_effect=lambda **_kwargs: events.append("signature-verify") or True, + ) as verify_signature: + api = module.PluginUploadFileApi() + result, status_code = unwrap(api.post)(api) + + assert status_code == 201 + assert result["reference"] == build_file_reference(record_id="file-id") + assert events == [ + "membership-session-enter", + "membership-session-exit", + "signature-verify", + "file-read", + "storage-create-file", + ] + mock_get_user.assert_not_called() + verify_signature.assert_called_once_with( + filename="report.pdf", + mimetype="application/pdf", + tenant_id="tenant-1", + user_id="account-1", + conversation_id=None, + user_from="account", + timestamp="123", + nonce="abc", + sign="sig", + ) + tool_file_manager.create_file_by_raw.assert_called_once_with( + user_id="account-1", + tenant_id="tenant-1", + file_binary=b"account-owned", + mimetype="application/pdf", + filename="report.pdf", + conversation_id=None, + ) + + @patch.object(module, "verify_plugin_file_signature") + @patch.object(module, "get_user") + @patch.object(module, "ToolFileManager") + @pytest.mark.parametrize("sqlite_session", [(Account, TenantAccountJoin)], indirect=True) + def test_account_upload_rejects_owner_outside_tenant( + self, + mock_tool_file_manager, + mock_get_user, + mock_verify_signature, + monkeypatch: pytest.MonkeyPatch, + sqlite_session: Session, + ): + _persist_account_memberships(sqlite_session) + events: list[str] = [] + + @contextmanager + def membership_session(): + events.append("membership-session-enter") + try: + yield sqlite_session + finally: + events.append("membership-session-exit") + + monkeypatch.setattr(module.session_factory, "create_session", membership_session) + monkeypatch.setattr( + module, + "request", + fake_request( + { + "timestamp": "123", + "nonce": "abc", + "sign": "sig", + "tenant_id": "tenant-1", + "user_id": "account-outside-tenant", + "user_from": "account", + }, + file=DummyFile(), + ), + ) + + api = module.PluginUploadFileApi() + with pytest.raises(Forbidden): + unwrap(api.post)(api) + + assert events == ["membership-session-enter", "membership-session-exit"] + mock_get_user.assert_not_called() + mock_verify_signature.assert_not_called() + mock_tool_file_manager.assert_not_called() + def test_missing_file(self): module.request = fake_request( { diff --git a/api/tests/unit_tests/controllers/inner_api/test_agent_files.py b/api/tests/unit_tests/controllers/inner_api/test_agent_files.py index d01f9b5512f..a34ca510220 100644 --- a/api/tests/unit_tests/controllers/inner_api/test_agent_files.py +++ b/api/tests/unit_tests/controllers/inner_api/test_agent_files.py @@ -7,7 +7,11 @@ from unittest.mock import MagicMock, patch import pytest from flask import Flask -from controllers.inner_api.agent.files import AgentFileDownloadRequestApi, AgentFileUploadRequestApi +from controllers.inner_api.agent.files import ( + AgentFileDownloadRequestApi, + AgentFileRequestHttpError, + AgentFileUploadRequestApi, +) from core.workflow.file_reference import build_file_reference from services.file_request_service import DownloadFileRequestResult @@ -46,9 +50,74 @@ def test_upload_request_returns_origin_free_uri(app: Flask) -> None: tenant_id="tenant-1", user_id="canonical-end-user-1", conversation_id="conversation-1", + user_from=None, ) +def test_upload_request_preserves_tenant_scoped_account_owner(app: Flask) -> None: + payload = { + "tenant_id": "tenant-1", + "user_id": "account-1", + "user_from": "account", + "filename": "report.pdf", + "mimetype": "application/pdf", + "conversation_id": "conversation-1", + } + tenant = SimpleNamespace(id="tenant-1") + session = MagicMock() + with app.test_request_context("/", method="POST", json=payload): + with ( + patch(f"{MODULE}.TenantService") as tenant_service, + patch(f"{MODULE}.get_user") as get_user, + patch(f"{MODULE}.get_signed_file_uri_for_plugin", return_value="/files/upload/for-plugin?sign=1") as sign, + ): + tenant_service.get_tenant_by_id.return_value = tenant + tenant_service.account_belongs_to_tenant.return_value = True + response = _raw(AgentFileUploadRequestApi.post)(AgentFileUploadRequestApi(), session) + + assert response == {"upload_uri": "/files/upload/for-plugin?sign=1"} + get_user.assert_not_called() + tenant_service.account_belongs_to_tenant.assert_called_once_with("account-1", "tenant-1", session=session) + sign.assert_called_once_with( + filename="report.pdf", + mimetype="application/pdf", + tenant_id="tenant-1", + user_id="account-1", + conversation_id="conversation-1", + user_from="account", + ) + + +def test_upload_request_rejects_account_outside_tenant_without_signing(app: Flask) -> None: + payload = { + "tenant_id": "tenant-1", + "user_id": "account-outside-tenant", + "user_from": "account", + "filename": "report.pdf", + "mimetype": "application/pdf", + } + tenant = SimpleNamespace(id="tenant-1") + session = MagicMock() + with app.test_request_context("/", method="POST", json=payload): + with ( + patch(f"{MODULE}.TenantService") as tenant_service, + patch(f"{MODULE}.get_user") as get_user, + patch(f"{MODULE}.get_signed_file_uri_for_plugin") as sign, + ): + tenant_service.get_tenant_by_id.return_value = tenant + tenant_service.account_belongs_to_tenant.return_value = False + with pytest.raises(AgentFileRequestHttpError) as exc_info: + _raw(AgentFileUploadRequestApi.post)(AgentFileUploadRequestApi(), session) + + assert exc_info.value.error_code == "user_not_found" + assert exc_info.value.code == 404 + tenant_service.account_belongs_to_tenant.assert_called_once_with( + "account-outside-tenant", "tenant-1", session=session + ) + get_user.assert_not_called() + sign.assert_not_called() + + def test_download_request_returns_origin_free_uri_for_sandbox(app: Flask) -> None: reference = build_file_reference(record_id="tool-file-1") payload = { diff --git a/api/tests/unit_tests/core/tools/test_signature.py b/api/tests/unit_tests/core/tools/test_signature.py index 369c61455f8..142b82902bf 100644 --- a/api/tests/unit_tests/core/tools/test_signature.py +++ b/api/tests/unit_tests/core/tools/test_signature.py @@ -187,6 +187,36 @@ def test_get_signed_file_uri_for_plugin_and_verify_roundtrip(monkeypatch: pytest ) +def test_plugin_upload_signature_binds_account_user_from(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("core.tools.signature.time.time", lambda: 1700000000) + monkeypatch.setattr("core.tools.signature.os.urandom", lambda _: b"\x09" * 16) + monkeypatch.setattr("core.tools.signature.dify_config.SECRET_KEY", "unit-secret") + monkeypatch.setattr("core.tools.signature.dify_config.FILES_ACCESS_TIMEOUT", 60) + + uri = get_signed_file_uri_for_plugin( + filename="report.pdf", + mimetype="application/pdf", + tenant_id="tenant-id", + user_id="account-id", + user_from="account", + ) + query = parse_qs(urlparse(uri).query) + + assert query["user_from"] == ["account"] + signed = { + "filename": "report.pdf", + "mimetype": "application/pdf", + "tenant_id": "tenant-id", + "user_id": "account-id", + "timestamp": query["timestamp"][0], + "nonce": query["nonce"][0], + "sign": query["sign"][0], + } + assert verify_plugin_file_signature(**signed, user_from="account") is True + assert verify_plugin_file_signature(**signed, user_from="end-user") is False + assert verify_plugin_file_signature(**signed) is False + + def test_verify_plugin_file_signature_rejects_invalid_signatures(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr("core.tools.signature.time.time", lambda: 1700000000) monkeypatch.setattr("core.tools.signature.os.urandom", lambda _: b"\x07" * 16) diff --git a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_session_store.py b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_session_store.py index 74ddb25d197..1d12821b934 100644 --- a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_session_store.py +++ b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_session_store.py @@ -199,9 +199,13 @@ def test_load_or_create_persists_binding_on_node_execution(monkeypatch, home_sna session=session, ) - assert resolved.id == "binding-1" + assert resolved.backend_binding_ref == "backend-binding-1" + assert resolved.agent_id == "agent-1" + assert resolved.agent_config_version_id == "config-1" + assert resolved.agent_config_version_kind == "snapshot" owner_scope = get_active.call_args.kwargs["expected_owner_scope"] assert owner_scope.owner_scope_key == "node-1:workflow-binding-1" + session.rollback.assert_called_once_with() def test_load_existing_pointer_rejects_missing_workflow_identity(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/api/tests/unit_tests/services/test_agent_app_sandbox_service.py b/api/tests/unit_tests/services/test_agent_app_sandbox_service.py index 292b0aa8372..578d6cbbe74 100644 --- a/api/tests/unit_tests/services/test_agent_app_sandbox_service.py +++ b/api/tests/unit_tests/services/test_agent_app_sandbox_service.py @@ -1,24 +1,39 @@ -from contextlib import nullcontext +import json +from contextlib import contextmanager, nullcontext from datetime import datetime from types import SimpleNamespace +from typing import cast from unittest.mock import MagicMock import pytest -from dify_agent.protocol import WorkspaceListResponse, WorkspaceReadResponse +from dify_agent.client import Client +from dify_agent.protocol import BindingFileDownloadResponse, BindingFileListResponse, BindingFileReadResponse from sqlalchemy.orm import Session +from graphon.enums import WorkflowNodeExecutionStatus from models.agent import ( + Agent, + AgentConfigDraft, + AgentConfigDraftType, AgentConfigVersionKind, + AgentKind, + AgentScope, + AgentSource, + AgentStatus, AgentWorkingResourceStatus, AgentWorkspace, AgentWorkspaceBinding, AgentWorkspaceOwnerType, ) -from models.enums import ConversationFromSource +from models.agent_config_entities import AgentSoulConfig +from models.enums import ConversationFromSource, CreatorUserRole from models.model import App, AppMode, Conversation, IconType +from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom +from services import agent_app_sandbox_service as sandbox_module from services.agent.workspace_service import AgentWorkspaceService from services.agent_app_sandbox_service import ( AgentAppSandboxService, + AgentSandboxDownload, AgentSandboxInspectorError, WorkflowAgentSandboxService, ) @@ -104,6 +119,79 @@ def _use_session(monkeypatch: pytest.MonkeyPatch, session: Session) -> None: ) +def _add_app(session: Session, *, app_id: str, tenant_id: str) -> None: + session.add( + App( + id=app_id, + tenant_id=tenant_id, + name=f"App {app_id}", + description="", + mode=AppMode.AGENT, + icon_type=IconType.EMOJI, + icon="robot", + icon_background="#FFFFFF", + enable_site=False, + enable_api=False, + max_active_requests=0, + ) + ) + + +def _add_binding( + session: Session, + *, + binding_id: str, + workspace_id: str, + tenant_id: str = "tenant-1", + app_id: str = "app-1", + agent_id: str = "agent-1", + owner_type: AgentWorkspaceOwnerType, + owner_id: str, + owner_scope_key: str = "root", + status: AgentWorkingResourceStatus = AgentWorkingResourceStatus.ACTIVE, +) -> AgentWorkspaceBinding: + active_guard = 1 if status is AgentWorkingResourceStatus.ACTIVE else None + workspace = AgentWorkspace( + id=workspace_id, + tenant_id=tenant_id, + app_id=app_id, + owner_type=owner_type, + owner_id=owner_id, + owner_scope_key=owner_scope_key, + backend_workspace_ref=f"{workspace_id}-ref", + status=status, + active_guard=active_guard, + ) + binding = AgentWorkspaceBinding( + id=binding_id, + tenant_id=tenant_id, + app_id=app_id, + workspace_id=workspace_id, + agent_id=agent_id, + base_home_snapshot_id=None, + agent_config_version_id=f"{binding_id}-config", + agent_config_version_kind=AgentConfigVersionKind.SNAPSHOT, + backend_binding_ref=f"{binding_id}-ref", + status=status, + ) + session.add_all([workspace, binding]) + return binding + + +def _download_client() -> MagicMock: + client = MagicMock() + client.download_binding_file_sync.return_value = BindingFileDownloadResponse(reference="dify-file-ref:canonical") + return client + + +def _stub_download_response(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + sandbox_module, + "_download_response", + lambda **_kwargs: AgentSandboxDownload(url="https://files.example/report.txt"), + ) + + @pytest.mark.parametrize( "sqlite_session", [(AgentWorkspace, AgentWorkspaceBinding, App, Conversation)], @@ -117,8 +205,8 @@ def test_agent_app_file_browsing_uses_conversation_pointer( sqlite_session.commit() _use_session(monkeypatch, sqlite_session) client = MagicMock() - response = WorkspaceListResponse(path=".", entries=[], truncated=False) - client.list_workspace_files_sync.return_value = response + response = BindingFileListResponse(path=".", entries=[], truncated=False) + client.list_binding_files_sync.return_value = response result = AgentAppSandboxService(client_factory=lambda: nullcontext(client)).list_files( tenant_id="tenant-1", @@ -131,7 +219,7 @@ def test_agent_app_file_browsing_uses_conversation_pointer( ) assert result is response - client.list_workspace_files_sync.assert_called_once_with(expected.backend_binding_ref, ".") + client.list_binding_files_sync.assert_called_once_with(expected.backend_binding_ref, ".") @pytest.mark.parametrize( @@ -160,7 +248,443 @@ def test_agent_app_file_browsing_rejects_other_account( ) assert exc_info.value.code == "no_active_binding" - client.list_workspace_files_sync.assert_not_called() + client.list_binding_files_sync.assert_not_called() + + +@pytest.mark.parametrize( + "sqlite_session", + [(AgentWorkspace, AgentWorkspaceBinding, App, Conversation)], + indirect=True, +) +def test_agent_conversation_download_resolves_only_exact_active_owner_chain( + monkeypatch: pytest.MonkeyPatch, + sqlite_session: Session, +) -> None: + _add_app(sqlite_session, app_id="app-1", tenant_id="tenant-1") + _add_app(sqlite_session, app_id="app-other", tenant_id="tenant-other") + valid = _add_binding( + sqlite_session, + binding_id="binding-valid", + workspace_id="workspace-valid", + owner_type=AgentWorkspaceOwnerType.CONVERSATION, + owner_id="conversation-valid", + ) + wrong_owner = _add_binding( + sqlite_session, + binding_id="binding-wrong-owner", + workspace_id="workspace-wrong-owner", + owner_type=AgentWorkspaceOwnerType.CONVERSATION, + owner_id="conversation-not-the-caller", + ) + retired = _add_binding( + sqlite_session, + binding_id="binding-retired", + workspace_id="workspace-retired", + owner_type=AgentWorkspaceOwnerType.CONVERSATION, + owner_id="conversation-retired", + status=AgentWorkingResourceStatus.RETIRED, + ) + cross_tenant = _add_binding( + sqlite_session, + binding_id="binding-cross-tenant", + workspace_id="workspace-cross-tenant", + tenant_id="tenant-other", + app_id="app-other", + owner_type=AgentWorkspaceOwnerType.CONVERSATION, + owner_id="conversation-cross-tenant", + ) + conversations = [ + Conversation( + id="conversation-valid", + app_id="app-1", + mode=AppMode.AGENT, + name="Valid", + from_source=ConversationFromSource.CONSOLE, + from_account_id="account-1", + is_deleted=False, + agent_workspace_binding_id=valid.id, + ), + Conversation( + id="conversation-wrong-owner", + app_id="app-1", + mode=AppMode.AGENT, + name="Wrong owner", + from_source=ConversationFromSource.CONSOLE, + from_account_id="account-1", + is_deleted=False, + agent_workspace_binding_id=wrong_owner.id, + ), + Conversation( + id="conversation-retired", + app_id="app-1", + mode=AppMode.AGENT, + name="Retired", + from_source=ConversationFromSource.CONSOLE, + from_account_id="account-1", + is_deleted=False, + agent_workspace_binding_id=retired.id, + ), + Conversation( + id="conversation-cross-tenant", + app_id="app-other", + mode=AppMode.AGENT, + name="Cross tenant", + from_source=ConversationFromSource.CONSOLE, + from_account_id="account-other", + is_deleted=False, + agent_workspace_binding_id=cross_tenant.id, + ), + ] + for conversation in conversations: + conversation._inputs = {} + sqlite_session.add_all(conversations) + sqlite_session.commit() + _use_session(monkeypatch, sqlite_session) + _stub_download_response(monkeypatch) + client = _download_client() + service = AgentAppSandboxService(client_factory=lambda: nullcontext(cast(Client, client))) + + result = service.download_file( + tenant_id="tenant-1", + app_id="app-1", + agent_id="agent-1", + caller_type="conversation", + caller_id="conversation-valid", + account_id="account-1", + path="report.txt", + ) + + assert result.url == "https://files.example/report.txt" + request = client.download_binding_file_sync.call_args.args[0] + assert request.backend_binding_ref == "binding-valid-ref" + client.download_binding_file_sync.reset_mock() + + rejected_locators = [ + {"account_id": "account-other"}, + {"app_id": "app-other"}, + {"caller_id": "conversation-wrong-owner"}, + {"caller_id": "conversation-retired"}, + { + "tenant_id": "tenant-1", + "app_id": "app-other", + "caller_id": "conversation-cross-tenant", + "account_id": "account-other", + }, + ] + for override in rejected_locators: + locator = { + "tenant_id": "tenant-1", + "app_id": "app-1", + "agent_id": "agent-1", + "caller_type": "conversation", + "caller_id": "conversation-valid", + "account_id": "account-1", + "path": "report.txt", + } + locator.update(override) + with pytest.raises(AgentSandboxInspectorError, match="active Agent Workspace Binding"): + service.download_file(**locator) # type: ignore[arg-type] + + client.download_binding_file_sync.assert_not_called() + + +@pytest.mark.parametrize( + "sqlite_session", + [(Agent, AgentConfigDraft, AgentWorkspace, AgentWorkspaceBinding)], + indirect=True, +) +def test_agent_build_draft_download_resolves_only_exact_active_owner_chain( + monkeypatch: pytest.MonkeyPatch, + sqlite_session: Session, +) -> None: + sqlite_session.add_all( + [ + Agent( + id="agent-1", + tenant_id="tenant-1", + name="Agent", + description="", + agent_kind=AgentKind.DIFY_AGENT, + scope=AgentScope.ROSTER, + source=AgentSource.AGENT_APP, + app_id="app-1", + status=AgentStatus.ACTIVE, + ), + Agent( + id="agent-cross-tenant", + tenant_id="tenant-other", + name="Other tenant Agent", + description="", + agent_kind=AgentKind.DIFY_AGENT, + scope=AgentScope.ROSTER, + source=AgentSource.AGENT_APP, + app_id="app-other", + status=AgentStatus.ACTIVE, + ), + ] + ) + valid = _add_binding( + sqlite_session, + binding_id="binding-build-valid", + workspace_id="workspace-build-valid", + owner_type=AgentWorkspaceOwnerType.BUILD_DRAFT, + owner_id="draft-valid", + ) + wrong_owner = _add_binding( + sqlite_session, + binding_id="binding-build-wrong-owner", + workspace_id="workspace-build-wrong-owner", + owner_type=AgentWorkspaceOwnerType.BUILD_DRAFT, + owner_id="draft-not-the-caller", + ) + retired = _add_binding( + sqlite_session, + binding_id="binding-build-retired", + workspace_id="workspace-build-retired", + owner_type=AgentWorkspaceOwnerType.BUILD_DRAFT, + owner_id="draft-retired", + status=AgentWorkingResourceStatus.RETIRED, + ) + drafts = [ + AgentConfigDraft( + id="draft-valid", + tenant_id="tenant-1", + agent_id="agent-1", + draft_type=AgentConfigDraftType.DEBUG_BUILD, + account_id="account-1", + draft_owner_key="account-1", + agent_workspace_binding_id=valid.id, + config_snapshot=AgentSoulConfig(), + ), + AgentConfigDraft( + id="draft-wrong-owner", + tenant_id="tenant-1", + agent_id="agent-1", + draft_type=AgentConfigDraftType.DEBUG_BUILD, + account_id="account-2", + draft_owner_key="account-2", + agent_workspace_binding_id=wrong_owner.id, + config_snapshot=AgentSoulConfig(), + ), + AgentConfigDraft( + id="draft-retired", + tenant_id="tenant-1", + agent_id="agent-1", + draft_type=AgentConfigDraftType.DEBUG_BUILD, + account_id="account-3", + draft_owner_key="account-3", + agent_workspace_binding_id=retired.id, + config_snapshot=AgentSoulConfig(), + ), + AgentConfigDraft( + id="draft-cross-tenant", + tenant_id="tenant-other", + agent_id="agent-cross-tenant", + draft_type=AgentConfigDraftType.DEBUG_BUILD, + account_id="account-other", + draft_owner_key="account-other", + agent_workspace_binding_id=None, + config_snapshot=AgentSoulConfig(), + ), + ] + sqlite_session.add_all(drafts) + sqlite_session.commit() + _use_session(monkeypatch, sqlite_session) + _stub_download_response(monkeypatch) + client = _download_client() + service = AgentAppSandboxService(client_factory=lambda: nullcontext(cast(Client, client))) + + result = service.download_file( + tenant_id="tenant-1", + app_id="app-1", + agent_id="agent-1", + caller_type="build_draft", + caller_id="draft-valid", + account_id="account-1", + path="report.txt", + ) + + assert result.url == "https://files.example/report.txt" + assert client.download_binding_file_sync.call_args.args[0].backend_binding_ref == "binding-build-valid-ref" + client.download_binding_file_sync.reset_mock() + + rejected_locators = [ + {"account_id": "account-other"}, + {"app_id": "app-other"}, + {"caller_id": "draft-wrong-owner", "account_id": "account-2"}, + {"caller_id": "draft-retired", "account_id": "account-3"}, + { + "tenant_id": "tenant-1", + "app_id": "app-other", + "agent_id": "agent-cross-tenant", + "caller_id": "draft-cross-tenant", + "account_id": "account-other", + }, + ] + for override in rejected_locators: + locator = { + "tenant_id": "tenant-1", + "app_id": "app-1", + "agent_id": "agent-1", + "caller_type": "build_draft", + "caller_id": "draft-valid", + "account_id": "account-1", + "path": "report.txt", + } + locator.update(override) + with pytest.raises(AgentSandboxInspectorError, match="active Agent Workspace Binding"): + service.download_file(**locator) # type: ignore[arg-type] + + client.download_binding_file_sync.assert_not_called() + + +def _workflow_execution( + *, + execution_id: str, + tenant_id: str = "tenant-1", + app_id: str = "app-1", + workflow_run_id: str = "run-1", + node_id: str = "node-1", + binding_id: str, + created_by: str = "historical-account", +) -> WorkflowNodeExecutionModel: + return WorkflowNodeExecutionModel( + id=execution_id, + tenant_id=tenant_id, + app_id=app_id, + workflow_id="workflow-1", + triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN, + workflow_run_id=workflow_run_id, + index=1, + node_id=node_id, + node_type="agent", + title=node_id, + agent_workspace_binding_id=binding_id, + inputs=None, + process_data=json.dumps({"workflow_agent_binding_id": "workflow-binding-1"}), + outputs=None, + status=WorkflowNodeExecutionStatus.SUCCEEDED, + error=None, + created_by_role=CreatorUserRole.ACCOUNT, + created_by=created_by, + ) + + +@pytest.mark.parametrize( + "sqlite_session", + [(WorkflowNodeExecutionModel, AgentWorkspace, AgentWorkspaceBinding)], + indirect=True, +) +def test_workflow_download_resolves_only_exact_active_owner_chain( + monkeypatch: pytest.MonkeyPatch, + sqlite_session: Session, +) -> None: + valid = _add_binding( + sqlite_session, + binding_id="binding-workflow-valid", + workspace_id="workspace-workflow-valid", + owner_type=AgentWorkspaceOwnerType.WORKFLOW_RUN, + owner_id="run-1", + owner_scope_key="node-1:workflow-binding-1", + ) + wrong_owner = _add_binding( + sqlite_session, + binding_id="binding-workflow-wrong-owner", + workspace_id="workspace-workflow-wrong-owner", + owner_type=AgentWorkspaceOwnerType.WORKFLOW_RUN, + owner_id="run-not-the-caller", + owner_scope_key="node-1:workflow-binding-1", + ) + retired = AgentWorkspaceBinding( + id="binding-workflow-retired", + tenant_id="tenant-1", + app_id="app-1", + workspace_id=valid.workspace_id, + agent_id="agent-1", + base_home_snapshot_id=None, + agent_config_version_id="binding-workflow-retired-config", + agent_config_version_kind=AgentConfigVersionKind.SNAPSHOT, + backend_binding_ref="binding-workflow-retired-ref", + status=AgentWorkingResourceStatus.RETIRED, + ) + sqlite_session.add(retired) + sqlite_session.add_all( + [ + _workflow_execution(execution_id="execution-valid", binding_id=valid.id), + _workflow_execution( + execution_id="execution-cross-tenant", + tenant_id="tenant-other", + binding_id=valid.id, + ), + _workflow_execution(execution_id="execution-wrong-app", app_id="app-other", binding_id=valid.id), + _workflow_execution(execution_id="execution-wrong-run", workflow_run_id="run-other", binding_id=valid.id), + _workflow_execution(execution_id="execution-wrong-node", node_id="node-other", binding_id=valid.id), + _workflow_execution(execution_id="execution-wrong-owner", binding_id=wrong_owner.id), + _workflow_execution(execution_id="execution-retired", binding_id=retired.id), + ] + ) + sqlite_session.commit() + persisted_execution = sqlite_session.get(WorkflowNodeExecutionModel, "execution-valid") + assert persisted_execution is not None + assert persisted_execution.created_by == "historical-account" + _use_session(monkeypatch, sqlite_session) + client = _download_client() + request_download = MagicMock( + return_value=SimpleNamespace(download_uri="/files/tools/report.txt?timestamp=1&sign=2") + ) + monkeypatch.setattr( + sandbox_module, + "FileRequestService", + lambda: SimpleNamespace(request_download=request_download), + ) + monkeypatch.setattr(sandbox_module.dify_config, "FILES_URL", "https://files.example") + service = WorkflowAgentSandboxService(client_factory=lambda: nullcontext(cast(Client, client))) + + result = service.download_file( + tenant_id="tenant-1", + app_id="app-1", + workflow_run_id="run-1", + node_id="node-1", + node_execution_id="execution-valid", + account_id="authenticated-account", + path="report.txt", + ) + + assert result.url == "https://files.example/files/tools/report.txt?timestamp=1&sign=2&as_attachment=true" + request = client.download_binding_file_sync.call_args.args[0] + assert request.backend_binding_ref == "binding-workflow-valid-ref" + assert request.execution_context.user_id == "authenticated-account" + assert request.execution_context.user_from == "account" + request_download.assert_called_once_with( + tenant_id="tenant-1", + user_id="authenticated-account", + user_from="account", + invoke_from="debugger", + file_mapping={"transfer_method": "tool_file", "reference": "dify-file-ref:canonical"}, + ) + client.download_binding_file_sync.reset_mock() + + for node_execution_id in ( + "execution-cross-tenant", + "execution-wrong-app", + "execution-wrong-run", + "execution-wrong-node", + "execution-wrong-owner", + "execution-retired", + ): + with pytest.raises(AgentSandboxInspectorError, match="active Workspace Binding"): + service.download_file( + tenant_id="tenant-1", + app_id="app-1", + workflow_run_id="run-1", + node_id="node-1", + node_execution_id=node_execution_id, + account_id="authenticated-account", + path="report.txt", + ) + + client.download_binding_file_sync.assert_not_called() + assert request_download.call_count == 1 @pytest.mark.parametrize( @@ -185,12 +709,14 @@ def test_agent_app_file_browsing_uses_build_draft_caller( binding = SimpleNamespace( agent_id="agent-1", backend_binding_ref="binding-build-ref", + agent_config_version_id="config-1", + agent_config_version_kind=AgentConfigVersionKind.DRAFT, ) get_binding = MagicMock(return_value=binding) monkeypatch.setattr(AgentWorkspaceService, "get_active_binding", get_binding) client = MagicMock() - response = WorkspaceListResponse(path=".", entries=[], truncated=False) - client.list_workspace_files_sync.return_value = response + response = BindingFileListResponse(path=".", entries=[], truncated=False) + client.list_binding_files_sync.return_value = response result = AgentAppSandboxService(client_factory=lambda: nullcontext(client)).list_files( tenant_id="tenant-1", @@ -207,7 +733,7 @@ def test_agent_app_file_browsing_uses_build_draft_caller( assert owner_scope.app_id == runtime_app_id assert owner_scope.owner_type is AgentWorkspaceOwnerType.BUILD_DRAFT assert owner_scope.owner_id == "build-1" - client.list_workspace_files_sync.assert_called_once_with("binding-build-ref", ".") + client.list_binding_files_sync.assert_called_once_with("binding-build-ref", ".") def test_workflow_file_access_uses_node_execution_pointer(monkeypatch: pytest.MonkeyPatch) -> None: @@ -217,12 +743,17 @@ def test_workflow_file_access_uses_node_execution_pointer(monkeypatch: pytest.Mo ) session = MagicMock() session.scalar.return_value = execution - binding = SimpleNamespace(backend_binding_ref="binding-workflow-ref") + binding = SimpleNamespace( + agent_id="agent-1", + backend_binding_ref="binding-workflow-ref", + agent_config_version_id="config-1", + agent_config_version_kind=AgentConfigVersionKind.SNAPSHOT, + ) get_binding = MagicMock(return_value=binding) monkeypatch.setattr(AgentWorkspaceService, "get_active_binding", get_binding) client = MagicMock() - response = WorkspaceReadResponse(path="report.txt", size=2, truncated=False, binary=False, text="ok") - client.read_workspace_file_sync.return_value = response + response = BindingFileReadResponse(path="report.txt", size=2, truncated=False, binary=False, text="ok") + client.read_binding_file_sync.return_value = response result = WorkflowAgentSandboxService(client_factory=lambda: nullcontext(client)).read_file( tenant_id="tenant-1", @@ -236,4 +767,238 @@ def test_workflow_file_access_uses_node_execution_pointer(monkeypatch: pytest.Mo assert result is response assert get_binding.call_args.kwargs["binding_id"] == "binding-workflow" - client.read_workspace_file_sync.assert_called_once_with("binding-workflow-ref", "report.txt") + client.read_binding_file_sync.assert_called_once_with("binding-workflow-ref", "report.txt") + session.rollback.assert_called_once_with() + + +def test_workflow_download_uses_authenticated_account_and_trusted_file_request( + monkeypatch: pytest.MonkeyPatch, +) -> None: + execution = SimpleNamespace( + agent_workspace_binding_id="binding-workflow", + process_data_dict={"workflow_agent_binding_id": "workflow-binding-1"}, + ) + session = MagicMock() + session.scalar.return_value = execution + binding = SimpleNamespace( + agent_id="agent-1", + backend_binding_ref="binding-workflow-ref", + agent_config_version_id="config-1", + agent_config_version_kind=AgentConfigVersionKind.SNAPSHOT, + ) + monkeypatch.setattr(AgentWorkspaceService, "get_active_binding", MagicMock(return_value=binding)) + events: list[str] = [] + + @contextmanager + def session_scope(): + try: + yield session + finally: + events.append("session-exit") + + monkeypatch.setattr(sandbox_module.session_factory, "create_session", session_scope) + client = MagicMock() + client.download_binding_file_sync.side_effect = lambda _request: ( + events.append("client-download") or BindingFileDownloadResponse(reference="dify-file-ref:canonical") + ) + request_download = MagicMock( + side_effect=lambda **_kwargs: ( + events.append("file-request") or SimpleNamespace(download_uri="/files/tools/report.txt?timestamp=1&sign=2") + ) + ) + monkeypatch.setattr( + "services.agent_app_sandbox_service.FileRequestService", + lambda: SimpleNamespace(request_download=request_download), + ) + monkeypatch.setattr("services.agent_app_sandbox_service.dify_config.FILES_URL", "https://files.example") + + result = WorkflowAgentSandboxService(client_factory=lambda: nullcontext(client)).download_file( + tenant_id="tenant-1", + app_id="app-1", + workflow_run_id="run-1", + node_id="node-1", + node_execution_id="execution-1", + account_id="account-1", + path="report.txt", + ) + + request = client.download_binding_file_sync.call_args.args[0] + assert request.execution_context.user_id == "account-1" + assert request.execution_context.user_from == "account" + assert request.execution_context.node_execution_id == "execution-1" + session.rollback.assert_called_once_with() + assert events == ["session-exit", "client-download", "file-request"] + request_download.assert_called_once_with( + tenant_id="tenant-1", + user_id="account-1", + user_from="account", + invoke_from="debugger", + file_mapping={"transfer_method": "tool_file", "reference": "dify-file-ref:canonical"}, + ) + assert result.url == "https://files.example/files/tools/report.txt?timestamp=1&sign=2&as_attachment=true" + + +def test_agent_app_download_uses_complete_account_context_after_session_exit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + events: list[str] = [] + session = MagicMock() + session.scalar.side_effect = [ + SimpleNamespace(app_id="app-1", backing_app_id=None), + SimpleNamespace(agent_workspace_binding_id="binding-build"), + ] + + @contextmanager + def session_scope(): + try: + yield session + finally: + events.append("session-exit") + + monkeypatch.setattr(sandbox_module.session_factory, "create_session", session_scope) + binding = SimpleNamespace( + agent_id="agent-1", + backend_binding_ref="binding-build-ref", + agent_config_version_id="config-1", + agent_config_version_kind=AgentConfigVersionKind.DRAFT, + ) + monkeypatch.setattr(AgentWorkspaceService, "get_active_binding", MagicMock(return_value=binding)) + client = MagicMock() + client.download_binding_file_sync.side_effect = lambda _request: ( + events.append("client-download") or BindingFileDownloadResponse(reference="dify-file-ref:canonical") + ) + request_download = MagicMock( + side_effect=lambda **_kwargs: ( + events.append("file-request") or SimpleNamespace(download_uri="/files/tools/report.txt?timestamp=1&sign=2") + ) + ) + monkeypatch.setattr( + sandbox_module, + "FileRequestService", + lambda: SimpleNamespace(request_download=request_download), + ) + monkeypatch.setattr(sandbox_module.dify_config, "FILES_URL", "https://files.example") + + result = AgentAppSandboxService(client_factory=lambda: nullcontext(client)).download_file( + tenant_id="tenant-1", + app_id="app-1", + agent_id="agent-1", + caller_type="build_draft", + caller_id="build-1", + account_id="account-1", + path="report.txt", + ) + + request = client.download_binding_file_sync.call_args.args[0] + assert request.backend_binding_ref == "binding-build-ref" + assert request.path == "report.txt" + assert request.execution_context.model_dump(exclude_none=True) == { + "tenant_id": "tenant-1", + "user_id": "account-1", + "user_from": "account", + "app_id": "app-1", + "agent_id": "agent-1", + "agent_config_version_id": "config-1", + "agent_config_version_kind": "draft", + "agent_mode": "agent_app", + "invoke_from": "debugger", + } + assert events == ["session-exit", "client-download", "file-request"] + request_download.assert_called_once_with( + tenant_id="tenant-1", + user_id="account-1", + user_from="account", + invoke_from="debugger", + file_mapping={"transfer_method": "tool_file", "reference": "dify-file-ref:canonical"}, + ) + assert result.url == "https://files.example/files/tools/report.txt?timestamp=1&sign=2&as_attachment=true" + + +def test_file_request_rejection_maps_to_download_unavailable(monkeypatch: pytest.MonkeyPatch) -> None: + request_download = MagicMock(side_effect=ValueError("reference is not accessible")) + monkeypatch.setattr( + sandbox_module, + "FileRequestService", + lambda: SimpleNamespace(request_download=request_download), + ) + + with pytest.raises(AgentSandboxInspectorError) as exc_info: + sandbox_module._download_response( + tenant_id="tenant-1", + account_id="account-1", + reference="dify-file-ref:untrusted", + ) + + assert exc_info.value.code == "binding_file_download_unavailable" + assert exc_info.value.status_code == 502 + + +@pytest.mark.parametrize( + "execution", + [ + pytest.param( + SimpleNamespace( + agent_workspace_binding_id=None, + process_data_dict={"workflow_agent_binding_id": "workflow-binding-1"}, + ), + id="missing-binding-pointer", + ), + pytest.param( + SimpleNamespace( + agent_workspace_binding_id="binding-workflow", + process_data_dict={}, + ), + id="missing-process-data", + ), + ], +) +def test_workflow_download_rejects_missing_binding_metadata_before_network( + monkeypatch: pytest.MonkeyPatch, + execution: SimpleNamespace, +) -> None: + session = MagicMock() + session.scalar.return_value = execution + _use_session(monkeypatch, session) + client = MagicMock() + + with pytest.raises(AgentSandboxInspectorError) as exc_info: + WorkflowAgentSandboxService(client_factory=lambda: nullcontext(client)).download_file( + tenant_id="tenant-1", + app_id="app-1", + workflow_run_id="run-1", + node_id="node-1", + node_execution_id="execution-1", + account_id="account-1", + path="report.txt", + ) + + assert exc_info.value.code == "no_active_binding" + client.download_binding_file_sync.assert_not_called() + + +def test_workflow_download_rejects_non_active_or_mismatched_binding_before_network( + monkeypatch: pytest.MonkeyPatch, +) -> None: + execution = SimpleNamespace( + agent_workspace_binding_id="binding-workflow", + process_data_dict={"workflow_agent_binding_id": "workflow-binding-1"}, + ) + session = MagicMock() + session.scalar.return_value = execution + _use_session(monkeypatch, session) + monkeypatch.setattr(AgentWorkspaceService, "get_active_binding", MagicMock(return_value=None)) + client = MagicMock() + + with pytest.raises(AgentSandboxInspectorError) as exc_info: + WorkflowAgentSandboxService(client_factory=lambda: nullcontext(client)).download_file( + tenant_id="tenant-1", + app_id="app-1", + workflow_run_id="run-1", + node_id="node-1", + node_execution_id="execution-1", + account_id="account-1", + path="report.txt", + ) + + assert exc_info.value.code == "no_active_binding" + client.download_binding_file_sync.assert_not_called() diff --git a/dify-agent-runtime/cmd/dify-agent-cli/main.go b/dify-agent-runtime/cmd/dify-agent-cli/main.go index 176934886a1..2111d6cd4f1 100644 --- a/dify-agent-runtime/cmd/dify-agent-cli/main.go +++ b/dify-agent-runtime/cmd/dify-agent-cli/main.go @@ -103,14 +103,21 @@ func newFileCommand() *cobra.Command { Short: "Upload or download workflow files through the Agent Stub.", } + var noDownloadLink bool upload := &cobra.Command{ Use: "upload PATH", Short: "Upload one sandbox-local file as a ToolFile output reference.", Args: cobra.ExactArgs(1), RunE: withEnv(func(env *agentcli.Environment, args []string, _ *cobra.Command) error { - return agentcli.RunFileUpload(env, args[0]) + return agentcli.RunFileUpload(env, args[0], noDownloadLink) }), } + upload.Flags().BoolVar( + &noDownloadLink, + "no-download-link", + false, + "Skip creating a public download link after upload.", + ) var downloadTo string download := &cobra.Command{ diff --git a/dify-agent-runtime/cmd/dify-agent-cli/main_test.go b/dify-agent-runtime/cmd/dify-agent-cli/main_test.go index 89b2677e96d..8e024122c6a 100644 --- a/dify-agent-runtime/cmd/dify-agent-cli/main_test.go +++ b/dify-agent-runtime/cmd/dify-agent-cli/main_test.go @@ -53,7 +53,12 @@ func TestCommandHelp(t *testing.T) { { name: "file upload", args: []string{"file", "upload", "--help"}, - want: []string{"dify-agent file upload", "Upload one sandbox-local file"}, + want: []string{ + "dify-agent file upload", + "Upload one sandbox-local file", + "--no-download-link", + "Skip creating a public download link after upload.", + }, }, { name: "file download", diff --git a/dify-agent-runtime/internal/agentcli/file.go b/dify-agent-runtime/internal/agentcli/file.go index 93dd3aabe83..b1f5179bb60 100644 --- a/dify-agent-runtime/internal/agentcli/file.go +++ b/dify-agent-runtime/internal/agentcli/file.go @@ -2,6 +2,7 @@ package agentcli import ( "context" + "encoding/base64" "encoding/json" "fmt" "io" @@ -15,7 +16,7 @@ import ( type FileUploadResponse struct { TransferMethod string `json:"transfer_method"` Reference string `json:"reference"` - PublicDownloadURL string `json:"public_download_url"` + PublicDownloadURL string `json:"public_download_url,omitempty"` } // FileDownloadResponse is the response from a file download request. @@ -27,14 +28,14 @@ type FileDownloadResponse struct { } // RunFileUpload executes the `file upload` command. -func RunFileUpload(env *Environment, path string) error { +func RunFileUpload(env *Environment, path string, noDownloadLink bool) error { client, err := NewStubClient(env) if err != nil { return err } defer func() { _ = client.Close() }() - return runFileUpload(client, path, os.Stdout) + return runFileUpload(client, path, noDownloadLink, os.Stdout) } type fileUploadClient interface { @@ -48,7 +49,7 @@ type fileUploadClient interface { ) (*FileDownloadResponse, error) } -func runFileUpload(client fileUploadClient, path string, output io.Writer) error { +func runFileUpload(client fileUploadClient, path string, noDownloadLink bool, output io.Writer) error { absPath, err := filepath.Abs(path) if err != nil { return fmt.Errorf("resolve path: %w", err) @@ -83,24 +84,47 @@ func runFileUpload(client fileUploadClient, path string, output io.Writer) error if reference == "" { return fmt.Errorf("signed file upload response is missing reference") } - - // Step 3: Request download URL for the uploaded file - ref := reference - dlResp, err := client.CreateFileDownloadURL(ctx, "tool_file", &ref, nil, true) - if err != nil { - return err + if noDownloadLink && !isCanonicalDifyFileReference(reference) { + return fmt.Errorf("signed file upload response has invalid reference") } result := FileUploadResponse{ - TransferMethod: "tool_file", - Reference: reference, - PublicDownloadURL: dlResp.DownloadURL, + TransferMethod: "tool_file", + Reference: reference, + } + if !noDownloadLink { + // Step 3: Request a browser-visible URL unless the caller only needs the + // canonical ToolFile reference. + ref := reference + dlResp, err := client.CreateFileDownloadURL(ctx, "tool_file", &ref, nil, true) + if err != nil { + return err + } + result.PublicDownloadURL = dlResp.DownloadURL } out, _ := json.Marshal(result) _, _ = fmt.Fprintln(output, string(out)) return nil } +func isCanonicalDifyFileReference(reference string) bool { + encodedPayload, found := strings.CutPrefix(reference, "dify-file-ref:") + if !found || encodedPayload == "" { + return false + } + payloadJSON, err := base64.URLEncoding.DecodeString(encodedPayload) + if err != nil { + return false + } + var payload struct { + RecordID string `json:"record_id"` + } + if err := json.Unmarshal(payloadJSON, &payload); err != nil { + return false + } + return payload.RecordID != "" +} + // RunFileDownload executes the `file download` command. func RunFileDownload(env *Environment, transferMethod string, referenceOrURL string, localDir string) error { var reference *string diff --git a/dify-agent-runtime/internal/agentcli/file_test.go b/dify-agent-runtime/internal/agentcli/file_test.go index a76fcffb7f5..a610457e6c2 100644 --- a/dify-agent-runtime/internal/agentcli/file_test.go +++ b/dify-agent-runtime/internal/agentcli/file_test.go @@ -13,24 +13,48 @@ import ( ) type fakeFileUploadClient struct { - forFrontend bool + forFrontend bool + downloadRequestCall int + uploadResponse []byte + calls []string + filename string + mimetype string + uploadURL string + uploadedBytes []byte + downloadReference string } func (f *fakeFileUploadClient) CreateFileUploadURL(_ context.Context, filename, mimetype string) (string, error) { + f.calls = append(f.calls, "upload-request") + f.filename = filename + f.mimetype = mimetype return "https://sandbox-files.example.com/files/upload/for-plugin?sign=1", nil } func (f *fakeFileUploadClient) UploadFileToURL(uploadURL, filePath, filename, mimetype string) ([]byte, error) { - return []byte(`{"reference":"dify-file-ref:canonical"}`), nil + f.calls = append(f.calls, "multipart-upload") + f.uploadURL = uploadURL + f.filename = filename + f.mimetype = mimetype + f.uploadedBytes, _ = os.ReadFile(filePath) + if f.uploadResponse != nil { + return f.uploadResponse, nil + } + return []byte(`{"reference":"dify-file-ref:eyJyZWNvcmRfaWQiOiJ0b29sLTEifQ=="}`), nil } func (f *fakeFileUploadClient) CreateFileDownloadURL( _ context.Context, _ string, - _, _ *string, + reference, _ *string, forFrontend bool, ) (*FileDownloadResponse, error) { + f.calls = append(f.calls, "download-request") f.forFrontend = forFrontend + f.downloadRequestCall++ + if reference != nil { + f.downloadReference = *reference + } return &FileDownloadResponse{ Filename: "report.pdf", MimeType: "application/pdf", @@ -47,20 +71,136 @@ func TestRunFileUploadReturnsFrontendDisplayURL(t *testing.T) { client := &fakeFileUploadClient{} var output bytes.Buffer - if err := runFileUpload(client, filePath, &output); err != nil { + if err := runFileUpload(client, filePath, false, &output); err != nil { t.Fatalf("run file upload: %v", err) } if !client.forFrontend { t.Fatal("download request did not select frontend display URL") } + if got, want := strings.Join(client.calls, ","), "upload-request,multipart-upload,download-request"; got != want { + t.Fatalf("call order = %s, want %s", got, want) + } + if client.filename != "report.pdf" || client.mimetype != "application/pdf" { + t.Fatalf("upload metadata = (%q, %q), want report.pdf/application/pdf", client.filename, client.mimetype) + } + if client.uploadURL != "https://sandbox-files.example.com/files/upload/for-plugin?sign=1" { + t.Fatalf("upload URL = %q", client.uploadURL) + } + if string(client.uploadedBytes) != "report" { + t.Fatalf("uploaded bytes = %q, want report", client.uploadedBytes) + } + if client.downloadReference != "dify-file-ref:eyJyZWNvcmRfaWQiOiJ0b29sLTEifQ==" { + t.Fatalf("download reference = %q", client.downloadReference) + } got := strings.TrimSpace(output.String()) - want := `{"transfer_method":"tool_file","reference":"dify-file-ref:canonical","public_download_url":"/files/tools/report.pdf?sign=2"}` + want := `{"transfer_method":"tool_file","reference":"dify-file-ref:eyJyZWNvcmRfaWQiOiJ0b29sLTEifQ==","public_download_url":"/files/tools/report.pdf?sign=2"}` if got != want { t.Fatalf("output = %s, want %s", got, want) } } +func TestRunFileUploadWithoutDownloadLinkReturnsOnlyCanonicalMapping(t *testing.T) { + filePath := filepath.Join(t.TempDir(), "report.pdf") + if err := os.WriteFile(filePath, []byte("report"), 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + + client := &fakeFileUploadClient{} + var output bytes.Buffer + if err := runFileUpload(client, filePath, true, &output); err != nil { + t.Fatalf("run file upload: %v", err) + } + + if client.downloadRequestCall != 0 { + t.Fatalf("download request calls = %d, want 0", client.downloadRequestCall) + } + if got, want := strings.Join(client.calls, ","), "upload-request,multipart-upload"; got != want { + t.Fatalf("call order = %s, want %s", got, want) + } + got := strings.TrimSpace(output.String()) + want := `{"transfer_method":"tool_file","reference":"dify-file-ref:eyJyZWNvcmRfaWQiOiJ0b29sLTEifQ=="}` + if got != want { + t.Fatalf("output = %s, want %s", got, want) + } +} + +func TestRunFileUploadDefaultAcceptsLegacyNonemptyReference(t *testing.T) { + filePath := filepath.Join(t.TempDir(), "report.pdf") + if err := os.WriteFile(filePath, []byte("report"), 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + + client := &fakeFileUploadClient{uploadResponse: []byte(`{"reference":"raw-id"}`)} + var output bytes.Buffer + err := runFileUpload(client, filePath, false, &output) + if err != nil { + t.Fatalf("run file upload: %v", err) + } + if client.downloadRequestCall != 1 || client.downloadReference != "raw-id" { + t.Fatalf("download request = (%d, %q), want legacy reference", client.downloadRequestCall, client.downloadReference) + } + if !strings.Contains(output.String(), `"reference":"raw-id"`) { + t.Fatalf("output = %q, want legacy reference", output.String()) + } +} + +func TestRunFileUploadWithoutDownloadLinkRejectsNonCanonicalReference(t *testing.T) { + filePath := filepath.Join(t.TempDir(), "report.pdf") + if err := os.WriteFile(filePath, []byte("report"), 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + + client := &fakeFileUploadClient{uploadResponse: []byte(`{"reference":"raw-id"}`)} + var output bytes.Buffer + err := runFileUpload(client, filePath, true, &output) + if err == nil || !strings.Contains(err.Error(), "invalid reference") { + t.Fatalf("error = %v, want invalid reference", err) + } + if client.downloadRequestCall != 0 { + t.Fatalf("download request calls = %d, want 0", client.downloadRequestCall) + } + if output.Len() != 0 { + t.Fatalf("output = %q, want empty", output.String()) + } +} + +func TestRunFileUploadRejectsMissingReferenceInBothModes(t *testing.T) { + filePath := filepath.Join(t.TempDir(), "report.pdf") + if err := os.WriteFile(filePath, []byte("report"), 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + + for _, noDownloadLink := range []bool{false, true} { + client := &fakeFileUploadClient{uploadResponse: []byte(`{"reference":""}`)} + var output bytes.Buffer + err := runFileUpload(client, filePath, noDownloadLink, &output) + if err == nil || !strings.Contains(err.Error(), "missing reference") { + t.Fatalf("noDownloadLink=%t error = %v, want missing reference", noDownloadLink, err) + } + if client.downloadRequestCall != 0 { + t.Fatalf("noDownloadLink=%t download request calls = %d, want 0", noDownloadLink, client.downloadRequestCall) + } + } +} + +func TestRunFileUploadRejectsInvalidUploadResponseBeforeDownloadRequest(t *testing.T) { + filePath := filepath.Join(t.TempDir(), "report.pdf") + if err := os.WriteFile(filePath, []byte("report"), 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + + client := &fakeFileUploadClient{uploadResponse: []byte("not-json")} + var output bytes.Buffer + err := runFileUpload(client, filePath, true, &output) + if err == nil || !strings.Contains(err.Error(), "parse upload result") { + t.Fatalf("error = %v, want parse upload result failure", err) + } + if client.downloadRequestCall != 0 { + t.Fatalf("download request calls = %d, want 0", client.downloadRequestCall) + } +} + func TestRunFileDownloadRequestsSandboxURLAndWritesFile(t *testing.T) { var requestPayload map[string]json.RawMessage var server *httptest.Server diff --git a/dify-agent-runtime/internal/agentcli/httpclient.go b/dify-agent-runtime/internal/agentcli/httpclient.go index cc17d224657..0c2c156c16b 100644 --- a/dify-agent-runtime/internal/agentcli/httpclient.go +++ b/dify-agent-runtime/internal/agentcli/httpclient.go @@ -2,7 +2,9 @@ package agentcli import ( "bytes" + "context" "encoding/json" + "errors" "fmt" "io" "mime/multipart" @@ -14,29 +16,45 @@ import ( // HTTPClient wraps HTTP interactions with the Agent Stub server. type HTTPClient struct { - baseURL string - authJWE string - client *http.Client + baseURL string + authJWE string + client *http.Client + openUploadFile func(string) (io.ReadCloser, error) + doUploadRequest func(*http.Request) (*http.Response, error) } +var errUploadRequestAborted = errors.New("upload request aborted") + // NewHTTPClient creates a new HTTP client for the Agent Stub API. func NewHTTPClient(env *Environment) *HTTPClient { return &HTTPClient{ - baseURL: env.URL, - authJWE: env.AuthJWE, - client: &http.Client{Timeout: 30 * time.Second}, + baseURL: env.URL, + authJWE: env.AuthJWE, + client: &http.Client{Timeout: 30 * time.Second}, + openUploadFile: openUploadSource, + doUploadRequest: doUploadRequest, } } // NewHTTPClientWithTimeout creates a client with a custom timeout. func NewHTTPClientWithTimeout(env *Environment, timeout time.Duration) *HTTPClient { return &HTTPClient{ - baseURL: env.URL, - authJWE: env.AuthJWE, - client: &http.Client{Timeout: timeout}, + baseURL: env.URL, + authJWE: env.AuthJWE, + client: &http.Client{Timeout: timeout}, + openUploadFile: openUploadSource, + doUploadRequest: doUploadRequest, } } +func openUploadSource(path string) (io.ReadCloser, error) { + return os.Open(path) +} + +func doUploadRequest(req *http.Request) (*http.Response, error) { + return (&http.Client{Timeout: 120 * time.Second}).Do(req) +} + // postJSON sends a POST request with JSON body and returns the response body. func (c *HTTPClient) postJSON(path string, payload any) ([]byte, int, error) { body, err := json.Marshal(payload) @@ -149,49 +167,105 @@ func (c *HTTPClient) putJSON(path string, payload any) ([]byte, int, error) { // uploadFile uploads a file to a signed URL using multipart form. func (c *HTTPClient) uploadFile(uploadURL string, filePath string, filename string, mimetype string) ([]byte, error) { - file, err := os.Open(filePath) + file, err := c.openUploadFile(filePath) if err != nil { return nil, fmt.Errorf("open file: %w", err) } - defer func() { _ = file.Close() }() + pipeReader, pipeWriter := io.Pipe() + + req, err := http.NewRequest("POST", uploadURL, pipeReader) + if err != nil { + _ = pipeReader.Close() + _ = pipeWriter.Close() + _ = file.Close() + return nil, errors.New("create upload request: invalid signed upload URL") + } + multipartWriter := multipart.NewWriter(pipeWriter) + req.Header.Set("Content-Type", multipartWriter.FormDataContentType()) + + writerDone := make(chan error, 1) + go func() { + writeErr := writeMultipartFile(multipartWriter, file, filename, mimetype) + if closeErr := file.Close(); writeErr == nil && closeErr != nil { + writeErr = fmt.Errorf("close file: %w", closeErr) + } + if writeErr != nil { + _ = pipeWriter.CloseWithError(writeErr) + } else { + writeErr = pipeWriter.Close() + } + writerDone <- writeErr + }() + + resp, requestErr := c.doUploadRequest(req) + _ = pipeReader.CloseWithError(errUploadRequestAborted) + + const maxUploadResponseBytes = 1024 * 1024 + var respBody []byte + var responseErr error + if resp != nil { + defer func() { _ = resp.Body.Close() }() + respBody, responseErr = io.ReadAll(io.LimitReader(resp.Body, maxUploadResponseBytes+1)) + } + + writerErr := <-writerDone + if resp != nil && (resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices) { + if len(respBody) > maxUploadResponseBytes { + respBody = respBody[:maxUploadResponseBytes] + } + return nil, fmt.Errorf("upload failed with status %d: %s", resp.StatusCode, string(respBody)) + } + if requestErr != nil { + if writerErr != nil && !isUploadWriterAbort(writerErr) { + return nil, writerErr + } + if errors.Is(requestErr, context.DeadlineExceeded) || os.IsTimeout(requestErr) { + return nil, errors.New("upload request timed out") + } + return nil, errors.New("upload request failed") + } + if writerErr != nil && !isUploadWriterAbort(writerErr) { + return nil, writerErr + } + if isUploadWriterAbort(writerErr) { + return nil, errors.New("upload request completed before multipart body was fully written") + } + if responseErr != nil { + return nil, fmt.Errorf("read upload response: %w", responseErr) + } + if len(respBody) > maxUploadResponseBytes { + return nil, fmt.Errorf("upload response exceeds %d bytes", maxUploadResponseBytes) + } + return respBody, nil +} + +func isUploadWriterAbort(err error) bool { + return errors.Is(err, errUploadRequestAborted) || errors.Is(err, io.ErrClosedPipe) +} + +func writeMultipartFile( + writer *multipart.Writer, + file io.Reader, + filename string, + mimetype string, +) (resultErr error) { + defer func() { + if closeErr := writer.Close(); resultErr == nil && closeErr != nil { + resultErr = fmt.Errorf("close multipart writer: %w", closeErr) + } + }() - var buf bytes.Buffer - writer := multipart.NewWriter(&buf) h := make(textproto.MIMEHeader) h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, filename)) h.Set("Content-Type", mimetype) part, err := writer.CreatePart(h) if err != nil { - return nil, fmt.Errorf("create form file: %w", err) + return fmt.Errorf("create form file: %w", err) } if _, err := io.Copy(part, file); err != nil { - return nil, fmt.Errorf("copy file content: %w", err) + return fmt.Errorf("copy file content: %w", err) } - if err := writer.Close(); err != nil { - return nil, fmt.Errorf("close multipart writer: %w", err) - } - - uploadClient := &http.Client{Timeout: 120 * time.Second} - req, err := http.NewRequest("POST", uploadURL, &buf) - if err != nil { - return nil, fmt.Errorf("create upload request: %w", err) - } - req.Header.Set("Content-Type", writer.FormDataContentType()) - - resp, err := uploadClient.Do(req) - if err != nil { - return nil, fmt.Errorf("upload request failed: %w", err) - } - defer func() { _ = resp.Body.Close() }() - - respBody, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("read upload response: %w", err) - } - if resp.StatusCode >= 400 { - return nil, fmt.Errorf("upload failed with status %d: %s", resp.StatusCode, string(respBody)) - } - return respBody, nil + return nil } // downloadFromURL downloads bytes from a signed URL. diff --git a/dify-agent-runtime/internal/agentcli/httpclient_test.go b/dify-agent-runtime/internal/agentcli/httpclient_test.go new file mode 100644 index 00000000000..67ac9dbd11e --- /dev/null +++ b/dify-agent-runtime/internal/agentcli/httpclient_test.go @@ -0,0 +1,678 @@ +package agentcli + +import ( + "bytes" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +const fifoTestDeadline = 3 * time.Second + +func receiveWithin[T any](ch <-chan T, timeout time.Duration) (T, bool) { + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case value := <-ch: + return value, true + case <-timer.C: + var zero T + return zero, false + } +} + +type multipartWriteRecorder struct { + headerErr error + cleanupErr error + source *terminalRecordingReader + headerAttempted bool + headerFailed bool + cleanupAttempted bool +} + +func (w *multipartWriteRecorder) Write(p []byte) (int, error) { + if w.headerFailed || (w.source != nil && w.source.finished) { + w.cleanupAttempted = true + if w.cleanupErr != nil { + return 0, w.cleanupErr + } + return len(p), nil + } + if !w.headerAttempted { + w.headerAttempted = true + if w.headerErr != nil { + w.headerFailed = true + return 0, w.headerErr + } + } + return len(p), nil +} + +type terminalRecordingReader struct { + reader io.Reader + terminalErr error + finished bool +} + +func (r *terminalRecordingReader) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + if errors.Is(err, io.EOF) { + r.finished = true + if r.terminalErr != nil { + return n, r.terminalErr + } + } + return n, err +} + +type dataThenErrorReader struct { + data []byte + sourceErr error + delivered bool +} + +type closeRecordingBody struct { + reader io.Reader + closed bool +} + +type gatedUploadSource struct { + payload []byte + started chan struct{} + release chan struct{} + completed chan struct{} + releaseOnce sync.Once + delivered bool + closed bool +} + +func newGatedUploadSource(payload []byte) *gatedUploadSource { + return &gatedUploadSource{ + payload: payload, + started: make(chan struct{}), + release: make(chan struct{}), + completed: make(chan struct{}), + } +} + +func (s *gatedUploadSource) Read(p []byte) (int, error) { + if s.delivered { + return 0, io.EOF + } + s.delivered = true + close(s.started) + <-s.release + n := copy(p, s.payload) + close(s.completed) + return n, nil +} + +func (s *gatedUploadSource) Close() error { + s.unblock() + s.closed = true + return nil +} + +func (s *gatedUploadSource) unblock() { + s.releaseOnce.Do(func() { close(s.release) }) +} + +type releaseOnReadBody struct { + reader io.Reader + release func() + closed bool +} + +func (b *releaseOnReadBody) Read(p []byte) (int, error) { + b.release() + return b.reader.Read(p) +} + +func (b *releaseOnReadBody) Close() error { + b.release() + b.closed = true + return nil +} + +func (b *closeRecordingBody) Read(p []byte) (int, error) { + return b.reader.Read(p) +} + +func (b *closeRecordingBody) Close() error { + b.closed = true + return nil +} + +func (r *dataThenErrorReader) Read(p []byte) (int, error) { + if r.delivered { + return 0, r.sourceErr + } + r.delivered = true + return copy(p, r.data), nil +} + +func TestUploadFileStreamsMultipartBody(t *testing.T) { + payload := bytes.Repeat([]byte("streamed-payload-"), 128*1024) + filePath := filepath.Join(t.TempDir(), "payload.bin") + if err := os.WriteFile(filePath, payload, 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.ContentLength != -1 { + t.Errorf("content length = %d, want -1 for streamed request", r.ContentLength) + } + reader, err := r.MultipartReader() + if err != nil { + http.Error(w, fmt.Sprintf("create multipart reader: %v", err), http.StatusBadRequest) + return + } + part, err := reader.NextPart() + if err != nil { + http.Error(w, fmt.Sprintf("read multipart part: %v", err), http.StatusBadRequest) + return + } + defer func() { _ = part.Close() }() + if part.FormName() != "file" || part.FileName() != "payload.bin" { + http.Error(w, "unexpected multipart metadata", http.StatusBadRequest) + return + } + if got := part.Header.Get("Content-Type"); got != "application/octet-stream" { + http.Error(w, "unexpected multipart content type: "+got, http.StatusBadRequest) + return + } + got, err := io.ReadAll(part) + if err != nil { + http.Error(w, fmt.Sprintf("read multipart content: %v", err), http.StatusBadRequest) + return + } + if !bytes.Equal(got, payload) { + http.Error(w, "multipart content mismatch", http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"reference":"dify-file-ref:canonical"}`)) + })) + defer server.Close() + + client := NewHTTPClient(&Environment{}) + body, err := client.uploadFile(server.URL, filePath, "payload.bin", "application/octet-stream") + if err != nil { + t.Fatalf("upload file: %v", err) + } + if string(body) != `{"reference":"dify-file-ref:canonical"}` { + t.Fatalf("response body = %s", body) + } +} + +func TestUploadFileStartsRequestBeforeSourceEOF(t *testing.T) { + if testing.Short() { + t.Skip("uses a local HTTP server and FIFO coordination") + } + mkfifo, err := exec.LookPath("mkfifo") + if err != nil { + t.Skip("mkfifo is unavailable") + } + fifoPath := filepath.Join(t.TempDir(), "stream.bin") + if err := exec.Command(mkfifo, "-m", "600", fifoPath).Run(); err != nil { + t.Fatalf("create FIFO: %v", err) + } + + requestStarted := make(chan struct{}) + continueSource := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + close(requestStarted) + reader, err := r.MultipartReader() + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + part, err := reader.NextPart() + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + defer func() { _ = part.Close() }() + body, err := io.ReadAll(part) + if err != nil || string(body) != "prefix-suffix" { + http.Error(w, "unexpected streamed body", http.StatusBadRequest) + return + } + _, _ = w.Write([]byte(`{"reference":"dify-file-ref:eyJyZWNvcmRfaWQiOiJ0b29sLTEifQ=="}`)) + })) + defer server.Close() + + writerDone := make(chan error, 1) + go func() { + file, err := os.OpenFile(fifoPath, os.O_WRONLY, 0) + if err != nil { + writerDone <- err + return + } + if _, err = file.WriteString("prefix-"); err == nil { + <-continueSource + _, err = file.WriteString("suffix") + } + if closeErr := file.Close(); err == nil { + err = closeErr + } + writerDone <- err + }() + + uploadDone := make(chan error, 1) + go func() { + client := NewHTTPClient(&Environment{}) + _, err := client.uploadFile(server.URL, fifoPath, "stream.bin", "application/octet-stream") + uploadDone <- err + }() + + releaseSource := sync.OnceFunc(func() { close(continueSource) }) + writerJoined := false + uploadJoined := false + defer func() { + releaseSource() + server.CloseClientConnections() + if !writerJoined { + if err, ok := receiveWithin(writerDone, fifoTestDeadline); !ok { + t.Errorf("cleanup: FIFO writer did not finish within %s", fifoTestDeadline) + } else if err != nil { + t.Errorf("cleanup: FIFO writer failed: %v", err) + } + } + if !uploadJoined { + if err, ok := receiveWithin(uploadDone, fifoTestDeadline); !ok { + t.Errorf("cleanup: upload did not finish within %s", fifoTestDeadline) + } else if err != nil { + t.Errorf("cleanup: upload failed: %v", err) + } + } + }() + + if _, ok := receiveWithin(requestStarted, fifoTestDeadline); !ok { + t.Fatalf("HTTP request did not start within %s before the source reached EOF", fifoTestDeadline) + } + releaseSource() + + writerErr, ok := receiveWithin(writerDone, fifoTestDeadline) + if !ok { + t.Fatalf("FIFO writer did not finish within %s after source release", fifoTestDeadline) + } + writerJoined = true + if writerErr != nil { + t.Fatalf("write FIFO: %v", writerErr) + } + uploadErr, ok := receiveWithin(uploadDone, fifoTestDeadline) + if !ok { + t.Fatalf("upload did not finish within %s after source EOF", fifoTestDeadline) + } + uploadJoined = true + if uploadErr != nil { + t.Fatalf("upload FIFO: %v", uploadErr) + } +} + +func TestUploadFileTransportFailureDoesNotBlockWriter(t *testing.T) { + const querySecret = "signed-upload-credential" + uploadURL := "https://upload.example/path?X-Amz-Credential=" + querySecret + source := &closeRecordingBody{reader: strings.NewReader("payload")} + client := NewHTTPClient(&Environment{}) + client.openUploadFile = func(string) (io.ReadCloser, error) { + return source, nil + } + requestBodyReady := make(chan io.ReadCloser, 1) + client.doUploadRequest = func(req *http.Request) (*http.Response, error) { + requestBodyReady <- req.Body + return nil, errors.New("deterministic transport failure") + } + + uploadDone := make(chan error, 1) + go func() { + _, err := client.uploadFile(uploadURL, "source-path", "payload.bin", "application/octet-stream") + uploadDone <- err + }() + + var requestBody io.ReadCloser + uploadJoined := false + defer func() { + _ = source.Close() + if requestBody == nil { + requestBody, _ = receiveWithin(requestBodyReady, fifoTestDeadline) + } + if requestBody != nil { + _ = requestBody.Close() + } + if !uploadJoined { + if _, ok := receiveWithin(uploadDone, fifoTestDeadline); !ok { + t.Errorf("cleanup: transport-failure upload did not finish within %s", fifoTestDeadline) + } + } + }() + + var ok bool + requestBody, ok = receiveWithin(requestBodyReady, fifoTestDeadline) + if !ok { + t.Fatalf("upload request did not start within %s", fifoTestDeadline) + } + + err, ok := receiveWithin(uploadDone, fifoTestDeadline) + if !ok { + t.Fatalf("transport-failure upload did not finish within %s", fifoTestDeadline) + } + uploadJoined = true + + if err == nil || err.Error() != "upload request failed" { + t.Fatalf("error = %v, want upload request failure", err) + } + if !source.closed { + t.Fatal("source file was not closed after transport failure") + } + if strings.Contains(err.Error(), uploadURL) || strings.Contains(err.Error(), querySecret) { + t.Fatalf("error leaked signed upload URL credentials: %v", err) + } +} + +func TestUploadFileInvalidSignedURLDoesNotLeakCredentials(t *testing.T) { + filePath := filepath.Join(t.TempDir(), "payload.txt") + if err := os.WriteFile(filePath, []byte("payload"), 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + const querySecret = "signed-upload-credential" + uploadURL := "http://example.test/upload?X-Amz-Credential=" + querySecret + "\n" + + client := NewHTTPClient(&Environment{}) + _, err := client.uploadFile(uploadURL, filePath, "payload.txt", "text/plain") + if err == nil || err.Error() != "create upload request: invalid signed upload URL" { + t.Fatalf("error = %v, want invalid signed upload URL failure", err) + } + if strings.Contains(err.Error(), uploadURL) || strings.Contains(err.Error(), querySecret) { + t.Fatalf("error leaked signed upload URL credentials: %v", err) + } +} + +func TestUploadFileRejectsOversizedResponse(t *testing.T) { + filePath := filepath.Join(t.TempDir(), "payload.txt") + if err := os.WriteFile(filePath, []byte("payload"), 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + _, _ = w.Write(bytes.Repeat([]byte("x"), 1024*1024+1)) + })) + defer server.Close() + + client := NewHTTPClient(&Environment{}) + _, err := client.uploadFile(server.URL, filePath, "payload.txt", "text/plain") + if err == nil || !strings.Contains(err.Error(), "upload response exceeds") { + t.Fatalf("error = %v, want bounded-response failure", err) + } +} + +func TestUploadFileReturnsNonSuccessStatus(t *testing.T) { + filePath := filepath.Join(t.TempDir(), "payload.txt") + if err := os.WriteFile(filePath, []byte("payload"), 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + http.Error(w, "too large", http.StatusRequestEntityTooLarge) + })) + defer server.Close() + + client := NewHTTPClient(&Environment{}) + _, err := client.uploadFile(server.URL, filePath, "payload.txt", "text/plain") + if err == nil || !strings.Contains(err.Error(), "upload failed with status 413") { + t.Fatalf("error = %v, want non-success status", err) + } +} + +func uploadFileWithGatedEarlyResponse(t *testing.T, statusCode int, body string) error { + t.Helper() + source := newGatedUploadSource([]byte("payload")) + responseBody := &releaseOnReadBody{reader: strings.NewReader(body), release: source.unblock} + requestDone := make(chan error, 1) + var orderingErr error + client := NewHTTPClient(&Environment{}) + client.openUploadFile = func(string) (io.ReadCloser, error) { + return source, nil + } + client.doUploadRequest = func(req *http.Request) (*http.Response, error) { + go func() { + _, err := io.Copy(io.Discard, req.Body) + requestDone <- err + }() + if _, ok := receiveWithin(source.started, fifoTestDeadline); !ok { + orderingErr = errors.New("upload source did not start before the early response") + source.unblock() + return nil, orderingErr + } + select { + case <-source.completed: + orderingErr = errors.New("upload source completed before the early response") + return nil, orderingErr + default: + } + return &http.Response{StatusCode: statusCode, Body: responseBody}, nil + } + + uploadDone := make(chan error, 1) + go func() { + _, err := client.uploadFile("https://upload.example/path", "source-path", "payload.bin", "application/octet-stream") + uploadDone <- err + }() + uploadJoined := false + requestJoined := false + defer func() { + source.unblock() + if !uploadJoined { + if _, ok := receiveWithin(uploadDone, fifoTestDeadline); !ok { + t.Errorf("cleanup: early-response upload did not finish within %s", fifoTestDeadline) + } + } + if !requestJoined { + if _, ok := receiveWithin(requestDone, fifoTestDeadline); !ok { + t.Errorf("cleanup: early-response request drain did not finish within %s", fifoTestDeadline) + } + } + }() + + uploadErr, ok := receiveWithin(uploadDone, fifoTestDeadline) + if !ok { + t.Fatalf("early-response upload did not finish within %s", fifoTestDeadline) + } + uploadJoined = true + if _, ok := receiveWithin(requestDone, fifoTestDeadline); !ok { + t.Fatalf("early-response request drain did not finish within %s", fifoTestDeadline) + } + requestJoined = true + if orderingErr != nil { + t.Fatalf("invalid early-response ordering: %v", orderingErr) + } + if _, ok := receiveWithin(source.completed, fifoTestDeadline); !ok { + t.Fatalf("upload source did not finish within %s after response processing", fifoTestDeadline) + } + if !source.closed { + t.Fatal("upload source was not closed before uploadFile returned") + } + if !responseBody.closed { + t.Fatal("early response body was not closed before uploadFile returned") + } + return uploadErr +} + +func TestUploadFileReturnsEarlyNonSuccessStatusInsteadOfWriterAbort(t *testing.T) { + err := uploadFileWithGatedEarlyResponse( + t, + http.StatusRequestEntityTooLarge, + "too large without reading body\n", + ) + if err == nil || !strings.Contains(err.Error(), "upload failed with status 413: too large without reading body") { + t.Fatalf("error = %v, want early HTTP 413 response", err) + } + if strings.Contains(err.Error(), "upload request aborted") || strings.Contains(err.Error(), "closed pipe") { + t.Fatalf("error exposed internal multipart abort: %v", err) + } +} + +func TestUploadFileRejectsEarlySuccessBeforeMultipartCompletes(t *testing.T) { + err := uploadFileWithGatedEarlyResponse(t, http.StatusOK, `{"reference":"incomplete"}`) + if err == nil || err.Error() != "upload request completed before multipart body was fully written" { + t.Fatalf("error = %v, want incomplete multipart failure", err) + } +} + +func TestUploadFilePropagatesSourceReadFailure(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + _, _ = w.Write([]byte(`{"reference":"unused"}`)) + })) + defer server.Close() + + client := NewHTTPClient(&Environment{}) + _, err := client.uploadFile(server.URL, t.TempDir(), "directory", "application/octet-stream") + if err == nil || !strings.Contains(err.Error(), "copy file content") { + t.Fatalf("error = %v, want source read failure", err) + } +} + +func TestUploadFileClosesResourcesAndJoinsWriterOnResponseOutcomes(t *testing.T) { + responseReadErr := errors.New("response read failed") + tests := []struct { + name string + statusCode int + responseReader io.Reader + wantError string + }{ + { + name: "success", + statusCode: http.StatusOK, + responseReader: strings.NewReader(`{"reference":"dify-file-ref:canonical"}`), + }, + { + name: "non-2xx", + statusCode: http.StatusRequestEntityTooLarge, + responseReader: strings.NewReader("too large"), + wantError: "upload failed with status 413", + }, + { + name: "response body read error", + statusCode: http.StatusOK, + responseReader: &dataThenErrorReader{data: []byte("partial"), sourceErr: responseReadErr}, + wantError: "read upload response: response read failed", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + source := &closeRecordingBody{reader: strings.NewReader("payload")} + var responseBody *closeRecordingBody + client := NewHTTPClient(&Environment{}) + client.openUploadFile = func(path string) (io.ReadCloser, error) { + if path != "source-path" { + t.Fatalf("source path = %q", path) + } + return source, nil + } + client.doUploadRequest = func(req *http.Request) (*http.Response, error) { + if _, err := io.Copy(io.Discard, req.Body); err != nil { + t.Fatalf("drain request body: %v", err) + } + responseBody = &closeRecordingBody{reader: tt.responseReader} + return &http.Response{StatusCode: tt.statusCode, Body: responseBody}, nil + } + + _, err := client.uploadFile("https://upload.example/path", "source-path", "payload.txt", "text/plain") + + if tt.wantError == "" { + if err != nil { + t.Fatalf("upload file: %v", err) + } + } else if err == nil || !strings.Contains(err.Error(), tt.wantError) { + t.Fatalf("error = %v, want containing %q", err, tt.wantError) + } + if !source.closed { + t.Fatal("source file was not closed before uploadFile returned") + } + if responseBody == nil || !responseBody.closed { + t.Fatal("response body was not closed before uploadFile returned") + } + }) + } +} + +func TestWriteMultipartFileAlwaysAttemptsCloseAndPreservesPrimaryError(t *testing.T) { + t.Run("create part failure", func(t *testing.T) { + createErr := errors.New("create part failed") + closeErr := errors.New("close failed") + destination := &multipartWriteRecorder{headerErr: createErr, cleanupErr: closeErr} + writer := multipart.NewWriter(destination) + + err := writeMultipartFile( + writer, + strings.NewReader("payload"), + "payload.txt", + "text/plain", + ) + + if !errors.Is(err, createErr) { + t.Fatalf("error = %v, want CreatePart failure", err) + } + if !destination.headerAttempted || !destination.cleanupAttempted { + t.Fatal("multipart header and cleanup were not both attempted") + } + }) + + t.Run("copy failure", func(t *testing.T) { + sourceErr := errors.New("source read failed") + closeErr := errors.New("close failed") + source := &terminalRecordingReader{reader: strings.NewReader("payload"), terminalErr: sourceErr} + destination := &multipartWriteRecorder{cleanupErr: closeErr, source: source} + writer := multipart.NewWriter(destination) + + err := writeMultipartFile( + writer, + source, + "payload.txt", + "text/plain", + ) + + if !errors.Is(err, sourceErr) { + t.Fatalf("error = %v, want source copy failure", err) + } + if !destination.cleanupAttempted { + t.Fatal("multipart cleanup was not attempted after the source read failure") + } + }) + + t.Run("close failure", func(t *testing.T) { + closeErr := errors.New("close failed") + source := &terminalRecordingReader{reader: strings.NewReader("payload")} + destination := &multipartWriteRecorder{cleanupErr: closeErr, source: source} + writer := multipart.NewWriter(destination) + + err := writeMultipartFile( + writer, + source, + "payload.txt", + "text/plain", + ) + + if !errors.Is(err, closeErr) || !strings.Contains(err.Error(), "close multipart writer") { + t.Fatalf("error = %v, want multipart Close failure", err) + } + if !destination.cleanupAttempted { + t.Fatal("multipart cleanup was not attempted after source EOF") + } + }) +} diff --git a/dify-agent/.example.env b/dify-agent/.example.env index d713539fe6b..31220ebb52c 100644 --- a/dify-agent/.example.env +++ b/dify-agent/.example.env @@ -48,9 +48,6 @@ DIFY_AGENT_E2B_TEMPLATE=difys-default-team/dify-agent-local-sandbox DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS=3600 DIFY_AGENT_E2B_SHELLCTL_AUTH_TOKEN= DIFY_AGENT_E2B_SHELLCTL_PORT=5004 -# Maximum whole-file size read through /workspace/files for an Agent Stub ToolFile upload (50 MiB). -# The environment variable keeps its existing SANDBOX name for deployment compatibility. -DIFY_AGENT_SANDBOX_FILE_UPLOAD_MAX_BYTES=52428800 # JSON array of regex patterns to redact from shell output shown to the agent. DIFY_AGENT_SHELL_REDACT_PATTERNS= diff --git a/dify-agent/docs/dify-agent/concepts/runtime-resources/index.md b/dify-agent/docs/dify-agent/concepts/runtime-resources/index.md index ec68abaf5e5..0745f1f9fce 100644 --- a/dify-agent/docs/dify-agent/concepts/runtime-resources/index.md +++ b/dify-agent/docs/dify-agent/concepts/runtime-resources/index.md @@ -29,7 +29,7 @@ backend ref to the `dify.runtime` layer: flowchart LR EC["dify.execution_context
request identity"] RT["dify.runtime
opaque backend_binding_ref"] - SH["dify.shell
commands, files, and jobs"] + SH["dify.shell
commands and jobs"] EC --> SH RT --> SH @@ -41,8 +41,8 @@ the resulting `RuntimeLease` only while that context is active. The layer does not create, retire, or destroy persistent resources, and it stores no backend SDK object in an Agenton session snapshot. -The Shell layer consumes `RuntimeLease.commands`, `RuntimeLease.files`, and -`RuntimeLease.layout`. It tracks only request-local shell job ids and offsets. +The Shell layer consumes `RuntimeLease.commands` and `RuntimeLease.layout`. It +tracks only request-local shell job ids and offsets. Closing a run clears that job state; it does not retire the Binding. ## State ownership @@ -105,7 +105,7 @@ composition contains: Each Agent request acquires that ref for the duration of the run and releases it afterward. Local release closes the operation's shellctl connection. E2B release also pauses the underlying E2B resource with memory preserved. A later request -or Workspace file operation acquires a new lease for the same Binding ref. If a +or Binding file operation acquires a new lease for the same Binding ref. If a backend confirms the resource is gone, acquisition fails; it does not create an empty replacement Workspace. @@ -146,7 +146,7 @@ returning success. For example, E2B kills a Sandbox when its initialization fails, and Local removes paths created by an incomplete operation. This backend-local cleanup does not cross the database commit boundary. -## Workspace file boundary +## Binding file boundary Dify API's public file APIs accept a product locator, not a Binding id or backend ref: a Conversation, a debug Build Draft, or a Workflow Node Execution. @@ -154,21 +154,20 @@ Dify API authorizes that object and resolves its associated active Binding. It does not select the latest Binding or fall back to another product context. The resolved request reaches Dify Agent through its private -`POST /workspace/files/list`, `POST /workspace/files/read`, and -`POST /workspace/files/upload` endpoints. Each operation receives a +`POST /execution-bindings/files/list`, `POST /execution-bindings/files/read`, +and `POST /execution-bindings/files/download` endpoints. Each operation receives a `backend_binding_ref`, acquires a fresh RuntimeLease, performs the file action, and releases the lease. -`WorkspaceFileService` forwards the request path unchanged to the current -RuntimeLease's file capability. The backend interprets that path in its own -filesystem namespace; the service does not require a Workspace-relative path -or enforce `workspace_dir` containment. `~` and `~/...` can therefore address -the lease's Home. Whether an absolute path or a path containing `..` is -accessible is determined by the backend and its path-isolation policy, such as -Local shellctl and Landlock isolation. Whole-file capture for Agent Stub upload -is bounded by -`DIFY_AGENT_SANDBOX_FILE_UPLOAD_MAX_BYTES`; the environment variable keeps its -existing name even though the route now operates through a Binding. +`BindingFileService` resolves relative paths from `workspace_dir`, `~` and +`~/...` from `home_dir`, and leaves absolute paths in the Binding filesystem +namespace. It does not enforce Workspace containment or reject `..`; the +selected backend's isolation policy remains authoritative. List and preview +run bounded inspection scripts through `RuntimeLease.commands`. Download runs +`dify-agent file upload --no-download-link` inside the Binding so bytes stream +directly from the runtime to Dify's existing ToolFile endpoint. Dify Agent +returns only the canonical ToolFile reference and releases the lease before +Dify API signs a browser URL. `RuntimeLayout.home_dir` and `RuntimeLayout.workspace_dir` are canonical paths inside the backend execution namespace. They are not host paths, product ids, diff --git a/dify-agent/docs/dify-agent/get-started/index.md b/dify-agent/docs/dify-agent/get-started/index.md index b672c5fdb41..71990baad5a 100644 --- a/dify-agent/docs/dify-agent/get-started/index.md +++ b/dify-agent/docs/dify-agent/get-started/index.md @@ -84,7 +84,6 @@ DIFY_AGENT_LOCAL_SANDBOX_AUTH_TOKEN= # DIFY_AGENT_LOCAL_SANDBOX_MATERIALIZED_HOME_ROOT=/tmp/dify-agent/materialized-homes # DIFY_AGENT_LOCAL_SANDBOX_WORKSPACE_ROOT=/tmp/dify-agent/workspaces # DIFY_AGENT_LOCAL_SANDBOX_HOME_SNAPSHOT_ROOT=/tmp/dify-agent/home-snapshots -DIFY_AGENT_SANDBOX_FILE_UPLOAD_MAX_BYTES=52428800 ``` E2B requires `DIFY_AGENT_E2B_API_KEY` and defaults to the prepared diff --git a/dify-agent/docs/dify-agent/guide/index.md b/dify-agent/docs/dify-agent/guide/index.md index 2474d3d0d82..75f5a4f2ba6 100644 --- a/dify-agent/docs/dify-agent/guide/index.md +++ b/dify-agent/docs/dify-agent/guide/index.md @@ -54,7 +54,6 @@ also reads `.env` and `dify-agent/.env` when present. | `DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS` | `3600` | Maximum continuous active time for the RuntimeLease spanning one complete Agent run. Binding resources pause on timeout. This is not a retention TTL. | | `DIFY_AGENT_E2B_SHELLCTL_AUTH_TOKEN` | empty | Optional bearer token expected by shellctl inside the E2B template. | | `DIFY_AGENT_E2B_SHELLCTL_PORT` | `5004` | shellctl port exposed by the E2B template. | -| `DIFY_AGENT_SANDBOX_FILE_UPLOAD_MAX_BYTES` | `52428800` | Standalone Dify Agent maximum for whole-file Workspace upload capture; 50 MiB by default. Docker Compose derives it from `PLUGIN_MAX_FILE_SIZE`. | | `DIFY_AGENT_SHELL_REDACT_PATTERNS` | empty | JSON array of additional regex patterns redacted from Shell output. | | `DIFY_AGENT_STUB_API_BASE_URL` | empty | HTTP(S) Agent Stub API base URL reachable from the Sandbox. It may be the service root or `/agent-stub`. Enables `DIFY_AGENT_STUB_*` env injection for user `shell.run` jobs. | | `DIFY_AGENT_SANDBOX_FILES_BASE_URL` | empty | Dify API base URL reachable from the Sandbox for signed `/files/*` upload/download bytes, including Config file and skill pulls. Required when Agent Stub file operations are enabled. May include an ingress path prefix, but not a query or fragment. | @@ -85,7 +84,6 @@ DIFY_AGENT_LOCAL_SANDBOX_AUTH_TOKEN=replace-with-shellctl-token DIFY_AGENT_LOCAL_SANDBOX_MATERIALIZED_HOME_ROOT=/tmp/dify-agent/materialized-homes DIFY_AGENT_LOCAL_SANDBOX_WORKSPACE_ROOT=/tmp/dify-agent/workspaces DIFY_AGENT_LOCAL_SANDBOX_HOME_SNAPSHOT_ROOT=/tmp/dify-agent/home-snapshots -DIFY_AGENT_SANDBOX_FILE_UPLOAD_MAX_BYTES=52428800 DIFY_AGENT_STUB_API_BASE_URL=https://agent.example.com/agent-stub DIFY_AGENT_SANDBOX_FILES_BASE_URL=https://dify.example.com # This is security-sensitive: it derives the JWE encryption key for Agent Stub bearer tokens. @@ -125,11 +123,6 @@ must use `DIFY_AGENT_LOCAL_SANDBOX_ENDPOINT` and `DIFY_AGENT_LOCAL_SANDBOX_AUTH_TOKEN`. There is no compatibility setting for the removed shell-provider selector. -The example above is for a standalone Dify Agent process, where the byte limit -can be set directly. In a Docker deployment, set `PLUGIN_MAX_FILE_SIZE` in -`docker/.env`; Compose maps it to -`DIFY_AGENT_SANDBOX_FILE_UPLOAD_MAX_BYTES` inside `agent_backend`. - The backend selection is deployment-private. Shell-enabled run requests use an Execution Context, `dify.runtime`, and `dify.shell` graph. Runtime config carries only the opaque `backend_binding_ref` resolved by Dify API. See diff --git a/dify-agent/docs/dify-agent/user-manual/shell-layer/index.md b/dify-agent/docs/dify-agent/user-manual/shell-layer/index.md index 91b83927ff3..0169af0369d 100644 --- a/dify-agent/docs/dify-agent/user-manual/shell-layer/index.md +++ b/dify-agent/docs/dify-agent/user-manual/shell-layer/index.md @@ -1,6 +1,6 @@ # Shell layer -The `dify.shell` layer exposes shellctl-backed commands and files to an Agent. +The `dify.shell` layer exposes shellctl-backed commands to an Agent. It does not select a backend or own persistent Home, Workspace, or Binding resources. It consumes the operation-scoped `RuntimeLease` opened by a sibling `dify.runtime` layer. @@ -115,6 +115,12 @@ or a same-origin `/files/...` relative URI when `FILES_URL` is empty. The CLI does not access this field. The former ambiguous `download_url` upload-output key is now named `public_download_url`. +Server-side Binding downloads use +`dify-agent file upload --no-download-link `. This additive mode performs +the same streaming ToolFile upload but skips the download-request step and +prints only `transfer_method` plus the canonical `reference`. The regular +`file upload` command keeps the link-producing behavior shown above. + ## Request graph A shell-enabled run contains Execution Context, Runtime, and Shell layers: @@ -127,7 +133,7 @@ flowchart LR `DifyRuntimeLayer` acquires the Binding when the run's resource context opens and releases it when that operation exits. `DifyShellLayer` uses the active -lease's commands, files, Home path, and Workspace path. It performs only +lease's commands, Home path, and Workspace path. It performs only best-effort cleanup of shell jobs; the persistent Binding lifecycle remains in Dify API. @@ -227,11 +233,14 @@ sent in the run request. Shell commands start in `workspace_dir`, while `HOME` is forced to `home_dir`; `~` therefore resolves to the current Binding's materialized Home. -Workspace files persist with the Workspace until Dify API retires and collects -it. Releasing a RuntimeLease ends only the current operation. Dify API can later -browse the current Workspace through Dify Agent's private -`/workspace/files/list`, `/workspace/files/read`, and -`/workspace/files/upload` routes, each of which acquires a fresh lease. +Workspace content persists with the Workspace until Dify API retires and +collects it. Releasing a RuntimeLease ends only the current operation. Dify API can later +browse the Binding filesystem through Dify Agent's private +`/execution-bindings/files/list`, `/execution-bindings/files/read`, and +`/execution-bindings/files/download` routes, each of which acquires a fresh +lease. Relative paths start in the Workspace, while `~` starts in the Binding's +Home. Download uses the installed CLI to stream a ToolFile upload and returns +only its canonical reference to Dify API. On Local, multiple Bindings may share a Workspace while each receives a separate materialized Home. Those directories may be siblings in one shellctl diff --git a/dify-agent/src/dify_agent/adapters/shell/__init__.py b/dify-agent/src/dify_agent/adapters/shell/__init__.py index 251c0462ae0..c09839b8d7b 100644 --- a/dify-agent/src/dify_agent/adapters/shell/__init__.py +++ b/dify-agent/src/dify_agent/adapters/shell/__init__.py @@ -9,7 +9,6 @@ from dify_agent.adapters.shell.protocols import ( ShellCommandProtocol, ShellCommandResult, ShellCommandStatus, - ShellFileTransferProtocol, ShellPromptObservation, ShellProviderError, ) @@ -28,7 +27,6 @@ __all__ = [ "ShellCommandProtocol", "ShellCommandResult", "ShellCommandStatus", - "ShellFileTransferProtocol", "ShellPromptObservation", "ShellProviderError", ] diff --git a/dify-agent/src/dify_agent/adapters/shell/protocols.py b/dify-agent/src/dify_agent/adapters/shell/protocols.py index f5f4e827cee..52b5436292a 100644 --- a/dify-agent/src/dify_agent/adapters/shell/protocols.py +++ b/dify-agent/src/dify_agent/adapters/shell/protocols.py @@ -105,9 +105,3 @@ class ShellCommandProtocol(Protocol): force: bool = False, grace_seconds: float | None = None, ) -> None: ... - - -class ShellFileTransferProtocol(Protocol): - async def upload(self, *, content: bytes, remote_path: str, cwd: str | None = None) -> None: ... - - async def download(self, *, remote_path: str, cwd: str | None = None) -> bytes: ... diff --git a/dify-agent/src/dify_agent/adapters/shell/shellctl.py b/dify-agent/src/dify_agent/adapters/shell/shellctl.py index d4cf4dfd663..cea953a4282 100644 --- a/dify-agent/src/dify_agent/adapters/shell/shellctl.py +++ b/dify-agent/src/dify_agent/adapters/shell/shellctl.py @@ -1,24 +1,14 @@ -"""Shellctl command and file data-plane adapters for RuntimeLease objects. +"""Shellctl command adapter for RuntimeLease objects. The built-in shellctl SDK owns the HTTP timeout policy for long-polling shellctl requests. This adapter translates SDK and transport failures into -``ShellProviderError``. Browse paths are interpreted directly inside the -backend-provided filesystem namespace. Whole-file reads return bytes to the -control plane; they never create another runtime-visible upload path. +``ShellProviderError``. """ from __future__ import annotations -import base64 -import binascii -import json -import logging import posixpath -import re -import shlex -import time -from collections.abc import Awaitable -from collections.abc import Callable +from collections.abc import Awaitable, Callable from dataclasses import dataclass from typing import Protocol, TypeVar, cast @@ -30,169 +20,15 @@ from dify_agent.adapters.shell.protocols import ( ShellCommandProtocol, ShellCommandResult, ShellCommandStatus, - ShellFileTransferProtocol, ShellProviderError, ) -from dify_agent.runtime_backend.errors import ( - WorkspaceFileTooLargeError, - WorkspacePathError, - WorkspaceUnavailableError, -) -from dify_agent.runtime_backend.protocols import ( - WorkspaceFileEntry, - WorkspaceFileContent, - WorkspaceListResult, - WorkspaceReadResult, -) - -logger = logging.getLogger(__name__) ResultT = TypeVar("ResultT") _DEFAULT_TIMEOUT_SECONDS = 30.0 _READ_OUTPUT_TIMEOUT_SECONDS = 0.0 _DEFAULT_TERMINATE_GRACE_SECONDS = 10.0 -_FILE_TRANSFER_TIMEOUT_SECONDS = 60.0 _SHELLCTL_OUTPUT_LIMIT_BYTES = 16 * 1024 -_TRANSFER_BEGIN = "<<>>" -_TRANSFER_END = "<<>>" -_DOWNLOAD_MISSING_EXIT_CODE = 66 -_WORKSPACE_PAYLOAD_BEGIN = "<<>>" -_WORKSPACE_PAYLOAD_END = "<<>>" - -_LIST_WORKSPACE_SCRIPT = r""" -import base64 -import json -import os -import stat -import sys - -path = sys.argv[1] -limit = int(sys.argv[2]) -directory_fd = None -try: - directory_fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) - # DIFY_WORKSPACE_CHECKPOINT: directory_opened - names = sorted(os.listdir(directory_fd)) - entries = [] - for name in names[:limit]: - child_stat = os.stat(name, dir_fd=directory_fd, follow_symlinks=False) - mode = child_stat.st_mode - entry_type = ( - "symlink" if stat.S_ISLNK(mode) else - "dir" if stat.S_ISDIR(mode) else - "file" if stat.S_ISREG(mode) else - "other" - ) - entries.append({ - "name": name, - "type": entry_type, - "size": int(child_stat.st_size), - "mtime": int(child_stat.st_mtime), - }) -finally: - if directory_fd is not None: - os.close(directory_fd) - -payload = {"path": path, "entries": entries, "truncated": len(names) > limit} -blob = base64.b64encode(json.dumps(payload).encode()).decode() -print("<<>>" + blob + "<<>>") -""" - -_READ_WORKSPACE_SCRIPT = r""" -import base64 -import json -import os -import stat -import sys - -path = sys.argv[1] -max_bytes = int(sys.argv[2]) -file_fd = None -try: - file_fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) - # DIFY_WORKSPACE_CHECKPOINT: file_opened - file_stat = os.fstat(file_fd) - if not stat.S_ISREG(file_stat.st_mode): - raise FileNotFoundError(path) - size = int(file_stat.st_size) - data = os.read(file_fd, max_bytes + 1) -finally: - if file_fd is not None: - os.close(file_fd) - -truncated = len(data) > max_bytes -data = data[:max_bytes] -try: - text = data.decode("utf-8") - binary = False -except UnicodeDecodeError: - text = None - binary = True -payload = { - "path": path, - "size": size, - "truncated": truncated, - "binary": binary, - "text": text, -} -blob = base64.b64encode(json.dumps(payload).encode()).decode() -print("<<>>" + blob + "<<>>") -""" - -_READ_WORKSPACE_BYTES_SCRIPT = r""" -import base64 -import json -import os -import stat -import sys - -path = sys.argv[1] -max_bytes = int(sys.argv[2]) -# DIFY_WORKSPACE_CHECKPOINT: arguments_loaded -file_fd = None -try: - file_fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) - # DIFY_WORKSPACE_CHECKPOINT: file_opened - file_stat = os.fstat(file_fd) - if not stat.S_ISREG(file_stat.st_mode): - raise FileNotFoundError(path) - size = int(file_stat.st_size) - # DIFY_WORKSPACE_CHECKPOINT: file_size_captured - content = None - if size <= max_bytes: - chunks = [] - remaining = max_bytes + 1 - while remaining > 0: - chunk = os.read(file_fd, min(1024 * 1024, remaining)) - if not chunk: - break - chunks.append(chunk) - remaining -= len(chunk) - captured = b"".join(chunks) - if len(captured) > max_bytes: - size = max(size, len(captured)) - else: - content = captured -finally: - if file_fd is not None: - os.close(file_fd) - -if content is None: - payload = {"path": path, "size": size, "too_large": True} -else: - payload = { - "path": path, - "size": size, - "content_base64": base64.b64encode(content).decode("ascii"), - } -blob = base64.b64encode(json.dumps(payload).encode()).decode() -print("<<>>" + blob + "<<>>") -""" - - -class ShellFileTransferError(RuntimeError): - """Raised when a file cannot be uploaded or downloaded through shellctl.""" class ShellctlJobResult(Protocol): @@ -344,160 +180,6 @@ class ShellctlCommands(ShellCommandProtocol): raise -@dataclass(slots=True) -class ShellctlFileTransfer(ShellFileTransferProtocol): - client: ShellctlClientProtocol - cwd: str | None = None - home_dir: str | None = None - timeout: float = _FILE_TRANSFER_TIMEOUT_SECONDS - - async def list_directory( - self, - *, - path: str, - limit: int, - ) -> WorkspaceListResult: - payload = await self._run_workspace_script( - _LIST_WORKSPACE_SCRIPT, - args=[self._resolve_path(path), str(limit)], - ) - raw_entries = payload.get("entries") - if not isinstance(raw_entries, list): - raise WorkspaceUnavailableError("workspace list returned an invalid entries payload") - entries: list[WorkspaceFileEntry] = [] - for raw_entry in raw_entries: - if not isinstance(raw_entry, dict): - raise WorkspaceUnavailableError("workspace list returned an invalid entry") - entries.append( - WorkspaceFileEntry( - name=str(raw_entry.get("name", "")), - type=str(raw_entry.get("type", "other")), - size=int(raw_entry["size"]) if isinstance(raw_entry.get("size"), int) else None, - mtime=int(raw_entry["mtime"]) if isinstance(raw_entry.get("mtime"), int) else None, - ) - ) - return WorkspaceListResult( - path=path, - entries=tuple(entries), - truncated=payload.get("truncated") is True, - ) - - async def read_file( - self, - *, - path: str, - max_bytes: int, - ) -> WorkspaceReadResult: - payload = await self._run_workspace_script( - _READ_WORKSPACE_SCRIPT, - args=[self._resolve_path(path), str(max_bytes)], - ) - size = payload.get("size") - if not isinstance(size, int): - raise WorkspaceUnavailableError("workspace read returned an invalid size") - text = payload.get("text") - return WorkspaceReadResult( - path=path, - size=size, - truncated=payload.get("truncated") is True, - binary=payload.get("binary") is True, - text=text if isinstance(text, str) else None, - ) - - async def read_bytes( - self, - *, - path: str, - max_bytes: int, - ) -> WorkspaceFileContent: - if max_bytes < 1: - raise ValueError("max_bytes must be positive") - payload = await self._run_workspace_script( - _READ_WORKSPACE_BYTES_SCRIPT, - args=[self._resolve_path(path), str(max_bytes)], - ) - size = payload.get("size") - if payload.get("too_large") is True: - if not isinstance(size, int): - raise WorkspaceUnavailableError("workspace bytes read returned an invalid size") - raise WorkspaceFileTooLargeError(path=path, size=size, max_bytes=max_bytes) - encoded = payload.get("content_base64") - if not isinstance(size, int) or not isinstance(encoded, str): - raise WorkspaceUnavailableError("workspace bytes read returned an invalid payload") - try: - content = base64.b64decode(encoded, validate=True) - except (ValueError, binascii.Error) as exc: - raise WorkspaceUnavailableError("workspace bytes read returned invalid base64") from exc - if len(content) != size: - raise WorkspaceUnavailableError("workspace bytes read returned an invalid size") - return WorkspaceFileContent( - path=path, - size=size, - content=content, - ) - - async def _run_workspace_script(self, source: str, *, args: list[str]) -> dict[str, object]: - command = _python_stdin_command(source, args=args) - completed = await _run_to_completion( - self.client, - command, - cwd=_resolve_lease_cwd(self.cwd, home_dir=self.home_dir, workspace_dir=self.cwd), - env=_lease_env(None, home_dir=self.home_dir), - timeout=self.timeout, - ) - if completed.exit_code != 0: - detail = _output_tail(completed.output) - if "PermissionError" in completed.output: - raise WorkspacePathError(detail) - raise WorkspaceUnavailableError(detail) - return _decode_workspace_payload(completed.output) - - def _resolve_path(self, path: str) -> str: - if self.home_dir is None: - return path - if path == "~": - return self.home_dir - if path.startswith("~/"): - return f"{self.home_dir.rstrip('/')}/{path[2:]}" - return path - - async def upload(self, *, content: bytes, remote_path: str, cwd: str | None = None) -> None: - encoded = base64.b64encode(content).decode("ascii") - completed = await _run_to_completion( - self.client, - _upload_script(remote_path=remote_path, encoded=encoded), - cwd=_resolve_lease_cwd(cwd, home_dir=self.home_dir, workspace_dir=self.cwd), - env=_lease_env(None, home_dir=self.home_dir), - timeout=self.timeout, - ) - if completed.exit_code != 0: - raise ShellFileTransferError( - f"Failed to upload to {remote_path!r}: exit_code={completed.exit_code}, " - f"output={_output_tail(completed.output)!r}" - ) - - async def download(self, *, remote_path: str, cwd: str | None = None) -> bytes: - completed = await _run_to_completion( - self.client, - _download_script(remote_path=remote_path), - cwd=_resolve_lease_cwd(cwd, home_dir=self.home_dir, workspace_dir=self.cwd), - env=_lease_env(None, home_dir=self.home_dir), - timeout=self.timeout, - ) - if completed.exit_code == _DOWNLOAD_MISSING_EXIT_CODE: - raise ShellFileTransferError(f"Remote path not found: {remote_path!r}.") - if completed.exit_code != 0: - raise ShellFileTransferError( - f"Failed to download {remote_path!r}: exit_code={completed.exit_code}, " - f"output={_output_tail(completed.output)!r}" - ) - encoded = _extract_transfer_payload(completed.output) - try: - return base64.b64decode(encoded.encode("ascii"), validate=True) - except (ValueError, binascii.Error) as exc: - raise ShellFileTransferError(f"Downloaded payload for {remote_path!r} was not valid base64.") from exc - - def create_default_shellctl_client_factory( *, entrypoint: str, @@ -518,13 +200,6 @@ def create_default_shellctl_client_factory( return factory -@dataclass(frozen=True, slots=True) -class _CompletedShellctlJob: - job_id: str - exit_code: int | None - output: str - - async def _run_client_call(awaitable: Awaitable[ResultT]) -> ResultT: """Map shellctl client boundary failures into provider-layer errors.""" @@ -578,94 +253,6 @@ def _status_name(status: object) -> str: return str(status) -async def _run_to_completion( - client: ShellctlClientProtocol, - script: str, - *, - cwd: str | None, - env: dict[str, str] | None, - timeout: float, -) -> _CompletedShellctlJob: - deadline = time.monotonic() + timeout - job_id: str | None = None - try: - result = await _run_client_call(client.run(script, cwd=cwd, env=env, timeout=_remaining_timeout(deadline))) - parts = [result.output] - job_id = result.job_id - while not result.done or result.truncated: - result = await _run_client_call( - client.wait(job_id, offset=result.offset, timeout=_remaining_timeout(deadline)) - ) - parts.append(result.output) - return _CompletedShellctlJob(job_id=job_id, exit_code=result.exit_code, output="".join(parts)) - finally: - if job_id is not None: - try: - await _run_client_call(client.delete(job_id, force=True)) - except RuntimeError as exc: - logger.warning("Failed to delete shellctl job %s: %s", job_id, exc) - - -def _upload_script(*, remote_path: str, encoded: str) -> str: - return ( - "set -eu\n" - f'mkdir -p "$(dirname -- {_shquote(remote_path)})"\n' - f"printf %s {_shquote(encoded)} | base64 -d > {_shquote(remote_path)}" - ) - - -def _download_script(*, remote_path: str) -> str: - return "\n".join( - [ - "set -eu", - f"path={_shquote(remote_path)}", - 'if [ ! -f "$path" ]; then exit 66; fi', - f"printf %s {_shquote(_TRANSFER_BEGIN)}", - 'base64 < "$path" | tr -d "\\n"', - f"printf %s {_shquote(_TRANSFER_END)}", - ] - ) - - -def _python_stdin_command(source: str, *, args: list[str]) -> str: - quoted_args = " ".join(shlex.quote(value) for value in args) - return f"python3 - {quoted_args} <<'PY'\n{source.strip()}\nPY" - - -def _decode_workspace_payload(output: str) -> dict[str, object]: - begin = output.find(_WORKSPACE_PAYLOAD_BEGIN) - end = output.find(_WORKSPACE_PAYLOAD_END, begin + len(_WORKSPACE_PAYLOAD_BEGIN)) if begin >= 0 else -1 - if begin < 0 or end < 0: - raise WorkspaceUnavailableError("workspace command returned no framed payload") - encoded = "".join(output[begin + len(_WORKSPACE_PAYLOAD_BEGIN) : end].split()) - try: - value = json.loads(base64.b64decode(encoded, validate=True).decode("utf-8")) - except (binascii.Error, UnicodeDecodeError, ValueError) as exc: - raise WorkspaceUnavailableError("workspace command returned an invalid framed payload") from exc - if not isinstance(value, dict): - raise WorkspaceUnavailableError("workspace command returned a non-object payload") - return cast(dict[str, object], value) - - -def _extract_transfer_payload(output: str) -> str: - pattern = re.escape(_TRANSFER_BEGIN) + r"(.*?)" + re.escape(_TRANSFER_END) - match = re.search(pattern, output, re.DOTALL) - if match is None: - raise ShellFileTransferError("Transfer payload markers were missing from shell output.") - return "".join(match.group(1).split()) - - -def _output_tail(output: str, *, limit: int = 256) -> str: - return output[-limit:] - - -def _remaining_timeout(deadline: float) -> float: - remaining = deadline - time.monotonic() - if remaining <= 0.0: - raise ShellProviderError("Shellctl command timed out before completion.", code="timeout") - return remaining - - def _lease_env(env: dict[str, str] | None, *, home_dir: str | None) -> dict[str, str] | None: if home_dir is None: return env @@ -699,15 +286,9 @@ def _resolve_lease_cwd( return candidate -def _shquote(value: str) -> str: - return "'" + value.replace("'", "'\\''") + "'" - - __all__ = [ - "ShellFileTransferError", "ShellctlClientFactory", "ShellctlClientProtocol", "ShellctlCommands", - "ShellctlFileTransfer", "create_default_shellctl_client_factory", ] diff --git a/dify-agent/src/dify_agent/agent_stub/server/agent_stub_files.py b/dify-agent/src/dify_agent/agent_stub/server/agent_stub_files.py index 5a8842665fe..636a88247d1 100644 --- a/dify-agent/src/dify_agent/agent_stub/server/agent_stub_files.py +++ b/dify-agent/src/dify_agent/agent_stub/server/agent_stub_files.py @@ -102,7 +102,7 @@ class DifyApiAgentStubFileRequestHandler: """Call Dify API inner file request endpoints on behalf of the sandbox. The upload path calls ``/inner/api/agent/files/upload-request`` and injects the - authenticated execution context's ``tenant_id``, ``user_id``, and optional + authenticated execution context's ``tenant_id``, ``user_id``, ``user_from``, and optional ``conversation_id`` along with the requested filename and mimetype. The download path calls ``/inner/api/agent/files/download-request`` and injects ``tenant_id``, ``user_id``, ``user_from``, and ``invoke_from`` plus the validated public @@ -143,6 +143,7 @@ class DifyApiAgentStubFileRequestHandler: payload = { "tenant_id": execution_context.tenant_id, "user_id": execution_context.user_id, + "user_from": execution_context.user_from, "filename": request.filename, "mimetype": request.mimetype, "conversation_id": execution_context.conversation_id, diff --git a/dify-agent/src/dify_agent/client/_client.py b/dify-agent/src/dify_agent/client/_client.py index d72eb975c82..0cd14a68711 100644 --- a/dify-agent/src/dify_agent/client/_client.py +++ b/dify-agent/src/dify_agent/client/_client.py @@ -28,6 +28,12 @@ from pydantic_ai.messages import FunctionToolResultEvent from dify_agent.protocol import ( CancelRunRequest, CancelRunResponse, + BindingFileDownloadRequest, + BindingFileDownloadResponse, + BindingFileListRequest, + BindingFileListResponse, + BindingFileReadRequest, + BindingFileReadResponse, CreateRunRequest, CreateRunResponse, CreateExecutionBindingRequest, @@ -40,17 +46,12 @@ from dify_agent.protocol import ( RunEvent, RunEventsResponse, RunStatusResponse, - WorkspaceListRequest, - WorkspaceListResponse, - WorkspaceReadRequest, - WorkspaceReadResponse, - WorkspaceUploadRequest, - WorkspaceUploadResponse, ) _ResponseModelT = TypeVar("_ResponseModelT", bound=BaseModel) _TERMINAL_EVENT_TYPES = {"run_succeeded", "run_failed", "run_cancelled"} _TERMINAL_RUN_STATUSES = {"succeeded", "failed", "cancelled"} +_BINDING_FILE_DOWNLOAD_TIMEOUT_SECONDS = 90.0 _function_tool_result_payload_key_cache: str | None = None @@ -260,7 +261,7 @@ class Client: headers, timeout settings, optional external HTTPX clients, and lazy-owned clients for whichever sync/async side is used. It is the shared transport boundary for both run-management endpoints (create/status/events/cancel) and - sandbox-file endpoints (list/read/upload). External clients are never closed + Binding-file endpoints (list/read/download). External clients are never closed by this wrapper. Owned sync clients close via ``close_sync`` or the sync context manager; owned async clients close via ``aclose`` or the async context manager. @@ -474,43 +475,53 @@ class Client: raise DifyAgentClientError(f"get_events_sync request failed: {exc}") from exc return _parse_model_response(response, RunEventsResponse) - async def list_workspace_files(self, backend_binding_ref: str, path: str) -> WorkspaceListResponse: - request_model = WorkspaceListRequest(backend_binding_ref=backend_binding_ref, path=path) - response = await self._post_async_json("list_workspace_files", "/workspace/files/list", request_model) - return _parse_model_response(response, WorkspaceListResponse) + async def list_binding_files(self, backend_binding_ref: str, path: str) -> BindingFileListResponse: + request_model = BindingFileListRequest(backend_binding_ref=backend_binding_ref, path=path) + response = await self._post_async_json("list_binding_files", "/execution-bindings/files/list", request_model) + return _parse_model_response(response, BindingFileListResponse) - def list_workspace_files_sync(self, backend_binding_ref: str, path: str) -> WorkspaceListResponse: - request_model = WorkspaceListRequest(backend_binding_ref=backend_binding_ref, path=path) - response = self._post_sync_json("list_workspace_files_sync", "/workspace/files/list", request_model) - return _parse_model_response(response, WorkspaceListResponse) + def list_binding_files_sync(self, backend_binding_ref: str, path: str) -> BindingFileListResponse: + request_model = BindingFileListRequest(backend_binding_ref=backend_binding_ref, path=path) + response = self._post_sync_json("list_binding_files_sync", "/execution-bindings/files/list", request_model) + return _parse_model_response(response, BindingFileListResponse) - async def read_workspace_file( + async def read_binding_file( self, backend_binding_ref: str, path: str, max_bytes: int = 262144, - ) -> WorkspaceReadResponse: - request_model = WorkspaceReadRequest(backend_binding_ref=backend_binding_ref, path=path, max_bytes=max_bytes) - response = await self._post_async_json("read_workspace_file", "/workspace/files/read", request_model) - return _parse_model_response(response, WorkspaceReadResponse) + ) -> BindingFileReadResponse: + request_model = BindingFileReadRequest(backend_binding_ref=backend_binding_ref, path=path, max_bytes=max_bytes) + response = await self._post_async_json("read_binding_file", "/execution-bindings/files/read", request_model) + return _parse_model_response(response, BindingFileReadResponse) - def read_workspace_file_sync( + def read_binding_file_sync( self, backend_binding_ref: str, path: str, max_bytes: int = 262144, - ) -> WorkspaceReadResponse: - request_model = WorkspaceReadRequest(backend_binding_ref=backend_binding_ref, path=path, max_bytes=max_bytes) - response = self._post_sync_json("read_workspace_file_sync", "/workspace/files/read", request_model) - return _parse_model_response(response, WorkspaceReadResponse) + ) -> BindingFileReadResponse: + request_model = BindingFileReadRequest(backend_binding_ref=backend_binding_ref, path=path, max_bytes=max_bytes) + response = self._post_sync_json("read_binding_file_sync", "/execution-bindings/files/read", request_model) + return _parse_model_response(response, BindingFileReadResponse) - async def upload_workspace_file(self, request: WorkspaceUploadRequest) -> WorkspaceUploadResponse: - response = await self._post_async_json("upload_workspace_file", "/workspace/files/upload", request) - return _parse_model_response(response, WorkspaceUploadResponse) + async def download_binding_file(self, request: BindingFileDownloadRequest) -> BindingFileDownloadResponse: + response = await self._post_async_json( + "download_binding_file", + "/execution-bindings/files/download", + request, + timeout=_BINDING_FILE_DOWNLOAD_TIMEOUT_SECONDS, + ) + return _parse_model_response(response, BindingFileDownloadResponse) - def upload_workspace_file_sync(self, request: WorkspaceUploadRequest) -> WorkspaceUploadResponse: - response = self._post_sync_json("upload_workspace_file_sync", "/workspace/files/upload", request) - return _parse_model_response(response, WorkspaceUploadResponse) + def download_binding_file_sync(self, request: BindingFileDownloadRequest) -> BindingFileDownloadResponse: + response = self._post_sync_json( + "download_binding_file_sync", + "/execution-bindings/files/download", + request, + timeout=_BINDING_FILE_DOWNLOAD_TIMEOUT_SECONDS, + ) + return _parse_model_response(response, BindingFileDownloadResponse) async def create_execution_binding(self, request: CreateExecutionBindingRequest) -> CreateExecutionBindingResponse: response = await self._post_async_json("create_execution_binding", "/execution-bindings", request) @@ -862,26 +873,40 @@ class Client: headers.update(extra) return headers - async def _post_async_json(self, operation: str, path: str, request_model: BaseModel) -> httpx.Response: + async def _post_async_json( + self, + operation: str, + path: str, + request_model: BaseModel, + *, + timeout: float | httpx.Timeout | None = None, + ) -> httpx.Response: try: return await self._get_async_http_client().post( self._url(path), content=request_model.model_dump_json(), headers=self._merged_headers({"Content-Type": "application/json"}), - timeout=self._timeout, + timeout=self._timeout if timeout is None else timeout, ) except httpx.TimeoutException as exc: raise DifyAgentTimeoutError(f"{operation} timed out") from exc except httpx.RequestError as exc: raise DifyAgentClientError(f"{operation} request failed: {exc}") from exc - def _post_sync_json(self, operation: str, path: str, request_model: BaseModel) -> httpx.Response: + def _post_sync_json( + self, + operation: str, + path: str, + request_model: BaseModel, + *, + timeout: float | httpx.Timeout | None = None, + ) -> httpx.Response: try: return self._get_sync_http_client().post( self._url(path), content=request_model.model_dump_json(), headers=self._merged_headers({"Content-Type": "application/json"}), - timeout=self._timeout, + timeout=self._timeout if timeout is None else timeout, ) except httpx.TimeoutException as exc: raise DifyAgentTimeoutError(f"{operation} timed out") from exc diff --git a/dify-agent/src/dify_agent/layers/_agent_cli_help.json b/dify-agent/src/dify_agent/layers/_agent_cli_help.json index 1361d1a4170..66578c3d9e0 100644 --- a/dify-agent/src/dify_agent/layers/_agent_cli_help.json +++ b/dify-agent/src/dify_agent/layers/_agent_cli_help.json @@ -21,5 +21,5 @@ "drive push": "Upload one local file or directory into the agent drive.\n\nUsage:\n dify-agent drive push LOCAL_PATH REMOTE_PATH [flags]\n\nFlags:\n -h, --help help for push\n --json Accepted for consistency; drive push output is already emitted as JSON.\n --kind string Directory upload kind: skill or dir.", "file": "Upload or download workflow files through the Agent Stub.\n\nUsage:\n dify-agent file [command]\n\nAvailable Commands:\n download Download one workflow file mapping into the local sandbox directory.\n upload Upload one sandbox-local file as a ToolFile output reference.\n\nFlags:\n -h, --help help for file\n\nUse \"dify-agent file [command] --help\" for more information about a command.", "file download": "Download one workflow file mapping into the local sandbox directory.\n\nUsage:\n dify-agent file download TRANSFER_METHOD REFERENCE_OR_URL [flags]\n\nFlags:\n -h, --help help for download\n --to string Local directory for the downloaded file.", - "file upload": "Upload one sandbox-local file as a ToolFile output reference.\n\nUsage:\n dify-agent file upload PATH [flags]\n\nFlags:\n -h, --help help for upload" + "file upload": "Upload one sandbox-local file as a ToolFile output reference.\n\nUsage:\n dify-agent file upload PATH [flags]\n\nFlags:\n -h, --help help for upload\n --no-download-link Skip creating a public download link after upload." } diff --git a/dify-agent/src/dify_agent/layers/shell/layer.py b/dify-agent/src/dify_agent/layers/shell/layer.py index 852fcefcaea..112a7def742 100644 --- a/dify-agent/src/dify_agent/layers/shell/layer.py +++ b/dify-agent/src/dify_agent/layers/shell/layer.py @@ -6,9 +6,8 @@ from collections.abc import Sequence import json import logging import re -import time from dataclasses import dataclass, field -from typing import ClassVar, Literal, NotRequired, Protocol, TypedDict, runtime_checkable +from typing import ClassVar, NotRequired, Protocol, TypedDict, runtime_checkable from pydantic import BaseModel, ConfigDict, Field, NonNegativeInt, field_validator, model_validator from pydantic_ai import Tool @@ -35,6 +34,7 @@ from dify_agent.layers.execution_context import DifyExecutionContextLayerConfig from dify_agent.layers.runtime.layer import DifyRuntimeLayer from dify_agent.layers.shell.configs import DIFY_SHELL_LAYER_TYPE_ID, DifyShellLayerConfig from dify_agent.layers.shell.output_text import normalized_output_text, utf8_prefix, utf8_suffix +from dify_agent.runtime.command_runner import execute_complete_with_commands from dify_agent.runtime_backend import RuntimeLease @@ -545,77 +545,6 @@ class DifyShellLayer(PydanticAILayer[DifyShellLayerDeps, object, DifyShellLayerC return text -async def execute_complete_with_commands( - commands: ShellCommandProtocol, - script: str, - *, - cwd: str | None, - env: dict[str, str] | None, - timeout: float, - max_output_bytes: int, -) -> CompleteShellCommandResult: - deadline = time.monotonic() + timeout - job_id: str | None = None - result: ShellCommandResult | None = None - output_parts: list[str] = [] - captured_bytes = 0 - incomplete_reason: Literal["output_limit", "timeout"] | None = None - try: - result = await commands.run(script, cwd=cwd, env=env, timeout=_remaining_time(deadline)) - job_id = result.job_id - while True: - remaining_bytes = max(max_output_bytes - captured_bytes, 0) - limited_output = utf8_prefix(result.output, remaining_bytes) - output_parts.append(limited_output) - captured_bytes += len(limited_output.encode("utf-8")) - if limited_output != result.output: - incomplete_reason = "output_limit" - break - if captured_bytes >= max_output_bytes and (result.truncated or not result.done): - incomplete_reason = "output_limit" - break - if result.truncated: - result = await commands.read_output(result.job_id, offset=result.offset) - continue - if result.done: - break - remaining_time = _remaining_time(deadline) - if remaining_time <= 0.0: - incomplete_reason = "timeout" - break - result = await commands.wait(result.job_id, offset=result.offset, timeout=remaining_time) - - assert result is not None - final_status = result.status - final_done = result.done - final_exit_code = result.exit_code - final_offset = result.offset - final_output_path = result.output_path - if incomplete_reason is not None and not result.done: - terminal_status = await commands.interrupt(result.job_id, grace_seconds=DEFAULT_TERMINATE_GRACE_SECONDS) - final_status = terminal_status.status - final_done = terminal_status.done - final_exit_code = terminal_status.exit_code - final_offset = terminal_status.offset - return CompleteShellCommandResult( - job_id=result.job_id, - status=final_status, - done=final_done, - exit_code=final_exit_code, - output="".join(output_parts), - output_complete=incomplete_reason is None, - incomplete_reason=incomplete_reason, - offset=final_offset, - output_path=final_output_path, - ) - finally: - if job_id is not None: - try: - await commands.delete(job_id, force=True) - except RuntimeError as exc: - logger.warning("Failed to delete transient shell job %s: %s", job_id, exc) - - async def render_prompt_observation_from_result( commands: ShellCommandProtocol, result: ShellCommandResult, @@ -765,10 +694,6 @@ def _tagged_shell_observation(metadata: dict[str, object], output: str) -> str: return f"\n{compact_metadata}\n\n\n\n{output}\n" -def _remaining_time(deadline: float) -> float: - return max(0.0, deadline - time.monotonic()) - - __all__ = [ "CompleteRemoteCommandResult", "DifyShellLayer", @@ -776,6 +701,5 @@ __all__ = [ "DifyShellRuntimeState", "DEFAULT_TERMINATE_GRACE_SECONDS", "DEFAULT_TIMEOUT_SECONDS", - "execute_complete_with_commands", "render_prompt_observation_from_result", ] diff --git a/dify-agent/src/dify_agent/protocol/__init__.py b/dify-agent/src/dify_agent/protocol/__init__.py index 32ce269ee3d..0eb18b57a36 100644 --- a/dify-agent/src/dify_agent/protocol/__init__.py +++ b/dify-agent/src/dify_agent/protocol/__init__.py @@ -43,24 +43,30 @@ from .execution_binding import ( CreateExecutionBindingResponse, DestroyExecutionBindingRequest, ) +from .binding_file import ( + BindingFileDownloadRequest, + BindingFileDownloadResponse, + BindingFileEntry, + BindingFileListRequest, + BindingFileListResponse, + BindingFileReadRequest, + BindingFileReadResponse, +) from .home_snapshot import ( CreateHomeSnapshotFromBindingRequest, DeleteHomeSnapshotRequest, HomeSnapshotResponse, ) -from .workspace import ( - WorkspaceFileEntry, - WorkspaceListRequest, - WorkspaceListResponse, - WorkspaceReadRequest, - WorkspaceReadResponse, - WorkspaceUploadRequest, - WorkspaceUploadResponse, - WorkspaceUploadedFile, -) __all__ = [ "BaseRunEvent", + "BindingFileDownloadRequest", + "BindingFileDownloadResponse", + "BindingFileEntry", + "BindingFileListRequest", + "BindingFileListResponse", + "BindingFileReadRequest", + "BindingFileReadResponse", "AgentRunUsage", "CancelRunRequest", "CancelRunResponse", @@ -96,14 +102,6 @@ __all__ = [ "RunStatusResponse", "RunSucceededEvent", "RunSucceededEventData", - "WorkspaceFileEntry", - "WorkspaceListRequest", - "WorkspaceListResponse", - "WorkspaceReadRequest", - "WorkspaceReadResponse", - "WorkspaceUploadRequest", - "WorkspaceUploadResponse", - "WorkspaceUploadedFile", "normalize_composition", "utc_now", ] diff --git a/dify-agent/src/dify_agent/protocol/workspace.py b/dify-agent/src/dify_agent/protocol/binding_file.py similarity index 56% rename from dify-agent/src/dify_agent/protocol/workspace.py rename to dify-agent/src/dify_agent/protocol/binding_file.py index ad1d909fdee..0de2af7574e 100644 --- a/dify-agent/src/dify_agent/protocol/workspace.py +++ b/dify-agent/src/dify_agent/protocol/binding_file.py @@ -1,4 +1,4 @@ -"""Private Workspace file DTOs resolved through an Execution Binding ref.""" +"""Private Binding file DTOs resolved through an Execution Binding ref.""" from typing import ClassVar, Literal @@ -6,8 +6,10 @@ from pydantic import BaseModel, ConfigDict, Field from dify_agent.layers.execution_context import DifyExecutionContextLayerConfig +_BINDING_FILE_PREVIEW_MAX_BYTES = 262144 -class WorkspaceFileEntry(BaseModel): + +class BindingFileEntry(BaseModel): name: str type: Literal["file", "dir", "symlink", "other"] size: int | None = None @@ -16,30 +18,34 @@ class WorkspaceFileEntry(BaseModel): model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid") -class WorkspaceListRequest(BaseModel): +class BindingFileListRequest(BaseModel): backend_binding_ref: str = Field(min_length=1) path: str = "." model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid") -class WorkspaceListResponse(BaseModel): +class BindingFileListResponse(BaseModel): path: str - entries: list[WorkspaceFileEntry] + entries: list[BindingFileEntry] truncated: bool model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid") -class WorkspaceReadRequest(BaseModel): +class BindingFileReadRequest(BaseModel): backend_binding_ref: str = Field(min_length=1) - path: str - max_bytes: int = 262144 + path: str = Field(min_length=1) + max_bytes: int = Field( + default=_BINDING_FILE_PREVIEW_MAX_BYTES, + ge=1, + le=_BINDING_FILE_PREVIEW_MAX_BYTES, + ) model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid") -class WorkspaceReadResponse(BaseModel): +class BindingFileReadResponse(BaseModel): path: str size: int | None = None truncated: bool @@ -49,36 +55,26 @@ class WorkspaceReadResponse(BaseModel): model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid") -class WorkspaceUploadedFile(BaseModel): - transfer_method: Literal["tool_file"] = "tool_file" - reference: str - download_url: str - - model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid") - - -class WorkspaceUploadRequest(BaseModel): +class BindingFileDownloadRequest(BaseModel): backend_binding_ref: str = Field(min_length=1) - path: str + path: str = Field(min_length=1) execution_context: DifyExecutionContextLayerConfig model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid") -class WorkspaceUploadResponse(BaseModel): - path: str - file: WorkspaceUploadedFile +class BindingFileDownloadResponse(BaseModel): + reference: str model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid") __all__ = [ - "WorkspaceFileEntry", - "WorkspaceListRequest", - "WorkspaceListResponse", - "WorkspaceReadRequest", - "WorkspaceReadResponse", - "WorkspaceUploadRequest", - "WorkspaceUploadResponse", - "WorkspaceUploadedFile", + "BindingFileDownloadRequest", + "BindingFileDownloadResponse", + "BindingFileEntry", + "BindingFileListRequest", + "BindingFileListResponse", + "BindingFileReadRequest", + "BindingFileReadResponse", ] diff --git a/dify-agent/src/dify_agent/runtime/command_runner.py b/dify-agent/src/dify_agent/runtime/command_runner.py new file mode 100644 index 00000000000..1ceec51a0cf --- /dev/null +++ b/dify-agent/src/dify_agent/runtime/command_runner.py @@ -0,0 +1,97 @@ +"""Bounded execution of one transient command through a RuntimeLease.""" + +from __future__ import annotations + +import logging +import time +from typing import Literal + +from dify_agent.adapters.shell.protocols import ( + CompleteShellCommandResult, + ShellCommandProtocol, + ShellCommandResult, +) +from dify_agent.layers.shell.output_text import utf8_prefix + +logger = logging.getLogger(__name__) + +_TERMINATE_GRACE_SECONDS = 10.0 + + +async def execute_complete_with_commands( + commands: ShellCommandProtocol, + script: str, + *, + cwd: str | None, + env: dict[str, str] | None, + timeout: float, + max_output_bytes: int, +) -> CompleteShellCommandResult: + """Run a command to completion with bounded output and deterministic cleanup.""" + + deadline = time.monotonic() + timeout + job_id: str | None = None + result: ShellCommandResult | None = None + output_parts: list[str] = [] + captured_bytes = 0 + incomplete_reason: Literal["output_limit", "timeout"] | None = None + try: + result = await commands.run(script, cwd=cwd, env=env, timeout=_remaining_time(deadline)) + job_id = result.job_id + while True: + remaining_bytes = max(max_output_bytes - captured_bytes, 0) + limited_output = utf8_prefix(result.output, remaining_bytes) + output_parts.append(limited_output) + captured_bytes += len(limited_output.encode("utf-8")) + if limited_output != result.output: + incomplete_reason = "output_limit" + break + if captured_bytes >= max_output_bytes and (result.truncated or not result.done): + incomplete_reason = "output_limit" + break + if result.truncated: + result = await commands.read_output(result.job_id, offset=result.offset) + continue + if result.done: + break + remaining_time = _remaining_time(deadline) + if remaining_time <= 0.0: + incomplete_reason = "timeout" + break + result = await commands.wait(result.job_id, offset=result.offset, timeout=remaining_time) + + final_status = result.status + final_done = result.done + final_exit_code = result.exit_code + final_offset = result.offset + final_output_path = result.output_path + if incomplete_reason is not None and not result.done: + terminal_status = await commands.interrupt(result.job_id, grace_seconds=_TERMINATE_GRACE_SECONDS) + final_status = terminal_status.status + final_done = terminal_status.done + final_exit_code = terminal_status.exit_code + final_offset = terminal_status.offset + return CompleteShellCommandResult( + job_id=result.job_id, + status=final_status, + done=final_done, + exit_code=final_exit_code, + output="".join(output_parts), + output_complete=incomplete_reason is None, + incomplete_reason=incomplete_reason, + offset=final_offset, + output_path=final_output_path, + ) + finally: + if job_id is not None: + try: + await commands.delete(job_id, force=True) + except RuntimeError as exc: + logger.warning("Failed to delete transient shell job %s: %s", job_id, exc) + + +def _remaining_time(deadline: float) -> float: + return max(0.0, deadline - time.monotonic()) + + +__all__ = ["execute_complete_with_commands"] diff --git a/dify-agent/src/dify_agent/runtime_backend/__init__.py b/dify-agent/src/dify_agent/runtime_backend/__init__.py index 422b65f48f1..bca6748166f 100644 --- a/dify-agent/src/dify_agent/runtime_backend/__init__.py +++ b/dify-agent/src/dify_agent/runtime_backend/__init__.py @@ -9,8 +9,6 @@ from .errors import ( HomeSnapshotNotFoundError, RuntimeBackendError, SharedWorkspaceUnsupportedError, - WorkspaceFileTooLargeError, - WorkspacePathError, WorkspacePreservationUnsupportedError, WorkspaceUnavailableError, ) @@ -19,16 +17,11 @@ from .protocols import ( ExecutionBindingBackend, ExecutionBindingCreateSpec, ExecutionBindingDestroySpec, - FileSystem, HomeSnapshotBackend, HomeSnapshotCreateSpec, RuntimeBackendProfile, RuntimeLayout, RuntimeLease, - WorkspaceFileContent, - WorkspaceFileEntry, - WorkspaceListResult, - WorkspaceReadResult, ) __all__ = [ @@ -40,7 +33,6 @@ __all__ = [ "ExecutionBindingBackend", "ExecutionBindingCreateSpec", "ExecutionBindingDestroySpec", - "FileSystem", "HomeSnapshotBackend", "HomeSnapshotCreateError", "HomeSnapshotCreateSpec", @@ -50,12 +42,6 @@ __all__ = [ "RuntimeLayout", "RuntimeLease", "SharedWorkspaceUnsupportedError", - "WorkspaceFileContent", - "WorkspaceFileEntry", - "WorkspaceFileTooLargeError", - "WorkspaceListResult", - "WorkspacePathError", "WorkspacePreservationUnsupportedError", - "WorkspaceReadResult", "WorkspaceUnavailableError", ] diff --git a/dify-agent/src/dify_agent/runtime_backend/e2b.py b/dify-agent/src/dify_agent/runtime_backend/e2b.py index f20a9e06aee..930ea317dcf 100644 --- a/dify-agent/src/dify_agent/runtime_backend/e2b.py +++ b/dify-agent/src/dify_agent/runtime_backend/e2b.py @@ -30,7 +30,6 @@ from dify_agent.runtime_backend.protocols import ( ExecutionBindingAllocation, ExecutionBindingCreateSpec, ExecutionBindingDestroySpec, - FileSystem, HomeSnapshotCreateSpec, RuntimeLayout, RuntimeLease, @@ -358,10 +357,6 @@ class E2BRuntimeLease: def commands(self) -> ShellCommandProtocol: return self.data_plane.commands - @property - def files(self) -> FileSystem: - return self.data_plane.files - async def _wait_for_shellctl_ready(client: ShellctlClientProtocol) -> None: for attempt in range(_SHELLCTL_READY_MAX_ATTEMPTS): diff --git a/dify-agent/src/dify_agent/runtime_backend/enterprise.py b/dify-agent/src/dify_agent/runtime_backend/enterprise.py index a87e879c826..f86867d778e 100644 --- a/dify-agent/src/dify_agent/runtime_backend/enterprise.py +++ b/dify-agent/src/dify_agent/runtime_backend/enterprise.py @@ -31,7 +31,6 @@ from dify_agent.runtime_backend.protocols import ( ExecutionBindingAllocation, ExecutionBindingCreateSpec, ExecutionBindingDestroySpec, - FileSystem, HomeSnapshotCreateSpec, RuntimeLayout, RuntimeLease, @@ -255,10 +254,6 @@ class EnterpriseRuntimeLease: def commands(self) -> ShellCommandProtocol: return self.data_plane.commands - @property - def files(self) -> FileSystem: - return self.data_plane.files - def _is_missing_sandbox(exc: ShellProviderError) -> bool: return exc.status_code == 404 or (exc.code or "").casefold() in { diff --git a/dify-agent/src/dify_agent/runtime_backend/errors.py b/dify-agent/src/dify_agent/runtime_backend/errors.py index b7aa48b6fcd..260844b976f 100644 --- a/dify-agent/src/dify_agent/runtime_backend/errors.py +++ b/dify-agent/src/dify_agent/runtime_backend/errors.py @@ -41,22 +41,6 @@ class WorkspaceUnavailableError(RuntimeBackendError): pass -class WorkspacePathError(RuntimeBackendError): - pass - - -class WorkspaceFileTooLargeError(RuntimeBackendError): - path: str - size: int - max_bytes: int - - def __init__(self, *, path: str, size: int, max_bytes: int) -> None: - self.path = path - self.size = size - self.max_bytes = max_bytes - super().__init__(f"Workspace file {path!r} exceeds the {max_bytes}-byte ToolFile upload limit") - - __all__ = [ "BindingAcquireError", "BindingCreateError", @@ -66,8 +50,6 @@ __all__ = [ "HomeSnapshotNotFoundError", "RuntimeBackendError", "SharedWorkspaceUnsupportedError", - "WorkspaceFileTooLargeError", - "WorkspacePathError", "WorkspacePreservationUnsupportedError", "WorkspaceUnavailableError", ] diff --git a/dify-agent/src/dify_agent/runtime_backend/protocols.py b/dify-agent/src/dify_agent/runtime_backend/protocols.py index b6e52d19861..7acd7da3dbe 100644 --- a/dify-agent/src/dify_agent/runtime_backend/protocols.py +++ b/dify-agent/src/dify_agent/runtime_backend/protocols.py @@ -28,51 +28,6 @@ class RuntimeLayout: workspace_dir: str -@dataclass(frozen=True, slots=True) -class WorkspaceFileEntry: - name: str - type: str - size: int | None - mtime: int | None - - -@dataclass(frozen=True, slots=True) -class WorkspaceListResult: - path: str - entries: tuple[WorkspaceFileEntry, ...] - truncated: bool - - -@dataclass(frozen=True, slots=True) -class WorkspaceReadResult: - path: str - size: int - truncated: bool - binary: bool - text: str | None - - -@dataclass(frozen=True, slots=True) -class WorkspaceFileContent: - path: str - size: int - content: bytes - - -class FileSystem(Protocol): - """File operations interpreted in the current RuntimeLease namespace.""" - - async def list_directory(self, *, path: str, limit: int) -> WorkspaceListResult: ... - - async def read_file(self, *, path: str, max_bytes: int) -> WorkspaceReadResult: ... - - async def read_bytes(self, *, path: str, max_bytes: int) -> WorkspaceFileContent: ... - - async def upload(self, *, content: bytes, remote_path: str, cwd: str | None = None) -> None: ... - - async def download(self, *, remote_path: str, cwd: str | None = None) -> bytes: ... - - class RuntimeLease(Protocol): """Invocation-local data-plane access to one persistent Binding.""" @@ -82,9 +37,6 @@ class RuntimeLease(Protocol): @property def commands(self) -> ShellCommandProtocol: ... - @property - def files(self) -> FileSystem: ... - @dataclass(frozen=True, slots=True) class ExecutionBindingCreateSpec: @@ -203,14 +155,9 @@ __all__ = [ "ExecutionBindingBackend", "ExecutionBindingCreateSpec", "ExecutionBindingDestroySpec", - "FileSystem", "HomeSnapshotBackend", "HomeSnapshotCreateSpec", "RuntimeBackendProfile", "RuntimeLayout", "RuntimeLease", - "WorkspaceFileContent", - "WorkspaceFileEntry", - "WorkspaceListResult", - "WorkspaceReadResult", ] diff --git a/dify-agent/src/dify_agent/runtime_backend/shellctl.py b/dify-agent/src/dify_agent/runtime_backend/shellctl.py index 257157dfc82..f947ea0d9ba 100644 --- a/dify-agent/src/dify_agent/runtime_backend/shellctl.py +++ b/dify-agent/src/dify_agent/runtime_backend/shellctl.py @@ -11,10 +11,9 @@ from dify_agent.adapters.shell.shellctl import ( ShellctlClientFactory, ShellctlClientProtocol, ShellctlCommands, - ShellctlFileTransfer, create_default_shellctl_client_factory, ) -from dify_agent.runtime_backend.protocols import FileSystem, RuntimeLayout +from dify_agent.runtime_backend.protocols import RuntimeLayout _CONTROL_COMMAND_OUTPUT_LIMIT = 256 * 1024 logger = logging.getLogger(__name__) @@ -32,7 +31,6 @@ class ShellctlRuntimeLease: layout: RuntimeLayout client: ShellctlClientProtocol commands: ShellCommandProtocol - files: FileSystem owned_transport: AsyncCloseable | None = None _closed: bool = field(default=False, init=False) @@ -77,11 +75,6 @@ def create_shellctl_lease( home_dir=layout.home_dir, workspace_dir=layout.workspace_dir, ), - files=ShellctlFileTransfer( - client=client, - cwd=layout.workspace_dir, - home_dir=layout.home_dir, - ), owned_transport=owned_transport, ) diff --git a/dify-agent/src/dify_agent/server/app.py b/dify-agent/src/dify_agent/server/app.py index 058c451fbf8..567397f0cdc 100644 --- a/dify-agent/src/dify_agent/server/app.py +++ b/dify-agent/src/dify_agent/server/app.py @@ -31,9 +31,9 @@ from dify_agent.server.observability import configure_server_observability from dify_agent.server.routes.runs import create_runs_router from dify_agent.server.routes.execution_bindings import create_execution_bindings_router from dify_agent.server.routes.home_snapshots import create_home_snapshots_router -from dify_agent.server.routes.workspace_files import create_workspace_files_router +from dify_agent.server.routes.binding_files import create_binding_files_router from dify_agent.server.execution_bindings import ExecutionBindingService -from dify_agent.server.workspace_files import AgentStubWorkspaceFileUploader, WorkspaceFileService +from dify_agent.server.binding_files import BindingFileService from dify_agent.server.home_snapshots import HomeSnapshotService from dify_agent.server.settings import ServerSettings from dify_agent.storage.redis_run_store import RedisRunStore @@ -72,15 +72,11 @@ def create_app(settings: ServerSettings | None = None) -> FastAPI: agent_stub_api_base_url=resolved_settings.agent_stub_api_base_url, agent_stub_token_factory=agent_stub_token_factory, ) - workspace_file_service = ( - WorkspaceFileService( + binding_file_service = ( + BindingFileService( execution_bindings=runtime_backend_profile.execution_bindings, - upload_max_bytes=resolved_settings.sandbox_file_upload_max_bytes, - file_uploader=( - AgentStubWorkspaceFileUploader(file_request_handler=agent_stub_file_request_handler) - if agent_stub_file_request_handler is not None - else None - ), + agent_stub_api_base_url=resolved_settings.agent_stub_api_base_url, + agent_stub_token_factory=agent_stub_token_factory, ) if runtime_backend_profile is not None else None @@ -145,7 +141,7 @@ def create_app(settings: ServerSettings | None = None) -> FastAPI: ) app.include_router(create_execution_bindings_router(lambda: execution_binding_service)) app.include_router(create_home_snapshots_router(lambda: home_snapshot_service)) - app.include_router(create_workspace_files_router(lambda: workspace_file_service)) + app.include_router(create_binding_files_router(lambda: binding_file_service)) app.include_router( create_agent_stub_router( token_codec=agent_stub_token_codec, diff --git a/dify-agent/src/dify_agent/server/binding_files.py b/dify-agent/src/dify_agent/server/binding_files.py new file mode 100644 index 00000000000..acb24094eaf --- /dev/null +++ b/dify-agent/src/dify_agent/server/binding_files.py @@ -0,0 +1,347 @@ +"""Binding filesystem operations through an operation-scoped RuntimeLease.""" + +from __future__ import annotations + +import base64 +import json +import logging +import posixpath +import re +import shlex +from dataclasses import dataclass +from typing import ClassVar, Literal + +from pydantic import BaseModel, ConfigDict, ValidationError + +from dify_agent.adapters.shell.protocols import CompleteShellCommandResult, ShellProviderError +from dify_agent.agent_stub.protocol import is_canonical_dify_file_reference +from dify_agent.agent_stub.shell_env import ShellAgentStubTokenFactory, build_shell_agent_stub_env +from dify_agent.protocol import ( + BindingFileDownloadRequest, + BindingFileDownloadResponse, + BindingFileListRequest, + BindingFileListResponse, + BindingFileReadRequest, + BindingFileReadResponse, +) +from dify_agent.runtime.command_runner import execute_complete_with_commands +from dify_agent.runtime_backend import ( + BindingAcquireError, + BindingLostError, + ExecutionBindingBackend, + RuntimeLayout, + WorkspaceUnavailableError, +) +from dify_agent.runtime_backend.leases import open_runtime_lease + +logger = logging.getLogger(__name__) + +_LIST_MAX_ENTRIES = 1000 +_BROWSE_TIMEOUT_SECONDS = 60.0 +_BROWSE_OUTPUT_MAX_BYTES = 1024 * 1024 +_DOWNLOAD_TIMEOUT_SECONDS = 60.0 +_DOWNLOAD_OUTPUT_MAX_BYTES = 32 * 1024 +_PAYLOAD_BEGIN = "<<>>" +_PAYLOAD_END = "<<>>" + +_LIST_BINDING_FILES_SCRIPT = r""" +import base64 +import json +import os +import stat +import sys + +path = sys.argv[1] +response_path = sys.argv[2] +limit = int(sys.argv[3]) +directory_fd = None +try: + directory_fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + names = sorted(os.listdir(directory_fd)) + entries = [] + for name in names[:limit]: + child_stat = os.stat(name, dir_fd=directory_fd, follow_symlinks=False) + mode = child_stat.st_mode + entry_type = ( + "symlink" if stat.S_ISLNK(mode) else + "dir" if stat.S_ISDIR(mode) else + "file" if stat.S_ISREG(mode) else + "other" + ) + entries.append({ + "name": name, + "type": entry_type, + "size": int(child_stat.st_size), + "mtime": int(child_stat.st_mtime), + }) +finally: + if directory_fd is not None: + os.close(directory_fd) + +payload = {"path": response_path, "entries": entries, "truncated": len(names) > limit} +blob = base64.b64encode(json.dumps(payload, separators=(",", ":")).encode()).decode() +print("<<>>" + blob + "<<>>") +""" + +_READ_BINDING_FILE_SCRIPT = r""" +import base64 +import json +import os +import stat +import sys + +path = sys.argv[1] +response_path = sys.argv[2] +max_bytes = int(sys.argv[3]) +file_fd = None +try: + file_fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + file_stat = os.fstat(file_fd) + if not stat.S_ISREG(file_stat.st_mode): + raise FileNotFoundError(path) + size = int(file_stat.st_size) + data = os.read(file_fd, max_bytes + 1) +finally: + if file_fd is not None: + os.close(file_fd) + +truncated = len(data) > max_bytes +data = data[:max_bytes] +try: + text = data.decode("utf-8") + binary = False +except UnicodeDecodeError: + text = None + binary = True +payload = { + "path": response_path, + "size": size, + "truncated": truncated, + "binary": binary, + "text": text, +} +blob = base64.b64encode(json.dumps(payload, separators=(",", ":")).encode()).decode() +print("<<>>" + blob + "<<>>") +""" + + +class BindingFileError(Exception): + code: str + message: str + status_code: int + + def __init__(self, code: str, message: str, *, status_code: int = 400) -> None: + super().__init__(message) + self.code = code + self.message = message + self.status_code = status_code + + +class _CliUploadResult(BaseModel): + transfer_method: Literal["tool_file"] + reference: str + + model_config: ClassVar[ConfigDict] = ConfigDict(extra="ignore") + + +@dataclass(slots=True) +class BindingFileService: + execution_bindings: ExecutionBindingBackend + agent_stub_api_base_url: str | None + agent_stub_token_factory: ShellAgentStubTokenFactory | None + + async def list_files(self, request: BindingFileListRequest) -> BindingFileListResponse: + try: + async with open_runtime_lease(self.execution_bindings, request.backend_binding_ref) as lease: + resolved_path = resolve_binding_path(request.path, lease.layout) + result = await execute_complete_with_commands( + lease.commands, + _python_command( + _LIST_BINDING_FILES_SCRIPT, + resolved_path, + request.path, + str(_LIST_MAX_ENTRIES), + ), + cwd=lease.layout.workspace_dir, + env={"HOME": lease.layout.home_dir}, + timeout=_BROWSE_TIMEOUT_SECONDS, + max_output_bytes=_BROWSE_OUTPUT_MAX_BYTES, + ) + payload = _require_browse_payload(result, operation="list") + try: + return BindingFileListResponse.model_validate(payload) + except ValidationError as exc: + raise WorkspaceUnavailableError("Binding file list returned an invalid response") from exc + except BindingFileError: + raise + except Exception as exc: + raise _normalize_binding_file_error(exc) from exc + + async def read_file(self, request: BindingFileReadRequest) -> BindingFileReadResponse: + try: + async with open_runtime_lease(self.execution_bindings, request.backend_binding_ref) as lease: + resolved_path = resolve_binding_path(request.path, lease.layout) + result = await execute_complete_with_commands( + lease.commands, + _python_command( + _READ_BINDING_FILE_SCRIPT, + resolved_path, + request.path, + str(request.max_bytes), + ), + cwd=lease.layout.workspace_dir, + env={"HOME": lease.layout.home_dir}, + timeout=_BROWSE_TIMEOUT_SECONDS, + max_output_bytes=_BROWSE_OUTPUT_MAX_BYTES, + ) + payload = _require_browse_payload(result, operation="read") + try: + return BindingFileReadResponse.model_validate(payload) + except ValidationError as exc: + raise WorkspaceUnavailableError("Binding file read returned an invalid response") from exc + except BindingFileError: + raise + except Exception as exc: + raise _normalize_binding_file_error(exc) from exc + + async def download_file(self, request: BindingFileDownloadRequest) -> BindingFileDownloadResponse: + context = request.execution_context + if not context.user_id or not context.user_from: + raise BindingFileError( + "invalid_execution_context", + "Binding file download requires user_id and user_from", + status_code=400, + ) + if self.agent_stub_api_base_url is None or self.agent_stub_token_factory is None: + raise BindingFileError( + "agent_stub_upload_unavailable", + "Agent Stub file upload is not configured", + status_code=503, + ) + + try: + agent_stub_env = build_shell_agent_stub_env( + agent_stub_api_base_url=self.agent_stub_api_base_url, + execution_context=context, + token_factory=self.agent_stub_token_factory, + session_id=None, + ) + if agent_stub_env is None: + raise BindingFileError( + "agent_stub_upload_unavailable", + "Agent Stub file upload is not configured", + status_code=503, + ) + async with open_runtime_lease(self.execution_bindings, request.backend_binding_ref) as lease: + resolved_path = resolve_binding_path(request.path, lease.layout) + env = {"HOME": lease.layout.home_dir, **agent_stub_env} + try: + result = await execute_complete_with_commands( + lease.commands, + f"dify-agent file upload --no-download-link {shlex.quote(resolved_path)}", + cwd=lease.layout.workspace_dir, + env=env, + timeout=_DOWNLOAD_TIMEOUT_SECONDS, + max_output_bytes=_DOWNLOAD_OUTPUT_MAX_BYTES, + ) + except ShellProviderError as exc: + if exc.code == "timeout": + raise _download_failed() from exc + raise + if result.exit_code != 0 or not result.output_complete: + _log_download_failure(result.output, agent_stub_env) + raise _download_failed() + try: + payload = json.loads(result.output) + uploaded = _CliUploadResult.model_validate(payload) + except (json.JSONDecodeError, ValidationError, TypeError) as exc: + raise _download_failed() from exc + if not is_canonical_dify_file_reference(uploaded.reference): + raise _download_failed() + return BindingFileDownloadResponse(reference=uploaded.reference) + except BindingFileError: + raise + except Exception as exc: + normalized = _normalize_binding_file_error(exc) + if normalized.code in {"binding_not_found", "binding_unavailable"}: + raise normalized from exc + raise _download_failed() from exc + + +def resolve_binding_path(path: str, layout: RuntimeLayout) -> str: + """Resolve convenient Binding paths without adding a containment policy.""" + + if path == "~": + candidate = layout.home_dir + elif path.startswith("~/"): + candidate = posixpath.join(layout.home_dir, path[2:]) + elif posixpath.isabs(path): + candidate = path + else: + candidate = posixpath.join(layout.workspace_dir, path or ".") + return posixpath.normpath(candidate) + + +def _python_command(source: str, *args: str) -> str: + return " ".join(["python3", "-c", shlex.quote(source), *(shlex.quote(arg) for arg in args)]) + + +def _require_browse_payload(result: CompleteShellCommandResult, *, operation: str) -> dict[str, object]: + exit_code = result.exit_code + output_complete = result.output_complete + output = result.output + if exit_code != 0: + if any( + name in output + for name in ("FileNotFoundError", "NotADirectoryError", "IsADirectoryError", "PermissionError") + ): + raise BindingFileError( + "invalid_binding_path", + f"Binding file {operation} path is unavailable", + status_code=400, + ) + raise WorkspaceUnavailableError(f"Binding file {operation} command failed") + if not output_complete: + raise WorkspaceUnavailableError(f"Binding file {operation} output was incomplete") + match = re.search(re.escape(_PAYLOAD_BEGIN) + r"(.*?)" + re.escape(_PAYLOAD_END), output, flags=re.DOTALL) + if match is None: + raise WorkspaceUnavailableError(f"Binding file {operation} returned no framed payload") + try: + decoded = base64.b64decode("".join(match.group(1).split()), validate=True) + payload = json.loads(decoded) + except (ValueError, json.JSONDecodeError) as exc: + raise WorkspaceUnavailableError(f"Binding file {operation} returned an invalid payload") from exc + if not isinstance(payload, dict): + raise WorkspaceUnavailableError(f"Binding file {operation} returned a non-object payload") + return payload + + +def _normalize_binding_file_error(exc: Exception) -> BindingFileError: + if isinstance(exc, BindingFileError): + return exc + if isinstance(exc, ValueError): + return BindingFileError("invalid_binding_path", "Binding file path or payload is invalid", status_code=400) + if isinstance(exc, BindingLostError): + return BindingFileError("binding_not_found", "Execution Binding was not found", status_code=404) + if isinstance(exc, BindingAcquireError | WorkspaceUnavailableError): + return BindingFileError("binding_unavailable", "Execution Binding is unavailable", status_code=502) + return BindingFileError("binding_unavailable", "Execution Binding file operation failed", status_code=502) + + +def _download_failed() -> BindingFileError: + return BindingFileError( + "binding_file_download_failed", + "Binding file could not be converted to a ToolFile", + status_code=502, + ) + + +def _log_download_failure(output: str, env: dict[str, str]) -> None: + redacted = output + for secret in env.values(): + if len(secret) > 8: + redacted = redacted.replace(secret, "***") + logger.warning("Binding file upload command failed: %s", redacted[-1024:]) + + +__all__ = ["BindingFileError", "BindingFileService", "resolve_binding_path"] diff --git a/dify-agent/src/dify_agent/server/routes/binding_files.py b/dify-agent/src/dify_agent/server/routes/binding_files.py new file mode 100644 index 00000000000..c8ab0c2bc24 --- /dev/null +++ b/dify-agent/src/dify_agent/server/routes/binding_files.py @@ -0,0 +1,107 @@ +"""Private Binding file routes used by Dify API.""" + +from collections.abc import Callable, Coroutine +from typing import Annotated, Any, override + +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from fastapi.routing import APIRoute +from starlette.responses import Response + +from dify_agent.protocol import ( + BindingFileDownloadRequest, + BindingFileDownloadResponse, + BindingFileListRequest, + BindingFileListResponse, + BindingFileReadRequest, + BindingFileReadResponse, +) +from dify_agent.server.binding_files import BindingFileError, BindingFileService + +_INVALID_BINDING_PATH_MESSAGE = "Binding file path or payload is invalid" +_BROWSE_PATHS = frozenset( + { + "/execution-bindings/files/list", + "/execution-bindings/files/read", + } +) + + +class _BindingFileValidationRoute(APIRoute): + @override + def get_route_handler(self) -> Callable[[Request], Coroutine[Any, Any, Response]]: + route_handler = super().get_route_handler() + + async def handle(request: Request) -> Response: + try: + return await route_handler(request) + except RequestValidationError: + if self.path not in _BROWSE_PATHS: + raise + return JSONResponse( + status_code=400, + content={ + "detail": { + "code": "invalid_binding_path", + "message": _INVALID_BINDING_PATH_MESSAGE, + } + }, + ) + + return handle + + +def create_binding_files_router(get_service: Callable[[], BindingFileService | None]) -> APIRouter: + router = APIRouter( + prefix="/execution-bindings/files", + tags=["execution-bindings"], + route_class=_BindingFileValidationRoute, + ) + + def service_dep() -> BindingFileService: + service = get_service() + if service is None: + raise HTTPException( + status_code=503, + detail={"code": "runtime_backend_unavailable", "message": "Binding file service is not configured"}, + ) + return service + + def raise_http(exc: BindingFileError) -> HTTPException: + return HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": exc.message}) + + @router.post("/list", response_model=BindingFileListResponse) + async def list_files( + request: BindingFileListRequest, + service: Annotated[BindingFileService, Depends(service_dep)], + ) -> BindingFileListResponse: + try: + return await service.list_files(request) + except BindingFileError as exc: + raise raise_http(exc) from exc + + @router.post("/read", response_model=BindingFileReadResponse) + async def read_file( + request: BindingFileReadRequest, + service: Annotated[BindingFileService, Depends(service_dep)], + ) -> BindingFileReadResponse: + try: + return await service.read_file(request) + except BindingFileError as exc: + raise raise_http(exc) from exc + + @router.post("/download", response_model=BindingFileDownloadResponse) + async def download_file( + request: BindingFileDownloadRequest, + service: Annotated[BindingFileService, Depends(service_dep)], + ) -> BindingFileDownloadResponse: + try: + return await service.download_file(request) + except BindingFileError as exc: + raise raise_http(exc) from exc + + return router + + +__all__ = ["create_binding_files_router"] diff --git a/dify-agent/src/dify_agent/server/routes/workspace_files.py b/dify-agent/src/dify_agent/server/routes/workspace_files.py deleted file mode 100644 index a1eb8d794f7..00000000000 --- a/dify-agent/src/dify_agent/server/routes/workspace_files.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Private Workspace file routes used by Dify API.""" - -from collections.abc import Callable -from typing import Annotated - -from fastapi import APIRouter, Depends, HTTPException - -from dify_agent.protocol import ( - WorkspaceListRequest, - WorkspaceListResponse, - WorkspaceReadRequest, - WorkspaceReadResponse, - WorkspaceUploadRequest, - WorkspaceUploadResponse, -) -from dify_agent.server.workspace_files import WorkspaceFileError, WorkspaceFileService - - -def create_workspace_files_router(get_service: Callable[[], WorkspaceFileService | None]) -> APIRouter: - router = APIRouter(prefix="/workspace", tags=["workspace"]) - - def service_dep() -> WorkspaceFileService: - service = get_service() - if service is None: - raise HTTPException( - status_code=503, - detail={"code": "runtime_backend_unavailable", "message": "Workspace service is not configured"}, - ) - return service - - def raise_http(exc: WorkspaceFileError) -> HTTPException: - return HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": exc.message}) - - @router.post("/files/list", response_model=WorkspaceListResponse) - async def list_files( - request: WorkspaceListRequest, - service: Annotated[WorkspaceFileService, Depends(service_dep)], - ) -> WorkspaceListResponse: - try: - return await service.list_files(request) - except WorkspaceFileError as exc: - raise raise_http(exc) from exc - - @router.post("/files/read", response_model=WorkspaceReadResponse) - async def read_file( - request: WorkspaceReadRequest, - service: Annotated[WorkspaceFileService, Depends(service_dep)], - ) -> WorkspaceReadResponse: - try: - return await service.read_file(request) - except WorkspaceFileError as exc: - raise raise_http(exc) from exc - - @router.post("/files/upload", response_model=WorkspaceUploadResponse) - async def upload_file( - request: WorkspaceUploadRequest, - service: Annotated[WorkspaceFileService, Depends(service_dep)], - ) -> WorkspaceUploadResponse: - try: - return await service.upload_file(request) - except WorkspaceFileError as exc: - raise raise_http(exc) from exc - - return router - - -__all__ = ["create_workspace_files_router"] diff --git a/dify-agent/src/dify_agent/server/settings.py b/dify-agent/src/dify_agent/server/settings.py index f793044f2ff..177456dc47a 100644 --- a/dify-agent/src/dify_agent/server/settings.py +++ b/dify-agent/src/dify_agent/server/settings.py @@ -71,7 +71,6 @@ class ServerSettings(BaseSettings): ) e2b_shellctl_auth_token: str = "" e2b_shellctl_port: int = Field(default=5004, ge=1, le=65535) - sandbox_file_upload_max_bytes: int = Field(default=50 * 1024 * 1024, ge=1) agent_stub_api_base_url: str | None = Field(default=None, validation_alias="DIFY_AGENT_STUB_API_BASE_URL") sandbox_files_base_url: str | None = Field( default=None, diff --git a/dify-agent/src/dify_agent/server/workspace_files.py b/dify-agent/src/dify_agent/server/workspace_files.py deleted file mode 100644 index 6e6863fa035..00000000000 --- a/dify-agent/src/dify_agent/server/workspace_files.py +++ /dev/null @@ -1,222 +0,0 @@ -"""Workspace file access through an operation-scoped RuntimeLease. - -Paths are passed directly to the backend FileSystem. Dify Agent does not rebase -them to ``layout.workspace_dir`` or impose another containment boundary. -""" - -from __future__ import annotations - -from dataclasses import asdict, dataclass -import mimetypes -from pathlib import PurePosixPath -from typing import ClassVar, Protocol - -import httpx -from pydantic import BaseModel, ConfigDict, ValidationError - -from dify_agent.agent_stub.protocol import ( - AgentStubFileDownloadRequest, - AgentStubFileMapping, - AgentStubFileUploadRequest, -) -from dify_agent.agent_stub.server.agent_stub_files import AgentStubFileRequestError, AgentStubFileRequestHandler -from dify_agent.agent_stub.server.tokens.agent_stub import AgentStubPrincipal -from dify_agent.layers.execution_context import DifyExecutionContextLayerConfig -from dify_agent.protocol import ( - WorkspaceListRequest, - WorkspaceListResponse, - WorkspaceReadRequest, - WorkspaceReadResponse, - WorkspaceUploadRequest, - WorkspaceUploadResponse, - WorkspaceUploadedFile, -) -from dify_agent.runtime_backend import ( - BindingAcquireError, - BindingLostError, - ExecutionBindingBackend, - WorkspaceFileTooLargeError, - WorkspacePathError, - WorkspaceUnavailableError, -) -from dify_agent.runtime_backend.leases import open_runtime_lease - -_LIST_MAX_ENTRIES = 1000 -_UPLOAD_TIMEOUT_SECONDS = 30.0 - - -class _SignedUploadResponse(BaseModel): - reference: str - - model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid") - - -class WorkspaceFileError(Exception): - code: str - message: str - status_code: int - - def __init__(self, code: str, message: str, *, status_code: int = 400) -> None: - super().__init__(message) - self.code = code - self.message = message - self.status_code = status_code - - -class WorkspaceFileUploader(Protocol): - async def upload( - self, - *, - execution_context: DifyExecutionContextLayerConfig, - filename: str, - mimetype: str, - content: bytes, - ) -> WorkspaceUploadedFile: ... - - -@dataclass(slots=True) -class AgentStubWorkspaceFileUploader: - file_request_handler: AgentStubFileRequestHandler - timeout: httpx.Timeout | float = _UPLOAD_TIMEOUT_SECONDS - transport: httpx.AsyncBaseTransport | None = None - - async def upload( - self, - *, - execution_context: DifyExecutionContextLayerConfig, - filename: str, - mimetype: str, - content: bytes, - ) -> WorkspaceUploadedFile: - principal = AgentStubPrincipal( - execution_context=execution_context, - session_id=None, - scope=[], - token_id="workspace-file-upload", - ) - try: - upload_request = await self.file_request_handler.create_upload_request( - principal=principal, - request=AgentStubFileUploadRequest(filename=filename, mimetype=mimetype), - ) - async with httpx.AsyncClient( - timeout=self.timeout, - follow_redirects=True, - trust_env=False, - transport=self.transport, - ) as client: - response = await client.post( - upload_request.upload_url, - files={"file": (filename, content, mimetype)}, - ) - _ = response.raise_for_status() - payload = _SignedUploadResponse.model_validate(response.json()) - mapping = AgentStubFileMapping(transfer_method="tool_file", reference=payload.reference) - download_request = await self.file_request_handler.create_download_request( - principal=principal, - request=AgentStubFileDownloadRequest(file=mapping, for_frontend=False), - ) - return WorkspaceUploadedFile( - reference=payload.reference, - download_url=download_request.download_url, - ) - except AgentStubFileRequestError as exc: - raise WorkspaceFileError("agent_stub_upload_failed", str(exc.detail), status_code=exc.status_code) from exc - except httpx.TimeoutException as exc: - raise WorkspaceFileError( - "agent_stub_upload_failed", "signed file upload timed out", status_code=504 - ) from exc - except httpx.HTTPStatusError as exc: - raise WorkspaceFileError( - "agent_stub_upload_failed", - f"signed file upload failed with status {exc.response.status_code}", - status_code=exc.response.status_code, - ) from exc - except httpx.RequestError as exc: - raise WorkspaceFileError( - "agent_stub_upload_failed", f"signed file upload failed: {exc}", status_code=502 - ) from exc - except (ValidationError, ValueError) as exc: - raise WorkspaceFileError( - "agent_stub_upload_failed", "signed file upload returned invalid data", status_code=502 - ) from exc - - -@dataclass(slots=True) -class WorkspaceFileService: - execution_bindings: ExecutionBindingBackend - upload_max_bytes: int - file_uploader: WorkspaceFileUploader | None = None - - async def list_files(self, request: WorkspaceListRequest) -> WorkspaceListResponse: - try: - async with open_runtime_lease(self.execution_bindings, request.backend_binding_ref) as lease: - result = await lease.files.list_directory(path=request.path, limit=_LIST_MAX_ENTRIES) - return WorkspaceListResponse.model_validate( - { - "path": result.path, - "entries": [asdict(entry) for entry in result.entries], - "truncated": result.truncated, - } - ) - except Exception as exc: - raise _normalize_file_error(exc) from exc - - async def read_file(self, request: WorkspaceReadRequest) -> WorkspaceReadResponse: - try: - async with open_runtime_lease(self.execution_bindings, request.backend_binding_ref) as lease: - result = await lease.files.read_file(path=request.path, max_bytes=request.max_bytes) - return WorkspaceReadResponse( - path=result.path, - size=result.size, - truncated=result.truncated, - binary=result.binary, - text=result.text, - ) - except Exception as exc: - raise _normalize_file_error(exc) from exc - - async def upload_file(self, request: WorkspaceUploadRequest) -> WorkspaceUploadResponse: - uploader = self.file_uploader - if uploader is None: - raise WorkspaceFileError( - "agent_stub_upload_unavailable", "Agent Stub file upload is not configured", status_code=503 - ) - try: - async with open_runtime_lease(self.execution_bindings, request.backend_binding_ref) as lease: - result = await lease.files.read_bytes(path=request.path, max_bytes=self.upload_max_bytes) - filename = PurePosixPath(result.path).name or "file" - mimetype = mimetypes.guess_type(filename)[0] or "application/octet-stream" - uploaded_file = await uploader.upload( - execution_context=request.execution_context, - filename=filename, - mimetype=mimetype, - content=result.content, - ) - return WorkspaceUploadResponse(path=result.path, file=uploaded_file) - except WorkspaceFileError: - raise - except Exception as exc: - raise _normalize_file_error(exc) from exc - - -def _normalize_file_error(exc: Exception) -> WorkspaceFileError: - if isinstance(exc, WorkspaceFileError): - return exc - if isinstance(exc, WorkspacePathError): - return WorkspaceFileError("invalid_workspace_path", str(exc), status_code=400) - if isinstance(exc, WorkspaceFileTooLargeError): - return WorkspaceFileError("file_too_large", str(exc), status_code=413) - if isinstance(exc, BindingLostError): - return WorkspaceFileError("binding_not_found", str(exc), status_code=404) - if isinstance(exc, (WorkspaceUnavailableError, BindingAcquireError)): - return WorkspaceFileError("workspace_unavailable", str(exc), status_code=502) - return WorkspaceFileError("workspace_file_failed", str(exc), status_code=502) - - -__all__ = [ - "AgentStubWorkspaceFileUploader", - "WorkspaceFileError", - "WorkspaceFileService", - "WorkspaceFileUploader", -] diff --git a/dify-agent/tests/integration/dify_agent/runtime_backend/test_working_environment.py b/dify-agent/tests/integration/dify_agent/runtime_backend/test_working_environment.py index 8340beefd1a..93a67c2e816 100644 --- a/dify-agent/tests/integration/dify_agent/runtime_backend/test_working_environment.py +++ b/dify-agent/tests/integration/dify_agent/runtime_backend/test_working_environment.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +import shlex import sys import uuid @@ -20,10 +21,25 @@ from dify_agent.runtime_backend.e2b import ( E2BSDKControlPlane, ) from dify_agent.runtime_backend.local import LocalExecutionBindingBackend +from dify_agent.runtime.command_runner import execute_complete_with_commands pytestmark = pytest.mark.integration +async def _run(lease, script: str, *, cwd: str) -> str: + result = await execute_complete_with_commands( + lease.commands, + script, + cwd=cwd, + env={"HOME": lease.layout.home_dir}, + timeout=30.0, + max_output_bytes=4096, + ) + assert result.exit_code == 0 + assert result.output_complete + return result.output + + def _required_env(name: str, purpose: str) -> str: value = os.environ.get(name, "").strip() if not value: @@ -53,9 +69,7 @@ async def test_local_two_agents_share_workspace_but_not_home() -> None: allocations.append(first) first_lease = await bindings.acquire(first.binding_ref) active_leases.append(first_lease) - await first_lease.files.upload( - content=b"shared", remote_path="shared.txt", cwd=first_lease.layout.workspace_dir - ) + await _run(first_lease, "printf shared > shared.txt", cwd=first_lease.layout.workspace_dir) await bindings.release(first_lease) active_leases.remove(first_lease) @@ -72,8 +86,8 @@ async def test_local_two_agents_share_workspace_but_not_home() -> None: allocations.append(second) second_lease = await bindings.acquire(second.binding_ref) active_leases.append(second_lease) - shared = await second_lease.files.read_bytes(path="shared.txt", max_bytes=1024) - assert shared.content == b"shared" + shared = await _run(second_lease, "cat shared.txt", cwd=second_lease.layout.workspace_dir) + assert shared == "shared" assert second_lease.layout.home_dir != first_lease.layout.home_dir assert second_lease.layout.workspace_dir == first_lease.layout.workspace_dir await bindings.release(second_lease) @@ -133,9 +147,9 @@ async def test_e2b_binding_checkpoint_and_collection() -> None: ) ) lease = await bindings.acquire(allocation.binding_ref) - await lease.files.upload(content=b"e2b", remote_path="probe.txt", cwd=lease.layout.workspace_dir) - await lease.files.upload(content=b"checkpoint-home", remote_path=".checkpoint-probe", cwd=lease.layout.home_dir) - assert (await lease.files.read_bytes(path="probe.txt", max_bytes=1024)).content == b"e2b" + await _run(lease, "printf e2b > probe.txt", cwd=lease.layout.workspace_dir) + await _run(lease, "printf checkpoint-home > .checkpoint-probe", cwd=lease.layout.home_dir) + assert await _run(lease, "cat probe.txt", cwd=lease.layout.workspace_dir) == "e2b" checkpoint_ref = await snapshots.create_from_runtime( spec=HomeSnapshotCreateSpec( tenant_id="integration-tenant", @@ -158,8 +172,9 @@ async def test_e2b_binding_checkpoint_and_collection() -> None: ) ) checkpoint_lease = await bindings.acquire(checkpoint_allocation.binding_ref) - restored = await checkpoint_lease.files.read_bytes(path="~/.checkpoint-probe", max_bytes=1024) - assert restored.content == b"checkpoint-home" + checkpoint_path = shlex.quote(f"{checkpoint_lease.layout.home_dir}/.checkpoint-probe") + restored = await _run(checkpoint_lease, f"cat {checkpoint_path}", cwd=checkpoint_lease.layout.workspace_dir) + assert restored == "checkpoint-home" await bindings.release(checkpoint_lease) checkpoint_lease = None finally: diff --git a/dify-agent/tests/local/dify_agent/adapters/shell/test_shellctl.py b/dify-agent/tests/local/dify_agent/adapters/shell/test_shellctl.py index 44f897a17c0..6984d3946ad 100644 --- a/dify-agent/tests/local/dify_agent/adapters/shell/test_shellctl.py +++ b/dify-agent/tests/local/dify_agent/adapters/shell/test_shellctl.py @@ -1,43 +1,26 @@ -"""Local tests for the shellctl shell adapter and env-driven provider factory.""" +"""Local tests for the shellctl command adapter.""" from __future__ import annotations import asyncio -import base64 -from collections.abc import Callable from dataclasses import dataclass, field -import json -import os -from pathlib import Path -import signal -import subprocess -import sys from typing import cast import httpx2 as httpx import pytest from shellctl.client import ShellctlClientError -from dify_agent.adapters.shell import shellctl -from dify_agent.adapters.shell.protocols import ShellCommandResult, ShellProviderError -from dify_agent.adapters.shell.shellctl import ( - ShellctlClientProtocol, - ShellctlCommands, - ShellFileTransferError, - ShellctlFileTransfer, -) -from dify_agent.runtime_backend.errors import WorkspaceFileTooLargeError - -_WORKSPACE_SCRIPT_TIMEOUT_SECONDS = 5.0 +from dify_agent.adapters.shell.protocols import ShellProviderError +from dify_agent.adapters.shell.shellctl import ShellctlClientProtocol, ShellctlCommands @dataclass(slots=True) class _Job: - job_id: str - status: str = "running" + job_id: str = "job-1" + status: str = "exited" done: bool = True - output: str = "" - offset: int = 0 + output: str = "ok" + offset: int = 2 truncated: bool = False exit_code: int | None = 0 output_path: str | None = "/tmp/output.log" @@ -45,664 +28,115 @@ class _Job: @dataclass(slots=True) class _Status: - job_id: str + job_id: str = "job-1" status: str = "terminated" done: bool = True - offset: int = 0 + offset: int = 2 exit_code: int | None = 130 @dataclass(slots=True) -class _RunCall: - script: str - cwd: str | None - env: dict[str, str] | None - timeout: float - - -type _RunHandler = Callable[[str, str | None, dict[str, str] | None, float], _Job] -type _WaitHandler = Callable[[str, int, float], _Job] -type _InputHandler = Callable[[str, str, int, float], _Job] -type _TerminateHandler = Callable[[str, float], _Status] - - -@dataclass(slots=True) -class FakeShellctlClient: - run_handler: _RunHandler | None = None - wait_handler: _WaitHandler | None = None - input_handler: _InputHandler | None = None - tail_handler: Callable[[str], _Job] | None = None - terminate_handler: _TerminateHandler | None = None - run_calls: list[_RunCall] = field(default_factory=list) +class _Client: + run_result: object = field(default_factory=_Job) + delete_error: Exception | None = None + run_calls: list[tuple[str, str | None, dict[str, str] | None, float]] = field(default_factory=list) wait_calls: list[tuple[str, int, float]] = field(default_factory=list) - input_calls: list[tuple[str, str, int, float]] = field(default_factory=list) - terminate_calls: list[tuple[str, float]] = field(default_factory=list) delete_calls: list[tuple[str, bool, float | None]] = field(default_factory=list) - closed: bool = False - async def run( - self, - script: str, - *, - cwd: str | None = None, - env: dict[str, str] | None = None, - timeout: float = 30.0, - ) -> _Job: - self.run_calls.append(_RunCall(script=script, cwd=cwd, env=env, timeout=timeout)) - if self.run_handler is not None: - return self.run_handler(script, cwd, env, timeout) - return _Job(job_id="job", status="exited", done=True, exit_code=0) + async def run(self, script: str, *, cwd=None, env=None, timeout=30.0): + self.run_calls.append((script, cwd, env, timeout)) + if isinstance(self.run_result, Exception): + raise self.run_result + return self.run_result - async def wait(self, job_id: str, *, offset: int, timeout: float = 30.0) -> _Job: + async def wait(self, job_id: str, *, offset: int, timeout=30.0): self.wait_calls.append((job_id, offset, timeout)) - if self.wait_handler is not None: - return self.wait_handler(job_id, offset, timeout) - return _Job(job_id=job_id, status="exited", done=True, offset=offset, exit_code=0) + return _Job(job_id=job_id) - async def input( - self, - job_id: str, - text: str, - *, - offset: int, - timeout: float = 30.0, - ) -> _Job: - self.input_calls.append((job_id, text, offset, timeout)) - if self.input_handler is not None: - return self.input_handler(job_id, text, offset, timeout) - return _Job(job_id=job_id, status="exited", done=True, offset=offset, exit_code=0) + async def input(self, job_id: str, text: str, *, offset: int, timeout=30.0): + return _Job(job_id=job_id) - async def tail(self, job_id: str) -> _Job: - if self.tail_handler is not None: - return self.tail_handler(job_id) - return _Job(job_id=job_id, status="exited", done=True, output="", exit_code=0) + async def tail(self, job_id: str): + return _Job(job_id=job_id) - async def terminate(self, job_id: str, grace_seconds: float = 10.0) -> _Status: - self.terminate_calls.append((job_id, grace_seconds)) - if self.terminate_handler is not None: - return self.terminate_handler(job_id, grace_seconds) + async def terminate(self, job_id: str, grace_seconds=10.0): return _Status(job_id=job_id) - async def delete( - self, - job_id: str, - *, - force: bool = False, - grace_seconds: float | None = None, - ) -> None: + async def delete(self, job_id: str, *, force=False, grace_seconds=None): self.delete_calls.append((job_id, force, grace_seconds)) - return None + if self.delete_error is not None: + raise self.delete_error + return object() async def close(self) -> None: - self.closed = True + return None -def _client_protocol(client: FakeShellctlClient) -> ShellctlClientProtocol: +def _client(client: _Client) -> ShellctlClientProtocol: return cast(ShellctlClientProtocol, cast(object, client)) -def _kill_process_group(process: subprocess.Popen[str]) -> None: - try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - - -def _run_workspace_source(source: str, args: list[str]) -> dict[str, object]: - process = subprocess.Popen( - [sys.executable, "-c", source, *args], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - start_new_session=True, - ) - try: - stdout, stderr = process.communicate(timeout=_WORKSPACE_SCRIPT_TIMEOUT_SECONDS) - except subprocess.TimeoutExpired: - _kill_process_group(process) - _ = process.communicate() - pytest.fail(f"workspace script exceeded {_WORKSPACE_SCRIPT_TIMEOUT_SECONDS:g}s test timeout") - except BaseException: - _kill_process_group(process) - _ = process.communicate() - raise - assert process.returncode == 0, stderr - begin = stdout.find(shellctl._WORKSPACE_PAYLOAD_BEGIN) - end = stdout.find(shellctl._WORKSPACE_PAYLOAD_END, begin + len(shellctl._WORKSPACE_PAYLOAD_BEGIN)) - assert begin >= 0 and end >= 0 - encoded = "".join(stdout[begin + len(shellctl._WORKSPACE_PAYLOAD_BEGIN) : end].split()) - payload = json.loads(base64.b64decode(encoded, validate=True)) - assert isinstance(payload, dict) - return cast(dict[str, object], payload) - - -def _inject_workspace_checkpoint(source: str, checkpoint: str, injected_source: str) -> str: - marker = f"# DIFY_WORKSPACE_CHECKPOINT: {checkpoint}" - assert source.count(marker) == 1 - marker_index = source.index(marker) - line_start = source.rfind("\n", 0, marker_index) + 1 - indentation = source[line_start:marker_index] - assert not indentation.strip() - target = f"{indentation}{marker}" - injected = "\n".join(f"{indentation}{line}" if line else "" for line in injected_source.splitlines()) - instrumented = source.replace(target, f"{injected}\n{target}", 1) - assert instrumented != source - assert instrumented.count(marker) == 1 - return instrumented - - -def test_workspace_read_holds_open_fd_across_concurrent_symlink_swap(tmp_path: Path) -> None: - workspace = tmp_path / "workspace" - reports = workspace / "reports" - reports.mkdir(parents=True) - source = reports / "result.pdf" - _ = source.write_text("workspace-content") - outside = tmp_path / "outside.pdf" - _ = outside.write_text("outside-content") - instrumented = _inject_workspace_checkpoint( - shellctl._READ_WORKSPACE_SCRIPT, - "file_opened", - "os.unlink(sys.argv[3])\nos.symlink(sys.argv[4], sys.argv[3])", - ) - - payload = _run_workspace_source( - instrumented, - [str(source), "1024", str(source), str(outside)], - ) - - assert payload["text"] == "workspace-content" - assert source.is_symlink() - - -def test_workspace_read_bytes_holds_open_fd_across_same_uid_symlink_swap(tmp_path: Path) -> None: - workspace = tmp_path / "workspace" - reports = workspace / "reports" - reports.mkdir(parents=True) - source = reports / "result.pdf" - source.write_bytes(b"workspace-content\x00") - outside = tmp_path / "outside.pdf" - outside.write_bytes(b"outside-content") - instrumented = _inject_workspace_checkpoint( - shellctl._READ_WORKSPACE_BYTES_SCRIPT, - "file_opened", - "os.unlink(sys.argv[3])\nos.symlink(sys.argv[4], sys.argv[3])", - ) - - payload = _run_workspace_source( - instrumented, - [str(source), "1024", str(source), str(outside)], - ) - - encoded = payload["content_base64"] - assert isinstance(encoded, str) - assert base64.b64decode(encoded) == b"workspace-content\x00" - assert source.is_symlink() - - -def test_workspace_read_bytes_accepts_file_at_size_limit(tmp_path: Path) -> None: - workspace = tmp_path / "workspace" - workspace.mkdir() - source = workspace / "result.bin" - source.write_bytes(b"12345") - - payload = _run_workspace_source( - shellctl._READ_WORKSPACE_BYTES_SCRIPT, - [str(source), "5"], - ) - - encoded = payload["content_base64"] - assert isinstance(encoded, str) - assert base64.b64decode(encoded) == b"12345" - assert payload["size"] == 5 - - -def test_workspace_read_bytes_rejects_oversize_before_remote_read(tmp_path: Path) -> None: - workspace = tmp_path / "workspace" - workspace.mkdir() - source = workspace / "result.bin" - source.write_bytes(b"123456") - instrumented = _inject_workspace_checkpoint( - shellctl._READ_WORKSPACE_BYTES_SCRIPT, - "arguments_loaded", - "os.read = lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError('must not read'))\n\n", - ) - - payload = _run_workspace_source( - instrumented, - [str(source), "5"], - ) - - assert payload == {"path": str(source), "size": 6, "too_large": True} - - -def test_workspace_read_bytes_caps_capture_when_file_grows_after_fstat(tmp_path: Path) -> None: - workspace = tmp_path / "workspace" - workspace.mkdir() - source = workspace / "result.bin" - source.write_bytes(b"12345") - instrumented = _inject_workspace_checkpoint( - shellctl._READ_WORKSPACE_BYTES_SCRIPT, - "arguments_loaded", - "real_read = os.read\n" - "captured_bytes = 0\n" - "def bounded_read(fd, count):\n" - " global captured_bytes\n" - " data = real_read(fd, count)\n" - " captured_bytes += len(data)\n" - " assert captured_bytes <= int(sys.argv[2]) + 1\n" - " return data\n" - "os.read = bounded_read", - ) - instrumented = _inject_workspace_checkpoint( - instrumented, - "file_size_captured", - "with open(sys.argv[3], 'ab') as growing:\n growing.write(b'x' * 10000)", - ) - - payload = _run_workspace_source( - instrumented, - [str(source), "5", str(source)], - ) - - assert payload == {"path": str(source), "size": 6, "too_large": True} - - -def test_workspace_list_holds_open_directory_fd_across_concurrent_symlink_swap(tmp_path: Path) -> None: - workspace = tmp_path / "workspace" - reports = workspace / "reports" - reports.mkdir(parents=True) - _ = (reports / "safe.txt").write_text("safe") - outside = tmp_path / "outside" - outside.mkdir() - _ = (outside / "secret.txt").write_text("secret") - moved = workspace / "opened-reports" - instrumented = _inject_workspace_checkpoint( - shellctl._LIST_WORKSPACE_SCRIPT, - "directory_opened", - "os.rename(sys.argv[3], sys.argv[4])\nos.symlink(sys.argv[5], sys.argv[3])", - ) - - payload = _run_workspace_source( - instrumented, - [str(reports), "100", str(reports), str(moved), str(outside)], - ) - - entries = payload["entries"] - assert isinstance(entries, list) - assert [entry["name"] for entry in entries if isinstance(entry, dict)] == ["safe.txt"] - - -def test_commands_forward_parameters_and_map_metadata() -> None: - client = FakeShellctlClient( - run_handler=lambda script, cwd, env, timeout: _Job( - job_id="run-job", - status="running", - done=False, - output="abc", - offset=3, - truncated=True, - exit_code=None, - output_path="/tmp/run.log", - ), - wait_handler=lambda job_id, offset, timeout: _Job( - job_id=job_id, - status="running", - done=False, - output="def", - offset=6, - truncated=False, - exit_code=None, - output_path="/tmp/run.log", - ), - input_handler=lambda job_id, text, offset, timeout: _Job( - job_id=job_id, - status="exited", - done=True, - output="ghi", - offset=9, - truncated=False, - exit_code=0, - output_path="/tmp/run.log", - ), - tail_handler=lambda job_id: _Job( - job_id=job_id, - status="exited", - done=True, - output="tail", - offset=11, - truncated=False, - exit_code=0, - output_path="/tmp/tail.log", - ), - terminate_handler=lambda job_id, grace_seconds: _Status( - job_id=job_id, - status="terminated", - done=True, - offset=12, - exit_code=130, - ), - ) +def test_commands_apply_runtime_layout_and_home_environment() -> None: + client = _Client() async def scenario() -> None: - commands = ShellctlCommands(_client_protocol(client)) - run_result = await commands.run("pwd", cwd="~/workspace/abc12ff", env={"FOO": "bar"}, timeout=2.5) - wait_result = await commands.wait("run-job", offset=3, timeout=4.0) - read_result = await commands.read_output("run-job", offset=6) - input_result = await commands.input("run-job", "ls\n", offset=6, timeout=5.0) - interrupt_result = await commands.interrupt("run-job", grace_seconds=1.5) - tail_result = await commands.tail("run-job") - await commands.delete("run-job", force=True, grace_seconds=2.0) - - assert run_result == ShellCommandResult( - job_id="run-job", - status="running", - done=False, - exit_code=None, - output="abc", - offset=3, - truncated=True, - output_path="/tmp/run.log", - ) - assert wait_result.offset == 6 - assert read_result.offset == 6 - assert input_result.exit_code == 0 - assert interrupt_result.status == "terminated" - assert tail_result.output_path == "/tmp/tail.log" + commands = ShellctlCommands(_client(client), home_dir="/home/binding", workspace_dir="/workspace") + result = await commands.run("pwd", cwd="reports", env={"TOKEN": "value"}, timeout=2.5) + assert result.output == "ok" asyncio.run(scenario()) - - assert client.run_calls == [_RunCall(script="pwd", cwd="~/workspace/abc12ff", env={"FOO": "bar"}, timeout=2.5)] - assert client.wait_calls == [ - ("run-job", 3, 4.0), - ("run-job", 6, 0.0), - ] - assert client.input_calls == [("run-job", "ls\n", 6, 5.0)] - assert client.terminate_calls == [("run-job", 1.5)] - assert client.delete_calls == [("run-job", True, 2.0)] + assert client.run_calls == [("pwd", "/workspace/reports", {"TOKEN": "value", "HOME": "/home/binding"}, 2.5)] -def test_commands_enforce_runtime_lease_home_and_cwd_namespace() -> None: - client = FakeShellctlClient() - +def test_commands_reject_cwd_outside_runtime_layout() -> None: async def scenario() -> None: - commands = ShellctlCommands( - _client_protocol(client), - home_dir="/homes/binding-b", - workspace_dir="/workspaces/shared", - ) - await commands.run("pwd", env={"HOME": "/homes/binding-a", "FOO": "bar"}, timeout=2.5) - await commands.run("pwd", cwd="~/project", timeout=2.5) + commands = ShellctlCommands(_client(_Client()), home_dir="/home/binding", workspace_dir="/workspace") with pytest.raises(ValueError, match="outside this RuntimeLease"): - await commands.run("cat secret", cwd="/homes/binding-a", timeout=2.5) + await commands.run("pwd", cwd="/var/private", timeout=2.5) asyncio.run(scenario()) - assert client.run_calls == [ - _RunCall( - script="pwd", - cwd="/workspaces/shared", - env={"HOME": "/homes/binding-b", "FOO": "bar"}, - timeout=2.5, - ), - _RunCall( - script="pwd", - cwd="/homes/binding-b/project", - env={"HOME": "/homes/binding-b"}, - timeout=2.5, - ), - ] + +def test_read_output_uses_nonblocking_wait() -> None: + client = _Client() + + async def scenario() -> None: + commands = ShellctlCommands(_client(client)) + result = await commands.read_output("job-1", offset=7) + assert result.job_id == "job-1" + + asyncio.run(scenario()) + assert client.wait_calls == [("job-1", 7, 0.0)] -def test_commands_map_http_timeout_to_shell_provider_error() -> None: +def test_commands_map_http_and_structured_errors() -> None: request = httpx.Request("POST", "http://shellctl.example/v1/jobs") - client = FakeShellctlClient( - run_handler=lambda script, cwd, env, timeout: (_ for _ in ()).throw( - httpx.ReadTimeout("timed out", request=request) + + async def scenario() -> None: + timeout_commands = ShellctlCommands( + _client(_Client(run_result=httpx.ReadTimeout("timed out", request=request))) ) - ) + with pytest.raises(ShellProviderError) as timeout_error: + await timeout_commands.run("pwd", timeout=2.5) + assert timeout_error.value.code == "timeout" - async def scenario() -> None: - commands = ShellctlCommands(_client_protocol(client)) - with pytest.raises(ShellProviderError, match="timed out") as exc_info: - await commands.run("pwd", timeout=2.5) - assert exc_info.value.code == "timeout" - - asyncio.run(scenario()) - - -def test_commands_map_http_request_error_to_shell_provider_error() -> None: - request = httpx.Request("POST", "http://shellctl.example/v1/jobs/run") - client = FakeShellctlClient( - wait_handler=lambda job_id, offset, timeout: (_ for _ in ()).throw( - httpx.ConnectError("connection failed", request=request) + missing_commands = ShellctlCommands( + _client(_Client(run_result=ShellctlClientError(404, "sandbox_not_found", "expired"))) ) - ) - - async def scenario() -> None: - commands = ShellctlCommands(_client_protocol(client)) - with pytest.raises(ShellProviderError, match="connection failed") as exc_info: - await commands.wait("run-job", offset=3, timeout=4.0) - assert exc_info.value.code == "request_error" + with pytest.raises(ShellProviderError) as missing_error: + await missing_commands.run("pwd", timeout=2.5) + assert missing_error.value.code == "sandbox_not_found" + assert missing_error.value.status_code == 404 asyncio.run(scenario()) -def test_commands_preserve_shellctl_structured_error_fields() -> None: - client = FakeShellctlClient( - run_handler=lambda script, cwd, env, timeout: (_ for _ in ()).throw( - ShellctlClientError(404, "sandbox_not_found", "sandbox expired") - ) - ) +def test_delete_treats_missing_job_as_already_deleted() -> None: + client = _Client(delete_error=ShellctlClientError(404, "job_not_found", "missing")) async def scenario() -> None: - commands = ShellctlCommands(_client_protocol(client)) - with pytest.raises(ShellProviderError, match="sandbox expired") as exc_info: - await commands.run("pwd", timeout=2.5) - assert exc_info.value.status_code == 404 - assert exc_info.value.code == "sandbox_not_found" - - asyncio.run(scenario()) - - -def test_read_bytes_maps_oversize_payload_to_domain_error() -> None: - payload = base64.b64encode(b'{"path":"reports/large.bin","size":6,"too_large":true}').decode("ascii") - client = FakeShellctlClient( - run_handler=lambda script, cwd, env, timeout: _Job( - job_id="read-job", - status="exited", - done=True, - exit_code=0, - output=f"{shellctl._WORKSPACE_PAYLOAD_BEGIN}{payload}{shellctl._WORKSPACE_PAYLOAD_END}", - ) - ) - - async def scenario() -> None: - files = ShellctlFileTransfer(_client_protocol(client), cwd="/workspace", home_dir="/home/dify") - with pytest.raises(WorkspaceFileTooLargeError) as exc_info: - await files.read_bytes(path="reports/large.bin", max_bytes=5) - assert exc_info.value.size == 6 - assert exc_info.value.max_bytes == 5 - - asyncio.run(scenario()) - - -def test_file_operations_use_runtime_lease_namespace() -> None: - payload = base64.b64encode(b'{"path":"/homes/binding-b/report.txt","size":2,"content_base64":"b2s="}').decode( - "ascii" - ) - client = FakeShellctlClient( - run_handler=lambda script, cwd, env, timeout: _Job( - job_id="read-job", - status="exited", - done=True, - exit_code=0, - output=f"{shellctl._WORKSPACE_PAYLOAD_BEGIN}{payload}{shellctl._WORKSPACE_PAYLOAD_END}", - ) - ) - - async def scenario() -> None: - files = ShellctlFileTransfer( - _client_protocol(client), - cwd="/workspaces/shared", - home_dir="/homes/binding-b", - ) - result = await files.read_bytes(path="~/report.txt", max_bytes=10) - assert result.content == b"ok" - with pytest.raises(ValueError, match="outside this RuntimeLease"): - await files.download(remote_path="secret", cwd="/homes/binding-a") - - asyncio.run(scenario()) - - assert client.run_calls[0].cwd == "/workspaces/shared" - assert client.run_calls[0].env == {"HOME": "/homes/binding-b"} - - -def test_delete_maps_http_timeout_to_shell_provider_error() -> None: - request = httpx.Request("DELETE", "http://shellctl.example/v1/jobs/run-job") - - @dataclass(slots=True) - class DeleteTimeoutClient(FakeShellctlClient): - async def delete(self, job_id, *, force=False, grace_seconds=None): - self.delete_calls.append((job_id, force, grace_seconds)) - raise httpx.ReadTimeout("delete timed out", request=request) - - client = DeleteTimeoutClient() - - async def scenario() -> None: - commands = ShellctlCommands(_client_protocol(client)) - with pytest.raises(ShellProviderError, match="delete timed out") as exc_info: - await commands.delete("run-job", force=True, grace_seconds=2.0) - assert exc_info.value.code == "timeout" - - asyncio.run(scenario()) - assert client.delete_calls == [("run-job", True, 2.0)] - - -def test_delete_maps_http_request_error_to_shell_provider_error() -> None: - request = httpx.Request("DELETE", "http://shellctl.example/v1/jobs/run-job") - - @dataclass(slots=True) - class DeleteRequestErrorClient(FakeShellctlClient): - async def delete(self, job_id, *, force=False, grace_seconds=None): - self.delete_calls.append((job_id, force, grace_seconds)) - raise httpx.ConnectError("delete connection failed", request=request) - - client = DeleteRequestErrorClient() - - async def scenario() -> None: - commands = ShellctlCommands(_client_protocol(client)) - with pytest.raises(ShellProviderError, match="delete connection failed") as exc_info: - await commands.delete("run-job", force=True, grace_seconds=2.0) - assert exc_info.value.code == "request_error" - - asyncio.run(scenario()) - assert client.delete_calls == [("run-job", True, 2.0)] - - -def test_files_upload_and_download_still_work() -> None: - content = b"hello \x00 world" - encoded = base64.b64encode(content).decode("ascii") - client = FakeShellctlClient( - run_handler=lambda script, cwd, env, timeout: ( - _Job(job_id="ul-job", status="exited", done=True, exit_code=0) - if "base64 -d" in script - else _Job( - job_id="dl-job", - status="exited", - done=True, - exit_code=0, - output=f"noise{shellctl._TRANSFER_BEGIN}{encoded}{shellctl._TRANSFER_END}tail", - ) - ) - ) - - async def scenario() -> None: - files = ShellctlFileTransfer(_client_protocol(client)) - await files.upload(content=content, remote_path="out.bin", cwd="~/workspace/abc12ff") - downloaded = await files.download(remote_path="report.txt", cwd="~/workspace/abc12ff") - assert downloaded == content - - asyncio.run(scenario()) - - -def test_file_transfer_timeout_is_an_end_to_end_budget(monkeypatch: pytest.MonkeyPatch) -> None: - clock = {"value": 100.0} - - def fake_monotonic() -> float: - return clock["value"] - - monkeypatch.setattr(shellctl.time, "monotonic", fake_monotonic) - - def run_handler(script: str, cwd: str | None, env: dict[str, str] | None, timeout: float) -> _Job: - del script, cwd, env - assert timeout == pytest.approx(5.0, rel=0, abs=0.01) - clock["value"] = 103.5 - return _Job(job_id="upload-job", status="running", done=False, output="part-1", offset=6, exit_code=None) - - def wait_handler(job_id: str, offset: int, timeout: float) -> _Job: - assert job_id == "upload-job" - assert offset == 6 - assert timeout == pytest.approx(1.5, rel=0, abs=0.01) - return _Job(job_id=job_id, status="exited", done=True, output="part-2", offset=12, exit_code=0) - - client = FakeShellctlClient(run_handler=run_handler, wait_handler=wait_handler) - - async def scenario() -> None: - transfer = shellctl.ShellctlFileTransfer( - client=_client_protocol(client), - timeout=5.0, - ) - await transfer.upload(content=b"payload", remote_path="out.bin") - - asyncio.run(scenario()) - assert client.delete_calls == [("upload-job", True, None)] - - -def test_file_transfer_timeout_exhaustion_raises_timeout_and_still_deletes_job( - monkeypatch: pytest.MonkeyPatch, -) -> None: - clock = {"value": 100.0} - - def fake_monotonic() -> float: - return clock["value"] - - monkeypatch.setattr(shellctl.time, "monotonic", fake_monotonic) - - def run_handler(script: str, cwd: str | None, env: dict[str, str] | None, timeout: float) -> _Job: - del script, cwd, env - assert timeout == pytest.approx(5.0, rel=0, abs=0.01) - clock["value"] = 106.0 - return _Job(job_id="upload-job", status="running", done=False, output="part-1", offset=6, exit_code=None) - - client = FakeShellctlClient(run_handler=run_handler) - - async def scenario() -> None: - transfer = shellctl.ShellctlFileTransfer( - client=_client_protocol(client), - timeout=5.0, - ) - with pytest.raises(ShellProviderError, match="timed out") as exc_info: - await transfer.upload(content=b"payload", remote_path="out.bin") - assert exc_info.value.code == "timeout" - - asyncio.run(scenario()) - assert client.delete_calls == [("upload-job", True, None)] - - -def test_download_missing_file_raises() -> None: - client = FakeShellctlClient( - run_handler=lambda script, cwd, env, timeout: _Job( - job_id="dl-job", - status="exited", - done=True, - output="", - exit_code=shellctl._DOWNLOAD_MISSING_EXIT_CODE, - ) - ) - - async def scenario() -> None: - files = ShellctlFileTransfer(_client_protocol(client)) - with pytest.raises(ShellFileTransferError, match="not found"): - await files.download(remote_path="missing.txt") + commands = ShellctlCommands(_client(client)) + await commands.delete("job-1", force=True) asyncio.run(scenario()) + assert client.delete_calls == [("job-1", True, None)] diff --git a/dify-agent/tests/local/dify_agent/agent_stub/server/test_agent_stub_files.py b/dify-agent/tests/local/dify_agent/agent_stub/server/test_agent_stub_files.py index 6fae0a6d0dd..2f204ac2b71 100644 --- a/dify-agent/tests/local/dify_agent/agent_stub/server/test_agent_stub_files.py +++ b/dify-agent/tests/local/dify_agent/agent_stub/server/test_agent_stub_files.py @@ -62,6 +62,7 @@ def test_upload_request_uses_agent_inner_endpoint_and_binds_sandbox_base(monkeyp assert json.loads(request.content) == { "tenant_id": "tenant-1", "user_id": "user-1", + "user_from": "account", "filename": "report.pdf", "mimetype": "application/pdf", "conversation_id": "conversation-1", diff --git a/dify-agent/tests/local/dify_agent/client/test_client.py b/dify-agent/tests/local/dify_agent/client/test_client.py index d87c192d783..e08c39d9009 100644 --- a/dify-agent/tests/local/dify_agent/client/test_client.py +++ b/dify-agent/tests/local/dify_agent/client/test_client.py @@ -22,6 +22,10 @@ from dify_agent.client import ( DifyAgentValidationError, ) from dify_agent.protocol import ( + BindingFileDownloadRequest, + BindingFileDownloadResponse, + BindingFileListResponse, + BindingFileReadResponse, CancelRunRequest, CancelRunResponse, CreateExecutionBindingRequest, @@ -37,10 +41,6 @@ from dify_agent.protocol import ( RunStartedEvent, RunSucceededEvent, RunSucceededEventData, - WorkspaceListResponse, - WorkspaceReadResponse, - WorkspaceUploadRequest, - WorkspaceUploadResponse, ) @@ -83,12 +83,13 @@ def _run_status_json(status: str) -> dict[str, object]: return {"run_id": "run-1", "status": status, "created_at": now, "updated_at": now, "error": None} -def _workspace_upload_request(path: str = "report.txt") -> WorkspaceUploadRequest: - return WorkspaceUploadRequest( +def _binding_file_download_request(path: str = "report.txt") -> BindingFileDownloadRequest: + return BindingFileDownloadRequest( backend_binding_ref="binding-ref", path=path, execution_context=DifyExecutionContextLayerConfig( tenant_id="tenant-1", + user_id="account-1", user_from="account", agent_mode="agent_app", invoke_from="debugger", @@ -96,6 +97,11 @@ def _workspace_upload_request(path: str = "report.txt") -> WorkspaceUploadReques ) +def _assert_binding_download_timeout(request: httpx.Request) -> None: + timeout = cast(dict[str, float], request.extensions["timeout"]) + assert timeout == {"connect": 90.0, "read": 90.0, "write": 90.0, "pool": 90.0} + + def _function_tool_result_payload(key: str) -> dict[str, object]: return { "type": "pydantic_ai_event", @@ -239,85 +245,70 @@ def test_async_methods_and_wait_run_parse_protocol_dtos() -> None: asyncio.run(scenario()) -def test_sync_workspace_methods_post_dtos_and_parse_responses() -> None: +def test_sync_binding_file_methods_post_dtos_and_parse_responses() -> None: def handler(request: httpx.Request) -> httpx.Response: - if request.url.path == "/workspace/files/list": + if request.url.path == "/execution-bindings/files/list": payload = cast(dict[str, object], json.loads(request.content)) assert payload["path"] == "." assert payload["backend_binding_ref"] == "binding-ref" return httpx.Response(200, json={"path": ".", "entries": [], "truncated": False}) - if request.url.path == "/workspace/files/read": + if request.url.path == "/execution-bindings/files/read": payload = cast(dict[str, object], json.loads(request.content)) assert payload["path"] == "note.txt" assert payload["max_bytes"] == 128 return httpx.Response( 200, json={"path": "note.txt", "size": 5, "truncated": False, "binary": False, "text": "hello"} ) - if request.url.path == "/workspace/files/upload": + if request.url.path == "/execution-bindings/files/download": + _assert_binding_download_timeout(request) payload = cast(dict[str, object], json.loads(request.content)) assert payload["path"] == "report.txt" return httpx.Response( 200, - json={ - "path": "report.txt", - "file": { - "transfer_method": "tool_file", - "reference": "dify-file-ref:file-1", - "download_url": "https://files.example.com/report.txt", - }, - }, + json={"reference": "dify-file-ref:file-1"}, ) raise AssertionError(f"unexpected request: {request.method} {request.url}") client = Client(base_url="http://testserver", sync_http_client=httpx.Client(transport=httpx.MockTransport(handler))) - listing = client.list_workspace_files_sync("binding-ref", ".") - preview = client.read_workspace_file_sync("binding-ref", "note.txt", max_bytes=128) - uploaded = client.upload_workspace_file_sync(_workspace_upload_request()) + listing = client.list_binding_files_sync("binding-ref", ".") + preview = client.read_binding_file_sync("binding-ref", "note.txt", max_bytes=128) + downloaded = client.download_binding_file_sync(_binding_file_download_request()) - assert isinstance(listing, WorkspaceListResponse) + assert isinstance(listing, BindingFileListResponse) assert listing.path == "." - assert isinstance(preview, WorkspaceReadResponse) + assert isinstance(preview, BindingFileReadResponse) assert preview.text == "hello" - assert isinstance(uploaded, WorkspaceUploadResponse) - assert uploaded.file.reference == "dify-file-ref:file-1" - assert uploaded.file.download_url == "https://files.example.com/report.txt" + assert isinstance(downloaded, BindingFileDownloadResponse) + assert downloaded.reference == "dify-file-ref:file-1" -def test_async_workspace_methods_post_dtos_and_parse_responses() -> None: +def test_async_binding_file_methods_post_dtos_and_parse_responses() -> None: def handler(request: httpx.Request) -> httpx.Response: - if request.url.path == "/workspace/files/list": + if request.url.path == "/execution-bindings/files/list": return httpx.Response(200, json={"path": ".", "entries": [], "truncated": False}) - if request.url.path == "/workspace/files/read": + if request.url.path == "/execution-bindings/files/read": + payload = cast(dict[str, object], json.loads(request.content)) + assert payload["max_bytes"] == 262144 return httpx.Response( 200, json={"path": "note.txt", "size": 5, "truncated": False, "binary": False, "text": "hello"} ) - if request.url.path == "/workspace/files/upload": - return httpx.Response( - 200, - json={ - "path": "report.txt", - "file": { - "transfer_method": "tool_file", - "reference": "dify-file-ref:file-1", - "download_url": "https://files.example.com/report.txt", - }, - }, - ) + if request.url.path == "/execution-bindings/files/download": + _assert_binding_download_timeout(request) + return httpx.Response(200, json={"reference": "dify-file-ref:file-1"}) raise AssertionError(f"unexpected request: {request.method} {request.url}") async def scenario() -> None: http_client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) client = Client(base_url="http://testserver", async_http_client=http_client) - listing = await client.list_workspace_files("binding-ref", ".") - preview = await client.read_workspace_file("binding-ref", "note.txt") - uploaded = await client.upload_workspace_file(_workspace_upload_request()) + listing = await client.list_binding_files("binding-ref", ".") + preview = await client.read_binding_file("binding-ref", "note.txt") + downloaded = await client.download_binding_file(_binding_file_download_request()) assert listing.path == "." assert preview.text == "hello" - assert uploaded.file.reference == "dify-file-ref:file-1" - assert uploaded.file.download_url == "https://files.example.com/report.txt" + assert downloaded.reference == "dify-file-ref:file-1" await http_client.aclose() asyncio.run(scenario()) @@ -445,28 +436,22 @@ def test_home_snapshot_client_maps_sync_validation_and_async_http_errors() -> No asyncio.run(scenario()) -def test_sync_upload_workspace_file_rejects_missing_download_url() -> None: +def test_sync_download_binding_file_rejects_missing_reference() -> None: def handler(request: httpx.Request) -> httpx.Response: - if request.url.path != "/workspace/files/upload": + if request.url.path != "/execution-bindings/files/download": raise AssertionError(f"unexpected request: {request.method} {request.url}") return httpx.Response( 200, - json={ - "path": "report.txt", - "file": { - "transfer_method": "tool_file", - "reference": "dify-file-ref:file-1", - }, - }, + json={}, ) client = Client(base_url="http://testserver", sync_http_client=httpx.Client(transport=httpx.MockTransport(handler))) with pytest.raises(DifyAgentValidationError): - _ = client.upload_workspace_file_sync(_workspace_upload_request()) + _ = client.download_binding_file_sync(_binding_file_download_request()) -def test_sync_workspace_methods_map_invalid_json_to_validation_error() -> None: +def test_sync_binding_file_methods_map_invalid_json_to_validation_error() -> None: responses = iter([httpx.Response(200, text="not-json"), httpx.Response(404, json={"detail": "missing"})]) def handler(_request: httpx.Request) -> httpx.Response: @@ -475,10 +460,10 @@ def test_sync_workspace_methods_map_invalid_json_to_validation_error() -> None: client = Client(base_url="http://testserver", sync_http_client=httpx.Client(transport=httpx.MockTransport(handler))) with pytest.raises(DifyAgentValidationError): - _ = client.list_workspace_files_sync("binding-ref", ".") + _ = client.list_binding_files_sync("binding-ref", ".") with pytest.raises(DifyAgentHTTPError) as http_error: - _ = client.read_workspace_file_sync("binding-ref", "missing.txt") + _ = client.read_binding_file_sync("binding-ref", "missing.txt") assert http_error.value.status_code == 404 diff --git a/dify-agent/tests/local/dify_agent/layers/shell/test_layer.py b/dify-agent/tests/local/dify_agent/layers/shell/test_layer.py index 626b377c423..d8b07395774 100644 --- a/dify-agent/tests/local/dify_agent/layers/shell/test_layer.py +++ b/dify-agent/tests/local/dify_agent/layers/shell/test_layer.py @@ -9,6 +9,7 @@ from typing import cast import pytest import dify_agent.layers.shell.layer as shell_layer_module +import dify_agent.runtime.command_runner as command_runner_module from dify_agent.layers.shell import ( DIFY_SHELL_LAYER_TYPE_ID, DifyShellCliToolConfig, @@ -25,7 +26,6 @@ from dify_agent.layers.shell.layer import ( from dify_agent.adapters.shell.protocols import ( ShellCommandResult, ShellCommandStatus, - ShellFileTransferProtocol, ShellProviderError, ) from dify_agent.layers.execution_context import DifyExecutionContextLayerConfig @@ -36,9 +36,6 @@ from dify_agent.runtime_backend import ( ExecutionBindingBackend, RuntimeLayout, RuntimeLease, - WorkspaceFileContent, - WorkspaceListResult, - WorkspaceReadResult, ) @@ -155,24 +152,6 @@ class _UnexpectedToolError(Exception): pass -class FakeFiles(ShellFileTransferProtocol): - async def list_directory(self, *, path: str, limit: int) -> WorkspaceListResult: - raise AssertionError("resource.files should not be used by shell layer logic") - - async def read_file(self, *, path: str, max_bytes: int) -> WorkspaceReadResult: - raise AssertionError("resource.files should not be used by shell layer logic") - - async def read_bytes(self, *, path: str, max_bytes: int) -> WorkspaceFileContent: - del max_bytes - raise AssertionError("resource.files should not be used by shell layer logic") - - async def upload(self, *, content: bytes, remote_path: str, cwd: str | None = None) -> None: - raise AssertionError("resource.files should not be used by production shell layer logic") - - async def download(self, *, remote_path: str, cwd: str | None = None) -> bytes: - raise AssertionError("resource.files should not be used by production shell layer logic") - - @dataclass(slots=True) class FakeCommands: run_handler: Callable[[str, str | None, Mapping[str, str] | None, float], ShellCommandResult] | None = None @@ -233,7 +212,6 @@ class FakeCommands: @dataclass(slots=True) class FakeResource: commands: FakeCommands - files: FakeFiles = field(default_factory=FakeFiles) handle: str = "sandbox-1" layout: RuntimeLayout = field( default_factory=lambda: RuntimeLayout( @@ -1099,7 +1077,7 @@ def test_run_remote_script_complete_returns_incomplete_reason_for_timeout( def fake_monotonic() -> float: return now - monkeypatch.setattr(shell_layer_module.time, "monotonic", fake_monotonic) + monkeypatch.setattr(command_runner_module.time, "monotonic", fake_monotonic) def run_handler(script: str, cwd: str | None, env: Mapping[str, str] | None, timeout: float) -> ShellCommandResult: nonlocal now diff --git a/dify-agent/tests/local/dify_agent/layers/test_runtime_layer.py b/dify-agent/tests/local/dify_agent/layers/test_runtime_layer.py index cdab88e90f2..de6ed027b7a 100644 --- a/dify-agent/tests/local/dify_agent/layers/test_runtime_layer.py +++ b/dify-agent/tests/local/dify_agent/layers/test_runtime_layer.py @@ -27,15 +27,17 @@ class _Backend: async def test_runtime_layer_acquires_and_releases_operation_scoped_lease() -> None: lease = cast( RuntimeLease, - type( - "Lease", - (), - { - "layout": RuntimeLayout(home_dir="/home/agent", workspace_dir="/workspace"), - "commands": object(), - "files": object(), - }, - )(), + cast( + object, + type( + "Lease", + (), + { + "layout": RuntimeLayout(home_dir="/home/agent", workspace_dir="/workspace"), + "commands": object(), + }, + )(), + ), ) backend = _Backend(lease=lease) layer = DifyRuntimeLayer.from_config_with_backend( diff --git a/dify-agent/tests/local/dify_agent/protocol/test_working_environment.py b/dify-agent/tests/local/dify_agent/protocol/test_working_environment.py index f438d30fe64..cdb217c1e29 100644 --- a/dify-agent/tests/local/dify_agent/protocol/test_working_environment.py +++ b/dify-agent/tests/local/dify_agent/protocol/test_working_environment.py @@ -2,10 +2,11 @@ import pytest from pydantic import ValidationError from dify_agent.protocol import ( + BindingFileListRequest, + BindingFileReadRequest, CreateExecutionBindingRequest, CreateHomeSnapshotFromBindingRequest, DestroyExecutionBindingRequest, - WorkspaceListRequest, ) @@ -64,7 +65,26 @@ def test_snapshot_and_file_requests_locate_binding_directly() -> None: home_snapshot_id="home-2", backend_binding_ref="binding-ref", ) - listing = WorkspaceListRequest(backend_binding_ref="binding-ref", path="~/files") + listing = BindingFileListRequest(backend_binding_ref="binding-ref", path="~/files") assert snapshot.backend_binding_ref == "binding-ref" assert listing.path == "~/files" + + +def test_binding_file_read_preview_uses_bounded_default_and_limit() -> None: + assert BindingFileReadRequest(backend_binding_ref="binding-ref", path="report.txt").max_bytes == 262144 + assert ( + BindingFileReadRequest( + backend_binding_ref="binding-ref", + path="report.txt", + max_bytes=262144, + ).max_bytes + == 262144 + ) + + with pytest.raises(ValidationError, match="max_bytes"): + BindingFileReadRequest( + backend_binding_ref="binding-ref", + path="report.txt", + max_bytes=262145, + ) diff --git a/dify-agent/tests/local/dify_agent/runtime/test_command_runner.py b/dify-agent/tests/local/dify_agent/runtime/test_command_runner.py new file mode 100644 index 00000000000..2a8e9f6b658 --- /dev/null +++ b/dify-agent/tests/local/dify_agent/runtime/test_command_runner.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import asyncio +from contextlib import suppress +from dataclasses import dataclass, field + +import pytest + +from dify_agent.adapters.shell.protocols import ShellCommandResult +from dify_agent.runtime.command_runner import execute_complete_with_commands + + +@dataclass(slots=True) +class _BlockingCommands: + wait_started: asyncio.Event = field(default_factory=asyncio.Event) + wait_forever: asyncio.Event = field(default_factory=asyncio.Event) + deletes: list[tuple[str, bool]] = field(default_factory=list) + + async def run(self, script: str, *, cwd: str | None, env: dict[str, str] | None, timeout: float): + assert script == "long-running" + assert cwd == "/workspace" + assert env == {"HOME": "/home/agent"} + assert timeout > 0 + return ShellCommandResult( + job_id="job-1", + status="running", + done=False, + exit_code=None, + output="started", + offset=7, + truncated=False, + ) + + async def wait(self, job_id: str, *, offset: int, timeout: float): + assert (job_id, offset) == ("job-1", 7) + assert timeout > 0 + self.wait_started.set() + await self.wait_forever.wait() + raise AssertionError("wait must remain blocked until cancellation") + + async def read_output(self, job_id: str, *, offset: int): + raise AssertionError("unexpected read_output") + + async def input(self, job_id: str, text: str, *, offset: int, timeout: float): + raise AssertionError("unexpected input") + + async def interrupt(self, job_id: str, *, grace_seconds: float): + raise AssertionError("unexpected interrupt") + + async def tail(self, job_id: str): + raise AssertionError("unexpected tail") + + async def delete(self, job_id: str, *, force: bool = False, grace_seconds: float | None = None) -> None: + assert grace_seconds is None + self.deletes.append((job_id, force)) + + +@pytest.mark.anyio +async def test_cancellation_deletes_job_returned_before_blocking_wait() -> None: + commands = _BlockingCommands() + task = asyncio.create_task( + execute_complete_with_commands( + commands, # pyright: ignore[reportArgumentType] + "long-running", + cwd="/workspace", + env={"HOME": "/home/agent"}, + timeout=60.0, + max_output_bytes=4096, + ) + ) + try: + await asyncio.wait_for(commands.wait_started.wait(), timeout=1) + + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=1) + finally: + if not task.done(): + task.cancel() + with suppress(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=1) + + assert commands.deletes == [("job-1", True)] diff --git a/dify-agent/tests/local/dify_agent/server/test_app.py b/dify-agent/tests/local/dify_agent/server/test_app.py index df845519ef3..c68252d6c2b 100644 --- a/dify-agent/tests/local/dify_agent/server/test_app.py +++ b/dify-agent/tests/local/dify_agent/server/test_app.py @@ -270,6 +270,15 @@ def test_create_app_creates_scheduler_and_closes_after_shutdown(monkeypatch: pyt getattr(route, "path", None) == "/agent-stub/drive/manifest" for route in create_app(settings).routes ) assert any(getattr(route, "path", None) == "/agent-stub/drive/commit" for route in create_app(settings).routes) + route_paths = create_app(settings).openapi()["paths"] + assert { + "/execution-bindings/files/list", + "/execution-bindings/files/read", + "/execution-bindings/files/download", + }.issubset(route_paths) + assert "/workspace/files/list" not in route_paths + assert "/workspace/files/read" not in route_paths + assert "/workspace/files/upload" not in route_paths assert FakeRunScheduler.created[0].shutdown_called is True assert FakeRunScheduler.created[0].dify_api_http_client.is_closed is True diff --git a/dify-agent/tests/local/dify_agent/server/test_binding_files.py b/dify-agent/tests/local/dify_agent/server/test_binding_files.py new file mode 100644 index 00000000000..21e93ea334b --- /dev/null +++ b/dify-agent/tests/local/dify_agent/server/test_binding_files.py @@ -0,0 +1,765 @@ +from __future__ import annotations + +import asyncio +import base64 +import json +import os +import shlex +from contextlib import suppress +from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal, cast +from unittest.mock import AsyncMock, patch + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from dify_agent.adapters.shell.protocols import ShellCommandResult, ShellProviderError +from dify_agent.layers.execution_context import DifyExecutionContextLayerConfig +from dify_agent.protocol import BindingFileDownloadRequest, BindingFileListRequest, BindingFileReadRequest +from dify_agent.runtime_backend import BindingAcquireError, BindingLostError, RuntimeLayout, RuntimeLease +from dify_agent.server import binding_files as binding_files_module +from dify_agent.server.binding_files import BindingFileError, BindingFileService, resolve_binding_path +from dify_agent.server.routes.binding_files import create_binding_files_router + +_REFERENCE = "dify-file-ref:eyJyZWNvcmRfaWQiOiJ0b29sLTEifQ==" + + +def _framed(payload: dict[str, object]) -> str: + encoded = base64.b64encode(json.dumps(payload).encode()).decode() + return f"<<>>{encoded}<<>>" + + +@dataclass(slots=True) +class _Commands: + outputs: list[str] + exit_codes: list[int] = field(default_factory=list) + calls: list[tuple[str, str | None, dict[str, str] | None, float]] = field(default_factory=list) + deletes: list[str] = field(default_factory=list) + + async def run(self, script: str, *, cwd: str | None = None, env=None, timeout: float) -> ShellCommandResult: + self.calls.append((script, cwd, env, timeout)) + output = self.outputs.pop(0) + exit_code = self.exit_codes.pop(0) if self.exit_codes else 0 + return ShellCommandResult( + job_id=f"job-{len(self.calls)}", + status="exited", + done=True, + exit_code=exit_code, + output=output, + offset=len(output), + truncated=False, + ) + + async def wait(self, job_id: str, *, offset: int, timeout: float) -> ShellCommandResult: + raise AssertionError("unexpected wait") + + async def read_output(self, job_id: str, *, offset: int): + raise AssertionError("unexpected read_output") + + async def input(self, job_id: str, text: str, *, offset: int, timeout: float): + raise AssertionError("unexpected input") + + async def interrupt(self, job_id: str, *, grace_seconds: float): + raise AssertionError("unexpected interrupt") + + async def tail(self, job_id: str): + raise AssertionError("unexpected tail") + + async def delete(self, job_id: str, *, force: bool = False, grace_seconds: float | None = None) -> None: + assert force is True + self.deletes.append(job_id) + + +@dataclass(slots=True) +class _ProviderErrorCommands(_Commands): + phase: Literal["run", "wait"] = "run" + error_code: str = "timeout" + + async def run(self, script: str, *, cwd: str | None = None, env=None, timeout: float) -> ShellCommandResult: + self.calls.append((script, cwd, env, timeout)) + if self.phase == "run": + raise ShellProviderError("shell provider failed", code=self.error_code) + return ShellCommandResult( + job_id="job-1", + status="running", + done=False, + exit_code=None, + output="", + offset=0, + truncated=False, + ) + + async def wait(self, job_id: str, *, offset: int, timeout: float) -> ShellCommandResult: + assert (job_id, offset) == ("job-1", 0) + assert timeout > 0 + raise ShellProviderError("shell provider failed", code=self.error_code) + + +@dataclass(slots=True) +class _LocalCommands(_Commands): + async def run(self, script: str, *, cwd: str | None = None, env=None, timeout: float) -> ShellCommandResult: + self.calls.append((script, cwd, env, timeout)) + process = await asyncio.create_subprocess_shell( + script, + cwd=cwd, + env={**os.environ, **(env or {})}, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + ) + output, _ = await asyncio.wait_for(process.communicate(), timeout=timeout) + text = output.decode(errors="replace") + return ShellCommandResult( + job_id=f"local-job-{len(self.calls)}", + status="exited", + done=True, + exit_code=process.returncode, + output=text, + offset=len(text), + truncated=False, + ) + + +@dataclass(slots=True) +class _Lease: + commands: _Commands + layout: RuntimeLayout = RuntimeLayout(home_dir="/home/agent", workspace_dir="/workspace") + + +@dataclass(slots=True) +class _Backend: + lease: RuntimeLease + acquired: list[str] = field(default_factory=list) + releases: int = 0 + + async def acquire(self, binding_ref: str) -> RuntimeLease: + self.acquired.append(binding_ref) + return self.lease + + async def release(self, lease: RuntimeLease) -> None: + assert lease is self.lease + self.releases += 1 + + +def _context() -> DifyExecutionContextLayerConfig: + return DifyExecutionContextLayerConfig( + tenant_id="tenant-1", + user_id="account-1", + user_from="account", + agent_mode="agent_app", + invoke_from="debugger", + ) + + +def _service(commands: _Commands, *, configured: bool = True) -> tuple[BindingFileService, _Backend]: + backend = _Backend(lease=cast(RuntimeLease, _Lease(commands=commands))) + service = BindingFileService( + execution_bindings=backend, # pyright: ignore[reportArgumentType] + agent_stub_api_base_url="http://stub/agent-stub" if configured else None, + agent_stub_token_factory=(lambda execution_context, *, session_id: "secret-jwe") if configured else None, + ) + return service, backend + + +def _local_service(tmp_path: Path) -> tuple[BindingFileService, _Backend, _LocalCommands, Path, Path]: + workspace = tmp_path / "workspace" + home = tmp_path / "home" + workspace.mkdir() + home.mkdir() + commands = _LocalCommands(outputs=[]) + backend = _Backend( + lease=cast( + RuntimeLease, + _Lease( + commands=commands, + layout=RuntimeLayout(home_dir=str(home), workspace_dir=str(workspace)), + ), + ) + ) + service = BindingFileService( + execution_bindings=backend, # pyright: ignore[reportArgumentType] + agent_stub_api_base_url=None, + agent_stub_token_factory=None, + ) + return service, backend, commands, workspace, home + + +def test_resolve_binding_path_supports_workspace_home_absolute_and_parent_paths() -> None: + layout = RuntimeLayout(home_dir="/home/agent", workspace_dir="/workspace") + + assert resolve_binding_path("", layout) == "/workspace" + assert resolve_binding_path("reports/out.csv", layout) == "/workspace/reports/out.csv" + assert resolve_binding_path("~", layout) == "/home/agent" + assert resolve_binding_path("~/outputs/out.csv", layout) == "/home/agent/outputs/out.csv" + assert resolve_binding_path("/var/data/out.csv", layout) == "/var/data/out.csv" + assert resolve_binding_path("../shared/out.csv", layout) == "/shared/out.csv" + + +@pytest.mark.anyio +async def test_list_and_read_use_commands_with_consistent_binding_paths_and_release_leases() -> None: + commands = _Commands( + outputs=[ + _framed( + { + "path": "reports", + "entries": [{"name": "note.txt", "type": "file", "size": 4, "mtime": 1}], + "truncated": False, + } + ), + _framed( + { + "path": "~/note.txt", + "size": 4, + "truncated": False, + "binary": False, + "text": "note", + } + ), + ] + ) + service, backend = _service(commands) + + listing = await service.list_files(BindingFileListRequest(backend_binding_ref="binding-ref", path="reports")) + preview = await service.read_file(BindingFileReadRequest(backend_binding_ref="binding-ref", path="~/note.txt")) + + assert listing.entries[0].name == "note.txt" + assert preview.text == "note" + assert "/workspace/reports" in commands.calls[0][0] + assert "/home/agent/note.txt" in commands.calls[1][0] + assert all(call[1] == "/workspace" for call in commands.calls) + assert all(call[2] == {"HOME": "/home/agent"} for call in commands.calls) + assert backend.acquired == ["binding-ref", "binding-ref"] + assert backend.releases == 2 + assert commands.deletes == ["job-1", "job-2"] + + +@pytest.mark.anyio +async def test_real_list_script_caps_1001_entries_at_1000(tmp_path: Path) -> None: + service, backend, commands, workspace, _ = _local_service(tmp_path) + for index in range(1001): + (workspace / f"{index:04d}.txt").write_bytes(b"x") + + listing = await service.list_files(BindingFileListRequest(backend_binding_ref="binding-ref", path=".")) + + assert len(listing.entries) == 1000 + assert listing.entries[0].name == "0000.txt" + assert listing.entries[-1].name == "0999.txt" + assert listing.truncated is True + assert commands.deletes == ["local-job-1"] + assert backend.releases == 1 + + +@pytest.mark.anyio +async def test_real_read_script_handles_boundary_truncation_and_binary(tmp_path: Path) -> None: + service, backend, commands, workspace, _ = _local_service(tmp_path) + (workspace / "boundary.txt").write_bytes(b"a" * 262144) + (workspace / "truncated.txt").write_bytes(b"b" * 262145) + (workspace / "binary.bin").write_bytes(b"\xff\x00") + + boundary = await service.read_file( + BindingFileReadRequest(backend_binding_ref="binding-ref", path="boundary.txt", max_bytes=262144) + ) + truncated = await service.read_file( + BindingFileReadRequest(backend_binding_ref="binding-ref", path="truncated.txt", max_bytes=262144) + ) + binary = await service.read_file( + BindingFileReadRequest(backend_binding_ref="binding-ref", path="binary.bin", max_bytes=262144) + ) + + assert boundary.size == 262144 + assert boundary.truncated is False + assert boundary.binary is False + assert boundary.text == "a" * 262144 + assert truncated.size == 262145 + assert truncated.truncated is True + assert truncated.binary is False + assert truncated.text == "b" * 262144 + assert binary.size == 2 + assert binary.truncated is False + assert binary.binary is True + assert binary.text is None + assert commands.deletes == ["local-job-1", "local-job-2", "local-job-3"] + assert backend.releases == 3 + + +@pytest.mark.anyio +async def test_real_browse_script_output_over_command_cap_normalizes_to_unavailable( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, backend, commands, workspace, _ = _local_service(tmp_path) + (workspace / "report.txt").write_bytes(b"report") + monkeypatch.setattr(binding_files_module, "_BROWSE_OUTPUT_MAX_BYTES", 64) + + with pytest.raises(BindingFileError) as exc_info: + await service.list_files(BindingFileListRequest(backend_binding_ref="binding-ref", path=".")) + + assert exc_info.value.code == "binding_unavailable" + assert exc_info.value.status_code == 502 + assert commands.deletes == ["local-job-1"] + assert backend.releases == 1 + + +@pytest.mark.parametrize("operation", ["list", "read"]) +@pytest.mark.anyio +async def test_browse_preserves_binding_file_error_and_releases_lease(operation: str) -> None: + commands = _Commands(outputs=["FileNotFoundError: missing"], exit_codes=[1]) + service, backend = _service(commands) + + with pytest.raises(BindingFileError) as exc_info: + if operation == "list": + await service.list_files(BindingFileListRequest(backend_binding_ref="binding-ref", path="missing")) + else: + await service.read_file(BindingFileReadRequest(backend_binding_ref="binding-ref", path="missing")) + + assert exc_info.value.code == "invalid_binding_path" + assert exc_info.value.status_code == 400 + assert backend.releases == 1 + + +@pytest.mark.parametrize("operation", ["list", "read"]) +@pytest.mark.anyio +async def test_browse_maps_malformed_backend_response_to_unavailable_and_releases_lease(operation: str) -> None: + commands = _Commands(outputs=[_framed({"path": "."})]) + service, backend = _service(commands) + + with pytest.raises(BindingFileError) as exc_info: + if operation == "list": + await service.list_files(BindingFileListRequest(backend_binding_ref="binding-ref", path=".")) + else: + await service.read_file(BindingFileReadRequest(backend_binding_ref="binding-ref", path="report.txt")) + + assert exc_info.value.code == "binding_unavailable" + assert exc_info.value.status_code == 502 + assert backend.releases == 1 + + +@pytest.mark.parametrize("missing_field", ["user_id", "user_from"]) +@pytest.mark.anyio +async def test_download_rejects_each_missing_identity_before_token_or_lease(missing_field: str) -> None: + context = _context().model_copy(update={missing_field: None}) + request = BindingFileDownloadRequest( + backend_binding_ref="binding-ref", + path="report.txt", + execution_context=context, + ) + commands = _Commands(outputs=[]) + service, backend = _service(commands) + issued_tokens: list[tuple[DifyExecutionContextLayerConfig, str | None]] = [] + + def issue_token(execution_context: DifyExecutionContextLayerConfig, *, session_id: str | None) -> str: + issued_tokens.append((execution_context, session_id)) + return "must-not-be-issued" + + service.agent_stub_token_factory = issue_token + + with pytest.raises(BindingFileError) as identity_error: + await service.download_file(request) + + assert identity_error.value.code == "invalid_execution_context" + assert identity_error.value.status_code == 400 + assert issued_tokens == [] + assert backend.acquired == [] + assert commands.calls == [] + + +@pytest.mark.anyio +async def test_download_rejects_missing_configuration_before_acquiring_lease() -> None: + request = BindingFileDownloadRequest( + backend_binding_ref="binding-ref", + path="report.txt", + execution_context=_context(), + ) + commands = _Commands(outputs=[]) + unavailable_service, unavailable_backend = _service(commands, configured=False) + + with pytest.raises(BindingFileError) as unavailable_error: + await unavailable_service.download_file(request) + + assert unavailable_error.value.code == "agent_stub_upload_unavailable" + assert unavailable_error.value.status_code == 503 + assert unavailable_backend.acquired == [] + assert commands.calls == [] + + +@pytest.mark.anyio +async def test_download_shell_quotes_resolved_path_and_returns_only_reference_inside_lease() -> None: + commands = _Commands( + outputs=[json.dumps({"transfer_method": "tool_file", "reference": _REFERENCE, "public_download_url": "bad"})] + ) + service, backend = _service(commands) + issued_tokens: list[tuple[DifyExecutionContextLayerConfig, str | None]] = [] + + def issue_token(execution_context: DifyExecutionContextLayerConfig, *, session_id: str | None) -> str: + issued_tokens.append((execution_context, session_id)) + return "secret-jwe" + + service.agent_stub_token_factory = issue_token + context = _context() + + result = await service.download_file( + BindingFileDownloadRequest( + backend_binding_ref="binding-ref", + path="../shared/report $(touch should-not-run); final.txt", + execution_context=context, + ) + ) + + assert result.reference == _REFERENCE + script, cwd, env, timeout = commands.calls[0] + assert shlex.split(script) == [ + "dify-agent", + "file", + "upload", + "--no-download-link", + "/shared/report $(touch should-not-run); final.txt", + ] + assert cwd == "/workspace" + assert env == { + "HOME": "/home/agent", + "DIFY_AGENT_STUB_API_BASE_URL": "http://stub/agent-stub", + "DIFY_AGENT_STUB_AUTH_JWE": "secret-jwe", + "DIFY_AGENT_STUB_DRIVE_BASE": "/mnt/drive", + } + assert timeout == pytest.approx(60.0, rel=0, abs=0.01) + assert issued_tokens == [(context, None)] + assert issued_tokens[0][0].model_dump() == context.model_dump() + assert backend.releases == 1 + + +@pytest.mark.parametrize( + ("output", "exit_code"), + [ + ("upload failed", 1), + ("not-json", 0), + (json.dumps({"transfer_method": "url", "reference": _REFERENCE}), 0), + (json.dumps({"transfer_method": "tool_file", "reference": "raw-id"}), 0), + ("x" * (32 * 1024 + 1), 0), + ], +) +@pytest.mark.anyio +async def test_download_normalizes_cli_failures_and_releases_lease(output: str, exit_code: int) -> None: + commands = _Commands(outputs=[output], exit_codes=[exit_code]) + service, backend = _service(commands) + + with pytest.raises(BindingFileError) as exc_info: + await service.download_file( + BindingFileDownloadRequest( + backend_binding_ref="binding-ref", + path="report.txt", + execution_context=_context(), + ) + ) + + assert exc_info.value.code == "binding_file_download_failed" + assert backend.releases == 1 + + +@pytest.mark.anyio +async def test_download_command_timeout_releases_lease_and_returns_download_failed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + commands = _Commands(outputs=[]) + service, backend = _service(commands) + + async def timed_out(*_args, **_kwargs): + return type( + "TimedOutResult", + (), + {"exit_code": None, "output_complete": False, "output": "", "incomplete_reason": "timeout"}, + )() + + monkeypatch.setattr(binding_files_module, "execute_complete_with_commands", timed_out) + + with pytest.raises(BindingFileError) as exc_info: + await service.download_file( + BindingFileDownloadRequest( + backend_binding_ref="binding-ref", + path="report.txt", + execution_context=_context(), + ) + ) + + assert exc_info.value.code == "binding_file_download_failed" + assert exc_info.value.status_code == 502 + assert backend.releases == 1 + + +@pytest.mark.parametrize( + ("phase", "error_code", "expected_code", "expected_deletes"), + [ + ("run", "timeout", "binding_file_download_failed", []), + ("wait", "timeout", "binding_file_download_failed", ["job-1"]), + ("wait", "request_error", "binding_unavailable", ["job-1"]), + ], +) +@pytest.mark.anyio +async def test_download_maps_only_shell_provider_timeout_to_download_failed_and_cleans_up( + phase: Literal["run", "wait"], + error_code: str, + expected_code: str, + expected_deletes: list[str], +) -> None: + commands = _ProviderErrorCommands(outputs=[], phase=phase, error_code=error_code) + service, backend = _service(commands) + + with pytest.raises(BindingFileError) as exc_info: + await service.download_file( + BindingFileDownloadRequest( + backend_binding_ref="binding-ref", + path="report.txt", + execution_context=_context(), + ) + ) + + assert exc_info.value.code == expected_code + assert exc_info.value.status_code == 502 + assert commands.deletes == expected_deletes + assert backend.acquired == ["binding-ref"] + assert backend.releases == 1 + + +@pytest.mark.anyio +async def test_download_cancellation_releases_lease(monkeypatch: pytest.MonkeyPatch) -> None: + commands = _Commands(outputs=[]) + service, backend = _service(commands) + command_started = asyncio.Event() + never_finishes = asyncio.Event() + + async def block(*_args, **_kwargs): + command_started.set() + await never_finishes.wait() + raise AssertionError("unreachable") + + monkeypatch.setattr(binding_files_module, "execute_complete_with_commands", block) + task = asyncio.create_task( + service.download_file( + BindingFileDownloadRequest( + backend_binding_ref="binding-ref", + path="report.txt", + execution_context=_context(), + ) + ) + ) + try: + await asyncio.wait_for(command_started.wait(), timeout=1) + task.cancel() + + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=1) + finally: + if not task.done(): + task.cancel() + with suppress(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=1) + + assert backend.releases == 1 + + +@pytest.mark.parametrize( + ("backend_error", "expected_code", "expected_status"), + [ + (BindingLostError("lost"), "binding_not_found", 404), + (BindingAcquireError("unavailable"), "binding_unavailable", 502), + ], +) +@pytest.mark.anyio +async def test_download_maps_binding_acquire_errors( + backend_error: Exception, + expected_code: str, + expected_status: int, +) -> None: + class FailingBackend: + releases = 0 + + async def acquire(self, _binding_ref: str) -> RuntimeLease: + raise backend_error + + async def release(self, _lease: RuntimeLease) -> None: + self.releases += 1 + + backend = FailingBackend() + service = BindingFileService( + execution_bindings=backend, # pyright: ignore[reportArgumentType] + agent_stub_api_base_url="http://stub/agent-stub", + agent_stub_token_factory=lambda execution_context, *, session_id: "secret-jwe", + ) + + with pytest.raises(BindingFileError) as exc_info: + await service.download_file( + BindingFileDownloadRequest( + backend_binding_ref="binding-ref", + path="report.txt", + execution_context=_context(), + ) + ) + + assert exc_info.value.code == expected_code + assert exc_info.value.status_code == expected_status + assert backend.releases == 0 + + +@pytest.mark.parametrize( + ("error", "expected_status", "expected_code"), + [ + (BindingFileError("binding_not_found", "missing", status_code=404), 404, "binding_not_found"), + (BindingFileError("binding_unavailable", "unavailable", status_code=502), 502, "binding_unavailable"), + ], +) +def test_binding_file_route_preserves_structured_error_status_and_code( + error: BindingFileError, + expected_status: int, + expected_code: str, +) -> None: + class FailingService: + async def download_file(self, _request): + raise error + + app = FastAPI() + app.include_router(create_binding_files_router(lambda: cast(BindingFileService, cast(object, FailingService())))) + + response = TestClient(app).post( + "/execution-bindings/files/download", + json={ + "backend_binding_ref": "binding-ref", + "path": "report.txt", + "execution_context": { + "tenant_id": "tenant-1", + "user_id": "account-1", + "user_from": "account", + "agent_mode": "agent_app", + "invoke_from": "debugger", + }, + }, + ) + + assert response.status_code == expected_status + assert response.json() == {"detail": {"code": expected_code, "message": error.message}} + + +@pytest.mark.parametrize( + ("path", "payload"), + [ + ("/execution-bindings/files/list", {"backend_binding_ref": "", "path": "."}), + ( + "/execution-bindings/files/read", + {"backend_binding_ref": "binding-ref", "path": "report.txt", "max_bytes": 262145}, + ), + ], +) +def test_list_and_read_route_validation_returns_structured_400_without_calling_service( + path: str, + payload: object, +) -> None: + service, backend = _service(_Commands(outputs=[])) + app = FastAPI() + app.include_router(create_binding_files_router(lambda: service)) + + with ( + patch.object(BindingFileService, "list_files", new_callable=AsyncMock) as list_files, + patch.object(BindingFileService, "read_file", new_callable=AsyncMock) as read_file, + ): + response = TestClient(app).post(path, json=payload) + + assert response.status_code == 400 + assert response.json() == { + "detail": { + "code": "invalid_binding_path", + "message": "Binding file path or payload is invalid", + } + } + list_files.assert_not_awaited() + read_file.assert_not_awaited() + assert backend.acquired == [] + assert backend.releases == 0 + + +def test_list_route_redacts_unexpected_field_value_from_validation_error() -> None: + service, backend = _service(_Commands(outputs=[])) + app = FastAPI() + app.include_router(create_binding_files_router(lambda: service)) + + with patch.object(BindingFileService, "list_files", new_callable=AsyncMock) as list_files: + response = TestClient(app).post( + "/execution-bindings/files/list", + json={"backend_binding_ref": "binding-ref", "path": ".", "unexpected": "top-secret"}, + ) + + assert response.status_code == 400 + assert response.json() == { + "detail": { + "code": "invalid_binding_path", + "message": "Binding file path or payload is invalid", + } + } + assert "top-secret" not in response.text + list_files.assert_not_awaited() + assert backend.acquired == [] + assert backend.releases == 0 + + +def test_download_route_validation_returns_422_without_calling_service() -> None: + service, backend = _service(_Commands(outputs=[])) + app = FastAPI() + app.include_router(create_binding_files_router(lambda: service)) + + with patch.object(BindingFileService, "download_file", new_callable=AsyncMock) as download_file: + download_response = TestClient(app).post( + "/execution-bindings/files/download", + json={"backend_binding_ref": "", "path": "", "execution_context": {}}, + ) + + assert download_response.status_code == 422 + assert isinstance(download_response.json()["detail"], list) + download_file.assert_not_awaited() + assert backend.acquired == [] + assert backend.releases == 0 + + +def test_read_route_accepts_preview_size_limit() -> None: + commands = _Commands( + outputs=[ + _framed( + { + "path": "report.txt", + "size": 262144, + "truncated": False, + "binary": False, + "text": "preview", + } + ) + ] + ) + service, backend = _service(commands) + app = FastAPI() + app.include_router(create_binding_files_router(lambda: service)) + + response = TestClient(app).post( + "/execution-bindings/files/read", + json={"backend_binding_ref": "binding-ref", "path": "report.txt", "max_bytes": 262144}, + ) + + assert response.status_code == 200 + assert response.json()["text"] == "preview" + assert "262144" in commands.calls[0][0] + assert backend.acquired == ["binding-ref"] + assert backend.releases == 1 + + +def test_binding_file_validation_override_preserves_openapi_request_schemas() -> None: + app = FastAPI() + app.include_router(create_binding_files_router(lambda: None)) + openapi = app.openapi() + paths = openapi["paths"] + + assert paths["/execution-bindings/files/list"]["post"]["requestBody"]["content"]["application/json"]["schema"] == { + "$ref": "#/components/schemas/BindingFileListRequest" + } + assert paths["/execution-bindings/files/read"]["post"]["requestBody"]["content"]["application/json"]["schema"] == { + "$ref": "#/components/schemas/BindingFileReadRequest" + } + max_bytes_schema = openapi["components"]["schemas"]["BindingFileReadRequest"]["properties"]["max_bytes"] + assert max_bytes_schema["default"] == 262144 + assert max_bytes_schema["minimum"] == 1 + assert max_bytes_schema["maximum"] == 262144 diff --git a/dify-agent/tests/local/dify_agent/server/test_settings.py b/dify-agent/tests/local/dify_agent/server/test_settings.py index 7db759feb6f..282542b4cca 100644 --- a/dify-agent/tests/local/dify_agent/server/test_settings.py +++ b/dify-agent/tests/local/dify_agent/server/test_settings.py @@ -313,12 +313,6 @@ def test_build_runtime_backend_profile_passes_e2b_active_timeout() -> None: assert profile.execution_bindings.template == "difys-default-team/dify-agent-local-sandbox" -def test_sandbox_file_upload_limit_defaults_to_tool_file_limit() -> None: - settings = ServerSettings() - - assert settings.sandbox_file_upload_max_bytes == 50 * 1024 * 1024 - - def test_build_runtime_backend_profile_rejects_missing_enterprise_endpoint( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/dify-agent/tests/local/dify_agent/server/test_workspace_files.py b/dify-agent/tests/local/dify_agent/server/test_workspace_files.py deleted file mode 100644 index 78af3bee9ed..00000000000 --- a/dify-agent/tests/local/dify_agent/server/test_workspace_files.py +++ /dev/null @@ -1,123 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import cast - -import pytest - -from dify_agent.layers.execution_context import DifyExecutionContextLayerConfig -from dify_agent.protocol import ( - WorkspaceListRequest, - WorkspaceReadRequest, - WorkspaceUploadRequest, - WorkspaceUploadedFile, -) -from dify_agent.runtime_backend import ( - RuntimeLayout, - RuntimeLease, - WorkspaceFileContent, - WorkspaceFileEntry, - WorkspaceListResult, - WorkspaceReadResult, -) -from dify_agent.server.workspace_files import WorkspaceFileService - - -@dataclass(slots=True) -class _Files: - calls: list[tuple[str, str]] = field(default_factory=list) - - async def list_directory(self, *, path: str, limit: int) -> WorkspaceListResult: - self.calls.append(("list", path)) - assert limit == 1000 - return WorkspaceListResult( - path=path, - entries=(WorkspaceFileEntry(name="note.txt", type="file", size=4, mtime=1),), - truncated=False, - ) - - async def read_file(self, *, path: str, max_bytes: int) -> WorkspaceReadResult: - self.calls.append(("read", path)) - return WorkspaceReadResult(path=path, size=4, truncated=False, binary=False, text="note") - - async def read_bytes(self, *, path: str, max_bytes: int) -> WorkspaceFileContent: - self.calls.append(("bytes", path)) - return WorkspaceFileContent(path=path, size=4, content=b"note") - - -@dataclass(slots=True) -class _Lease: - files: _Files = field(default_factory=_Files) - layout: RuntimeLayout = RuntimeLayout(home_dir="/home/agent", workspace_dir="/workspace") - commands: object = field(default_factory=object) - - -@dataclass(slots=True) -class _Backend: - lease: RuntimeLease - acquired: list[str] = field(default_factory=list) - releases: int = 0 - - async def acquire(self, binding_ref: str) -> RuntimeLease: - self.acquired.append(binding_ref) - return self.lease - - async def release(self, lease: RuntimeLease) -> None: - assert lease is self.lease - self.releases += 1 - - -@dataclass(slots=True) -class _Uploader: - uploads: list[tuple[str, str, bytes]] = field(default_factory=list) - - async def upload( - self, - *, - execution_context: DifyExecutionContextLayerConfig, - filename: str, - mimetype: str, - content: bytes, - ) -> WorkspaceUploadedFile: - del execution_context - self.uploads.append((filename, mimetype, content)) - return WorkspaceUploadedFile(reference="tool-file-1", download_url="https://files/note.txt") - - -@pytest.mark.anyio -async def test_workspace_service_passes_paths_directly_and_leases_each_operation() -> None: - lease = _Lease() - backend = _Backend(lease=cast(RuntimeLease, lease)) - uploader = _Uploader() - service = WorkspaceFileService( - execution_bindings=backend, # pyright: ignore[reportArgumentType] - upload_max_bytes=1024, - file_uploader=uploader, - ) - - listing = await service.list_files(WorkspaceListRequest(backend_binding_ref="binding-ref", path="/var/data")) - preview = await service.read_file(WorkspaceReadRequest(backend_binding_ref="binding-ref", path="~/note.txt")) - uploaded = await service.upload_file( - WorkspaceUploadRequest( - backend_binding_ref="binding-ref", - path="../outside.txt", - execution_context=DifyExecutionContextLayerConfig( - tenant_id="tenant-1", - user_from="account", - agent_mode="agent_app", - invoke_from="debugger", - ), - ) - ) - - assert listing.path == "/var/data" - assert preview.path == "~/note.txt" - assert uploaded.path == "../outside.txt" - assert lease.files.calls == [ - ("list", "/var/data"), - ("read", "~/note.txt"), - ("bytes", "../outside.txt"), - ] - assert backend.acquired == ["binding-ref", "binding-ref", "binding-ref"] - assert backend.releases == 3 - assert uploader.uploads == [("outside.txt", "text/plain", b"note")] diff --git a/docker/.env.example b/docker/.env.example index 2066467ae04..34491c1d082 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -224,7 +224,6 @@ PLUGIN_DAEMON_PORT=5002 PLUGIN_DAEMON_KEY=lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi PLUGIN_DAEMON_URL=http://plugin_daemon:5002 PLUGIN_MAX_PACKAGE_SIZE=52428800 -# Compose maps this byte value to DIFY_AGENT_SANDBOX_FILE_UPLOAD_MAX_BYTES. PLUGIN_MAX_FILE_SIZE=52428800 PLUGIN_MODEL_SCHEMA_CACHE_TTL=3600 PLUGIN_PPROF_ENABLED=false diff --git a/docker/docker-compose-template.yaml b/docker/docker-compose-template.yaml index 5401d0694b4..4ac4e0161ff 100644 --- a/docker/docker-compose-template.yaml +++ b/docker/docker-compose-template.yaml @@ -680,7 +680,6 @@ services: DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS: ${DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS:-3600} DIFY_AGENT_E2B_SHELLCTL_AUTH_TOKEN: ${DIFY_AGENT_E2B_SHELLCTL_AUTH_TOKEN:-} DIFY_AGENT_E2B_SHELLCTL_PORT: ${DIFY_AGENT_E2B_SHELLCTL_PORT:-5004} - DIFY_AGENT_SANDBOX_FILE_UPLOAD_MAX_BYTES: ${PLUGIN_MAX_FILE_SIZE:-52428800} DIFY_AGENT_STUB_API_BASE_URL: ${DIFY_AGENT_STUB_API_BASE_URL:-http://agent_backend:5050/agent-stub} DIFY_AGENT_SANDBOX_FILES_BASE_URL: ${DIFY_AGENT_SANDBOX_FILES_BASE_URL:-http://api:5001} # This is security-sensitive: it derives the JWE encryption key for Agent Stub bearer tokens. diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 27c24149fdb..289076b9364 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -686,7 +686,6 @@ services: DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS: ${DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS:-3600} DIFY_AGENT_E2B_SHELLCTL_AUTH_TOKEN: ${DIFY_AGENT_E2B_SHELLCTL_AUTH_TOKEN:-} DIFY_AGENT_E2B_SHELLCTL_PORT: ${DIFY_AGENT_E2B_SHELLCTL_PORT:-5004} - DIFY_AGENT_SANDBOX_FILE_UPLOAD_MAX_BYTES: ${PLUGIN_MAX_FILE_SIZE:-52428800} DIFY_AGENT_STUB_API_BASE_URL: ${DIFY_AGENT_STUB_API_BASE_URL:-http://agent_backend:5050/agent-stub} DIFY_AGENT_SANDBOX_FILES_BASE_URL: ${DIFY_AGENT_SANDBOX_FILES_BASE_URL:-http://api:5001} # This is security-sensitive: it derives the JWE encryption key for Agent Stub bearer tokens. diff --git a/docker/envs/core-services/dify-agent.env.example b/docker/envs/core-services/dify-agent.env.example index 9d0562aaa3c..46b950bc9f7 100644 --- a/docker/envs/core-services/dify-agent.env.example +++ b/docker/envs/core-services/dify-agent.env.example @@ -33,9 +33,6 @@ DIFY_AGENT_E2B_TEMPLATE=difys-default-team/dify-agent-local-sandbox DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS=3600 DIFY_AGENT_E2B_SHELLCTL_AUTH_TOKEN= DIFY_AGENT_E2B_SHELLCTL_PORT=5004 -# Standalone/direct-container byte limit. Full Docker Compose derives this from -# PLUGIN_MAX_FILE_SIZE in docker/.env. -DIFY_AGENT_SANDBOX_FILE_UPLOAD_MAX_BYTES=52428800 # Sandbox-reachable Dify API base for signed /files/* transfers. DIFY_AGENT_SANDBOX_FILES_BASE_URL=http://api:5001 DIFY_AGENT_STUB_API_BASE_URL=http://agent_backend:5050/agent-stub diff --git a/packages/contracts/generated/api/console/agent/orpc.gen.ts b/packages/contracts/generated/api/console/agent/orpc.gen.ts index 2cc590c433d..7d10e128fa0 100644 --- a/packages/contracts/generated/api/console/agent/orpc.gen.ts +++ b/packages/contracts/generated/api/console/agent/orpc.gen.ts @@ -157,9 +157,9 @@ import { zPostAgentByAgentIdPublishBody, zPostAgentByAgentIdPublishPath, zPostAgentByAgentIdPublishResponse, - zPostAgentByAgentIdSandboxFilesUploadBody, - zPostAgentByAgentIdSandboxFilesUploadPath, - zPostAgentByAgentIdSandboxFilesUploadResponse, + zPostAgentByAgentIdSandboxFilesDownloadBody, + zPostAgentByAgentIdSandboxFilesDownloadPath, + zPostAgentByAgentIdSandboxFilesDownloadResponse, zPostAgentByAgentIdSkillsBySlugInferToolsPath, zPostAgentByAgentIdSkillsBySlugInferToolsResponse, zPostAgentByAgentIdSkillsUploadBody, @@ -1179,6 +1179,30 @@ export const referencingWorkflows = { get: get28, } +/** + * Create a ToolFile from one Agent App Binding file and return its download URL + */ +export const post17 = oc + .route({ + description: 'Create a ToolFile from one Agent App Binding file and return its download URL', + inputStructure: 'detailed', + method: 'POST', + operationId: 'postAgentByAgentIdSandboxFilesDownload', + path: '/agent/{agent_id}/sandbox/files/download', + tags: ['console'], + }) + .input( + z.object({ + body: zPostAgentByAgentIdSandboxFilesDownloadBody, + params: zPostAgentByAgentIdSandboxFilesDownloadPath, + }), + ) + .output(zPostAgentByAgentIdSandboxFilesDownloadResponse) + +export const download5 = { + post: post17, +} + /** * Read a text/binary preview file in an Agent App conversation sandbox */ @@ -1203,30 +1227,6 @@ export const read = { get: get29, } -/** - * Upload one Agent App sandbox file and return a signed download URL - */ -export const post17 = oc - .route({ - description: 'Upload one Agent App sandbox file and return a signed download URL', - inputStructure: 'detailed', - method: 'POST', - operationId: 'postAgentByAgentIdSandboxFilesUpload', - path: '/agent/{agent_id}/sandbox/files/upload', - tags: ['console'], - }) - .input( - z.object({ - body: zPostAgentByAgentIdSandboxFilesUploadBody, - params: zPostAgentByAgentIdSandboxFilesUploadPath, - }), - ) - .output(zPostAgentByAgentIdSandboxFilesUploadResponse) - -export const upload2 = { - post: post17, -} - /** * List a directory in an Agent App conversation sandbox */ @@ -1249,8 +1249,8 @@ export const get30 = oc export const files5 = { get: get30, + download: download5, read, - upload: upload2, } /** @@ -1294,7 +1294,7 @@ export const post18 = oc ) .output(zPostAgentByAgentIdSkillsUploadResponse) -export const upload3 = { +export const upload2 = { post: post18, } @@ -1338,7 +1338,7 @@ export const bySlug = { } export const skills3 = { - upload: upload3, + upload: upload2, bySlug, } diff --git a/packages/contracts/generated/api/console/agent/types.gen.ts b/packages/contracts/generated/api/console/agent/types.gen.ts index 9e1a46e8f95..cfabaa87f32 100644 --- a/packages/contracts/generated/api/console/agent/types.gen.ts +++ b/packages/contracts/generated/api/console/agent/types.gen.ts @@ -433,6 +433,16 @@ export type SandboxListResponse = { truncated?: boolean } +export type AgentSandboxDownloadPayload = { + caller_id: string + caller_type: 'build_draft' | 'conversation' + path: string +} + +export type SandboxDownloadResponse = { + url: string +} + export type SandboxReadResponse = { binary: boolean path: string @@ -441,16 +451,6 @@ export type SandboxReadResponse = { truncated: boolean } -export type AgentSandboxUploadPayload = { - caller_id: string - caller_type: 'build_draft' | 'conversation' - path: string -} - -export type SandboxUploadResponse = { - url: string -} - export type AgentSkillUploadResponse = { manifest: SkillManifest skill: AgentUploadedSkillResponse @@ -3117,6 +3117,22 @@ export type GetAgentByAgentIdSandboxFilesResponses = { export type GetAgentByAgentIdSandboxFilesResponse = GetAgentByAgentIdSandboxFilesResponses[keyof GetAgentByAgentIdSandboxFilesResponses] +export type PostAgentByAgentIdSandboxFilesDownloadData = { + body: AgentSandboxDownloadPayload + path: { + agent_id: string + } + query?: never + url: '/agent/{agent_id}/sandbox/files/download' +} + +export type PostAgentByAgentIdSandboxFilesDownloadResponses = { + 200: SandboxDownloadResponse +} + +export type PostAgentByAgentIdSandboxFilesDownloadResponse = + PostAgentByAgentIdSandboxFilesDownloadResponses[keyof PostAgentByAgentIdSandboxFilesDownloadResponses] + export type GetAgentByAgentIdSandboxFilesReadData = { body?: never path: { @@ -3137,22 +3153,6 @@ export type GetAgentByAgentIdSandboxFilesReadResponses = { export type GetAgentByAgentIdSandboxFilesReadResponse = GetAgentByAgentIdSandboxFilesReadResponses[keyof GetAgentByAgentIdSandboxFilesReadResponses] -export type PostAgentByAgentIdSandboxFilesUploadData = { - body: AgentSandboxUploadPayload - path: { - agent_id: string - } - query?: never - url: '/agent/{agent_id}/sandbox/files/upload' -} - -export type PostAgentByAgentIdSandboxFilesUploadResponses = { - 200: SandboxUploadResponse -} - -export type PostAgentByAgentIdSandboxFilesUploadResponse = - PostAgentByAgentIdSandboxFilesUploadResponses[keyof PostAgentByAgentIdSandboxFilesUploadResponses] - export type PostAgentByAgentIdSkillsUploadData = { body: { file: Blob | File diff --git a/packages/contracts/generated/api/console/agent/zod.gen.ts b/packages/contracts/generated/api/console/agent/zod.gen.ts index f3b35522b5d..9ad871a9582 100644 --- a/packages/contracts/generated/api/console/agent/zod.gen.ts +++ b/packages/contracts/generated/api/console/agent/zod.gen.ts @@ -200,6 +200,22 @@ export const zSandboxInfoResponse = z.object({ workspace_cwd: z.string(), }) +/** + * AgentSandboxDownloadPayload + */ +export const zAgentSandboxDownloadPayload = z.object({ + caller_id: z.string().min(1), + caller_type: z.enum(['build_draft', 'conversation']), + path: z.string().min(1), +}) + +/** + * SandboxDownloadResponse + */ +export const zSandboxDownloadResponse = z.object({ + url: z.string(), +}) + /** * SandboxReadResponse */ @@ -211,22 +227,6 @@ export const zSandboxReadResponse = z.object({ truncated: z.boolean(), }) -/** - * AgentSandboxUploadPayload - */ -export const zAgentSandboxUploadPayload = z.object({ - caller_id: z.string().min(1), - caller_type: z.enum(['build_draft', 'conversation']), - path: z.string().min(1), -}) - -/** - * SandboxUploadResponse - */ -export const zSandboxUploadResponse = z.object({ - url: z.string(), -}) - /** * AgentConfigSnapshotRestoreResponse */ @@ -3500,6 +3500,17 @@ export const zGetAgentByAgentIdSandboxFilesQuery = z.object({ */ export const zGetAgentByAgentIdSandboxFilesResponse = zSandboxListResponse +export const zPostAgentByAgentIdSandboxFilesDownloadBody = zAgentSandboxDownloadPayload + +export const zPostAgentByAgentIdSandboxFilesDownloadPath = z.object({ + agent_id: z.uuid(), +}) + +/** + * Download URL returned + */ +export const zPostAgentByAgentIdSandboxFilesDownloadResponse = zSandboxDownloadResponse + export const zGetAgentByAgentIdSandboxFilesReadPath = z.object({ agent_id: z.uuid(), }) @@ -3515,17 +3526,6 @@ export const zGetAgentByAgentIdSandboxFilesReadQuery = z.object({ */ export const zGetAgentByAgentIdSandboxFilesReadResponse = zSandboxReadResponse -export const zPostAgentByAgentIdSandboxFilesUploadBody = zAgentSandboxUploadPayload - -export const zPostAgentByAgentIdSandboxFilesUploadPath = z.object({ - agent_id: z.uuid(), -}) - -/** - * Uploaded - */ -export const zPostAgentByAgentIdSandboxFilesUploadResponse = zSandboxUploadResponse - export const zPostAgentByAgentIdSkillsUploadBody = z.object({ file: z.custom((value) => value instanceof Blob || value instanceof File), }) diff --git a/packages/contracts/generated/api/console/apps/orpc.gen.ts b/packages/contracts/generated/api/console/apps/orpc.gen.ts index b22e0a56886..ede274a1f60 100644 --- a/packages/contracts/generated/api/console/apps/orpc.gen.ts +++ b/packages/contracts/generated/api/console/apps/orpc.gen.ts @@ -406,9 +406,9 @@ import { zPostAppsByAppIdWorkflowCommentsByCommentIdResolveResponse, zPostAppsByAppIdWorkflowCommentsPath, zPostAppsByAppIdWorkflowCommentsResponse, - zPostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesUploadBody, - zPostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesUploadPath, - zPostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesUploadResponse, + zPostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesDownloadBody, + zPostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesDownloadPath, + zPostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesDownloadResponse, zPostAppsByAppIdWorkflowRunsTasksByTaskIdStopPath, zPostAppsByAppIdWorkflowRunsTasksByTaskIdStopResponse, zPostAppsByAppIdWorkflowsByWorkflowIdRestorePath, @@ -3050,6 +3050,31 @@ export const byRunId = { nodeExecutions, } +/** + * Create a ToolFile from one workflow Agent Binding file and return its download URL + */ +export const post42 = oc + .route({ + description: + 'Create a ToolFile from one workflow Agent Binding file and return its download URL', + inputStructure: 'detailed', + method: 'POST', + operationId: 'postAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesDownload', + path: '/apps/{app_id}/workflow-runs/{workflow_run_id}/agent-nodes/{node_id}/sandbox/files/download', + tags: ['console'], + }) + .input( + z.object({ + body: zPostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesDownloadBody, + params: zPostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesDownloadPath, + }), + ) + .output(zPostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesDownloadResponse) + +export const download5 = { + post: post42, +} + /** * Read a text/binary preview file in a workflow Agent node sandbox */ @@ -3074,30 +3099,6 @@ export const read = { get: get58, } -/** - * Upload one workflow Agent sandbox file and return a signed download URL - */ -export const post42 = oc - .route({ - description: 'Upload one workflow Agent sandbox file and return a signed download URL', - inputStructure: 'detailed', - method: 'POST', - operationId: 'postAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesUpload', - path: '/apps/{app_id}/workflow-runs/{workflow_run_id}/agent-nodes/{node_id}/sandbox/files/upload', - tags: ['console'], - }) - .input( - z.object({ - body: zPostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesUploadBody, - params: zPostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesUploadPath, - }), - ) - .output(zPostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesUploadResponse) - -export const upload3 = { - post: post42, -} - /** * List a directory in a workflow Agent node sandbox */ @@ -3120,8 +3121,8 @@ export const get59 = oc export const files5 = { get: get59, + download: download5, read, - upload: upload3, } export const sandbox = { diff --git a/packages/contracts/generated/api/console/apps/types.gen.ts b/packages/contracts/generated/api/console/apps/types.gen.ts index c73766e3e0f..ff09e5e48f0 100644 --- a/packages/contracts/generated/api/console/apps/types.gen.ts +++ b/packages/contracts/generated/api/console/apps/types.gen.ts @@ -865,6 +865,15 @@ export type SandboxListResponse = { truncated?: boolean } +export type WorkflowAgentSandboxDownloadPayload = { + node_execution_id: string + path: string +} + +export type SandboxDownloadResponse = { + url: string +} + export type SandboxReadResponse = { binary: boolean path: string @@ -873,15 +882,6 @@ export type SandboxReadResponse = { truncated: boolean } -export type WorkflowAgentSandboxUploadPayload = { - node_execution_id: string - path: string -} - -export type SandboxUploadResponse = { - url: string -} - export type WorkflowCommentBasicList = { data: Array } @@ -5700,6 +5700,25 @@ export type GetAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFi export type GetAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesResponse = GetAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesResponses[keyof GetAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesResponses] +export type PostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesDownloadData = { + body: WorkflowAgentSandboxDownloadPayload + path: { + app_id: string + node_id: string + workflow_run_id: string + } + query?: never + url: '/apps/{app_id}/workflow-runs/{workflow_run_id}/agent-nodes/{node_id}/sandbox/files/download' +} + +export type PostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesDownloadResponses = + { + 200: SandboxDownloadResponse + } + +export type PostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesDownloadResponse = + PostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesDownloadResponses[keyof PostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesDownloadResponses] + export type GetAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesReadData = { body?: never path: { @@ -5721,25 +5740,6 @@ export type GetAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFi export type GetAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesReadResponse = GetAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesReadResponses[keyof GetAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesReadResponses] -export type PostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesUploadData = { - body: WorkflowAgentSandboxUploadPayload - path: { - app_id: string - node_id: string - workflow_run_id: string - } - query?: never - url: '/apps/{app_id}/workflow-runs/{workflow_run_id}/agent-nodes/{node_id}/sandbox/files/upload' -} - -export type PostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesUploadResponses = - { - 200: SandboxUploadResponse - } - -export type PostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesUploadResponse = - PostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesUploadResponses[keyof PostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesUploadResponses] - export type GetAppsByAppIdWorkflowCommentsData = { body?: never path: { diff --git a/packages/contracts/generated/api/console/apps/zod.gen.ts b/packages/contracts/generated/api/console/apps/zod.gen.ts index 2bc2afbc978..0693ecd3dbe 100644 --- a/packages/contracts/generated/api/console/apps/zod.gen.ts +++ b/packages/contracts/generated/api/console/apps/zod.gen.ts @@ -545,6 +545,21 @@ export const zWorkflowRunExportResponse = z.object({ status: z.string(), }) +/** + * WorkflowAgentSandboxDownloadPayload + */ +export const zWorkflowAgentSandboxDownloadPayload = z.object({ + node_execution_id: z.string().min(1), + path: z.string().min(1), +}) + +/** + * SandboxDownloadResponse + */ +export const zSandboxDownloadResponse = z.object({ + url: z.string(), +}) + /** * SandboxReadResponse */ @@ -556,21 +571,6 @@ export const zSandboxReadResponse = z.object({ truncated: z.boolean(), }) -/** - * WorkflowAgentSandboxUploadPayload - */ -export const zWorkflowAgentSandboxUploadPayload = z.object({ - node_execution_id: z.string().min(1), - path: z.string().min(1), -}) - -/** - * SandboxUploadResponse - */ -export const zSandboxUploadResponse = z.object({ - url: z.string(), -}) - /** * WorkflowCommentCreatePayload */ @@ -5960,6 +5960,22 @@ export const zGetAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandbox export const zGetAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesResponse = zSandboxListResponse +export const zPostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesDownloadBody = + zWorkflowAgentSandboxDownloadPayload + +export const zPostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesDownloadPath = + z.object({ + app_id: z.uuid(), + node_id: z.string(), + workflow_run_id: z.uuid(), + }) + +/** + * Download URL returned + */ +export const zPostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesDownloadResponse = + zSandboxDownloadResponse + export const zGetAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesReadPath = z.object({ app_id: z.uuid(), @@ -5979,22 +5995,6 @@ export const zGetAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandbox export const zGetAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesReadResponse = zSandboxReadResponse -export const zPostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesUploadBody = - zWorkflowAgentSandboxUploadPayload - -export const zPostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesUploadPath = - z.object({ - app_id: z.uuid(), - node_id: z.string(), - workflow_run_id: z.uuid(), - }) - -/** - * Uploaded - */ -export const zPostAppsByAppIdWorkflowRunsByWorkflowRunIdAgentNodesByNodeIdSandboxFilesUploadResponse = - zSandboxUploadResponse - export const zGetAppsByAppIdWorkflowCommentsPath = z.object({ app_id: z.uuid(), }) diff --git a/packages/contracts/sandbox-contract.smoke.test.ts b/packages/contracts/sandbox-contract.smoke.test.ts index c1e20b62905..b7f272b1ded 100644 --- a/packages/contracts/sandbox-contract.smoke.test.ts +++ b/packages/contracts/sandbox-contract.smoke.test.ts @@ -9,6 +9,7 @@ describe('generated sandbox contracts', () => { ])('exposes the %s file operations', (_, sandbox) => { expect(sandbox.files.get).toBeDefined() expect(sandbox.files.read.get).toBeDefined() - expect(sandbox.files.upload.post).toBeDefined() + expect(sandbox.files.download.post).toBeDefined() + expect('upload' in sandbox.files).toBe(false) }) }) diff --git a/web/app/components/workflow/nodes/agent-v2/components/__tests__/agent-orchestrate-panel-content.spec.tsx b/web/app/components/workflow/nodes/agent-v2/components/__tests__/agent-orchestrate-panel-content.spec.tsx index c43bf34f385..6f213095996 100644 --- a/web/app/components/workflow/nodes/agent-v2/components/__tests__/agent-orchestrate-panel-content.spec.tsx +++ b/web/app/components/workflow/nodes/agent-v2/components/__tests__/agent-orchestrate-panel-content.spec.tsx @@ -14,6 +14,8 @@ const mocks = vi.hoisted(() => ({ checkoutBuildDraft: vi.fn(), completeBuildConversation: undefined as (() => void) | undefined, deleteBuildDraft: vi.fn(), + downloadAgentSandboxFile: vi.fn(), + downloadWorkflowSandboxFile: vi.fn(), loadBuildDraft: vi.fn(), applyBuildDraft: vi.fn(), finalizeBuildChat: vi.fn(), @@ -333,6 +335,11 @@ vi.mock('@/service/client', async () => { }), }, }, + download: { + post: { + mutationOptions: () => ({ mutationFn: mocks.downloadAgentSandboxFile }), + }, + }, upload: { post: { mutationOptions: () => ({ mutationFn: mocks.uploadAgentSandboxFile }), @@ -377,6 +384,13 @@ vi.mock('@/service/client', async () => { }), }, }, + download: { + post: { + mutationOptions: () => ({ + mutationFn: mocks.downloadWorkflowSandboxFile, + }), + }, + }, upload: { post: { mutationOptions: () => ({ mutationFn: mocks.uploadWorkflowSandboxFile }), diff --git a/web/features/agent-v2/agent-detail/configure/components/preview/__tests__/working-directory-panel.spec.tsx b/web/features/agent-v2/agent-detail/configure/components/preview/__tests__/working-directory-panel.spec.tsx index 4f5dcfb924d..2ad21cf7832 100644 --- a/web/features/agent-v2/agent-detail/configure/components/preview/__tests__/working-directory-panel.spec.tsx +++ b/web/features/agent-v2/agent-detail/configure/components/preview/__tests__/working-directory-panel.spec.tsx @@ -1,3 +1,4 @@ +import type { AgentWorkingDirectorySource } from '../working-directory-panel' import { toast } from '@langgenius/dify-ui/toast' import { QueryClient, QueryClientProvider } from '@tanstack/react-query' import { render, screen, waitFor } from '@testing-library/react' @@ -17,32 +18,82 @@ const mocks = vi.hoisted(() => ({ sandboxInfoQueryOptions: vi.fn(), sandboxFilesQueryOptions: vi.fn(), sandboxFileReadQueryOptions: vi.fn(), - sandboxFileUploadMutationFn: vi.fn(async (_input: unknown) => ({ + sandboxFileDownloadMutationFn: vi.fn(async (_input: unknown) => ({ url: 'https://example.com/sandbox-file', })), workflowSandboxFilesQueryOptions: vi.fn(), workflowSandboxFileReadQueryOptions: vi.fn(), - workflowSandboxFileUploadMutationFn: vi.fn(async (_input: unknown) => ({ + workflowSandboxFileDownloadMutationFn: vi.fn(async (_input: unknown) => ({ url: 'https://example.com/workflow-sandbox-file', })), - sandboxFileUploadClientPost: vi.fn(async (_input: unknown) => ({ + sandboxFileDownloadClientPost: vi.fn(async (_input: unknown) => ({ url: 'https://example.com/chart.png', })), - workflowSandboxFileUploadClientPost: vi.fn(async (_input: unknown) => ({ + workflowSandboxFileDownloadClientPost: vi.fn(async (_input: unknown) => ({ url: 'https://example.com/workflow-chart.png', })), downloadUrl: vi.fn(), toastSuccess: vi.fn(), })) +const agentSource = { + type: 'agent', + agentId: 'agent-1', + callerType: 'conversation', + callerId: 'conversation-1', +} satisfies AgentWorkingDirectorySource + +const workflowSource = { + type: 'workflow-node', + appId: 'app-1', + workflowRunId: 'run-1', + nodeId: 'node-1', + nodeExecutionId: 'execution-1', +} satisfies AgentWorkingDirectorySource + +const previewSourceCases = [ + { + label: 'Agent', + source: agentSource, + identitySource: { + ...agentSource, + callerId: 'conversation-2', + } satisfies AgentWorkingDirectorySource, + imagePaths: ['workspace/chart-a.png', 'workspace/chart-b.png'], + nonImagePath: 'workspace/model.bin', + previewClient: mocks.sandboxFileDownloadClientPost, + urls: [ + 'https://example.com/agent-chart-a.png', + 'https://example.com/agent-chart-caller-b.png', + 'https://example.com/agent-chart-path-c.png', + ], + }, + { + label: 'Workflow', + source: workflowSource, + identitySource: { + ...workflowSource, + nodeExecutionId: 'execution-2', + } satisfies AgentWorkingDirectorySource, + imagePaths: ['chart-a.png', 'chart-b.png'], + nonImagePath: 'model.bin', + previewClient: mocks.workflowSandboxFileDownloadClientPost, + urls: [ + 'https://example.com/workflow-chart-a.png', + 'https://example.com/workflow-chart-execution-b.png', + 'https://example.com/workflow-chart-path-c.png', + ], + }, +] as const + vi.mock('@/service/client', () => ({ consoleClient: { agent: { byAgentId: { sandbox: { files: { - upload: { - post: mocks.sandboxFileUploadClientPost, + download: { + post: mocks.sandboxFileDownloadClientPost, }, }, }, @@ -56,8 +107,8 @@ vi.mock('@/service/client', () => ({ byNodeId: { sandbox: { files: { - upload: { - post: mocks.workflowSandboxFileUploadClientPost, + download: { + post: mocks.workflowSandboxFileDownloadClientPost, }, }, }, @@ -84,9 +135,9 @@ vi.mock('@/service/client', () => ({ queryOptions: mocks.sandboxFileReadQueryOptions, }, }, - upload: { + download: { post: { - mutationOptions: () => ({ mutationFn: mocks.sandboxFileUploadMutationFn }), + mutationOptions: () => ({ mutationFn: mocks.sandboxFileDownloadMutationFn }), }, }, }, @@ -109,10 +160,10 @@ vi.mock('@/service/client', () => ({ queryOptions: mocks.workflowSandboxFileReadQueryOptions, }, }, - upload: { + download: { post: { mutationOptions: () => ({ - mutationFn: mocks.workflowSandboxFileUploadMutationFn, + mutationFn: mocks.workflowSandboxFileDownloadMutationFn, }), }, }, @@ -139,57 +190,114 @@ vi.mock('@langgenius/dify-ui/toast', () => ({ function createDeferred() { let resolve!: (value: T) => void - const promise = new Promise((promiseResolve) => { + let reject!: (reason?: unknown) => void + const promise = new Promise((promiseResolve, promiseReject) => { resolve = promiseResolve + reject = promiseReject }) - return { promise, resolve } + return { promise, reject, resolve } } -function renderWorkingDirectoryPanel() { +type RenderWorkingDirectoryPanelOptions = { + open?: boolean + source?: AgentWorkingDirectorySource +} + +function renderWorkingDirectoryPanel({ + open = true, + source = agentSource, +}: RenderWorkingDirectoryPanelOptions = {}) { const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false }, }, }) - return render( + const rendered = render( - + , ) + return { + ...rendered, + rerenderPanel: (nextOptions: Required) => { + rendered.rerender( + + + , + ) + }, + } } function renderWorkflowWorkingDirectoryPanel() { - const queryClient = new QueryClient({ - defaultOptions: { - queries: { retry: false }, + return renderWorkingDirectoryPanel({ source: workflowSource }) +} + +function mockFileListEntries( + source: AgentWorkingDirectorySource, + entries: Array<{ name: string; type: 'file' }>, +) { + const queryOptions = + source.type === 'agent' + ? mocks.sandboxFilesQueryOptions + : mocks.workflowSandboxFilesQueryOptions + queryOptions.mockImplementation(({ input }: QueryOptionsInput) => ({ + queryKey: [`${source.type}-sandbox-files`, input, entries], + queryFn: async () => ({ + path: input.query?.path ?? (source.type === 'agent' ? '~/workspace' : '.'), + entries, + }), + })) +} + +function mockFileReadAsBinary(source: AgentWorkingDirectorySource) { + const queryOptions = + source.type === 'agent' + ? mocks.sandboxFileReadQueryOptions + : mocks.workflowSandboxFileReadQueryOptions + queryOptions.mockImplementation(({ input }: QueryOptionsInput) => ({ + queryKey: [`${source.type}-sandbox-file-read`, input], + queryFn: async () => ({ + binary: true, + path: input.query?.path ?? '', + text: null, + truncated: false, + }), + })) +} + +function expectedImagePreviewRequest(source: AgentWorkingDirectorySource, path: string) { + if (source.type === 'agent') { + return { + params: { agent_id: source.agentId }, + body: { + caller_type: source.callerType, + caller_id: source.callerId, + path: `~/${path}`, + }, + } + } + + return { + params: { + app_id: source.appId, + workflow_run_id: source.workflowRunId, + node_id: source.nodeId, }, - }) - const rendered = render( - - - , - ) - return { ...rendered, queryClient } + body: { + node_execution_id: source.nodeExecutionId, + path: `~/${path}`, + }, + } +} + +function fileName(path: string) { + return path.slice(path.lastIndexOf('/') + 1) } describe('AgentWorkingDirectoryPanel', () => { @@ -244,8 +352,8 @@ describe('AgentWorkingDirectoryPanel', () => { it('should download the selected working directory file from the preview header download action', async () => { const user = userEvent.setup() - const upload = createDeferred<{ url: string }>() - mocks.sandboxFileUploadMutationFn.mockReturnValueOnce(upload.promise) + const download = createDeferred<{ url: string }>() + mocks.sandboxFileDownloadMutationFn.mockReturnValueOnce(download.promise) renderWorkingDirectoryPanel() await user.click(await screen.findByText('notes.md')) @@ -259,12 +367,14 @@ describe('AgentWorkingDirectoryPanel', () => { name: /common\.operation\.downloading.*notes\.md/i, }) expect(downloadingButton.querySelector('.animate-spin')).toBeInTheDocument() + await user.click(downloadingButton) + expect(mocks.sandboxFileDownloadMutationFn).toHaveBeenCalledTimes(1) - upload.resolve({ url: 'https://example.com/sandbox-file' }) + download.resolve({ url: 'https://example.com/sandbox-file' }) await waitFor(() => { - expect(mocks.sandboxFileUploadMutationFn).toHaveBeenCalled() - expect(mocks.sandboxFileUploadMutationFn.mock.calls[0]?.[0]).toEqual({ + expect(mocks.sandboxFileDownloadMutationFn).toHaveBeenCalled() + expect(mocks.sandboxFileDownloadMutationFn.mock.calls[0]?.[0]).toEqual({ params: { agent_id: 'agent-1', }, @@ -284,8 +394,8 @@ describe('AgentWorkingDirectoryPanel', () => { it('should download binary working directory files from the unsupported preview download link', async () => { const user = userEvent.setup() - const upload = createDeferred<{ url: string }>() - mocks.sandboxFileUploadMutationFn.mockReturnValueOnce(upload.promise) + const download = createDeferred<{ url: string }>() + mocks.sandboxFileDownloadMutationFn.mockReturnValueOnce(download.promise) renderWorkingDirectoryPanel() await user.click(await screen.findByText('model.bin')) @@ -304,11 +414,11 @@ describe('AgentWorkingDirectoryPanel', () => { }) expect(headerDownloadButton.querySelector('.animate-spin')).not.toBeInTheDocument() - upload.resolve({ url: 'https://example.com/sandbox-file' }) + download.resolve({ url: 'https://example.com/sandbox-file' }) await waitFor(() => { - expect(mocks.sandboxFileUploadMutationFn).toHaveBeenCalled() - expect(mocks.sandboxFileUploadMutationFn.mock.calls[0]?.[0]).toEqual({ + expect(mocks.sandboxFileDownloadMutationFn).toHaveBeenCalled() + expect(mocks.sandboxFileDownloadMutationFn.mock.calls[0]?.[0]).toEqual({ params: { agent_id: 'agent-1', }, @@ -326,22 +436,22 @@ describe('AgentWorkingDirectoryPanel', () => { }) }) - it('should preview sandbox images with the uploaded file url', async () => { + it('should preview sandbox images with the downloaded file url', async () => { const user = userEvent.setup() - const upload = createDeferred<{ url: string }>() - mocks.sandboxFileUploadClientPost.mockReturnValueOnce(upload.promise) + const download = createDeferred<{ url: string }>() + mocks.sandboxFileDownloadClientPost.mockReturnValueOnce(download.promise) renderWorkingDirectoryPanel() await user.click(await screen.findByText('chart.png')) await waitFor(() => { - expect(mocks.sandboxFileUploadClientPost).toHaveBeenCalled() + expect(mocks.sandboxFileDownloadClientPost).toHaveBeenCalled() }) - upload.resolve({ url: 'https://example.com/chart.png' }) + download.resolve({ url: 'https://example.com/chart.png' }) const image = await screen.findByAltText('chart.png') expect(image).toHaveAttribute('src', 'https://example.com/chart.png') - expect(mocks.sandboxFileUploadClientPost).toHaveBeenCalledWith({ + expect(mocks.sandboxFileDownloadClientPost).toHaveBeenCalledWith({ params: { agent_id: 'agent-1', }, @@ -357,27 +467,186 @@ describe('AgentWorkingDirectoryPanel', () => { expect(mocks.downloadUrl).not.toHaveBeenCalled() }) - it('should scope workflow image previews to the exact node execution', async () => { - const { queryClient } = renderWorkflowWorkingDirectoryPanel() + it.each(previewSourceCases)( + 'should refresh $label image previews when caller ownership or path changes', + async ({ source, identitySource, imagePaths, previewClient, urls }) => { + const user = userEvent.setup() + const [initialImagePath, nextImagePath] = imagePaths + const [initialUrl, identityUrl, pathUrl] = urls + mockFileListEntries( + source, + imagePaths.map((name) => ({ name, type: 'file' })), + ) + previewClient + .mockResolvedValueOnce({ url: initialUrl }) + .mockResolvedValueOnce({ url: identityUrl }) + .mockResolvedValueOnce({ url: pathUrl }) + const { rerenderPanel } = renderWorkingDirectoryPanel({ source }) - await waitFor(() => { - expect(mocks.workflowSandboxFileUploadClientPost).toHaveBeenCalled() - }) + expect(await screen.findByAltText(fileName(initialImagePath))).toHaveAttribute( + 'src', + initialUrl, + ) + expect(previewClient).toHaveBeenCalledTimes(1) + + rerenderPanel({ open: true, source: identitySource }) + + await waitFor(() => { + expect(previewClient).toHaveBeenCalledTimes(2) + expect(screen.getByAltText(fileName(initialImagePath))).toHaveAttribute('src', identityUrl) + }) + expect(previewClient).toHaveBeenNthCalledWith( + 2, + expectedImagePreviewRequest(identitySource, initialImagePath), + ) + + await user.click(await screen.findByText(fileName(nextImagePath))) + + await waitFor(() => { + expect(previewClient).toHaveBeenCalledTimes(3) + expect(screen.getByAltText(fileName(nextImagePath))).toHaveAttribute('src', pathUrl) + }) + expect(previewClient).toHaveBeenNthCalledWith( + 3, + expectedImagePreviewRequest(identitySource, nextImagePath), + ) + }, + ) + + it.each(previewSourceCases)( + 'should disable $label image preview requests while closed and for non-images', + async ({ source, identitySource, imagePaths, nonImagePath, previewClient, urls }) => { + const [imagePath] = imagePaths + const [firstUrl] = urls + mockFileListEntries(source, [{ name: imagePath, type: 'file' }]) + previewClient.mockResolvedValueOnce({ url: firstUrl }) + const { rerenderPanel, unmount } = renderWorkingDirectoryPanel({ source }) + + expect(await screen.findByAltText(fileName(imagePath))).toHaveAttribute('src', firstUrl) + expect(previewClient).toHaveBeenCalledTimes(1) + previewClient.mockClear() + + rerenderPanel({ open: false, source: identitySource }) + await waitFor(() => { + expect(screen.queryByAltText(fileName(imagePath))).not.toBeInTheDocument() + }) + expect(previewClient).toHaveBeenCalledTimes(0) + unmount() + + mockFileListEntries(source, [{ name: nonImagePath, type: 'file' }]) + mockFileReadAsBinary(source) + renderWorkingDirectoryPanel({ source }) + expect( + await screen.findByText('agentV2.agentDetail.configure.files.preview.unsupported'), + ).toBeInTheDocument() + expect(previewClient).toHaveBeenCalledTimes(0) + }, + ) + + it('should download workflow files from the exact node execution', async () => { + const user = userEvent.setup() + const download = createDeferred<{ url: string }>() + mocks.workflowSandboxFileDownloadMutationFn.mockReturnValueOnce(download.promise) + renderWorkflowWorkingDirectoryPanel() + + await user.click( + await screen.findByRole('button', { + name: /common\.operation\.download.*chart\.png/i, + }), + ) expect( - queryClient.getQueryCache().find({ - queryKey: [ - 'agent-v2', - 'working-directory', - 'image-preview', - 'workflow-node', - 'app-1', - 'run-1', - 'node-1', - 'execution-1', - 'chart.png', - ], + await screen.findByRole('button', { + name: /common\.operation\.downloading.*chart\.png/i, }), - ).toBeDefined() + ).toBeInTheDocument() + await user.click( + screen.getByRole('button', { + name: /common\.operation\.downloading.*chart\.png/i, + }), + ) + expect(mocks.workflowSandboxFileDownloadMutationFn).toHaveBeenCalledTimes(1) + + download.resolve({ url: 'https://example.com/workflow-sandbox-file' }) + + await waitFor(() => { + expect(mocks.workflowSandboxFileDownloadMutationFn).toHaveBeenCalled() + expect(mocks.workflowSandboxFileDownloadMutationFn.mock.calls[0]?.[0]).toEqual({ + params: { + app_id: 'app-1', + workflow_run_id: 'run-1', + node_id: 'node-1', + }, + body: { + node_execution_id: 'execution-1', + path: '~/chart.png', + }, + }) + expect(mocks.downloadUrl).toHaveBeenCalledWith({ + url: 'https://example.com/workflow-sandbox-file', + fileName: 'chart.png', + }) + expect(toast.success).toHaveBeenCalledWith('common.operation.downloadSuccess') + }) + }) + + it('should clear Agent download pending state without reporting success after failure', async () => { + const user = userEvent.setup() + const download = createDeferred<{ url: string }>() + mocks.sandboxFileDownloadMutationFn.mockReturnValueOnce(download.promise) + renderWorkingDirectoryPanel() + + await user.click( + await screen.findByRole('button', { + name: /common\.operation\.download.*report\.md/i, + }), + ) + expect( + await screen.findByRole('button', { + name: /common\.operation\.downloading.*report\.md/i, + }), + ).toBeInTheDocument() + + download.reject(new Error('download failed')) + + await waitFor(() => { + expect( + screen.getByRole('button', { + name: /common\.operation\.download.*report\.md/i, + }), + ).toBeInTheDocument() + }) + expect(mocks.downloadUrl).not.toHaveBeenCalled() + expect(toast.success).not.toHaveBeenCalled() + }) + + it('should clear workflow download pending state without reporting success after failure', async () => { + const user = userEvent.setup() + const download = createDeferred<{ url: string }>() + mocks.workflowSandboxFileDownloadMutationFn.mockReturnValueOnce(download.promise) + renderWorkflowWorkingDirectoryPanel() + + await user.click( + await screen.findByRole('button', { + name: /common\.operation\.download.*chart\.png/i, + }), + ) + expect( + await screen.findByRole('button', { + name: /common\.operation\.downloading.*chart\.png/i, + }), + ).toBeInTheDocument() + + download.reject(new Error('download failed')) + + await waitFor(() => { + expect( + screen.getByRole('button', { + name: /common\.operation\.download.*chart\.png/i, + }), + ).toBeInTheDocument() + }) + expect(mocks.downloadUrl).not.toHaveBeenCalled() + expect(toast.success).not.toHaveBeenCalled() }) }) diff --git a/web/features/agent-v2/agent-detail/configure/components/preview/working-directory-panel.tsx b/web/features/agent-v2/agent-detail/configure/components/preview/working-directory-panel.tsx index f7c37395ea2..a30273d3f44 100644 --- a/web/features/agent-v2/agent-detail/configure/components/preview/working-directory-panel.tsx +++ b/web/features/agent-v2/agent-detail/configure/components/preview/working-directory-panel.tsx @@ -445,18 +445,18 @@ export function AgentWorkingDirectoryPanel({ }, retry: false, }) - const agentSandboxUploadMutation = useMutation( - consoleQuery.agent.byAgentId.sandbox.files.upload.post.mutationOptions(), + const agentSandboxDownloadMutation = useMutation( + consoleQuery.agent.byAgentId.sandbox.files.download.post.mutationOptions(), ) - const workflowSandboxUploadMutation = useMutation( - consoleQuery.apps.byAppId.workflowRuns.byWorkflowRunId.agentNodes.byNodeId.sandbox.files.upload.post.mutationOptions(), + const workflowSandboxDownloadMutation = useMutation( + consoleQuery.apps.byAppId.workflowRuns.byWorkflowRunId.agentNodes.byNodeId.sandbox.files.download.post.mutationOptions(), ) - const { mutateAsync: uploadAgentSandboxFile } = agentSandboxUploadMutation + const { mutateAsync: downloadAgentSandboxFile } = agentSandboxDownloadMutation const isImagePreviewFile = selectedWorkingDirectoryFile?.icon === 'image' const selectedWorkingDirectoryFilePath = selectedWorkingDirectoryFile?.id - const { mutateAsync: uploadWorkflowSandboxFile } = workflowSandboxUploadMutation + const { mutateAsync: downloadWorkflowSandboxFile } = workflowSandboxDownloadMutation const isFileDownloadPending = - agentSandboxUploadMutation.isPending || workflowSandboxUploadMutation.isPending + agentSandboxDownloadMutation.isPending || workflowSandboxDownloadMutation.isPending const isFileReadLoading = !!selectedWorkingDirectoryFile && !isImagePreviewFile && fileReadQuery.isPending const imagePreviewQuery = useQuery({ @@ -476,7 +476,7 @@ export function AgentWorkingDirectoryPanel({ throw new Error('Missing selected working directory file') if (source.type === 'agent') { - return consoleClient.agent.byAgentId.sandbox.files.upload.post({ + return consoleClient.agent.byAgentId.sandbox.files.download.post({ params: { agent_id: source.agentId, }, @@ -488,7 +488,7 @@ export function AgentWorkingDirectoryPanel({ }) } - return consoleClient.apps.byAppId.workflowRuns.byWorkflowRunId.agentNodes.byNodeId.sandbox.files.upload.post( + return consoleClient.apps.byAppId.workflowRuns.byWorkflowRunId.agentNodes.byNodeId.sandbox.files.download.post( { params: { app_id: source.appId, @@ -511,7 +511,7 @@ export function AgentWorkingDirectoryPanel({ if (source.type === 'agent') { setDownloadActionLoadingTarget(action) try { - const result = await uploadAgentSandboxFile({ + const result = await downloadAgentSandboxFile({ params: { agent_id: source.agentId, }, @@ -523,6 +523,8 @@ export function AgentWorkingDirectoryPanel({ }) downloadUrl({ url: result.url, fileName: selectedWorkingDirectoryFile.name }) toast.success(tCommon(($) => $['operation.downloadSuccess'])) + } catch { + // The generated client reports the mutation failure through its shared error handler. } finally { setDownloadActionLoadingTarget(null) } @@ -531,7 +533,7 @@ export function AgentWorkingDirectoryPanel({ setDownloadActionLoadingTarget(action) try { - const result = await uploadWorkflowSandboxFile({ + const result = await downloadWorkflowSandboxFile({ params: { app_id: source.appId, workflow_run_id: source.workflowRunId, @@ -544,6 +546,8 @@ export function AgentWorkingDirectoryPanel({ }) downloadUrl({ url: result.url, fileName: selectedWorkingDirectoryFile.name }) toast.success(tCommon(($) => $['operation.downloadSuccess'])) + } catch { + // The generated client reports the mutation failure through its shared error handler. } finally { setDownloadActionLoadingTarget(null) } @@ -553,8 +557,8 @@ export function AgentWorkingDirectoryPanel({ selectedWorkingDirectoryFile, source, tCommon, - uploadAgentSandboxFile, - uploadWorkflowSandboxFile, + downloadAgentSandboxFile, + downloadWorkflowSandboxFile, ], )