fix(agent): bound streams and cancel zombie workflow runs (#39186)

api/.env.example

@@ -677,6 +677,9 @@ INNER_API_KEY_FOR_PLUGIN=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y
 
 # Dify Agent backend
 AGENT_BACKEND_BASE_URL=http://localhost:5050
+AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS=30
+AGENT_BACKEND_STREAM_MAX_RECONNECTS=3
+AGENT_BACKEND_RUN_TIMEOUT_SECONDS=1200
 
 # Marketplace configuration
 MARKETPLACE_ENABLED=true

api/clients/agent_backend/client.py

@@ -8,7 +8,7 @@ creating another wire contract.
 
 from __future__ import annotations
 
-from collections.abc import Iterator
+from collections.abc import Callable, Iterator
 from typing import Protocol
 
 from dify_agent.client import (
@@ -45,7 +45,13 @@ class AgentBackendRunClient(Protocol):
     def cancel_run(self, run_id: str, request: CancelRunRequest | None = None) -> CancelRunResponse:
         """Request explicit cancellation for one Agent backend run."""
 
-    def stream_events(self, run_id: str, *, after: str | None = None) -> Iterator[RunEvent]:
+    def stream_events(
+        self,
+        run_id: str,
+        *,
+        after: str | None = None,
+        should_stop: Callable[[], bool] | None = None,
+    ) -> Iterator[RunEvent]:
         """Yield public ``dify-agent`` run events in stream order."""
 
     def wait_run(self, run_id: str, *, timeout_seconds: float | None = None) -> RunStatusResponse:
@@ -61,7 +67,15 @@ class _DifyAgentSyncClient(Protocol):
     def cancel_run_sync(self, run_id: str, request: CancelRunRequest | None = None) -> CancelRunResponse:
         """Cancel one run synchronously."""
 
-    def stream_events_sync(self, run_id: str, *, after: str | None = None) -> Iterator[RunEvent]:
+    def stream_events_sync(
+        self,
+        run_id: str,
+        *,
+        after: str | None = None,
+        max_reconnects: int | None = None,
+        timeout_seconds: float | None = None,
+        should_stop: Callable[[], bool] | None = None,
+    ) -> Iterator[RunEvent]:
         """Stream run events synchronously."""
 
     def wait_run_sync(self, run_id: str, *, timeout_seconds: float | None = None) -> RunStatusResponse:
@@ -73,8 +87,16 @@ class DifyAgentBackendRunClient:
 
     client: _DifyAgentSyncClient
 
-    def __init__(self, client: _DifyAgentSyncClient) -> None:
+    def __init__(
+        self,
+        client: _DifyAgentSyncClient,
+        *,
+        stream_max_reconnects: int = 3,
+        stream_timeout_seconds: float = 1200,
+    ) -> None:
         self.client = client
+        self._stream_max_reconnects = stream_max_reconnects
+        self._stream_timeout_seconds = stream_timeout_seconds
 
     def create_run(self, request: CreateRunRequest) -> CreateRunResponse:
         """Create one run through ``POST /runs`` and normalize client exceptions."""
@@ -90,10 +112,22 @@ class DifyAgentBackendRunClient:
         except Exception as exc:
             raise _normalize_dify_agent_error(exc) from exc
 
-    def stream_events(self, run_id: str, *, after: str | None = None) -> Iterator[RunEvent]:
+    def stream_events(
+        self,
+        run_id: str,
+        *,
+        after: str | None = None,
+        should_stop: Callable[[], bool] | None = None,
+    ) -> Iterator[RunEvent]:
         """Stream run events from ``/events/sse`` with the wrapped client's reconnect policy."""
         try:
-            yield from self.client.stream_events_sync(run_id, after=after)
+            yield from self.client.stream_events_sync(
+                run_id,
+                after=after,
+                max_reconnects=self._stream_max_reconnects,
+                timeout_seconds=self._stream_timeout_seconds,
+                should_stop=should_stop,
+            )
         except Exception as exc:
             raise _normalize_dify_agent_error(exc) from exc
 

api/clients/agent_backend/factory.py

@@ -13,10 +13,17 @@ def create_agent_backend_run_client(
     base_url: str | None = None,
     use_fake: bool = False,
     fake_scenario: str | FakeAgentBackendScenario = FakeAgentBackendScenario.SUCCESS,
+    stream_read_timeout_seconds: float = 30,
+    stream_max_reconnects: int = 3,
+    stream_run_timeout_seconds: float = 1200,
 ) -> AgentBackendRunClient:
     """Create the API-side run client without hiding the ``dify-agent`` protocol."""
     if use_fake:
         return FakeAgentBackendRunClient(scenario=FakeAgentBackendScenario(fake_scenario))
     if base_url is None:
         raise ValueError("base_url is required when creating a real Agent backend client")
-    return DifyAgentBackendRunClient(Client(base_url=base_url))
+    return DifyAgentBackendRunClient(
+        Client(base_url=base_url, stream_timeout=stream_read_timeout_seconds),
+        stream_max_reconnects=stream_max_reconnects,
+        stream_timeout_seconds=stream_run_timeout_seconds,
+    )

api/clients/agent_backend/fake_client.py

@@ -7,7 +7,7 @@ separate ``agent-backend.v1`` event stream.
 
 from __future__ import annotations
 
-from collections.abc import Iterator
+from collections.abc import Callable, Iterator
 from datetime import UTC, datetime
 from enum import StrEnum
 
@@ -69,9 +69,17 @@ class FakeAgentBackendRunClient:
         del request
         return CancelRunResponse(run_id=run_id, status="cancelled")
 
-    def stream_events(self, run_id: str, *, after: str | None = None) -> Iterator[RunEvent]:
+    def stream_events(
+        self,
+        run_id: str,
+        *,
+        after: str | None = None,
+        should_stop: Callable[[], bool] | None = None,
+    ) -> Iterator[RunEvent]:
         """Yield the deterministic public ``RunEvent`` sequence for ``run_id``."""
         for event in self._events(run_id):
+            if should_stop is not None and should_stop():
+                return
             if after is not None and event.id is not None and event.id <= after:
                 continue
             yield event

api/configs/extra/agent_backend_config.py

@@ -1,4 +1,4 @@
-from pydantic import Field, NonNegativeFloat
+from pydantic import Field, NonNegativeFloat, NonNegativeInt, PositiveFloat
 from pydantic_settings import BaseSettings
 
 
@@ -22,6 +22,21 @@ class AgentBackendConfig(BaseSettings):
         default="success",
     )
 
+    AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS: PositiveFloat = Field(
+        description="Read timeout for one Agent backend SSE connection.",
+        default=30,
+    )
+
+    AGENT_BACKEND_STREAM_MAX_RECONNECTS: NonNegativeInt = Field(
+        description="Maximum Agent backend SSE reconnects before failing the run.",
+        default=3,
+    )
+
+    AGENT_BACKEND_RUN_TIMEOUT_SECONDS: PositiveFloat = Field(
+        description="Total deadline for one Agent backend run event stream.",
+        default=1200,
+    )
+
     AGENT_SHELL_ENABLED: bool = Field(
         description=(
             "Inject the dify.shell layer (sandboxed bash workspace) into Agent runs. "

api/core/app/apps/agent_app/app_generator.py

@@ -540,6 +540,9 @@ class AgentAppGenerator(MessageBasedAppGenerator):
                 base_url=dify_config.AGENT_BACKEND_BASE_URL,
                 use_fake=dify_config.AGENT_BACKEND_USE_FAKE,
                 fake_scenario=dify_config.AGENT_BACKEND_FAKE_SCENARIO,
+                stream_read_timeout_seconds=dify_config.AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS,
+                stream_max_reconnects=dify_config.AGENT_BACKEND_STREAM_MAX_RECONNECTS,
+                stream_run_timeout_seconds=dify_config.AGENT_BACKEND_RUN_TIMEOUT_SECONDS,
             ),
             event_adapter=AgentBackendRunEventAdapter(),
             session_store=AgentAppRuntimeSessionStore(),

api/core/app/apps/agent_app/app_runner.py

@@ -941,48 +941,64 @@ class AgentAppRunner:
             if pending_text:
                 persist_answer_text(pending_text)
 
-        for public_event in self._agent_backend_client.stream_events(run_id):
-            if queue_manager.is_stopped():
-                flush_pending_agent_message_text()
-                self._cancel_run(run_id)
-                raise GenerateTaskStoppedError()
-            for internal_event in self._event_adapter.adapt(public_event):
+        try:
+            public_events = self._agent_backend_client.stream_events(
+                run_id,
+                should_stop=queue_manager.is_stopped,
+            )
+            for public_event in public_events:
                 if queue_manager.is_stopped():
                     flush_pending_agent_message_text()
                     self._cancel_run(run_id)
                     raise GenerateTaskStoppedError()
-                if internal_event.type in (
-                    AgentBackendInternalEventType.RUN_STARTED,
-                    AgentBackendInternalEventType.STREAM_EVENT,
-                    AgentBackendInternalEventType.AGENT_MESSAGE_DELTA,
-                ):
-                    if isinstance(internal_event, AgentBackendAgentMessageDeltaInternalEvent):
-                        debounced_delta = text_delta_debouncer.push(internal_event.delta)
-                        if debounced_delta:
-                            persist_answer_text(debounced_delta)
-                        continue
-
-                    if isinstance(internal_event, AgentBackendStreamInternalEvent):
+                for internal_event in self._event_adapter.adapt(public_event):
+                    if queue_manager.is_stopped():
                         flush_pending_agent_message_text()
-                        try:
-                            process_recorder.handle_stream_event(internal_event)
-                        except Exception:
-                            db.session.rollback()
-                            logger.warning(
-                                "Failed to persist Agent App process event: run_id=%s message_id=%s event_kind=%s",
-                                run_id,
-                                message_id,
-                                internal_event.event_kind,
-                                exc_info=True,
-                            )
+                        self._cancel_run(run_id)
+                        raise GenerateTaskStoppedError()
+                    if internal_event.type in (
+                        AgentBackendInternalEventType.RUN_STARTED,
+                        AgentBackendInternalEventType.STREAM_EVENT,
+                        AgentBackendInternalEventType.AGENT_MESSAGE_DELTA,
+                    ):
+                        if isinstance(internal_event, AgentBackendAgentMessageDeltaInternalEvent):
+                            debounced_delta = text_delta_debouncer.push(internal_event.delta)
+                            if debounced_delta:
+                                persist_answer_text(debounced_delta)
+                            continue
+
+                        if isinstance(internal_event, AgentBackendStreamInternalEvent):
+                            flush_pending_agent_message_text()
+                            try:
+                                process_recorder.handle_stream_event(internal_event)
+                            except Exception:
+                                db.session.rollback()
+                                logger.warning(
+                                    "Failed to persist Agent App process event: run_id=%s message_id=%s event_kind=%s",
+                                    run_id,
+                                    message_id,
+                                    internal_event.event_kind,
+                                    exc_info=True,
+                                )
+                            continue
                         continue
-                    continue
-                flush_pending_agent_message_text()
-                terminal = internal_event
-                break
-            if terminal is not None:
-                break
+                    flush_pending_agent_message_text()
+                    terminal = internal_event
+                    break
+                if terminal is not None:
+                    break
+        except GenerateTaskStoppedError:
+            raise
+        except Exception as error:
+            flush_pending_agent_message_text()
+            self._cancel_run(run_id)
+            if queue_manager.is_stopped():
+                raise GenerateTaskStoppedError() from error
+            raise
         flush_pending_agent_message_text()
+        if queue_manager.is_stopped():
+            self._cancel_run(run_id)
+            raise GenerateTaskStoppedError()
         return terminal, process_recorder
 
     def _cancel_run(self, run_id: str) -> None:

api/core/app/apps/base_app_queue_manager.py

@@ -21,6 +21,7 @@ from core.app.entities.queue_entities import (
     WorkflowQueueMessage,
 )
 from extensions.ext_redis import redis_client
+from graphon.graph_engine.manager import GraphEngineManager
 from graphon.runtime import GraphRuntimeState
 
 logger = logging.getLogger(__name__)
@@ -51,6 +52,9 @@ class AppQueueManager(ABC):
         self._graph_runtime_state: GraphRuntimeState | None = None
         self._stopped_cache: TTLCache[tuple, bool] = TTLCache(maxsize=1, ttl=1)
         self._cache_lock = threading.Lock()
+        self._execution_terminal = threading.Event()
+        self._abort_sent = threading.Event()
+        self._lifecycle_lock = threading.Lock()
 
     def listen(self):
         """
@@ -59,7 +63,7 @@ class AppQueueManager(ABC):
         """
         # wait for APP_MAX_EXECUTION_TIME seconds to stop listen
         listen_timeout = dify_config.APP_MAX_EXECUTION_TIME
-        start_time = time.time()
+        start_time = time.monotonic()
         last_ping_time: int | float = 0
         try:
             while True:
@@ -72,8 +76,14 @@ class AppQueueManager(ABC):
                 except queue.Empty:
                     continue
                 finally:
-                    elapsed_time = time.time() - start_time
-                    if elapsed_time >= listen_timeout or self._is_stopped():
+                    elapsed_time = time.monotonic() - start_time
+                    timed_out = elapsed_time >= listen_timeout
+                    manually_stopped = self._is_stopped()
+                    if not self._execution_terminal.is_set() and (timed_out or manually_stopped):
+                        reason = (
+                            f"App execution exceeded {listen_timeout} seconds" if timed_out else "App task was stopped"
+                        )
+                        self._abort_execution(reason)
                         # publish two messages to make sure the client can receive the stop signal
                         # and stop listening after the stop signal processed
                         self.publish(
@@ -84,16 +94,33 @@ class AppQueueManager(ABC):
                         self.publish(QueuePingEvent(), PublishFrom.TASK_PIPELINE)
                         last_ping_time = elapsed_time // 10
         finally:
+            if not self._execution_terminal.is_set():
+                self._abort_execution("Client response stream closed before app execution completed")
             self._graph_runtime_state = None  # Release reference once consumers finish or close the generator.
 
-    def stop_listen(self):
+    def stop_listen(self, *, execution_terminal: bool = False):
         """
         Stop listen to queue
         :return:
         """
+        if execution_terminal:
+            self._execution_terminal.set()
         self._clear_task_belong_cache()
         self._q.put(None)
 
+    def _abort_execution(self, reason: str) -> None:
+        """Propagate response timeout/disconnect to legacy and GraphEngine runners."""
+        with self._lifecycle_lock:
+            if self._execution_terminal.is_set() or self._abort_sent.is_set():
+                return
+            self._abort_sent.set()
+
+        try:
+            self.set_stop_flag_no_user_check(self._task_id)
+            GraphEngineManager(redis_client).send_stop_command(self._task_id, reason=reason)
+        except Exception:
+            logger.exception("Failed to abort app execution for task %s", self._task_id)
+
     def _clear_task_belong_cache(self) -> None:
         """
         Remove the task belong cache key once listening is finished.

api/core/app/apps/message_based_app_queue_manager.py

@@ -45,7 +45,7 @@ class MessageBasedAppQueueManager(AppQueueManager):
         if isinstance(
             event, QueueStopEvent | QueueErrorEvent | QueueMessageEndEvent | QueueAdvancedChatMessageEndEvent
         ):
-            self.stop_listen()
+            self.stop_listen(execution_terminal=True)
 
         if pub_from == PublishFrom.APPLICATION_MANAGER and self._is_stopped():
             if self._app_mode == AppMode.ADVANCED_CHAT.value:

api/core/app/apps/pipeline/pipeline_queue_manager.py

@@ -42,7 +42,7 @@ class PipelineQueueManager(AppQueueManager):
             | QueueWorkflowFailedEvent
             | QueueWorkflowPartialSuccessEvent,
         ):
-            self.stop_listen()
+            self.stop_listen(execution_terminal=True)
 
         if pub_from == PublishFrom.APPLICATION_MANAGER and self._is_stopped():
             raise GenerateTaskStoppedError()

api/core/app/apps/workflow/app_queue_manager.py

@@ -41,4 +41,4 @@ class WorkflowAppQueueManager(AppQueueManager):
             | QueueWorkflowFailedEvent
             | QueueWorkflowPartialSuccessEvent,
         ):
-            self.stop_listen()
+            self.stop_listen(execution_terminal=True)

api/core/app/apps/workflow_app_runner.py

@@ -26,6 +26,7 @@ from core.app.entities.queue_entities import (
     QueueNodeSucceededEvent,
     QueueReasoningChunkEvent,
     QueueRetrieverResourcesEvent,
+    QueueStopEvent,
     QueueTextChunkEvent,
     QueueWorkflowFailedEvent,
     QueueWorkflowPartialSuccessEvent,
@@ -424,7 +425,12 @@ class WorkflowBasedAppRunner:
                     QueueWorkflowFailedEvent(error=event.error, exceptions_count=event.exceptions_count)
                 )
             case GraphRunAbortedEvent():
-                self._publish_event(QueueWorkflowFailedEvent(error=event.reason or "Unknown error", exceptions_count=0))
+                self._publish_event(
+                    QueueStopEvent(
+                        stopped_by=QueueStopEvent.StopBy.USER_MANUAL,
+                        reason=event.reason or "Workflow execution aborted",
+                    )
+                )
             case GraphRunPausedEvent():
                 runtime_state = workflow_entry.graph_engine.graph_runtime_state
                 paused_nodes = runtime_state.get_paused_nodes()

api/core/app/entities/queue_entities.py

@@ -500,11 +500,15 @@ class QueueStopEvent(AppQueueEvent):
 
     event: QueueEvent = QueueEvent.STOP
     stopped_by: StopBy
+    reason: str | None = None
 
     def get_stop_reason(self) -> str:
         """
         To stop reason
         """
+        if self.reason:
+            return self.reason
+
         reason_mapping = {
             QueueStopEvent.StopBy.USER_MANUAL: "Stopped by user.",
             QueueStopEvent.StopBy.ANNOTATION_REPLY: "Stopped by annotation reply.",

api/core/workflow/node_factory.py

@@ -499,6 +499,9 @@ class DifyNodeFactory(NodeFactory):
                     base_url=dify_config.AGENT_BACKEND_BASE_URL,
                     use_fake=dify_config.AGENT_BACKEND_USE_FAKE,
                     fake_scenario=dify_config.AGENT_BACKEND_FAKE_SCENARIO,
+                    stream_read_timeout_seconds=dify_config.AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS,
+                    stream_max_reconnects=dify_config.AGENT_BACKEND_STREAM_MAX_RECONNECTS,
+                    stream_run_timeout_seconds=dify_config.AGENT_BACKEND_RUN_TIMEOUT_SECONDS,
                 ),
                 "event_adapter": AgentBackendRunEventAdapter(),
                 # Agent Files §4.6: reback file outputs from the ToolFile row so

api/core/workflow/nodes/agent_v2/agent_node.py

@@ -5,6 +5,7 @@ from collections.abc import Generator, Mapping, Sequence
 from typing import TYPE_CHECKING, Any, override
 
 from agenton.compositor import CompositorSessionSnapshot
+from dify_agent.protocol import CancelRunRequest
 
 from clients.agent_backend import (
     AgentBackendAgentMessageDeltaInternalEvent,
@@ -473,7 +474,10 @@ class DifyAgentNode(Node[DifyAgentNodeData]):
         """
         stream_event_count = 0
         try:
-            for public_event in self._agent_backend_client.stream_events(run_id):
+            for public_event in self._agent_backend_client.stream_events(
+                run_id,
+                should_stop=self._is_graph_aborted,
+            ):
                 stream_event_count += 1
                 for internal_event in self._event_adapter.adapt(public_event):
                     if internal_event.type == AgentBackendInternalEventType.RUN_STARTED:
@@ -501,6 +505,7 @@ class DifyAgentNode(Node[DifyAgentNodeData]):
                         | AgentBackendDeferredToolCallInternalEvent,
                     ):
                         return internal_event, None
+                    self._cancel_backend_run(run_id, reason="unexpected_event")
                     return None, self._failure_event(
                         inputs={},
                         process_data={},
@@ -509,6 +514,7 @@ class DifyAgentNode(Node[DifyAgentNodeData]):
                         error_type="agent_backend_stream_error",
                     )
         except AgentBackendError as error:
+            self._cancel_backend_run(run_id, reason=self._stream_stop_reason())
             return None, self._failure_event(
                 inputs={},
                 process_data={},
@@ -517,6 +523,7 @@ class DifyAgentNode(Node[DifyAgentNodeData]):
                 error_type=self._agent_backend_error_type(error),
             )
         except Exception as error:
+            self._cancel_backend_run(run_id, reason=self._stream_stop_reason())
             return None, self._failure_event(
                 inputs={},
                 process_data={},
@@ -525,8 +532,28 @@ class DifyAgentNode(Node[DifyAgentNodeData]):
                 error_type="agent_backend_stream_error",
             )
 
+        self._cancel_backend_run(run_id, reason="stream_ended_without_terminal_event")
         return None, None
 
+    def _is_graph_aborted(self) -> bool:
+        """Let Agent SSE consumption observe GraphEngine's cooperative abort state."""
+        try:
+            return self.graph_runtime_state.graph_execution.aborted
+        except (AttributeError, RuntimeError):
+            return False
+
+    def _stream_stop_reason(self) -> str:
+        return "workflow_graph_aborted" if self._is_graph_aborted() else "event_stream_failed"
+
+    def _cancel_backend_run(self, run_id: str, *, reason: str) -> None:
+        try:
+            self._agent_backend_client.cancel_run(
+                run_id,
+                CancelRunRequest(reason=reason, message="Workflow Agent event consumption stopped"),
+            )
+        except Exception:
+            logger.warning("Failed to cancel Workflow Agent backend run: run_id=%s", run_id, exc_info=True)
+
     @staticmethod
     def _record_type_check_metadata(metadata: dict[str, Any], outcome: OutputTypeCheckOutcome) -> None:
         # Surface enough detail in metadata for Inspector / debug logs without

api/tasks/agent_backend_session_cleanup_task.py

@@ -24,6 +24,9 @@ def _create_agent_backend_client():
         base_url=dify_config.AGENT_BACKEND_BASE_URL,
         use_fake=dify_config.AGENT_BACKEND_USE_FAKE,
         fake_scenario=dify_config.AGENT_BACKEND_FAKE_SCENARIO,
+        stream_read_timeout_seconds=dify_config.AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS,
+        stream_max_reconnects=dify_config.AGENT_BACKEND_STREAM_MAX_RECONNECTS,
+        stream_run_timeout_seconds=dify_config.AGENT_BACKEND_RUN_TIMEOUT_SECONDS,
     )
 
 

api/tests/unit_tests/clients/agent_backend/test_client.py

@@ -1,4 +1,4 @@
-from collections.abc import Iterator
+from collections.abc import Callable, Iterator
 from typing import override
 
 import pytest
@@ -47,6 +47,8 @@ def _request():
 
 
 class _SuccessfulClient:
+    stream_options: tuple[int | None, float | None, Callable[[], bool] | None] | None = None
+
     def create_run_sync(self, request: CreateRunRequest) -> CreateRunResponse:
         assert isinstance(request, CreateRunRequest)
         return CreateRunResponse(run_id="run-1", status="running")
@@ -55,8 +57,17 @@ class _SuccessfulClient:
         del request
         return CancelRunResponse(run_id=run_id, status="cancelled")
 
-    def stream_events_sync(self, run_id: str, *, after: str | None = None) -> Iterator[RunEvent]:
+    def stream_events_sync(
+        self,
+        run_id: str,
+        *,
+        after: str | None = None,
+        max_reconnects: int | None = None,
+        timeout_seconds: float | None = None,
+        should_stop: Callable[[], bool] | None = None,
+    ) -> Iterator[RunEvent]:
         del after
+        self.stream_options = (max_reconnects, timeout_seconds, should_stop)
         yield RunStartedEvent(id="1-0", run_id=run_id)
 
     def wait_run_sync(self, run_id: str, *, timeout_seconds: float | None = None) -> RunStatusResponse:
@@ -72,17 +83,22 @@ class _SuccessfulClient:
 
 
 def test_dify_agent_backend_run_client_delegates_sync_methods():
-    client = DifyAgentBackendRunClient(_SuccessfulClient())
+    wrapped = _SuccessfulClient()
+    client = DifyAgentBackendRunClient(wrapped, stream_max_reconnects=2, stream_timeout_seconds=45)
+
+    def should_stop() -> bool:
+        return False
 
     created = client.create_run(_request())
     cancelled = client.cancel_run(created.run_id)
-    events = list(client.stream_events(created.run_id))
+    events = list(client.stream_events(created.run_id, should_stop=should_stop))
     status = client.wait_run(created.run_id)
 
     assert created.run_id == "run-1"
     assert cancelled.status == "cancelled"
     assert events[0].type == "run_started"
     assert status.status == "succeeded"
+    assert wrapped.stream_options == (2, 45, should_stop)
 
 
 def test_dify_agent_backend_run_client_maps_validation_error():
@@ -125,7 +141,16 @@ def test_dify_agent_backend_run_client_maps_timeout_error():
 def test_dify_agent_backend_run_client_maps_stream_error():
     class StreamClient(_SuccessfulClient):
         @override
-        def stream_events_sync(self, run_id: str, *, after: str | None = None) -> Iterator[RunEvent]:
+        def stream_events_sync(
+            self,
+            run_id: str,
+            *,
+            after: str | None = None,
+            max_reconnects: int | None = None,
+            timeout_seconds: float | None = None,
+            should_stop: Callable[[], bool] | None = None,
+        ) -> Iterator[RunEvent]:
+            del run_id, after, max_reconnects, timeout_seconds, should_stop
             raise DifyAgentStreamError("bad stream")
             yield
 

api/tests/unit_tests/core/app/apps/agent_app/test_app_runner.py

@@ -4,7 +4,7 @@ saved, using the deterministic fake backend client (no live stack)."""
 
 from __future__ import annotations
 
-from collections.abc import Iterator
+from collections.abc import Callable, Iterator
 from datetime import UTC, datetime
 from types import SimpleNamespace
 from typing import Any, override
@@ -106,8 +106,14 @@ class _RecordingFakeAgentBackendRunClient(FakeAgentBackendRunClient):
 
 class _StreamingFakeAgentBackendRunClient(FakeAgentBackendRunClient):
     @override
-    def stream_events(self, run_id: str, *, after: str | None = None) -> Iterator[RunEvent]:
-        del after
+    def stream_events(
+        self,
+        run_id: str,
+        *,
+        after: str | None = None,
+        should_stop: Callable[[], bool] | None = None,
+    ) -> Iterator[RunEvent]:
+        del after, should_stop
         created_at = datetime(2026, 1, 1, tzinfo=UTC)
         yield RunStartedEvent(id="1-0", run_id=run_id, created_at=created_at)
         yield PydanticAIStreamRunEvent(
@@ -138,8 +144,14 @@ class _StreamingFakeAgentBackendRunClient(FakeAgentBackendRunClient):
 
 class _StreamingRecordingFakeAgentBackendRunClient(_RecordingFakeAgentBackendRunClient):
     @override
-    def stream_events(self, run_id: str, *, after: str | None = None) -> Iterator[RunEvent]:
-        del after
+    def stream_events(
+        self,
+        run_id: str,
+        *,
+        after: str | None = None,
+        should_stop: Callable[[], bool] | None = None,
+    ) -> Iterator[RunEvent]:
+        del after, should_stop
         created_at = datetime(2026, 1, 1, tzinfo=UTC)
         yield RunStartedEvent(id="1-0", run_id=run_id, created_at=created_at)
         yield PydanticAIStreamRunEvent(
@@ -173,8 +185,14 @@ class _StreamingStopAfterFirstDeltaFakeAgentBackendRunClient(_RecordingFakeAgent
         self._queue_manager = queue_manager
 
     @override
-    def stream_events(self, run_id: str, *, after: str | None = None) -> Iterator[RunEvent]:
-        del after
+    def stream_events(
+        self,
+        run_id: str,
+        *,
+        after: str | None = None,
+        should_stop: Callable[[], bool] | None = None,
+    ) -> Iterator[RunEvent]:
+        del after, should_stop
         created_at = datetime(2026, 1, 1, tzinfo=UTC)
         yield RunStartedEvent(id="1-0", run_id=run_id, created_at=created_at)
         yield PydanticAIStreamRunEvent(
@@ -196,8 +214,14 @@ class _StreamingStopAfterFirstDeltaFakeAgentBackendRunClient(_RecordingFakeAgent
 
 class _StreamingSingleAgentMessageDeltaFakeAgentBackendRunClient(FakeAgentBackendRunClient):
     @override
-    def stream_events(self, run_id: str, *, after: str | None = None) -> Iterator[RunEvent]:
-        del after
+    def stream_events(
+        self,
+        run_id: str,
+        *,
+        after: str | None = None,
+        should_stop: Callable[[], bool] | None = None,
+    ) -> Iterator[RunEvent]:
+        del after, should_stop
         created_at = datetime(2026, 1, 1, tzinfo=UTC)
         yield RunStartedEvent(id="1-0", run_id=run_id, created_at=created_at)
         yield PydanticAIStreamRunEvent(
@@ -220,8 +244,14 @@ class _StreamingSingleAgentMessageDeltaFakeAgentBackendRunClient(FakeAgentBacken
 
 class _NullOutputFakeAgentBackendRunClient(FakeAgentBackendRunClient):
     @override
-    def stream_events(self, run_id: str, *, after: str | None = None) -> Iterator[RunEvent]:
-        del after
+    def stream_events(
+        self,
+        run_id: str,
+        *,
+        after: str | None = None,
+        should_stop: Callable[[], bool] | None = None,
+    ) -> Iterator[RunEvent]:
+        del after, should_stop
         created_at = datetime(2026, 1, 1, tzinfo=UTC)
         yield RunStartedEvent(id="1-0", run_id=run_id, created_at=created_at)
         yield RunSucceededEvent(
@@ -237,8 +267,14 @@ class _NullOutputFakeAgentBackendRunClient(FakeAgentBackendRunClient):
 
 class _StreamingTextNullOutputFakeAgentBackendRunClient(FakeAgentBackendRunClient):
     @override
-    def stream_events(self, run_id: str, *, after: str | None = None) -> Iterator[RunEvent]:
-        del after
+    def stream_events(
+        self,
+        run_id: str,
+        *,
+        after: str | None = None,
+        should_stop: Callable[[], bool] | None = None,
+    ) -> Iterator[RunEvent]:
+        del after, should_stop
         created_at = datetime(2026, 1, 1, tzinfo=UTC)
         yield RunStartedEvent(id="1-0", run_id=run_id, created_at=created_at)
         yield PydanticAIStreamRunEvent(
@@ -261,8 +297,14 @@ class _StreamingTextNullOutputFakeAgentBackendRunClient(FakeAgentBackendRunClien
 
 class _AgentAnswerStreamingFakeAgentBackendRunClient(FakeAgentBackendRunClient):
     @override
-    def stream_events(self, run_id: str, *, after: str | None = None) -> Iterator[RunEvent]:
-        del after
+    def stream_events(
+        self,
+        run_id: str,
+        *,
+        after: str | None = None,
+        should_stop: Callable[[], bool] | None = None,
+    ) -> Iterator[RunEvent]:
+        del after, should_stop
         created_at = datetime(2026, 1, 1, tzinfo=UTC)
         yield RunStartedEvent(id="1-0", run_id=run_id, created_at=created_at)
         yield PydanticAIStreamRunEvent(
@@ -292,8 +334,14 @@ class _AgentAnswerStreamingFakeAgentBackendRunClient(FakeAgentBackendRunClient):
 
 class _ProcessStreamingFakeAgentBackendRunClient(FakeAgentBackendRunClient):
     @override
-    def stream_events(self, run_id: str, *, after: str | None = None) -> Iterator[RunEvent]:
-        del after
+    def stream_events(
+        self,
+        run_id: str,
+        *,
+        after: str | None = None,
+        should_stop: Callable[[], bool] | None = None,
+    ) -> Iterator[RunEvent]:
+        del after, should_stop
         created_at = datetime(2026, 1, 1, tzinfo=UTC)
         yield RunStartedEvent(id="1-0", run_id=run_id, created_at=created_at)
         yield PydanticAIStreamRunEvent(

api/tests/unit_tests/core/app/apps/test_base_app_queue_manager.py

@@ -61,16 +61,37 @@ class TestBaseAppQueueManager:
             manager._check_for_sqlalchemy_models(bad)
 
     def test_stop_listen_defers_graph_runtime_state_cleanup_until_listener_exits(self):
-        with patch("core.app.apps.base_app_queue_manager.redis_client") as mock_redis:
+        with (
+            patch("core.app.apps.base_app_queue_manager.redis_client") as mock_redis,
+            patch("core.app.apps.base_app_queue_manager.GraphEngineManager") as graph_engine_manager,
+        ):
             mock_redis.setex.return_value = True
             mock_redis.get.return_value = None
             manager = DummyQueueManager(task_id="t1", user_id="u1", invoke_from=InvokeFrom.SERVICE_API)
+            runtime_state = SimpleNamespace(name="runtime-state")
+            manager.graph_runtime_state = runtime_state
 
-        runtime_state = SimpleNamespace(name="runtime-state")
-        manager.graph_runtime_state = runtime_state
+            manager.stop_listen()
 
-        manager.stop_listen()
+            assert manager.graph_runtime_state is runtime_state
+            assert list(manager.listen()) == []
+            assert manager.graph_runtime_state is None
+            graph_engine_manager.return_value.send_stop_command.assert_called_once_with(
+                "t1",
+                reason="Client response stream closed before app execution completed",
+            )
 
-        assert manager.graph_runtime_state is runtime_state
-        assert list(manager.listen()) == []
-        assert manager.graph_runtime_state is None
+    def test_abort_execution_is_idempotent_when_graph_stop_command_fails(self, caplog):
+        with (
+            patch("core.app.apps.base_app_queue_manager.redis_client") as mock_redis,
+            patch("core.app.apps.base_app_queue_manager.GraphEngineManager") as graph_engine_manager,
+        ):
+            mock_redis.setex.return_value = True
+            graph_engine_manager.return_value.send_stop_command.side_effect = RuntimeError("redis unavailable")
+            manager = DummyQueueManager(task_id="t1", user_id="u1", invoke_from=InvokeFrom.SERVICE_API)
+
+            manager._abort_execution("stream closed")
+            manager._abort_execution("duplicate")
+
+        graph_engine_manager.return_value.send_stop_command.assert_called_once_with("t1", reason="stream closed")
+        assert "Failed to abort app execution for task t1" in caplog.text

api/tests/unit_tests/core/app/apps/test_workflow_app_runner_core.py

@@ -17,6 +17,7 @@ from core.app.entities.queue_entities import (
     QueueNodeRetryEvent,
     QueueNodeSucceededEvent,
     QueueReasoningChunkEvent,
+    QueueStopEvent,
     QueueTextChunkEvent,
     QueueWorkflowPausedEvent,
     QueueWorkflowStartedEvent,
@@ -27,6 +28,7 @@ from core.workflow.system_variables import default_system_variables
 from graphon.entities.pause_reason import HitlRequired
 from graphon.enums import BuiltinNodeTypes
 from graphon.graph_events import (
+    GraphRunAbortedEvent,
     GraphRunPausedEvent,
     GraphRunStartedEvent,
     GraphRunSucceededEvent,
@@ -373,6 +375,23 @@ class TestWorkflowBasedAppRunner:
         assert paused_event.paused_nodes == ["node-1"]
         assert emails
 
+    def test_handle_graph_aborted_publishes_stopped_terminal(self):
+        published: list[object] = []
+
+        class _QueueManager:
+            def publish(self, event, publish_from):
+                del publish_from
+                published.append(event)
+
+        runner = WorkflowBasedAppRunner(queue_manager=_QueueManager(), app_id="app")
+        workflow_entry = SimpleNamespace()
+
+        runner._handle_event(workflow_entry, GraphRunAbortedEvent(reason="User requested stop", outputs={}))
+
+        event = published[-1]
+        assert isinstance(event, QueueStopEvent)
+        assert event.get_stop_reason() == "User requested stop"
+
     def test_handle_node_events_publishes_queue_events(self):
         published: list[object] = []
 

api/tests/unit_tests/core/app/apps/workflow/test_app_queue_manager.py

@@ -5,7 +5,7 @@ from unittest.mock import patch
 from core.app.apps.base_app_queue_manager import PublishFrom
 from core.app.apps.workflow.app_queue_manager import WorkflowAppQueueManager
 from core.app.entities.app_invoke_entities import InvokeFrom
-from core.app.entities.queue_entities import QueueMessageEndEvent, QueuePingEvent
+from core.app.entities.queue_entities import QueueMessageEndEvent, QueuePingEvent, QueueStopEvent
 
 
 class TestWorkflowAppQueueManager:
@@ -35,3 +35,67 @@ class TestWorkflowAppQueueManager:
         )
 
         manager._publish(QueuePingEvent(), PublishFrom.TASK_PIPELINE)
+
+    def test_listener_close_aborts_unfinished_execution(self):
+        with (
+            patch("core.app.apps.base_app_queue_manager.redis_client") as redis_client,
+            patch("core.app.apps.base_app_queue_manager.GraphEngineManager") as graph_engine_manager,
+        ):
+            redis_client.get.return_value = None
+            manager = WorkflowAppQueueManager(
+                task_id="task",
+                user_id="user",
+                invoke_from=InvokeFrom.DEBUGGER,
+                app_mode="workflow",
+            )
+            manager.publish(QueuePingEvent(), PublishFrom.TASK_PIPELINE)
+            listener = manager.listen()
+
+            assert isinstance(next(listener).event, QueuePingEvent)
+            listener.close()
+
+            graph_engine_manager.return_value.send_stop_command.assert_called_once_with(
+                "task",
+                reason="Client response stream closed before app execution completed",
+            )
+
+    def test_execution_timeout_aborts_graph_before_stop_event(self):
+        with (
+            patch("core.app.apps.base_app_queue_manager.redis_client") as redis_client,
+            patch("core.app.apps.base_app_queue_manager.GraphEngineManager") as graph_engine_manager,
+            patch("core.app.apps.base_app_queue_manager.dify_config.APP_MAX_EXECUTION_TIME", 0),
+        ):
+            redis_client.get.return_value = None
+            manager = WorkflowAppQueueManager(
+                task_id="task",
+                user_id="user",
+                invoke_from=InvokeFrom.DEBUGGER,
+                app_mode="workflow",
+            )
+            manager.publish(QueuePingEvent(), PublishFrom.TASK_PIPELINE)
+
+            messages = list(manager.listen())
+
+            assert any(isinstance(message.event, QueueStopEvent) for message in messages)
+            graph_engine_manager.return_value.send_stop_command.assert_called_once_with(
+                "task",
+                reason="App execution exceeded 0 seconds",
+            )
+
+    def test_terminal_event_does_not_abort_completed_execution(self):
+        with (
+            patch("core.app.apps.base_app_queue_manager.redis_client") as redis_client,
+            patch("core.app.apps.base_app_queue_manager.GraphEngineManager") as graph_engine_manager,
+        ):
+            redis_client.get.return_value = None
+            manager = WorkflowAppQueueManager(
+                task_id="task",
+                user_id="user",
+                invoke_from=InvokeFrom.DEBUGGER,
+                app_mode="workflow",
+            )
+            manager.publish(QueueMessageEndEvent(llm_result=None), PublishFrom.APPLICATION_MANAGER)
+
+            _ = list(manager.listen())
+
+            graph_engine_manager.return_value.send_stop_command.assert_not_called()

api/tests/unit_tests/core/app/entities/test_queue_entities.py

@@ -6,6 +6,13 @@ class TestQueueEntities:
         event = QueueStopEvent(stopped_by=QueueStopEvent.StopBy.USER_MANUAL)
         assert event.get_stop_reason() == "Stopped by user."
 
+    def test_get_stop_reason_prefers_explicit_reason(self):
+        event = QueueStopEvent(
+            stopped_by=QueueStopEvent.StopBy.USER_MANUAL,
+            reason="Workflow execution timed out",
+        )
+        assert event.get_stop_reason() == "Workflow execution timed out"
+
     def test_get_stop_reason_for_unknown_stop_by(self):
         event = QueueStopEvent(stopped_by=QueueStopEvent.StopBy.USER_MANUAL)
         event.stopped_by = "unknown"

api/tests/unit_tests/core/workflow/nodes/agent_v2/test_agent_node.py

@@ -1,3 +1,4 @@
+from collections.abc import Callable, Iterator
 from datetime import UTC, datetime
 from types import SimpleNamespace
 from typing import cast
@@ -5,11 +6,21 @@ from unittest.mock import MagicMock, patch
 
 from agenton.compositor import CompositorSessionSnapshot
 from dify_agent.layers.ask_human import AskHumanToolResult
-from dify_agent.protocol import PydanticAIStreamRunEvent, RunStartedEvent, RunSucceededEvent, RunSucceededEventData
+from dify_agent.protocol import (
+    CancelRunRequest,
+    CancelRunResponse,
+    PydanticAIStreamRunEvent,
+    RunEvent,
+    RunStartedEvent,
+    RunSucceededEvent,
+    RunSucceededEventData,
+)
 from pydantic_ai.messages import PartDeltaEvent, TextPartDelta
 
 from clients.agent_backend import (
+    AgentBackendInternalEventType,
     AgentBackendRunEventAdapter,
+    AgentBackendStreamError,
     AgentBackendStreamInternalEvent,
     FakeAgentBackendRunClient,
     FakeAgentBackendScenario,
@@ -216,6 +227,59 @@ class AgentMessageDeltaBackendClient(FakeAgentBackendRunClient):
         )
 
 
+class FailingStreamBackendClient(FakeAgentBackendRunClient):
+    def __init__(self) -> None:
+        super().__init__()
+        self.cancel_requests: list[CancelRunRequest | None] = []
+
+    def stream_events(
+        self,
+        run_id: str,
+        *,
+        after: str | None = None,
+        should_stop: Callable[[], bool] | None = None,
+    ) -> Iterator[RunEvent]:
+        del run_id, after, should_stop
+        raise AgentBackendStreamError("stream reconnect attempts exhausted")
+        yield
+
+    def cancel_run(self, run_id: str, request: CancelRunRequest | None = None) -> CancelRunResponse:
+        self.cancel_requests.append(request)
+        return CancelRunResponse(run_id=run_id, status="cancelled")
+
+
+class EmptyStreamBackendClient(FailingStreamBackendClient):
+    def stream_events(
+        self,
+        run_id: str,
+        *,
+        after: str | None = None,
+        should_stop: Callable[[], bool] | None = None,
+    ) -> Iterator[RunEvent]:
+        del run_id, after, should_stop
+        return
+        yield
+
+
+class GenericFailingStreamBackendClient(FailingStreamBackendClient):
+    def stream_events(
+        self,
+        run_id: str,
+        *,
+        after: str | None = None,
+        should_stop: Callable[[], bool] | None = None,
+    ) -> Iterator[RunEvent]:
+        del run_id, after, should_stop
+        raise RuntimeError("unexpected stream failure")
+        yield
+
+
+class CancelFailingStreamBackendClient(FailingStreamBackendClient):
+    def cancel_run(self, run_id: str, request: CancelRunRequest | None = None) -> CancelRunResponse:
+        self.cancel_requests.append(request)
+        raise RuntimeError(f"failed to cancel {run_id}")
+
+
 def _node(
     *,
     scenario: FakeAgentBackendScenario = FakeAgentBackendScenario.SUCCESS,
@@ -668,6 +732,78 @@ def test_agent_node_repauses_when_resumed_form_still_waiting(monkeypatch):
     assert client.request is None  # no second Agent run was created
 
 
+def test_agent_node_cancels_backend_run_when_stream_fails():
+    client = FailingStreamBackendClient()
+    node = _node(agent_backend_client=client)
+
+    terminal, failure = node._consume_event_stream("run-1", {"agent_backend": {}})
+
+    assert terminal is None
+    assert failure is not None
+    assert len(client.cancel_requests) == 1
+    assert client.cancel_requests[0] is not None
+    assert client.cancel_requests[0].reason == "event_stream_failed"
+
+
+def test_agent_node_cancels_backend_run_when_stream_ends_without_terminal_event():
+    client = EmptyStreamBackendClient()
+    node = _node(agent_backend_client=client)
+
+    terminal, failure = node._consume_event_stream("run-1", {"agent_backend": {}})
+
+    assert terminal is None
+    assert failure is None
+    assert client.cancel_requests[0] is not None
+    assert client.cancel_requests[0].reason == "stream_ended_without_terminal_event"
+
+
+def test_agent_node_cancels_backend_run_when_stream_raises_unexpected_error():
+    client = GenericFailingStreamBackendClient()
+    node = _node(agent_backend_client=client)
+
+    terminal, failure = node._consume_event_stream("run-1", {"agent_backend": {}})
+
+    assert terminal is None
+    assert failure is not None
+    assert failure.node_run_result.error == "unexpected stream failure"
+    assert client.cancel_requests[0] is not None
+    assert client.cancel_requests[0].reason == "event_stream_failed"
+
+
+def test_agent_node_uses_graph_abort_reason_when_cancel_request_fails(caplog):
+    client = CancelFailingStreamBackendClient()
+    node = _node(agent_backend_client=client)
+    node.graph_runtime_state.graph_execution = SimpleNamespace(aborted=True)
+
+    terminal, failure = node._consume_event_stream("run-1", {"agent_backend": {}})
+
+    assert terminal is None
+    assert failure is not None
+    assert client.cancel_requests[0] is not None
+    assert client.cancel_requests[0].reason == "workflow_graph_aborted"
+    assert "Failed to cancel Workflow Agent backend run" in caplog.text
+
+
+def test_agent_node_cancels_backend_run_for_unexpected_internal_event():
+    client = FakeAgentBackendRunClient()
+    node = _node(agent_backend_client=client)
+    node._agent_backend_client.cancel_run = MagicMock(  # type: ignore[method-assign]
+        return_value=CancelRunResponse(run_id="run-1", status="cancelled")
+    )
+    node._event_adapter.adapt = MagicMock(  # type: ignore[method-assign]
+        return_value=[SimpleNamespace(type=AgentBackendInternalEventType.RUN_FAILED)]
+    )
+
+    terminal, failure = node._consume_event_stream("run-1", {"agent_backend": {}})
+
+    assert terminal is None
+    assert failure is not None
+    assert failure.node_run_result.error == (
+        "Unexpected internal event type <AgentBackendInternalEventType.RUN_FAILED: 'run_failed'>"
+    )
+    node._agent_backend_client.cancel_run.assert_called_once()
+
+
 def test_agent_node_records_stream_usage_metadata():
     metadata = {"agent_backend": {"run_id": "run-1"}}