refactor(api): clarify DSL import and plugin migration boundaries (#38483)

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

api/tests/unit_tests/services/plugin/test_plugin_migration.py

@@ -1,3 +1,4 @@
+import json
 from unittest.mock import MagicMock, patch
 
 import pytest
@@ -8,17 +9,17 @@ from services.plugin.plugin_migration import PluginMigration
 MIGRATION_MODULE = "services.plugin.plugin_migration"
 
 
-def test_fetch_plugin_unique_identifier_returns_none_when_disabled(mocker: MockerFixture) -> None:
+def test_fetch_latest_package_identifier_returns_none_when_disabled(mocker: MockerFixture) -> None:
     mocker.patch("services.plugin.plugin_migration.dify_config.MARKETPLACE_ENABLED", False)
     batch_fetch = mocker.patch("services.plugin.plugin_migration.marketplace.batch_fetch_plugin_manifests")
 
-    result = PluginMigration._fetch_plugin_unique_identifier("langgenius/openai")
+    result = PluginMigration._fetch_latest_package_identifier("langgenius/openai")
 
     assert result is None
     batch_fetch.assert_not_called()
 
 
-def test_fetch_plugin_unique_identifier_calls_marketplace_when_enabled(mocker: MockerFixture) -> None:
+def test_fetch_latest_package_identifier_calls_marketplace_when_enabled(mocker: MockerFixture) -> None:
     mocker.patch("services.plugin.plugin_migration.dify_config.MARKETPLACE_ENABLED", True)
     manifest = mocker.MagicMock()
     manifest.latest_package_identifier = "langgenius/openai:1.0.0@abc"
@@ -27,7 +28,7 @@ def test_fetch_plugin_unique_identifier_calls_marketplace_when_enabled(mocker: M
         return_value=[manifest],
     )
 
-    result = PluginMigration._fetch_plugin_unique_identifier("langgenius/openai")
+    result = PluginMigration._fetch_latest_package_identifier("langgenius/openai")
 
     assert result == "langgenius/openai:1.0.0@abc"
 
@@ -75,7 +76,27 @@ class TestHandlePluginInstanceInstall:
 
         mock_marketplace.download_plugin_pkg.assert_called_once()
         invalidate_cache.assert_called_once_with("tenant1")
-        assert "success" in result or "failed" in result
+        assert result["success"] == ["langgenius/openai"]
+        assert result["failed"] == []
+
+    def test_reports_failed_plugin_ids_when_install_batch_raises(self) -> None:
+        with (
+            patch(f"{MIGRATION_MODULE}.dify_config") as mock_cfg,
+            patch(f"{MIGRATION_MODULE}.marketplace") as mock_marketplace,
+            patch(f"{MIGRATION_MODULE}.PluginInstaller") as mock_installer_cls,
+        ):
+            mock_cfg.MARKETPLACE_ENABLED = True
+            mock_marketplace.download_plugin_pkg.return_value = b"pkg_data"
+            mock_installer = MagicMock()
+            mock_installer_cls.return_value = mock_installer
+            mock_installer.install_from_identifiers.side_effect = RuntimeError("install failed")
+
+            result = PluginMigration.handle_plugin_instance_install(
+                "tenant1", {"langgenius/openai": "langgenius/openai:1.0.0@abc"}
+            )
+
+        assert result["success"] == []
+        assert result["failed"] == ["langgenius/openai"]
 
     def test_install_plugins_invalidates_cache_after_direct_tenant_install(self, tmp_path) -> None:
         extracted_plugins = tmp_path / "plugins.jsonl"
@@ -102,3 +123,30 @@ class TestHandlePluginInstanceInstall:
 
         mock_installer.install_from_identifiers.assert_called_once()
         invalidate_cache.assert_called_once_with("tenant1")
+
+    def test_install_plugins_reports_missing_plugin_ids(self, tmp_path) -> None:
+        extracted_plugins = tmp_path / "plugins.jsonl"
+        output_file = tmp_path / "output.json"
+        extracted_plugins.write_text('{"tenant_id":"tenant1","plugins":["langgenius/openai","langgenius/missing"]}\n')
+
+        with (
+            patch(
+                f"{MIGRATION_MODULE}.PluginMigration.extract_unique_plugins",
+                return_value={
+                    "plugins": {"langgenius/openai": "langgenius/openai:1.0.0@abc"},
+                    "plugin_not_exist": ["langgenius/missing"],
+                },
+            ),
+            patch(f"{MIGRATION_MODULE}.PluginMigration.handle_plugin_instance_install", return_value={}),
+            patch(f"{MIGRATION_MODULE}.PluginInstaller") as mock_installer_cls,
+            patch(f"{MIGRATION_MODULE}.PluginService.invalidate_plugin_model_providers_cache"),
+        ):
+            mock_installer = MagicMock()
+            mock_installer.list_plugins.return_value = []
+            mock_installer_cls.return_value = mock_installer
+
+            PluginMigration.install_plugins(str(extracted_plugins), str(output_file), workers=1)
+
+        assert json.loads(output_file.read_text())["not_installed"] == [
+            {"tenant_id": "tenant1", "plugin_not_exist": ["langgenius/missing"]}
+        ]

api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_dsl_service.py

@@ -643,6 +643,19 @@ def test_import_rag_pipeline_yaml_content_requires_mapping() -> None:
     assert "content must be a mapping" in result.error
 
 
+def test_import_rag_pipeline_rejects_oversized_yaml_content_by_bytes(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr("services.rag_pipeline.rag_pipeline_dsl_service.DSL_MAX_SIZE", 1)
+    service = RagPipelineDslService(session=Mock())
+    account = Mock(current_tenant_id="t1")
+
+    result = service.import_rag_pipeline(account=account, import_mode="yaml-content", yaml_content="é")
+
+    assert result.status == ImportStatus.FAILED
+    assert "10MB" in result.error
+
+
 def test_confirm_import_returns_failed_when_pending_data_is_invalid_type(mocker: MockerFixture) -> None:
     mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.redis_client.get", return_value=object())
     service = RagPipelineDslService(session=Mock())
@@ -901,6 +914,46 @@ def test_import_rag_pipeline_url_size_exceeds_limit(mocker: MockerFixture) -> No
     assert "10MB" in result.error
 
 
+def test_import_rag_pipeline_rejects_oversized_yaml_url_bytes_before_decode(
+    mocker: MockerFixture,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr("services.rag_pipeline.rag_pipeline_dsl_service.DSL_MAX_SIZE", 1)
+    response = Mock()
+    response.raise_for_status.return_value = None
+    response.content = b"\xff\xff"
+    mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.remote_fetcher.make_request", return_value=response)
+    service = RagPipelineDslService(session=Mock())
+    account = Mock(current_tenant_id="t1")
+
+    result = service.import_rag_pipeline(
+        account=account,
+        import_mode="yaml-url",
+        yaml_url="https://example.com/pipeline.yaml",
+    )
+
+    assert result.status == ImportStatus.FAILED
+    assert "10MB" in result.error
+
+
+def test_import_rag_pipeline_returns_decode_error_for_invalid_yaml_url_bytes(mocker: MockerFixture) -> None:
+    response = Mock()
+    response.raise_for_status.return_value = None
+    response.content = b"\xff"
+    mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.remote_fetcher.make_request", return_value=response)
+    service = RagPipelineDslService(session=Mock())
+    account = Mock(current_tenant_id="t1")
+
+    result = service.import_rag_pipeline(
+        account=account,
+        import_mode="yaml-url",
+        yaml_url="https://example.com/pipeline.yaml",
+    )
+
+    assert result.status == ImportStatus.FAILED
+    assert "utf-8" in result.error
+
+
 def test_import_rag_pipeline_fails_when_rag_pipeline_data_missing() -> None:
     service = RagPipelineDslService(session=Mock())
     account = Mock(current_tenant_id="t1")

api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_transform_service.py

@@ -89,7 +89,7 @@ def test_deal_dependencies_installs_missing_marketplace_plugins(mocker: MockerFi
     installer_cls.return_value.list_plugins.return_value = [SimpleNamespace(plugin_id="installed-plugin")]
 
     migration_cls = mocker.patch("services.rag_pipeline.rag_pipeline_transform_service.PluginMigration")
-    migration_cls.return_value._fetch_plugin_unique_identifier.return_value = "missing-plugin:1.0.0"
+    migration_cls.return_value._fetch_latest_package_identifier.return_value = "missing-plugin:1.0.0"
 
     install_mock = mocker.patch(
         "services.rag_pipeline.rag_pipeline_transform_service.PluginService.install_from_marketplace_pkg"
@@ -518,7 +518,7 @@ def test_deal_dependencies_installs_when_enabled(mocker: MockerFixture) -> None:
     installer = mocker.patch("services.rag_pipeline.rag_pipeline_transform_service.PluginInstaller").return_value
     installer.list_plugins.return_value = []
     migration = mocker.patch("services.rag_pipeline.rag_pipeline_transform_service.PluginMigration").return_value
-    migration._fetch_plugin_unique_identifier.return_value = "langgenius/openai:1.0.0@abc"
+    migration._fetch_latest_package_identifier.return_value = "langgenius/openai:1.0.0@abc"
     install_call = mocker.patch(
         "services.rag_pipeline.rag_pipeline_transform_service.PluginService.install_from_marketplace_pkg"
     )

api/tests/unit_tests/services/test_app_dsl_service.py

@@ -0,0 +1,53 @@
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+from services.app_dsl_service import AppDslService, ImportStatus
+
+
+def test_import_app_rejects_oversized_yaml_content_by_bytes(monkeypatch) -> None:
+    monkeypatch.setattr("services.app_dsl_service.DSL_MAX_SIZE", 1)
+    service = AppDslService(session=SimpleNamespace())
+
+    result = service.import_app(
+        account=SimpleNamespace(current_tenant_id="tenant-1"),
+        import_mode="yaml-content",
+        yaml_content="é",
+    )
+
+    assert result.status == ImportStatus.FAILED
+    assert "10MB" in result.error
+
+
+def test_import_app_rejects_oversized_yaml_url_bytes_before_decode(monkeypatch) -> None:
+    monkeypatch.setattr("services.app_dsl_service.DSL_MAX_SIZE", 1)
+    response = Mock()
+    response.raise_for_status.return_value = None
+    response.content = b"\xff\xff"
+    monkeypatch.setattr("services.app_dsl_service.remote_fetcher.make_request", Mock(return_value=response))
+    service = AppDslService(session=SimpleNamespace())
+
+    result = service.import_app(
+        account=SimpleNamespace(current_tenant_id="tenant-1"),
+        import_mode="yaml-url",
+        yaml_url="https://example.com/app.yaml",
+    )
+
+    assert result.status == ImportStatus.FAILED
+    assert "10MB" in result.error
+
+
+def test_import_app_returns_decode_error_for_invalid_yaml_url_bytes(monkeypatch) -> None:
+    response = Mock()
+    response.raise_for_status.return_value = None
+    response.content = b"\xff"
+    monkeypatch.setattr("services.app_dsl_service.remote_fetcher.make_request", Mock(return_value=response))
+    service = AppDslService(session=SimpleNamespace())
+
+    result = service.import_app(
+        account=SimpleNamespace(current_tenant_id="tenant-1"),
+        import_mode="yaml-url",
+        yaml_url="https://example.com/app.yaml",
+    )
+
+    assert result.status == ImportStatus.FAILED
+    assert "utf-8" in result.error

api/tests/unit_tests/services/test_snippet_dsl_service.py

@@ -95,7 +95,7 @@ def test_import_snippet_rejects_oversized_yaml_url_content(monkeypatch: pytest.M
     monkeypatch.setattr("services.snippet_dsl_service.DSL_MAX_SIZE", 3)
     monkeypatch.setattr(
         "services.snippet_dsl_service.ssrf_proxy.get",
-        Mock(return_value=SimpleNamespace(status_code=200, text="too large")),
+        Mock(return_value=SimpleNamespace(status_code=200, content=b"too large")),
     )
 
     result = service.import_snippet(
@@ -108,6 +108,43 @@ def test_import_snippet_rejects_oversized_yaml_url_content(monkeypatch: pytest.M
     assert "YAML content size exceeds maximum limit" in result.error
 
 
+def test_import_snippet_rejects_oversized_yaml_url_bytes_before_decode(monkeypatch: pytest.MonkeyPatch) -> None:
+    service = SnippetDslService(session=SimpleNamespace())
+    monkeypatch.setattr("services.snippet_dsl_service.DSL_MAX_SIZE", 1)
+    monkeypatch.setattr(
+        "services.snippet_dsl_service.ssrf_proxy.get",
+        Mock(return_value=SimpleNamespace(status_code=200, content=b"\xff\xff")),
+    )
+
+    result = service.import_snippet(
+        account=SimpleNamespace(current_tenant_id="tenant-1"),
+        import_mode=ImportMode.YAML_URL.value,
+        yaml_url="https://example.com/snippet.yaml",
+    )
+
+    assert result.status == ImportStatus.FAILED
+    assert "YAML content size exceeds maximum limit" in result.error
+
+
+def test_import_snippet_returns_decode_error_for_invalid_yaml_url_bytes(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    service = SnippetDslService(session=SimpleNamespace())
+    monkeypatch.setattr(
+        "services.snippet_dsl_service.ssrf_proxy.get",
+        Mock(return_value=SimpleNamespace(status_code=200, content=b"\xff")),
+    )
+
+    result = service.import_snippet(
+        account=SimpleNamespace(current_tenant_id="tenant-1"),
+        import_mode=ImportMode.YAML_URL.value,
+        yaml_url="https://example.com/snippet.yaml",
+    )
+
+    assert result.status == ImportStatus.FAILED
+    assert "utf-8" in result.error
+
+
 def test_import_snippet_returns_failed_when_yaml_url_fetch_raises(monkeypatch: pytest.MonkeyPatch) -> None:
     service = SnippetDslService(session=SimpleNamespace())
     monkeypatch.setattr(
@@ -127,12 +164,12 @@ def test_import_snippet_returns_failed_when_yaml_url_fetch_raises(monkeypatch: p
 
 def test_import_snippet_rejects_oversized_yaml_content(monkeypatch: pytest.MonkeyPatch) -> None:
     service = SnippetDslService(session=SimpleNamespace())
-    monkeypatch.setattr("services.snippet_dsl_service.DSL_MAX_SIZE", 3)
+    monkeypatch.setattr("services.snippet_dsl_service.DSL_MAX_SIZE", 1)
 
     result = service.import_snippet(
         account=SimpleNamespace(current_tenant_id="tenant-1"),
         import_mode=ImportMode.YAML_CONTENT.value,
-        yaml_content="too large",
+        yaml_content="é",
     )
 
     assert result.status == ImportStatus.FAILED