fix: handle long text URLs in webscraper (#40757)

api/core/rag/extractor/extract_processor.py

@@ -132,7 +132,6 @@ class ExtractProcessor:
                     storage.download(upload_file.key, file_path)
                 input_file = Path(file_path)
                 file_extension = input_file.suffix.lower()
-                assert upload_file is not None, "upload_file is required"
                 etl_type = dify_config.ETL_TYPE
                 extractor: BaseExtractor | None = None
                 if etl_type == "Unstructured":
@@ -140,6 +139,7 @@ class ExtractProcessor:
                     unstructured_api_key = dify_config.UNSTRUCTURED_API_KEY or ""
 
                     if file_extension in {".xlsx", ".xls"}:
+                        assert upload_file is not None, "upload_file is required"
                         extractor = ExcelExtractor(
                             file_path,
                             upload_file.tenant_id,
@@ -187,6 +187,7 @@ class ExtractProcessor:
                         extractor = TextExtractor(file_path, autodetect_encoding=True)
                 else:
                     if file_extension in {".xlsx", ".xls"}:
+                        assert upload_file is not None, "upload_file is required"
                         extractor = ExcelExtractor(
                             file_path,
                             upload_file.tenant_id,

api/tests/unit_tests/core/rag/extractor/test_extract_processor.py

@@ -118,6 +118,16 @@ class TestExtractProcessorLoaders:
         assert len(docs) == 2
         assert text == "u1\nu2"
 
+    def test_load_from_url_extracts_long_text_without_upload_file(self, monkeypatch: pytest.MonkeyPatch):
+        content = "a" * 100_000
+        response = SimpleNamespace(headers={"Content-Type": "text/plain"}, content=content.encode())
+        monkeypatch.setattr(processor_module.remote_fetcher, "make_request", lambda *args, **kwargs: response)
+        monkeypatch.setattr(processor_module.dify_config, "ETL_TYPE", "SelfHosted")
+
+        text = ExtractProcessor.load_from_url("https://example.com/response.txt", return_text=True)
+
+        assert text == content
+
 
 class TestExtractProcessorFileRouting:
     @pytest.fixture(autouse=True)