diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.py index 567f6e253..86a32a1fd 100644 --- a/apps/python-sdk/firecrawl/firecrawl.py +++ b/apps/python-sdk/firecrawl/firecrawl.py @@ -151,7 +151,7 @@ class ScrapeOptions(pydantic.BaseModel): excludeTags: Optional[List[str]] = None onlyMainContent: Optional[bool] = None waitFor: Optional[int] = None - timeout: Optional[int] = None + timeout: Optional[int] = 30000 location: Optional[LocationConfig] = None mobile: Optional[bool] = None skipTlsVerification: Optional[bool] = None @@ -305,7 +305,7 @@ class MapParams(pydantic.BaseModel): includeSubdomains: Optional[bool] = None sitemapOnly: Optional[bool] = None limit: Optional[int] = None - timeout: Optional[int] = None + timeout: Optional[int] = 30000 useIndex: Optional[bool] = None class MapResponse(pydantic.BaseModel): @@ -469,7 +469,7 @@ class FirecrawlApp: exclude_tags: Optional[List[str]] = None, only_main_content: Optional[bool] = None, wait_for: Optional[int] = None, - timeout: Optional[int] = None, + timeout: Optional[int] = 30000, location: Optional[LocationConfig] = None, mobile: Optional[bool] = None, skip_tls_verification: Optional[bool] = None, @@ -593,7 +593,7 @@ class FirecrawlApp: f'{self.api_url}/v1/scrape', headers=_headers, json=scrape_params, - timeout=(timeout + 5000 if timeout else None) + timeout=(timeout / 1000.0 + 5 if timeout is not None else None) ) if response.status_code == 200: @@ -620,7 +620,7 @@ class FirecrawlApp: lang: Optional[str] = None, country: Optional[str] = None, location: Optional[str] = None, - timeout: Optional[int] = None, + timeout: Optional[int] = 30000, scrape_options: Optional[ScrapeOptions] = None, **kwargs) -> SearchResponse: """ @@ -1189,7 +1189,7 @@ class FirecrawlApp: include_subdomains: Optional[bool] = None, sitemap_only: Optional[bool] = None, limit: Optional[int] = None, - timeout: Optional[int] = None, + timeout: Optional[int] = 30000, use_index: Optional[bool] = None, **kwargs) -> MapResponse: """ @@ -1280,7 +1280,7 @@ class FirecrawlApp: exclude_tags: Optional[List[str]] = None, only_main_content: Optional[bool] = None, wait_for: Optional[int] = None, - timeout: Optional[int] = None, + timeout: Optional[int] = 30000, location: Optional[LocationConfig] = None, mobile: Optional[bool] = None, skip_tls_verification: Optional[bool] = None, @@ -1421,7 +1421,7 @@ class FirecrawlApp: exclude_tags: Optional[List[str]] = None, only_main_content: Optional[bool] = None, wait_for: Optional[int] = None, - timeout: Optional[int] = None, + timeout: Optional[int] = 30000, location: Optional[LocationConfig] = None, mobile: Optional[bool] = None, skip_tls_verification: Optional[bool] = None, @@ -1561,7 +1561,7 @@ class FirecrawlApp: exclude_tags: Optional[List[str]] = None, only_main_content: Optional[bool] = None, wait_for: Optional[int] = None, - timeout: Optional[int] = None, + timeout: Optional[int] = 30000, location: Optional[LocationConfig] = None, mobile: Optional[bool] = None, skip_tls_verification: Optional[bool] = None, @@ -2204,7 +2204,7 @@ class FirecrawlApp: requests.RequestException: If the request fails after the specified retries. """ for attempt in range(retries): - response = requests.post(url, headers=headers, json=data, timeout=((data["timeout"] + 5000) if "timeout" in data else None)) + response = requests.post(url, headers=headers, json=data, timeout=((data["timeout"] / 1000.0 + 5) if "timeout" in data and data["timeout"] is not None else None)) if response.status_code == 502: time.sleep(backoff_factor * (2 ** attempt)) else: @@ -2975,7 +2975,7 @@ class AsyncFirecrawlApp(FirecrawlApp): exclude_tags: Optional[List[str]] = None, only_main_content: Optional[bool] = None, wait_for: Optional[int] = None, - timeout: Optional[int] = None, + timeout: Optional[int] = 30000, location: Optional[LocationConfig] = None, mobile: Optional[bool] = None, skip_tls_verification: Optional[bool] = None, @@ -3109,7 +3109,7 @@ class AsyncFirecrawlApp(FirecrawlApp): exclude_tags: Optional[List[str]] = None, only_main_content: Optional[bool] = None, wait_for: Optional[int] = None, - timeout: Optional[int] = None, + timeout: Optional[int] = 30000, location: Optional[LocationConfig] = None, mobile: Optional[bool] = None, skip_tls_verification: Optional[bool] = None, @@ -3247,7 +3247,7 @@ class AsyncFirecrawlApp(FirecrawlApp): exclude_tags: Optional[List[str]] = None, only_main_content: Optional[bool] = None, wait_for: Optional[int] = None, - timeout: Optional[int] = None, + timeout: Optional[int] = 30000, location: Optional[LocationConfig] = None, mobile: Optional[bool] = None, skip_tls_verification: Optional[bool] = None, @@ -3732,7 +3732,7 @@ class AsyncFirecrawlApp(FirecrawlApp): include_subdomains: Optional[bool] = None, sitemap_only: Optional[bool] = None, limit: Optional[int] = None, - timeout: Optional[int] = None, + timeout: Optional[int] = 30000, params: Optional[MapParams] = None) -> MapResponse: """ Asynchronously map and discover links from a URL. @@ -4467,7 +4467,7 @@ class AsyncFirecrawlApp(FirecrawlApp): lang: Optional[str] = None, country: Optional[str] = None, location: Optional[str] = None, - timeout: Optional[int] = None, + timeout: Optional[int] = 30000, scrape_options: Optional[ScrapeOptions] = None, params: Optional[Union[Dict[str, Any], SearchParams]] = None, **kwargs) -> SearchResponse: diff --git a/apps/python-sdk/tests/test_timeout_conversion.py b/apps/python-sdk/tests/test_timeout_conversion.py new file mode 100644 index 000000000..3c09c34bd --- /dev/null +++ b/apps/python-sdk/tests/test_timeout_conversion.py @@ -0,0 +1,117 @@ +import unittest +from unittest.mock import patch, MagicMock +import os +from firecrawl import FirecrawlApp + + +class TestTimeoutConversion(unittest.TestCase): + + @patch('requests.post') + def test_scrape_url_timeout_conversion(self, mock_post): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + 'success': True, + 'data': { + 'markdown': 'Test content' + } + } + mock_post.return_value = mock_response + + app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing')) + app.scrape_url('https://example.com', timeout=60000) + + args, kwargs = mock_post.call_args + self.assertEqual(kwargs['timeout'], 65.0) + + @patch('requests.post') + def test_scrape_url_default_timeout(self, mock_post): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + 'success': True, + 'data': { + 'markdown': 'Test content' + } + } + mock_post.return_value = mock_response + + app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing')) + app.scrape_url('https://example.com') + + args, kwargs = mock_post.call_args + self.assertEqual(kwargs['timeout'], 35.0) + + @patch('requests.post') + def test_post_request_timeout_conversion(self, mock_post): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_post.return_value = mock_response + + app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing')) + + data = {'timeout': 30000} + headers = {'Content-Type': 'application/json'} + + app._post_request('https://example.com/api', data, headers) + + args, kwargs = mock_post.call_args + self.assertEqual(kwargs['timeout'], 35.0) + + @patch('requests.post') + def test_post_request_default_timeout(self, mock_post): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_post.return_value = mock_response + + app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing')) + + data = {'timeout': 30000, 'url': 'https://example.com'} + headers = {'Content-Type': 'application/json'} + + app._post_request('https://example.com/api', data, headers) + + args, kwargs = mock_post.call_args + self.assertEqual(kwargs['timeout'], 35.0) + + @patch('requests.post') + def test_timeout_edge_cases(self, mock_post): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + 'success': True, + 'data': { + 'markdown': 'Test content' + } + } + mock_post.return_value = mock_response + + app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing')) + + app.scrape_url('https://example.com', timeout=1000) + args, kwargs = mock_post.call_args + self.assertEqual(kwargs['timeout'], 6.0) + + app.scrape_url('https://example.com', timeout=0) + args, kwargs = mock_post.call_args + self.assertEqual(kwargs['timeout'], 5.0) + + @patch('requests.post') + def test_post_request_no_timeout_key(self, mock_post): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_post.return_value = mock_response + + app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing')) + + data = {'url': 'https://example.com'} + headers = {'Content-Type': 'application/json'} + + app._post_request('https://example.com/api', data, headers) + + args, kwargs = mock_post.call_args + self.assertIsNone(kwargs['timeout']) + + +if __name__ == '__main__': + unittest.main()