diff --git a/apps/api/requests/v2/map.requests.http b/apps/api/requests/v2/map.requests.http index bbd7820e6..eb96552a5 100644 --- a/apps/api/requests/v2/map.requests.http +++ b/apps/api/requests/v2/map.requests.http @@ -10,7 +10,7 @@ Authorization: Bearer {{$dotenv TEST_API_KEY}} content-type: application/json { - "url": "https://firecrawl.dev", + "url": "https://firecrawl.dev" } # { diff --git a/apps/python-sdk/example.py b/apps/python-sdk/example.py index 848a55ef5..b9a53c537 100644 --- a/apps/python-sdk/example.py +++ b/apps/python-sdk/example.py @@ -81,5 +81,9 @@ def main(): print(search_response) + # map example + map_response = firecrawl.map("https://firecrawl.dev") + print(map_response) + if __name__ == "__main__": main() \ No newline at end of file diff --git a/apps/python-sdk/firecrawl/TODO.md b/apps/python-sdk/firecrawl/TODO.md new file mode 100644 index 000000000..743e15e75 --- /dev/null +++ b/apps/python-sdk/firecrawl/TODO.md @@ -0,0 +1,89 @@ +- [ ] improve error handling for 500s + + +============================================================================ FAILURES ============================================================================ +_____________________________________________________ TestCrawlE2E.test_get_active_crawls_with_running_crawl _____________________________________________________ + +self = + + def test_get_active_crawls_with_running_crawl(self): + """Test getting active crawls when there's a running crawl.""" + # Start a crawl + start_job = self.client.start_crawl("https://docs.firecrawl.dev", limit=5) + assert start_job.id is not None + + # Get active crawls +> active_crawls_response = self.client.active_crawls() + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +firecrawl/__tests__/e2e/v2/test_crawl.py:149: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +firecrawl/v2/client.py:386: in active_crawls + return self.get_active_crawls() + ^^^^^^^^^^^^^^^^^^^^^^^^ +firecrawl/v2/client.py:377: in get_active_crawls + return crawl_module.get_active_crawls(self.http_client) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +firecrawl/v2/methods/crawl.py:446: in get_active_crawls + handle_response_error(response, "get active crawls") +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +response = , action = 'get active crawls' + + def handle_response_error(response: requests.Response, action: str) -> None: + """ + Handle API response errors and raise appropriate exceptions. + + Args: + response: The HTTP response object + action: Description of the action being performed + + Raises: + FirecrawlError: Appropriate error based on status code + """ + try: + response_json = response.json() + error_message = response_json.get('error', 'No error message provided.') + error_details = response_json.get('details', 'No additional error details provided.') + except: + # If we can't parse JSON, provide a helpful error message + try: + response_text = response.text[:500] # Limit to first 500 chars + if response_text.strip(): + error_message = f"Server returned non-JSON response: {response_text}" + error_details = f"Full response status: {response.status_code}" + else: + error_message = f"Server returned empty response with status {response.status_code}" + error_details = "No additional details available" + except: + error_message = f"Server returned unreadable response with status {response.status_code}" + error_details = "No additional details available" + + # Create appropriate error message + if response.status_code == 400: + message = f"Bad Request: Failed to {action}. {error_message} - {error_details}" + raise BadRequestError(message, response.status_code, response) + elif response.status_code == 401: + message = f"Unauthorized: Failed to {action}. {error_message} - {error_details}" + raise UnauthorizedError(message, response.status_code, response) + elif response.status_code == 402: + message = f"Payment Required: Failed to {action}. {error_message} - {error_details}" + raise PaymentRequiredError(message, response.status_code, response) + elif response.status_code == 403: + message = f"Website Not Supported: Failed to {action}. {error_message} - {error_details}" + raise WebsiteNotSupportedError(message, response.status_code, response) + elif response.status_code == 408: + message = f"Request Timeout: Failed to {action} as the request timed out. {error_message} - {error_details}" + raise RequestTimeoutError(message, response.status_code, response) + elif response.status_code == 429: + message = f"Rate Limit Exceeded: Failed to {action}. {error_message} - {error_details}" + raise RateLimitError(message, response.status_code, response) + elif response.status_code == 500: + message = f"Internal Server Error: Failed to {action}. {error_message} - {error_details}" +> raise InternalServerError(message, response.status_code, response) +E firecrawl.v2.utils.error_handler.InternalServerError: Internal Server Error: Failed to get active crawls. An unexpected error occurred. Please contact help@firecrawl.com for help. Your exception ID is c775af539e3a44f286664e55c488638e - No additional error details provided. + +firecrawl/v2/utils/error_handler.py:104: InternalServerError +==================================================================== short test summary info ===================================================================== +FAILED firecrawl/__tests__/e2e/v2/test_crawl.py::TestCrawlE2E::test_get_active_crawls_with_running_crawl - firecrawl.v2.utils.error_handler.InternalServerError: Internal Server Error: Failed to get active crawls. An unexpected error occurred. Please contact help@f... +======================================================================= 1 failed in 1.17s ======================================================================== diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py index 3a70e5e55..8f6d41c11 100644 --- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py +++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py @@ -194,7 +194,7 @@ class TestCrawlE2E: def test_crawl_with_scrape_options(self): """Test crawl with scrape options.""" scrape_opts = ScrapeOptions( - formats=["markdown"], + formats=["markdown", "links"], only_main_content=False, mobile=True, ) @@ -207,6 +207,25 @@ class TestCrawlE2E: assert crawl_job.id is not None + def test_crawl_with_json_format_object(self): + """Crawl with scrape_options including a JSON format object (prompt + schema).""" + json_schema = { + "type": "object", + "properties": { + "title": {"type": "string"} + }, + "required": ["title"], + } + scrape_opts = ScrapeOptions( + formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}] + ) + crawl_job = self.client.start_crawl( + "https://docs.firecrawl.dev", + limit=2, + scrape_options=scrape_opts + ) + assert crawl_job.id is not None + def test_crawl_all_parameters(self): """Test crawl with all possible parameters.""" scrape_opts = ScrapeOptions( diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_map.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_map.py new file mode 100644 index 000000000..02f6f70bc --- /dev/null +++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_map.py @@ -0,0 +1,68 @@ +import os +from dotenv import load_dotenv +from firecrawl import Firecrawl + + +load_dotenv() + +if not os.getenv("API_KEY"): + raise ValueError("API_KEY is not set") + +if not os.getenv("API_URL"): + raise ValueError("API_URL is not set") + + +import pytest + + +class TestMapE2E: + """End-to-end tests for map functionality (v2).""" + + def setup_method(self): + self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL")) + + def test_map_minimal_request(self): + resp = self.client.map("https://docs.firecrawl.dev") + + assert hasattr(resp, "success") and resp.success is True + assert hasattr(resp, "data") and resp.data is not None + assert hasattr(resp.data, "links") + assert isinstance(resp.data.links, list) + + # Basic sanity checks on at least one link + if len(resp.data.links) > 0: + first = resp.data.links[0] + assert hasattr(first, "url") + assert isinstance(first.url, str) and first.url.startswith("http") + + @pytest.mark.parametrize( + "sitemap_only,ignore_sitemap", + [ + (True, None), # sitemap: only + (None, True), # sitemap: skip + (None, None), # sitemap: include (default) + ], + ) + def test_map_with_options(self, sitemap_only, ignore_sitemap): + kwargs = { + "search": "docs", + "include_subdomains": True, + "limit": 10, + } + if sitemap_only is not None: + kwargs["sitemap_only"] = sitemap_only + if ignore_sitemap is not None: + kwargs["ignore_sitemap"] = ignore_sitemap + + resp = self.client.map("https://docs.firecrawl.dev", **kwargs) + + assert hasattr(resp, "success") and resp.success is True + assert hasattr(resp, "data") and resp.data is not None + assert isinstance(resp.data.links, list) + + # Limit should be respected (server-side) + assert len(resp.data.links) <= 10 + + for link in resp.data.links: + assert hasattr(link, "url") + assert isinstance(link.url, str) and link.url.startswith("http") diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_scrape.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_scrape.py index 8c9ca7ebc..7acaa9d75 100644 --- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_scrape.py +++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_scrape.py @@ -2,6 +2,8 @@ import os import pytest from dotenv import load_dotenv from firecrawl import Firecrawl +import json as _json +import pytest from firecrawl.v2.types import Viewport, ScreenshotAction, Document load_dotenv() @@ -56,6 +58,55 @@ class TestScrapeE2E: ) self._assert_valid_document(doc) + @pytest.mark.parametrize("fmt,expect_field", [ + ("markdown", "markdown"), + ("html", "html"), + ("raw_html", "raw_html"), + ("links", "links"), + ("screenshot", "screenshot"), + ]) + def test_scrape_basic_formats(self, fmt, expect_field): + """Verify basic formats request succeeds and expected fields are present when applicable.""" + doc = self.client.scrape( + "https://docs.firecrawl.dev", + formats=[fmt], + ) + # For formats that are not content (links/screenshot/json), skip main-content assertion + if expect_field not in {"links", "screenshot"}: + self._assert_valid_document(doc) + if expect_field == "markdown": + assert doc.markdown is not None + elif expect_field == "html": + assert doc.html is not None + elif expect_field == "raw_html": + assert doc.raw_html is not None + elif expect_field == "screenshot": + assert doc.screenshot is not None + elif expect_field == "links": + assert isinstance(doc.links, list) + assert len(doc.links) > 0 + + def test_scrape_with_json_format_object(self): + """Scrape with JSON format object (requires prompt and schema).""" + json_schema = { + "type": "object", + "properties": { + "title": {"type": "string"} + }, + "required": ["title"], + } + doc = self.client.scrape( + "https://docs.firecrawl.dev", + formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}], + only_main_content=True, + ) + # JSON format may not include main content fields; ensure request succeeded + assert isinstance(doc, Document) + # If backend returns extracted json content, it should be present under `json` + # (Do not fail if backend omits it; existence depends on implementation) + # if hasattr(doc, 'json'): + # assert doc.json is not None + def test_scrape_invalid_url(self): """Scrape should fail with empty or invalid URLs.""" with pytest.raises(ValueError, match="URL cannot be empty"): diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py index 7dbbc54f4..deb309c1b 100644 --- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py +++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py @@ -117,7 +117,15 @@ def test_search_all_parameters(): ignore_invalid_urls=True, timeout=60000, scrape_options=ScrapeOptions( - formats=["markdown", "html"], + formats=[ + "markdown", + "html", + { + "type": "json", + "prompt": "Extract the title and description from the page", + "schema": schema + } + ], headers={"User-Agent": "Firecrawl-Test/1.0"}, include_tags=["h1", "h2", "p"], exclude_tags=["nav", "footer"], @@ -208,4 +216,23 @@ def test_search_formats_flexibility(): assert isinstance(results1, SearchData) assert isinstance(results2, SearchData) assert results1.web is not None - assert results2.web is not None \ No newline at end of file + assert results2.web is not None + +def test_search_with_json_format_object(): + """Search with scrape_options including a JSON format object (prompt + schema).""" + json_schema = { + "type": "object", + "properties": { + "title": {"type": "string"} + }, + "required": ["title"], + } + results = firecrawl.search( + query="site:docs.firecrawl.dev", + limit=1, + scrape_options=ScrapeOptions( + formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}] + ), + ) + assert isinstance(results, SearchData) + assert results.web is not None and len(results.web) >= 0 \ No newline at end of file diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/map/test_map_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/map/test_map_request_preparation.py new file mode 100644 index 000000000..d4f8a6a4e --- /dev/null +++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/map/test_map_request_preparation.py @@ -0,0 +1,51 @@ +import pytest +from firecrawl.v2.types import MapOptions +from firecrawl.v2.methods.map import _prepare_map_request + + +class TestMapRequestPreparation: + """Unit tests for map request preparation.""" + + def test_basic_request_preparation(self): + data = _prepare_map_request("https://example.com") + assert data["url"] == "https://example.com" + # Default sitemap handling should be "include" when no flags provided + assert "sitemap" not in data # we only send when options provided + + def test_sitemap_transformations(self): + # sitemap_only -> sitemap: "only" + opts = MapOptions(sitemap_only=True) + data = _prepare_map_request("https://example.com", opts) + assert data["sitemap"] == "only" + + # ignore_sitemap -> sitemap: "skip" + opts = MapOptions(ignore_sitemap=True) + data = _prepare_map_request("https://example.com", opts) + assert data["sitemap"] == "skip" + + # default when options present but neither flag set -> include + opts = MapOptions(search="docs") + data = _prepare_map_request("https://example.com", opts) + assert data["sitemap"] == "include" + + def test_field_conversions(self): + opts = MapOptions( + search="docs", + include_subdomains=True, + limit=25, + sitemap_only=True, + ) + data = _prepare_map_request("https://example.com", opts) + + assert data["url"] == "https://example.com" + assert data["search"] == "docs" + assert data["includeSubdomains"] is True + assert data["limit"] == 25 + assert data["sitemap"] == "only" + + def test_invalid_url(self): + with pytest.raises(ValueError): + _prepare_map_request("") + with pytest.raises(ValueError): + _prepare_map_request(" ") + diff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.py index b812481f7..31d04da3d 100644 --- a/apps/python-sdk/firecrawl/client.py +++ b/apps/python-sdk/firecrawl/client.py @@ -62,7 +62,7 @@ class V2Proxy: self.start_crawl = client_instance.start_crawl self.crawl_params_preview = client_instance.crawl_params_preview # self.batch_scrape = client_instance.batch_scrape - # self.map = client_instance.map + self.map = client_instance.map def __getattr__(self, name): """Forward attribute access to the underlying client.""" @@ -141,7 +141,7 @@ class Firecrawl: self.active_crawls = self._v2_client.active_crawls # self.batch_scrape = self._v2_client.batch_scrape - # self.map = self._v2_client.map + self.map = self._v2_client.map self.search = self._v2_client.search class AsyncFirecrawl: diff --git a/apps/python-sdk/firecrawl/v2/__init__.py b/apps/python-sdk/firecrawl/v2/__init__.py index bc45080fd..0904fc44b 100644 --- a/apps/python-sdk/firecrawl/v2/__init__.py +++ b/apps/python-sdk/firecrawl/v2/__init__.py @@ -1,261 +1,3 @@ -from typing import Optional, List, Union, Dict, Any -from ..types import ( - SearchData, Document, ScrapeOptions, SearchRequest, - SourceOption, FormatOption, CrawlRequest, CrawlJob, CrawlResponse, - CrawlParamsRequest, CrawlParamsData, CrawlErrorsResponse, ActiveCrawlsResponse, - WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction, Location -) -from .methods.search import search as search_method -from .methods.crawl import ( - crawl as crawl_method, - start_crawl as start_crawl_method, - cancel_crawl as cancel_crawl_method, - get_crawl_status as get_crawl_status_method, - crawl_params_preview as crawl_params_preview_method, - get_crawl_errors as get_crawl_errors_method, - get_active_crawls as get_active_crawls_method -) -from .methods.scrape import scrape as scrape_method -from .utils.http_client import HttpClient +from .client import FirecrawlClient -class FirecrawlClient: - """ - Firecrawl v2 API client. - """ - - def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"): - self.api_key = api_key - self.api_url = api_url - self._client = HttpClient(api_key=self.api_key, api_url=self.api_url) - - def scrape( - self, - url: str, - formats: Optional[List[FormatOption]] = None, - headers: Optional[Dict[str, str]] = None, - include_tags: Optional[List[str]] = None, - exclude_tags: Optional[List[str]] = None, - only_main_content: Optional[bool] = None, - timeout: Optional[int] = None, - wait_for: Optional[int] = None, - mobile: Optional[bool] = None, - parsers: Optional[List[str]] = None, - actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction]]] = None, - location: Optional[Location] = None, - skip_tls_verification: Optional[bool] = None, - remove_base64_images: Optional[bool] = None, - fast_mode: Optional[bool] = None, - use_mock: Optional[str] = None, - block_ads: Optional[bool] = None, - proxy: Optional[str] = None, - max_age: Optional[int] = None, - store_in_cache: Optional[bool] = None, - ) -> Document: - """Scrape a single URL and return the document.""" - options = ScrapeOptions( - formats=formats, - headers=headers, - include_tags=include_tags, - exclude_tags=exclude_tags, - only_main_content=only_main_content, - timeout=timeout, - wait_for=wait_for, - mobile=mobile, - parsers=parsers, - actions=actions, - location=location, - skip_tls_verification=skip_tls_verification, - remove_base64_images=remove_base64_images, - fast_mode=fast_mode, - use_mock=use_mock, - block_ads=block_ads, - proxy=proxy, - max_age=max_age, - store_in_cache=store_in_cache - ) - return scrape_method(self._client, url, options) - - # batch-scrape - def batch_scrape( - self - ): - pass - - # start-batch-scrape - def start_batch_scrape( - self - ): - pass - - # get-batch-scrape-status - def get_batch_scrape_status( - self - ): - pass - - # cancel-batch-scrape - def cancel_batch_scrape( - self - ): - pass - # get-batch-scrape-errors - def get_batch_scrape_errors( - self - ): - pass - - def crawl( - self, - url: str, - prompt: Optional[str] = None, - include_paths: Optional[List[str]] = None, - exclude_paths: Optional[List[str]] = None, - max_discovery_depth: Optional[int] = None, - ignore_sitemap: bool = False, - ignore_query_parameters: bool = False, - limit: Optional[int] = None, - crawl_entire_domain: bool = False, - allow_external_links: bool = False, - allow_subdomains: bool = False, - delay: Optional[int] = None, - max_concurrency: Optional[int] = None, - webhook: Optional[Dict[str, Any]] = None, - scrape_options: Optional[ScrapeOptions] = None, - zero_data_retention: bool = False, - poll_interval: int = 2, - timeout: Optional[int] = None - ) -> CrawlJob: - """Start a crawl job and wait for it to complete.""" - request = CrawlRequest( - url=url, - prompt=prompt, - include_paths=include_paths, - exclude_paths=exclude_paths, - max_discovery_depth=max_discovery_depth, - ignore_sitemap=ignore_sitemap, - ignore_query_parameters=ignore_query_parameters, - limit=limit, - crawl_entire_domain=crawl_entire_domain, - allow_external_links=allow_external_links, - allow_subdomains=allow_subdomains, - delay=delay, - max_concurrency=max_concurrency, - webhook=webhook, - scrape_options=scrape_options, - zero_data_retention=zero_data_retention - ) - return crawl_method(self._client, request, poll_interval, timeout) - - def start_crawl( - self, - url: str, - prompt: Optional[str] = None, - include_paths: Optional[List[str]] = None, - exclude_paths: Optional[List[str]] = None, - max_discovery_depth: Optional[int] = None, - ignore_sitemap: bool = False, - ignore_query_parameters: bool = False, - limit: Optional[int] = None, - crawl_entire_domain: bool = False, - allow_external_links: bool = False, - allow_subdomains: bool = False, - delay: Optional[int] = None, - max_concurrency: Optional[int] = None, - webhook: Optional[Dict[str, Any]] = None, - scrape_options: Optional[ScrapeOptions] = None, - zero_data_retention: bool = False - ) -> CrawlResponse: - """Start a crawl job and return immediately.""" - request = CrawlRequest( - url=url, - prompt=prompt, - include_paths=include_paths, - exclude_paths=exclude_paths, - max_discovery_depth=max_discovery_depth, - ignore_sitemap=ignore_sitemap, - ignore_query_parameters=ignore_query_parameters, - limit=limit, - crawl_entire_domain=crawl_entire_domain, - allow_external_links=allow_external_links, - allow_subdomains=allow_subdomains, - delay=delay, - max_concurrency=max_concurrency, - webhook=webhook, - scrape_options=scrape_options, - zero_data_retention=zero_data_retention - ) - return start_crawl_method(self._client, request) - - def get_crawl_status( - self, - job_id: str - ) -> CrawlJob: - """Get the status of a crawl job.""" - return get_crawl_status_method(self._client, job_id) - - def cancel_crawl( - self, - job_id: str - ) -> bool: - """Cancel a running crawl job.""" - return cancel_crawl_method(self._client, job_id) - - def crawl_params_preview( - self, - url: str, - prompt: str - ) -> CrawlParamsData: - """Get crawl parameters from LLM based on URL and prompt.""" - request = CrawlParamsRequest(url=url, prompt=prompt) - return crawl_params_preview_method(self._client, request) - - def active_crawls( - self - ) -> ActiveCrawlsResponse: - """Get a list of active crawl jobs.""" - return get_active_crawls_method(self._client) - - def get_crawl_errors( - self, - crawl_id: str - ) -> CrawlErrorsResponse: - """Get errors from a crawl job.""" - return get_crawl_errors_method(self._client, crawl_id) - - # map - def map( - self - ): - pass - - def search( - self, - query: str, - sources: Optional[List[SourceOption]] = None, - limit: Optional[int] = 5, - tbs: Optional[str] = None, - location: Optional[str] = None, - ignore_invalid_urls: Optional[bool] = True, - timeout: Optional[int] = 60000, - scrape_options: Optional[ScrapeOptions] = None, - ) -> SearchData: - """Search for documents.""" - request = SearchRequest( - query=query, - sources=sources, - limit=limit, - tbs=tbs, - location=location, - ignore_invalid_urls=ignore_invalid_urls, - timeout=timeout, - scrape_options=scrape_options, - ) - return search_method(self._client, request) - - # credit-usage - def credit_usage( - self - ): - pass - -__all__ = ['FirecrawlClient'] \ No newline at end of file +__all__ = ["FirecrawlClient"] \ No newline at end of file diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py index c06007de4..af3f505b5 100644 --- a/apps/python-sdk/firecrawl/v2/client.py +++ b/apps/python-sdk/firecrawl/v2/client.py @@ -7,11 +7,33 @@ This module provides the main client class that orchestrates all v2 functionalit import os from typing import Optional, List, Dict, Any, Callable, Union from .types import ( - ClientConfig, ScrapeOptions, CrawlOptions, MapOptions, ExtractOptions, - ScrapeResponse, CrawlResponse, CrawlStatusResponse, BatchScrapeResponse, - BatchScrapeStatusResponse, MapResponse, ExtractResponse, Document, - SearchRequest, SearchResponse, CrawlRequest, WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse, - FormatOption, WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction, Location, + ClientConfig, + ScrapeOptions, + Document, + SearchRequest, + SearchData, + SourceOption, + CrawlRequest, + CrawlResponse, + CrawlJob, + CrawlParamsRequest, + CrawlParamsData, + WebhookConfig, + CrawlErrorsResponse, + ActiveCrawlsResponse, + MapOptions, + MapResponse, + FormatOption, + WaitAction, + ScreenshotAction, + ClickAction, + WriteAction, + PressAction, + ScrollAction, + ScrapeAction, + ExecuteJavascriptAction, + PDFAction, + Location, ) from .utils.http_client import HttpClient from .utils.error_handler import FirecrawlError @@ -19,6 +41,7 @@ from .methods import scrape as scrape_module from .methods import crawl as crawl_module from .methods import batch as batch_module from .methods import search as search_module +from .methods import map as map_module class FirecrawlClient: """ @@ -143,12 +166,14 @@ class FirecrawlClient: self, query: str, *, + sources: Optional[List[SourceOption]] = None, limit: Optional[int] = None, tbs: Optional[str] = None, location: Optional[str] = None, + ignore_invalid_urls: Optional[bool] = None, timeout: Optional[int] = None, - page_options: Optional[ScrapeOptions] = None, - ) -> SearchResponse: + scrape_options: Optional[ScrapeOptions] = None, + ) -> SearchData: """ Search for documents. @@ -161,17 +186,20 @@ class FirecrawlClient: page_options: Options for scraping individual pages Returns: - SearchResponse containing the search results + SearchData containing the search results """ - options = SearchRequest( + request = SearchRequest( + query=query, + sources=sources, limit=limit, tbs=tbs, location=location, + ignore_invalid_urls=ignore_invalid_urls, timeout=timeout, - page_options=page_options + scrape_options=scrape_options, ) - - return search_module.search(self.http_client, query, options) + + return search_module.search(self.http_client, request) def crawl( self, @@ -194,7 +222,7 @@ class FirecrawlClient: zero_data_retention: bool = False, poll_interval: int = 2, timeout: Optional[int] = None - ) -> CrawlStatusResponse: + ) -> CrawlJob: """ Start a crawl job and wait for it to complete. @@ -219,7 +247,7 @@ class FirecrawlClient: timeout: Maximum seconds to wait (None for no timeout) Returns: - CrawlStatusResponse when job completes + CrawlJob when job completes Raises: ValueError: If request is invalid @@ -321,7 +349,7 @@ class FirecrawlClient: return crawl_module.start_crawl(self.http_client, request) - def get_crawl_status(self, job_id: str) -> CrawlStatusResponse: + def get_crawl_status(self, job_id: str) -> CrawlJob: """ Get the status of a crawl job. @@ -329,24 +357,24 @@ class FirecrawlClient: job_id: ID of the crawl job Returns: - CrawlStatusResponse with current status and data + CrawlJob with current status and data Raises: Exception: If the status check fails """ return crawl_module.get_crawl_status(self.http_client, job_id) - def check_crawl_errors(self, crawl_id: str) -> CrawlErrorsResponse: + def get_crawl_errors(self, crawl_id: str) -> CrawlErrorsResponse: """ - Get errors from a crawl job. + Retrieve error details and robots.txt blocks for a given crawl job. Args: crawl_id: The ID of the crawl job - + Returns: - CrawlErrorsResponse containing errors and robots blocked URLs + CrawlErrorsResponse containing per-URL errors and robots-blocked URLs """ - return crawl_module.check_crawl_errors(self.http_client, crawl_id) + return crawl_module.get_crawl_errors(self.http_client, crawl_id) def get_active_crawls(self) -> ActiveCrawlsResponse: """ @@ -357,6 +385,38 @@ class FirecrawlClient: """ return crawl_module.get_active_crawls(self.http_client) + def active_crawls(self) -> ActiveCrawlsResponse: + """ + List currently active crawl jobs for the authenticated team. + + Returns: + ActiveCrawlsResponse containing the list of active crawl jobs + """ + return self.get_active_crawls() + + def map( + self, + url: str, + *, + search: Optional[str] = None, + include_subdomains: Optional[bool] = None, + limit: Optional[int] = None, + ignore_sitemap: Optional[bool] = None, + sitemap_only: Optional[bool] = None, + ) -> MapResponse: + """ + Map a URL and return discovered links (with optional titles/descriptions). + """ + options = MapOptions( + search=search, + include_subdomains=include_subdomains, + limit=limit, + ignore_sitemap=ignore_sitemap, + sitemap_only=sitemap_only, + ) if any(v is not None for v in [search, include_subdomains, limit, ignore_sitemap, sitemap_only]) else None + + return map_module.map(self.http_client, url, options) + def cancel_crawl(self, crawl_id: str) -> bool: """ Cancel a crawl job. @@ -368,4 +428,11 @@ class FirecrawlClient: bool: True if the crawl was cancelled, False otherwise """ return crawl_module.cancel_crawl(self.http_client, crawl_id) + + def crawl_params_preview(self, url: str, prompt: str) -> CrawlParamsData: + """ + Get crawl parameters from LLM based on URL and prompt. + """ + request = CrawlParamsRequest(url=url, prompt=prompt) + return crawl_module.crawl_params_preview(self.http_client, request) \ No newline at end of file diff --git a/apps/python-sdk/firecrawl/v2/methods/batch.py b/apps/python-sdk/firecrawl/v2/methods/batch.py index 2a19eda02..b10ba770a 100644 --- a/apps/python-sdk/firecrawl/v2/methods/batch.py +++ b/apps/python-sdk/firecrawl/v2/methods/batch.py @@ -4,11 +4,11 @@ Batch scraping functionality for Firecrawl v2 API. import time from typing import Optional, List, Callable -from .types import ( +from ..types import ( BatchScrapeRequest, BatchScrapeResponse, BatchScrapeJob, BatchScrapeData, ScrapeOptions, Document ) -from .utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options +from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options def start_batch_scrape( @@ -34,7 +34,7 @@ def start_batch_scrape( request_data = prepare_batch_request(urls, options) # Make the API request - response = client.post("/v1/batch/scrape", request_data) + response = client.post("/v2/batch/scrape", request_data) # Handle errors if not response.ok: @@ -77,7 +77,7 @@ def get_batch_scrape_status( FirecrawlError: If the status check fails """ # Make the API request - response = client.get(f"/v1/batch/scrape/{job_id}") + response = client.get(f"/v2/batch/scrape/{job_id}") # Handle errors if not response.ok: @@ -132,7 +132,7 @@ def cancel_batch_scrape( FirecrawlError: If the cancellation fails """ # Make the API request - response = client.delete(f"/v1/batch/scrape/{job_id}") + response = client.delete(f"/v2/batch/scrape/{job_id}") # Handle errors if not response.ok: @@ -235,7 +235,7 @@ def batch_scrape_and_wait( batch_response = start_batch_scrape(client, urls, options) if not batch_response.success or not batch_response.data: - return BatchScrapeStatusResponse( + return BatchScrapeResponse( success=False, error=batch_response.error or "Failed to start batch scrape" ) diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.py index 1894723f1..b6b2ee1ac 100644 --- a/apps/python-sdk/firecrawl/v2/methods/crawl.py +++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py @@ -8,7 +8,7 @@ from ..types import ( CrawlRequest, CrawlJob, CrawlResponse, Document, CrawlParamsRequest, CrawlParamsResponse, CrawlParamsData, - WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse + WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse, ActiveCrawl ) from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options @@ -416,13 +416,19 @@ def get_crawl_errors(http_client: HttpClient, crawl_id: str) -> CrawlErrorsRespo Exception: If the request fails """ response = http_client.get(f"/v2/crawl/{crawl_id}/errors") - - if response.status_code != 200: + + if not response.ok: handle_response_error(response, "check crawl errors") - + try: - data = response.json() - return CrawlErrorsResponse(**data) + body = response.json() + payload = body.get("data", body) + # Manual key normalization since we avoid Pydantic aliases + normalized = { + "errors": payload.get("errors", []), + "robots_blocked": payload.get("robotsBlocked", payload.get("robots_blocked", [])), + } + return CrawlErrorsResponse(**normalized) except Exception as e: raise Exception(f"Failed to parse crawl errors response: {e}") @@ -441,13 +447,23 @@ def get_active_crawls(client: HttpClient) -> ActiveCrawlsResponse: Exception: If the request fails """ response = client.get("/v2/crawl/active") - + if not response.ok: handle_response_error(response, "get active crawls") - - response_data = response.json() - - if response_data.get("success"): - return ActiveCrawlsResponse(**response_data.get("data", {})) - else: - raise Exception(response_data.get("error", "Unknown error occurred")) + + body = response.json() + if not body.get("success"): + raise Exception(body.get("error", "Unknown error occurred")) + + data = body.get("data", {}) + crawls_in = data.get("crawls", []) + normalized_crawls = [] + for c in crawls_in: + if isinstance(c, dict): + normalized_crawls.append({ + "id": c.get("id"), + "team_id": c.get("teamId", c.get("team_id")), + "url": c.get("url"), + "options": c.get("options"), + }) + return ActiveCrawlsResponse(success=True, crawls=[ActiveCrawl(**nc) for nc in normalized_crawls]) diff --git a/apps/python-sdk/firecrawl/v2/methods/map.py b/apps/python-sdk/firecrawl/v2/methods/map.py new file mode 100644 index 000000000..7daae4457 --- /dev/null +++ b/apps/python-sdk/firecrawl/v2/methods/map.py @@ -0,0 +1,69 @@ +""" +Mapping functionality for Firecrawl v2 API. +""" + +from typing import Optional, Dict, Any +from ..types import MapOptions, MapResponse, MapData, LinkResult +from ..utils import HttpClient, handle_response_error + + +def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dict[str, Any]: + if not url or not url.strip(): + raise ValueError("URL cannot be empty") + + payload: Dict[str, Any] = {"url": url.strip()} + + if options is not None: + # Transform ignoreSitemap and sitemapOnly to the new unified sitemap parameter + # sitemap: "only" | "include" | "skip" (default include) + sitemap: str = "include" + if options.sitemap_only: + sitemap = "only" + elif options.ignore_sitemap: + sitemap = "skip" + + data: Dict[str, Any] = {} + data["sitemap"] = sitemap + + if options.search is not None: + data["search"] = options.search + if options.include_subdomains is not None: + data["includeSubdomains"] = options.include_subdomains + if options.limit is not None: + data["limit"] = options.limit + payload.update(data) + + return payload + + +def map(client: HttpClient, url: str, options: Optional[MapOptions] = None) -> MapResponse: + """ + Map a URL and return a list of links with optional titles/descriptions. + """ + request_data = _prepare_map_request(url, options) + + response = client.post("/v2/map", request_data) + + if not response.ok: + handle_response_error(response, "map") + + body = response.json() + if not body.get("success"): + raise Exception(body.get("error", "Unknown error occurred")) + + data = body.get("data", {}) + result_links: list[LinkResult] = [] + for item in data.get("links", []): + if isinstance(item, dict): + result_links.append( + LinkResult( + url=item.get("url", ""), + title=item.get("title"), + description=item.get("description"), + ) + ) + elif isinstance(item, str): + result_links.append(LinkResult(url=item)) + + return MapResponse(success=True, data=MapData(links=result_links)) + diff --git a/apps/python-sdk/firecrawl/v2/methods/scrape.py b/apps/python-sdk/firecrawl/v2/methods/scrape.py index 0eaf7e720..408d8d378 100644 --- a/apps/python-sdk/firecrawl/v2/methods/scrape.py +++ b/apps/python-sdk/firecrawl/v2/methods/scrape.py @@ -59,4 +59,10 @@ def scrape(client: HttpClient, url: str, options: Optional[ScrapeOptions] = None raise Exception(body.get("error", "Unknown error occurred")) document_data = body.get("data", {}) - return Document(**document_data) \ No newline at end of file + # Normalize keys for Document (no Pydantic aliases) + normalized = dict(document_data) + if 'rawHtml' in normalized and 'raw_html' not in normalized: + normalized['raw_html'] = normalized.pop('rawHtml') + if 'changeTracking' in normalized and 'change_tracking' not in normalized: + normalized['change_tracking'] = normalized.pop('changeTracking') + return Document(**normalized) \ No newline at end of file diff --git a/apps/python-sdk/firecrawl/v2/methods/search.py b/apps/python-sdk/firecrawl/v2/methods/search.py index 6ac1d76c8..1495b0f34 100644 --- a/apps/python-sdk/firecrawl/v2/methods/search.py +++ b/apps/python-sdk/firecrawl/v2/methods/search.py @@ -3,7 +3,7 @@ Search functionality for Firecrawl v2 API. """ from typing import Optional, Dict, Any, Union -from ...types import SearchRequest, SearchData, SearchResult, Document +from ..types import SearchRequest, SearchData, SearchResult, Document from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options @@ -46,9 +46,19 @@ def search( results = [] for doc_data in source_documents: if isinstance(doc_data, dict): - if any(key in doc_data for key in ['markdown', 'html', 'content', 'screenshot']): - results.append(Document(**doc_data)) + # If page scraping options were provided, API returns full Document objects + if request.scrape_options is not None and any( + key in doc_data for key in ['markdown', 'html', 'rawHtml', 'links', 'screenshot', 'changeTracking'] + ): + # Normalize keys for Document (no Pydantic aliases) + normalized = dict(doc_data) + if 'rawHtml' in normalized and 'raw_html' not in normalized: + normalized['raw_html'] = normalized.pop('rawHtml') + if 'changeTracking' in normalized and 'change_tracking' not in normalized: + normalized['change_tracking'] = normalized.pop('changeTracking') + results.append(Document(**normalized)) else: + # Minimal search result shape results.append(SearchResult( url=doc_data.get('url', ''), title=doc_data.get('title'), @@ -56,8 +66,7 @@ def search( )) elif isinstance(doc_data, str): results.append(SearchResult(url=doc_data)) - - # Set the appropriate field based on source type + if hasattr(search_data, source_type): setattr(search_data, source_type, results) diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py index dc9a15cb9..f5a265193 100644 --- a/apps/python-sdk/firecrawl/v2/types.py +++ b/apps/python-sdk/firecrawl/v2/types.py @@ -15,6 +15,7 @@ warnings.filterwarnings("ignore", message="Field name \"schema\" in \"Format\" s warnings.filterwarnings("ignore", message="Field name \"schema\" in \"JsonFormat\" shadows an attribute in parent \"Format\"") warnings.filterwarnings("ignore", message="Field name \"schema\" in \"ChangeTrackingFormat\" shadows an attribute in parent \"Format\"") warnings.filterwarnings("ignore", message="Field name \"json\" in \"ScrapeFormats\" shadows an attribute in parent \"BaseModel\"") +warnings.filterwarnings("ignore", message="Field name \"json\" in \"Document\" shadows an attribute in parent \"BaseModel\"") T = TypeVar('T') @@ -34,25 +35,26 @@ class DocumentMetadata(BaseModel): language: Optional[str] = None keywords: Optional[Union[str, List[str]]] = None robots: Optional[str] = None - og_title: Optional[str] = Field(None, alias="ogTitle") - og_description: Optional[str] = Field(None, alias="ogDescription") - og_url: Optional[str] = Field(None, alias="ogUrl") - og_image: Optional[str] = Field(None, alias="ogImage") - source_url: Optional[str] = Field(None, alias="sourceURL") - status_code: Optional[int] = Field(None, alias="statusCode") + og_title: Optional[str] = None + og_description: Optional[str] = None + og_url: Optional[str] = None + og_image: Optional[str] = None + source_url: Optional[str] = None + status_code: Optional[int] = None error: Optional[str] = None class Document(BaseModel): """A scraped document.""" markdown: Optional[str] = None html: Optional[str] = None - raw_html: Optional[str] = Field(None, alias="rawHtml") + raw_html: Optional[str] = None + json: Optional[Any] = None metadata: Optional[DocumentMetadata] = None links: Optional[List[str]] = None screenshot: Optional[str] = None actions: Optional[Dict[str, Any]] = None warning: Optional[str] = None - change_tracking: Optional[Dict[str, Any]] = Field(None, alias="changeTracking") + change_tracking: Optional[Dict[str, Any]] = None # Webhook types class WebhookConfig(BaseModel): @@ -64,7 +66,7 @@ class WebhookConfig(BaseModel): class WebhookData(BaseModel): """Data sent to webhooks.""" - job_id: str = Field(alias="jobId") + job_id: str status: str current: Optional[int] = None total: Optional[int] = None @@ -120,7 +122,7 @@ class ScrapeFormats(BaseModel): formats: Optional[List[FormatOption]] = None markdown: bool = True html: bool = False - raw_html: bool = Field(False, alias="rawHtml") + raw_html: bool = False links: bool = False screenshot: bool = False change_tracking: bool = False @@ -148,7 +150,7 @@ class ScrapeFormats(BaseModel): class ScrapeOptions(BaseModel): """Options for scraping operations.""" - formats: Optional[List[FormatOption]] = None + formats: Optional[Union['ScrapeFormats', List[FormatOption]]] = None headers: Optional[Dict[str, str]] = None include_tags: Optional[List[str]] = None exclude_tags: Optional[List[str]] = None @@ -174,15 +176,10 @@ class ScrapeOptions(BaseModel): """Validate and normalize formats input.""" if v is None: return v - - # If it's already a ScrapeFormats object, return as is if isinstance(v, ScrapeFormats): return v - - # If it's a list, keep it as a list (don't convert to ScrapeFormats) if isinstance(v, list): return v - raise ValueError(f"Invalid formats type: {type(v)}. Expected ScrapeFormats or List[FormatOption]") class ScrapeRequest(BaseModel): @@ -236,6 +233,7 @@ class CrawlJob(BaseModel): class SearchDocument(Document): """A document from a search operation with URL and description.""" url: str + title: Optional[str] = None description: Optional[str] = None class MapDocument(Document): @@ -283,8 +281,8 @@ class BatchScrapeJob(BaseModel): status: Literal["scraping", "completed", "failed"] current: Optional[int] = None total: Optional[int] = None - created_at: Optional[datetime] = Field(None, alias="createdAt") - completed_at: Optional[datetime] = Field(None, alias="completedAt") + created_at: Optional[datetime] = None + completed_at: Optional[datetime] = None class BatchScrapeData(BaseModel): """Batch scrape results data.""" @@ -301,8 +299,9 @@ class BatchScrapeResponse(BaseResponse[BatchScrapeData]): class MapOptions(BaseModel): """Options for mapping operations.""" search: Optional[str] = None - ignore_sitemap: bool = Field(False, alias="ignoreSitemap") - include_subdomains: bool = Field(False, alias="includeSubdomains") + ignore_sitemap: Optional[bool] = None + sitemap_only: Optional[bool] = None + include_subdomains: Optional[bool] = None limit: Optional[int] = None class MapRequest(BaseModel): @@ -312,7 +311,7 @@ class MapRequest(BaseModel): class MapData(BaseModel): """Map results data.""" - links: List[str] + links: List['SearchResult'] class MapResponse(BaseResponse[MapData]): """Response for map operations.""" @@ -406,17 +405,20 @@ class SearchRequest(BaseModel): return normalized_sources -class SearchResult(BaseModel): - """A search result with basic information.""" +class LinkResult(BaseModel): + """A generic link result with optional metadata (used by search and map).""" url: str title: Optional[str] = None description: Optional[str] = None + +# Backward-compatible alias for existing tests/usages +SearchResult = LinkResult class SearchData(BaseModel): """Search results grouped by source type.""" - web: Optional[List[Union[SearchResult, SearchDocument]]] = None - news: Optional[List[Union[SearchResult, SearchDocument]]] = None - images: Optional[List[Union[SearchResult, SearchDocument]]] = None + web: Optional[List[Union[LinkResult, SearchDocument]]] = None + news: Optional[List[Union[LinkResult, SearchDocument]]] = None + images: Optional[List[Union[LinkResult, SearchDocument]]] = None class SearchResponse(BaseResponse[SearchData]): """Response from search operation.""" @@ -442,19 +444,19 @@ class JobStatus(BaseModel): status: Literal["pending", "scraping", "completed", "failed"] current: Optional[int] = None total: Optional[int] = None - created_at: Optional[datetime] = Field(None, alias="createdAt") - completed_at: Optional[datetime] = Field(None, alias="completedAt") - expires_at: Optional[datetime] = Field(None, alias="expiresAt") + created_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + expires_at: Optional[datetime] = None class CrawlErrorsResponse(BaseModel): """Response from crawl error monitoring.""" - errors: List[Dict[str, str]] = Field(description="List of errors with fields: id, timestamp, url, error") - robots_blocked: List[str] = Field(alias="robotsBlocked", description="List of URLs blocked by robots.txt") + errors: List[Dict[str, str]] + robots_blocked: List[str] class ActiveCrawl(BaseModel): """Information about an active crawl job.""" id: str - team_id: str = Field(alias="teamId") + team_id: str url: str options: Optional[Dict[str, Any]] = None diff --git a/apps/python-sdk/firecrawl/v2/utils/validation.py b/apps/python-sdk/firecrawl/v2/utils/validation.py index 39bb6acc7..655048104 100644 --- a/apps/python-sdk/firecrawl/v2/utils/validation.py +++ b/apps/python-sdk/firecrawl/v2/utils/validation.py @@ -2,8 +2,8 @@ Shared validation functions for Firecrawl v2 API. """ -from typing import Optional, Dict, Any -from ..types import ScrapeOptions +from typing import Optional, Dict, Any, List +from ..types import ScrapeOptions, ScrapeFormats def _convert_format_string(format_str: str) -> str: @@ -144,34 +144,112 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[st if value is not None: if key == "formats": # Handle formats conversion - converted_formats = [] - for fmt in value: - if isinstance(fmt, str): - # Handle format strings - if fmt == "json": - raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields") - converted_formats.append(_convert_format_string(fmt)) - elif isinstance(fmt, dict): - # Handle format objects - if fmt.get('type') == 'json': - validated_json = _validate_json_format(fmt) - converted_formats.append(validated_json) + converted_formats: List[Any] = [] + + # Prefer using original object to detect ScrapeFormats vs list + original_formats = getattr(options, 'formats', None) + + if isinstance(original_formats, ScrapeFormats): + # Include explicit list first + if original_formats.formats: + for fmt in original_formats.formats: + if isinstance(fmt, str): + if fmt == "json": + raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields") + converted_formats.append(_convert_format_string(fmt)) + elif isinstance(fmt, dict): + fmt_type = _convert_format_string(fmt.get('type')) if fmt.get('type') else None + if fmt_type == 'json': + validated_json = _validate_json_format({**fmt, 'type': 'json'}) + converted_formats.append(validated_json) + elif fmt_type == 'screenshot': + # Normalize screenshot options + normalized = {**fmt, 'type': 'screenshot'} + if 'full_page' in normalized: + normalized['fullPage'] = normalized.pop('full_page') + # Normalize viewport if it's a model instance + vp = normalized.get('viewport') + if hasattr(vp, 'model_dump'): + normalized['viewport'] = vp.model_dump(exclude_none=True) + converted_formats.append(normalized) + else: + if 'type' in fmt: + fmt['type'] = fmt_type or fmt['type'] + converted_formats.append(fmt) + elif hasattr(fmt, 'type'): + if fmt.type == 'json': + converted_formats.append(fmt.model_dump()) + else: + converted_formats.append(_convert_format_string(fmt.type)) + else: + converted_formats.append(fmt) + + # Add booleans from ScrapeFormats + if original_formats.markdown: + converted_formats.append("markdown") + if original_formats.html: + converted_formats.append("html") + if original_formats.raw_html: + converted_formats.append("rawHtml") + if original_formats.links: + converted_formats.append("links") + if original_formats.screenshot: + converted_formats.append("screenshot") + if original_formats.change_tracking: + converted_formats.append("changeTracking") + # Note: We intentionally do not auto-include 'json' when boolean is set, + # because JSON requires an object with schema/prompt. The caller must + # supply the full json format object explicitly. + elif isinstance(original_formats, list): + for fmt in original_formats: + if isinstance(fmt, str): + if fmt == "json": + raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields") + converted_formats.append(_convert_format_string(fmt)) + elif isinstance(fmt, dict): + fmt_type = _convert_format_string(fmt.get('type')) if fmt.get('type') else None + if fmt_type == 'json': + validated_json = _validate_json_format({**fmt, 'type': 'json'}) + converted_formats.append(validated_json) + elif fmt_type == 'screenshot': + normalized = {**fmt, 'type': 'screenshot'} + if 'full_page' in normalized: + normalized['fullPage'] = normalized.pop('full_page') + vp = normalized.get('viewport') + if hasattr(vp, 'model_dump'): + normalized['viewport'] = vp.model_dump(exclude_none=True) + converted_formats.append(normalized) + else: + if 'type' in fmt: + fmt['type'] = fmt_type or fmt['type'] + converted_formats.append(fmt) + elif hasattr(fmt, 'type'): + if fmt.type == 'json': + converted_formats.append(fmt.model_dump()) + elif fmt.type == 'screenshot': + normalized = {'type': 'screenshot'} + if getattr(fmt, 'full_page', None) is not None: + normalized['fullPage'] = fmt.full_page + if getattr(fmt, 'quality', None) is not None: + normalized['quality'] = fmt.quality + vp = getattr(fmt, 'viewport', None) + if vp is not None: + normalized['viewport'] = vp.model_dump(exclude_none=True) if hasattr(vp, 'model_dump') else vp + converted_formats.append(normalized) + else: + converted_formats.append(_convert_format_string(fmt.type)) else: - # Convert other format objects - if 'type' in fmt: - fmt['type'] = _convert_format_string(fmt['type']) converted_formats.append(fmt) - elif hasattr(fmt, 'type'): - # Handle Format objects - if fmt.type == 'json': - # For json format, we need the full object - converted_formats.append(fmt.model_dump()) - else: - # For other formats, just convert the type - converted_formats.append(_convert_format_string(fmt.type)) - else: - converted_formats.append(fmt) - scrape_data["formats"] = converted_formats + else: + # Fallback: try to iterate over value if it's a list-like + try: + for fmt in value: + converted_formats.append(fmt) + except TypeError: + pass + + if converted_formats: + scrape_data["formats"] = converted_formats elif key == "actions": # Handle actions conversion converted_actions = []