diff --git a/apps/python-sdk/example.py b/apps/python-sdk/example.py index 93f489718..77b038c1d 100644 --- a/apps/python-sdk/example.py +++ b/apps/python-sdk/example.py @@ -21,24 +21,24 @@ def main(): firecrawl = Firecrawl(api_key=api_key, api_url=api_url) - # # Scrape - # doc = firecrawl.scrape("https://docs.firecrawl.dev", formats=["markdown"]) - # print("scrape:", doc.markdown) + # Scrape + doc = firecrawl.scrape("https://docs.firecrawl.dev", formats=["markdown"]) + print("scrape:", doc.markdown) - # # Crawl (waits until terminal state) - # crawl_job = firecrawl.crawl("https://docs.firecrawl.dev", limit=3, poll_interval=1, timeout=120) - # print("crawl:", crawl_job.status, crawl_job.completed, "/", crawl_job.total) + # Crawl (waits until terminal state) + crawl_job = firecrawl.crawl("https://docs.firecrawl.dev", limit=3, poll_interval=1, timeout=120) + print("crawl:", crawl_job.status, crawl_job.completed, "/", crawl_job.total) - # # Batch scrape - # batch = firecrawl.batch_scrape([ - # "https://docs.firecrawl.dev", - # "https://firecrawl.dev", - # ], formats=["markdown"], poll_interval=1, wait_timeout=120) - # print("batch:", batch.status, batch.completed, "/", batch.total) + # Batch scrape + batch = firecrawl.batch_scrape([ + "https://docs.firecrawl.dev", + "https://firecrawl.dev", + ], formats=["markdown"], poll_interval=1, wait_timeout=120) + print("batch:", batch.status, batch.completed, "/", batch.total) - # # Search - # search_response = firecrawl.search(query="What is the capital of France?", limit=5) - # print("search web results:", len(getattr(search_response, "web", []) or [])) + # Search + search_response = firecrawl.search(query="What is the capital of France?", limit=5) + print("search web results:", len(getattr(search_response, "web", []) or [])) # Map map_response = firecrawl.map("https://firecrawl.dev") diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py index 08e90493c..c9512ff45 100644 --- a/apps/python-sdk/firecrawl/v2/client.py +++ b/apps/python-sdk/firecrawl/v2/client.py @@ -44,6 +44,7 @@ from .methods import search as search_module from .methods import map as map_module from .methods import batch as batch_methods from .methods import usage as usage_methods +from .methods import extract as extract_module from .watcher import Watcher class FirecrawlClient: @@ -439,25 +440,24 @@ class FirecrawlClient: request = CrawlParamsRequest(url=url, prompt=prompt) return crawl_module.crawl_params_preview(self.http_client, request) - def extract( + def start_extract( self, urls: Optional[List[str]] = None, *, prompt: Optional[str] = None, - schema: Optional[Any] = None, + schema: Optional[Dict[str, Any]] = None, system_prompt: Optional[str] = None, - allow_external_links: Optional[bool] = False, - enable_web_search: Optional[bool] = False, - show_sources: Optional[bool] = False, - agent: Optional[Dict[str, Any]] = None, - **kwargs, + allow_external_links: Optional[bool] = None, + enable_web_search: Optional[bool] = None, + show_sources: Optional[bool] = None, + scrape_options: Optional['ScrapeOptions'] = None, + ignore_invalid_urls: Optional[bool] = None, ): """ - Extract data from a list of URLs. + Start an extract job (non-blocking). Use get_extract_status to poll. """ - from ..v1.client import V1FirecrawlApp - v1 = V1FirecrawlApp(api_key=self.config.api_key, api_url=self.config.api_url) - return v1.extract( + return extract_module.start_extract( + self.http_client, urls, prompt=prompt, schema=schema, @@ -465,8 +465,41 @@ class FirecrawlClient: allow_external_links=allow_external_links, enable_web_search=enable_web_search, show_sources=show_sources, - agent=agent, - **kwargs, + scrape_options=scrape_options, + ignore_invalid_urls=ignore_invalid_urls, + ) + + def extract( + self, + urls: Optional[List[str]] = None, + *, + prompt: Optional[str] = None, + schema: Optional[Dict[str, Any]] = None, + system_prompt: Optional[str] = None, + allow_external_links: Optional[bool] = None, + enable_web_search: Optional[bool] = None, + show_sources: Optional[bool] = None, + scrape_options: Optional['ScrapeOptions'] = None, + ignore_invalid_urls: Optional[bool] = None, + poll_interval: int = 2, + timeout: Optional[int] = None, + ): + """ + Extract structured data using the v2 client surface, waiting until completion. + """ + return extract_module.extract( + self.http_client, + urls, + prompt=prompt, + schema=schema, + system_prompt=system_prompt, + allow_external_links=allow_external_links, + enable_web_search=enable_web_search, + show_sources=show_sources, + scrape_options=scrape_options, + ignore_invalid_urls=ignore_invalid_urls, + poll_interval=poll_interval, + timeout=timeout, ) def start_batch_scrape( @@ -553,10 +586,7 @@ class FirecrawlClient: return batch_methods.get_batch_scrape_errors(self.http_client, job_id) def get_extract_status(self, job_id: str): - """Proxy to v1 get_extract_status while v2 extract is not implemented.""" - from ..v1.client import V1FirecrawlApp - v1 = V1FirecrawlApp(api_key=self.config.api_key, api_url=self.config.api_url) - return v1.get_extract_status(job_id) + return extract_module.get_extract_status(self.http_client, job_id) def get_concurrency(self): """Get current concurrency and maximum allowed for this team/key (v2).""" diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.py index 8e53059a2..3a8102801 100644 --- a/apps/python-sdk/firecrawl/v2/client_async.py +++ b/apps/python-sdk/firecrawl/v2/client_async.py @@ -41,6 +41,7 @@ from .methods.aio import crawl as async_crawl # type: ignore[attr-defined] from .methods.aio import search as async_search # type: ignore[attr-defined] from .methods.aio import map as async_map # type: ignore[attr-defined] from .methods.aio import usage as async_usage # type: ignore[attr-defined] +from .methods.aio import extract as async_extract # type: ignore[attr-defined] from .watcher_async import AsyncWatcher @@ -169,17 +170,18 @@ class AsyncFirecrawlClient: urls: Optional[List[str]] = None, *, prompt: Optional[str] = None, - schema: Optional[Any] = None, + schema: Optional[Dict[str, Any]] = None, system_prompt: Optional[str] = None, - allow_external_links: Optional[bool] = False, - enable_web_search: Optional[bool] = False, - show_sources: Optional[bool] = False, - agent: Optional[Dict[str, Any]] = None, - **kwargs, + allow_external_links: Optional[bool] = None, + enable_web_search: Optional[bool] = None, + show_sources: Optional[bool] = None, + scrape_options: Optional['ScrapeOptions'] = None, + ignore_invalid_urls: Optional[bool] = None, + poll_interval: int = 2, + timeout: Optional[int] = None, ): - from ..v1.client import AsyncV1FirecrawlApp - v1 = AsyncV1FirecrawlApp(api_key=self.http_client.api_key, api_url=self.http_client.api_url) - return await v1.extract( + return await async_extract.extract( + self.async_http_client, urls, prompt=prompt, schema=schema, @@ -187,14 +189,40 @@ class AsyncFirecrawlClient: allow_external_links=allow_external_links, enable_web_search=enable_web_search, show_sources=show_sources, - agent=agent, - **kwargs, + scrape_options=scrape_options, + ignore_invalid_urls=ignore_invalid_urls, + poll_interval=poll_interval, + timeout=timeout, ) async def get_extract_status(self, job_id: str): - from ..v1.client import AsyncV1FirecrawlApp - v1 = AsyncV1FirecrawlApp(api_key=self.http_client.api_key, api_url=self.http_client.api_url) - return await v1.get_extract_status(job_id) + return await async_extract.get_extract_status(self.async_http_client, job_id) + + async def start_extract( + self, + urls: Optional[List[str]] = None, + *, + prompt: Optional[str] = None, + schema: Optional[Dict[str, Any]] = None, + system_prompt: Optional[str] = None, + allow_external_links: Optional[bool] = None, + enable_web_search: Optional[bool] = None, + show_sources: Optional[bool] = None, + scrape_options: Optional['ScrapeOptions'] = None, + ignore_invalid_urls: Optional[bool] = None, + ): + return await async_extract.start_extract( + self.async_http_client, + urls, + prompt=prompt, + schema=schema, + system_prompt=system_prompt, + allow_external_links=allow_external_links, + enable_web_search=enable_web_search, + show_sources=show_sources, + scrape_options=scrape_options, + ignore_invalid_urls=ignore_invalid_urls, + ) # Usage endpoints async def get_concurrency(self): diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/__init__.py b/apps/python-sdk/firecrawl/v2/methods/aio/__init__.py index 85f6a99e2..5cc6d202e 100644 --- a/apps/python-sdk/firecrawl/v2/methods/aio/__init__.py +++ b/apps/python-sdk/firecrawl/v2/methods/aio/__init__.py @@ -1,2 +1 @@ -# Async (aio) method modules for v2 - +# Async (aio) method modules for v2 \ No newline at end of file diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/extract.py b/apps/python-sdk/firecrawl/v2/methods/aio/extract.py new file mode 100644 index 000000000..981829dca --- /dev/null +++ b/apps/python-sdk/firecrawl/v2/methods/aio/extract.py @@ -0,0 +1,126 @@ +from typing import Any, Dict, List, Optional +import asyncio + +from ...types import ExtractResponse, ScrapeOptions +from ...utils.http_client_async import AsyncHttpClient +from ...utils.validation import prepare_scrape_options + + +def _prepare_extract_request( + urls: Optional[List[str]], + *, + prompt: Optional[str] = None, + schema: Optional[Dict[str, Any]] = None, + system_prompt: Optional[str] = None, + allow_external_links: Optional[bool] = None, + enable_web_search: Optional[bool] = None, + show_sources: Optional[bool] = None, + scrape_options: Optional[ScrapeOptions] = None, + ignore_invalid_urls: Optional[bool] = None, +) -> Dict[str, Any]: + body: Dict[str, Any] = {} + if urls is not None: + body["urls"] = urls + if prompt is not None: + body["prompt"] = prompt + if schema is not None: + body["schema"] = schema + if system_prompt is not None: + body["systemPrompt"] = system_prompt + if allow_external_links is not None: + body["allowExternalLinks"] = allow_external_links + if enable_web_search is not None: + body["enableWebSearch"] = enable_web_search + if show_sources is not None: + body["showSources"] = show_sources + if ignore_invalid_urls is not None: + body["ignoreInvalidURLs"] = ignore_invalid_urls + if scrape_options is not None: + prepared = prepare_scrape_options(scrape_options) + if prepared: + body["scrapeOptions"] = prepared + return body + + +async def start_extract( + client: AsyncHttpClient, + urls: Optional[List[str]], + *, + prompt: Optional[str] = None, + schema: Optional[Dict[str, Any]] = None, + system_prompt: Optional[str] = None, + allow_external_links: Optional[bool] = None, + enable_web_search: Optional[bool] = None, + show_sources: Optional[bool] = None, + scrape_options: Optional[ScrapeOptions] = None, + ignore_invalid_urls: Optional[bool] = None, +) -> ExtractResponse: + body = _prepare_extract_request( + urls, + prompt=prompt, + schema=schema, + system_prompt=system_prompt, + allow_external_links=allow_external_links, + enable_web_search=enable_web_search, + show_sources=show_sources, + scrape_options=scrape_options, + ignore_invalid_urls=ignore_invalid_urls, + ) + resp = await client.post("/v2/extract", body) + return ExtractResponse(**resp.json()) + + +async def get_extract_status(client: AsyncHttpClient, job_id: str) -> ExtractResponse: + resp = await client.get(f"/v2/extract/{job_id}") + return ExtractResponse(**resp.json()) + + +async def wait_extract( + client: AsyncHttpClient, + job_id: str, + *, + poll_interval: int = 2, + timeout: Optional[int] = None, +) -> ExtractResponse: + start_ts = asyncio.get_event_loop().time() + while True: + status = await get_extract_status(client, job_id) + if status.status in ("completed", "failed", "cancelled"): + return status + if timeout is not None and (asyncio.get_event_loop().time() - start_ts) > timeout: + return status + await asyncio.sleep(max(1, poll_interval)) + + +async def extract( + client: AsyncHttpClient, + urls: Optional[List[str]], + *, + prompt: Optional[str] = None, + schema: Optional[Dict[str, Any]] = None, + system_prompt: Optional[str] = None, + allow_external_links: Optional[bool] = None, + enable_web_search: Optional[bool] = None, + show_sources: Optional[bool] = None, + scrape_options: Optional[ScrapeOptions] = None, + ignore_invalid_urls: Optional[bool] = None, + poll_interval: int = 2, + timeout: Optional[int] = None, +) -> ExtractResponse: + started = await start_extract( + client, + urls, + prompt=prompt, + schema=schema, + system_prompt=system_prompt, + allow_external_links=allow_external_links, + enable_web_search=enable_web_search, + show_sources=show_sources, + scrape_options=scrape_options, + ignore_invalid_urls=ignore_invalid_urls, + ) + job_id = getattr(started, "id", None) + if not job_id: + return started + return await wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout) + diff --git a/apps/python-sdk/firecrawl/v2/methods/extract.py b/apps/python-sdk/firecrawl/v2/methods/extract.py new file mode 100644 index 000000000..d863b1e1e --- /dev/null +++ b/apps/python-sdk/firecrawl/v2/methods/extract.py @@ -0,0 +1,131 @@ +from typing import Any, Dict, List, Optional +import time + +from ..types import ExtractResponse, ScrapeOptions +from ..utils.http_client import HttpClient +from ..utils.validation import prepare_scrape_options +from ..utils.error_handler import handle_response_error + + +def _prepare_extract_request( + urls: Optional[List[str]], + *, + prompt: Optional[str] = None, + schema: Optional[Dict[str, Any]] = None, + system_prompt: Optional[str] = None, + allow_external_links: Optional[bool] = None, + enable_web_search: Optional[bool] = None, + show_sources: Optional[bool] = None, + scrape_options: Optional[ScrapeOptions] = None, + ignore_invalid_urls: Optional[bool] = None, +) -> Dict[str, Any]: + body: Dict[str, Any] = {} + if urls is not None: + body["urls"] = urls + if prompt is not None: + body["prompt"] = prompt + if schema is not None: + body["schema"] = schema + if system_prompt is not None: + body["systemPrompt"] = system_prompt + if allow_external_links is not None: + body["allowExternalLinks"] = allow_external_links + if enable_web_search is not None: + body["enableWebSearch"] = enable_web_search + if show_sources is not None: + body["showSources"] = show_sources + if ignore_invalid_urls is not None: + body["ignoreInvalidURLs"] = ignore_invalid_urls + if scrape_options is not None: + prepared = prepare_scrape_options(scrape_options) + if prepared: + body["scrapeOptions"] = prepared + return body + + +def start_extract( + client: HttpClient, + urls: Optional[List[str]], + *, + prompt: Optional[str] = None, + schema: Optional[Dict[str, Any]] = None, + system_prompt: Optional[str] = None, + allow_external_links: Optional[bool] = None, + enable_web_search: Optional[bool] = None, + show_sources: Optional[bool] = None, + scrape_options: Optional[ScrapeOptions] = None, + ignore_invalid_urls: Optional[bool] = None, +) -> ExtractResponse: + body = _prepare_extract_request( + urls, + prompt=prompt, + schema=schema, + system_prompt=system_prompt, + allow_external_links=allow_external_links, + enable_web_search=enable_web_search, + show_sources=show_sources, + scrape_options=scrape_options, + ignore_invalid_urls=ignore_invalid_urls, + ) + resp = client.post("/v2/extract", body) + if not resp.ok: + handle_response_error(resp, "extract") + return ExtractResponse(**resp.json()) + + +def get_extract_status(client: HttpClient, job_id: str) -> ExtractResponse: + resp = client.get(f"/v2/extract/{job_id}") + if not resp.ok: + handle_response_error(resp, "extract-status") + return ExtractResponse(**resp.json()) + + +def wait_extract( + client: HttpClient, + job_id: str, + *, + poll_interval: int = 2, + timeout: Optional[int] = None, +) -> ExtractResponse: + start_ts = time.time() + while True: + status = get_extract_status(client, job_id) + if status.status in ("completed", "failed", "cancelled"): + return status + if timeout is not None and (time.time() - start_ts) > timeout: + return status + time.sleep(max(1, poll_interval)) + + +def extract( + client: HttpClient, + urls: Optional[List[str]], + *, + prompt: Optional[str] = None, + schema: Optional[Dict[str, Any]] = None, + system_prompt: Optional[str] = None, + allow_external_links: Optional[bool] = None, + enable_web_search: Optional[bool] = None, + show_sources: Optional[bool] = None, + scrape_options: Optional[ScrapeOptions] = None, + ignore_invalid_urls: Optional[bool] = None, + poll_interval: int = 2, + timeout: Optional[int] = None, +) -> ExtractResponse: + started = start_extract( + client, + urls, + prompt=prompt, + schema=schema, + system_prompt=system_prompt, + allow_external_links=allow_external_links, + enable_web_search=enable_web_search, + show_sources=show_sources, + scrape_options=scrape_options, + ignore_invalid_urls=ignore_invalid_urls, + ) + job_id = getattr(started, "id", None) + if not job_id: + return started + return wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout) + diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py index ce786a3aa..eff1008df 100644 --- a/apps/python-sdk/firecrawl/v2/types.py +++ b/apps/python-sdk/firecrawl/v2/types.py @@ -341,6 +341,18 @@ class MapResponse(BaseResponse[MapData]): """Response for map operations.""" pass +# Extract types +class ExtractResponse(BaseModel): + """Response for extract operations (start/status/final).""" + success: Optional[bool] = None + id: Optional[str] = None + status: Optional[Literal["processing", "completed", "failed", "cancelled"]] = None + data: Optional[Any] = None + error: Optional[str] = None + warning: Optional[str] = None + sources: Optional[Dict[str, Any]] = None + expires_at: Optional[datetime] = None + # Usage/limits types class ConcurrencyCheck(BaseModel): """Current concurrency and limits for the team/API key."""