diff --git a/apps/api/src/__tests__/snips/v1/lib.ts b/apps/api/src/__tests__/snips/v1/lib.ts index 9c905c466..72ed02213 100644 --- a/apps/api/src/__tests__/snips/v1/lib.ts +++ b/apps/api/src/__tests__/snips/v1/lib.ts @@ -63,7 +63,7 @@ export async function scrape( ): Promise { const raw = await scrapeRaw(body, identity); expectScrapeToSucceed(raw); - if (body.proxy === "stealth") { + if (body.proxy === "stealth" || body.proxy === "enhanced") { expect(raw.body.data.metadata.proxyUsed).toBe("stealth"); } else if (!body.proxy || body.proxy === "basic") { expect(raw.body.data.metadata.proxyUsed).toBe("basic"); diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index f2a6be45c..dc7156407 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -515,7 +515,7 @@ const baseScrapeOptions = z.strictObject({ fastMode: z.boolean().prefault(false), useMock: z.string().optional(), blockAds: z.boolean().prefault(true), - proxy: z.enum(["basic", "stealth", "auto"]).prefault("basic"), + proxy: z.enum(["basic", "stealth", "enhanced", "auto"]).prefault("basic"), maxAge: z .int() .gte(0) @@ -575,7 +575,9 @@ const extractTransform = (obj: ScrapeOptions) => { } if ( - (obj.proxy === "stealth" || obj.proxy === "auto") && + (obj.proxy === "stealth" || + obj.proxy === "enhanced" || + obj.proxy === "auto") && obj.timeout === 30000 ) { obj = { ...obj, timeout: 120000 }; diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts index f6b813c84..218285bc9 100644 --- a/apps/api/src/controllers/v2/types.ts +++ b/apps/api/src/controllers/v2/types.ts @@ -541,7 +541,7 @@ const baseScrapeOptions = z.strictObject({ fastMode: z.boolean().prefault(false), useMock: z.string().optional(), blockAds: z.boolean().prefault(true), - proxy: z.enum(["basic", "stealth", "auto"]).prefault("auto"), + proxy: z.enum(["basic", "stealth", "enhanced", "auto"]).prefault("auto"), maxAge: z.int().gte(0).optional(), minAge: z.int().gte(0).optional(), storeInCache: z.boolean().prefault(true), @@ -596,7 +596,9 @@ const extractTransformImpl = ( } if ( - (obj.proxy === "stealth" || obj.proxy === "auto") && + (obj.proxy === "stealth" || + obj.proxy === "enhanced" || + obj.proxy === "auto") && obj.timeout === 30000 ) { result = { ...result, timeout: 120000 }; diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts index 42071bfa6..5e2d90e3d 100644 --- a/apps/api/src/scraper/scrapeURL/index.ts +++ b/apps/api/src/scraper/scrapeURL/index.ts @@ -175,7 +175,7 @@ function buildFeatureFlags( flags.add("useFastMode"); } - if (options.proxy === "stealth") { + if (options.proxy === "stealth" || options.proxy === "enhanced") { flags.add("stealthProxy"); } @@ -254,10 +254,7 @@ async function buildMetaObject( const abortHandle = options.timeout !== undefined ? setTimeout( - () => - abortController.abort( - new ScrapeJobTimeoutError(), - ), + () => abortController.abort(new ScrapeJobTimeoutError()), options.timeout, ) : undefined; diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index 6b28489f7..941be779b 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "4.11.5", + "version": "4.12.0", "description": "JavaScript SDK for Firecrawl API", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.ts index ed72cb763..77c78e21f 100644 --- a/apps/js-sdk/firecrawl/src/v1/index.ts +++ b/apps/js-sdk/firecrawl/src/v1/index.ts @@ -131,7 +131,7 @@ export interface CrawlScrapeOptions { skipTlsVerification?: boolean; removeBase64Images?: boolean; blockAds?: boolean; - proxy?: "basic" | "stealth" | "auto"; + proxy?: "basic" | "stealth" | "enhanced" | "auto"; storeInCache?: boolean; maxAge?: number; parsePDF?: boolean; diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts index b3ec7523e..df9c79de8 100644 --- a/apps/js-sdk/firecrawl/src/v2/types.ts +++ b/apps/js-sdk/firecrawl/src/v2/types.ts @@ -153,7 +153,7 @@ export interface ScrapeOptions { fastMode?: boolean; useMock?: string; blockAds?: boolean; - proxy?: 'basic' | 'stealth' | 'auto' | string; + proxy?: 'basic' | 'stealth' | 'enhanced' | 'auto' | string; maxAge?: number; minAge?: number; storeInCache?: boolean; diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py index fbea47891..5edca90e0 100644 --- a/apps/python-sdk/firecrawl/__init__.py +++ b/apps/python-sdk/firecrawl/__init__.py @@ -17,7 +17,7 @@ from .v1 import ( V1ChangeTrackingOptions, ) -__version__ = "4.13.4" +__version__ = "4.14.0" # Define the logger for the Firecrawl project logger: logging.Logger = logging.getLogger("firecrawl") diff --git a/apps/python-sdk/firecrawl/v1/client.py b/apps/python-sdk/firecrawl/v1/client.py index 797cb8b6b..bc21729da 100644 --- a/apps/python-sdk/firecrawl/v1/client.py +++ b/apps/python-sdk/firecrawl/v1/client.py @@ -150,7 +150,7 @@ class V1ScrapeOptions(pydantic.BaseModel): skipTlsVerification: Optional[bool] = None removeBase64Images: Optional[bool] = None blockAds: Optional[bool] = None - proxy: Optional[Literal["basic", "stealth", "auto"]] = None + proxy: Optional[Literal["basic", "stealth", "enhanced", "auto"]] = None changeTrackingOptions: Optional[V1ChangeTrackingOptions] = None maxAge: Optional[int] = None storeInCache: Optional[bool] = None @@ -542,7 +542,7 @@ class V1FirecrawlApp: skip_tls_verification: Optional[bool] = None, remove_base64_images: Optional[bool] = None, block_ads: Optional[bool] = None, - proxy: Optional[Literal["basic", "stealth", "auto"]] = None, + proxy: Optional[Literal["basic", "stealth", "enhanced", "auto"]] = None, parse_pdf: Optional[bool] = None, extract: Optional[V1JsonConfig] = None, json_options: Optional[V1JsonConfig] = None, @@ -1441,7 +1441,7 @@ class V1FirecrawlApp: skip_tls_verification: Optional[bool] = None, remove_base64_images: Optional[bool] = None, block_ads: Optional[bool] = None, - proxy: Optional[Literal["basic", "stealth", "auto"]] = None, + proxy: Optional[Literal["basic", "stealth", "enhanced", "auto"]] = None, extract: Optional[V1JsonConfig] = None, json_options: Optional[V1JsonConfig] = None, actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None, @@ -1582,7 +1582,7 @@ class V1FirecrawlApp: skip_tls_verification: Optional[bool] = None, remove_base64_images: Optional[bool] = None, block_ads: Optional[bool] = None, - proxy: Optional[Literal["basic", "stealth", "auto"]] = None, + proxy: Optional[Literal["basic", "stealth", "enhanced", "auto"]] = None, extract: Optional[V1JsonConfig] = None, json_options: Optional[V1JsonConfig] = None, actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None, @@ -1722,7 +1722,7 @@ class V1FirecrawlApp: skip_tls_verification: Optional[bool] = None, remove_base64_images: Optional[bool] = None, block_ads: Optional[bool] = None, - proxy: Optional[Literal["basic", "stealth", "auto"]] = None, + proxy: Optional[Literal["basic", "stealth", "enhanced", "auto"]] = None, extract: Optional[V1JsonConfig] = None, json_options: Optional[V1JsonConfig] = None, actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None, @@ -3523,7 +3523,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp): skip_tls_verification: Optional[bool] = None, remove_base64_images: Optional[bool] = None, block_ads: Optional[bool] = None, - proxy: Optional[Literal["basic", "stealth", "auto"]] = None, + proxy: Optional[Literal["basic", "stealth", "enhanced", "auto"]] = None, parse_pdf: Optional[bool] = None, extract: Optional[V1JsonConfig] = None, json_options: Optional[V1JsonConfig] = None, @@ -3657,7 +3657,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp): skip_tls_verification: Optional[bool] = None, remove_base64_images: Optional[bool] = None, block_ads: Optional[bool] = None, - proxy: Optional[Literal["basic", "stealth", "auto"]] = None, + proxy: Optional[Literal["basic", "stealth", "enhanced", "auto"]] = None, extract: Optional[V1JsonConfig] = None, json_options: Optional[V1JsonConfig] = None, actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None, @@ -3796,7 +3796,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp): skip_tls_verification: Optional[bool] = None, remove_base64_images: Optional[bool] = None, block_ads: Optional[bool] = None, - proxy: Optional[Literal["basic", "stealth", "auto"]] = None, + proxy: Optional[Literal["basic", "stealth", "enhanced", "auto"]] = None, extract: Optional[V1JsonConfig] = None, json_options: Optional[V1JsonConfig] = None, actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None, diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py index 6439c3142..33c3e9001 100644 --- a/apps/python-sdk/firecrawl/v2/types.py +++ b/apps/python-sdk/firecrawl/v2/types.py @@ -523,7 +523,7 @@ class ScrapeOptions(BaseModel): fast_mode: Optional[bool] = None use_mock: Optional[str] = None block_ads: Optional[bool] = None - proxy: Optional[Literal["basic", "stealth", "auto"]] = None + proxy: Optional[Literal["basic", "stealth", "enhanced", "auto"]] = None max_age: Optional[int] = None min_age: Optional[int] = None store_in_cache: Optional[bool] = None