From 7bea613ec093bf365ef5ba49b16e2e766d15702a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 29 Aug 2025 20:13:58 -0400 Subject: [PATCH] feat: add maxPages parameter to PDF parser in v2 scrape API (#2047) * feat: add maxPages parameter to PDF parser - Extend parsersSchema to support both string array ['pdf'] and object array [{'type':'pdf','maxPages':10}] formats - Add shouldParsePDF and getPDFMaxPages helper functions for consistent parser handling - Update PDF processing to respect maxPages limit in both RunPod MU and PdfParse processors - Modify billing calculation to use actual pages processed instead of total pages - Add comprehensive tests for object format parsers, page limiting, and validation - Maintain backward compatibility with existing string array format The maxPages parameter is optional and defaults to unlimited when not specified. Page limiting occurs before processing to avoid unnecessary computation and billing is based on the effective page count for fairness. Co-Authored-By: thomas@sideguide.dev * fix: correct parsersSchema to handle individual parser items - Change union from array-level to item-level in parsersSchema - Now accepts array where each item is either string 'pdf' or object {'type':'pdf','maxPages':10} - When parser is string 'pdf', maxPages is undefined (no limit) - When parser is object, use specified maxPages value - Maintains backward compatibility with existing ['pdf'] format Co-Authored-By: thomas@sideguide.dev * fix: remove maxPages logic from scrapePDFWithParsePDF per PR feedback - Remove maxPages parameter and truncation logic from scrapePDFWithParsePDF - Keep maxPages logic only in scrapePDFWithRunPodMU where it provides cost savings - Addresses feedback from mogery: pdf-parse doesn't cost anything extra to process all pages Co-Authored-By: thomas@sideguide.dev * test: add maxPages parameter tests for crawl and search endpoints - Add crawl endpoint test with PDF maxPages parameter - Add search endpoint test with PDF maxPages parameter - Verify maxPages works end-to-end across all endpoints (scrape, crawl, search) - Ensure schema inheritance and data flow work correctly Co-Authored-By: thomas@sideguide.dev * fix: remove problematic crawl and search tests for maxPages - Remove crawl test that incorrectly uses direct PDF URL - Remove search test that relies on unreliable external search results - maxPages functionality verified through schema inheritance and data flow analysis - Comprehensive tests already exist in parsers.test.ts for core functionality Co-Authored-By: thomas@sideguide.dev * feat: add maxPages parameter support to Python and JavaScript SDKs - Add PDFParser class to Python SDK with max_pages field validation (1-1000) - Update Python SDK parsers field to support Union[List[str], List[Union[str, PDFParser]]] - Add parsers preprocessing in Python SDK to convert snake_case to camelCase - Update JavaScript SDK parsers type to Array - Add maxPages validation to JavaScript SDK ensureValidScrapeOptions - Maintain backward compatibility with existing ['pdf'] string array format - Support mixed formats in both SDKs - Add comprehensive test files for both SDKs Addresses GitHub comment requesting SDK support for maxPages parameter. Co-Authored-By: thomas@sideguide.dev * cleanup: remove temporary test files Co-Authored-By: thomas@sideguide.dev * fix: correct parsers schema to support mixed string and object arrays - Fix parsers schema to properly handle mixed arrays like ['pdf', {type: 'pdf', maxPages: 5}] - Resolves backward compatibility issue that was causing webhook test failures - All parser formats now work: ['pdf'], [{type: 'pdf'}], [{type: 'pdf', maxPages: 10}], mixed arrays Co-Authored-By: thomas@sideguide.dev * Delete SDK_MAXPAGES_IMPLEMENTATION.md * feat: increase maxPages limit from 1000 to 10000 pages - Update backend Zod schema validation in types.ts - Update JavaScript SDK client-side validation - Update API test cases to use new 10000 limit - Addresses GitHub comment feedback from nickscamara Co-Authored-By: thomas@sideguide.dev * fix: update Python SDK maxPages limit from 1000 to 10000 - Fix validation discrepancy between Python SDK (1000) and backend/JS SDK (10000) - Ensures consistent maxPages validation across all SDKs - Addresses critical bug identified in PR review Co-Authored-By: thomas@sideguide.dev * fix: remove SDK-side maxPages validation per PR feedback - Remove maxPages range validation from JavaScript SDK validation.ts - Remove maxPages range validation from Python SDK types.py - Keep backend API validation as single source of truth - Addresses GitHub comment from mogery Co-Authored-By: thomas@sideguide.dev * Nick: --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: thomas@sideguide.dev Co-authored-by: Nicolas --- apps/api/src/__tests__/snips/v2/crawl.test.ts | 1 + .../src/__tests__/snips/v2/parsers.test.ts | 71 ++++++++++++++++++- .../api/src/__tests__/snips/v2/search.test.ts | 1 + apps/api/src/controllers/v2/types.ts | 39 +++++++++- apps/api/src/lib/scrape-billing.ts | 6 +- .../scraper/scrapeURL/engines/index/index.ts | 7 +- .../scraper/scrapeURL/engines/pdf/index.ts | 16 +++-- apps/js-sdk/firecrawl/package.json | 2 +- apps/js-sdk/firecrawl/src/v2/types.ts | 2 +- apps/python-sdk/firecrawl/__init__.py | 2 +- apps/python-sdk/firecrawl/v2/client.py | 7 +- apps/python-sdk/firecrawl/v2/types.py | 29 +++++++- .../firecrawl/v2/utils/validation.py | 16 ++++- 13 files changed, 177 insertions(+), 22 deletions(-) diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.ts index 2e2d9b182..5f121056b 100644 --- a/apps/api/src/__tests__/snips/v2/crawl.test.ts +++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts @@ -272,6 +272,7 @@ describe("Crawl tests", () => { }, 8 * scrapeTimeout); }); } + }); describe("Robots.txt FFI Integration tests", () => { diff --git a/apps/api/src/__tests__/snips/v2/parsers.test.ts b/apps/api/src/__tests__/snips/v2/parsers.test.ts index b9257f5cf..303172a6f 100644 --- a/apps/api/src/__tests__/snips/v2/parsers.test.ts +++ b/apps/api/src/__tests__/snips/v2/parsers.test.ts @@ -57,6 +57,42 @@ describe("Parsers parameter tests", () => { }, scrapeTimeout); }); + describe("Object format", () => { + it.concurrent("accepts parsers: [{type: 'pdf'}] and parses PDF", async () => { + const response = await scrape({ + url: pdfUrl, + parsers: [{ type: "pdf" }], + }, identity); + + expect(response.markdown).toBeDefined(); + expect(response.markdown).toContain("PDF Test File"); + expect(response.metadata.numPages).toBeGreaterThan(0); + }, scrapeTimeout * 2); + + it.concurrent("accepts parsers: [{type: 'pdf', maxPages: 1}] and limits pages", async () => { + const response = await scrape({ + url: pdfUrl, + parsers: [{ type: "pdf", maxPages: 1 }], + }, identity); + + expect(response.markdown).toBeDefined(); + expect(response.markdown).toContain("PDF Test File"); + expect(response.metadata.numPages).toBe(1); + }, scrapeTimeout * 2); + + it.concurrent("handles maxPages larger than actual pages", async () => { + const response = await scrape({ + url: pdfUrl, + parsers: [{ type: "pdf", maxPages: 10000 }], + }, identity); + + expect(response.markdown).toBeDefined(); + expect(response.markdown).toContain("PDF Test File"); + expect(response.metadata.numPages).toBeGreaterThan(0); + expect(response.metadata.numPages).toBeLessThan(10000); + }, scrapeTimeout * 2); + }); + describe("Default behavior", () => { it.concurrent("parses PDF by default when parsers not specified", async () => { @@ -103,6 +139,28 @@ describe("Parsers parameter tests", () => { expect(raw.body.success).toBe(false); expect(raw.body.error).toBe("Bad Request"); }, scrapeTimeout); + + it.concurrent("rejects negative maxPages", async () => { + const raw = await scrapeRaw({ + url: pdfUrl, + parsers: [{ type: "pdf", maxPages: -1 }], + }, identity); + + expect(raw.statusCode).toBe(400); + expect(raw.body.success).toBe(false); + expect(raw.body.error).toBe("Bad Request"); + }, scrapeTimeout); + + it.concurrent("rejects maxPages over limit", async () => { + const raw = await scrapeRaw({ + url: pdfUrl, + parsers: [{ type: "pdf", maxPages: 10001 }], + }, identity); + + expect(raw.statusCode).toBe(400); + expect(raw.body.success).toBe(false); + expect(raw.body.error).toBe("Bad Request"); + }, scrapeTimeout); }); describe("Billing implications", () => { @@ -125,5 +183,16 @@ describe("Parsers parameter tests", () => { // Should bill flat rate (1 credit) when PDF parsing is disabled expect(response.metadata.creditsUsed).toBe(1); }, scrapeTimeout * 2); + + it.concurrent("bills based on limited pages with maxPages", async () => { + const response = await scrape({ + url: pdfUrl, + parsers: [{ type: "pdf", maxPages: 1 }], + }, identity); + + // Should bill based on limited pages (1 page = 1 credit) + expect(response.metadata.creditsUsed).toBe(1); + expect(response.metadata.numPages).toBe(1); + }, scrapeTimeout * 2); }); -}); \ No newline at end of file +}); diff --git a/apps/api/src/__tests__/snips/v2/search.test.ts b/apps/api/src/__tests__/snips/v2/search.test.ts index d1a880b23..ca61d49e5 100644 --- a/apps/api/src/__tests__/snips/v2/search.test.ts +++ b/apps/api/src/__tests__/snips/v2/search.test.ts @@ -33,4 +33,5 @@ describe("Search tests", () => { expect(doc.markdown).toBeDefined(); } }, 125000); + }); diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts index f7057df56..e670df22b 100644 --- a/apps/api/src/controllers/v2/types.ts +++ b/apps/api/src/controllers/v2/types.ts @@ -235,10 +235,47 @@ export type FormatObject = | ScreenshotFormatWithOptions | AttributesFormatWithOptions -export const parsersSchema = z.array(z.enum(["pdf"])).default(["pdf"]); +export const pdfParserWithOptions = z.object({ + type: z.literal("pdf"), + maxPages: z.number().int().positive().finite().max(10000).optional(), +}).strict(); + +export const parsersSchema = z + .array( + z.union([ + z.literal("pdf"), + pdfParserWithOptions, + ]) + ) + .default(["pdf"]); export type Parsers = z.infer; +export function shouldParsePDF(parsers?: Parsers): boolean { + if (!parsers) return true; + return parsers.some(parser => { + if (parser === "pdf") return true; + if (typeof parser === "object" && parser !== null && "type" in parser) { + return (parser as any).type === "pdf"; + } + return false; + }); +} + +export function getPDFMaxPages(parsers?: Parsers): number | undefined { + if (!parsers) return undefined; + const pdfParser = parsers.find(parser => { + if (typeof parser === "object" && parser !== null && "type" in parser) { + return (parser as any).type === "pdf"; + } + return false; + }); + if (pdfParser && typeof pdfParser === "object" && "maxPages" in pdfParser) { + return (pdfParser as any).maxPages; + } + return undefined; +} + function transformIframeSelector(selector: string): string { return selector.replace(/(?:^|[\s,])iframe(?=\s|$|[.#\[:,])/g, (match) => { const prefix = match.match(/^[\s,]/)?.[0] || ''; diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts index ec47a82c5..2e82754e6 100644 --- a/apps/api/src/lib/scrape-billing.ts +++ b/apps/api/src/lib/scrape-billing.ts @@ -1,5 +1,5 @@ import { InternalOptions } from "src/scraper/scrapeURL"; -import { Document, ScrapeOptions, TeamFlags } from "../controllers/v2/types"; +import { Document, ScrapeOptions, TeamFlags, shouldParsePDF } from "../controllers/v2/types"; import { CostTracking } from "./extract/extraction-service"; import { hasFormatOfType } from "./format-utils"; @@ -34,8 +34,8 @@ export async function calculateCreditsToBeBilled(options: ScrapeOptions, interna creditsToBeBilled += (flags?.zdrCost ?? 1); } - const shouldParsePDF = options.parsers?.includes("pdf") ?? true; - if (shouldParsePDF && document.metadata?.numPages !== undefined && document.metadata.numPages > 1) { + const shouldParse = shouldParsePDF(options.parsers); + if (shouldParse && document.metadata?.numPages !== undefined && document.metadata.numPages > 1) { creditsToBeBilled += creditsPerPDFPage * (document.metadata.numPages - 1); } diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts index b076da894..fbd7ec8bf 100644 --- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts @@ -3,6 +3,7 @@ import { EngineScrapeResult } from ".."; import { Meta } from "../.."; import { getIndexFromGCS, hashURL, index_supabase_service, normalizeURLForIndex, saveIndexToGCS, generateURLSplits, addIndexInsertJob, generateDomainSplits, addOMCEJob, addDomainFrequencyJob } from "../../../../services"; import { EngineError, IndexMissError } from "../../error"; +import { shouldParsePDF } from "../../../../controllers/v2/types"; import crypto from "crypto"; export async function sendDocumentToIndex(meta: Meta, document: Document) { @@ -10,7 +11,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) { && !meta.internalOptions.zeroDataRetention && meta.winnerEngine !== "index" && meta.winnerEngine !== "index;documents" - && !(meta.winnerEngine === "pdf" && meta.options.parsers?.includes("pdf") === false) + && !(meta.winnerEngine === "pdf" && !shouldParsePDF(meta.options.parsers)) && ( meta.internalOptions.teamId === "sitemap" || ( @@ -249,13 +250,13 @@ export async function scrapeURLWithIndex(meta: Meta): Promise { meta.logger.debug("Processing PDF document with RunPod MU", { tempFilePath, @@ -66,6 +68,7 @@ async function scrapePDFWithRunPodMU( filename: path.basename(tempFilePath) + ".pdf", timeout: meta.abort.scrapeTimeout(), created_at: Date.now(), + ...(maxPages !== undefined && { max_pages: maxPages }), }, }, logger: meta.logger.child({ @@ -162,9 +165,10 @@ async function scrapePDFWithParsePDF( export async function scrapePDF( meta: Meta, ): Promise { - const shouldParsePDF = meta.options.parsers?.includes("pdf") ?? true; + const shouldParse = shouldParsePDF(meta.options.parsers); + const maxPages = getPDFMaxPages(meta.options.parsers); - if (!shouldParsePDF) { + if (!shouldParse) { if (meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null) { const content = (await readFile(meta.pdfPrefetch.filePath)).toString( "base64", @@ -228,11 +232,12 @@ export async function scrapePDF( } const pdfMetadata = await getPDFMetadata(tempFilePath); + const effectivePageCount = maxPages ? Math.min(pdfMetadata.numPages, maxPages) : pdfMetadata.numPages; - if (pdfMetadata.numPages * MILLISECONDS_PER_PAGE > (meta.abort.scrapeTimeout() ?? Infinity)) { + if (effectivePageCount * MILLISECONDS_PER_PAGE > (meta.abort.scrapeTimeout() ?? Infinity)) { throw new PDFInsufficientTimeError( - pdfMetadata.numPages, - pdfMetadata.numPages * MILLISECONDS_PER_PAGE + 5000, + effectivePageCount, + effectivePageCount * MILLISECONDS_PER_PAGE + 5000, ); } @@ -256,6 +261,7 @@ export async function scrapePDF( }, tempFilePath, base64Content, + maxPages, ); } catch (error) { if ( diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index f9abd453c..313af37ff 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "4.0.0", + "version": "4.1.0", "description": "JavaScript SDK for Firecrawl API", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts index 56d83cfd6..1139df6f0 100644 --- a/apps/js-sdk/firecrawl/src/v2/types.ts +++ b/apps/js-sdk/firecrawl/src/v2/types.ts @@ -133,7 +133,7 @@ export interface ScrapeOptions { timeout?: number; waitFor?: number; mobile?: boolean; - parsers?: string[]; + parsers?: Array; actions?: ActionOption[]; location?: LocationConfig; skipTlsVerification?: boolean; diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py index 54b89235e..55e843736 100644 --- a/apps/python-sdk/firecrawl/__init__.py +++ b/apps/python-sdk/firecrawl/__init__.py @@ -17,7 +17,7 @@ from .v1 import ( V1ChangeTrackingOptions, ) -__version__ = "4.0.0" +__version__ = "4.1.0" # Define the logger for the Firecrawl project logger: logging.Logger = logging.getLogger("firecrawl") diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py index 3c37afc0e..a4bc2f60b 100644 --- a/apps/python-sdk/firecrawl/v2/client.py +++ b/apps/python-sdk/firecrawl/v2/client.py @@ -18,6 +18,7 @@ from .types import ( CrawlResponse, CrawlJob, CrawlParamsRequest, + PDFParser, CrawlParamsData, WebhookConfig, CrawlErrorsResponse, @@ -105,7 +106,7 @@ class FirecrawlClient: timeout: Optional[int] = None, wait_for: Optional[int] = None, mobile: Optional[bool] = None, - parsers: Optional[List[str]] = None, + parsers: Optional[Union[List[str], List[Union[str, PDFParser]]]] = None, actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None, location: Optional['Location'] = None, skip_tls_verification: Optional[bool] = None, @@ -571,7 +572,7 @@ class FirecrawlClient: timeout: Optional[int] = None, wait_for: Optional[int] = None, mobile: Optional[bool] = None, - parsers: Optional[List[str]] = None, + parsers: Optional[Union[List[str], List[Union[str, PDFParser]]]] = None, actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None, location: Optional['Location'] = None, skip_tls_verification: Optional[bool] = None, @@ -759,7 +760,7 @@ class FirecrawlClient: timeout: Optional[int] = None, wait_for: Optional[int] = None, mobile: Optional[bool] = None, - parsers: Optional[List[str]] = None, + parsers: Optional[Union[List[str], List[Union[str, PDFParser]]]] = None, actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None, location: Optional['Location'] = None, skip_tls_verification: Optional[bool] = None, diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py index 319162e14..411295d8e 100644 --- a/apps/python-sdk/firecrawl/v2/types.py +++ b/apps/python-sdk/firecrawl/v2/types.py @@ -278,7 +278,7 @@ class ScrapeOptions(BaseModel): timeout: Optional[int] = None wait_for: Optional[int] = None mobile: Optional[bool] = None - parsers: Optional[List[str]] = None + parsers: Optional[Union[List[str], List[Union[str, 'PDFParser']]]] = None actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None location: Optional['Location'] = None skip_tls_verification: Optional[bool] = None @@ -536,6 +536,11 @@ class PDFAction(BaseModel): landscape: Optional[bool] = None scale: Optional[float] = None +class PDFParser(BaseModel): + """PDF parser configuration with optional page limit.""" + type: Literal["pdf"] = "pdf" + max_pages: Optional[int] = None + # Location types class Location(BaseModel): """Location configuration for scraping.""" @@ -594,6 +599,26 @@ class SearchRequest(BaseModel): return normalized_categories + @field_validator('parsers') + @classmethod + def validate_parsers(cls, v): + """Validate and normalize parsers input.""" + if v is None: + return v + + normalized_parsers = [] + for parser in v: + if isinstance(parser, str): + normalized_parsers.append(parser) + elif isinstance(parser, dict): + normalized_parsers.append(PDFParser(**parser)) + elif isinstance(parser, PDFParser): + normalized_parsers.append(parser) + else: + raise ValueError(f"Invalid parser format: {parser}") + + return normalized_parsers + class LinkResult(BaseModel): """A generic link result with optional metadata (used by search and map).""" url: str @@ -686,4 +711,4 @@ AnyResponse = Union[ MapResponse, SearchResponse, ErrorResponse, -] \ No newline at end of file +] diff --git a/apps/python-sdk/firecrawl/v2/utils/validation.py b/apps/python-sdk/firecrawl/v2/utils/validation.py index f380a02cf..075ad9a2a 100644 --- a/apps/python-sdk/firecrawl/v2/utils/validation.py +++ b/apps/python-sdk/firecrawl/v2/utils/validation.py @@ -311,6 +311,20 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[st converted_action[action_key] = action_value converted_actions.append(converted_action) scrape_data["actions"] = converted_actions + elif key == "parsers": + converted_parsers = [] + for parser in value: + if isinstance(parser, str): + converted_parsers.append(parser) + elif isinstance(parser, dict): + converted_parsers.append(parser) + else: + parser_data = parser.model_dump(exclude_none=True) + # Convert snake_case to camelCase for API + if "max_pages" in parser_data: + parser_data["maxPages"] = parser_data.pop("max_pages") + converted_parsers.append(parser_data) + scrape_data["parsers"] = converted_parsers elif key == "location": # Handle location conversion if isinstance(value, dict): @@ -321,4 +335,4 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[st # For fields that don't need conversion, use as-is scrape_data[key] = value - return scrape_data \ No newline at end of file + return scrape_data \ No newline at end of file