diff --git a/apps/api/requests/v2/crawl.requests.http b/apps/api/requests/v2/crawl.requests.http index 8ddc0cb3f..47d34fdd2 100644 --- a/apps/api/requests/v2/crawl.requests.http +++ b/apps/api/requests/v2/crawl.requests.http @@ -31,8 +31,8 @@ content-type: application/json # "success": true, # "data": { # "url": "https://firecrawl.dev", -# "includes": ["blog", "news"], -# "excludes": [], +# "includePaths": ["blog", "news"], +# "excludePaths": [], # "limit": 50, # "maxDepth": 2 # } @@ -46,5 +46,5 @@ content-type: application/json { "url": "https://firecrawl.dev", - "includes": ["blog", "news"] + "includePaths": ["blog", "news"] } diff --git a/apps/api/src/controllers/v2/crawl-params-preview.ts b/apps/api/src/controllers/v2/crawl-params-preview.ts new file mode 100644 index 000000000..b951784e9 --- /dev/null +++ b/apps/api/src/controllers/v2/crawl-params-preview.ts @@ -0,0 +1,110 @@ +import { Response } from "express"; +import { z } from "zod"; +import { + RequestWithAuth, + crawlRequestSchema, +} from "./types"; +import { logger as _logger } from "../../lib/logger"; +import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract"; +import { CostTracking } from "../../lib/extract/extraction-service"; + +// Define the request schema for params preview +// Only url and prompt are required/relevant for preview +const crawlParamsPreviewRequestSchema = z.object({ + url: z.string().url(), + prompt: z.string().max(10000), +}); + +type CrawlParamsPreviewRequest = z.infer; + +type CrawlParamsPreviewResponse = { + success: boolean; + data?: { + url: string; + includePaths?: string[]; + excludePaths?: string[]; + maxDepth?: number; + maxDiscoveryDepth?: number; + crawlEntireDomain?: boolean; + allowExternalLinks?: boolean; + allowSubdomains?: boolean; + ignoreSitemap?: boolean; + ignoreQueryParameters?: boolean; + deduplicateSimilarURLs?: boolean; + delay?: number; + limit?: number; + }; + error?: string; +}; + +export async function crawlParamsPreviewController( + req: RequestWithAuth<{}, CrawlParamsPreviewResponse, CrawlParamsPreviewRequest>, + res: Response, +) { + const logger = _logger.child({ + module: "api/v2", + method: "crawlParamsPreviewController", + teamId: req.auth.team_id, + }); + + try { + // Parse and validate request body + const parsedBody = crawlParamsPreviewRequestSchema.parse(req.body); + + logger.debug("Crawl params preview request", { + url: parsedBody.url, + prompt: parsedBody.prompt, + }); + + // Generate crawler options from prompt + const costTracking = new CostTracking(); + const { extract } = await generateCrawlerOptionsFromPrompt( + parsedBody.prompt, + logger, + costTracking + ); + + const generatedOptions = extract || {}; + + logger.debug("Generated crawler options from prompt", { + prompt: parsedBody.prompt, + generatedOptions: generatedOptions, + }); + + // Prepare response data + const responseData = { + url: parsedBody.url, + ...generatedOptions, + }; + + // Remove any undefined values for cleaner response + Object.keys(responseData).forEach(key => { + if (responseData[key] === undefined) { + delete responseData[key]; + } + }); + + return res.status(200).json({ + success: true, + data: responseData, + }); + + } catch (error) { + if (error instanceof z.ZodError) { + return res.status(400).json({ + success: false, + error: "Invalid request parameters: " + error.errors.map(e => e.message).join(", "), + }); + } + + logger.error("Failed to generate crawler params preview", { + error: error.message, + prompt: req.body.prompt, + }); + + return res.status(400).json({ + success: false, + error: "Failed to process natural language prompt. Please try rephrasing.", + }); + } +} \ No newline at end of file diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts index 64b9b5c94..732fd4a30 100644 --- a/apps/api/src/routes/v2.ts +++ b/apps/api/src/routes/v2.ts @@ -5,6 +5,7 @@ import { searchController } from "../controllers/v2/search"; import { scrapeController } from "../controllers/v2/scrape"; import { batchScrapeController } from "../controllers/v2/batch-scrape"; import { crawlController } from "../controllers/v2/crawl"; +import { crawlParamsPreviewController } from "../controllers/v2/crawl-params-preview"; import { crawlStatusController } from "../controllers/v2/crawl-status"; import { mapV2Controller } from "../controllers/v2/map"; import { @@ -56,6 +57,13 @@ v2Router.post( wrap(crawlController), ); +v2Router.post( + "/crawl/params-preview", + authMiddleware(RateLimiterMode.Crawl), + checkCreditsMiddleware(), + wrap(crawlParamsPreviewController), +); + v2Router.get( "/crawl/:jobId", authMiddleware(RateLimiterMode.CrawlStatus),