diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts index 7eba40a60..0e1fffc95 100644 --- a/apps/api/src/controllers/v1/extract.ts +++ b/apps/api/src/controllers/v1/extract.ts @@ -14,6 +14,7 @@ import { performExtraction_F0 } from "../../lib/extract/fire-0/extraction-servic import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings"; import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist"; import { logger as _logger } from "../../lib/logger"; +import { fromV1ScrapeOptions } from "../v2/types"; export async function oldExtract( req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>, @@ -89,8 +90,15 @@ export async function extractController( zeroDataRetention: req.acuc?.flags?.forceZDR, }); + const scrapeOptions = req.body.scrapeOptions + ? fromV1ScrapeOptions(req.body.scrapeOptions, req.body.scrapeOptions.timeout, req.auth.team_id).scrapeOptions + : undefined; + const jobData = { - request: req.body, + request: { + ...req.body, + scrapeOptions, + }, teamId: req.auth.team_id, subId: req.acuc?.sub_id, extractId, diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.ts new file mode 100644 index 000000000..33e5a7cb4 --- /dev/null +++ b/apps/api/src/controllers/v2/extract.ts @@ -0,0 +1,87 @@ +import { Response } from "express"; +import { + RequestWithAuth, + ExtractRequest, + extractRequestSchema, + ExtractResponse, +} from "./types"; +import { getExtractQueue } from "../../services/queue-service"; +import { saveExtract } from "../../lib/extract/extract-redis"; +import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings"; +import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist"; +import { logger as _logger } from "../../lib/logger"; + +/** + * Extracts data from the provided URLs based on the request parameters. + * Currently in beta. + * @param req - The request object containing authentication and extraction details. + * @param res - The response object to send the extraction results. + * @returns A promise that resolves when the extraction process is complete. + */ +export async function extractController( + req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>, + res: Response, +) { + const originalRequest = { ...req.body }; + req.body = extractRequestSchema.parse(req.body); + + if (req.acuc?.flags?.forceZDR) { + return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on extract. Please contact support@firecrawl.com to unblock this feature." }); + } + + const invalidURLs: string[] = req.body.urls?.filter((url: string) => isUrlBlocked(url, req.acuc?.flags ?? null)) ?? []; + + if (invalidURLs.length > 0 && !req.body.ignoreInvalidURLs) { + if (!res.headersSent) { + return res.status(403).json({ + success: false, + error: BLOCKLISTED_URL_MESSAGE, + }); + } + } + + const extractId = crypto.randomUUID(); + + _logger.info("Extract starting...", { + request: req.body, + originalRequest, + teamId: req.auth.team_id, + team_id: req.auth.team_id, + subId: req.acuc?.sub_id, + extractId, + zeroDataRetention: req.acuc?.flags?.forceZDR, + }); + + const jobData = { + request: req.body, + teamId: req.auth.team_id, + subId: req.acuc?.sub_id, + extractId, + agent: req.body.agent, + }; + + await saveExtract(extractId, { + id: extractId, + team_id: req.auth.team_id, + createdAt: Date.now(), + status: "processing", + showSteps: req.body.__experimental_streamSteps, + showLLMUsage: req.body.__experimental_llmUsage, + showSources: req.body.__experimental_showSources || req.body.showSources, + showCostTracking: req.body.__experimental_showCostTracking, + zeroDataRetention: req.acuc?.flags?.forceZDR, + }); + + await getExtractQueue().add(extractId, jobData, { + jobId: extractId, + }); + + return res.status(200).json({ + success: true, + id: extractId, + urlTrace: [], + ...(invalidURLs.length > 0 && req.body.ignoreInvalidURLs ? { + invalidURLs, + } : {}), + }); +} diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts index ee6c80a30..3f833ccd0 100644 --- a/apps/api/src/controllers/v2/types.ts +++ b/apps/api/src/controllers/v2/types.ts @@ -9,7 +9,7 @@ import { Document as V0Document, WebSearchResult, } from "../../lib/entities"; -import { ScrapeOptions as V1ScrapeOptions } from "../v1/types"; +import { agentOptionsExtract, ScrapeOptions as V1ScrapeOptions } from "../v1/types"; import { InternalOptions } from "../../scraper/scrapeURL"; export enum IntegrationEnum { @@ -391,7 +391,7 @@ import type { CostTracking } from "../../lib/extract/extraction-service"; const ajv = new Ajv(); -export const extractV1Options = z +export const extractOptions = z .object({ urls: url .array() @@ -426,6 +426,7 @@ export const extractV1Options = z integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null), urlTrace: z.boolean().default(false), timeout: z.number().int().positive().finite().safe().optional(), + agent: agentOptionsExtract.optional(), __experimental_streamSteps: z.boolean().default(false), __experimental_llmUsage: z.boolean().default(false), __experimental_showSources: z.boolean().default(false), @@ -457,8 +458,8 @@ export const extractV1Options = z : x.scrapeOptions, })); -export type ExtractV1Options = z.infer; -export const extractRequestSchema = extractV1Options; +export type ExtractOptions = z.infer; +export const extractRequestSchema = extractOptions; export type ExtractRequest = z.infer; export type ExtractRequestInput = z.input; diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.ts index fe2475086..1b84733b2 100644 --- a/apps/api/src/lib/extract/document-scraper.ts +++ b/apps/api/src/lib/extract/document-scraper.ts @@ -1,13 +1,10 @@ -import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions as scrapeOptionsSchema } from "../../controllers/v1/types"; -import { logger } from "../logger"; +import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions as scrapeOptionsSchema } from "../../controllers/v2/types"; import { getScrapeQueue } from "../../services/queue-service"; import { waitForJob } from "../../services/queue-jobs"; import { addScrapeJob } from "../../services/queue-jobs"; import { getJobPriority } from "../job-priority"; import type { Logger } from "winston"; -import { getJobFromGCS } from "../gcs-jobs"; import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist"; -import { fromV1ScrapeOptions } from "../../controllers/v2/types"; interface ScrapeDocumentOptions { url: string; @@ -42,19 +39,16 @@ export async function scrapeDocument( from_extract: true, }); - const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(scrapeOptionsSchema.parse({ - ...internalScrapeOptions, - maxAge: 4 * 60 * 60 * 1000, - }), internalScrapeOptions.timeout, options.teamId) - await addScrapeJob( { url: options.url, mode: "single_urls", team_id: options.teamId, - scrapeOptions: scrapeOptions, + scrapeOptions: scrapeOptionsSchema.parse({ + ...internalScrapeOptions, + maxAge: 4 * 60 * 60 * 1000, + }), internalOptions: { - ...internalOptions, teamId: options.teamId, saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false, bypassBilling: true, diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.ts index 8c2eb3801..65215c60b 100644 --- a/apps/api/src/lib/extract/extraction-service.ts +++ b/apps/api/src/lib/extract/extraction-service.ts @@ -1,10 +1,10 @@ import { Document, ExtractRequest, - isAgentExtractModelValid, TokenUsage, URLTrace, -} from "../../controllers/v1/types"; +} from "../../controllers/v2/types"; +import { isAgentExtractModelValid } from "../../controllers/v1/types"; import { logger as _logger } from "../logger"; import { generateBasicCompletion, processUrl } from "./url-processor"; import { scrapeDocument } from "./document-scraper"; diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts index c22828c5a..a4b524a64 100644 --- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts +++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts @@ -1,12 +1,10 @@ -import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions as scrapeOptionsSchema } from "../../../controllers/v1/types"; -import { logger } from "../../logger"; +import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions as scrapeOptionsSchema } from "../../../controllers/v2/types"; import { getScrapeQueue } from "../../../services/queue-service"; import { waitForJob } from "../../../services/queue-jobs"; import { addScrapeJob } from "../../../services/queue-jobs"; import { getJobPriority } from "../../job-priority"; import type { Logger } from "winston"; import { isUrlBlocked } from "../../../scraper/WebScraper/utils/blocklist"; -import { fromV1ScrapeOptions } from "../../../controllers/v2/types"; interface ScrapeDocumentOptions { url: string; @@ -41,10 +39,10 @@ export async function scrapeDocument_F0( from_extract: true, }); - const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(scrapeOptionsSchema.parse({ + const scrapeOptions = scrapeOptionsSchema.parse({ ...internalScrapeOptions, maxAge: 4 * 60 * 60 * 1000, - }), internalScrapeOptions.timeout, options.teamId); + }); await addScrapeJob( { @@ -53,7 +51,6 @@ export async function scrapeDocument_F0( team_id: options.teamId, scrapeOptions, internalOptions: { - ...internalOptions, teamId: options.teamId, bypassBilling: true, }, diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts index f03de5cc4..079489238 100644 --- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts +++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts @@ -3,7 +3,7 @@ import { ExtractRequest, TokenUsage, URLTrace, - } from "../../../controllers/v1/types"; + } from "../../../controllers/v2/types"; import { logger as _logger } from "../../logger"; import { scrapeDocument_F0 } from "./document-scraper-f0"; import { billTeam } from "../../../services/billing/credit_billing";