From a4ffd546775c9a8f781995914a28ed3972bb27aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Mon, 4 Aug 2025 01:09:29 +0200 Subject: [PATCH] v2: scrape-status --- apps/api/src/controllers/v2/scrape-status.ts | 53 ++++++++++++++++++++ apps/api/src/routes/v2.ts | 7 +++ 2 files changed, 60 insertions(+) create mode 100644 apps/api/src/controllers/v2/scrape-status.ts diff --git a/apps/api/src/controllers/v2/scrape-status.ts b/apps/api/src/controllers/v2/scrape-status.ts new file mode 100644 index 000000000..8d75a8c66 --- /dev/null +++ b/apps/api/src/controllers/v2/scrape-status.ts @@ -0,0 +1,53 @@ +import { supabaseGetJobByIdOnlyData } from "../../lib/supabase-jobs"; +import { getJob } from "./crawl-status"; +import { logger as _logger } from "../../lib/logger"; + +export async function scrapeStatusController(req: any, res: any) { + const logger = _logger.child({ + module: "scrape-status", + method: "scrapeStatusController", + teamId: req.auth.team_id, + jobId: req.params.jobId, + scrapeId: req.params.jobId, + zeroDataRetention: req.acuc?.flags?.forceZDR, + }); + + if (req.acuc?.flags?.forceZDR) { + return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on scrape status. Please contact support@firecrawl.com to unblock this feature." }); + } + + const job = await supabaseGetJobByIdOnlyData(req.params.jobId, logger); + + if (!job) { + return res.status(404).json({ + success: false, + error: "Job not found.", + }); + } + + if ( + job?.team_id !== req.auth.team_id + ) { + return res.status(403).json({ + success: false, + error: "You are not allowed to access this resource.", + }); + } + + const jobData = await getJob(req.params.jobId); + const data = Array.isArray(jobData?.returnvalue) + ? jobData?.returnvalue[0] + : jobData?.returnvalue; + + if (!data) { + return res.status(404).json({ + success: false, + error: "Job not found.", + }); + } + + return res.status(200).json({ + success: true, + data, + }); +} diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts index ead47a4b2..b819d420b 100644 --- a/apps/api/src/routes/v2.ts +++ b/apps/api/src/routes/v2.ts @@ -10,6 +10,7 @@ import { crawlStatusController } from "../controllers/v2/crawl-status"; import { mapV2Controller } from "../controllers/v2/map"; import { crawlErrorsController } from "../controllers/v2/crawl-errors"; import { ongoingCrawlsController } from "../controllers/v2/crawl-ongoing"; +import { scrapeStatusController } from "../controllers/v2/scrape-status"; import { authMiddleware, checkCreditsMiddleware, @@ -36,6 +37,12 @@ v2Router.post( wrap(scrapeController), ); +v2Router.get( + "/scrape/:jobId", + authMiddleware(RateLimiterMode.CrawlStatus), + wrap(scrapeStatusController), +); + v2Router.post( "/batch/scrape", authMiddleware(RateLimiterMode.Scrape),