From 2dd1a34407de82d8850a5c686ed656632894173c Mon Sep 17 00:00:00 2001
From: "firecrawl-spring[bot]"
 <254786068+firecrawl-spring[bot]@users.noreply.github.com>
Date: Fri, 30 Jan 2026 23:26:32 -0800
Subject: [PATCH] chore: remove firecrawl_jobs cleanup from ZDR cleaner (#2777)

Remove the cleanUpFirecrawlJobs function that cleaned up legacy
firecrawl_jobs table data. The ZDR cleaner now only handles the
new requests/scrapes tables.

Co-authored-by: firecrawl-spring[bot] <254786068+firecrawl-spring[bot]@users.noreply.github.com>
Co-authored-by: micahstairs <micah@sideguide.dev>
---
 .../src/controllers/v0/admin/zdrcleaner.ts    | 96 +------------------
 1 file changed, 1 insertion(+), 95 deletions(-)

diff --git a/apps/api/src/controllers/v0/admin/zdrcleaner.ts b/apps/api/src/controllers/v0/admin/zdrcleaner.ts
index 7e1ad206d..504094501 100644
--- a/apps/api/src/controllers/v0/admin/zdrcleaner.ts
+++ b/apps/api/src/controllers/v0/admin/zdrcleaner.ts
@@ -9,97 +9,6 @@ async function cleanUpJob(jobId: string) {
   await removeJobFromGCS(jobId);
 }
 
-async function cleanUpFirecrawlJobs(
-  specificTeamId: string | null,
-  _logger: Logger,
-) {
-  const logger = _logger.child({
-    ...(specificTeamId ? { teamId: specificTeamId } : {}),
-    method: "cleanUpFirecrawlJobs",
-  });
-
-  const cleanedUp: number[] = [];
-
-  try {
-    for (let i = 0; ; i++) {
-      let selector = supabase_service
-        .from("firecrawl_jobs")
-        .select("id, job_id");
-
-      if (specificTeamId) {
-        selector = selector
-          .eq("team_id", specificTeamId)
-          .not("dr_clean_by", "is", null);
-      } else {
-        selector = selector
-          .lte("dr_clean_by", new Date().toISOString())
-          .gte(
-            "dr_clean_by",
-            new Date(Date.now() - 1000 * 60 * 60 * 24 * 7).toISOString(),
-          );
-        // Explanation for the gte: since the cleaner should run every 5 minutes, it is very unlikely that
-        // the cleaner will be down for 7 days without anyone noticing.
-        // Since the firecrawl_jobs table is incredibly large, even with the index on dr_clean_by,
-        // not giving the select a lower bound guarantees that the select will not run with an empty result
-        // in reasonable time.
-        // Therefore, we give it a lower bound which should never cause problems.
-      }
-
-      const { data: jobs } = await selector
-        .range(i * 1000, (i + 1) * 1000)
-        .throwOnError();
-
-      if (jobs?.length === 0) {
-        break;
-      }
-
-      for (let i = 0; i < Math.ceil((jobs?.length ?? 0) / 50); i++) {
-        const theseJobs = (jobs ?? []).slice(i * 50, (i + 1) * 50);
-        await Promise.allSettled(
-          theseJobs.map(async job => {
-            try {
-              await cleanUpJob(job.job_id);
-              cleanedUp.push(job.id);
-            } catch (error) {
-              logger.error(`Error cleaning up job`, {
-                method: "cleanUpJob",
-                jobId: job.job_id,
-                scrapeId: job.job_id,
-                error,
-              });
-              throw error;
-            }
-          }) ?? [],
-        );
-      }
-
-      if ((jobs ?? []).length < 1000) {
-        break;
-      }
-    }
-  } catch (error) {
-    logger.error(`Error looping through jobs`, {
-      error,
-    });
-  }
-
-  if (cleanedUp.length > 0) {
-    try {
-      await supabase_service
-        .from("firecrawl_jobs")
-        .update({
-          dr_clean_by: null,
-        })
-        .in("id", cleanedUp)
-        .throwOnError();
-    } catch (error) {
-      logger.error(`Error setting cleanup value on team`, {
-        error,
-      });
-    }
-  }
-}
-
 async function cleanUpRequests(specificTeamId: string | null, _logger: Logger) {
   const logger = _logger.child({
     ...(specificTeamId ? { teamId: specificTeamId } : {}),
@@ -212,10 +121,7 @@ export async function zdrcleanerController(req: Request, res: Response) {
 
   const teamId = (req.query.teamId as string | undefined) ?? null;
 
-  // Clean up old firecrawl_jobs table (legacy)
-  await cleanUpFirecrawlJobs(teamId, logger);
-
-  // Clean up new requests/scrapes tables
+  // Clean up requests/scrapes tables
   await cleanUpRequests(teamId, logger);
 
   logger.info("ZDR Cleaner finished!");