From 322996347bf7788e4e773949671f38d065d6143c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Sun, 14 Sep 2025 13:24:57 +0200
Subject: [PATCH] fix(scrape-worker): cap sitemaps per crawl at 20 to avoid
 queue flooding

---
 apps/api/src/services/worker/scrape-worker.ts | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 02efe206c..1ddcf1ce1 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -713,6 +713,16 @@ async function addKickoffSitemapJob(
   sc: StoredCrawl,
   logger: Logger,
 ) {
+  // TEMP: max 20 sitemaps per crawl
+  if (
+    (await redisEvictConnection.scard(
+      "crawl:" + sourceJob.data.crawl_id + ":sitemaps",
+    )) >= 20
+  ) {
+    logger.debug("Sitemap limit reached, skipping...", { sitemap: sitemapUrl });
+    return;
+  }
+
   const sitemapLocked =
     (await redisEvictConnection.sadd(
       "crawl:" + sourceJob.data.crawl_id + ":sitemaps",