From 0365e3d76d33156c0bfad0230e951999d92bf8cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Sun, 10 Aug 2025 23:40:00 +0200
Subject: [PATCH 01/12] feat(v1/search): add query to logs to make debugging
 502s easier

---
 apps/api/src/controllers/v1/search.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index d4429e15b..d4a65f71f 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -203,6 +203,7 @@ export async function searchController(
     module: "search",
     method: "searchController",
     zeroDataRetention: req.acuc?.flags?.forceZDR,
+    searchQuery: req.body.query.slice(0, 100),
   });
 
   if (req.acuc?.flags?.forceZDR) {

From e2b27dc7cf3ffdfe51ad8b6b0f833a5e36787eb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Mon, 11 Aug 2025 12:27:08 +0200
Subject: [PATCH 02/12] fix(index-worker); precrawl stalling

---
 apps/api/src/services/indexing/index-worker.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index a041da939..bbefa3240 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -7,6 +7,7 @@ import {
   redisConnection,
   getBillingQueue,
   getPrecrawlQueue,
+  precrawlQueueName,
 } from "../queue-service";
 import { processBillingBatch, queueBillingOperation, startBillingBatchProcessing } from "../billing/batch_billing";
 import systemMonitor from "../system-monitor";
@@ -219,7 +220,7 @@ const workerFun = async (queue: Queue, jobProcessor: (token: string, job: Job) =
     connection: redisConnection,
     lockDuration: workerLockDuration,
     stalledInterval: workerStalledCheckInterval,
-    maxStalledCount: 10,
+    maxStalledCount: queue.name === precrawlQueueName ? 0 : 10,
   });
 
   worker.startStalledCheckTimer();

From 29c684b7412c5ab88c0a8342df895a71bba3655b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Mon, 11 Aug 2025 12:36:33 +0200
Subject: [PATCH 03/12] fix(index-worker): precrawl job never moved to
 completed

---
 apps/api/src/services/indexing/index-worker.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index bbefa3240..f39fef2a7 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -190,6 +190,7 @@ const processPrecrawlJobInternal = async (token: string, job: Job) => {
       }
     }
 
+    await job.moveToCompleted({ success: true }, token, false);
   } catch (error) {
     logger.error("Error processing precrawl job", { error });
     await job.moveToFailed(error, token, false);

From 5cd20c828fb78c1146cf911ec30467d2550ff3ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Tue, 12 Aug 2025 12:46:41 +0200
Subject: [PATCH 04/12] feat(v0): better usage tracking

---
 apps/api/src/controllers/v0/crawl-cancel.ts | 3 +++
 apps/api/src/controllers/v0/crawl-status.ts | 3 +++
 apps/api/src/controllers/v0/crawl.ts        | 7 +++++--
 apps/api/src/controllers/v0/keyAuth.ts      | 3 +++
 apps/api/src/controllers/v0/scrape.ts       | 7 +++++--
 apps/api/src/controllers/v0/search.ts       | 7 +++++--
 6 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/apps/api/src/controllers/v0/crawl-cancel.ts b/apps/api/src/controllers/v0/crawl-cancel.ts
index a454f700c..ff8b88040 100644
--- a/apps/api/src/controllers/v0/crawl-cancel.ts
+++ b/apps/api/src/controllers/v0/crawl-cancel.ts
@@ -26,6 +26,9 @@ export async function crawlCancelController(req: Request, res: Response) {
 
     redisEvictConnection.sadd("teams_using_v0", team_id)
       .catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id }));
+    
+    redisEvictConnection.sadd("teams_using_v0:" + team_id, "crawl:" + req.params.jobId + ":cancel")
+      .catch(error => logger.error("Failed to add team to teams_using_v0 (2)", { error, team_id }));
 
     const sc = await getCrawl(req.params.jobId);
     if (!sc) {
diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.ts
index 9f6e5b7bb..78c2ae0b9 100644
--- a/apps/api/src/controllers/v0/crawl-status.ts
+++ b/apps/api/src/controllers/v0/crawl-status.ts
@@ -87,6 +87,9 @@ export async function crawlStatusController(req: Request, res: Response) {
 
     redisEvictConnection.sadd("teams_using_v0", team_id)
       .catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id }));
+    
+    redisEvictConnection.sadd("teams_using_v0:" + team_id, "crawl:" + req.params.jobId + ":status")
+      .catch(error => logger.error("Failed to add team to teams_using_v0 (2)", { error, team_id }));
 
     const sc = await getCrawl(req.params.jobId);
     if (!sc) {
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index abd858a91..63b90e809 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -45,8 +45,13 @@ export async function crawlController(req: Request, res: Response) {
       return res.status(400).json({ error: "Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API." });
     }
 
+    const id = uuidv4();
+
     redisEvictConnection.sadd("teams_using_v0", team_id)
       .catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id }));
+    
+    redisEvictConnection.sadd("teams_using_v0:" + team_id, "crawl:" + id)
+      .catch(error => logger.error("Failed to add team to teams_using_v0 (2)", { error, team_id }));
 
     if (req.headers["x-idempotency-key"]) {
       const isIdempotencyValid = await validateIdempotencyKey(req);
@@ -154,8 +159,6 @@ export async function crawlController(req: Request, res: Response) {
     //   }
     // }
 
-    const id = uuidv4();
-
     await logCrawl(id, team_id);
 
     const { scrapeOptions, internalOptions } = fromLegacyScrapeOptions(
diff --git a/apps/api/src/controllers/v0/keyAuth.ts b/apps/api/src/controllers/v0/keyAuth.ts
index febac4115..87179e19f 100644
--- a/apps/api/src/controllers/v0/keyAuth.ts
+++ b/apps/api/src/controllers/v0/keyAuth.ts
@@ -19,6 +19,9 @@ export const keyAuthController = async (req: Request, res: Response) => {
 
     redisEvictConnection.sadd("teams_using_v0", auth.team_id)
       .catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id: auth.team_id }));
+    
+    redisEvictConnection.sadd("teams_using_v0:" + auth.team_id, "keyAuth")
+      .catch(error => logger.error("Failed to add team to teams_using_v0 (2)", { error, team_id: auth.team_id }));
 
     // if success, return success: true
     return res.status(200).json({ success: true });
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index d4bd73bbc..3e02e4e3d 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -181,9 +181,14 @@ export async function scrapeController(req: Request, res: Response) {
       return res.status(400).json({ error: "Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API." });
     }
 
+    const jobId = uuidv4();
+
     redisEvictConnection.sadd("teams_using_v0", team_id)
       .catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id }));
 
+    redisEvictConnection.sadd("teams_using_v0:" + team_id, "scrape:" + jobId)
+      .catch(error => logger.error("Failed to add team to teams_using_v0 (2)", { error, team_id }));
+
     const crawlerOptions = req.body.crawlerOptions ?? {};
     const pageOptions = { ...defaultPageOptions, ...req.body.pageOptions };
     const extractorOptions = {
@@ -228,8 +233,6 @@ export async function scrapeController(req: Request, res: Response) {
       });
     }
 
-    const jobId = uuidv4();
-
     const startTime = new Date().getTime();
     const result = await scrapeHelper(
       jobId,
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts
index aaff178c2..2341bac45 100644
--- a/apps/api/src/controllers/v0/search.ts
+++ b/apps/api/src/controllers/v0/search.ts
@@ -174,9 +174,14 @@ export async function searchController(req: Request, res: Response) {
       return res.status(400).json({ error: "Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API." });
     }
 
+    const jobId = uuidv4();
+
     redisEvictConnection.sadd("teams_using_v0", team_id)
       .catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id }));
     
+    redisEvictConnection.sadd("teams_using_v0:" + team_id, "search:" + jobId)
+      .catch(error => logger.error("Failed to add team to teams_using_v0 (2)", { error, team_id }));
+
     const crawlerOptions = req.body.crawlerOptions ?? {};
     const pageOptions = req.body.pageOptions ?? {
       includeHtml: req.body.pageOptions?.includeHtml ?? false,
@@ -189,8 +194,6 @@ export async function searchController(req: Request, res: Response) {
 
     const searchOptions = req.body.searchOptions ?? { limit: 5 };
 
-    const jobId = uuidv4();
-
     try {
       const { success: creditsCheckSuccess, message: creditsCheckMessage } =
         await checkTeamCredits(chunk, team_id, 1);

From 2fa17facb75b4e87efc5743ac705f17564a2054e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Tue, 12 Aug 2025 15:59:04 +0200
Subject: [PATCH 05/12] fix(rust-sdk): pkg vuln

---
 apps/rust-sdk/Cargo.lock | 147 ++++++++++++++++++---------------------
 apps/rust-sdk/Cargo.toml |   2 +-
 2 files changed, 70 insertions(+), 79 deletions(-)

diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lock
index 35d2f12e6..bb080ac77 100644
--- a/apps/rust-sdk/Cargo.lock
+++ b/apps/rust-sdk/Cargo.lock
@@ -52,9 +52,9 @@ dependencies = [
 
 [[package]]
 name = "anstream"
-version = "0.6.19"
+version = "0.6.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933"
+checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192"
 dependencies = [
  "anstyle",
  "anstyle-parse",
@@ -82,29 +82,29 @@ dependencies = [
 
 [[package]]
 name = "anstyle-query"
-version = "1.1.3"
+version = "1.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9"
+checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2"
 dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
 name = "anstyle-wincon"
-version = "3.0.9"
+version = "3.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882"
+checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a"
 dependencies = [
  "anstyle",
  "once_cell_polyfill",
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
 name = "anyhow"
-version = "1.0.98"
+version = "1.0.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"
+checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100"
 
 [[package]]
 name = "arrayref"
@@ -345,9 +345,9 @@ checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
 
 [[package]]
 name = "bytemuck"
-version = "1.23.1"
+version = "1.23.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c76a5792e44e4abe34d3abf15636779261d45a7450612059293d1d2cfc63422"
+checksum = "3995eaeebcdf32f91f980d360f78732ddc061097ab4e39991ae7a6ace9194677"
 
 [[package]]
 name = "byteorder"
@@ -369,9 +369,9 @@ checksum = "2e93abca9e28e0a1b9877922aacb20576e05d4679ffa78c3d6dc22a26a216659"
 
 [[package]]
 name = "cc"
-version = "1.2.30"
+version = "1.2.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7"
+checksum = "2352e5597e9c544d5e6d9c95190d5d27738ade584fa8db0a16e130e5c2b5296e"
 dependencies = [
  "jobserver",
  "libc",
@@ -399,9 +399,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.5.41"
+version = "4.5.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9"
+checksum = "1c1f056bae57e3e54c3375c41ff79619ddd13460a17d7438712bd0d83fda4ff8"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -409,9 +409,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.41"
+version = "4.5.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d"
+checksum = "b3e7f4214277f3c7aa526a59dd3fbe306a370daee1f8b7b8c987069cd8e888a8"
 dependencies = [
  "anstream",
  "anstyle",
@@ -625,9 +625,9 @@ checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
 
 [[package]]
 name = "dyn-clone"
-version = "1.0.19"
+version = "1.0.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005"
+checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
 
 [[package]]
 name = "either"
@@ -685,7 +685,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
 
 [[package]]
 name = "firecrawl"
-version = "1.2.0"
+version = "1.2.1"
 dependencies = [
  "assert_matches",
  "axum",
@@ -910,9 +910,9 @@ dependencies = [
 
 [[package]]
 name = "glob"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"
+checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
 
 [[package]]
 name = "globset"
@@ -943,9 +943,9 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.4.11"
+version = "0.4.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17da50a276f1e01e0ba6c029e47b7100754904ee8a278f886546e98575380785"
+checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386"
 dependencies = [
  "atomic-waker",
  "bytes",
@@ -968,9 +968,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 
 [[package]]
 name = "hashbrown"
-version = "0.15.4"
+version = "0.15.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5"
+checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
 
 [[package]]
 name = "heck"
@@ -1110,7 +1110,7 @@ dependencies = [
  "libc",
  "percent-encoding",
  "pin-project-lite",
- "socket2 0.6.0",
+ "socket2",
  "system-configuration",
  "tokio",
  "tower-service",
@@ -1273,7 +1273,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661"
 dependencies = [
  "equivalent",
- "hashbrown 0.15.4",
+ "hashbrown 0.15.5",
  "serde",
 ]
 
@@ -1347,9 +1347,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.174"
+version = "0.2.175"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"
+checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543"
 
 [[package]]
 name = "libgit2-sys"
@@ -1626,7 +1626,7 @@ checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5"
 dependencies = [
  "cfg-if",
  "libc",
- "redox_syscall 0.5.15",
+ "redox_syscall 0.5.17",
  "smallvec",
  "windows-targets 0.52.6",
 ]
@@ -1703,18 +1703,18 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.95"
+version = "1.0.97"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
+checksum = "d61789d7719defeb74ea5fe81f2fdfdbd28a803847077cecce2ff14e1472f6f1"
 dependencies = [
  "unicode-ident",
 ]
 
 [[package]]
 name = "quick-xml"
-version = "0.38.0"
+version = "0.38.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8927b0664f5c5a98265138b7e3f90aa19a6b21353182469ace36d4ac527b7b1b"
+checksum = "9845d9dccf565065824e69f9f235fafba1587031eda353c1f1561cd6a6be78f4"
 dependencies = [
  "memchr",
 ]
@@ -1791,9 +1791,9 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
 
 [[package]]
 name = "redox_syscall"
-version = "0.5.15"
+version = "0.5.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e8af0dde094006011e6a740d4879319439489813bd0bcdc7d821beaeeff48ec"
+checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77"
 dependencies = [
  "bitflags 2.9.1",
 ]
@@ -1860,9 +1860,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
 
 [[package]]
 name = "reqwest"
-version = "0.12.22"
+version = "0.12.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cbc931937e6ca3a06e3b6c0aa7841849b160a90351d6ab467a8b9b9959767531"
+checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb"
 dependencies = [
  "base64 0.22.1",
  "bytes",
@@ -1937,9 +1937,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-demangle"
-version = "0.1.25"
+version = "0.1.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f"
+checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace"
 
 [[package]]
 name = "rustix"
@@ -1956,9 +1956,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.23.29"
+version = "0.23.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2491382039b29b9b11ff08b76ff6c97cf287671dbb74f0be44bda389fffe9bd1"
+checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc"
 dependencies = [
  "once_cell",
  "rustls-pki-types",
@@ -1989,9 +1989,9 @@ dependencies = [
 
 [[package]]
 name = "rustversion"
-version = "1.0.21"
+version = "1.0.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d"
+checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
 
 [[package]]
 name = "ryu"
@@ -2133,9 +2133,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.141"
+version = "1.0.142"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3"
+checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7"
 dependencies = [
  "itoa",
  "memchr",
@@ -2239,9 +2239,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
 
 [[package]]
 name = "signal-hook-registry"
-version = "1.4.5"
+version = "1.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410"
+checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b"
 dependencies = [
  "libc",
 ]
@@ -2254,9 +2254,9 @@ checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa"
 
 [[package]]
 name = "slab"
-version = "0.4.10"
+version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d"
+checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589"
 
 [[package]]
 name = "smallvec"
@@ -2264,16 +2264,6 @@ version = "1.15.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
 
-[[package]]
-name = "socket2"
-version = "0.5.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
-dependencies = [
- "libc",
- "windows-sys 0.52.0",
-]
-
 [[package]]
 name = "socket2"
 version = "0.6.0"
@@ -2457,12 +2447,12 @@ dependencies = [
 
 [[package]]
 name = "terminal_size"
-version = "0.4.2"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45c6481c4829e4cc63825e62c49186a34538b7b2750b73b266581ffb612fb5ed"
+checksum = "60b8cb979cb11c32ce1603f8137b22262a9d131aaa5c37b5678025f22b8becd0"
 dependencies = [
  "rustix",
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
@@ -2528,9 +2518,9 @@ dependencies = [
 
 [[package]]
 name = "tokio"
-version = "1.46.1"
+version = "1.47.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17"
+checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038"
 dependencies = [
  "backtrace",
  "bytes",
@@ -2541,9 +2531,9 @@ dependencies = [
  "pin-project-lite",
  "signal-hook-registry",
  "slab",
- "socket2 0.5.10",
+ "socket2",
  "tokio-macros",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -2579,9 +2569,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.7.15"
+version = "0.7.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df"
+checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5"
 dependencies = [
  "bytes",
  "futures-core",
@@ -2759,9 +2749,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "uuid"
-version = "1.17.0"
+version = "1.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d"
+checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be"
 dependencies = [
  "getrandom 0.3.3",
  "js-sys",
@@ -3135,7 +3125,7 @@ version = "0.60.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
 dependencies = [
- "windows-targets 0.53.2",
+ "windows-targets 0.53.3",
 ]
 
 [[package]]
@@ -3171,10 +3161,11 @@ dependencies = [
 
 [[package]]
 name = "windows-targets"
-version = "0.53.2"
+version = "0.53.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef"
+checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91"
 dependencies = [
+ "windows-link",
  "windows_aarch64_gnullvm 0.53.0",
  "windows_aarch64_msvc 0.53.0",
  "windows_i686_gnu 0.53.0",
@@ -3446,9 +3437,9 @@ dependencies = [
 
 [[package]]
 name = "zerovec"
-version = "0.11.2"
+version = "0.11.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428"
+checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b"
 dependencies = [
  "yoke",
  "zerofrom",
diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml
index 6f886fa1d..a117deb67 100644
--- a/apps/rust-sdk/Cargo.toml
+++ b/apps/rust-sdk/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "firecrawl"
 author= "Mendable.ai"
-version = "1.2.0"
+version = "1.2.1"
 edition = "2021"
 license = "MIT"
 homepage = "https://www.firecrawl.dev/"

From b2c64254932edac77e1392007606ad6a6a531fa3 Mon Sep 17 00:00:00 2001
From: Thomas Kosmas <thomas510111@gmail.com>
Date: Tue, 12 Aug 2025 09:34:24 -0700
Subject: [PATCH 06/12] Test new mu version (#1958)

* Test new mu version

* ignore errors
---
 .../scraper/scrapeURL/engines/pdf/index.ts    | 31 +++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 7020848b7..156666157 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -61,6 +61,37 @@ async function scrapePDFWithRunPodMU(
 
   const abort = timeout ? AbortSignal.timeout(timeout) : undefined;
 
+  robustFetch({
+    url:
+      "https://api.runpod.ai/v2/" + process.env.RUNPOD_MU_POD_ID + "/runsync",
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${process.env.RUNPOD_MU_API_KEY}`,
+    },
+    body: {
+      input: {
+        file_content: base64Content,
+        filename: path.basename(tempFilePath) + ".pdf",
+        timeout,
+        created_at: Date.now(),
+      },
+    },
+    logger: meta.logger.child({
+      method: "scrapePDFWithRunPodMU/runsync/robustFetch",
+    }),
+    schema: z.object({
+      id: z.string(),
+      status: z.string(),
+      output: z
+        .object({
+          markdown: z.string(),
+        })
+        .optional(),
+    }),
+    mock: meta.mock,
+    abort,
+  }).catch(()=>{});
+
   const podStart = await robustFetch({
     url:
       "https://api.runpod.ai/v2/" + process.env.RUNPOD_MUV2_POD_ID + "/runsync",

From efede5e39f7add0db33bf4906c7f9c71c9b8b4bc Mon Sep 17 00:00:00 2001
From: Thomas Kosmas <thomas510111@gmail.com>
Date: Tue, 12 Aug 2025 10:17:43 -0700
Subject: [PATCH 07/12] Update mu (#1959)

---
 .../scraper/scrapeURL/engines/pdf/index.ts    | 32 +------------------
 1 file changed, 1 insertion(+), 31 deletions(-)

diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 156666157..420e632b2 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -61,40 +61,10 @@ async function scrapePDFWithRunPodMU(
 
   const abort = timeout ? AbortSignal.timeout(timeout) : undefined;
 
-  robustFetch({
-    url:
-      "https://api.runpod.ai/v2/" + process.env.RUNPOD_MU_POD_ID + "/runsync",
-    method: "POST",
-    headers: {
-      Authorization: `Bearer ${process.env.RUNPOD_MU_API_KEY}`,
-    },
-    body: {
-      input: {
-        file_content: base64Content,
-        filename: path.basename(tempFilePath) + ".pdf",
-        timeout,
-        created_at: Date.now(),
-      },
-    },
-    logger: meta.logger.child({
-      method: "scrapePDFWithRunPodMU/runsync/robustFetch",
-    }),
-    schema: z.object({
-      id: z.string(),
-      status: z.string(),
-      output: z
-        .object({
-          markdown: z.string(),
-        })
-        .optional(),
-    }),
-    mock: meta.mock,
-    abort,
-  }).catch(()=>{});
 
   const podStart = await robustFetch({
     url:
-      "https://api.runpod.ai/v2/" + process.env.RUNPOD_MUV2_POD_ID + "/runsync",
+      "https://api.runpod.ai/v2/" + process.env.RUNPOD_MU_POD_ID + "/runsync",
     method: "POST",
     headers: {
       Authorization: `Bearer ${process.env.RUNPOD_MU_API_KEY}`,

From 298fd14980e38d9b0c8eaaa1ef861dd5df4fbea7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Wed, 13 Aug 2025 00:40:36 +0200
Subject: [PATCH 08/12] feat(api): log network info for debug log correlation

---
 apps/api/src/index.ts | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 3ff680558..9144d0c8a 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -38,6 +38,10 @@ const { ExpressAdapter } = require("@bull-board/express");
 const numCPUs = process.env.ENV === "local" ? 2 : os.cpus().length;
 logger.info(`Number of CPUs: ${numCPUs} available`);
 
+logger.info("Network info dump", {
+  networkInterfaces: os.networkInterfaces(),
+});
+
 // Install cacheable lookup for all other requests
 cacheableLookup.install(http.globalAgent);
 cacheableLookup.install(https.globalAgent);

From e907a9a7d4ce30f1512aed97957000671d82a8b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Wed, 13 Aug 2025 18:28:15 +0200
Subject: [PATCH 09/12] feat(api/admin): prometheus metrics about cc limit
 queue (#1963)

---
 apps/api/src/controllers/v0/admin/metrics.ts | 25 ++++++++++++++++++++
 apps/api/src/routes/admin.ts                 |  6 +++++
 2 files changed, 31 insertions(+)
 create mode 100644 apps/api/src/controllers/v0/admin/metrics.ts

diff --git a/apps/api/src/controllers/v0/admin/metrics.ts b/apps/api/src/controllers/v0/admin/metrics.ts
new file mode 100644
index 000000000..9e167d708
--- /dev/null
+++ b/apps/api/src/controllers/v0/admin/metrics.ts
@@ -0,0 +1,25 @@
+import type { Request, Response } from "express";
+import { redisEvictConnection } from "../../../services/redis";
+
+export async function metricsController(_: Request, res: Response) {
+    let cursor: string = "0";
+    const metrics: Record<string, number> = {};
+    do {
+        const res = await redisEvictConnection.scan(cursor, "MATCH", "concurrency-limit-queue:*");
+        cursor = res[0];
+
+        const keys = res[1];
+
+        for (const key of keys) {
+            const teamId = key.split(":")[1];
+            const jobCount = await redisEvictConnection.zcard(key);
+            metrics[teamId] = jobCount;
+        }
+    } while (cursor !== "0");
+
+    res.contentType("text/plain").send(`\
+# HELP concurrency_limit_queue_job_count The number of jobs in the concurrency limit queue per team
+# TYPE concurrency_limit_queue_job_count gauge
+${Object.entries(metrics).map(([key, value]) => `concurrency_limit_queue_job_count{team_id="${key}"} ${value}`).join("\n")}
+`);
+}
\ No newline at end of file
diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.ts
index fc8a3f8dc..8655e0439 100644
--- a/apps/api/src/routes/admin.ts
+++ b/apps/api/src/routes/admin.ts
@@ -13,6 +13,7 @@ import { cclogController } from "../controllers/v0/admin/cclog";
 import { indexQueuePrometheus } from "../controllers/v0/admin/index-queue-prometheus";
 import { zdrcleanerController } from "../controllers/v0/admin/zdrcleaner";
 import { triggerPrecrawl } from "../controllers/v0/admin/precrawl";
+import { metricsController } from "../controllers/v0/admin/metrics";
 
 export const adminRouter = express.Router();
 
@@ -68,3 +69,8 @@ adminRouter.get(
   `/admin/${process.env.BULL_AUTH_KEY}/precrawl`,
   wrap(triggerPrecrawl),
 );
+
+adminRouter.get(
+  `/admin/${process.env.BULL_AUTH_KEY}/metrics`,
+  wrap(metricsController),
+);
\ No newline at end of file

From 5e64ecba070562f210b378c211e9e5d94ca82003 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Wed, 13 Aug 2025 18:33:22 +0200
Subject: [PATCH 10/12] fix(api): stop using bulljobs_teams table (#1962)

---
 apps/api/src/controllers/v0/crawl-cancel.ts | 18 ++---------------
 apps/api/src/controllers/v0/crawl.ts        |  3 ---
 apps/api/src/controllers/v1/batch-scrape.ts |  5 -----
 apps/api/src/controllers/v1/crawl-cancel.ts | 19 ++----------------
 apps/api/src/controllers/v1/crawl.ts        |  3 ---
 apps/api/src/services/logging/crawl_log.ts  | 22 ---------------------
 6 files changed, 4 insertions(+), 66 deletions(-)
 delete mode 100644 apps/api/src/services/logging/crawl_log.ts

diff --git a/apps/api/src/controllers/v0/crawl-cancel.ts b/apps/api/src/controllers/v0/crawl-cancel.ts
index ff8b88040..3d5249b9c 100644
--- a/apps/api/src/controllers/v0/crawl-cancel.ts
+++ b/apps/api/src/controllers/v0/crawl-cancel.ts
@@ -1,7 +1,6 @@
 import { Request, Response } from "express";
 import { authenticateUser } from "../auth";
 import { RateLimiterMode } from "../../../src/types";
-import { supabase_service } from "../../../src/services/supabase";
 import { logger } from "../../../src/lib/logger";
 import { getCrawl, saveCrawl } from "../../../src/lib/crawl-redis";
 import * as Sentry from "@sentry/node";
@@ -11,8 +10,6 @@ configDotenv();
 
 export async function crawlCancelController(req: Request, res: Response) {
   try {
-    const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
-
     const auth = await authenticateUser(req, res, RateLimiterMode.CrawlStatus);
     if (!auth.success) {
       return res.status(auth.status).json({ error: auth.error });
@@ -36,19 +33,8 @@ export async function crawlCancelController(req: Request, res: Response) {
     }
 
     // check if the job belongs to the team
-    if (useDbAuthentication) {
-      const { data, error: supaError } = await supabase_service
-        .from("bulljobs_teams")
-        .select("*")
-        .eq("job_id", req.params.jobId)
-        .eq("team_id", team_id);
-      if (supaError) {
-        return res.status(500).json({ error: supaError.message });
-      }
-
-      if (data.length === 0) {
-        return res.status(403).json({ error: "Unauthorized" });
-      }
+    if (sc.team_id !== team_id) {
+      return res.status(403).json({ error: "Unauthorized" });
     }
 
     try {
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index 63b90e809..65e5c4ed1 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -4,7 +4,6 @@ import { authenticateUser } from "../auth";
 import { RateLimiterMode } from "../../../src/types";
 import { addScrapeJob } from "../../../src/services/queue-jobs";
 import { isUrlBlocked } from "../../../src/scraper/WebScraper/utils/blocklist";
-import { logCrawl } from "../../../src/services/logging/crawl_log";
 import { validateIdempotencyKey } from "../../../src/services/idempotency/validate";
 import { createIdempotencyKey } from "../../../src/services/idempotency/create";
 import {
@@ -159,8 +158,6 @@ export async function crawlController(req: Request, res: Response) {
     //   }
     // }
 
-    await logCrawl(id, team_id);
-
     const { scrapeOptions, internalOptions } = fromLegacyScrapeOptions(
       pageOptions,
       undefined,
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index 9a034c150..57f6c4499 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -17,7 +17,6 @@ import {
   saveCrawl,
   StoredCrawl,
 } from "../../lib/crawl-redis";
-import { logCrawl } from "../../services/logging/crawl_log";
 import { getJobPriority } from "../../lib/job-priority";
 import { addScrapeJobs } from "../../services/queue-jobs";
 import { callWebhook } from "../../services/webhook";
@@ -96,10 +95,6 @@ export async function batchScrapeController(
     account: req.account,
   });
 
-  if (!req.body.appendToId) {
-    await logCrawl(id, req.auth.team_id);
-  }
-
   const sc: StoredCrawl = req.body.appendToId
     ? ((await getCrawl(req.body.appendToId)) as StoredCrawl)
     : {
diff --git a/apps/api/src/controllers/v1/crawl-cancel.ts b/apps/api/src/controllers/v1/crawl-cancel.ts
index 00af8b31f..8ddd9fdba 100644
--- a/apps/api/src/controllers/v1/crawl-cancel.ts
+++ b/apps/api/src/controllers/v1/crawl-cancel.ts
@@ -1,5 +1,4 @@
 import { Response } from "express";
-import { supabase_service } from "../../services/supabase";
 import { logger } from "../../lib/logger";
 import { getCrawl, saveCrawl } from "../../lib/crawl-redis";
 import * as Sentry from "@sentry/node";
@@ -12,27 +11,13 @@ export async function crawlCancelController(
   res: Response,
 ) {
   try {
-    const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
-
     const sc = await getCrawl(req.params.jobId);
     if (!sc) {
       return res.status(404).json({ error: "Job not found" });
     }
 
-    // check if the job belongs to the team
-    if (useDbAuthentication) {
-      const { data, error: supaError } = await supabase_service
-        .from("bulljobs_teams")
-        .select("*")
-        .eq("job_id", req.params.jobId)
-        .eq("team_id", req.auth.team_id);
-      if (supaError) {
-        return res.status(500).json({ error: supaError.message });
-      }
-
-      if (data.length === 0) {
-        return res.status(403).json({ error: "Unauthorized" });
-      }
+    if (sc.team_id !== req.auth.team_id) {
+      return res.status(403).json({ error: "Unauthorized" });
     }
 
     try {
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index 1f75bc0ab..ab5d2decd 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -8,7 +8,6 @@ import {
   toLegacyCrawlerOptions,
 } from "./types";
 import { crawlToCrawler, saveCrawl, StoredCrawl } from "../../lib/crawl-redis";
-import { logCrawl } from "../../services/logging/crawl_log";
 import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";
 import { logger as _logger } from "../../lib/logger";
 
@@ -43,8 +42,6 @@ export async function crawlController(
     account: req.account,
   });
 
-  await logCrawl(id, req.auth.team_id);
-
   let { remainingCredits } = req.account!;
   const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
   if (!useDbAuthentication) {
diff --git a/apps/api/src/services/logging/crawl_log.ts b/apps/api/src/services/logging/crawl_log.ts
deleted file mode 100644
index 86f885293..000000000
--- a/apps/api/src/services/logging/crawl_log.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-import { supabase_service } from "../supabase";
-import { logger } from "../../../src/lib/logger";
-import { configDotenv } from "dotenv";
-configDotenv();
-
-export async function logCrawl(job_id: string, team_id: string) {
-  const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
-  if (useDbAuthentication) {
-    try {
-      const { data, error } = await supabase_service
-        .from("bulljobs_teams")
-        .insert([
-          {
-            job_id: job_id,
-            team_id: team_id,
-          },
-        ]);
-    } catch (error) {
-      logger.error(`Error logging crawl job to supabase:\n${error}`);
-    }
-  }
-}

From a9141a41b81611c189a23439c6355a7f71844972 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Wed, 13 Aug 2025 19:17:39 +0200
Subject: [PATCH 11/12] fix(api/admin/metrics): store queues in a set

---
 apps/api/src/controllers/v0/admin/metrics.ts | 11 ++++++++---
 apps/api/src/lib/concurrency-limit.ts        |  8 +++-----
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/apps/api/src/controllers/v0/admin/metrics.ts b/apps/api/src/controllers/v0/admin/metrics.ts
index 9e167d708..43b6288d4 100644
--- a/apps/api/src/controllers/v0/admin/metrics.ts
+++ b/apps/api/src/controllers/v0/admin/metrics.ts
@@ -5,15 +5,20 @@ export async function metricsController(_: Request, res: Response) {
     let cursor: string = "0";
     const metrics: Record<string, number> = {};
     do {
-        const res = await redisEvictConnection.scan(cursor, "MATCH", "concurrency-limit-queue:*");
+        const res = await redisEvictConnection.sscan("concurrency-limit-queues", cursor);
         cursor = res[0];
 
         const keys = res[1];
 
         for (const key of keys) {
-            const teamId = key.split(":")[1];
             const jobCount = await redisEvictConnection.zcard(key);
-            metrics[teamId] = jobCount;
+
+            if (jobCount === 0) {
+                await redisEvictConnection.srem("concurrency-limit-queues", key);
+            } else {
+                const teamId = key.split(":")[1];
+                metrics[teamId] = jobCount;
+            }
         }
     } while (cursor !== "0");
 
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index 022f3523b..7fbb7837b 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -75,11 +75,9 @@ export async function pushConcurrencyLimitedJob(
   timeout: number,
   now: number = Date.now(),
 ) {
-  await redisEvictConnection.zadd(
-    constructQueueKey(team_id),
-    now + timeout,
-    JSON.stringify(job),
-  );
+  const queueKey = constructQueueKey(team_id);
+  await redisEvictConnection.zadd(queueKey, now + timeout, JSON.stringify(job));
+  await redisEvictConnection.sadd("concurrency-limit-queues", queueKey);
 }
 
 export async function getConcurrencyLimitedJobs(

From 6e59bbd265acc2494dc30a9955e3fc2a24dac16c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Wed, 13 Aug 2025 19:57:57 +0200
Subject: [PATCH 12/12] fix(map): returning excessive query parameters

---
 apps/api/src/controllers/v1/map.ts   | 2 +-
 apps/api/src/controllers/v1/types.ts | 2 ++
 apps/api/src/lib/validateUrl.ts      | 7 +++++--
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index 6145beced..c33f07fc3 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -258,7 +258,7 @@ export async function getMapResults({
     links = links
       .map((x) => {
         try {
-          return checkAndUpdateURLForMap(x).url.trim();
+          return checkAndUpdateURLForMap(x, crawlerOptions.ignoreQueryParameters ?? true).url.trim();
         } catch (_) {
           return null;
         }
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index cf859271f..8571612ee 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -795,12 +795,14 @@ export type CrawlRequest = z.infer<typeof crawlRequestSchema>;
 export type CrawlRequestInput = z.input<typeof crawlRequestSchema>;
 
 export const mapRequestSchema = crawlerOptions
+  .omit({ ignoreQueryParameters: true })
   .extend({
     url,
     origin: z.string().optional().default("api"),
     integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
     includeSubdomains: z.boolean().default(true),
     search: z.string().optional(),
+    ignoreQueryParameters: z.boolean().default(true),
     ignoreSitemap: z.boolean().default(false),
     sitemapOnly: z.boolean().default(false),
     limit: z.number().min(1).max(30000).default(5000),
diff --git a/apps/api/src/lib/validateUrl.ts b/apps/api/src/lib/validateUrl.ts
index 6216e0ec5..e8fe866b5 100644
--- a/apps/api/src/lib/validateUrl.ts
+++ b/apps/api/src/lib/validateUrl.ts
@@ -126,7 +126,7 @@ export function isSameSubdomain(url: string, baseUrl: string) {
   return domain1 === domain2 && subdomain1 === subdomain2;
 }
 
-export const checkAndUpdateURLForMap = (url: string) => {
+export const checkAndUpdateURLForMap = (url: string, ignoreQueryParameters: boolean = false) => {
   if (!protocolIncluded(url)) {
     url = `http://${url}`;
   }
@@ -147,7 +147,10 @@ export const checkAndUpdateURLForMap = (url: string) => {
   }
 
   // remove any query params
-  // url = url.split("?")[0].trim();
+  if (ignoreQueryParameters) {
+    url = url.split("?")[0].trim();
+    typedUrlObj.search = "";
+  }
 
   return { urlObj: typedUrlObj, url: url };
 };