From 0c8a54105371e63fefda307facccbe5d33687ae3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Fri, 8 Aug 2025 12:36:33 +0200
Subject: [PATCH] feat(scrapeURL): optional A/B test to staging

---
 apps/api/src/scraper/scrapeURL/index.ts | 37 +++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 5ae390364..266ae3677 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -34,6 +34,7 @@ import { LLMRefusalError } from "./transformers/llmExtract";
 import { urlSpecificParams } from "./lib/urlSpecificParams";
 import { loadMock, MockState } from "./lib/mock";
 import { CostTracking } from "../../lib/extract/extraction-service";
+import { robustFetch } from "./lib/fetch";
 import { addIndexRFInsertJob, generateDomainSplits, hashURL, index_supabase_service, normalizeURLForIndex, useIndex } from "../../services/index";
 import { checkRobotsTxt } from "../../lib/robots-txt";
 
@@ -590,6 +591,42 @@ export async function scrapeURL(
     });
   }
 
+  // Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATE
+  try {
+    const abRateEnv = process.env.SCRAPEURL_AB_RATE;
+    const abRate = abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;
+    const shouldABTest = !meta.internalOptions.zeroDataRetention && abRate > 0 && Math.random() <= abRate;
+    if (shouldABTest) {
+      (async () => {
+        try {
+          const abLogger = meta.logger.child({ method: "scrapeURL/abTestToStaging" });
+          abLogger.info("A/B-testing scrapeURL to staging");
+          const abort = AbortSignal.timeout(Math.min(60000, (meta.options.timeout ?? 30000) + 10000));
+          await robustFetch({
+            url: `http://firecrawl-app-staging:3002/v1/scrape`,
+            method: "POST",
+            body: {
+              url: meta.url,
+              ...meta.options,
+              origin: (meta.options as any).origin ?? "api",
+              timeout: meta.options.timeout ?? 30000,
+            },
+            logger: abLogger,
+            tryCount: 1,
+            ignoreResponse: true,
+            mock: null,
+            abort,
+          });
+          abLogger.info("A/B-testing scrapeURL (staging) request sent");
+        } catch (error) {
+          meta.logger.warn("A/B-testing scrapeURL (staging) failed", { error });
+        }
+      })();
+    }
+  } catch (error) {
+    meta.logger.warn("Failed to initiate A/B test to staging", { error });
+  }
+
   try {
     while (true) {
       try {