From cf31c78d4f8c9ec3894ef57b53d44aa94871f179 Mon Sep 17 00:00:00 2001
From: Nicolas <nicolascamara29@gmail.com>
Date: Sun, 12 Oct 2025 02:52:11 -0300
Subject: [PATCH] Update sendToSearchIndex.ts

---
 .../transformers/sendToSearchIndex.ts         | 91 +++++++++----------
 1 file changed, 45 insertions(+), 46 deletions(-)

diff --git a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
index 07ac02613..d630a5f28 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
@@ -1,9 +1,9 @@
 /**
  * Transformer: Send Document to Search Index
- * 
+ *
  * Integrates with the existing scraper transformer stack.
  * Queues documents for real-time search indexing.
- * 
+ *
  * Sampling: Controlled via SEARCH_INDEX_SAMPLE_RATE (0.0-1.0)
  * - 0.1 = 10% of documents indexed (recommended for initial rollout)
  * - 1.0 = 100% of documents indexed (full production)
@@ -18,38 +18,35 @@ import { Meta } from "..";
  * Check if document should be indexed for search
  */
 function shouldIndexForSearch(meta: Meta, document: Document): boolean {
-
-  
   if (meta.internalOptions.zeroDataRetention) {
     return false;
   }
-  
+
   const statusCode = document.metadata.statusCode;
   if (statusCode < 200 || statusCode >= 300) {
     return false;
   }
-  
+
   // Check if markdown content exists and is substantial
   const markdown = document.markdown ?? "";
   if (markdown.length < 200) {
     return false;
   }
-  
+
   // Don't index if has auth headers (private content)
   if (
     meta.options.headers &&
-    (meta.options.headers["Authorization"] ||
-      meta.options.headers["Cookie"])
+    (meta.options.headers["Authorization"] || meta.options.headers["Cookie"])
   ) {
     return false;
   }
-  
+
   // Don't index PDFs without parsing
   const isPdf = document.metadata.contentType?.includes("pdf");
   if (isPdf && !document.markdown) {
     return false;
   }
-  
+
   return true;
 }
 
@@ -60,7 +57,7 @@ function shouldSampleDocument(): boolean {
   // Get sample rate from environment (default 10% for safe rollout)
   const sampleRateStr = process.env.SEARCH_INDEX_SAMPLE_RATE || "0.1";
   const sampleRate = parseFloat(sampleRateStr);
-  
+
   // Validate sample rate
   if (isNaN(sampleRate) || sampleRate < 0 || sampleRate > 1) {
     _logger.warn("Invalid SEARCH_INDEX_SAMPLE_RATE, using 0.1 (10%)", {
@@ -68,7 +65,7 @@ function shouldSampleDocument(): boolean {
     });
     return Math.random() < 0.1;
   }
-  
+
   // Sample based on rate
   return Math.random() < sampleRate;
 }
@@ -84,16 +81,16 @@ export async function sendDocumentToSearchIndex(
   const searchIndexEnabled =
     process.env.ENABLE_SEARCH_INDEX === "true" &&
     process.env.SEARCH_SERVICE_URL;
-  
+
   meta.logger.debug("Sending document to search index", {
     url: meta.url,
     searchIndexEnabled,
   });
-  
+
   if (!searchIndexEnabled) {
     return document;
   }
-  
+
   // Apply sampling (canary rollout)
   if (!shouldSampleDocument()) {
     meta.logger.debug("Document not sampled for search indexing", {
@@ -102,7 +99,7 @@ export async function sendDocumentToSearchIndex(
     });
     return document;
   }
-  
+
   // Check if document should be indexed
   if (!shouldIndexForSearch(meta, document)) {
     meta.logger.debug("Document not suitable for search index", {
@@ -112,40 +109,43 @@ export async function sendDocumentToSearchIndex(
     });
     return document;
   }
-  
+
   // Get the indexId from document metadata (set by sendDocumentToIndex transformer)
   // Format as GCS path: {indexId}.json
-  const gcsPath = document.metadata.indexId 
-  
+  const gcsPath = document.metadata.indexId;
+
   // Remove indexId from metadata after extracting it (internal field, shouldn't be exposed to user)
   delete document.metadata.indexId;
-  
+
   // Send to search service via HTTP (async, don't block scraper)
   (async () => {
     try {
-      await indexDocumentIfEnabled({
-        url: meta.url,
-        resolvedUrl:
-          document.metadata.url ??
-          document.metadata.sourceURL ??
-          meta.rewrittenUrl ??
-          meta.url,
-        title:
-          document.metadata.title ?? document.metadata.ogTitle ?? undefined,
-        description:
-          document.metadata.description ??
-          document.metadata.ogDescription ??
-          undefined,
-        markdown: document.markdown ?? "",
-        html: document.rawHtml ?? "",
-        statusCode: document.metadata.statusCode,
-        gcsPath: gcsPath,
-        screenshotUrl: document.screenshot ?? undefined,
-        language: document.metadata.language ?? "en",
-        country: meta.options.location?.country ?? undefined,
-        isMobile: meta.options.mobile ?? false,
-      }, meta.logger);
-      
+      await indexDocumentIfEnabled(
+        {
+          url: meta.url,
+          resolvedUrl:
+            document.metadata.url ??
+            document.metadata.sourceURL ??
+            meta.rewrittenUrl ??
+            meta.url,
+          title:
+            document.metadata.title ?? document.metadata.ogTitle ?? undefined,
+          description:
+            document.metadata.description ??
+            document.metadata.ogDescription ??
+            undefined,
+          markdown: document.markdown ?? "",
+          html: document.rawHtml ?? "",
+          statusCode: document.metadata.statusCode,
+          gcsPath: gcsPath,
+          screenshotUrl: document.screenshot ?? undefined,
+          language: document.metadata.language ?? "en",
+          country: meta.options.location?.country ?? undefined,
+          isMobile: meta.options.mobile ?? false,
+        },
+        meta.logger,
+      );
+
       meta.logger.debug("Sent document to search service", {
         url: meta.url,
       });
@@ -156,7 +156,6 @@ export async function sendDocumentToSearchIndex(
       });
     }
   })();
-  
+
   return document;
 }
-