From 24a5a719210e37131d1ca1f555e15c0aa04f17c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Tue, 15 Jul 2025 22:00:30 +0200
Subject: [PATCH] timeouts for index engine

---
 .../scraper/scrapeURL/engines/index/index.ts    | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 4c12c3601..2e37b2ba7 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -2,7 +2,7 @@ import { Document } from "../../../../controllers/v1/types";
 import { EngineScrapeResult } from "..";
 import { Meta } from "../..";
 import { getIndexFromGCS, hashURL, index_supabase_service, normalizeURLForIndex, saveIndexToGCS, generateURLSplits, addIndexInsertJob, generateDomainSplits, addOMCEJob } from "../../../../services";
-import { EngineError, IndexMissError } from "../../error";
+import { EngineError, IndexMissError, TimeoutError } from "../../error";
 import crypto from "crypto";
 
 export async function sendDocumentToIndex(meta: Meta, document: Document) {
@@ -133,7 +133,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
 
 const errorCountToRegister = 3;
 
-export async function scrapeURLWithIndex(meta: Meta): Promise<EngineScrapeResult> {
+export async function scrapeURLWithIndex(meta: Meta, timeToRun: number | undefined): Promise<EngineScrapeResult> {
     const normalizedURL = normalizeURLForIndex(meta.url);
     const urlHash = hashURL(normalizedURL);
 
@@ -162,11 +162,16 @@ export async function scrapeURLWithIndex(meta: Meta): Promise<EngineScrapeResult
         selector = selector.is("location_languages", null);
     }
 
-    const { data, error } = await selector
-        .order("created_at", { ascending: false })
-        .limit(5);
+    const { data, error } = await Promise.race([
+        selector
+            .order("created_at", { ascending: false })
+            .limit(5),
+        new Promise<{ data: { id: any; created_at: any; status: any }[], error: any }>((resolve, reject) => {
+            setTimeout(() => reject(new TimeoutError()), timeToRun ?? 10000);
+        }),
+    ]);
 
-    if (error) {
+    if (error || !data) {
         throw new EngineError("Failed to retrieve URL from DB index", {
             cause: error,
         });