From 3c3d382c8dd65f5b3c9fed2d907a03208ec7bd16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Mon, 15 Sep 2025 20:59:01 +0200
Subject: [PATCH] feat(scrapeURL/index): index metrics (#2160)

Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com>
---
 .../src/scraper/scrapeURL/engines/index/index.ts | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 538c33a1a..95eb1d0ae 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -12,11 +12,9 @@ import {
   addIndexInsertJob,
   generateDomainSplits,
   addOMCEJob,
-  addDomainFrequencyJob,
 } from "../../../../services";
 import { EngineError, IndexMissError } from "../../error";
 import { shouldParsePDF } from "../../../../controllers/v2/types";
-import { storage } from "../../../../lib/gcs-jobs";
 
 export async function sendDocumentToIndex(meta: Meta, document: Document) {
   const shouldCache =
@@ -300,6 +298,12 @@ export async function scrapeURLWithIndex(
   }
 
   if (selectedRow === null || selectedRow === undefined) {
+    meta.logger.debug("Index metrics", {
+      module: "index/metrics",
+      hit: false,
+      maxAge,
+      dynamicMaxAge: meta.options.maxAge === undefined,
+    });
     throw new IndexMissError();
   }
 
@@ -333,6 +337,14 @@ export async function scrapeURLWithIndex(
     }
   }
 
+  meta.logger.debug("Index metrics", {
+    module: "index/metrics",
+    hit: true,
+    age: Date.now() - new Date(selectedRow.created_at).getTime(),
+    maxAge,
+    dynamicMaxAge: meta.options.maxAge === undefined,
+  });
+
   return {
     url: doc.url,
     html: doc.html,