diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts index 538c33a1a..95eb1d0ae 100644 --- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts @@ -12,11 +12,9 @@ import { addIndexInsertJob, generateDomainSplits, addOMCEJob, - addDomainFrequencyJob, } from "../../../../services"; import { EngineError, IndexMissError } from "../../error"; import { shouldParsePDF } from "../../../../controllers/v2/types"; -import { storage } from "../../../../lib/gcs-jobs"; export async function sendDocumentToIndex(meta: Meta, document: Document) { const shouldCache = @@ -300,6 +298,12 @@ export async function scrapeURLWithIndex( } if (selectedRow === null || selectedRow === undefined) { + meta.logger.debug("Index metrics", { + module: "index/metrics", + hit: false, + maxAge, + dynamicMaxAge: meta.options.maxAge === undefined, + }); throw new IndexMissError(); } @@ -333,6 +337,14 @@ export async function scrapeURLWithIndex( } } + meta.logger.debug("Index metrics", { + module: "index/metrics", + hit: true, + age: Date.now() - new Date(selectedRow.created_at).getTime(), + maxAge, + dynamicMaxAge: meta.options.maxAge === undefined, + }); + return { url: doc.url, html: doc.html,