diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts index 7327a92f4..201b18464 100644 --- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts @@ -45,6 +45,11 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) { return document; } + // Generate indexId synchronously and set it on document immediately + // so it's available to other transformers (e.g., search index) + const indexId = crypto.randomUUID(); + document.metadata.indexId = indexId; + (async () => { try { const normalizedURL = normalizeURLForIndex(meta.url); @@ -60,8 +65,6 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) { const domainSplits = generateDomainSplits(hostname, fakeDomain); const domainSplitsHash = domainSplits.map(split => hashURL(split)); - const indexId = crypto.randomUUID(); - try { await saveIndexToGCS(indexId, { url: @@ -84,9 +87,6 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) { contentType: document.metadata.contentType, postprocessorsUsed: document.metadata.postprocessorsUsed, }); - - // Store the indexId in document metadata so it can be used by other transformers (e.g., search index) - document.metadata.indexId = indexId; } catch (error) { meta.logger.error("Failed to save document to index", { error,