From a7669cccb98fbd15355e995c23f1ccdec18d26e5 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Fri, 10 Oct 2025 16:49:29 -0300 Subject: [PATCH] Nick: gcs index id fix --- apps/api/src/controllers/v1/types.ts | 1 + apps/api/src/controllers/v2/types.ts | 1 + apps/api/src/lib/search-index/service.ts | 1 + apps/api/src/scraper/scrapeURL/engines/index/index.ts | 3 +++ .../scraper/scrapeURL/transformers/sendToSearchIndex.ts | 9 ++++++++- 5 files changed, 14 insertions(+), 1 deletion(-) diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index 4455e3b64..fc7c15109 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -1018,6 +1018,7 @@ export type Document = { cachedAt?: string; creditsUsed?: number; postprocessorsUsed?: string[]; + indexId?: string; // ID used to store the document in the index (GCS) // [key: string]: string | string[] | number | { smartScrape: number; other: number; total: number } | undefined; }; serpResults?: { diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts index c9adb71ed..deb59f7ec 100644 --- a/apps/api/src/controllers/v2/types.ts +++ b/apps/api/src/controllers/v2/types.ts @@ -924,6 +924,7 @@ export type Document = { cachedAt?: string; creditsUsed?: number; postprocessorsUsed?: string[]; + indexId?: string; // ID used to store the document in the index (GCS) // [key: string]: string | string[] | number | { smartScrape: number; other: number; total: number } | undefined; }; serpResults?: { diff --git a/apps/api/src/lib/search-index/service.ts b/apps/api/src/lib/search-index/service.ts index fe2b943e2..32e07e83a 100644 --- a/apps/api/src/lib/search-index/service.ts +++ b/apps/api/src/lib/search-index/service.ts @@ -306,6 +306,7 @@ export async function indexDocumentForSearch( description: input.description?.slice(0, 500) ?? null, language: input.language ?? "en", gcs_path: input.gcsPath ?? null, + gcp_path_id: input.gcsPath ?? null, screenshot_url: input.screenshotUrl ?? null, content_ts: contentTsVector, domain, diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts index b104a8d91..7327a92f4 100644 --- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts @@ -84,6 +84,9 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) { contentType: document.metadata.contentType, postprocessorsUsed: document.metadata.postprocessorsUsed, }); + + // Store the indexId in document metadata so it can be used by other transformers (e.g., search index) + document.metadata.indexId = indexId; } catch (error) { meta.logger.error("Failed to save document to index", { error, diff --git a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts index 4fa44075b..e71c7e868 100644 --- a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts +++ b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts @@ -112,6 +112,13 @@ export async function sendDocumentToSearchIndex( return document; } + // Get the indexId from document metadata (set by sendDocumentToIndex transformer) + // Format as GCS path: {indexId}.json + const gcsPath = document.metadata.indexId + + // Remove indexId from metadata after extracting it (internal field, shouldn't be exposed to user) + delete document.metadata.indexId; + // Queue job for async processing (don't block scraper) (async () => { try { @@ -131,7 +138,7 @@ export async function sendDocumentToSearchIndex( markdown: document.markdown ?? "", html: document.rawHtml ?? "", statusCode: document.metadata.statusCode, - gcsPath: undefined, // Can be populated from GCS if needed + gcsPath: gcsPath, screenshotUrl: document.screenshot ?? undefined, language: document.metadata.language ?? "en", country: meta.options.location?.country ?? undefined,