From a5e3efc77a8f9fd7383d1baa036156c78948611f Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Tue, 30 Dec 2025 13:17:35 -0800 Subject: [PATCH] Add request_id to Markdown Logs (#2621) * Add request_id to Markdown Logs * remove unused param --- apps/api/src/lib/html-to-markdown-client.ts | 30 ++++++++++++++----- apps/api/src/lib/html-to-markdown.ts | 29 +++++++++++++----- apps/api/src/scraper/scrapeURL/index.ts | 3 ++ .../scrapeURL/lib/extractSmartScrape.ts | 9 +++++- .../scraper/scrapeURL/transformers/agent.ts | 6 +++- .../scraper/scrapeURL/transformers/index.ts | 12 ++++++-- 6 files changed, 70 insertions(+), 19 deletions(-) diff --git a/apps/api/src/lib/html-to-markdown-client.ts b/apps/api/src/lib/html-to-markdown-client.ts index 73bb8caeb..a4e9bc263 100644 --- a/apps/api/src/lib/html-to-markdown-client.ts +++ b/apps/api/src/lib/html-to-markdown-client.ts @@ -8,6 +8,7 @@ import axios, { AxiosInstance, AxiosError } from "axios"; import { config } from "../config"; import { logger } from "./logger"; +import type { Logger } from "winston"; import * as Sentry from "@sentry/node"; interface ConvertRequest { @@ -27,19 +28,24 @@ interface ErrorResponse { /** * Convert HTML to Markdown using direct axios call * @param html HTML string to convert - * @param serviceUrl URL of the HTML to Markdown service (default: http://localhost:8080) + * @param context Optional context with logger and requestId * @returns Markdown string * @throws Error if conversion fails */ export async function convertHTMLToMarkdownWithHttpService( html: string, - serviceUrl?: string, + context?: { + logger?: Logger; + requestId?: string; + }, ): Promise { if (!html || html.trim() === "") { return ""; } - const url = serviceUrl || config.HTML_TO_MARKDOWN_SERVICE_URL; + const contextLogger = context?.logger || logger; + const requestId = context?.requestId; + const url = config.HTML_TO_MARKDOWN_SERVICE_URL; const startTime = Date.now(); try { @@ -62,7 +68,7 @@ export async function convertHTMLToMarkdownWithHttpService( throw new Error("Conversion was not successful"); } - logger.debug("HTML to Markdown conversion successful", { + contextLogger.debug("HTML to Markdown conversion successful", { duration_ms: duration, input_size: html.length, output_size: response.data.markdown.length, @@ -79,7 +85,7 @@ export async function convertHTMLToMarkdownWithHttpService( axiosError.response?.data?.error || axiosError.message; const statusCode = axiosError.response?.status; - logger.error("HTML to Markdown conversion failed", { + contextLogger.error("HTML to Markdown conversion failed", { error: errorMessage, statusCode, duration_ms: duration, @@ -91,6 +97,7 @@ export async function convertHTMLToMarkdownWithHttpService( tags: { service: "html-to-markdown", status_code: statusCode, + ...(requestId ? { request_id: requestId } : {}), }, extra: { serviceUrl: url, @@ -101,10 +108,17 @@ export async function convertHTMLToMarkdownWithHttpService( throw new Error(`HTML to Markdown conversion failed: ${errorMessage}`); } else { - logger.error("Unexpected error during HTML to Markdown conversion", { - error, + contextLogger.error( + "Unexpected error during HTML to Markdown conversion", + { + error, + }, + ); + Sentry.captureException(error, { + tags: { + ...(requestId ? { request_id: requestId } : {}), + }, }); - Sentry.captureException(error); throw error; } } diff --git a/apps/api/src/lib/html-to-markdown.ts b/apps/api/src/lib/html-to-markdown.ts index b7d2f3848..69bfa6941 100644 --- a/apps/api/src/lib/html-to-markdown.ts +++ b/apps/api/src/lib/html-to-markdown.ts @@ -3,6 +3,7 @@ import { config } from "../config"; import "../services/sentry"; import * as Sentry from "@sentry/node"; import { logger } from "./logger"; +import type { Logger } from "winston"; import { stat } from "fs/promises"; import { HTML_TO_MARKDOWN_PATH } from "../natives"; import { convertHTMLToMarkdownWithHttpService } from "./html-to-markdown-client"; @@ -52,25 +53,36 @@ class GoMarkdownConverter { export async function parseMarkdown( html: string | null | undefined, + context?: { + logger?: Logger; + requestId?: string; + }, ): Promise { if (!html) { return ""; } + const contextLogger = context?.logger || logger; + const requestId = context?.requestId; + // Try HTTP service first if enabled if (config.HTML_TO_MARKDOWN_SERVICE_URL) { try { - let markdownContent = await convertHTMLToMarkdownWithHttpService(html); + let markdownContent = await convertHTMLToMarkdownWithHttpService(html, { + logger: contextLogger, + requestId, + }); markdownContent = await postProcessMarkdown(markdownContent); return markdownContent; } catch (error) { - logger.error( + contextLogger.error( "Error converting HTML to Markdown with HTTP service, falling back to original parser", { error }, ); Sentry.captureException(error, { tags: { fallback: "original_parser", + ...(requestId ? { request_id: requestId } : {}), }, }); } @@ -81,7 +93,6 @@ export async function parseMarkdown( const converter = await GoMarkdownConverter.getInstance(); let markdownContent = await converter.convertHTMLToMarkdown(html); markdownContent = await postProcessMarkdown(markdownContent); - // logger.info(`HTML to Markdown conversion using Go parser successful`); return markdownContent; } } catch (error) { @@ -89,12 +100,16 @@ export async function parseMarkdown( !(error instanceof Error) || error.message !== "Go shared library not found" ) { - Sentry.captureException(error); - logger.error( + Sentry.captureException(error, { + tags: { + ...(requestId ? { request_id: requestId } : {}), + }, + }); + contextLogger.error( `Error converting HTML to Markdown with Go parser: ${error}`, ); } else { - logger.warn( + contextLogger.warn( "Tried to use Go parser, but it doesn't exist in the file system.", { HTML_TO_MARKDOWN_PATH }, ); @@ -129,7 +144,7 @@ export async function parseMarkdown( return markdownContent; } catch (error) { - logger.error("Error converting HTML to Markdown", { error }); + contextLogger.error("Error converting HTML to Markdown", { error }); return ""; // Optionally return an empty string or handle the error as needed } } diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts index 7ef3b0db8..a725b071f 100644 --- a/apps/api/src/scraper/scrapeURL/index.ts +++ b/apps/api/src/scraper/scrapeURL/index.ts @@ -367,12 +367,14 @@ async function scrapeURLLoopIter( } else if (!needsMarkdown) { checkMarkdown = engineResult.html?.trim() ?? ""; } else { + const requestId = meta.id || meta.internalOptions.crawlId; checkMarkdown = await parseMarkdown( await htmlTransform( engineResult.html, meta.url, scrapeOptions.parse({ onlyMainContent: true }), ), + { logger: meta.logger, requestId }, ); if (checkMarkdown.trim().length === 0) { @@ -382,6 +384,7 @@ async function scrapeURLLoopIter( meta.url, scrapeOptions.parse({ onlyMainContent: false }), ), + { logger: meta.logger, requestId }, ); } } diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts index 29f4091d1..c91cb16a6 100644 --- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts +++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts @@ -443,8 +443,15 @@ export async function extractData({ // console.log("scrapedPages", scrapedPages); const htmls = scrapedPages.flat().map(page => page.html); // console.log("htmls", htmls); + const requestId = extractId || scrapeId; const markdowns = await Promise.all( - htmls.map(async html => await parseMarkdown(html)), + htmls.map( + async html => + await parseMarkdown(html, { + logger, + requestId, + }), + ), ); // console.log("markdowns", markdowns); extractedData = await Promise.all( diff --git a/apps/api/src/scraper/scrapeURL/transformers/agent.ts b/apps/api/src/scraper/scrapeURL/transformers/agent.ts index 0268d02be..ab120ec7e 100644 --- a/apps/api/src/scraper/scrapeURL/transformers/agent.ts +++ b/apps/api/src/scraper/scrapeURL/transformers/agent.ts @@ -57,7 +57,11 @@ export async function performAgent( ].html; if (hasFormatOfType(meta.options.formats, "markdown")) { - const markdown = await parseMarkdown(html); + const requestId = meta.id || meta.internalOptions.crawlId; + const markdown = await parseMarkdown(html, { + logger: meta.logger, + requestId, + }); document.markdown = markdown; } if (hasFormatOfType(meta.options.formats, "html")) { diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts index 65a5ca9cc..f2a4a6ede 100644 --- a/apps/api/src/scraper/scrapeURL/transformers/index.ts +++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts @@ -98,7 +98,12 @@ async function deriveMarkdownFromHTML( return document; } - document.markdown = await parseMarkdown(document.html); + // Use scrape ID or crawl ID as request_id for tracing + const requestId = meta.id || meta.internalOptions.crawlId; + document.markdown = await parseMarkdown(document.html, { + logger: meta.logger, + requestId, + }); if ( meta.options.onlyMainContent === true && @@ -117,7 +122,10 @@ async function deriveMarkdownFromHTML( }; document = await deriveHTMLFromRawHTML(fallbackMeta, document); - document.markdown = await parseMarkdown(document.html); + document.markdown = await parseMarkdown(document.html, { + logger: meta.logger, + requestId, + }); meta.logger.info("Fallback to full content extraction completed", { markdownLength: document.markdown?.length || 0,