diff --git a/apps/api/src/lib/html-to-markdown-client.ts b/apps/api/src/lib/html-to-markdown-client.ts
index 73bb8caeb..a4e9bc263 100644
--- a/apps/api/src/lib/html-to-markdown-client.ts
+++ b/apps/api/src/lib/html-to-markdown-client.ts
@@ -8,6 +8,7 @@
import axios, { AxiosInstance, AxiosError } from "axios";
import { config } from "../config";
import { logger } from "./logger";
+import type { Logger } from "winston";
import * as Sentry from "@sentry/node";
interface ConvertRequest {
@@ -27,19 +28,24 @@ interface ErrorResponse {
/**
* Convert HTML to Markdown using direct axios call
* @param html HTML string to convert
- * @param serviceUrl URL of the HTML to Markdown service (default: http://localhost:8080)
+ * @param context Optional context with logger and requestId
* @returns Markdown string
* @throws Error if conversion fails
*/
export async function convertHTMLToMarkdownWithHttpService(
html: string,
- serviceUrl?: string,
+ context?: {
+ logger?: Logger;
+ requestId?: string;
+ },
): Promise {
if (!html || html.trim() === "") {
return "";
}
- const url = serviceUrl || config.HTML_TO_MARKDOWN_SERVICE_URL;
+ const contextLogger = context?.logger || logger;
+ const requestId = context?.requestId;
+ const url = config.HTML_TO_MARKDOWN_SERVICE_URL;
const startTime = Date.now();
try {
@@ -62,7 +68,7 @@ export async function convertHTMLToMarkdownWithHttpService(
throw new Error("Conversion was not successful");
}
- logger.debug("HTML to Markdown conversion successful", {
+ contextLogger.debug("HTML to Markdown conversion successful", {
duration_ms: duration,
input_size: html.length,
output_size: response.data.markdown.length,
@@ -79,7 +85,7 @@ export async function convertHTMLToMarkdownWithHttpService(
axiosError.response?.data?.error || axiosError.message;
const statusCode = axiosError.response?.status;
- logger.error("HTML to Markdown conversion failed", {
+ contextLogger.error("HTML to Markdown conversion failed", {
error: errorMessage,
statusCode,
duration_ms: duration,
@@ -91,6 +97,7 @@ export async function convertHTMLToMarkdownWithHttpService(
tags: {
service: "html-to-markdown",
status_code: statusCode,
+ ...(requestId ? { request_id: requestId } : {}),
},
extra: {
serviceUrl: url,
@@ -101,10 +108,17 @@ export async function convertHTMLToMarkdownWithHttpService(
throw new Error(`HTML to Markdown conversion failed: ${errorMessage}`);
} else {
- logger.error("Unexpected error during HTML to Markdown conversion", {
- error,
+ contextLogger.error(
+ "Unexpected error during HTML to Markdown conversion",
+ {
+ error,
+ },
+ );
+ Sentry.captureException(error, {
+ tags: {
+ ...(requestId ? { request_id: requestId } : {}),
+ },
});
- Sentry.captureException(error);
throw error;
}
}
diff --git a/apps/api/src/lib/html-to-markdown.ts b/apps/api/src/lib/html-to-markdown.ts
index b7d2f3848..69bfa6941 100644
--- a/apps/api/src/lib/html-to-markdown.ts
+++ b/apps/api/src/lib/html-to-markdown.ts
@@ -3,6 +3,7 @@ import { config } from "../config";
import "../services/sentry";
import * as Sentry from "@sentry/node";
import { logger } from "./logger";
+import type { Logger } from "winston";
import { stat } from "fs/promises";
import { HTML_TO_MARKDOWN_PATH } from "../natives";
import { convertHTMLToMarkdownWithHttpService } from "./html-to-markdown-client";
@@ -52,25 +53,36 @@ class GoMarkdownConverter {
export async function parseMarkdown(
html: string | null | undefined,
+ context?: {
+ logger?: Logger;
+ requestId?: string;
+ },
): Promise {
if (!html) {
return "";
}
+ const contextLogger = context?.logger || logger;
+ const requestId = context?.requestId;
+
// Try HTTP service first if enabled
if (config.HTML_TO_MARKDOWN_SERVICE_URL) {
try {
- let markdownContent = await convertHTMLToMarkdownWithHttpService(html);
+ let markdownContent = await convertHTMLToMarkdownWithHttpService(html, {
+ logger: contextLogger,
+ requestId,
+ });
markdownContent = await postProcessMarkdown(markdownContent);
return markdownContent;
} catch (error) {
- logger.error(
+ contextLogger.error(
"Error converting HTML to Markdown with HTTP service, falling back to original parser",
{ error },
);
Sentry.captureException(error, {
tags: {
fallback: "original_parser",
+ ...(requestId ? { request_id: requestId } : {}),
},
});
}
@@ -81,7 +93,6 @@ export async function parseMarkdown(
const converter = await GoMarkdownConverter.getInstance();
let markdownContent = await converter.convertHTMLToMarkdown(html);
markdownContent = await postProcessMarkdown(markdownContent);
- // logger.info(`HTML to Markdown conversion using Go parser successful`);
return markdownContent;
}
} catch (error) {
@@ -89,12 +100,16 @@ export async function parseMarkdown(
!(error instanceof Error) ||
error.message !== "Go shared library not found"
) {
- Sentry.captureException(error);
- logger.error(
+ Sentry.captureException(error, {
+ tags: {
+ ...(requestId ? { request_id: requestId } : {}),
+ },
+ });
+ contextLogger.error(
`Error converting HTML to Markdown with Go parser: ${error}`,
);
} else {
- logger.warn(
+ contextLogger.warn(
"Tried to use Go parser, but it doesn't exist in the file system.",
{ HTML_TO_MARKDOWN_PATH },
);
@@ -129,7 +144,7 @@ export async function parseMarkdown(
return markdownContent;
} catch (error) {
- logger.error("Error converting HTML to Markdown", { error });
+ contextLogger.error("Error converting HTML to Markdown", { error });
return ""; // Optionally return an empty string or handle the error as needed
}
}
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 7ef3b0db8..a725b071f 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -367,12 +367,14 @@ async function scrapeURLLoopIter(
} else if (!needsMarkdown) {
checkMarkdown = engineResult.html?.trim() ?? "";
} else {
+ const requestId = meta.id || meta.internalOptions.crawlId;
checkMarkdown = await parseMarkdown(
await htmlTransform(
engineResult.html,
meta.url,
scrapeOptions.parse({ onlyMainContent: true }),
),
+ { logger: meta.logger, requestId },
);
if (checkMarkdown.trim().length === 0) {
@@ -382,6 +384,7 @@ async function scrapeURLLoopIter(
meta.url,
scrapeOptions.parse({ onlyMainContent: false }),
),
+ { logger: meta.logger, requestId },
);
}
}
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
index 29f4091d1..c91cb16a6 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
@@ -443,8 +443,15 @@ export async function extractData({
// console.log("scrapedPages", scrapedPages);
const htmls = scrapedPages.flat().map(page => page.html);
// console.log("htmls", htmls);
+ const requestId = extractId || scrapeId;
const markdowns = await Promise.all(
- htmls.map(async html => await parseMarkdown(html)),
+ htmls.map(
+ async html =>
+ await parseMarkdown(html, {
+ logger,
+ requestId,
+ }),
+ ),
);
// console.log("markdowns", markdowns);
extractedData = await Promise.all(
diff --git a/apps/api/src/scraper/scrapeURL/transformers/agent.ts b/apps/api/src/scraper/scrapeURL/transformers/agent.ts
index 0268d02be..ab120ec7e 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/agent.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/agent.ts
@@ -57,7 +57,11 @@ export async function performAgent(
].html;
if (hasFormatOfType(meta.options.formats, "markdown")) {
- const markdown = await parseMarkdown(html);
+ const requestId = meta.id || meta.internalOptions.crawlId;
+ const markdown = await parseMarkdown(html, {
+ logger: meta.logger,
+ requestId,
+ });
document.markdown = markdown;
}
if (hasFormatOfType(meta.options.formats, "html")) {
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index 65a5ca9cc..f2a4a6ede 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -98,7 +98,12 @@ async function deriveMarkdownFromHTML(
return document;
}
- document.markdown = await parseMarkdown(document.html);
+ // Use scrape ID or crawl ID as request_id for tracing
+ const requestId = meta.id || meta.internalOptions.crawlId;
+ document.markdown = await parseMarkdown(document.html, {
+ logger: meta.logger,
+ requestId,
+ });
if (
meta.options.onlyMainContent === true &&
@@ -117,7 +122,10 @@ async function deriveMarkdownFromHTML(
};
document = await deriveHTMLFromRawHTML(fallbackMeta, document);
- document.markdown = await parseMarkdown(document.html);
+ document.markdown = await parseMarkdown(document.html, {
+ logger: meta.logger,
+ requestId,
+ });
meta.logger.info("Fallback to full content extraction completed", {
markdownLength: document.markdown?.length || 0,