From 162bce8dd87ae97fc08aacb963464af3bf46431e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Fri, 19 Sep 2025 13:11:33 +0200 Subject: [PATCH] fix(removeUnwantedElements): respect fakedomain --- .../scraper/scrapeURL/lib/removeUnwantedElements.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts index 68dcfc6cd..dead67ff6 100644 --- a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts +++ b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts @@ -71,12 +71,14 @@ export const htmlTransform = async ( url: string, scrapeOptions: ScrapeOptions, ) => { - let omce_signatures: string[] | undefined = undefined; + let omceSignatures: string[] | undefined = undefined; if (scrapeOptions.__experimental_omce) { try { - const hostname = new URL(url).hostname; - omce_signatures = await queryOMCESignatures(hostname); + const hostname = + scrapeOptions.__experimental_omceDomain || new URL(url).hostname; + omceSignatures = await queryOMCESignatures(hostname); + logger.info("Got OMCE signatures", { signatures: omceSignatures.length }); } catch (error) { logger.warn("Failed to get omce signatures.", { error, @@ -98,7 +100,7 @@ export const htmlTransform = async ( .map(x => x.trim()) .filter(x => x.length !== 0), onlyMainContent: scrapeOptions.onlyMainContent, - omceSignatures: omce_signatures, + omceSignatures, }); } catch (error) { logger.warn("Failed to call html-transformer! Falling back to cheerio...", {