From 9723b5137fecc75857146e4db646d1d847f0116a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Thu, 4 Sep 2025 15:01:49 +0200 Subject: [PATCH] feat(scrapeURL): make proxy selection error fatal --- apps/api/src/lib/error.ts | 1 + .../engines/fire-engine/checkStatus.ts | 6 +++++ .../scrapeURL/engines/fire-engine/index.ts | 4 +++- .../scrapeURL/engines/fire-engine/scrape.ts | 6 +++++ apps/api/src/scraper/scrapeURL/error.ts | 24 +++++++++++++++++++ apps/api/src/scraper/scrapeURL/index.ts | 6 ++++- 6 files changed, 45 insertions(+), 2 deletions(-) diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts index 45f79d604..7fdd0847c 100644 --- a/apps/api/src/lib/error.ts +++ b/apps/api/src/lib/error.ts @@ -5,6 +5,7 @@ export type ErrorCodes = | "SCRAPE_ALL_ENGINES_FAILED" | "SCRAPE_SSL_ERROR" | "SCRAPE_SITE_ERROR" + | "SCRAPE_PROXY_SELECTION_ERROR" | "SCRAPE_PDF_PREFETCH_FAILED" | "SCRAPE_ZDR_VIOLATION_ERROR" | "SCRAPE_DNS_RESOLUTION_ERROR" diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts index 7854614b8..d0391d77e 100644 --- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts +++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts @@ -11,6 +11,7 @@ import { UnsupportedFileError, DNSResolutionError, FEPageLoadFailed, + ProxySelectionError, } from "../../error"; import { MockState } from "../../lib/mock"; import { fireEngineStagingURL, fireEngineURL } from "./scrape"; @@ -182,6 +183,11 @@ export async function fireEngineCheckStatus( } else { throw new SiteError(code); } + } else if ( + typeof status.error === "string" && + status.error.includes("proxies available for") + ) { + throw new ProxySelectionError(); } else if ( typeof status.error === "string" && status.error.includes("Dns resolution error for hostname: ") diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts index 17d527d4d..899ee2c6a 100644 --- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts @@ -21,6 +21,7 @@ import { SSLError, UnsupportedFileError, FEPageLoadFailed, + ProxySelectionError, } from "../../error"; import * as Sentry from "@sentry/node"; import { specialtyScrapeCheck } from "../utils/specialtyHandler"; @@ -100,7 +101,8 @@ async function performFireEngineScrape< error instanceof DNSResolutionError || error instanceof ActionError || error instanceof UnsupportedFileError || - error instanceof FEPageLoadFailed + error instanceof FEPageLoadFailed || + error instanceof ProxySelectionError ) { fireEngineDelete( logger.child({ diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts index 565688ad7..7a3ea0b40 100644 --- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts +++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts @@ -11,6 +11,7 @@ import { DNSResolutionError, EngineError, FEPageLoadFailed, + ProxySelectionError, SSLError, SiteError, UnsupportedFileError, @@ -263,6 +264,11 @@ export async function fireEngineScrape< status.error.includes("Javascript execution failed")) ) { throw new ActionError(status.error.split("Error: ")[1]); + } else if ( + typeof status.error === "string" && + status.error.includes("proxies available for") + ) { + throw new ProxySelectionError(); } else { throw new EngineError("Scrape job failed", { cause: { diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts index 1f07e0ae9..faae68f1d 100644 --- a/apps/api/src/scraper/scrapeURL/error.ts +++ b/apps/api/src/scraper/scrapeURL/error.ts @@ -113,6 +113,30 @@ export class SiteError extends TransportableError { } } +export class ProxySelectionError extends TransportableError { + constructor() { + super( + "SCRAPE_PROXY_SELECTION_ERROR", + "Specified proxy cannot be selected. Change `location` or `proxy` in your scrape request.", + ); + } + + serialize() { + return { + ...super.serialize(), + }; + } + + static deserialize( + _: ErrorCodes, + data: ReturnType, + ) { + const x = new ProxySelectionError(); + x.stack = data.stack; + return x; + } +} + export class ActionError extends TransportableError { constructor(public errorCode: string) { super( diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts index 86076e0f6..688708a06 100644 --- a/apps/api/src/scraper/scrapeURL/index.ts +++ b/apps/api/src/scraper/scrapeURL/index.ts @@ -38,6 +38,7 @@ import { EngineSnipedError, WaterfallNextEngineSignal, EngineUnsuccessfulError, + ProxySelectionError, } from "./error"; import { executeTransformers } from "./transformers"; import { LLMRefusalError } from "./transformers/llmExtract"; @@ -528,7 +529,8 @@ async function scrapeURLLoop(meta: Meta): Promise { error.error instanceof ActionError || error.error instanceof UnsupportedFileError || error.error instanceof PDFAntibotError || - error.error instanceof PDFInsufficientTimeError + error.error instanceof PDFInsufficientTimeError || + error.error instanceof ProxySelectionError ) { throw error.error; } else if (error.error instanceof LLMRefusalError) { @@ -864,6 +866,8 @@ export async function scrapeURL( "scrapeURL: Failed to prefetch PDF that is protected by anti-bot", { error }, ); + } else if (error instanceof ProxySelectionError) { + meta.logger.warn("scrapeURL: Proxy selection error", { error }); } else if (error instanceof AbortManagerThrownError) { throw error.inner; } else {