From 588fe390a3b5bbedc5ea36eb0c906be421006629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Wed, 13 Aug 2025 17:45:37 +0200 Subject: [PATCH] feat(crawl-errors): take advantage of TransportableError --- apps/api/src/controllers/v1/crawl-errors.ts | 51 +++++++++++++++------ apps/api/src/controllers/v2/crawl-errors.ts | 51 +++++++++++++++------ apps/api/src/controllers/v2/types.ts | 1 + 3 files changed, 73 insertions(+), 30 deletions(-) diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.ts index 307efe4b0..45084c741 100644 --- a/apps/api/src/controllers/v1/crawl-errors.ts +++ b/apps/api/src/controllers/v1/crawl-errors.ts @@ -14,6 +14,8 @@ import { configDotenv } from "dotenv"; import { Job } from "bullmq"; import { supabase_rr_service } from "../../services/supabase"; import { logger } from "../../lib/logger"; +import { deserializeTransportableError } from "../../lib/error-serde"; +import { TransportableError } from "../../lib/error"; configDotenv(); export async function getJob(id: string) { @@ -57,15 +59,23 @@ export async function crawlErrorsController( } res.status(200).json({ - errors: (await getJobs(failedJobIDs)).map((x) => ({ - id: x.id, - timestamp: - x.finishedOn !== undefined - ? new Date(x.finishedOn).toISOString() - : undefined, - url: x.data.url, - error: x.failedReason, - })), + errors: (await getJobs(failedJobIDs)).map((x) => { + const error = deserializeTransportableError(x.failedReason) as TransportableError | null; + return { + id: x.id, + timestamp: + x.finishedOn !== undefined + ? new Date(x.finishedOn).toISOString() + : undefined, + url: x.data.url, + ...(error ? { + code: error.code, + error: error.message, + } : { + error: x.failedReason, + }), + }; + }), robotsBlocked: await redisEvictConnection.smembers( "crawl:" + req.params.jobId + ":robots_blocked", ), @@ -117,12 +127,23 @@ export async function crawlErrorsController( } res.status(200).json({ - errors: (failedJobs || []).map((job) => ({ - id: job.job_id, - timestamp: new Date(job.date_added).toISOString(), - url: job.page_options?.url || job.page_options?.urls?.[0] || "Unknown URL", - error: job.message || "Unknown error", - })), + errors: (failedJobs || []).map((job) => { + const error = deserializeTransportableError(job.message) as TransportableError | null; + return { + id: job.job_id, + timestamp: + job.finishedOn !== undefined + ? new Date(job.finishedOn).toISOString() + : undefined, + url: job.url, + ...(error ? { + code: error.code, + error: error.message, + } : { + error: job.message, + }), + }; + }), robotsBlocked: await redisEvictConnection.smembers( "crawl:" + req.params.jobId + ":robots_blocked", ), diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.ts index 307efe4b0..45084c741 100644 --- a/apps/api/src/controllers/v2/crawl-errors.ts +++ b/apps/api/src/controllers/v2/crawl-errors.ts @@ -14,6 +14,8 @@ import { configDotenv } from "dotenv"; import { Job } from "bullmq"; import { supabase_rr_service } from "../../services/supabase"; import { logger } from "../../lib/logger"; +import { deserializeTransportableError } from "../../lib/error-serde"; +import { TransportableError } from "../../lib/error"; configDotenv(); export async function getJob(id: string) { @@ -57,15 +59,23 @@ export async function crawlErrorsController( } res.status(200).json({ - errors: (await getJobs(failedJobIDs)).map((x) => ({ - id: x.id, - timestamp: - x.finishedOn !== undefined - ? new Date(x.finishedOn).toISOString() - : undefined, - url: x.data.url, - error: x.failedReason, - })), + errors: (await getJobs(failedJobIDs)).map((x) => { + const error = deserializeTransportableError(x.failedReason) as TransportableError | null; + return { + id: x.id, + timestamp: + x.finishedOn !== undefined + ? new Date(x.finishedOn).toISOString() + : undefined, + url: x.data.url, + ...(error ? { + code: error.code, + error: error.message, + } : { + error: x.failedReason, + }), + }; + }), robotsBlocked: await redisEvictConnection.smembers( "crawl:" + req.params.jobId + ":robots_blocked", ), @@ -117,12 +127,23 @@ export async function crawlErrorsController( } res.status(200).json({ - errors: (failedJobs || []).map((job) => ({ - id: job.job_id, - timestamp: new Date(job.date_added).toISOString(), - url: job.page_options?.url || job.page_options?.urls?.[0] || "Unknown URL", - error: job.message || "Unknown error", - })), + errors: (failedJobs || []).map((job) => { + const error = deserializeTransportableError(job.message) as TransportableError | null; + return { + id: job.job_id, + timestamp: + job.finishedOn !== undefined + ? new Date(job.finishedOn).toISOString() + : undefined, + url: job.url, + ...(error ? { + code: error.code, + error: error.message, + } : { + error: job.message, + }), + }; + }), robotsBlocked: await redisEvictConnection.smembers( "crawl:" + req.params.jobId + ":robots_blocked", ), diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts index 9068eda7a..64f0e7552 100644 --- a/apps/api/src/controllers/v2/types.ts +++ b/apps/api/src/controllers/v2/types.ts @@ -867,6 +867,7 @@ export type CrawlErrorsResponse = id: string; timestamp?: string; url: string; + code?: ErrorCodes; error: string; }[]; robotsBlocked: string[];