diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index ba1780a84..1265ebb80 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "4.44.1", + "version": "4.45.0", "description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/crawl-cancel-abort.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/crawl-cancel-abort.unit.test.ts new file mode 100644 index 000000000..e33f15d7c --- /dev/null +++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/crawl-cancel-abort.unit.test.ts @@ -0,0 +1,189 @@ +import { describe, test, expect } from "@jest/globals"; +import axios, { type AxiosAdapter } from "axios"; +import { FirecrawlClient } from "../../../v2/client"; +import { HttpClient } from "../../../v2/utils/httpClient"; +import { waitForCrawlCompletion } from "../../../v2/methods/crawl"; +import { SdkError } from "../../../v2/types"; + +const API_URL = "https://api.firecrawl.dev"; +const JOB_ID = "job-123"; + +type Sent = { method: string; path: string }; +type Reply = { status: number; data: unknown }; + +function withAdapter(instance: T, route: (req: Sent) => Reply): Sent[] { + const sent: Sent[] = []; + const adapter: AxiosAdapter = async (config) => { + const req = { + method: (config.method || "get").toLowerCase(), + path: new URL(axios.getUri(config)).pathname, + }; + sent.push(req); + const { status, data } = route(req); + const response = { data, status, statusText: String(status), headers: {}, config }; + const validate = config.validateStatus; + if (validate && !validate(status)) { + throw new axios.AxiosError(`Request failed with status code ${status}`, undefined, config, null, response); + } + return response; + }; + instance.defaults.adapter = adapter; + return sent; +} + +function makeClient(route: (req: Sent) => Reply) { + const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL }); + const sent = withAdapter((client as any).http.instance, route); + return { client, sent }; +} + +// The cancel runs in the background, so let its request reach the adapter. +const flush = () => new Promise((r) => setTimeout(r, 0)); + +const running: Reply = { status: 200, data: { success: true, status: "scraping", completed: 0, total: 1, data: [] } }; +const started: Reply = { status: 200, data: { success: true, id: JOB_ID, url: "https://example.com" } }; + +describe("v2 cancelCrawl", () => { + test("returns false when the crawl is already completed (409)", async () => { + const { client, sent } = makeClient(() => ({ status: 409, data: { error: "Crawl is already completed" } })); + await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(false); + expect(sent).toEqual([{ method: "delete", path: `/v2/crawl/${JOB_ID}` }]); + }); + + test("returns false on 409 when the transport does not throw for 4xx", async () => { + const { client } = makeClient(() => ({ status: 409, data: { error: "Crawl is already completed" } })); + (client as any).http.instance.defaults.validateStatus = () => true; + await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(false); + }); + + test("returns true when the crawl is cancelled", async () => { + const { client } = makeClient(() => ({ status: 200, data: { status: "cancelled" } })); + await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(true); + }); + + test("still throws for other errors", async () => { + const { client } = makeClient(() => ({ status: 404, data: { error: "Job not found" } })); + const err = await client.cancelCrawl(JOB_ID).catch((e) => e); + expect(err).toBeInstanceOf(SdkError); + expect(err.status).toBe(404); + }); +}); + +describe("v2 crawl with an AbortSignal", () => { + test("abort during polling stops polling, cancels the job, and rejects with the abort reason", async () => { + const controller = new AbortController(); + let polls = 0; + const { client, sent } = makeClient((req) => { + if (req.method === "post") return started; + if (req.method === "delete") return { status: 200, data: { status: "cancelled" } }; + polls += 1; + if (polls === 1) setTimeout(() => controller.abort(), 10); + return running; + }); + + const t0 = Date.now(); + const err = await client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal }).catch((e) => e); + + expect(Date.now() - t0).toBeLessThan(5000); + expect(err).toBe(controller.signal.reason); + expect((err as Error).name).toBe("AbortError"); + await flush(); + expect(sent.map((r) => r.method)).toEqual(["post", "get", "delete"]); + expect(sent[2]!.path).toBe(`/v2/crawl/${JOB_ID}`); + }); + + test("rejects with a custom abort reason and ignores a failing cancel", async () => { + const controller = new AbortController(); + const reason = new Error("caller gave up"); + const { client, sent } = makeClient((req) => { + if (req.method === "post") return started; + if (req.method === "delete") return { status: 500, data: { error: "boom" } }; + setTimeout(() => controller.abort(reason), 10); + return running; + }); + + await expect(client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal })).rejects.toBe(reason); + await flush(); + expect(sent.some((r) => r.method === "delete")).toBe(true); + }); + + test("a hanging cancel does not delay the rejection", async () => { + const controller = new AbortController(); + const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL }); + const methods: string[] = []; + (client as any).http.instance.defaults.adapter = (async (config) => { + methods.push(config.method!); + if (config.method === "delete") return new Promise(() => {}); + if (config.method === "get") setTimeout(() => controller.abort(), 10); + const data = config.method === "post" ? started.data : running.data; + return { data, status: 200, statusText: "OK", headers: {}, config }; + }) as AxiosAdapter; + + // Read the reason after the abort: before it, signal.reason is undefined. + const err = await client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal }).catch((e) => e); + expect(err).toBe(controller.signal.reason); + expect((err as Error).name).toBe("AbortError"); + await flush(); + expect(methods).toEqual(["post", "get", "delete"]); + }); + + test("abort while the start request is in flight cancels the job once the id arrives", async () => { + const controller = new AbortController(); + const { client, sent } = makeClient((req) => { + if (req.method === "post") { + controller.abort(); + return started; + } + if (req.method === "delete") return { status: 200, data: { status: "cancelled" } }; + return running; + }); + + const err = await client.crawl("https://example.com", { signal: controller.signal }).catch((e) => e); + expect(err).toBe(controller.signal.reason); + expect((err as Error).name).toBe("AbortError"); + await flush(); + expect(sent.map((r) => r.method)).toEqual(["post", "delete"]); + }); + + test("an already aborted signal sends no request", async () => { + const controller = new AbortController(); + controller.abort(); + const { client, sent } = makeClient(() => started); + + await expect(client.crawl("https://example.com", { signal: controller.signal })).rejects.toBe(controller.signal.reason); + expect(sent).toHaveLength(0); + }); + + test("the signal is not sent in the crawl request body", async () => { + const controller = new AbortController(); + const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL }); + const bodies: unknown[] = []; + (client as any).http.instance.defaults.adapter = (async (config) => { + if (config.method === "post") bodies.push(JSON.parse(config.data as string)); + const data = config.method === "post" ? started.data : { success: true, status: "completed", completed: 1, total: 1, data: [] }; + return { data, status: 200, statusText: "OK", headers: {}, config }; + }) as AxiosAdapter; + + const job = await client.crawl("https://example.com", { limit: 1, signal: controller.signal }); + expect(job.status).toBe("completed"); + expect(bodies[0]).toMatchObject({ url: "https://example.com", limit: 1 }); + expect(Object.keys(bodies[0] as object)).not.toContain("signal"); + }); +}); + +describe("v2 waitForCrawlCompletion with an AbortSignal", () => { + test("stops polling and rejects without cancelling the job", async () => { + const http = new HttpClient({ apiKey: "fc-test", apiUrl: API_URL }); + const controller = new AbortController(); + const sent = withAdapter((http as any).instance, () => { + setTimeout(() => controller.abort(), 10); + return running; + }); + + const err = await waitForCrawlCompletion(http, JOB_ID, 30, undefined, controller.signal).catch((e) => e); + expect(err).toBe(controller.signal.reason); + expect((err as Error).name).toBe("AbortError"); + await new Promise((r) => setTimeout(r, 50)); + expect(sent.map((r) => r.method)).toEqual(["get"]); + }); +}); diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.ts index 40c7c6bc2..694fe04e0 100644 --- a/apps/js-sdk/firecrawl/src/v2/client.ts +++ b/apps/js-sdk/firecrawl/src/v2/client.ts @@ -394,7 +394,7 @@ export class FirecrawlClient { /** * Cancel a crawl job. * @param jobId Crawl job id. - * @returns True if cancelled. + * @returns True if cancelled. False if the crawl was already completed (the API answers 409). */ async cancelCrawl(jobId: string): Promise { return cancelCrawl(this.http, jobId); @@ -402,11 +402,14 @@ export class FirecrawlClient { /** * Convenience waiter: start a crawl and poll until it finishes. * @param url Root URL to crawl. - * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds). + * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal). + * When `signal` aborts, polling stops, the job gets a best-effort cancel request, + * and the promise rejects with `signal.reason`. * @returns Final job snapshot. */ - async crawl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number } = {}): Promise { - return crawlWaiter(this.http, { url, ...req }, req.pollInterval, req.timeout); + async crawl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number; signal?: AbortSignal } = {}): Promise { + const { pollInterval, timeout, signal, ...options } = req; + return crawlWaiter(this.http, { url, ...options }, pollInterval, timeout, signal); } /** * Retrieve crawl errors and robots.txt blocks. @@ -724,7 +727,7 @@ export class FirecrawlClient { } /** @deprecated V1 compatibility alias for agent recovery. Prefer crawl(). */ - async crawlUrl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number } = {}): Promise { + async crawlUrl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number; signal?: AbortSignal } = {}): Promise { return this.crawl(url, req); } diff --git a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts index 834444c13..01405d702 100644 --- a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts +++ b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts @@ -107,28 +107,94 @@ export async function getCrawlStatus( } } +/** + * Cancel a crawl job. Resolves `false` when the API answers 409 because the + * crawl already finished, so there is nothing left to cancel. + */ export async function cancelCrawl(http: HttpClient, jobId: string): Promise { try { const res = await http.delete<{ status: string }>(`/v2/crawl/${jobId}`); + if (res.status === 409) return false; if (res.status !== 200) throwForBadResponse(res, "cancel crawl"); return res.data?.status === "cancelled"; } catch (err: any) { - if (err?.isAxiosError) return normalizeAxiosError(err, "cancel crawl"); + if (err?.isAxiosError) { + if (err.response?.status === 409) return false; + return normalizeAxiosError(err, "cancel crawl"); + } throw err; } } -export async function waitForCrawlCompletion(http: HttpClient, jobId: string, pollInterval = 2, timeout?: number): Promise { +function abortReason(signal: AbortSignal): unknown { + if (signal.reason !== undefined) return signal.reason; + return new DOMException("This operation was aborted", "AbortError"); +} + +/** Reject with the abort reason as soon as `signal` aborts. */ +function raceAbort(promise: Promise, signal?: AbortSignal): Promise { + if (!signal) return promise; + if (signal.aborted) { + promise.catch(() => {}); + return Promise.reject(abortReason(signal)); + } + return new Promise((resolve, reject) => { + const onAbort = () => { + promise.catch(() => {}); + reject(abortReason(signal)); + }; + signal.addEventListener("abort", onAbort, { once: true }); + promise.then( + (value) => { + signal.removeEventListener("abort", onAbort); + resolve(value); + }, + (err) => { + signal.removeEventListener("abort", onAbort); + reject(err); + }, + ); + }); +} + +function sleep(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal?.aborted) return reject(abortReason(signal)); + const onAbort = () => { + clearTimeout(timer); + reject(abortReason(signal!)); + }; + const timer = setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + resolve(); + }, ms); + signal?.addEventListener("abort", onAbort, { once: true }); + }); +} + +/** + * Poll a crawl until it reaches a terminal status. When `signal` aborts, polling + * stops and the promise rejects with the abort reason. The job is not cancelled. + */ +export async function waitForCrawlCompletion( + http: HttpClient, + jobId: string, + pollInterval = 2, + timeout?: number, + signal?: AbortSignal, +): Promise { const start = Date.now(); while (true) { + if (signal?.aborted) throw abortReason(signal); try { - const status = await getCrawlStatus(http, jobId); + const status = await raceAbort(getCrawlStatus(http, jobId), signal); if (["completed", "failed", "cancelled"].includes(status.status)) { return status; } } catch (err: any) { + if (signal?.aborted) throw abortReason(signal); // Don't retry on permanent errors (4xx) - re-throw immediately with jobId context if (!isRetryableError(err)) { // Create new error with jobId for better debugging (non-retryable errors like 404) @@ -151,13 +217,31 @@ export async function waitForCrawlCompletion(http: HttpClient, jobId: string, po throw new JobTimeoutError(jobId, timeout, 'crawl'); } - await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000))); + await sleep(Math.max(1000, pollInterval * 1000), signal); } } -export async function crawl(http: HttpClient, request: CrawlRequest, pollInterval = 2, timeout?: number): Promise { +/** + * Start a crawl and wait for it to finish. When `signal` aborts, polling stops, + * the job gets a best-effort cancel request, and the promise rejects with the + * abort reason at once, without waiting for the cancel. A signal that is already + * aborted rejects before any request. + */ +export async function crawl( + http: HttpClient, + request: CrawlRequest, + pollInterval = 2, + timeout?: number, + signal?: AbortSignal, +): Promise { + if (signal?.aborted) throw abortReason(signal); const started = await startCrawl(http, request); - return waitForCrawlCompletion(http, started.id, pollInterval, timeout); + try { + return await waitForCrawlCompletion(http, started.id, pollInterval, timeout, signal); + } catch (err) { + if (signal?.aborted) cancelCrawl(http, started.id).catch(() => {}); + throw err; + } } export async function getCrawlErrors(http: HttpClient, crawlId: string): Promise {