mirror of
https://github.com/firecrawl/firecrawl.git
synced 2026-10-13 09:26:21 +08:00
feat(js-sdk): abortable crawl() that cancels its job; cancelCrawl returns false on 409 (#4898)
* feat(js-sdk): abortable crawl() that cancels its job; cancelCrawl returns false on 409 cancelCrawl now resolves false when the API answers 409 because the crawl is already completed. Other errors still throw. crawl() accepts an optional AbortSignal. When it aborts, polling stops, the SDK sends a best-effort cancel request for the job, and the promise rejects with signal.reason. A signal that is already aborted rejects before any request. waitForCrawlCompletion also accepts the signal, but only stops polling and does not cancel the job. Bumps the JS SDK to 4.43.0. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01H5YqtAzmu8fHaWAUDQLZa4 * chore(js-sdk): bump to 4.45.0 for the crawl() abort signal Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(js-sdk): reject an aborted crawl() without waiting for the cancel request The best-effort cancel now runs in the background, so a slow or hanging DELETE no longer delays the abort rejection. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test(js-sdk): read the abort reason after the abort in the hanging-cancel test The test passed `controller.signal.reason` to `rejects.toBe()` before the abort happened, so the expected value was `undefined`. Catch the error first, then compare it with the reason. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01H5YqtAzmu8fHaWAUDQLZa4 --------- Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: mogery <mogery@sideguide.dev> Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
claude[bot]
co-authored by
Claude Opus 5.5
mogery
capy-ai[bot]
Claude Opus 5.5
mogery
capy-ai[bot]
parent
a73af1d5df
commit
23b68aefd2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@mendable/firecrawl-js",
|
||||
"version": "4.44.1",
|
||||
"version": "4.45.0",
|
||||
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
import { describe, test, expect } from "@jest/globals";
|
||||
import axios, { type AxiosAdapter } from "axios";
|
||||
import { FirecrawlClient } from "../../../v2/client";
|
||||
import { HttpClient } from "../../../v2/utils/httpClient";
|
||||
import { waitForCrawlCompletion } from "../../../v2/methods/crawl";
|
||||
import { SdkError } from "../../../v2/types";
|
||||
|
||||
const API_URL = "https://api.firecrawl.dev";
|
||||
const JOB_ID = "job-123";
|
||||
|
||||
type Sent = { method: string; path: string };
|
||||
type Reply = { status: number; data: unknown };
|
||||
|
||||
function withAdapter<T extends { defaults: any }>(instance: T, route: (req: Sent) => Reply): Sent[] {
|
||||
const sent: Sent[] = [];
|
||||
const adapter: AxiosAdapter = async (config) => {
|
||||
const req = {
|
||||
method: (config.method || "get").toLowerCase(),
|
||||
path: new URL(axios.getUri(config)).pathname,
|
||||
};
|
||||
sent.push(req);
|
||||
const { status, data } = route(req);
|
||||
const response = { data, status, statusText: String(status), headers: {}, config };
|
||||
const validate = config.validateStatus;
|
||||
if (validate && !validate(status)) {
|
||||
throw new axios.AxiosError(`Request failed with status code ${status}`, undefined, config, null, response);
|
||||
}
|
||||
return response;
|
||||
};
|
||||
instance.defaults.adapter = adapter;
|
||||
return sent;
|
||||
}
|
||||
|
||||
function makeClient(route: (req: Sent) => Reply) {
|
||||
const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });
|
||||
const sent = withAdapter((client as any).http.instance, route);
|
||||
return { client, sent };
|
||||
}
|
||||
|
||||
// The cancel runs in the background, so let its request reach the adapter.
|
||||
const flush = () => new Promise((r) => setTimeout(r, 0));
|
||||
|
||||
const running: Reply = { status: 200, data: { success: true, status: "scraping", completed: 0, total: 1, data: [] } };
|
||||
const started: Reply = { status: 200, data: { success: true, id: JOB_ID, url: "https://example.com" } };
|
||||
|
||||
describe("v2 cancelCrawl", () => {
|
||||
test("returns false when the crawl is already completed (409)", async () => {
|
||||
const { client, sent } = makeClient(() => ({ status: 409, data: { error: "Crawl is already completed" } }));
|
||||
await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(false);
|
||||
expect(sent).toEqual([{ method: "delete", path: `/v2/crawl/${JOB_ID}` }]);
|
||||
});
|
||||
|
||||
test("returns false on 409 when the transport does not throw for 4xx", async () => {
|
||||
const { client } = makeClient(() => ({ status: 409, data: { error: "Crawl is already completed" } }));
|
||||
(client as any).http.instance.defaults.validateStatus = () => true;
|
||||
await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(false);
|
||||
});
|
||||
|
||||
test("returns true when the crawl is cancelled", async () => {
|
||||
const { client } = makeClient(() => ({ status: 200, data: { status: "cancelled" } }));
|
||||
await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(true);
|
||||
});
|
||||
|
||||
test("still throws for other errors", async () => {
|
||||
const { client } = makeClient(() => ({ status: 404, data: { error: "Job not found" } }));
|
||||
const err = await client.cancelCrawl(JOB_ID).catch((e) => e);
|
||||
expect(err).toBeInstanceOf(SdkError);
|
||||
expect(err.status).toBe(404);
|
||||
});
|
||||
});
|
||||
|
||||
describe("v2 crawl with an AbortSignal", () => {
|
||||
test("abort during polling stops polling, cancels the job, and rejects with the abort reason", async () => {
|
||||
const controller = new AbortController();
|
||||
let polls = 0;
|
||||
const { client, sent } = makeClient((req) => {
|
||||
if (req.method === "post") return started;
|
||||
if (req.method === "delete") return { status: 200, data: { status: "cancelled" } };
|
||||
polls += 1;
|
||||
if (polls === 1) setTimeout(() => controller.abort(), 10);
|
||||
return running;
|
||||
});
|
||||
|
||||
const t0 = Date.now();
|
||||
const err = await client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal }).catch((e) => e);
|
||||
|
||||
expect(Date.now() - t0).toBeLessThan(5000);
|
||||
expect(err).toBe(controller.signal.reason);
|
||||
expect((err as Error).name).toBe("AbortError");
|
||||
await flush();
|
||||
expect(sent.map((r) => r.method)).toEqual(["post", "get", "delete"]);
|
||||
expect(sent[2]!.path).toBe(`/v2/crawl/${JOB_ID}`);
|
||||
});
|
||||
|
||||
test("rejects with a custom abort reason and ignores a failing cancel", async () => {
|
||||
const controller = new AbortController();
|
||||
const reason = new Error("caller gave up");
|
||||
const { client, sent } = makeClient((req) => {
|
||||
if (req.method === "post") return started;
|
||||
if (req.method === "delete") return { status: 500, data: { error: "boom" } };
|
||||
setTimeout(() => controller.abort(reason), 10);
|
||||
return running;
|
||||
});
|
||||
|
||||
await expect(client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal })).rejects.toBe(reason);
|
||||
await flush();
|
||||
expect(sent.some((r) => r.method === "delete")).toBe(true);
|
||||
});
|
||||
|
||||
test("a hanging cancel does not delay the rejection", async () => {
|
||||
const controller = new AbortController();
|
||||
const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });
|
||||
const methods: string[] = [];
|
||||
(client as any).http.instance.defaults.adapter = (async (config) => {
|
||||
methods.push(config.method!);
|
||||
if (config.method === "delete") return new Promise(() => {});
|
||||
if (config.method === "get") setTimeout(() => controller.abort(), 10);
|
||||
const data = config.method === "post" ? started.data : running.data;
|
||||
return { data, status: 200, statusText: "OK", headers: {}, config };
|
||||
}) as AxiosAdapter;
|
||||
|
||||
// Read the reason after the abort: before it, signal.reason is undefined.
|
||||
const err = await client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal }).catch((e) => e);
|
||||
expect(err).toBe(controller.signal.reason);
|
||||
expect((err as Error).name).toBe("AbortError");
|
||||
await flush();
|
||||
expect(methods).toEqual(["post", "get", "delete"]);
|
||||
});
|
||||
|
||||
test("abort while the start request is in flight cancels the job once the id arrives", async () => {
|
||||
const controller = new AbortController();
|
||||
const { client, sent } = makeClient((req) => {
|
||||
if (req.method === "post") {
|
||||
controller.abort();
|
||||
return started;
|
||||
}
|
||||
if (req.method === "delete") return { status: 200, data: { status: "cancelled" } };
|
||||
return running;
|
||||
});
|
||||
|
||||
const err = await client.crawl("https://example.com", { signal: controller.signal }).catch((e) => e);
|
||||
expect(err).toBe(controller.signal.reason);
|
||||
expect((err as Error).name).toBe("AbortError");
|
||||
await flush();
|
||||
expect(sent.map((r) => r.method)).toEqual(["post", "delete"]);
|
||||
});
|
||||
|
||||
test("an already aborted signal sends no request", async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
const { client, sent } = makeClient(() => started);
|
||||
|
||||
await expect(client.crawl("https://example.com", { signal: controller.signal })).rejects.toBe(controller.signal.reason);
|
||||
expect(sent).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("the signal is not sent in the crawl request body", async () => {
|
||||
const controller = new AbortController();
|
||||
const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });
|
||||
const bodies: unknown[] = [];
|
||||
(client as any).http.instance.defaults.adapter = (async (config) => {
|
||||
if (config.method === "post") bodies.push(JSON.parse(config.data as string));
|
||||
const data = config.method === "post" ? started.data : { success: true, status: "completed", completed: 1, total: 1, data: [] };
|
||||
return { data, status: 200, statusText: "OK", headers: {}, config };
|
||||
}) as AxiosAdapter;
|
||||
|
||||
const job = await client.crawl("https://example.com", { limit: 1, signal: controller.signal });
|
||||
expect(job.status).toBe("completed");
|
||||
expect(bodies[0]).toMatchObject({ url: "https://example.com", limit: 1 });
|
||||
expect(Object.keys(bodies[0] as object)).not.toContain("signal");
|
||||
});
|
||||
});
|
||||
|
||||
describe("v2 waitForCrawlCompletion with an AbortSignal", () => {
|
||||
test("stops polling and rejects without cancelling the job", async () => {
|
||||
const http = new HttpClient({ apiKey: "fc-test", apiUrl: API_URL });
|
||||
const controller = new AbortController();
|
||||
const sent = withAdapter((http as any).instance, () => {
|
||||
setTimeout(() => controller.abort(), 10);
|
||||
return running;
|
||||
});
|
||||
|
||||
const err = await waitForCrawlCompletion(http, JOB_ID, 30, undefined, controller.signal).catch((e) => e);
|
||||
expect(err).toBe(controller.signal.reason);
|
||||
expect((err as Error).name).toBe("AbortError");
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
expect(sent.map((r) => r.method)).toEqual(["get"]);
|
||||
});
|
||||
});
|
||||
@@ -394,7 +394,7 @@ export class FirecrawlClient {
|
||||
/**
|
||||
* Cancel a crawl job.
|
||||
* @param jobId Crawl job id.
|
||||
* @returns True if cancelled.
|
||||
* @returns True if cancelled. False if the crawl was already completed (the API answers 409).
|
||||
*/
|
||||
async cancelCrawl(jobId: string): Promise<boolean> {
|
||||
return cancelCrawl(this.http, jobId);
|
||||
@@ -402,11 +402,14 @@ export class FirecrawlClient {
|
||||
/**
|
||||
* Convenience waiter: start a crawl and poll until it finishes.
|
||||
* @param url Root URL to crawl.
|
||||
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
|
||||
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
|
||||
* When `signal` aborts, polling stops, the job gets a best-effort cancel request,
|
||||
* and the promise rejects with `signal.reason`.
|
||||
* @returns Final job snapshot.
|
||||
*/
|
||||
async crawl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number } = {}): Promise<CrawlJob> {
|
||||
return crawlWaiter(this.http, { url, ...req }, req.pollInterval, req.timeout);
|
||||
async crawl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number; signal?: AbortSignal } = {}): Promise<CrawlJob> {
|
||||
const { pollInterval, timeout, signal, ...options } = req;
|
||||
return crawlWaiter(this.http, { url, ...options }, pollInterval, timeout, signal);
|
||||
}
|
||||
/**
|
||||
* Retrieve crawl errors and robots.txt blocks.
|
||||
@@ -724,7 +727,7 @@ export class FirecrawlClient {
|
||||
}
|
||||
|
||||
/** @deprecated V1 compatibility alias for agent recovery. Prefer crawl(). */
|
||||
async crawlUrl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number } = {}): Promise<CrawlJob> {
|
||||
async crawlUrl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number; signal?: AbortSignal } = {}): Promise<CrawlJob> {
|
||||
return this.crawl(url, req);
|
||||
}
|
||||
|
||||
|
||||
@@ -107,28 +107,94 @@ export async function getCrawlStatus(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancel a crawl job. Resolves `false` when the API answers 409 because the
|
||||
* crawl already finished, so there is nothing left to cancel.
|
||||
*/
|
||||
export async function cancelCrawl(http: HttpClient, jobId: string): Promise<boolean> {
|
||||
try {
|
||||
const res = await http.delete<{ status: string }>(`/v2/crawl/${jobId}`);
|
||||
if (res.status === 409) return false;
|
||||
if (res.status !== 200) throwForBadResponse(res, "cancel crawl");
|
||||
return res.data?.status === "cancelled";
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "cancel crawl");
|
||||
if (err?.isAxiosError) {
|
||||
if (err.response?.status === 409) return false;
|
||||
return normalizeAxiosError(err, "cancel crawl");
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function waitForCrawlCompletion(http: HttpClient, jobId: string, pollInterval = 2, timeout?: number): Promise<CrawlJob> {
|
||||
function abortReason(signal: AbortSignal): unknown {
|
||||
if (signal.reason !== undefined) return signal.reason;
|
||||
return new DOMException("This operation was aborted", "AbortError");
|
||||
}
|
||||
|
||||
/** Reject with the abort reason as soon as `signal` aborts. */
|
||||
function raceAbort<T>(promise: Promise<T>, signal?: AbortSignal): Promise<T> {
|
||||
if (!signal) return promise;
|
||||
if (signal.aborted) {
|
||||
promise.catch(() => {});
|
||||
return Promise.reject(abortReason(signal));
|
||||
}
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const onAbort = () => {
|
||||
promise.catch(() => {});
|
||||
reject(abortReason(signal));
|
||||
};
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
promise.then(
|
||||
(value) => {
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
resolve(value);
|
||||
},
|
||||
(err) => {
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
reject(err);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
if (signal?.aborted) return reject(abortReason(signal));
|
||||
const onAbort = () => {
|
||||
clearTimeout(timer);
|
||||
reject(abortReason(signal!));
|
||||
};
|
||||
const timer = setTimeout(() => {
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
resolve();
|
||||
}, ms);
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Poll a crawl until it reaches a terminal status. When `signal` aborts, polling
|
||||
* stops and the promise rejects with the abort reason. The job is not cancelled.
|
||||
*/
|
||||
export async function waitForCrawlCompletion(
|
||||
http: HttpClient,
|
||||
jobId: string,
|
||||
pollInterval = 2,
|
||||
timeout?: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CrawlJob> {
|
||||
const start = Date.now();
|
||||
|
||||
while (true) {
|
||||
if (signal?.aborted) throw abortReason(signal);
|
||||
try {
|
||||
const status = await getCrawlStatus(http, jobId);
|
||||
const status = await raceAbort(getCrawlStatus(http, jobId), signal);
|
||||
|
||||
if (["completed", "failed", "cancelled"].includes(status.status)) {
|
||||
return status;
|
||||
}
|
||||
} catch (err: any) {
|
||||
if (signal?.aborted) throw abortReason(signal);
|
||||
// Don't retry on permanent errors (4xx) - re-throw immediately with jobId context
|
||||
if (!isRetryableError(err)) {
|
||||
// Create new error with jobId for better debugging (non-retryable errors like 404)
|
||||
@@ -151,13 +217,31 @@ export async function waitForCrawlCompletion(http: HttpClient, jobId: string, po
|
||||
throw new JobTimeoutError(jobId, timeout, 'crawl');
|
||||
}
|
||||
|
||||
await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));
|
||||
await sleep(Math.max(1000, pollInterval * 1000), signal);
|
||||
}
|
||||
}
|
||||
|
||||
export async function crawl(http: HttpClient, request: CrawlRequest, pollInterval = 2, timeout?: number): Promise<CrawlJob> {
|
||||
/**
|
||||
* Start a crawl and wait for it to finish. When `signal` aborts, polling stops,
|
||||
* the job gets a best-effort cancel request, and the promise rejects with the
|
||||
* abort reason at once, without waiting for the cancel. A signal that is already
|
||||
* aborted rejects before any request.
|
||||
*/
|
||||
export async function crawl(
|
||||
http: HttpClient,
|
||||
request: CrawlRequest,
|
||||
pollInterval = 2,
|
||||
timeout?: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CrawlJob> {
|
||||
if (signal?.aborted) throw abortReason(signal);
|
||||
const started = await startCrawl(http, request);
|
||||
return waitForCrawlCompletion(http, started.id, pollInterval, timeout);
|
||||
try {
|
||||
return await waitForCrawlCompletion(http, started.id, pollInterval, timeout, signal);
|
||||
} catch (err) {
|
||||
if (signal?.aborted) cancelCrawl(http, started.id).catch(() => {});
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getCrawlErrors(http: HttpClient, crawlId: string): Promise<CrawlErrorsResponse> {
|
||||
|
||||
Reference in New Issue
Block a user