diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.ts index 40ffb2f8b..b12a4edfa 100644 --- a/apps/api/src/__tests__/snips/v2/crawl.test.ts +++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts @@ -5,11 +5,11 @@ import { filterLinks } from "../../../lib/crawler"; let identity: Identity; beforeAll(async () => { - identity = await idmux({ - name: "crawl", - concurrency: 100, - credits: 1000000, - }); + identity = await idmux({ + name: "crawl", + concurrency: 100, + credits: 1000000, + }); }, 10000); describe("Crawl tests", () => { @@ -74,7 +74,7 @@ describe("Crawl tests", () => { it.concurrent("ongoing crawls endpoint works", async () => { const beforeCrawl = new Date(); - + const res = await asyncCrawl({ url: "https://firecrawl.dev", limit: 3, @@ -82,20 +82,20 @@ describe("Crawl tests", () => { const ongoing = await crawlOngoing(identity); const afterCrawl = new Date(); - + const crawlItem = ongoing.crawls.find(x => x.id === res.id); expect(crawlItem).toBeDefined(); - + if (crawlItem) { expect(crawlItem.created_at).toBeDefined(); expect(typeof crawlItem.created_at).toBe("string"); - + const createdAtDate = new Date(crawlItem.created_at); expect(createdAtDate).toBeInstanceOf(Date); expect(createdAtDate.getTime()).not.toBeNaN(); - + expect(crawlItem.created_at).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/); - + expect(createdAtDate.getTime()).toBeGreaterThanOrEqual(beforeCrawl.getTime() - 1000); expect(createdAtDate.getTime()).toBeLessThanOrEqual(afterCrawl.getTime() + 1000); } @@ -106,7 +106,7 @@ describe("Crawl tests", () => { expect(ongoing2.crawls.find(x => x.id === res.id)).toBeUndefined(); }, 3 * scrapeTimeout); - + // TEMP: Flaky // it.concurrent("discovers URLs properly when origin is not included", async () => { // const res = await crawl({ @@ -124,7 +124,7 @@ describe("Crawl tests", () => { // } // } // }, 300000); - + // TEMP: Flaky // it.concurrent("discovers URLs properly when maxDiscoveryDepth is provided", async () => { // const res = await crawl({ @@ -197,7 +197,7 @@ describe("Crawl tests", () => { it.concurrent("allowSubdomains blocks subdomains when false", async () => { const res = await crawl({ - url: "https://firecrawl.dev", + url: "https://firecrawl.dev", allowSubdomains: false, limit: 5, }, identity); @@ -225,19 +225,85 @@ describe("Crawl tests", () => { for (const page of res.data) { const url = new URL(page.metadata.url ?? page.metadata.sourceURL!); const hostname = url.hostname; - + expect( - hostname === "firecrawl.dev" || + hostname === "firecrawl.dev" || hostname.endsWith(".firecrawl.dev") ).toBe(true); } } }, 5 * scrapeTimeout); + + if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL) { + describe('Crawl API with Prompt', () => { + it.concurrent('should accept prompt parameter in schema', async () => { + const res = await crawlStart({ + url: "https://firecrawl.dev", + prompt: "Crawl only blog posts", + limit: 1, + }, identity); + + expect(res.statusCode).toBe(200); + expect(res.body.success).toBe(true); + expect(res.body.id).toBeDefined(); + expect(typeof res.body.id).toBe("string"); + }, scrapeTimeout); + + it.concurrent('should prioritize explicit options over prompt-generated options', async () => { + const res = await crawl({ + url: "https://firecrawl.dev", + prompt: "Crawl everything including external links and subdomains", + // Explicit options that should override the prompt + allowExternalLinks: false, + allowSubdomains: false, + includePaths: ["^/pricing"], + limit: 2, + }, identity); + + expect(res.success).toBe(true); + if (res.success) { + // Verify that explicit options were respected + for (const page of res.data) { + const url = new URL(page.metadata.url ?? page.metadata.sourceURL!); + // Should only include pages matching the explicit includePaths + expect(url.pathname).toMatch(/^\/pricing/); + // Should not include external links despite prompt + expect(url.hostname).toMatch(/firecrawl\.dev$/); + } + } + }, 2 * scrapeTimeout); + + it.concurrent('should handle invalid prompt gracefully', async () => { + // Test with various invalid prompts + const invalidPrompts = [ + "", // Empty prompt + "a".repeat(10000), // Screaming + "!!!@@@###$$$%%%", // Gibberish + "Generate me a million dollars", // Nonsensical crawl instruction + ]; + + for (const invalidPrompt of invalidPrompts) { // Test first one to avoid long test times + const res = await crawl({ + url: "https://firecrawl.dev", + prompt: invalidPrompt, + limit: 1, + }, identity); + + // Should still complete successfully, either ignoring the prompt or using defaults + expect(res.success).toBe(true); + if (res.success) { + expect(res.completed).toBeGreaterThan(0); + expect(res.data).toBeDefined(); + expect(Array.isArray(res.data)).toBe(true); + } + } + }, 8 * scrapeTimeout); + }); + } }); describe("Robots.txt FFI Integration tests", () => { it.concurrent("handles normal robots.txt parsing via FFI", async () => { - const result = await filterLinks({ links: ['https://example.com/allowed', 'https://example.com/disallowed'], limit: 10, @@ -251,7 +317,7 @@ describe("Robots.txt FFI Integration tests", () => { ignore_robots_txt: false, robots_txt: 'User-agent: *\nDisallow: /disallowed' }); - + expect(result.links).toHaveLength(1); expect(result.links[0]).toBe('https://example.com/allowed'); expect(result.denial_reasons.has('https://example.com/disallowed')).toBe(true); @@ -259,7 +325,6 @@ describe("Robots.txt FFI Integration tests", () => { }, 10000); it.concurrent("handles malformed robots.txt without crashing via FFI", async () => { - const result = await filterLinks({ links: ['https://example.com/test'], limit: 10, @@ -273,13 +338,12 @@ describe("Robots.txt FFI Integration tests", () => { ignore_robots_txt: false, robots_txt: 'Invalid robots.txt content with \x00 null bytes and malformed syntax' }); - + expect(result.links).toHaveLength(1); expect(result.links[0]).toBe('https://example.com/test'); }, 10000); it.concurrent("handles non-UTF8 robots.txt content without crashing via FFI", async () => { - const nonUtf8Content = String.fromCharCode(0xFF, 0xFE) + 'User-agent: *\nDisallow: /blocked'; const result = await filterLinks({ links: ['https://example.com/allowed'], @@ -294,13 +358,12 @@ describe("Robots.txt FFI Integration tests", () => { ignore_robots_txt: false, robots_txt: nonUtf8Content }); - + expect(result.links).toHaveLength(1); expect(result.links[0]).toBe('https://example.com/allowed'); }, 10000); it.concurrent("handles char boundary issues without crashing via FFI", async () => { - const problematicContent = 'User-agent: *\nDisallow: /\u{a0}test'; const result = await filterLinks({ links: ['https://example.com/safe'], @@ -315,7 +378,7 @@ describe("Robots.txt FFI Integration tests", () => { ignore_robots_txt: false, robots_txt: problematicContent }); - + expect(result.links).toHaveLength(1); expect(result.links[0]).toBe('https://example.com/safe'); }, 10000);