From 96438c3a54d36d030a525c50fe70783b11ba036c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 3 Jun 2025 19:02:07 -0700 Subject: [PATCH] feat(tools): add firecrawl search, add conditional sub-block rendering to tool input for agents (#456) * added firecrawl search * take into account conditional subblocks in tool input for agents * update turborepo * added docs --- apps/docs/content/docs/tools/firecrawl.mdx | 26 ++++++- .../components/tool-input/tool-input.tsx | 69 ++++++++++++++++++- apps/sim/blocks/blocks/firecrawl.ts | 62 +++++++++++++++-- apps/sim/tools/firecrawl/index.ts | 3 +- apps/sim/tools/firecrawl/search.ts | 57 +++++++++++++++ apps/sim/tools/firecrawl/types.ts | 28 ++++++++ apps/sim/tools/registry.ts | 3 +- bun.lock | 16 ++--- package.json | 2 +- 9 files changed, 244 insertions(+), 22 deletions(-) create mode 100644 apps/sim/tools/firecrawl/search.ts diff --git a/apps/docs/content/docs/tools/firecrawl.mdx b/apps/docs/content/docs/tools/firecrawl.mdx index 976530c343..1f19918a42 100644 --- a/apps/docs/content/docs/tools/firecrawl.mdx +++ b/apps/docs/content/docs/tools/firecrawl.mdx @@ -1,6 +1,6 @@ --- title: Firecrawl -description: Scrape website content +description: Scrape or search the web --- import { BlockInfoCard } from "@/components/ui/block-info-card" @@ -51,7 +51,7 @@ The Firecrawl integration allows your agents to access and process web content p ## Usage Instructions -Extract content from any website with advanced web scraping capabilities and content filtering. Retrieve clean, structured data from web pages with options to focus on main content. +Extract content from any website with advanced web scraping or search the web for information. Retrieve clean, structured data from web pages with options to focus on main content, or intelligently search for information across the web. @@ -77,6 +77,24 @@ Extract structured content from web pages with comprehensive metadata support. C | `html` | string | | `metadata` | string | +### `firecrawl_search` + +Search for information on the web using Firecrawl + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Firecrawl API key | +| `query` | string | Yes | The search query to use | + +#### Output + +| Parameter | Type | +| --------- | ---- | +| `data` | string | +| `warning` | string | + ## Block Configuration @@ -85,7 +103,7 @@ Extract structured content from web pages with comprehensive metadata support. C | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `apiKey` | string | Yes | API Key - Enter your Firecrawl API key | +| `apiKey` | string | Yes | | @@ -97,6 +115,8 @@ Extract structured content from web pages with comprehensive metadata support. C | ↳ `markdown` | string | markdown of the response | | ↳ `html` | any | html of the response | | ↳ `metadata` | json | metadata of the response | +| ↳ `data` | json | data of the response | +| ↳ `warning` | any | warning of the response | ## Notes diff --git a/apps/sim/app/w/[id]/components/workflow-block/components/sub-block/components/tool-input/tool-input.tsx b/apps/sim/app/w/[id]/components/workflow-block/components/sub-block/components/tool-input/tool-input.tsx index 2d90b3cc3e..f2e5136b05 100644 --- a/apps/sim/app/w/[id]/components/workflow-block/components/sub-block/components/tool-input/tool-input.tsx +++ b/apps/sim/app/w/[id]/components/workflow-block/components/sub-block/components/tool-input/tool-input.tsx @@ -77,6 +77,69 @@ const getToolDisplayParams = (toolId: string): ToolParam[] => { })) } +// Get filtered parameters based on block conditions +const getFilteredToolParams = ( + blockType: string, + toolId: string, + currentOperation?: string, + toolParams?: Record +): ToolParam[] => { + const tool = getTool(toolId) + const block = getAllBlocks().find((block) => block.type === blockType) + + if (!tool || !block) return [] + + const allParams = Object.entries(tool.params) + .filter(([_, param]) => param.requiredForToolCall || param.optionalToolInput) + .map(([paramId, param]) => ({ + id: paramId, + type: param.type, + description: param.description, + requiredForToolCall: param.requiredForToolCall ?? false, + optionalToolInput: param.optionalToolInput ?? false, + })) + + const evaluateCondition = (condition: any, currentValues: Record): boolean => { + const fieldValue = currentValues[condition.field] + let result = false + + if (Array.isArray(condition.value)) { + result = condition.value.includes(fieldValue) + } else { + result = fieldValue === condition.value + } + + if (condition.not) { + result = !result + } + + return result + } + + return allParams.filter((param) => { + const subBlock = block.subBlocks.find((sb) => sb.id === param.id) + + if (!subBlock || !subBlock.condition) { + return true + } + + const currentValues: Record = { + operation: currentOperation, + ...toolParams, + } + + const condition = subBlock.condition + let mainConditionResult = evaluateCondition(condition, currentValues) + + if (condition.and) { + const andConditionResult = evaluateCondition(condition.and, currentValues) + mainConditionResult = mainConditionResult && andConditionResult + } + + return mainConditionResult + }) +} + // Keep this for backward compatibility - only get strictly required parameters const getRequiredToolParams = (toolId: string): ToolParam[] => { const tool = getTool(toolId) @@ -335,7 +398,9 @@ export function ToolInput({ const defaultOperation = operationOptions.length > 0 ? operationOptions[0].id : undefined const toolId = getToolIdFromBlock(toolBlock.type) || toolBlock.type - const displayParams = toolId ? getToolDisplayParams(toolId) : [] + const displayParams = toolId + ? getFilteredToolParams(toolBlock.type, toolId, defaultOperation, {}) + : [] // Use the helper function to initialize parameters with blockId as instanceId const initialParams = initializeToolParams( @@ -729,7 +794,7 @@ export function ToolInput({ const requiredParams = isCustomTool ? getCustomToolParams(tool.schema) : toolId - ? getRequiredToolParams(toolId) + ? getFilteredToolParams(tool.type, toolId, tool.operation, tool.params) : [] // Check if the tool has any expandable content diff --git a/apps/sim/blocks/blocks/firecrawl.ts b/apps/sim/blocks/blocks/firecrawl.ts index 5b839e1d7f..20c0ab17e5 100644 --- a/apps/sim/blocks/blocks/firecrawl.ts +++ b/apps/sim/blocks/blocks/firecrawl.ts @@ -1,30 +1,62 @@ import { FirecrawlIcon } from '@/components/icons' -import type { ScrapeResponse } from '@/tools/firecrawl/types' +import type { ScrapeResponse, SearchResponse } from '@/tools/firecrawl/types' import type { BlockConfig } from '../types' -export const FirecrawlBlock: BlockConfig = { +type FirecrawlResponse = ScrapeResponse | SearchResponse + +export const FirecrawlBlock: BlockConfig = { type: 'firecrawl', name: 'Firecrawl', - description: 'Scrape website content', + description: 'Scrape or search the web', longDescription: - 'Extract content from any website with advanced web scraping capabilities and content filtering. Retrieve clean, structured data from web pages with options to focus on main content.', + 'Extract content from any website with advanced web scraping or search the web for information. Retrieve clean, structured data from web pages with options to focus on main content, or intelligently search for information across the web.', docsLink: 'https://docs.simstudio.ai/tools/firecrawl', category: 'tools', bgColor: '#181C1E', icon: FirecrawlIcon, subBlocks: [ + { + id: 'operation', + title: 'Operation', + type: 'dropdown', + layout: 'full', + options: [ + { label: 'Scrape', id: 'scrape' }, + { label: 'Search', id: 'search' }, + ], + value: () => 'scrape', + }, { id: 'url', title: 'Website URL', type: 'short-input', layout: 'full', placeholder: 'Enter the webpage URL to scrape', + condition: { + field: 'operation', + value: 'scrape', + }, }, { id: 'onlyMainContent', title: 'Only Main Content', type: 'switch', layout: 'half', + condition: { + field: 'operation', + value: 'scrape', + }, + }, + { + id: 'query', + title: 'Search Query', + type: 'short-input', + layout: 'full', + placeholder: 'Enter the search query', + condition: { + field: 'operation', + value: 'search', + }, }, { id: 'apiKey', @@ -36,19 +68,37 @@ export const FirecrawlBlock: BlockConfig = { }, ], tools: { - access: ['firecrawl_scrape'], + access: ['firecrawl_scrape', 'firecrawl_search'], + config: { + tool: (params) => { + switch (params.operation) { + case 'scrape': + return 'firecrawl_scrape' + case 'search': + return 'firecrawl_search' + default: + return 'firecrawl_scrape' + } + }, + }, }, inputs: { apiKey: { type: 'string', required: true }, - url: { type: 'string', required: true }, + operation: { type: 'string', required: true }, + url: { type: 'string', required: false }, + query: { type: 'string', required: false }, scrapeOptions: { type: 'json', required: false }, }, outputs: { response: { type: { + // Scrape output markdown: 'string', html: 'any', metadata: 'json', + // Search output + data: 'json', + warning: 'any', }, }, }, diff --git a/apps/sim/tools/firecrawl/index.ts b/apps/sim/tools/firecrawl/index.ts index 9aeb0795e1..a8a60b7ff3 100644 --- a/apps/sim/tools/firecrawl/index.ts +++ b/apps/sim/tools/firecrawl/index.ts @@ -1,3 +1,4 @@ import { scrapeTool } from './scrape' +import { searchTool } from './search' -export { scrapeTool } +export { scrapeTool, searchTool } diff --git a/apps/sim/tools/firecrawl/search.ts b/apps/sim/tools/firecrawl/search.ts new file mode 100644 index 0000000000..55646cf4a8 --- /dev/null +++ b/apps/sim/tools/firecrawl/search.ts @@ -0,0 +1,57 @@ +import type { ToolConfig } from '../types' +import type { SearchParams, SearchResponse } from './types' + +export const searchTool: ToolConfig = { + id: 'firecrawl_search', + name: 'Firecrawl Search', + description: 'Search for information on the web using Firecrawl', + version: '1.0.0', + + params: { + apiKey: { + type: 'string', + required: true, + requiredForToolCall: true, + description: 'Firecrawl API key', + }, + query: { + type: 'string', + required: true, + description: 'The search query to use', + }, + }, + + request: { + method: 'POST', + url: 'https://api.firecrawl.dev/v1/search', + headers: (params) => ({ + 'Content-Type': 'application/json', + Authorization: `Bearer ${params.apiKey}`, + }), + body: (params) => ({ + query: params.query, + }), + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!data.success) { + throw new Error(data.error?.message || 'Unknown error occurred') + } + + return { + success: true, + output: { + data: data.data, + warning: data.warning, + }, + } + }, + + transformError: (error) => { + const message = error.error?.message || error.message + const code = error.error?.type || error.code + return `${message} (${code})` + }, +} diff --git a/apps/sim/tools/firecrawl/types.ts b/apps/sim/tools/firecrawl/types.ts index 0e67f181a2..3cb5c579ec 100644 --- a/apps/sim/tools/firecrawl/types.ts +++ b/apps/sim/tools/firecrawl/types.ts @@ -9,6 +9,11 @@ export interface ScrapeParams { } } +export interface SearchParams { + apiKey: string + query: string +} + export interface ScrapeResponse extends ToolResponse { output: { markdown: string @@ -30,3 +35,26 @@ export interface ScrapeResponse extends ToolResponse { } } } + +export interface SearchResponse extends ToolResponse { + output: { + data: Array<{ + title: string + description: string + url: string + markdown?: string + html?: string + rawHtml?: string + links?: string[] + screenshot?: string + metadata: { + title: string + description: string + sourceURL: string + statusCode: number + error?: string + } + }> + warning?: string + } +} diff --git a/apps/sim/tools/registry.ts b/apps/sim/tools/registry.ts index b917d07c19..a81164e1bd 100644 --- a/apps/sim/tools/registry.ts +++ b/apps/sim/tools/registry.ts @@ -17,7 +17,7 @@ import { import { elevenLabsTtsTool } from './elevenlabs' import { exaAnswerTool, exaFindSimilarLinksTool, exaGetContentsTool, exaSearchTool } from './exa' import { fileParseTool } from './file' -import { scrapeTool } from './firecrawl' +import { scrapeTool, searchTool } from './firecrawl' import { functionExecuteTool } from './function' import { githubCommentTool, @@ -109,6 +109,7 @@ export const tools: Record = { vision_tool: visionTool, file_parser: fileParseTool, firecrawl_scrape: scrapeTool, + firecrawl_search: searchTool, google_search: googleSearchTool, jina_read_url: readUrlTool, linkup_search: linkupSearchTool, diff --git a/bun.lock b/bun.lock index f8629b599f..dd46f9ff60 100644 --- a/bun.lock +++ b/bun.lock @@ -18,7 +18,7 @@ "lint-staged": "16.0.0", "prettier": "^3.5.3", "prettier-plugin-tailwindcss": "^0.6.11", - "turbo": "2.5.3", + "turbo": "2.5.4", }, }, "apps/docs": { @@ -2920,19 +2920,19 @@ "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], - "turbo": ["turbo@2.5.3", "", { "optionalDependencies": { "turbo-darwin-64": "2.5.3", "turbo-darwin-arm64": "2.5.3", "turbo-linux-64": "2.5.3", "turbo-linux-arm64": "2.5.3", "turbo-windows-64": "2.5.3", "turbo-windows-arm64": "2.5.3" }, "bin": { "turbo": "bin/turbo" } }, "sha512-iHuaNcq5GZZnr3XDZNuu2LSyCzAOPwDuo5Qt+q64DfsTP1i3T2bKfxJhni2ZQxsvAoxRbuUK5QetJki4qc5aYA=="], + "turbo": ["turbo@2.5.4", "", { "optionalDependencies": { "turbo-darwin-64": "2.5.4", "turbo-darwin-arm64": "2.5.4", "turbo-linux-64": "2.5.4", "turbo-linux-arm64": "2.5.4", "turbo-windows-64": "2.5.4", "turbo-windows-arm64": "2.5.4" }, "bin": { "turbo": "bin/turbo" } }, "sha512-kc8ZibdRcuWUG1pbYSBFWqmIjynlD8Lp7IB6U3vIzvOv9VG+6Sp8bzyeBWE3Oi8XV5KsQrznyRTBPvrf99E4mA=="], - "turbo-darwin-64": ["turbo-darwin-64@2.5.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-YSItEVBUIvAGPUDpAB9etEmSqZI3T6BHrkBkeSErvICXn3dfqXUfeLx35LfptLDEbrzFUdwYFNmt8QXOwe9yaw=="], + "turbo-darwin-64": ["turbo-darwin-64@2.5.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-ah6YnH2dErojhFooxEzmvsoZQTMImaruZhFPfMKPBq8sb+hALRdvBNLqfc8NWlZq576FkfRZ/MSi4SHvVFT9PQ=="], - "turbo-darwin-arm64": ["turbo-darwin-arm64@2.5.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-5PefrwHd42UiZX7YA9m1LPW6x9YJBDErXmsegCkVp+GjmWrADfEOxpFrGQNonH3ZMj77WZB2PVE5Aw3gA+IOhg=="], + "turbo-darwin-arm64": ["turbo-darwin-arm64@2.5.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-2+Nx6LAyuXw2MdXb7pxqle3MYignLvS7OwtsP9SgtSBaMlnNlxl9BovzqdYAgkUW3AsYiQMJ/wBRb7d+xemM5A=="], - "turbo-linux-64": ["turbo-linux-64@2.5.3", "", { "os": "linux", "cpu": "x64" }, "sha512-M9xigFgawn5ofTmRzvjjLj3Lqc05O8VHKuOlWNUlnHPUltFquyEeSkpQNkE/vpPdOR14AzxqHbhhxtfS4qvb1w=="], + "turbo-linux-64": ["turbo-linux-64@2.5.4", "", { "os": "linux", "cpu": "x64" }, "sha512-5May2kjWbc8w4XxswGAl74GZ5eM4Gr6IiroqdLhXeXyfvWEdm2mFYCSWOzz0/z5cAgqyGidF1jt1qzUR8hTmOA=="], - "turbo-linux-arm64": ["turbo-linux-arm64@2.5.3", "", { "os": "linux", "cpu": "arm64" }, "sha512-auJRbYZ8SGJVqvzTikpg1bsRAsiI9Tk0/SDkA5Xgg0GdiHDH/BOzv1ZjDE2mjmlrO/obr19Dw+39OlMhwLffrw=="], + "turbo-linux-arm64": ["turbo-linux-arm64@2.5.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-/2yqFaS3TbfxV3P5yG2JUI79P7OUQKOUvAnx4MV9Bdz6jqHsHwc9WZPpO4QseQm+NvmgY6ICORnoVPODxGUiJg=="], - "turbo-windows-64": ["turbo-windows-64@2.5.3", "", { "os": "win32", "cpu": "x64" }, "sha512-arLQYohuHtIEKkmQSCU9vtrKUg+/1TTstWB9VYRSsz+khvg81eX6LYHtXJfH/dK7Ho6ck+JaEh5G+QrE1jEmCQ=="], + "turbo-windows-64": ["turbo-windows-64@2.5.4", "", { "os": "win32", "cpu": "x64" }, "sha512-EQUO4SmaCDhO6zYohxIjJpOKRN3wlfU7jMAj3CgcyTPvQR/UFLEKAYHqJOnJtymbQmiiM/ihX6c6W6Uq0yC7mA=="], - "turbo-windows-arm64": ["turbo-windows-arm64@2.5.3", "", { "os": "win32", "cpu": "arm64" }, "sha512-3JPn66HAynJ0gtr6H+hjY4VHpu1RPKcEwGATvGUTmLmYSYBQieVlnGDRMMoYN066YfyPqnNGCfhYbXfH92Cm0g=="], + "turbo-windows-arm64": ["turbo-windows-arm64@2.5.4", "", { "os": "win32", "cpu": "arm64" }, "sha512-oQ8RrK1VS8lrxkLriotFq+PiF7iiGgkZtfLKF4DDKsmdbPo0O9R2mQxm7jHLuXraRCuIQDWMIw6dpcr7Iykf4A=="], "type-fest": ["type-fest@0.7.1", "", {}, "sha512-Ne2YiiGN8bmrmJJEuTWTLJR32nh/JdL1+PSicowtNb0WFpn59GK8/lfD61bVtzguz7b3PBt74nxpv/Pw5po5Rg=="], diff --git a/package.json b/package.json index 40240f5164..ec682b3cbf 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "lint-staged": "16.0.0", "prettier": "^3.5.3", "prettier-plugin-tailwindcss": "^0.6.11", - "turbo": "2.5.3" + "turbo": "2.5.4" }, "lint-staged": { "*.{js,jsx,ts,tsx,json,css,scss,md}": [