From 3b57f4ff7ec1d6a8bec0eff4e1f95a9bb9e8bfe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Fri, 1 May 2026 17:12:15 +0200 Subject: [PATCH] feat(search): includeDomains/excldueDomains (#222) * feat(search): includeDomains/excldueDomains * Nick: * Update publish.yml --------- Co-authored-by: Nicolas <20311743+nickscamara@users.noreply.github.com> --- .github/workflows/publish.yml | 31 ++++++++------ package.json | 2 +- pnpm-lock.yaml | 29 ++++++------- src/index.ts | 79 ++++++++++++++++++++++++++++------- src/legacy/index.md | 18 ++++++++ 5 files changed, 117 insertions(+), 42 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f066733..bf65d10 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,31 +7,36 @@ on: jobs: publish: + name: Publish runs-on: ubuntu-latest permissions: id-token: write # Required for OIDC authentication with MCP registry contents: read steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v5 - - name: Use Node.js - uses: actions/setup-node@v3 + - name: Install pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4 with: - node-version: '20.x' - cache: 'pnpm' - registry-url: 'https://registry.npmjs.org' + version: 10 - - name: Install dependencies - run: pnpm install --frozen-lockfile + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "pnpm" - - name: Build - run: pnpm run build + - name: Authenticate + run: echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/.npmrc + env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - name: Publish to NPM - run: pnpm publish --access public - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + pnpm install + pnpm run build + pnpm publish --access public --no-git-checks - name: Install MCP Publisher run: | diff --git a/package.json b/package.json index f52e231..3fb3558 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ }, "license": "MIT", "dependencies": { - "@mendable/firecrawl-js": "4.17.0", + "@mendable/firecrawl-js": "4.21.0", "dotenv": "^17.2.2", "firecrawl-fastmcp": "^1.0.4", "typescript": "^5.9.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 81096e8..be6f031 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: dependencies: '@mendable/firecrawl-js': - specifier: 4.17.0 - version: 4.17.0 + specifier: 4.21.0 + version: 4.21.0 dotenv: specifier: ^17.2.2 version: 17.2.2 @@ -33,8 +33,8 @@ packages: '@borewit/text-codec@0.1.1': resolution: {integrity: sha512-5L/uBxmjaCIX5h8Z+uu+kA9BQLkc/Wl06UGR5ajNRxu+/XjonB5i8JpgFMrPj3LXTCPA0pv8yxUvbUi+QthGGA==} - '@mendable/firecrawl-js@4.17.0': - resolution: {integrity: sha512-4Dz2y8QLJMlf45qQIyCgvfjbz+cn9T5jRf0aTxFptBe+123373Vsker9vKYHriWIl2oO/SwRSILkJV6AsGlCMA==} + '@mendable/firecrawl-js@4.21.0': + resolution: {integrity: sha512-sUNGerLQmwMyC/IrPmgVsvjKLufXvelkHcTfzjJ5LB9yCWY3PWcplPWZ+zSxNHp19S44L00ZrNIlnRFqgOyuqQ==} engines: {node: '>=22.0.0'} '@modelcontextprotocol/sdk@1.18.0': @@ -79,8 +79,8 @@ packages: asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - axios@1.13.5: - resolution: {integrity: sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==} + axios@1.15.0: + resolution: {integrity: sha512-wWyJDlAatxk30ZJer+GeCWS209sA42X+N5jU2jy6oHTp7ufw8uzUTVFBX9+wTfAlhiJXGS0Bq7X6efruWjuK9Q==} body-parser@2.2.0: resolution: {integrity: sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg==} @@ -443,8 +443,9 @@ packages: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} - proxy-from-env@1.1.0: - resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + proxy-from-env@2.1.0: + resolution: {integrity: sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==} + engines: {node: '>=10'} punycode@2.3.1: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} @@ -649,9 +650,9 @@ snapshots: '@borewit/text-codec@0.1.1': {} - '@mendable/firecrawl-js@4.17.0': + '@mendable/firecrawl-js@4.21.0': dependencies: - axios: 1.13.5 + axios: 1.15.0 firecrawl: 4.16.0 typescript-event-target: 1.1.1 zod: 3.25.76 @@ -714,11 +715,11 @@ snapshots: asynckit@0.4.0: {} - axios@1.13.5: + axios@1.15.0: dependencies: follow-redirects: 1.15.11 form-data: 4.0.5 - proxy-from-env: 1.1.0 + proxy-from-env: 2.1.0 transitivePeerDependencies: - debug @@ -932,7 +933,7 @@ snapshots: firecrawl@4.16.0: dependencies: - axios: 1.13.5 + axios: 1.15.0 typescript-event-target: 1.1.1 zod: 3.25.76 zod-to-json-schema: 3.24.6(zod@3.25.76) @@ -1094,7 +1095,7 @@ snapshots: forwarded: 0.2.0 ipaddr.js: 1.9.1 - proxy-from-env@1.1.0: {} + proxy-from-env@2.1.0: {} punycode@2.3.1: {} diff --git a/src/index.ts b/src/index.ts index f489afe..ef3a88a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -53,6 +53,35 @@ function removeEmptyTopLevel>( return out; } +const searchDomainSchema = z + .string() + .trim() + .toLowerCase() + .regex( + /^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/, + 'Domain must be a valid hostname without protocol or path' + ); + +function buildSearchQueryWithDomains( + query: string, + includeDomains?: string[], + excludeDomains?: string[] +): string { + if (includeDomains?.length) { + return `${query} (${includeDomains + .map((domain) => `site:${domain}`) + .join(' OR ')})`; + } + + if (excludeDomains?.length) { + return `${query} ${excludeDomains + .map((domain) => `-site:${domain}`) + .join(' ')}`; + } + + return query; +} + class ConsoleLogger implements Logger { private shouldLog = process.env.CLOUD_SERVICE === 'true' || @@ -576,6 +605,7 @@ The query also supports search operators, that you can use if needed to refine t **Common mistakes:** Using crawl or map for open-ended questions (use search instead). **Prompt Example:** "Find the latest research papers on AI published in 2023." **Sources:** web, images, news, default to web unless needed images or news. +**Domain filters:** Use includeDomains to restrict results to specific domains, or excludeDomains to remove domains. Do not use both in the same request. Domains must be hostnames only, without protocol or path. **Scrape Options:** Only use scrapeOptions when you think it is absolutely necessary. When you do so default to a lower limit to avoid timeouts, 5 or lower. **Optimal Workflow:** Search first using firecrawl_search without formats, then after fetching the results, use the scrape tool to get the content of the relevantpage(s) that you want to scrape @@ -586,6 +616,7 @@ The query also supports search operators, that you can use if needed to refine t "arguments": { "query": "top AI companies", "limit": 5, + "includeDomains": ["example.com"], "sources": [ { "type": "web" } ] @@ -615,18 +646,28 @@ The query also supports search operators, that you can use if needed to refine t \`\`\` **Returns:** Array of search results (with optional scraped content). `, - parameters: z.object({ - query: z.string().min(1), - limit: z.number().optional(), - tbs: z.string().optional(), - filter: z.string().optional(), - location: z.string().optional(), - sources: z - .array(z.object({ type: z.enum(['web', 'images', 'news']) })) - .optional(), - scrapeOptions: scrapeParamsSchema.omit({ url: true }).partial().optional(), - enterprise: z.array(z.enum(['default', 'anon', 'zdr'])).optional(), - }), + parameters: z + .object({ + query: z.string().min(1), + limit: z.number().optional(), + tbs: z.string().optional(), + filter: z.string().optional(), + location: z.string().optional(), + includeDomains: z.array(searchDomainSchema).optional(), + excludeDomains: z.array(searchDomainSchema).optional(), + sources: z + .array(z.object({ type: z.enum(['web', 'images', 'news']) })) + .optional(), + scrapeOptions: scrapeParamsSchema + .omit({ url: true }) + .partial() + .optional(), + enterprise: z.array(z.enum(['default', 'anon', 'zdr'])).optional(), + }) + .refine( + (args) => !(args.includeDomains?.length && args.excludeDomains?.length), + 'includeDomains and excludeDomains cannot both be specified' + ), execute: async ( args: unknown, { session, log }: { session?: SessionData; log: Logger } @@ -635,6 +676,11 @@ The query also supports search operators, that you can use if needed to refine t const { query, ...opts } = args as Record; const searchOpts = { ...opts } as Record; + const includeDomains = searchOpts.includeDomains as string[] | undefined; + const excludeDomains = searchOpts.excludeDomains as string[] | undefined; + delete searchOpts.includeDomains; + delete searchOpts.excludeDomains; + if (searchOpts.scrapeOptions) { searchOpts.scrapeOptions = transformScrapeParams( searchOpts.scrapeOptions as Record @@ -642,8 +688,13 @@ The query also supports search operators, that you can use if needed to refine t } const cleaned = removeEmptyTopLevel(searchOpts); - log.info('Searching', { query: String(query) }); - const res = await client.search(query as string, { + const searchQuery = buildSearchQueryWithDomains( + query as string, + includeDomains, + excludeDomains + ); + log.info('Searching', { query: searchQuery }); + const res = await client.search(searchQuery, { ...(cleaned as any), origin: ORIGIN, }); diff --git a/src/legacy/index.md b/src/legacy/index.md index 64d54d4..2d8dc3b 100644 --- a/src/legacy/index.md +++ b/src/legacy/index.md @@ -499,6 +499,7 @@ Search the web and optionally extract content from search results. This is the m **Common mistakes:** Using crawl or map for open-ended questions (use search instead). **Prompt Example:** "Find the latest research papers on AI published in 2023." **Sources:** web, images, news, default to web unless needed images or news. +**Domain filters:** Use includeDomains to restrict results to specific domains, or excludeDomains to remove domains. Do not use both in the same request. Domains must be hostnames only, without protocol or path. **Usage Example:** \`\`\`json { @@ -508,6 +509,7 @@ Search the web and optionally extract content from search results. This is the m "limit": 5, "lang": "en", "country": "us", + "excludeDomains": ["example.com"], "sources": [ "web", "images", @@ -545,6 +547,22 @@ Search the web and optionally extract content from search results. This is the m type: 'string', description: 'Location parameter for search results', }, + includeDomains: { + type: 'array', + description: + 'Domains to include in search results. Cannot be used with excludeDomains. Domains must be hostnames only, without protocol or path.', + items: { + type: 'string', + }, + }, + excludeDomains: { + type: 'array', + description: + 'Domains to exclude from search results. Cannot be used with includeDomains. Domains must be hostnames only, without protocol or path.', + items: { + type: 'string', + }, + }, sources: { type: 'array', description: