diff --git a/apps/api/.env.example b/apps/api/.env.example index f560476e5..fcb564c50 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -93,5 +93,5 @@ CDP_API_KEY_ID="" CDP_API_KEY_SECRET="" # ^ KEYS taken from -> https://portal.cdp.coinbase.com/projects/api-keys -## ENDPOINT PRICING TEST -X402_ENDPOINT_PRICE_USD=$0.001 \ No newline at end of file +## ENDPOINT PRICING TEST $0.01 +X402_ENDPOINT_PRICE_USD=0.01 \ No newline at end of file diff --git a/apps/api/requests.http b/apps/api/requests.http index fd53ee545..6fd622268 100644 --- a/apps/api/requests.http +++ b/apps/api/requests.http @@ -132,32 +132,6 @@ content-type: application/json GET {{baseUrl}}/v1/crawl/active HTTP/1.1 Authorization: Bearer {{$dotenv TEST_API_KEY}} - - -### Generate Payment Header for Protected Data -# Generate a payment header for accessing protected data -# Duration options: 1h, 1d, 1w -# @name generatePaymentHeaderProtected -POST {{baseUrl}}/v1/generate-payment-header HTTP/1.1 -Authorization: Bearer {{$dotenv TEST_API_KEY}} -content-type: application/json - -{ - "privateKey": "0X-PRIVATE-KEY-PRIVATE-KEY", - "duration": "1h", - "resource": "protected-data" -} - -### Test Protected Data with Generated Payment Header -# Use the payment header from the previous request -@paymentHeaderProtected = {{generatePaymentHeaderProtected.response.body.data.paymentHeader}} -# @name testProtectedData -GET {{baseUrl}}/v1/protected-data HTTP/1.1 -Accept: application/json -Authorization: Bearer {{$dotenv TEST_API_KEY}} -X-Payment: {{paymentHeaderProtected}} - - ### Generate Payment Header for Search with Wallet # Generate a payment header for search with wallet # Duration options: 1h, 1d, 1w @@ -169,18 +143,18 @@ content-type: application/json { "privateKey": "0X-PRIVATE-KEY-PRIVATE-KEY", "duration": "1h", - "resource": "search-with-wallet" + "resource": "x402/search" } ### Search with Wallet @paymentHeaderSearch = {{generatePaymentHeaderSearch.response.body.data.paymentHeader}} # @name searchWithWallet -POST {{baseUrl}}/v1/search-with-wallet HTTP/1.1 +POST {{baseUrl}}/v1/x402/search HTTP/1.1 Authorization: Bearer {{$dotenv TEST_API_KEY}} content-type: application/json X-Payment: {{paymentHeaderSearch}} { - "query": "firecrawl", + "query": "Premier League fixtures for the 2025/26 season", "limit": 2 } \ No newline at end of file diff --git a/apps/api/src/controllers/v1/generate-payment-header.ts b/apps/api/src/controllers/v1/generate-payment-header.ts index 325bf20bd..14e736d29 100644 --- a/apps/api/src/controllers/v1/generate-payment-header.ts +++ b/apps/api/src/controllers/v1/generate-payment-header.ts @@ -139,20 +139,13 @@ export async function generatePaymentHeaderController(req: RequestWithAuth<{}, G const baseUrl = `${protocol}://${host}`; switch (resource) { - case 'search-with-wallet': - return { - ...baseRequirements, - maxAmountRequired: "10000", // $0.001 in USDC base units - matches X402_SEARCH_WALLET_PRICE_USD - resource: `${baseUrl}/v1/search-with-wallet`, - description: "Search with wallet payment API", - }; - case 'protected-data': + case 'x402/search': default: return { ...baseRequirements, - maxAmountRequired: "10000", // $0.01 in USDC base units - matches X402_ENDPOINT_PRICE_USD - resource: `${baseUrl}/v1/protected-data`, - description: "Access to protected data API", + maxAmountRequired: "10000", // $0.01 in USDC base units - matches X402_ENDPOINT_PRICE_USD ("20000" => $0.02) + resource: `${baseUrl}/v1/x402/search`, + description: "x402 search payment API", }; } }; @@ -194,7 +187,7 @@ export async function generatePaymentHeaderController(req: RequestWithAuth<{}, G notes: { timeSensitive: `This payment header is time-sensitive (valid for ${duration})`, walletRequirement: "Make sure your wallet has USDC on Base Sepolia", - paymentAmount: resource === 'search-with-wallet' ? "1000 USDC (in base units = $0.001)" : "10000 USDC (in base units = $0.01)", + paymentAmount: "10000 USDC (in base units = $0.01)", // 20000 USDC (in base units = $0.02) generatedAt: new Date().toISOString(), team_id: req.auth?.team_id || req.acuc?.team_id } diff --git a/apps/api/src/controllers/v1/protected-data.ts b/apps/api/src/controllers/v1/protected-data.ts deleted file mode 100644 index ed60698a5..000000000 --- a/apps/api/src/controllers/v1/protected-data.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { Request, Response } from "express"; -import { RequestWithAuth } from "./types"; - -export async function protectedDataController( - req: RequestWithAuth<{}, Response, any>, - res: Response, -) { - res.json({ - request: { - headers: req.headers, - body: req.body, - }, - success: true, - data: { - created_at: new Date().toISOString(), - person: { - name: "John Doe", - age: 30, - email: "john.doe@example.com", - }, - company: { - name: "Example Inc.", - industry: "Technology", - }, - }, - }); -} diff --git a/apps/api/src/controllers/v1/search-with-wallet.ts b/apps/api/src/controllers/v1/x402-search.ts similarity index 69% rename from apps/api/src/controllers/v1/search-with-wallet.ts rename to apps/api/src/controllers/v1/x402-search.ts index a1ff7b783..58349af64 100644 --- a/apps/api/src/controllers/v1/search-with-wallet.ts +++ b/apps/api/src/controllers/v1/x402-search.ts @@ -10,12 +10,16 @@ import { } from "./types"; import { v4 as uuidv4 } from "uuid"; import { addScrapeJob, waitForJob } from "../../services/queue-jobs"; +import { logJob } from "../../services/logging/log_job"; import { getJobPriority } from "../../lib/job-priority"; import { Mode } from "../../types"; import { getScrapeQueue } from "../../services/queue-service"; import { search } from "../../search"; import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist"; +import * as Sentry from "@sentry/node"; import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings"; +import { logger as _logger } from "../../lib/logger"; +import type { Logger } from "winston"; import { CostTracking } from "../../lib/extract/extraction-service"; import { supabase_service } from "../../services/supabase"; @@ -25,7 +29,7 @@ interface DocumentWithCostTracking { } // Used for deep research -export async function searchAndScrapeSearchResultWithWallet( +export async function searchAndScrapeX402SearchResult( query: string, options: { teamId: string; @@ -33,6 +37,7 @@ export async function searchAndScrapeSearchResultWithWallet( timeout: number; scrapeOptions: ScrapeOptions; }, + logger: Logger, flags: TeamFlags, ): Promise { try { @@ -43,13 +48,14 @@ export async function searchAndScrapeSearchResultWithWallet( const documentsWithCostTracking = await Promise.all( searchResults.map((result) => - scrapeSearchResultWithWallet( + scrapeX402SearchResult( { url: result.url, title: result.title, description: result.description, }, options, + logger, flags, ), ), @@ -61,7 +67,7 @@ export async function searchAndScrapeSearchResultWithWallet( } } -async function scrapeSearchResultWithWallet( +async function scrapeX402SearchResult( searchResult: { url: string; title: string; description: string }, options: { teamId: string; @@ -69,6 +75,7 @@ async function scrapeSearchResultWithWallet( timeout: number; scrapeOptions: ScrapeOptions; }, + logger: Logger, flags: TeamFlags, directToBullMQ: boolean = false, isSearchPreview: boolean = false, @@ -87,6 +94,13 @@ async function scrapeSearchResultWithWallet( if (isUrlBlocked(searchResult.url, flags)) { throw new Error("Could not scrape url: " + BLOCKLISTED_URL_MESSAGE); } + logger.info("Adding scrape job [x402]", { + scrapeId: jobId, + url: searchResult.url, + teamId: options.teamId, + origin: options.origin, + zeroDataRetention, + }); await addScrapeJob( { url: searchResult.url, @@ -110,6 +124,12 @@ async function scrapeSearchResultWithWallet( const doc: Document = await waitForJob(jobId, options.timeout); + logger.info("Scrape job [x402] completed", { + scrapeId: jobId, + url: searchResult.url, + teamId: options.teamId, + origin: options.origin, + }); await getScrapeQueue().remove(jobId); const document = { @@ -139,6 +159,11 @@ async function scrapeSearchResultWithWallet( costTracking, }; } catch (error) { + logger.error(`Error in scrapeSearchResult [x402]: ${error}`, { + scrapeId: jobId, + url: searchResult.url, + teamId: options.teamId, + }); let statusCode = 0; if (error?.message?.includes("Could not scrape url")) { @@ -163,29 +188,51 @@ async function scrapeSearchResultWithWallet( } } -export async function searchWithWalletController( +export async function x402SearchController( req: RequestWithAuth<{}, SearchResponse, SearchRequest>, res: Response, ) { + const jobId = uuidv4(); + let logger = _logger.child({ + jobId, + teamId: req.auth.team_id, + module: "x402-search", + method: "x402SearchController", + zeroDataRetention: req.acuc?.flags?.forceZDR, + }); + if (req.acuc?.flags?.forceZDR) { - return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on search-with-wallet. Please contact support@firecrawl.com to unblock this feature." }); + return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on x402/search. Please contact support@firecrawl.com to unblock this feature." }); } - + let responseData: SearchResponse = { success: true, data: [], }; const startTime = new Date().getTime(); const isSearchPreview = process.env.SEARCH_PREVIEW_TOKEN !== undefined && process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken; - + try { req.body = searchRequestSchema.parse(req.body); + + // IMPORTANT NOTE: Force results to be at most 10 even if a larger limit is requested + const MAX_RESULTS = 10; + if (req.body.limit > MAX_RESULTS) { + req.body.limit = MAX_RESULTS; + } + + logger = logger.child({ + query: req.body.query, + origin: req.body.origin, + }); let limit = req.body.limit; // Buffer results by 50% to account for filtered URLs const num_results_buffer = Math.floor(limit * 2); + logger.info("Searching [x402] for results"); + let searchResults = await search({ query: req.body.query, advanced: false, @@ -203,12 +250,17 @@ export async function searchWithWalletController( ); } + logger.info("Searching [x402] completed", { + num_results: searchResults.length, + }); + // Filter blocked URLs early to avoid unnecessary billing if (searchResults.length > limit) { searchResults = searchResults.slice(0, limit); } if (searchResults.length === 0) { + logger.info("No search [x402] results found"); responseData.warning = "No search results found"; } else if ( !req.body.scrapeOptions.formats || @@ -220,8 +272,9 @@ export async function searchWithWalletController( description: r.description, })) as Document[]; } else { + logger.info("Scraping search [x402] results"); const scrapePromises = searchResults.map((result) => - scrapeSearchResultWithWallet( + scrapeX402SearchResult( result, { teamId: req.auth.team_id, @@ -229,6 +282,7 @@ export async function searchWithWalletController( timeout: req.body.timeout, scrapeOptions: req.body.scrapeOptions, }, + logger, req.acuc?.flags ?? null, (req.acuc?.price_credits ?? 0) <= 3000, isSearchPreview, @@ -236,6 +290,9 @@ export async function searchWithWalletController( ); const docsWithCostTracking = await Promise.all(scrapePromises); + logger.info("Scraping [x402] completed", { + num_docs: docsWithCostTracking.length, + }); const docs = docsWithCostTracking.map(item => item.document); const filteredDocs = docs.filter( @@ -243,6 +300,10 @@ export async function searchWithWalletController( doc.serpResults || (doc.markdown && doc.markdown.trim().length > 0), ); + logger.info("Filtering [x402] completed", { + num_docs: filteredDocs.length, + }); + if (filteredDocs.length === 0) { responseData.data = docs; responseData.warning = "No content found in search results"; @@ -251,6 +312,39 @@ export async function searchWithWalletController( } } + const endTime = new Date().getTime(); + const timeTakenInSeconds = (endTime - startTime) / 1000; + + logger.info("Logging job", { + num_docs: responseData.data.length, + time_taken: timeTakenInSeconds, + }); + + logJob( + { + job_id: jobId, + success: true, + num_docs: responseData.data.length, + docs: responseData.data, + time_taken: timeTakenInSeconds, + team_id: req.auth.team_id, + mode: "x402-search", + url: req.body.query, + scrapeOptions: req.body.scrapeOptions, + crawlerOptions: { + ...req.body, + query: undefined, + scrapeOptions: undefined, + }, + origin: req.body.origin, + integration: req.body.integration, + credits_billed: 0, + zeroDataRetention: false, // not supported + }, + false, + isSearchPreview, + ); + return res.status(200).json(responseData); } catch (error) { if ( @@ -263,6 +357,8 @@ export async function searchWithWalletController( }); } + Sentry.captureException(error); + logger.error("Unhandled error occurred in search [x402]", { error }); return res.status(500).json({ success: false, error: error.message, diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.ts index f5e2f5f47..f1c4142d9 100644 --- a/apps/api/src/routes/v1.ts +++ b/apps/api/src/routes/v1.ts @@ -29,7 +29,7 @@ import { extractStatusController } from "../controllers/v1/extract-status"; import { creditUsageController } from "../controllers/v1/credit-usage"; import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings"; import { searchController } from "../controllers/v1/search"; -import { searchWithWalletController } from "../controllers/v1/search-with-wallet"; +import { x402SearchController } from "../controllers/v1/x402-search"; import { crawlErrorsController } from "../controllers/v1/crawl-errors"; import { generateLLMsTextController } from "../controllers/v1/generate-llmstxt"; import { generateLLMsTextStatusController } from "../controllers/v1/generate-llmstxt-status"; @@ -38,7 +38,6 @@ import { deepResearchStatusController } from "../controllers/v1/deep-research-st import { tokenUsageController } from "../controllers/v1/token-usage"; import { ongoingCrawlsController } from "../controllers/v1/crawl-ongoing"; import { generatePaymentHeaderController } from "../controllers/v1/generate-payment-header"; -import { protectedDataController } from '../controllers/v1/protected-data'; import { paymentMiddleware } from "x402-express"; function checkCreditsMiddleware( @@ -178,28 +177,20 @@ expressWs(express()); export const v1Router = express.Router(); -// First, configure the payment middleware with your routes +// Configure payment middleware to enable micropayment-protected endpoints +// This middleware handles payment verification and processing for premium API features +// x402 payments protocol - https://github.com/coinbase/x402 v1Router.use( paymentMiddleware( process.env.X402_PAY_TO_ADDRESS as `0x${string}` || "0x0000000000000000000000000000000000000000", { - // Define your routes and their payment requirements - "GET /protected-data": { - price: process.env.X402_ENDPOINT_PRICE_USD || "0.01", - network: "base-sepolia", + "POST /x402/search": { + price: process.env.X402_ENDPOINT_PRICE_USD as string, + network: process.env.X402_NETWORK as "base-sepolia" | "base" | "avalanche-fuji" | "avalanche" | "iotex", config: { - description: "Access premium protected data API", + description: "The search endpoint combines web search (SERP) with Firecrawl's scraping capabilities to return full page content for any query. Requires micropayment via X402 protocol", mimeType: "application/json", - maxTimeoutSeconds: 300, - } - }, - "POST /search-with-wallet": { - price: process.env.X402_ENDPOINT_PRICE_USD || "0.01", - network: "base-sepolia", - config: { - description: "Search with wallet payment API", - mimeType: "application/json", - maxTimeoutSeconds: 300, + maxTimeoutSeconds: 120, } }, }, @@ -381,13 +372,8 @@ v1Router.post( wrap(generatePaymentHeaderController), ); -v1Router.get("/protected-data", - authMiddleware(RateLimiterMode.Search), - wrap(protectedDataController), -); - v1Router.post( - "/search-with-wallet", + "/x402/search", authMiddleware(RateLimiterMode.Search), - wrap(searchWithWalletController), + wrap(x402SearchController), ); \ No newline at end of file