From c1b78d2cbd8ea43baad5872bf9806c9b971feeee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Sat, 27 Sep 2025 12:48:10 +0300 Subject: [PATCH] fix(otel): --- apps/api/package.json | 6 -- apps/api/pnpm-lock.yaml | 52 --------------- apps/api/src/index.ts | 63 +++---------------- .../api/src/lib/extract/extraction-service.ts | 11 ---- .../extract/fire-0/extraction-service-f0.ts | 11 ---- apps/api/src/services/langfuse.ts | 3 - apps/api/src/services/queue-worker.ts | 57 +++-------------- apps/api/src/services/worker/scrape-worker.ts | 62 +++--------------- 8 files changed, 26 insertions(+), 239 deletions(-) delete mode 100644 apps/api/src/services/langfuse.ts diff --git a/apps/api/package.json b/apps/api/package.json index 8d88296a1..c29baf861 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -79,12 +79,8 @@ "@openrouter/ai-sdk-provider": "^0.4.5", "@opentelemetry/auto-instrumentations-node": "^0.62.0", "@opentelemetry/core": "^2.0.1", - "@opentelemetry/exporter-trace-otlp-grpc": "^0.203.0", "@opentelemetry/exporter-trace-otlp-http": "^0.205.0", - "@opentelemetry/resources": "^2.0.1", "@opentelemetry/sdk-node": "^0.203.0", - "@opentelemetry/sdk-trace-node": "^2.0.1", - "@opentelemetry/semantic-conventions": "^1.36.0", "@sentry/cli": "^2.33.1", "@sentry/node": "^9.40.0", "@sentry/profiling-node": "^9.40.0", @@ -112,8 +108,6 @@ "joplin-turndown-plugin-gfm": "^1.0.12", "jsdom": "^26.0.0", "koffi": "^2.9.0", - "langfuse": "^3.38.4", - "langfuse-vercel": "^3.38.4", "lodash": "^4.17.21", "marked": "^14.1.2", "ollama-ai-provider": "^1.2.0", diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml index c4c67b301..6e7dc8b3d 100644 --- a/apps/api/pnpm-lock.yaml +++ b/apps/api/pnpm-lock.yaml @@ -66,24 +66,12 @@ importers: '@opentelemetry/core': specifier: ^2.0.1 version: 2.0.1(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-trace-otlp-grpc': - specifier: ^0.203.0 - version: 0.203.0(@opentelemetry/api@1.9.0) '@opentelemetry/exporter-trace-otlp-http': specifier: ^0.205.0 version: 0.205.0(@opentelemetry/api@1.9.0) - '@opentelemetry/resources': - specifier: ^2.0.1 - version: 2.0.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-node': specifier: ^0.203.0 version: 0.203.0(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-trace-node': - specifier: ^2.0.1 - version: 2.0.1(@opentelemetry/api@1.9.0) - '@opentelemetry/semantic-conventions': - specifier: ^1.36.0 - version: 1.36.0 '@sentry/cli': specifier: ^2.33.1 version: 2.33.1(encoding@0.1.13) @@ -165,12 +153,6 @@ importers: koffi: specifier: ^2.9.0 version: 2.9.0 - langfuse: - specifier: ^3.38.4 - version: 3.38.4 - langfuse-vercel: - specifier: ^3.38.4 - version: 3.38.4(ai@4.3.5(react@18.3.1)(zod@3.24.2)) lodash: specifier: ^4.17.21 version: 4.17.21 @@ -5222,20 +5204,6 @@ packages: kuler@2.0.0: resolution: {integrity: sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==} - langfuse-core@3.38.4: - resolution: {integrity: sha512-onTAqcEGhoXuBgqDFXe2t+bt9Vi+5YChRgdz3voM49JKoHwtVZQiUdqTfjSivGR75eSbYoiaIL8IRoio+jaqwg==} - engines: {node: '>=18'} - - langfuse-vercel@3.38.4: - resolution: {integrity: sha512-gyhQuim2b1qpI7yiGnww9qzqsdpD59XagD+QB3eGHIrX+1ao7vbcrwL14H0GlsLIUM10Nud9wtEbxiFZP1PB7g==} - engines: {node: '>=18'} - peerDependencies: - ai: '>=3.2.44' - - langfuse@3.38.4: - resolution: {integrity: sha512-2UqMeHLl3DGNX1Nh/cO4jGhk7TzDJ6gjQLlyS9rwFCKVO81xot6b58yeTsTB5YrWupWsOxQtMNoQYIQGOUlH9Q==} - engines: {node: '>=18'} - leac@0.6.0: resolution: {integrity: sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg==} @@ -5483,10 +5451,6 @@ packages: multiformats@9.9.0: resolution: {integrity: sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg==} - mustache@4.2.0: - resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==} - hasBin: true - mute-stream@2.0.0: resolution: {integrity: sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==} engines: {node: ^18.17.0 || >=20.5.0} @@ -13742,20 +13706,6 @@ snapshots: kuler@2.0.0: {} - langfuse-core@3.38.4: - dependencies: - mustache: 4.2.0 - - langfuse-vercel@3.38.4(ai@4.3.5(react@18.3.1)(zod@3.24.2)): - dependencies: - ai: 4.3.5(react@18.3.1)(zod@3.24.2) - langfuse: 3.38.4 - langfuse-core: 3.38.4 - - langfuse@3.38.4: - dependencies: - langfuse-core: 3.38.4 - leac@0.6.0: {} leven@3.1.0: {} @@ -13989,8 +13939,6 @@ snapshots: multiformats@9.9.0: {} - mustache@4.2.0: {} - mute-stream@2.0.0: {} nan@2.22.2: diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 3ccbbcc2c..598565536 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -31,13 +31,8 @@ import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup"; import { v2Router } from "./routes/v2"; import domainFrequencyRouter from "./routes/domain-frequency"; import { NodeSDK } from "@opentelemetry/sdk-node"; -import { LangfuseExporter } from "langfuse-vercel"; import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node"; -import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; -import { resourceFromAttributes } from "@opentelemetry/resources"; -import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node"; -import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc"; -import { OTLPTraceExporter as OTLPHTTPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; import { nuqShutdown } from "./services/worker/nuq"; import { getErrorContactMessage } from "./lib/deployment"; import { initializeBlocklist } from "./scraper/WebScraper/utils/blocklist"; @@ -58,48 +53,12 @@ logger.info("Network info dump", { cacheableLookup.install(http.globalAgent); cacheableLookup.install(https.globalAgent); -const shouldOtel = - process.env.LANGFUSE_PUBLIC_KEY || - process.env.OTEL_EXPORTER_OTLP_ENDPOINT || - process.env.HONEYCOMB_TEAM_ID; -const otelSdk = shouldOtel - ? new NodeSDK({ - resource: resourceFromAttributes({ - [ATTR_SERVICE_NAME]: "firecrawl-app", - }), - spanProcessors: [ - ...(process.env.LANGFUSE_PUBLIC_KEY - ? [new BatchSpanProcessor(new LangfuseExporter())] - : []), - ...(process.env.OTEL_EXPORTER_OTLP_ENDPOINT - ? [ - new BatchSpanProcessor( - new OTLPTraceExporter({ - url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT, - }), - ), - ] - : []), - ...(process.env.HONEYCOMB_TEAM_ID - ? [ - new BatchSpanProcessor( - new OTLPHTTPTraceExporter({ - url: "https://api.honeycomb.io", - headers: { - "x-honeycomb-team": process.env.HONEYCOMB_TEAM_ID, - }, - }), - ), - ] - : []), - ], - instrumentations: [getNodeAutoInstrumentations()], - }) - : null; +const otelSdk = new NodeSDK({ + traceExporter: new OTLPTraceExporter(), + instrumentations: [getNodeAutoInstrumentations()], +}); -if (otelSdk) { - otelSdk.start(); -} +otelSdk.start(); // Initialize Express with WebSocket support const expressApp = express(); @@ -183,14 +142,10 @@ async function startServer(port = DEFAULT_PORT) { logger.info("Server closed."); nuqShutdown().finally(() => { logger.info("NUQ shutdown complete"); - if (otelSdk) { - otelSdk.shutdown().finally(() => { - logger.info("OTEL shutdown"); - process.exit(0); - }); - } else { + otelSdk.shutdown().finally(() => { + logger.info("OTEL shutdown"); process.exit(0); - } + }); }); }); }; diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.ts index 4abb8cf9b..4e70fc516 100644 --- a/apps/api/src/lib/extract/extraction-service.ts +++ b/apps/api/src/lib/extract/extraction-service.ts @@ -40,7 +40,6 @@ import { normalizeUrl } from "../canonical-url"; import { search } from "../../search"; import { buildRephraseToSerpPrompt } from "./build-prompts"; import { getACUCTeam } from "../../controllers/auth"; -import { langfuse } from "../../services/langfuse"; import { CostLimitExceededError, CostTracking } from "../cost-tracking"; interface ExtractServiceOptions { @@ -103,16 +102,6 @@ export async function performExtraction( teamId, }); - langfuse.trace({ - id: "extract:" + extractId, - name: "performExtraction", - metadata: { - teamId, - extractId, - model: "fire-1", - }, - }); - try { // If no URLs are provided, generate URLs from the prompt if ((!request.urls || request.urls.length === 0) && request.prompt) { diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts index 8de1cb842..951c0a536 100644 --- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts +++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts @@ -39,7 +39,6 @@ import { } from "./usage/llm-cost-f0"; import { SourceTracker_F0 } from "./helpers/source-tracker-f0"; import { getACUCTeam } from "../../../controllers/auth"; -import { langfuse } from "../../../services/langfuse"; interface ExtractServiceOptions { request: ExtractRequest; @@ -94,16 +93,6 @@ export async function performExtraction_F0( teamId, }); - langfuse.trace({ - id: "extract:" + extractId, - name: "performExtraction", - metadata: { - teamId, - extractId, - model: "fire-0", - }, - }); - // If no URLs are provided, generate URLs from the prompt if ((!request.urls || request.urls.length === 0) && request.prompt) { logger.debug("Generating URLs from prompt...", { diff --git a/apps/api/src/services/langfuse.ts b/apps/api/src/services/langfuse.ts deleted file mode 100644 index 90aa50155..000000000 --- a/apps/api/src/services/langfuse.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { Langfuse } from "langfuse"; - -export const langfuse = new Langfuse(); diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts index bbae4a02b..e128d7ed1 100644 --- a/apps/api/src/services/queue-worker.ts +++ b/apps/api/src/services/queue-worker.ts @@ -30,13 +30,8 @@ import { cacheableLookup } from "../scraper/scrapeURL/lib/cacheableLookup"; import { robustFetch } from "../scraper/scrapeURL/lib/fetch"; import { NodeSDK } from "@opentelemetry/sdk-node"; import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node"; -import { LangfuseExporter } from "langfuse-vercel"; -import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node"; -import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; -import { resourceFromAttributes } from "@opentelemetry/resources"; -import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; import { BullMQOtel } from "bullmq-otel"; -import { pathToFileURL } from "url"; import { getErrorContactMessage } from "../lib/deployment"; import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist"; @@ -61,48 +56,12 @@ const runningJobs: Set = new Set(); cacheableLookup.install(http.globalAgent); cacheableLookup.install(https.globalAgent); -const shouldOtel = - process.env.LANGFUSE_PUBLIC_KEY || - process.env.OTEL_EXPORTER_OTLP_ENDPOINT || - process.env.HONEYCOMB_TEAM_ID; -const otelSdk = shouldOtel - ? new NodeSDK({ - resource: resourceFromAttributes({ - [ATTR_SERVICE_NAME]: "firecrawl-worker", - }), - spanProcessors: [ - ...(process.env.LANGFUSE_PUBLIC_KEY - ? [new BatchSpanProcessor(new LangfuseExporter())] - : []), - ...(process.env.OTEL_EXPORTER_OTLP_ENDPOINT - ? [ - new BatchSpanProcessor( - new OTLPTraceExporter({ - url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT, - }), - ), - ] - : []), - ...(process.env.HONEYCOMB_TEAM_ID - ? [ - new BatchSpanProcessor( - new OTLPTraceExporter({ - url: "https://api.honeycomb.io", - headers: { - "x-honeycomb-team": process.env.HONEYCOMB_TEAM_ID, - }, - }), - ), - ] - : []), - ], - instrumentations: [getNodeAutoInstrumentations()], - }) - : null; +const otelSdk = new NodeSDK({ + traceExporter: new OTLPTraceExporter(), + instrumentations: [getNodeAutoInstrumentations()], +}); -if (otelSdk) { - otelSdk.start(); -} +otelSdk.start(); const processExtractJobInternal = async ( token: string, @@ -529,8 +488,6 @@ app.listen(workerPort, () => { } console.log("All jobs finished. Worker out!"); - if (otelSdk) { - await otelSdk.shutdown(); - } + await otelSdk.shutdown(); process.exit(0); })(); diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts index 6ebd829d2..536a760c1 100644 --- a/apps/api/src/services/worker/scrape-worker.ts +++ b/apps/api/src/services/worker/scrape-worker.ts @@ -50,7 +50,6 @@ import { calculateCreditsToBeBilled } from "../../lib/scrape-billing"; import { getBillingQueue } from "../queue-service"; import type { Logger } from "winston"; import { finishCrawlIfNeeded } from "./crawl-logic"; -import { LangfuseExporter } from "langfuse-vercel"; import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node"; import { NodeSDK } from "@opentelemetry/sdk-node"; import { @@ -60,10 +59,7 @@ import { UnknownError, } from "../../lib/error"; import { serializeTransportableError } from "../../lib/error-serde"; -import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node"; -import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc"; -import { resourceFromAttributes } from "@opentelemetry/resources"; -import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; import type { NuQJob } from "./nuq"; import { ScrapeJobData, @@ -1144,56 +1140,18 @@ export const processJobInternal = async (job: NuQJob) => { } }; -const shouldOtel = - process.env.LANGFUSE_PUBLIC_KEY || - process.env.OTEL_EXPORTER_OTLP_ENDPOINT || - process.env.HONEYCOMB_TEAM_ID; -const otelSdk = shouldOtel - ? new NodeSDK({ - resource: resourceFromAttributes({ - [ATTR_SERVICE_NAME]: "firecrawl-worker-scrape", - }), - spanProcessors: [ - ...(process.env.LANGFUSE_PUBLIC_KEY - ? [new BatchSpanProcessor(new LangfuseExporter())] - : []), - ...(process.env.OTEL_EXPORTER_OTLP_ENDPOINT - ? [ - new BatchSpanProcessor( - new OTLPTraceExporter({ - url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT, - }), - ), - ] - : []), - ...(process.env.HONEYCOMB_TEAM_ID - ? [ - new BatchSpanProcessor( - new OTLPTraceExporter({ - url: "https://api.honeycomb.io", - headers: { - "x-honeycomb-team": process.env.HONEYCOMB_TEAM_ID, - }, - }), - ), - ] - : []), - ], - instrumentations: [getNodeAutoInstrumentations()], - }) - : null; +const otelSdk = new NodeSDK({ + traceExporter: new OTLPTraceExporter(), + instrumentations: [getNodeAutoInstrumentations()], +}); -if (otelSdk) { - otelSdk.start(); -} +otelSdk.start(); const exitHandler = () => { - if (otelSdk) { - otelSdk.shutdown().finally(() => { - _logger.debug("OTEL shutdown"); - process.exit(0); - }); - } + otelSdk.shutdown().finally(() => { + _logger.debug("OTEL shutdown"); + process.exit(0); + }); }; process.on("SIGINT", exitHandler);