chore: Add route-aware sampling for webhook traces (#33259)

packages/@n8n/config/src/configs/sentry.config.ts

@@ -46,6 +46,16 @@ export class SentryConfig {
 	@Env('N8N_SENTRY_TRACES_SLOW_SPAN_THRESHOLD_MS', z.number({ coerce: true }).int().positive())
 	tracesSlowSpanThresholdMs: number = 1000;
 
+	/**
+	 * Sample rate (0.0 to 1.0) for successful production webhook transaction
+	 * traces. These are by far the highest-volume route, so they are sampled
+	 * below the base rate. Errored webhook transactions are always kept.
+	 *
+	 * @default 0.05
+	 */
+	@Env('N8N_SENTRY_WEBHOOK_TRACES_SAMPLE_RATE', sampleRateSchema)
+	webhookTracesSampleRate: number = 0.05;
+
 	/**
 	 * Whether Sentry's native event-loop-block detection is enabled. When on, a
 	 * native watchdog (`@sentry/node-native`) captures the main thread's stack

packages/@n8n/config/test/config.test.ts

@@ -402,6 +402,7 @@ describe('GlobalConfig', () => {
 			profilesSampleRate: 0,
 			tracesSampleRate: 0,
 			tracesSlowSpanThresholdMs: 1000,
+			webhookTracesSampleRate: 0.05,
 			eventLoopBlockDetectionEnabled: false,
 			eventLoopBlockThreshold: 500,
 			eventLoopBlockMaxEventsPerHour: 5,

packages/cli/src/commands/base-command.ts

@@ -97,6 +97,7 @@ export abstract class BaseCommand<F = never> {
 			profilesSampleRate,
 			tracesSampleRate,
 			tracesSlowSpanThresholdMs,
+			webhookTracesSampleRate,
 			eventLoopBlockThreshold,
 			eventLoopBlockMaxEventsPerHour,
 			eventLoopBlockDetectionEnabled,
@@ -113,6 +114,8 @@ export abstract class BaseCommand<F = never> {
 			eventLoopBlockMaxEventsPerHour,
 			tracesSampleRate,
 			slowSpanThresholdMs: tracesSlowSpanThresholdMs,
+			webhookEndpoint: this.globalConfig.endpoints.webhook,
+			webhookTracesSampleRate,
 			profilesSampleRate,
 			healthEndpoint: resolveBackendHealthEndpointPath(this.globalConfig),
 			eligibleIntegrations: {

packages/core/src/errors/error-reporter.ts

@@ -48,6 +48,12 @@ type ErrorReporterInitOptions = {
 	/** Threshold in ms below which non-errored `db`/`http.client` spans are dropped. */
 	slowSpanThresholdMs?: number;
 
+	/** Production webhook endpoint path segment (e.g. `webhook`), used to sample webhook traces. */
+	webhookEndpoint?: string;
+
+	/** Sample rate (0.0 to 1.0) for successful production webhook transaction traces. */
+	webhookTracesSampleRate?: number;
+
 	/** Sample rate for Sentry profiling (0.0 to 1.0). 0 means disabled */
 	profilesSampleRate: number;
 
@@ -156,6 +162,8 @@ export class ErrorReporter {
 		profilesSampleRate,
 		tracesSampleRate,
 		slowSpanThresholdMs = DEFAULT_SLOW_SPAN_THRESHOLD_MS,
+		webhookEndpoint,
+		webhookTracesSampleRate,
 		eligibleIntegrations = {},
 		healthEndpoint = '/healthz',
 	}: ErrorReporterInitOptions) {
@@ -250,7 +258,12 @@ export class ErrorReporter {
 			...(isTracingEnabled
 				? {
 						tracesSampler: buildTracesSampler(tracesSampleRate),
-						beforeSendTransaction: buildBeforeSendTransaction(slowSpanThresholdMs),
+						beforeSendTransaction: buildBeforeSendTransaction(
+							slowSpanThresholdMs,
+							webhookEndpoint && webhookTracesSampleRate !== undefined
+								? { endpoint: webhookEndpoint, sampleRate: webhookTracesSampleRate }
+								: undefined,
+						),
 					}
 				: {}),
 			...(isProfilingEnabled ? { profilesSampleRate, profileLifecycle: 'trace' } : {}),

packages/core/src/observability/tracing/__tests__/span-sampling.test.ts

@@ -1,4 +1,5 @@
 import type { SpanJSON, TracesSamplerSamplingContext, TransactionEvent } from '@sentry/core';
+import { afterEach, describe, expect, it, vi } from 'vitest';
 
 import {
 	buildBeforeSendTransaction,
@@ -140,6 +141,64 @@ describe('buildBeforeSendTransaction', () => {
 		const event = { type: 'transaction' } as TransactionEvent;
 		expect(buildBeforeSendTransaction(THRESHOLD_MS)(event)).toBe(event);
 	});
+
+	describe('webhook trace sampling', () => {
+		const WEBHOOK = { endpoint: 'webhook', sampleRate: 0.05 };
+		const beforeSend = buildBeforeSendTransaction(THRESHOLD_MS, WEBHOOK);
+
+		/** Builds a webhook transaction with the given name and root status. */
+		function webhookTx(name: string, status?: SpanJSON['status']): TransactionEvent {
+			return {
+				type: 'transaction',
+				transaction: name,
+				contexts: { trace: { op: 'http.server', span_id: 'root', trace_id: 't', status } },
+			} as TransactionEvent;
+		}
+
+		afterEach(() => vi.restoreAllMocks());
+
+		it('drops a successful webhook transaction when the dice roll exceeds the rate', () => {
+			vi.spyOn(Math, 'random').mockReturnValue(0.5);
+			expect(beforeSend(webhookTx('POST /webhook/*path', 'ok'))).toBeNull();
+		});
+
+		it('keeps a successful webhook transaction when the dice roll is below the rate', () => {
+			vi.spyOn(Math, 'random').mockReturnValue(0.01);
+			expect(beforeSend(webhookTx('POST /webhook/*path', 'ok'))).not.toBeNull();
+		});
+
+		it('keeps errored webhook transactions regardless of the dice roll', () => {
+			vi.spyOn(Math, 'random').mockReturnValue(0.99);
+			expect(beforeSend(webhookTx('POST /webhook/*path', 'internal_error'))).not.toBeNull();
+		});
+
+		it('does not sample /rest transactions', () => {
+			vi.spyOn(Math, 'random').mockReturnValue(0.99);
+			expect(beforeSend(webhookTx('POST /rest/executions/:id/stop', 'ok'))).not.toBeNull();
+		});
+
+		it('does not sample test/waiting webhooks (production only)', () => {
+			vi.spyOn(Math, 'random').mockReturnValue(0.99);
+			expect(beforeSend(webhookTx('POST /webhook-test/*path', 'ok'))).not.toBeNull();
+			expect(beforeSend(webhookTx('POST /webhook-waiting/:path', 'ok'))).not.toBeNull();
+		});
+
+		it('does not sample webhooks when no webhook config is provided', () => {
+			vi.spyOn(Math, 'random').mockReturnValue(0.99);
+			expect(
+				buildBeforeSendTransaction(THRESHOLD_MS)(webhookTx('POST /webhook/*path', 'ok')),
+			).not.toBeNull();
+		});
+
+		it('does not sample webhooks when the rate is 1', () => {
+			vi.spyOn(Math, 'random').mockReturnValue(0.99);
+			const keepAll = buildBeforeSendTransaction(THRESHOLD_MS, {
+				endpoint: 'webhook',
+				sampleRate: 1,
+			});
+			expect(keepAll(webhookTx('POST /webhook/*path', 'ok'))).not.toBeNull();
+		});
+	});
 });
 
 describe('buildTracesSampler', () => {

packages/core/src/observability/tracing/span-sampling.ts

@@ -67,6 +67,23 @@ function shouldKeepSpan(span: SpanJSON, slowSpanThresholdMs: number): boolean {
 	return true;
 }
 
+/**
+ * Whether a transaction is a successful (non-errored) production webhook request.
+ * Sentry names Express transactions by route pattern, e.g. `POST /webhook/*path`,
+ * so we match on the configured webhook endpoint. `webhook-test`/`webhook-waiting`
+ * are deliberately excluded (only production webhooks are high-volume).
+ */
+function isSuccessfulWebhook(event: TransactionEvent, webhookEndpoint: string): boolean {
+	if (isErrored(event.contexts?.trace?.status)) return false;
+
+	const name = event.transaction;
+	if (!name) return false;
+
+	const path = name.slice(name.indexOf(' ') + 1); // strip the HTTP method prefix
+	const prefix = `/${webhookEndpoint}`;
+	return path === prefix || path.startsWith(`${prefix}/`);
+}
+
 function reparentChildSpans(
 	spans: SpanJSON[],
 	droppedSpan: SpanJSON,
@@ -90,14 +107,24 @@ function reparentChildSpans(
  * - Fast, non-errored `db`/`http.client` child spans (below `slowSpanThresholdMs`).
  *
  * Kept descendants of dropped child spans are reparented.
+ *
+ * When `webhook` is set, successful production webhook transactions (the highest-volume
+ * route) are randomly dropped down to `webhook.sampleRate`; errored ones are always kept.
  */
-export function buildBeforeSendTransaction(slowSpanThresholdMs: number) {
+export function buildBeforeSendTransaction(
+	slowSpanThresholdMs: number,
+	webhook?: { endpoint: string; sampleRate: number },
+) {
 	return (event: TransactionEvent): TransactionEvent | null => {
 		// DB operations (queries, pool connects) get op `db`. A `db`-rooted transaction
 		// ran outside any request/job span — high volume, no signal. Child db spans under
 		// real transactions keep a non-`db` root op, so they are untouched.
 		if (event.contexts?.trace?.op === 'db') return null;
 
+		if (webhook && webhook.sampleRate < 1 && isSuccessfulWebhook(event, webhook.endpoint)) {
+			if (Math.random() >= webhook.sampleRate) return null;
+		}
+
 		if (event.spans) {
 			const spans = event.spans;
 			event.spans = spans.filter((span) => {