feat(core): Add agent-evals PostHog feature flag and gating (no-changelog) (#34651)

packages/@n8n/api-types/src/frontend-settings.ts

@@ -272,6 +272,13 @@ export interface FrontendSettings {
 		 * `084_eval_collections` flag would otherwise never resolve.
 		 */
 		collectionsEnabled: boolean;
+		/**
+		 * Operator override (`N8N_AGENT_EVALS_ENABLED`) that force-enables the
+		 * agent-evals surface. Surfaced here so the frontend gate works even when
+		 * the in-browser PostHog client is disabled (telemetry off), where the
+		 * `101_agent_evals` flag would otherwise never resolve.
+		 */
+		agentEvalsEnabled: boolean;
 	};
 
 	/** Backend modules that were initialized during startup. */

packages/@n8n/api-types/src/index.ts

@@ -560,6 +560,8 @@ export {
 	type EvalVersionsResponse,
 } from './schemas/eval-collections.schema';
 
+export { AGENT_EVALS_FLAG } from './schemas/agent-evals.schema';
+
 export {
 	aiInsightsStatusSchema,
 	aiInsightsPayloadSchema,

packages/@n8n/api-types/src/schemas/agent-evals.schema.ts

@@ -0,0 +1,7 @@
+// PostHog rollout flag id gating the agent-evals feature surface. Every new
+// agent-eval endpoint + frontend entry point consults this; the flag-off
+// cohort sees no agent-eval surface at all. Single source of truth shared
+// between FE and BE so the two cannot drift. Follows the `NNN_<feature>`
+// numeric-prefix convention used by every other n8n PostHog flag (next
+// available after `100_n8n_credits_credential_selection`).
+export const AGENT_EVALS_FLAG = '101_agent_evals';

packages/@n8n/config/src/configs/evaluation.config.ts

@@ -22,4 +22,23 @@ export class EvaluationConfig {
 	 */
 	@Env('N8N_EVAL_COLLECTIONS_ENABLED')
 	collectionsEnabled: boolean = false;
+
+	/**
+	 * Force-enable the agent-evals feature surface.
+	 *
+	 * Acts as an operator-level override of the `101_agent_evals` PostHog
+	 * rollout flag. When `true`, every agent-eval endpoint + frontend entry
+	 * point is available regardless of PostHog cohort. When `false` (default),
+	 * PostHog remains the source of truth.
+	 *
+	 * Useful for:
+	 * - Local development without PostHog wiring.
+	 * - QA environments that need the feature on for a smoke run.
+	 * - Self-hosted deployments that want the feature without PostHog.
+	 *
+	 * Cannot force-disable: setting this to `false` falls back to PostHog,
+	 * not a kill-switch. Use PostHog itself to disable a rolled-out flag.
+	 */
+	@Env('N8N_AGENT_EVALS_ENABLED')
+	agentEvalsEnabled: boolean = false;
 }

packages/@n8n/config/test/config.test.ts

@@ -468,6 +468,7 @@ describe('GlobalConfig', () => {
 		},
 		evaluation: {
 			collectionsEnabled: false,
+			agentEvalsEnabled: false,
 		},
 		generic: {
 			timezone: 'America/New_York',

packages/cli/src/posthog/__tests__/posthog.test.ts

@@ -219,6 +219,7 @@ describe('PostHog', () => {
 				// Mutated per test; reset so test ordering doesn't leak override
 				// state into unrelated cases.
 				globalConfig.evaluation.collectionsEnabled = false;
+				globalConfig.evaluation.agentEvalsEnabled = false;
 			});
 
 			it('force-enables the eval-collections flag when N8N_EVAL_COLLECTIONS_ENABLED is set', async () => {
@@ -257,6 +258,30 @@ describe('PostHog', () => {
 
 				expect(flags).toEqual({ '084_eval_collections': true });
 			});
+
+			it('force-enables the agent-evals flag when N8N_AGENT_EVALS_ENABLED is set', async () => {
+				(PostHog.prototype.evaluateFlags as Mock).mockResolvedValue(mockEvaluatedFlags({}));
+				globalConfig.evaluation.agentEvalsEnabled = true;
+
+				const ph = new PostHogClient(instanceSettings, globalConfig);
+				await ph.init();
+
+				const flags = await ph.getFeatureFlags({ id: userId, createdAt });
+
+				expect(flags).toMatchObject({ '101_agent_evals': true });
+			});
+
+			it('falls back to the agent-evals override when PostHog throws', async () => {
+				(PostHog.prototype.evaluateFlags as Mock).mockRejectedValue(new Error('posthog down'));
+				globalConfig.evaluation.agentEvalsEnabled = true;
+
+				const ph = new PostHogClient(instanceSettings, globalConfig);
+				await ph.init();
+
+				const flags = await ph.getFeatureFlags({ id: userId, createdAt });
+
+				expect(flags).toEqual({ '101_agent_evals': true });
+			});
 		});
 	});
 });

packages/cli/src/posthog/index.ts

@@ -1,4 +1,4 @@
-import { EVAL_COLLECTIONS_FLAG } from '@n8n/api-types';
+import { AGENT_EVALS_FLAG, EVAL_COLLECTIONS_FLAG } from '@n8n/api-types';
 import { GlobalConfig } from '@n8n/config';
 import type { PublicUser } from '@n8n/db';
 import { Service } from '@n8n/di';
@@ -168,6 +168,9 @@ export class PostHogClient {
 		if (this.globalConfig.evaluation.collectionsEnabled) {
 			overrides[EVAL_COLLECTIONS_FLAG] = true;
 		}
+		if (this.globalConfig.evaluation.agentEvalsEnabled) {
+			overrides[AGENT_EVALS_FLAG] = true;
+		}
 		return Object.keys(overrides).length === 0 ? flags : { ...flags, ...overrides };
 	}
 }

packages/cli/src/services/frontend.service.ts

@@ -420,6 +420,7 @@ export class FrontendService {
 			evaluation: {
 				quota: this.licenseState.getMaxWorkflowsWithEvaluations(),
 				collectionsEnabled: this.globalConfig.evaluation.collectionsEnabled,
+				agentEvalsEnabled: this.globalConfig.evaluation.agentEvalsEnabled,
 			},
 			activeModules: this.moduleRegistry.getActiveModules(),
 			canvasOnly: this.globalConfig.canvasOnly,

packages/frontend/editor-ui/src/__tests__/defaults.ts

@@ -184,6 +184,7 @@ export const defaultSettings: FrontendSettings = {
 	evaluation: {
 		quota: 0,
 		collectionsEnabled: false,
+		agentEvalsEnabled: false,
 	},
 	activeModules: [],
 	canvasOnly: false,

packages/frontend/editor-ui/src/features/ai/evaluation.ee/composables/useAgentEvalsFlag.test.ts

@@ -0,0 +1,41 @@
+import { describe, expect, it, vi } from 'vitest';
+
+import { useAgentEvalsFlag } from './useAgentEvalsFlag';
+
+const settingsState = { agentEvalsEnabled: false };
+const posthogState = { enabled: false };
+
+vi.mock('@/app/stores/settings.store', () => ({
+	useSettingsStore: () => ({
+		settings: { evaluation: { agentEvalsEnabled: settingsState.agentEvalsEnabled } },
+	}),
+}));
+
+vi.mock('@/app/stores/posthog.store', () => ({
+	usePostHog: () => ({ isFeatureEnabled: () => posthogState.enabled }),
+}));
+
+describe('useAgentEvalsFlag', () => {
+	it('is enabled via the backend operator override even when PostHog is off', () => {
+		// The telemetry-off case: the in-browser PostHog client never initializes,
+		// so the flag must come from the settings-provided override.
+		settingsState.agentEvalsEnabled = true;
+		posthogState.enabled = false;
+
+		expect(useAgentEvalsFlag().value).toBe(true);
+	});
+
+	it('is enabled via the PostHog cohort flag when the override is off', () => {
+		settingsState.agentEvalsEnabled = false;
+		posthogState.enabled = true;
+
+		expect(useAgentEvalsFlag().value).toBe(true);
+	});
+
+	it('is disabled when neither signal is set', () => {
+		settingsState.agentEvalsEnabled = false;
+		posthogState.enabled = false;
+
+		expect(useAgentEvalsFlag().value).toBe(false);
+	});
+});

packages/frontend/editor-ui/src/features/ai/evaluation.ee/composables/useAgentEvalsFlag.ts

@@ -0,0 +1,29 @@
+import { AGENT_EVALS_FLAG } from '@n8n/api-types';
+import { computed } from 'vue';
+
+import { usePostHog } from '@/app/stores/posthog.store';
+import { useSettingsStore } from '@/app/stores/settings.store';
+
+/**
+ * Frontend gate for the agent-evals feature surface, matching the
+ * `101_agent_evals` flag the backend consults to gate its routes. It combines
+ * two independent signals:
+ *
+ *  - `settings.evaluation.agentEvalsEnabled` — the backend-provided operator
+ *    override (`N8N_AGENT_EVALS_ENABLED`). Delivered in the settings payload,
+ *    so it works even when the in-browser PostHog client never initializes
+ *    (telemetry off), where the flag would otherwise stay false.
+ *  - the PostHog client flag — carries per-cohort rollout when telemetry is on.
+ *
+ * Coerces to a strict boolean so `v-if="isAgentEvalsEnabled"` never
+ * undefined-flickers during the initial flag-fetch frame.
+ */
+export const useAgentEvalsFlag = () => {
+	const postHog = usePostHog();
+	const settingsStore = useSettingsStore();
+	return computed(
+		() =>
+			settingsStore.settings.evaluation?.agentEvalsEnabled === true ||
+			postHog.isFeatureEnabled(AGENT_EVALS_FLAG) === true,
+	);
+};