diff --git a/packages/testing/performance/README.md b/packages/testing/performance/README.md new file mode 100644 index 00000000000..b21061c8603 --- /dev/null +++ b/packages/testing/performance/README.md @@ -0,0 +1,144 @@ +# Performance Benchmarks + +Microbenchmarks for measuring and tracking performance of critical code paths. + +## When to Use Benchmarks + +**Good fit:** +- Hot paths executed thousands of times (expression evaluation, data transforms) +- Comparing implementation approaches (current vs proposed) +- Detecting regressions in critical code + +**Not a good fit:** +- API endpoint latency (use load testing - k6, artillery) +- Database query performance (use query analysis tools) +- Frontend rendering (use browser profiling) +- One-off operations (startup time, migrations) + +**Rule of thumb:** If it runs millions of times per day across all users, benchmark it. + +## Commands + +```bash +pnpm --filter=@n8n/performance bench # Run benchmarks +pnpm --filter=@n8n/performance bench:baseline # Save new baseline +pnpm --filter=@n8n/performance bench:ci # CI check (fails if >10% slower) +``` + +## Adding a Benchmark + +### 1. Create a bench file + +```typescript +// benchmarks/my-feature/thing.bench.ts +import { bench, describe } from 'vitest'; + +describe('My Feature', () => { + bench('operation name', () => { + // Code to measure - runs thousands of times + doTheThing(); + }); +}); +``` + +### 2. Add setup outside the bench function + +```typescript +// Setup runs once, not measured +const data = createTestData(); +const instance = new MyClass(); + +describe('My Feature', () => { + bench('with small input', () => { + instance.process(data.small); + }); + + bench('with large input', () => { + instance.process(data.large); + }); +}); +``` + +### 3. Add warmup if needed + +```typescript +// Warmup ensures JIT compilation is done before measuring +for (let i = 0; i < 1000; i++) { + instance.process(data.small); +} + +describe('My Feature', () => { + // Now benchmarks measure hot path, not JIT compilation +}); +``` + +## Reading Results + +``` +name hz min max mean p99 rme samples +my operation 20,000 0.04 0.20 0.05 0.10 ±0.5% 10000 +``` + +| Column | Meaning | +|--------|---------| +| hz | Operations per second (higher = faster) | +| mean | Average time per operation in ms | +| p99 | 99th percentile - worst case latency | +| rme | Margin of error - lower = more reliable | +| samples | Number of iterations run | + +## Regression Detection + +Benchmarks are compared against a saved baseline: + +- **>10% slower** = regression (CI fails) +- **>10% faster** = improvement (consider updating baseline) + +### Local Workflow + +```bash +# 1. Before making changes, save a baseline +pnpm --filter=@n8n/performance bench:baseline + +# 2. Make your changes/refactors + +# 3. Check for regressions +pnpm --filter=@n8n/performance bench:ci +``` + +### After Intentional Improvements + +```bash +# Save new baseline to reflect the improvement +pnpm --filter=@n8n/performance bench:baseline +``` + +## Current Benchmarks + +| Area | What it measures | Why it matters | +|------|------------------|----------------| +| Expression Engine | `={{ }}` evaluation speed | Runs for every node parameter | + +## Current Status + +This is a proof-of-concept for local regression detection. + +### CI Integration (TODO) + +Baselines are hardware-specific (an 8-core MacBook baseline is meaningless on a 2-core runner). CI needs its own baseline management: + +- **Option A:** Store baselines as CI artifacts, restore before comparison +- **Option B:** External storage (S3, dedicated benchmark service) +- **Option C:** Compare against previous CI run on same runner type + +## Known Limitations + +- **Local noise**: Background processes affect results. Run multiple times to verify. +- **Baselines are machine-specific**: Cannot commit baselines to git - they must be generated on the same hardware they'll be compared against. + +## Tips + +1. **Keep benchmarks focused** - one thing per bench, not workflows +2. **Use realistic data sizes** - 100 items is typical, 10k is stress test +3. **Compare approaches** - benchmark both before deciding +4. **Don't over-benchmark** - only critical hot paths need this diff --git a/packages/testing/performance/benchmarks/expression-engine/evaluation.bench.ts b/packages/testing/performance/benchmarks/expression-engine/evaluation.bench.ts new file mode 100644 index 00000000000..b375f803125 --- /dev/null +++ b/packages/testing/performance/benchmarks/expression-engine/evaluation.bench.ts @@ -0,0 +1,156 @@ +/** + * Expression Engine Benchmarks + * + * Answers: "What's the baseline performance of expression evaluation?" + * + * These benchmarks establish the hot-path performance for comparing + * alternative implementations (WASM sandbox, quickjs, etc.) + * + * Run: pnpm --filter=@n8n/performance bench + */ +import { bench, describe } from 'vitest'; +import { Workflow } from 'n8n-workflow'; +import type { INodeTypes, INodeType, INodeTypeDescription } from 'n8n-workflow'; + +// Minimal node types implementation for workflow instantiation +class TestNodeTypes implements INodeTypes { + getByName(nodeType: string): INodeType { + return { + description: { + name: nodeType, + displayName: 'Test', + group: ['transform'], + version: 1, + defaults: { name: 'Test' }, + inputs: ['main'], + outputs: ['main'], + properties: [], + description: '', + } as INodeTypeDescription, + execute: async () => [[{ json: {} }]], + }; + } + + getByNameAndVersion(): INodeType { + return this.getByName('test.set'); + } + + getKnownTypes(): Record> { + return {}; + } +} + +// Shared workflow instance (simulates production reuse) +const nodeTypes = new TestNodeTypes(); +const workflow = new Workflow({ + id: '1', + nodes: [ + { + name: 'node', + typeVersion: 1, + type: 'test.set', + id: 'uuid-1234', + position: [0, 0], + parameters: {}, + }, + ], + connections: {}, + active: false, + nodeTypes, +}); + +// Factory for fresh workflow instances +const createWorkflow = () => + new Workflow({ + id: '1', + nodes: [ + { + name: 'node', + typeVersion: 1, + type: 'test.set', + id: 'uuid-1234', + position: [0, 0], + parameters: {}, + }, + ], + connections: {}, + active: false, + nodeTypes: new TestNodeTypes(), + }); + +// Test data +const smallData = [ + { + json: { + name: 'test-user', + email: 'test@example.com', + items: Array(100) + .fill(null) + .map((_, i) => ({ id: i, value: i * 10, active: i % 2 === 0 })), + }, + }, +]; + +const largeData = [ + { + json: { + name: 'test-user', + items: Array(10000) + .fill(null) + .map((_, i) => ({ id: i, value: i * 10, active: i % 2 === 0 })), + }, + }, +]; + +const evaluate = (expr: string, data: typeof smallData | typeof largeData) => + workflow.expression.getParameterValue(expr, null, 0, 0, 'node', data, 'manual', {}); + +describe('Hot Path', () => { + // Baseline: simplest possible expression + bench('simple property access', () => { + evaluate('={{ $json.name }}', smallData); + }); + + // Typical: array transform + bench('array map (100 items)', () => { + evaluate('={{ $json.items.map(i => i.value) }}', smallData); + }); + + // Complex: chained operations + bench('method chain', () => { + evaluate('={{ $json.items.filter(i => i.active).map(i => i.id) }}', smallData); + }); +}); + +describe('Cold Start', () => { + // Answers: "What's the WASM sandbox init cost comparison?" + bench('first evaluation (fresh workflow)', () => { + const fresh = createWorkflow(); + fresh.expression.getParameterValue( + '={{ $json.name }}', + null, + 0, + 0, + 'node', + smallData, + 'manual', + {}, + ); + }); + + // Answers: "Should we pool expression workers?" + bench('reused workflow', () => { + evaluate('={{ $json.name }}', smallData); + }); +}); + +describe('Data Transfer', () => { + // Answers: "What's the overhead of data moving between wasm and node?" + bench('small context (100 items)', () => { + evaluate('={{ $json.items.map(i => i.id) }}', smallData); + }); + + bench('large context (10k items)', () => { + evaluate('={{ $json.items.map(i => i.id) }}', largeData); + }); +}); diff --git a/packages/testing/performance/package.json b/packages/testing/performance/package.json new file mode 100644 index 00000000000..48eef629f67 --- /dev/null +++ b/packages/testing/performance/package.json @@ -0,0 +1,15 @@ +{ + "name": "@n8n/performance", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "bench": "vitest bench --run", + "bench:baseline": "node scripts/save-baseline.mjs", + "bench:ci": "vitest bench --run --outputJson ./profiles/benchmark-results.json && node scripts/check-regression.mjs" + }, + "devDependencies": { + "vitest": "catalog:", + "n8n-workflow": "workspace:*" + } +} diff --git a/packages/testing/performance/profiles/.gitignore b/packages/testing/performance/profiles/.gitignore new file mode 100644 index 00000000000..4c84eba28ba --- /dev/null +++ b/packages/testing/performance/profiles/.gitignore @@ -0,0 +1,3 @@ +# Generated by benchmarks - CI manages baselines +baseline.json +benchmark-results.json diff --git a/packages/testing/performance/scripts/check-regression.mjs b/packages/testing/performance/scripts/check-regression.mjs new file mode 100644 index 00000000000..2e353f01c50 --- /dev/null +++ b/packages/testing/performance/scripts/check-regression.mjs @@ -0,0 +1,107 @@ +#!/usr/bin/env node +/** + * Benchmark Regression Checker + * + * Compares current benchmark results against baseline and fails if any + * benchmark regresses beyond the threshold. + * + * Exit codes: + * 0 = All benchmarks within threshold + * 1 = Regression detected + */ + +import { readFileSync, existsSync } from 'fs'; +import { resolve, dirname } from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const PROFILES_DIR = resolve(__dirname, '../profiles'); +const THRESHOLD = 0.10; // 10% + +const BASELINE_PATH = resolve(PROFILES_DIR, 'baseline.json'); +const CURRENT_PATH = resolve(PROFILES_DIR, 'benchmark-results.json'); + +if (!existsSync(BASELINE_PATH)) { + console.error('❌ No baseline found. Run: pnpm bench:baseline'); + process.exit(1); +} + +if (!existsSync(CURRENT_PATH)) { + console.error('❌ No current results found. Run bench:ci to generate them.'); + process.exit(1); +} + +const baseline = JSON.parse(readFileSync(BASELINE_PATH, 'utf-8')); +const current = JSON.parse(readFileSync(CURRENT_PATH, 'utf-8')); + +// Build lookup map from baseline +const baselineMap = new Map(); +for (const file of baseline.files) { + for (const group of file.groups) { + for (const bench of group.benchmarks) { + baselineMap.set(`${group.fullName}::${bench.name}`, bench); + } + } +} + +// Compare results +const results = []; +let hasRegression = false; + +for (const file of current.files) { + for (const group of file.groups) { + for (const bench of group.benchmarks) { + const key = `${group.fullName}::${bench.name}`; + const base = baselineMap.get(key); + + if (!base) { + results.push({ name: bench.name, status: 'new', current: bench.hz, baseline: null, ratio: null }); + continue; + } + + const ratio = bench.hz / base.hz; + const isRegression = ratio < (1 - THRESHOLD); + const isImprovement = ratio > (1 + THRESHOLD); + + if (isRegression) hasRegression = true; + + results.push({ + name: bench.name, + status: isRegression ? 'regression' : isImprovement ? 'improved' : 'ok', + current: bench.hz, + baseline: base.hz, + ratio, + }); + } + } +} + +// Print results +console.log(`\nBenchmark Comparison (±${(THRESHOLD * 100).toFixed(0)}% threshold)\n`); +console.log(''.padEnd(70, '─')); + +for (const r of results) { + const icon = r.status === 'regression' ? '❌' : r.status === 'improved' ? '✅' : r.status === 'new' ? '🆕' : ' '; + const changeStr = r.ratio !== null ? `${((r.ratio - 1) * 100).toFixed(1)}%` : 'new'; + const currentStr = r.current.toFixed(0).padStart(8); + const baselineStr = r.baseline !== null ? r.baseline.toFixed(0).padStart(8) : ' N/A'; + + console.log(`${icon} ${r.name.padEnd(35)} ${currentStr} hz (was ${baselineStr}) ${changeStr.padStart(7)}`); +} + +console.log(''.padEnd(70, '─')); + +const regressions = results.filter(r => r.status === 'regression'); +const improvements = results.filter(r => r.status === 'improved'); + +if (hasRegression) { + console.log(`\n❌ FAILED: ${regressions.length} regression(s) exceeded ${(THRESHOLD * 100).toFixed(0)}% threshold\n`); + process.exit(1); +} else { + console.log(`\n✅ PASSED: All benchmarks within threshold`); + if (improvements.length > 0) { + console.log(` ${improvements.length} improved - consider updating baseline with: pnpm bench:baseline`); + } + console.log(''); + process.exit(0); +} diff --git a/packages/testing/performance/scripts/save-baseline.mjs b/packages/testing/performance/scripts/save-baseline.mjs new file mode 100644 index 00000000000..76025c31587 --- /dev/null +++ b/packages/testing/performance/scripts/save-baseline.mjs @@ -0,0 +1,49 @@ +#!/usr/bin/env node +/** + * Save Baseline + * + * Runs benchmarks and saves results as the new baseline for regression detection. + * Sanitizes absolute paths so baseline can be committed. + */ + +import { readFileSync, writeFileSync, existsSync } from 'fs'; +import { resolve, dirname } from 'path'; +import { fileURLToPath } from 'url'; +import { execSync } from 'child_process'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const PROFILES_DIR = resolve(__dirname, '../profiles'); +const PACKAGE_DIR = resolve(__dirname, '..'); + +console.log('Running benchmarks...\n'); + +try { + execSync('pnpm vitest bench --run --outputJson ./profiles/benchmark-results.json', { + cwd: PACKAGE_DIR, + stdio: 'inherit', + }); +} catch { + console.error('\n❌ Benchmark run failed'); + process.exit(1); +} + +const resultsPath = resolve(PROFILES_DIR, 'benchmark-results.json'); +const baselinePath = resolve(PROFILES_DIR, 'baseline.json'); + +if (!existsSync(resultsPath)) { + console.error('\n❌ No benchmark results found'); + process.exit(1); +} + +// Load and sanitize paths +const results = JSON.parse(readFileSync(resultsPath, 'utf-8')); + +for (const file of results.files) { + // Convert absolute path to relative + if (file.filepath) { + file.filepath = file.filepath.replace(/^.*\/benchmarks\//, 'benchmarks/'); + } +} + +writeFileSync(baselinePath, JSON.stringify(results, null, '\t')); +console.log('\n✅ Saved baseline.json (paths sanitized)'); diff --git a/packages/testing/performance/vitest.config.ts b/packages/testing/performance/vitest.config.ts new file mode 100644 index 00000000000..4e8f047dc33 --- /dev/null +++ b/packages/testing/performance/vitest.config.ts @@ -0,0 +1,17 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + benchmark: { + include: ['benchmarks/**/*.bench.ts'], + // Run each benchmark longer for more stable results + // Default is 500ms - we use 1000ms for ~2x more samples + time: 1000, + // Warmup: ensure JIT compilation is complete before measuring + // Default is 5 iterations - we use 100 for more thorough warmup + warmupIterations: 100, + // Default warmup time is 100ms - we use 500ms for stability + warmupTime: 500, + }, + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 429e3741a9b..4234a38b169 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3627,6 +3627,15 @@ importers: specifier: ^11.11.0 version: 11.11.0 + packages/testing/performance: + devDependencies: + n8n-workflow: + specifier: workspace:* + version: link:../../workflow + vitest: + specifier: 'catalog:' + version: 3.1.3(@types/debug@4.1.12)(@types/node@20.19.21)(jiti@2.6.1)(jsdom@23.0.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(lightningcss@1.30.2)(sass@1.89.2)(terser@5.16.1)(tsx@4.19.3) + packages/testing/playwright: devDependencies: '@currents/playwright': @@ -31266,7 +31275,7 @@ snapshots: '@types/debug': 4.1.12 '@types/node': 20.19.21 '@types/tough-cookie': 4.0.5 - axios: 1.12.0(debug@4.4.3) + axios: 1.12.0 camelcase: 6.3.0 debug: 4.4.3(supports-color@8.1.1) dotenv: 16.6.1 @@ -31276,7 +31285,7 @@ snapshots: isstream: 0.1.2 jsonwebtoken: 9.0.3 mime-types: 2.1.35 - retry-axios: 2.6.0(axios@1.12.0) + retry-axios: 2.6.0(axios@1.12.0(debug@4.4.3)) tough-cookie: 4.1.4 transitivePeerDependencies: - supports-color @@ -35766,7 +35775,7 @@ snapshots: onetime: 5.1.2 signal-exit: 3.0.7 - retry-axios@2.6.0(axios@1.12.0): + retry-axios@2.6.0(axios@1.12.0(debug@4.4.3)): dependencies: axios: 1.12.0