mirror of
https://github.com/n8n-io/n8n.git
synced 2026-08-29 01:39:24 +08:00
test(benchmark): Add expression engine performance benchmarks (no-changelog) (#24199)
Co-authored-by: Danny Martini <danny@n8n.io>
This commit is contained in:
Declan Carroll
@@ -0,0 +1,144 @@
|
||||
# Performance Benchmarks
|
||||
|
||||
Microbenchmarks for measuring and tracking performance of critical code paths.
|
||||
|
||||
## When to Use Benchmarks
|
||||
|
||||
**Good fit:**
|
||||
- Hot paths executed thousands of times (expression evaluation, data transforms)
|
||||
- Comparing implementation approaches (current vs proposed)
|
||||
- Detecting regressions in critical code
|
||||
|
||||
**Not a good fit:**
|
||||
- API endpoint latency (use load testing - k6, artillery)
|
||||
- Database query performance (use query analysis tools)
|
||||
- Frontend rendering (use browser profiling)
|
||||
- One-off operations (startup time, migrations)
|
||||
|
||||
**Rule of thumb:** If it runs millions of times per day across all users, benchmark it.
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
pnpm --filter=@n8n/performance bench # Run benchmarks
|
||||
pnpm --filter=@n8n/performance bench:baseline # Save new baseline
|
||||
pnpm --filter=@n8n/performance bench:ci # CI check (fails if >10% slower)
|
||||
```
|
||||
|
||||
## Adding a Benchmark
|
||||
|
||||
### 1. Create a bench file
|
||||
|
||||
```typescript
|
||||
// benchmarks/my-feature/thing.bench.ts
|
||||
import { bench, describe } from 'vitest';
|
||||
|
||||
describe('My Feature', () => {
|
||||
bench('operation name', () => {
|
||||
// Code to measure - runs thousands of times
|
||||
doTheThing();
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### 2. Add setup outside the bench function
|
||||
|
||||
```typescript
|
||||
// Setup runs once, not measured
|
||||
const data = createTestData();
|
||||
const instance = new MyClass();
|
||||
|
||||
describe('My Feature', () => {
|
||||
bench('with small input', () => {
|
||||
instance.process(data.small);
|
||||
});
|
||||
|
||||
bench('with large input', () => {
|
||||
instance.process(data.large);
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### 3. Add warmup if needed
|
||||
|
||||
```typescript
|
||||
// Warmup ensures JIT compilation is done before measuring
|
||||
for (let i = 0; i < 1000; i++) {
|
||||
instance.process(data.small);
|
||||
}
|
||||
|
||||
describe('My Feature', () => {
|
||||
// Now benchmarks measure hot path, not JIT compilation
|
||||
});
|
||||
```
|
||||
|
||||
## Reading Results
|
||||
|
||||
```
|
||||
name hz min max mean p99 rme samples
|
||||
my operation 20,000 0.04 0.20 0.05 0.10 ±0.5% 10000
|
||||
```
|
||||
|
||||
| Column | Meaning |
|
||||
|--------|---------|
|
||||
| hz | Operations per second (higher = faster) |
|
||||
| mean | Average time per operation in ms |
|
||||
| p99 | 99th percentile - worst case latency |
|
||||
| rme | Margin of error - lower = more reliable |
|
||||
| samples | Number of iterations run |
|
||||
|
||||
## Regression Detection
|
||||
|
||||
Benchmarks are compared against a saved baseline:
|
||||
|
||||
- **>10% slower** = regression (CI fails)
|
||||
- **>10% faster** = improvement (consider updating baseline)
|
||||
|
||||
### Local Workflow
|
||||
|
||||
```bash
|
||||
# 1. Before making changes, save a baseline
|
||||
pnpm --filter=@n8n/performance bench:baseline
|
||||
|
||||
# 2. Make your changes/refactors
|
||||
|
||||
# 3. Check for regressions
|
||||
pnpm --filter=@n8n/performance bench:ci
|
||||
```
|
||||
|
||||
### After Intentional Improvements
|
||||
|
||||
```bash
|
||||
# Save new baseline to reflect the improvement
|
||||
pnpm --filter=@n8n/performance bench:baseline
|
||||
```
|
||||
|
||||
## Current Benchmarks
|
||||
|
||||
| Area | What it measures | Why it matters |
|
||||
|------|------------------|----------------|
|
||||
| Expression Engine | `={{ }}` evaluation speed | Runs for every node parameter |
|
||||
|
||||
## Current Status
|
||||
|
||||
This is a proof-of-concept for local regression detection.
|
||||
|
||||
### CI Integration (TODO)
|
||||
|
||||
Baselines are hardware-specific (an 8-core MacBook baseline is meaningless on a 2-core runner). CI needs its own baseline management:
|
||||
|
||||
- **Option A:** Store baselines as CI artifacts, restore before comparison
|
||||
- **Option B:** External storage (S3, dedicated benchmark service)
|
||||
- **Option C:** Compare against previous CI run on same runner type
|
||||
|
||||
## Known Limitations
|
||||
|
||||
- **Local noise**: Background processes affect results. Run multiple times to verify.
|
||||
- **Baselines are machine-specific**: Cannot commit baselines to git - they must be generated on the same hardware they'll be compared against.
|
||||
|
||||
## Tips
|
||||
|
||||
1. **Keep benchmarks focused** - one thing per bench, not workflows
|
||||
2. **Use realistic data sizes** - 100 items is typical, 10k is stress test
|
||||
3. **Compare approaches** - benchmark both before deciding
|
||||
4. **Don't over-benchmark** - only critical hot paths need this
|
||||
@@ -0,0 +1,156 @@
|
||||
/**
|
||||
* Expression Engine Benchmarks
|
||||
*
|
||||
* Answers: "What's the baseline performance of expression evaluation?"
|
||||
*
|
||||
* These benchmarks establish the hot-path performance for comparing
|
||||
* alternative implementations (WASM sandbox, quickjs, etc.)
|
||||
*
|
||||
* Run: pnpm --filter=@n8n/performance bench
|
||||
*/
|
||||
import { bench, describe } from 'vitest';
|
||||
import { Workflow } from 'n8n-workflow';
|
||||
import type { INodeTypes, INodeType, INodeTypeDescription } from 'n8n-workflow';
|
||||
|
||||
// Minimal node types implementation for workflow instantiation
|
||||
class TestNodeTypes implements INodeTypes {
|
||||
getByName(nodeType: string): INodeType {
|
||||
return {
|
||||
description: {
|
||||
name: nodeType,
|
||||
displayName: 'Test',
|
||||
group: ['transform'],
|
||||
version: 1,
|
||||
defaults: { name: 'Test' },
|
||||
inputs: ['main'],
|
||||
outputs: ['main'],
|
||||
properties: [],
|
||||
description: '',
|
||||
} as INodeTypeDescription,
|
||||
execute: async () => [[{ json: {} }]],
|
||||
};
|
||||
}
|
||||
|
||||
getByNameAndVersion(): INodeType {
|
||||
return this.getByName('test.set');
|
||||
}
|
||||
|
||||
getKnownTypes(): Record<string, Record<string, unknown>> {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
// Shared workflow instance (simulates production reuse)
|
||||
const nodeTypes = new TestNodeTypes();
|
||||
const workflow = new Workflow({
|
||||
id: '1',
|
||||
nodes: [
|
||||
{
|
||||
name: 'node',
|
||||
typeVersion: 1,
|
||||
type: 'test.set',
|
||||
id: 'uuid-1234',
|
||||
position: [0, 0],
|
||||
parameters: {},
|
||||
},
|
||||
],
|
||||
connections: {},
|
||||
active: false,
|
||||
nodeTypes,
|
||||
});
|
||||
|
||||
// Factory for fresh workflow instances
|
||||
const createWorkflow = () =>
|
||||
new Workflow({
|
||||
id: '1',
|
||||
nodes: [
|
||||
{
|
||||
name: 'node',
|
||||
typeVersion: 1,
|
||||
type: 'test.set',
|
||||
id: 'uuid-1234',
|
||||
position: [0, 0],
|
||||
parameters: {},
|
||||
},
|
||||
],
|
||||
connections: {},
|
||||
active: false,
|
||||
nodeTypes: new TestNodeTypes(),
|
||||
});
|
||||
|
||||
// Test data
|
||||
const smallData = [
|
||||
{
|
||||
json: {
|
||||
name: 'test-user',
|
||||
email: 'test@example.com',
|
||||
items: Array(100)
|
||||
.fill(null)
|
||||
.map((_, i) => ({ id: i, value: i * 10, active: i % 2 === 0 })),
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const largeData = [
|
||||
{
|
||||
json: {
|
||||
name: 'test-user',
|
||||
items: Array(10000)
|
||||
.fill(null)
|
||||
.map((_, i) => ({ id: i, value: i * 10, active: i % 2 === 0 })),
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const evaluate = (expr: string, data: typeof smallData | typeof largeData) =>
|
||||
workflow.expression.getParameterValue(expr, null, 0, 0, 'node', data, 'manual', {});
|
||||
|
||||
describe('Hot Path', () => {
|
||||
// Baseline: simplest possible expression
|
||||
bench('simple property access', () => {
|
||||
evaluate('={{ $json.name }}', smallData);
|
||||
});
|
||||
|
||||
// Typical: array transform
|
||||
bench('array map (100 items)', () => {
|
||||
evaluate('={{ $json.items.map(i => i.value) }}', smallData);
|
||||
});
|
||||
|
||||
// Complex: chained operations
|
||||
bench('method chain', () => {
|
||||
evaluate('={{ $json.items.filter(i => i.active).map(i => i.id) }}', smallData);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Cold Start', () => {
|
||||
// Answers: "What's the WASM sandbox init cost comparison?"
|
||||
bench('first evaluation (fresh workflow)', () => {
|
||||
const fresh = createWorkflow();
|
||||
fresh.expression.getParameterValue(
|
||||
'={{ $json.name }}',
|
||||
null,
|
||||
0,
|
||||
0,
|
||||
'node',
|
||||
smallData,
|
||||
'manual',
|
||||
{},
|
||||
);
|
||||
});
|
||||
|
||||
// Answers: "Should we pool expression workers?"
|
||||
bench('reused workflow', () => {
|
||||
evaluate('={{ $json.name }}', smallData);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Data Transfer', () => {
|
||||
// Answers: "What's the overhead of data moving between wasm and node?"
|
||||
bench('small context (100 items)', () => {
|
||||
evaluate('={{ $json.items.map(i => i.id) }}', smallData);
|
||||
});
|
||||
|
||||
bench('large context (10k items)', () => {
|
||||
evaluate('={{ $json.items.map(i => i.id) }}', largeData);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"name": "@n8n/performance",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"bench": "vitest bench --run",
|
||||
"bench:baseline": "node scripts/save-baseline.mjs",
|
||||
"bench:ci": "vitest bench --run --outputJson ./profiles/benchmark-results.json && node scripts/check-regression.mjs"
|
||||
},
|
||||
"devDependencies": {
|
||||
"vitest": "catalog:",
|
||||
"n8n-workflow": "workspace:*"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
# Generated by benchmarks - CI manages baselines
|
||||
baseline.json
|
||||
benchmark-results.json
|
||||
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Benchmark Regression Checker
|
||||
*
|
||||
* Compares current benchmark results against baseline and fails if any
|
||||
* benchmark regresses beyond the threshold.
|
||||
*
|
||||
* Exit codes:
|
||||
* 0 = All benchmarks within threshold
|
||||
* 1 = Regression detected
|
||||
*/
|
||||
|
||||
import { readFileSync, existsSync } from 'fs';
|
||||
import { resolve, dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const PROFILES_DIR = resolve(__dirname, '../profiles');
|
||||
const THRESHOLD = 0.10; // 10%
|
||||
|
||||
const BASELINE_PATH = resolve(PROFILES_DIR, 'baseline.json');
|
||||
const CURRENT_PATH = resolve(PROFILES_DIR, 'benchmark-results.json');
|
||||
|
||||
if (!existsSync(BASELINE_PATH)) {
|
||||
console.error('❌ No baseline found. Run: pnpm bench:baseline');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!existsSync(CURRENT_PATH)) {
|
||||
console.error('❌ No current results found. Run bench:ci to generate them.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const baseline = JSON.parse(readFileSync(BASELINE_PATH, 'utf-8'));
|
||||
const current = JSON.parse(readFileSync(CURRENT_PATH, 'utf-8'));
|
||||
|
||||
// Build lookup map from baseline
|
||||
const baselineMap = new Map();
|
||||
for (const file of baseline.files) {
|
||||
for (const group of file.groups) {
|
||||
for (const bench of group.benchmarks) {
|
||||
baselineMap.set(`${group.fullName}::${bench.name}`, bench);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Compare results
|
||||
const results = [];
|
||||
let hasRegression = false;
|
||||
|
||||
for (const file of current.files) {
|
||||
for (const group of file.groups) {
|
||||
for (const bench of group.benchmarks) {
|
||||
const key = `${group.fullName}::${bench.name}`;
|
||||
const base = baselineMap.get(key);
|
||||
|
||||
if (!base) {
|
||||
results.push({ name: bench.name, status: 'new', current: bench.hz, baseline: null, ratio: null });
|
||||
continue;
|
||||
}
|
||||
|
||||
const ratio = bench.hz / base.hz;
|
||||
const isRegression = ratio < (1 - THRESHOLD);
|
||||
const isImprovement = ratio > (1 + THRESHOLD);
|
||||
|
||||
if (isRegression) hasRegression = true;
|
||||
|
||||
results.push({
|
||||
name: bench.name,
|
||||
status: isRegression ? 'regression' : isImprovement ? 'improved' : 'ok',
|
||||
current: bench.hz,
|
||||
baseline: base.hz,
|
||||
ratio,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Print results
|
||||
console.log(`\nBenchmark Comparison (±${(THRESHOLD * 100).toFixed(0)}% threshold)\n`);
|
||||
console.log(''.padEnd(70, '─'));
|
||||
|
||||
for (const r of results) {
|
||||
const icon = r.status === 'regression' ? '❌' : r.status === 'improved' ? '✅' : r.status === 'new' ? '🆕' : ' ';
|
||||
const changeStr = r.ratio !== null ? `${((r.ratio - 1) * 100).toFixed(1)}%` : 'new';
|
||||
const currentStr = r.current.toFixed(0).padStart(8);
|
||||
const baselineStr = r.baseline !== null ? r.baseline.toFixed(0).padStart(8) : ' N/A';
|
||||
|
||||
console.log(`${icon} ${r.name.padEnd(35)} ${currentStr} hz (was ${baselineStr}) ${changeStr.padStart(7)}`);
|
||||
}
|
||||
|
||||
console.log(''.padEnd(70, '─'));
|
||||
|
||||
const regressions = results.filter(r => r.status === 'regression');
|
||||
const improvements = results.filter(r => r.status === 'improved');
|
||||
|
||||
if (hasRegression) {
|
||||
console.log(`\n❌ FAILED: ${regressions.length} regression(s) exceeded ${(THRESHOLD * 100).toFixed(0)}% threshold\n`);
|
||||
process.exit(1);
|
||||
} else {
|
||||
console.log(`\n✅ PASSED: All benchmarks within threshold`);
|
||||
if (improvements.length > 0) {
|
||||
console.log(` ${improvements.length} improved - consider updating baseline with: pnpm bench:baseline`);
|
||||
}
|
||||
console.log('');
|
||||
process.exit(0);
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Save Baseline
|
||||
*
|
||||
* Runs benchmarks and saves results as the new baseline for regression detection.
|
||||
* Sanitizes absolute paths so baseline can be committed.
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync, existsSync } from 'fs';
|
||||
import { resolve, dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { execSync } from 'child_process';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const PROFILES_DIR = resolve(__dirname, '../profiles');
|
||||
const PACKAGE_DIR = resolve(__dirname, '..');
|
||||
|
||||
console.log('Running benchmarks...\n');
|
||||
|
||||
try {
|
||||
execSync('pnpm vitest bench --run --outputJson ./profiles/benchmark-results.json', {
|
||||
cwd: PACKAGE_DIR,
|
||||
stdio: 'inherit',
|
||||
});
|
||||
} catch {
|
||||
console.error('\n❌ Benchmark run failed');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const resultsPath = resolve(PROFILES_DIR, 'benchmark-results.json');
|
||||
const baselinePath = resolve(PROFILES_DIR, 'baseline.json');
|
||||
|
||||
if (!existsSync(resultsPath)) {
|
||||
console.error('\n❌ No benchmark results found');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Load and sanitize paths
|
||||
const results = JSON.parse(readFileSync(resultsPath, 'utf-8'));
|
||||
|
||||
for (const file of results.files) {
|
||||
// Convert absolute path to relative
|
||||
if (file.filepath) {
|
||||
file.filepath = file.filepath.replace(/^.*\/benchmarks\//, 'benchmarks/');
|
||||
}
|
||||
}
|
||||
|
||||
writeFileSync(baselinePath, JSON.stringify(results, null, '\t'));
|
||||
console.log('\n✅ Saved baseline.json (paths sanitized)');
|
||||
@@ -0,0 +1,17 @@
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
benchmark: {
|
||||
include: ['benchmarks/**/*.bench.ts'],
|
||||
// Run each benchmark longer for more stable results
|
||||
// Default is 500ms - we use 1000ms for ~2x more samples
|
||||
time: 1000,
|
||||
// Warmup: ensure JIT compilation is complete before measuring
|
||||
// Default is 5 iterations - we use 100 for more thorough warmup
|
||||
warmupIterations: 100,
|
||||
// Default warmup time is 100ms - we use 500ms for stability
|
||||
warmupTime: 500,
|
||||
},
|
||||
},
|
||||
});
|
||||
Generated
+12
-3
@@ -3627,6 +3627,15 @@ importers:
|
||||
specifier: ^11.11.0
|
||||
version: 11.11.0
|
||||
|
||||
packages/testing/performance:
|
||||
devDependencies:
|
||||
n8n-workflow:
|
||||
specifier: workspace:*
|
||||
version: link:../../workflow
|
||||
vitest:
|
||||
specifier: 'catalog:'
|
||||
version: 3.1.3(@types/debug@4.1.12)(@types/node@20.19.21)(jiti@2.6.1)(jsdom@23.0.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(lightningcss@1.30.2)(sass@1.89.2)(terser@5.16.1)(tsx@4.19.3)
|
||||
|
||||
packages/testing/playwright:
|
||||
devDependencies:
|
||||
'@currents/playwright':
|
||||
@@ -31266,7 +31275,7 @@ snapshots:
|
||||
'@types/debug': 4.1.12
|
||||
'@types/node': 20.19.21
|
||||
'@types/tough-cookie': 4.0.5
|
||||
axios: 1.12.0(debug@4.4.3)
|
||||
axios: 1.12.0
|
||||
camelcase: 6.3.0
|
||||
debug: 4.4.3(supports-color@8.1.1)
|
||||
dotenv: 16.6.1
|
||||
@@ -31276,7 +31285,7 @@ snapshots:
|
||||
isstream: 0.1.2
|
||||
jsonwebtoken: 9.0.3
|
||||
mime-types: 2.1.35
|
||||
retry-axios: 2.6.0(axios@1.12.0)
|
||||
retry-axios: 2.6.0(axios@1.12.0(debug@4.4.3))
|
||||
tough-cookie: 4.1.4
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
@@ -35766,7 +35775,7 @@ snapshots:
|
||||
onetime: 5.1.2
|
||||
signal-exit: 3.0.7
|
||||
|
||||
retry-axios@2.6.0(axios@1.12.0):
|
||||
retry-axios@2.6.0(axios@1.12.0(debug@4.4.3)):
|
||||
dependencies:
|
||||
axios: 1.12.0
|
||||
|
||||
|
||||
Reference in New Issue
Block a user