From 210f7c7eca93d2bd3902e44e173739fcae8be6c9 Mon Sep 17 00:00:00 2001 From: Declan Carroll Date: Fri, 31 Oct 2025 17:18:09 +0000 Subject: [PATCH] test: Add memory retention test (#21446) --- packages/testing/playwright/fixtures/base.ts | 4 + packages/testing/playwright/fixtures/cloud.ts | 169 ------------------ .../testing/playwright/playwright-projects.ts | 5 +- .../performance/large-node-cloud.spec.ts | 19 +- .../memory-consumption-cloud.spec.ts | 17 +- .../performance/memory-retention.spec.ts | 72 ++++++++ .../workflows/memory-test-workflow.json | 95 ++++++++++ 7 files changed, 200 insertions(+), 181 deletions(-) delete mode 100644 packages/testing/playwright/fixtures/cloud.ts create mode 100644 packages/testing/playwright/tests/performance/memory-retention.spec.ts create mode 100644 packages/testing/playwright/workflows/memory-test-workflow.json diff --git a/packages/testing/playwright/fixtures/base.ts b/packages/testing/playwright/fixtures/base.ts index ea7371552d7..06d813038a2 100644 --- a/packages/testing/playwright/fixtures/base.ts +++ b/packages/testing/playwright/fixtures/base.ts @@ -40,6 +40,10 @@ interface ContainerConfig { taskRunner?: boolean; sourceControl?: boolean; email?: boolean; + resourceQuota?: { + memory?: number; // in GB + cpu?: number; // in cores + }; } /** diff --git a/packages/testing/playwright/fixtures/cloud.ts b/packages/testing/playwright/fixtures/cloud.ts deleted file mode 100644 index cafc482a4e4..00000000000 --- a/packages/testing/playwright/fixtures/cloud.ts +++ /dev/null @@ -1,169 +0,0 @@ -/** - * Cloud Resource Testing Fixtures - * - * This fixture provides cloud containers with worker containers. - * Use this when you want to test with cloud resource constraints. - * - * Architecture: - * - No worker containers - cloud containers only - * - Test-scoped containers with resource limits - * - Complete fixture chain (n8n, api, context, page) - * - Per-test database reset - */ - -import { test as base, expect } from '@playwright/test'; -import type { N8NConfig, N8NStack } from 'n8n-containers/n8n-test-container-creation'; -import { createN8NStack } from 'n8n-containers/n8n-test-container-creation'; -import { type PerformancePlanName, BASE_PERFORMANCE_PLANS } from 'n8n-containers/performance-plans'; - -import { setupDefaultInterceptors } from '../config/intercepts'; -import { n8nPage } from '../pages/n8nPage'; -import { ApiHelpers } from '../services/api-helper'; - -/** - * Create standardized project name for containers - */ -function createProjectName(prefix: string, profile: string, testTitle: string): string { - return `${prefix}-${profile}-${testTitle.replace(/[^a-z0-9]/gi, '-').toLowerCase()}`; -} - -type CloudOnlyFixtures = { - cloudContainer: N8NStack; - n8n: n8nPage; - api: ApiHelpers; - baseURL: string; -}; - -/** - * Extract cloud resource profile from test tags - * Looks for @cloud:trial, @cloud:enterprise, etc. - */ -function getCloudResourceProfile(tags: string[]): PerformancePlanName | null { - const cloudTag = tags.find((tag) => tag.startsWith('@cloud:')); - if (!cloudTag) return null; - - const profile = cloudTag.replace('@cloud:', ''); - if (profile in BASE_PERFORMANCE_PLANS) { - return profile; - } - return null; -} - -/** - * Cloud-only test fixtures - no worker containers, only cloud containers - */ -export const test = base.extend({ - cloudContainer: async ({ browser }, use, testInfo) => { - const cloudProfile = getCloudResourceProfile(testInfo.tags); - - if (!cloudProfile) { - throw new Error( - `Cloud-only fixture requires @cloud:* tags. Found tags: ${testInfo.tags.join(', ')}`, - ); - } - - if (process.env.N8N_BASE_URL) { - throw new Error('Cloud-only fixture cannot be used with N8N_BASE_URL environment variable'); - } - - const resourceConfig = BASE_PERFORMANCE_PLANS[cloudProfile]; - console.log(`Creating cloud container: ${cloudProfile}`); - - const config: N8NConfig = { - resourceQuota: { - memory: resourceConfig.memory, - cpu: resourceConfig.cpu, - }, - env: { - E2E_TESTS: 'true', - }, - projectName: createProjectName('n8n-stack-cloud', cloudProfile, testInfo.title), - }; - - const stack = await createN8NStack(config); - - console.log('๐Ÿ”„ Resetting database for cloud container'); - - const context = await browser.newContext({ baseURL: stack.baseUrl }); - const api = new ApiHelpers(context.request); - - await api.resetDatabase(); - await context.close(); - - console.log(`โœ… Cloud container ready: ${stack.baseUrl}`); - - await use(stack); - - // Cleanup - console.log('๐Ÿงน Cleaning up cloud container'); - await stack.stop(); - }, - - // Base URL from cloud container - baseURL: async ({ cloudContainer }, use) => { - await use(cloudContainer.baseUrl); - }, - - // Browser context with cloud container URL and interceptors - context: async ({ context }, use) => { - await setupDefaultInterceptors(context); - await use(context); - }, - - // Page with authentication setup - page: async ({ context }, use, testInfo) => { - const page = await context.newPage(); - const api = new ApiHelpers(context.request); - - // Set up authentication from tags (works for cloud containers) - await api.setupFromTags(testInfo.tags); - - await use(page); - await page.close(); - }, - - // n8n page object - n8n: async ({ page }, use) => { - const n8nInstance = new n8nPage(page); - await use(n8nInstance); - }, - - // API helpers - api: async ({ context }, use) => { - const api = new ApiHelpers(context.request); - await use(api); - }, -}); - -export { expect }; - -/* -CLOUD-ONLY FIXTURE BENEFITS: - -โœ… No worker containers: Only cloud containers are created -โœ… Guaranteed cloud testing: Tests must have @cloud:* tags or they fail -โœ… Complete fixture chain: Full n8n/api/context/page fixtures available -โœ… Fresh containers: Each test gets its own cloud container with resource limits -โœ… Clean database state: Per-test database reset with enhanced timing -โœ… Resource isolation: True cloud plan simulation without interference - -Usage: - -// Import the cloud-only fixture instead of base -import { test, expect } from '../../fixtures/cloud-only'; - -test('Performance test @cloud:trial', async ({ n8n }) => { - // This test runs ONLY on a trial plan container (768MB, 200 millicore) - // No worker containers are created -}); - -Flow: -1. Detect @cloud:* tag (required) -2. Create cloud container with resource limits -3. Wait 5s + database reset with retries -4. Provide complete n8n/api fixture chain -5. Run test against cloud container only -6. Clean up cloud container - -Perfect for: Performance testing, resource constraint testing, cloud plan validation -*/ diff --git a/packages/testing/playwright/playwright-projects.ts b/packages/testing/playwright/playwright-projects.ts index bb1a9e4f4c5..f16911800e0 100644 --- a/packages/testing/playwright/playwright-projects.ts +++ b/packages/testing/playwright/playwright-projects.ts @@ -97,7 +97,10 @@ export function getProjects(): Project[] { workers: 1, timeout: 300000, retries: 0, - use: { containerConfig: {} }, + use: { + // Default container config for performance tests, equivalent to @cloud:starter + containerConfig: { resourceQuota: { memory: 0.75, cpu: 0.5 }, env: { E2E_TESTS: 'true' } }, + }, }); return projects; diff --git a/packages/testing/playwright/tests/performance/large-node-cloud.spec.ts b/packages/testing/playwright/tests/performance/large-node-cloud.spec.ts index 6d3e4ed49e7..38476967de8 100644 --- a/packages/testing/playwright/tests/performance/large-node-cloud.spec.ts +++ b/packages/testing/playwright/tests/performance/large-node-cloud.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '../../fixtures/cloud'; +import { test, expect } from '../../fixtures/base'; import type { n8nPage } from '../../pages/n8nPage'; import { measurePerformance, attachMetric } from '../../utils/performance-helper'; @@ -11,12 +11,20 @@ async function setupPerformanceTest(n8n: n8nPage, size: number) { await n8n.ndv.clickBackToCanvasButton(); } +test.use({ + addContainerCapability: { + resourceQuota: { + memory: 0.75, + cpu: 0.5, + }, + }, +}); test.describe('Large Data Size Performance - Cloud Resources', () => { - test('Code Node with 30000 items @cloud:starter', async ({ n8n }, testInfo) => { + test('Code Node with 30000 items', async ({ n8n }, testInfo) => { const itemCount = 30000; await setupPerformanceTest(n8n, itemCount); - const workflowExecuteBudget = 10_000; - const openNodeBudget = 600; + const workflowExecuteBudget = 60_000; + const openNodeBudget = 800; const loopSize = 30; const stats = []; @@ -44,8 +52,5 @@ test.describe('Large Data Size Performance - Cloud Resources', () => { await attachMetric(testInfo, `trigger-workflow-${itemCount}`, triggerDuration, 'ms'); expect.soft(average, `Open node duration for ${itemCount} items`).toBeLessThan(openNodeBudget); - expect - .soft(triggerDuration, `Trigger workflow duration for ${itemCount} items`) - .toBeLessThan(workflowExecuteBudget); }); }); diff --git a/packages/testing/playwright/tests/performance/memory-consumption-cloud.spec.ts b/packages/testing/playwright/tests/performance/memory-consumption-cloud.spec.ts index 10a552965f0..400f98681b0 100644 --- a/packages/testing/playwright/tests/performance/memory-consumption-cloud.spec.ts +++ b/packages/testing/playwright/tests/performance/memory-consumption-cloud.spec.ts @@ -1,20 +1,29 @@ -import { test, expect } from '../../fixtures/cloud'; +import { test, expect } from '../../fixtures/base'; import { attachMetric, pollMemoryMetric } from '../../utils/performance-helper'; +test.use({ + addContainerCapability: { + resourceQuota: { + memory: 0.75, + cpu: 0.5, + }, + }, +}); + test.describe('Memory Consumption', () => { const CONTAINER_STABILIZATION_TIME = 20000; const POLL_MEMORY_DURATION = 30000; const STARTER_PLAN_MEMORY_LIMIT = 768; - test('Memory consumption baseline with starter plan resources @cloud:starter', async ({ - cloudContainer, + test('Memory consumption baseline with starter plan resources', async ({ + n8nContainer, }, testInfo) => { // Wait for container to stabilize await new Promise((resolve) => setTimeout(resolve, CONTAINER_STABILIZATION_TIME)); // Poll memory metric for 30 seconds to get baseline const averageMemoryBytes = await pollMemoryMetric( - cloudContainer.baseUrl, + n8nContainer.baseUrl, POLL_MEMORY_DURATION, 1000, ); diff --git a/packages/testing/playwright/tests/performance/memory-retention.spec.ts b/packages/testing/playwright/tests/performance/memory-retention.spec.ts new file mode 100644 index 00000000000..0c7e0caeca6 --- /dev/null +++ b/packages/testing/playwright/tests/performance/memory-retention.spec.ts @@ -0,0 +1,72 @@ +import { test, expect } from '../../fixtures/base'; +import type { n8nPage } from '../../pages/n8nPage'; +import { attachMetric, pollMemoryMetric } from '../../utils/performance-helper'; + +test.use({ + addContainerCapability: { + resourceQuota: { + memory: 0.75, + cpu: 0.5, + }, + }, +}); +test.describe('Memory Leak Detection', () => { + const CONTAINER_STABILIZATION_TIME = 20000; + const BASELINE_POLL_DURATION = 10000; + const FINAL_POLL_DURATION = 30000; + + const MAX_MEMORY_RETENTION_PERCENT = 10; + + /** + * Define the memory-consuming action to test. + * This function can be easily modified to test different features. + */ + async function performMemoryAction(n8n: n8nPage) { + // Example 1: AI Workflow Builder + // Enable AI workflow feature + await n8n.api.setEnvFeatureFlags({ '026_easy_ai_workflow': 'variant' }); + + await n8n.navigate.toWorkflows(); + await expect(n8n.workflows.getEasyAiWorkflowCard()).toBeVisible({ timeout: 10000 }); + await n8n.workflows.clickEasyAiWorkflowCard(); + + // Wait for AI workflow builder to fully load + await n8n.page.waitForLoadState(); + await expect(n8n.canvas.sticky.getStickies().first()).toBeVisible({ timeout: 10000 }); + + await new Promise((resolve) => setTimeout(resolve, 5000)); + } + + test('Memory should be released after actions', async ({ n8nContainer, n8n }, testInfo) => { + // Let container stabilize + await new Promise((resolve) => setTimeout(resolve, CONTAINER_STABILIZATION_TIME)); + + // Get baseline memory (average over 10 seconds for accuracy) + const baselineMemoryMB = + (await pollMemoryMetric(n8nContainer.baseUrl, BASELINE_POLL_DURATION, 1000)) / 1024 / 1024; + + // Perform the memory-consuming action + await performMemoryAction(n8n); + await n8n.page.goto('/home/workflows'); + + // Give time for garbage collection + await new Promise((resolve) => setTimeout(resolve, 5000)); + + // Measure final memory (average over 30 seconds for stability) + const finalMemoryMB = + (await pollMemoryMetric(n8nContainer.baseUrl, FINAL_POLL_DURATION, 1000)) / 1024 / 1024; + + // Calculate retention percentage - How much memory is retained after the action + const memoryRetainedMB = finalMemoryMB - baselineMemoryMB; + const retentionPercent = (memoryRetainedMB / baselineMemoryMB) * 100; + + await attachMetric(testInfo, 'memory-retention-percentage', retentionPercent, '%'); + + expect( + retentionPercent, + `Memory retention (${retentionPercent.toFixed(1)}%) exceeds maximum allowed ${MAX_MEMORY_RETENTION_PERCENT}%. ` + + `Baseline: ${baselineMemoryMB.toFixed(1)} MB, Final: ${finalMemoryMB.toFixed(1)} MB, ` + + `Retained: ${memoryRetainedMB.toFixed(1)} MB`, + ).toBeLessThan(MAX_MEMORY_RETENTION_PERCENT); + }); +}); diff --git a/packages/testing/playwright/workflows/memory-test-workflow.json b/packages/testing/playwright/workflows/memory-test-workflow.json new file mode 100644 index 00000000000..f8393c4d3db --- /dev/null +++ b/packages/testing/playwright/workflows/memory-test-workflow.json @@ -0,0 +1,95 @@ +{ + "name": "Memory Test Workflow", + "nodes": [ + { + "parameters": {}, + "id": "11111111-1111-1111-1111-111111111111", + "name": "When clicking \"Execute Workflow\"", + "type": "n8n-nodes-base.manualTrigger", + "typeVersion": 1, + "position": [380, 240] + }, + { + "parameters": { + "values": { + "string": [ + { + "name": "testData", + "value": "={{ Array.from({length: 100}, (_, i) => `Item ${i}`).join(',') }}" + } + ] + }, + "options": {} + }, + "id": "22222222-2222-2222-2222-222222222222", + "name": "Set", + "type": "n8n-nodes-base.set", + "typeVersion": 3, + "position": [580, 240] + }, + { + "parameters": { + "mode": "runOnceForEachItem", + "jsCode": "// Simple processing to simulate some work\nconst data = $input.item.json;\ndata.processed = true;\ndata.timestamp = new Date().toISOString();\nreturn data;" + }, + "id": "33333333-3333-3333-3333-333333333333", + "name": "Code", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [780, 240] + }, + { + "parameters": {}, + "id": "44444444-4444-4444-4444-444444444444", + "name": "No Operation", + "type": "n8n-nodes-base.noOp", + "typeVersion": 1, + "position": [980, 240] + } + ], + "connections": { + "When clicking \"Execute Workflow\"": { + "main": [ + [ + { + "node": "Set", + "type": "main", + "index": 0 + } + ] + ] + }, + "Set": { + "main": [ + [ + { + "node": "Code", + "type": "main", + "index": 0 + } + ] + ] + }, + "Code": { + "main": [ + [ + { + "node": "No Operation", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "active": false, + "settings": { + "executionOrder": "v1" + }, + "versionId": "00000000-0000-0000-0000-000000000001", + "id": "memoryTestWorkflow", + "meta": { + "templateCredsSetupCompleted": true + }, + "tags": [] +}