fix(core): Make workers memory constraints more container aware (#22698)

packages/@n8n/api-types/src/scaling.ts

@@ -13,6 +13,23 @@ export type RunningJobSummary = {
 export type WorkerStatus = {
 	senderId: string;
 	runningJobsSummary: RunningJobSummary[];
+	isInContainer: boolean;
+	process: {
+		memory: {
+			available: number;
+			constraint: number;
+			rss: number;
+			heapTotal: number;
+			heapUsed: number;
+		};
+		uptime: number;
+	};
+	host: {
+		memory: {
+			total: number;
+			free: number;
+		};
+	};
 	freeMem: number;
 	totalMem: number;
 	uptime: number;

packages/cli/src/scaling/worker-status.service.ee.ts

@@ -3,6 +3,7 @@ import { OnPubSubEvent } from '@n8n/decorators';
 import { Service } from '@n8n/di';
 import { InstanceSettings } from 'n8n-core';
 import os from 'node:os';
+import process from 'node:process';
 
 import { N8N_VERSION } from '@/constants';
 import { Push } from '@/push';
@@ -46,9 +47,29 @@ export class WorkerStatusService {
 	}
 
 	private generateStatus(): WorkerStatus {
+		const constrainedMemory = process.constrainedMemory();
+
+		// See https://github.com/nodejs/node/issues/59227 for information about why we cap at MAX_SAFE_INTEGER
+		// The number 18446744073709552000 does come back when running in a container with no constraints
+		const isInContainer = constrainedMemory > 0 && constrainedMemory < Number.MAX_SAFE_INTEGER;
 		return {
 			senderId: this.instanceSettings.hostId,
 			runningJobsSummary: this.jobProcessor.getRunningJobsSummary(),
+			isInContainer,
+			process: {
+				memory: {
+					available: process.availableMemory(),
+					constraint: process.constrainedMemory(),
+					...process.memoryUsage(),
+				},
+				uptime: process.uptime(),
+			},
+			host: {
+				memory: {
+					total: os.totalmem(),
+					free: os.freemem(),
+				},
+			},
 			freeMem: os.freemem(),
 			totalMem: os.totalmem(),
 			uptime: process.uptime(),
@@ -73,6 +94,6 @@ export class WorkerStatusService {
 
 		if (cpus.length === 0) return 'no CPU info';
 
-		return `${cpus.length}x ${cpus[0].model} - speed: ${cpus[0].speed}`;
+		return `${cpus.length}x ${cpus[0].model}`;
 	}
 }

packages/frontend/@n8n/i18n/src/locales/en.json

@@ -1081,6 +1081,7 @@
 	"workerList.item.jobList.empty": "No current jobs",
 	"workerList.item.jobListTitle": "Current Jobs",
 	"workerList.item.netListTitle": "Network Interfaces",
+	"workerList.item.memoryMonitorTitle": "Memory Monitoring",
 	"workerList.item.chartsTitle": "Performance Monitoring",
 	"workerList.item.copyAddressToClipboard": "Address copied to clipboard",
 	"workerList.actionBox.title": "Available on the Enterprise plan",

packages/frontend/editor-ui/src/features/settings/orchestration.ee/components/WorkerCard.vue

@@ -7,6 +7,7 @@ import { averageWorkerLoadFromLoadsAsString, memAsGb } from '../orchestration.ut
 import WorkerJobAccordion from './WorkerJobAccordion.vue';
 import WorkerNetAccordion from './WorkerNetAccordion.vue';
 import WorkerChartsAccordion from './WorkerChartsAccordion.vue';
+import WorkerMemoryMonitorAccordion from './WorkerMemoryMonitorAccordion.vue';
 import { sortByProperty } from '@n8n/utils/sort/sortByProperty';
 import { useI18n } from '@n8n/i18n';
 
@@ -69,8 +70,13 @@ onBeforeUnmount(() => {
 				data-test-id="worker-card-name"
 			>
 				Name: {{ worker.senderId }} ({{ worker.hostname }}) <br />
-				Average Load: {{ averageWorkerLoadFromLoadsAsString(worker.loadAvg ?? [0]) }} | Free Memory:
-				{{ memAsGb(worker.freeMem).toFixed(2) }}GB / {{ memAsGb(worker.totalMem).toFixed(2) }}GB
+				Average Load: {{ averageWorkerLoadFromLoadsAsString(worker.loadAvg ?? [0]) }} | Free memory:
+				{{ memAsGb(worker.process.memory.available) }}GB /
+				{{
+					memAsGb(
+						worker.isInContainer ? worker.process.memory.constraint : worker.host.memory.total,
+					)
+				}}GB
 				{{ stale ? ' (stale)' : '' }}
 			</N8nHeading>
 		</template>
@@ -81,9 +87,11 @@ onBeforeUnmount(() => {
 					ago | n8n-Version: {{ worker.version }} | Architecture: {{ worker.arch }} (
 					{{ worker.platform }}) | Uptime: {{ upTime(worker.uptime) }}</span
 				>
+				<br />
 				<WorkerJobAccordion :items="worker.runningJobsSummary" />
 				<WorkerNetAccordion :items="sortedWorkerInterfaces" />
 				<WorkerChartsAccordion :worker-id="worker.senderId" />
+				<WorkerMemoryMonitorAccordion :worker="worker" />
 			</N8nText>
 		</div>
 		<template #append>

packages/frontend/editor-ui/src/features/settings/orchestration.ee/components/WorkerChartsAccordion.vue

@@ -5,7 +5,7 @@ import { ref } from 'vue';
 import type { ChartData, ChartOptions } from 'chart.js';
 import type { ChartComponentRef } from 'vue-chartjs';
 import { Chart } from 'vue-chartjs';
-import { averageWorkerLoadFromLoads, memAsGb } from '../orchestration.utils';
+import { averageWorkerLoadFromLoads } from '../orchestration.utils';
 import { useI18n } from '@n8n/i18n';
 
 const props = defineProps<{
@@ -28,7 +28,7 @@ const blankDataSet = (label: string, color: string, prefill: number = 0) => ({
 const orchestrationStore = useOrchestrationStore();
 const chartRefJobs = ref<ChartComponentRef | undefined>(undefined);
 const chartRefCPU = ref<ChartComponentRef | undefined>(undefined);
-const chartRefMemory = ref<ChartComponentRef | undefined>(undefined);
+const chartRefMemoryUsage = ref<ChartComponentRef | undefined>(undefined);
 const optionsBase: () => Partial<ChartOptions<'line'>> = () => ({
 	responsive: true,
 	maintainAspectRatio: true,
@@ -49,9 +49,8 @@ const optionsBase: () => Partial<ChartOptions<'line'>> = () => ({
 const optionsJobs: Partial<ChartOptions<'line'>> = optionsBase();
 const optionsCPU: Partial<ChartOptions<'line'>> = optionsBase();
 if (optionsCPU.scales?.y) optionsCPU.scales.y.suggestedMax = 100;
-const maxMemory = memAsGb(orchestrationStore.workers[props.workerId]?.totalMem) ?? 1;
 const optionsMemory: Partial<ChartOptions<'line'>> = optionsBase();
-if (optionsMemory.scales?.y) optionsMemory.scales.y.suggestedMax = maxMemory;
+if (optionsMemory.scales?.y) optionsMemory.scales.y.suggestedMax = 100;
 
 // prefilled initial arrays
 const dataJobs = ref<ChartData>(
@@ -60,8 +59,8 @@ const dataJobs = ref<ChartData>(
 const dataCPU = ref<ChartData>(
 	blankDataSet('Processor Usage', 'rgb(19, 205, 103)', WORKER_HISTORY_LENGTH),
 );
-const dataMemory = ref<ChartData>(
-	blankDataSet('Memory Usage', 'rgb(244, 216, 174)', WORKER_HISTORY_LENGTH),
+const dataMemoryUsage = ref<ChartData>(
+	blankDataSet('Memory Usage (%)', 'rgb(244, 216, 174)', WORKER_HISTORY_LENGTH),
 );
 
 orchestrationStore.$onAction(({ name, store }) => {
@@ -74,22 +73,33 @@ orchestrationStore.$onAction(({ name, store }) => {
 			'rgb(19, 205, 103)',
 			prefillCount,
 		);
-		const newDataMemory: ChartData = blankDataSet(
-			'Memory Usage',
+		const newDataMemoryUsage: ChartData = blankDataSet(
+			'Memory Usage (%)',
 			'rgb(244, 216, 174)',
 			prefillCount,
 		);
+
 		store.workersHistory[props.workerId]?.forEach((item) => {
 			newDataJobs.datasets[0].data.push(item.data.runningJobsSummary.length);
 			newDataJobs.labels?.push(new Date(item.timestamp).toLocaleTimeString());
 			newDataCPU.datasets[0].data.push(averageWorkerLoadFromLoads(item.data.loadAvg));
+
+			//	Map the x axis timestamps to the labels
 			newDataCPU.labels = newDataJobs.labels;
-			newDataMemory.datasets[0].data.push(maxMemory - memAsGb(item.data.freeMem));
-			newDataMemory.labels = newDataJobs.labels;
+			newDataMemoryUsage.labels = newDataJobs.labels;
+
+			const totalMem = item.data.isInContainer
+				? item.data.process.memory.constraint
+				: item.data.host.memory.total;
+
+			const usedMem = totalMem - item.data.process.memory.available;
+
+			const usage = (usedMem / totalMem) * 100;
+			newDataMemoryUsage.datasets[0].data.push(usage);
 		});
 		dataJobs.value = newDataJobs;
 		dataCPU.value = newDataCPU;
-		dataMemory.value = newDataMemory;
+		dataMemoryUsage.value = newDataMemoryUsage;
 	}
 });
 </script>
@@ -116,9 +126,9 @@ orchestrationStore.$onAction(({ name, store }) => {
 					:class="$style.chart"
 				/>
 				<Chart
-					ref="chartRefMemory"
+					ref="chartRefMemoryUsage"
 					type="line"
-					:data="dataMemory"
+					:data="dataMemoryUsage"
 					:options="optionsMemory"
 					:class="$style.chart"
 				/>

packages/frontend/editor-ui/src/features/settings/orchestration.ee/components/WorkerMemoryMonitorAccordion.vue

@@ -0,0 +1,81 @@
+<script setup lang="ts">
+import type { WorkerStatus } from '@n8n/api-types';
+import WorkerAccordion from './WorkerAccordion.vue';
+import { useI18n } from '@n8n/i18n';
+import { memAsGb, memAsMb } from '@/features/settings/orchestration.ee/orchestration.utils';
+
+const props = defineProps<{
+	worker: WorkerStatus;
+}>();
+
+const i18n = useI18n();
+</script>
+
+<template>
+	<WorkerAccordion icon="list-checks" icon-color="text-dark" :initial-expanded="false">
+		<template #title>
+			{{ i18n.baseText('workerList.item.memoryMonitorTitle') }}
+		</template>
+		<template #content>
+			<div :class="$style['accordion-content']">
+				<strong>Host/OS Memory:</strong>
+				<table>
+					<tbody>
+						<tr>
+							<th>Total (os.totalmem)</th>
+							<td>{{ memAsGb(props.worker.host.memory.total) }}GB</td>
+						</tr>
+						<tr>
+							<th>Free (os.freemem)</th>
+							<td>{{ memAsGb(props.worker.host.memory.free) }}GB</td>
+						</tr>
+					</tbody>
+				</table>
+				<br />
+				<strong>Process Memory:</strong><br />
+				<table>
+					<tbody>
+						<tr v-if="worker.isInContainer">
+							<th>Constraint: (process.constrainedMemory)</th>
+							<td>{{ memAsMb(props.worker.process.memory.constraint) }}MB</td>
+						</tr>
+						<tr>
+							<th>Available: (process.availableMemory)</th>
+							<td>{{ memAsMb(props.worker.process.memory.available) }}MB</td>
+						</tr>
+						<tr>
+							<th>RSS: (process.memoryUsage().rss)</th>
+							<td>{{ memAsMb(props.worker.process.memory.rss) }}MB</td>
+						</tr>
+						<tr>
+							<th>Heap total: (process.memoryUsage().heapTotal)</th>
+							<td>{{ memAsMb(props.worker.process.memory.heapTotal) }}MB</td>
+						</tr>
+						<tr>
+							<th>Heap used: (process.memoryUsage().heapUsed)</th>
+							<td>{{ memAsMb(props.worker.process.memory.heapUsed) }}MB</td>
+						</tr>
+					</tbody>
+				</table>
+			</div>
+		</template>
+	</WorkerAccordion>
+</template>
+
+<style lang="scss" module>
+table {
+	th,
+	td {
+		text-align: left;
+		font-weight: normal;
+	}
+	td {
+		font-variant-numeric: tabular-nums;
+		padding-left: 8px;
+	}
+}
+
+.accordion-content {
+	padding: var(--spacing--2xs);
+}
+</style>

packages/frontend/editor-ui/src/features/settings/orchestration.ee/orchestration.utils.ts

@@ -6,6 +6,10 @@ export function averageWorkerLoadFromLoadsAsString(loads: number[]): string {
 	return averageWorkerLoadFromLoads(loads).toFixed(2);
 }
 
-export function memAsGb(mem: number): number {
-	return mem / 1024 / 1024 / 1024;
+export function memAsGb(mem: number, decimalPlaces: number = 2): number {
+	return Number((mem / 1024 / 1024 / 1024).toFixed(decimalPlaces));
+}
+
+export function memAsMb(mem: number, decimalPlaces: number = 2): number {
+	return Number((mem / 1024 / 1024).toFixed(decimalPlaces));
 }