refactor(core): Update binary data config (#22141)

packages/@n8n/api-types/src/frontend-settings.ts

@@ -95,7 +95,7 @@ export interface FrontendSettings {
 	authCookie: {
 		secure: boolean;
 	};
-	binaryDataMode: 'default' | 'filesystem' | 's3';
+	binaryDataMode: 'default' | 'filesystem' | 's3' | 'database';
 	releaseChannel: 'stable' | 'beta' | 'nightly' | 'dev';
 	n8nMetadata?: {
 		userId?: string;

packages/@n8n/db/src/connection/__tests__/db-connection.test.ts

@@ -3,7 +3,7 @@ import type { Logger } from '@n8n/backend-common';
 import type { DatabaseConfig } from '@n8n/config';
 import { DataSource, type DataSourceOptions } from '@n8n/typeorm';
 import { mock, mockDeep } from 'jest-mock-extended';
-import type { ErrorReporter } from 'n8n-core';
+import type { BinaryDataConfig, ErrorReporter } from 'n8n-core';
 import { DbConnectionTimeoutError } from 'n8n-workflow';
 
 import * as migrationHelper from '../../migrations/migration-helpers';
@@ -24,6 +24,10 @@ describe('DbConnection', () => {
 	const errorReporter = mock<ErrorReporter>();
 	const databaseConfig = mock<DatabaseConfig>();
 	const logger = mock<Logger>();
+	const binaryDataConfig = mock<BinaryDataConfig>({
+		availableModes: ['filesystem'],
+		dbMaxFileSize: 512,
+	});
 	const dataSource = mockDeep<DataSource>({ options: { migrations } });
 	const connectionOptions = mockDeep<DbConnectionOptions>();
 	const postgresOptions: DataSourceOptions = {
@@ -42,7 +46,13 @@ describe('DbConnection', () => {
 		connectionOptions.getOptions.mockReturnValue(postgresOptions);
 		(DataSource as jest.Mock) = jest.fn().mockImplementation(() => dataSource);
 
-		dbConnection = new DbConnection(errorReporter, connectionOptions, databaseConfig, logger);
+		dbConnection = new DbConnection(
+			errorReporter,
+			connectionOptions,
+			databaseConfig,
+			logger,
+			binaryDataConfig,
+		);
 	});
 
 	describe('init', () => {
@@ -196,6 +206,7 @@ describe('DbConnection', () => {
 						pingIntervalSeconds: 1,
 					}),
 					logger,
+					binaryDataConfig,
 				);
 
 				// eslint-disable-next-line @typescript-eslint/no-explicit-any

packages/@n8n/db/src/connection/db-connection.ts

@@ -4,7 +4,7 @@ import { Time } from '@n8n/constants';
 import { Memoized } from '@n8n/decorators';
 import { Container, Service } from '@n8n/di';
 import { DataSource } from '@n8n/typeorm';
-import { ErrorReporter } from 'n8n-core';
+import { BinaryDataConfig, ErrorReporter } from 'n8n-core';
 import { DbConnectionTimeoutError, ensureError, OperationalError } from 'n8n-workflow';
 import { setTimeout as setTimeoutP } from 'timers/promises';
 
@@ -33,6 +33,7 @@ export class DbConnection {
 		private readonly connectionOptions: DbConnectionOptions,
 		private readonly databaseConfig: DatabaseConfig,
 		private readonly logger: Logger,
+		private readonly binaryDataConfig: BinaryDataConfig,
 	) {
 		this.dataSource = new DataSource(this.options);
 		Container.set(DataSource, this.dataSource);
@@ -62,6 +63,21 @@ export class DbConnection {
 			throw error;
 		}
 
+		if (
+			(options.type === 'mysql' || options.type === 'mariadb') &&
+			this.binaryDataConfig.availableModes.includes('database')
+		) {
+			const maxAllowedPacket = this.binaryDataConfig.dbMaxFileSize * 1024 * 1024;
+			try {
+				await this.dataSource.query(`SET GLOBAL max_allowed_packet = ${maxAllowedPacket}`);
+			} catch {
+				this.logger.warn(
+					`Failed to set \`max_allowed_packet\` to ${maxAllowedPacket} bytes on your MySQL server. ` +
+						`Please set \`max_allowed_packet\` to at least ${this.binaryDataConfig.dbMaxFileSize} MiB in your MySQL server configuration.`,
+				);
+			}
+		}
+
 		connectionState.connected = true;
 		if (!inTest) this.scheduleNextPing();
 	}

packages/core/src/binary-data/binary-data.config.ts

@@ -5,13 +5,19 @@ import { z } from 'zod';
 
 import { InstanceSettings } from '@/instance-settings';
 
-const binaryDataModesSchema = z.enum(['default', 'filesystem', 's3']);
+export const BINARY_DATA_MODES = ['default', 'filesystem', 's3', 'database'] as const;
+
+const binaryDataModesSchema = z.enum(BINARY_DATA_MODES);
 
 const availableModesSchema = z
 	.string()
 	.transform((value) => value.split(','))
 	.pipe(binaryDataModesSchema.array());
 
+const dbMaxFileSizeSchema = z
+	.number()
+	.max(1024, 'Binary data max file size in `database` mode cannot exceed 1024 MiB'); // because of Postgres BYTEA hard limit
+
 @Config
 export class BinaryDataConfig {
 	/** Available modes of binary data storage, as comma separated strings. */
@@ -33,6 +39,10 @@ export class BinaryDataConfig {
 	@Env('N8N_BINARY_DATA_SIGNING_SECRET')
 	signingSecret: string;
 
+	/** Maximum file size (in MiB) for binary data in `database` mode. **/
+	@Env('N8N_BINARY_DATA_DATABASE_MAX_FILE_SIZE', dbMaxFileSizeSchema)
+	dbMaxFileSize: number = 512;
+
 	constructor({ encryptionKey, n8nFolder }: InstanceSettings) {
 		this.localStoragePath = path.join(n8nFolder, 'binaryData');
 		this.signingSecret = createHash('sha256')

packages/core/src/binary-data/binary-data.service.ts

@@ -1,7 +1,7 @@
 import { Container, Service } from '@n8n/di';
 import jwt from 'jsonwebtoken';
 import type { StringValue as TimeUnitValue } from 'ms';
-import { BINARY_ENCODING, UnexpectedError } from 'n8n-workflow';
+import { BINARY_ENCODING, UnexpectedError, UserError } from 'n8n-workflow';
 import type { INodeExecutionData, IBinaryData } from 'n8n-workflow';
 import { readFile, stat } from 'node:fs/promises';
 import prettyBytes from 'pretty-bytes';
@@ -11,9 +11,8 @@ import { ErrorReporter } from '@/errors';
 
 import { BinaryDataConfig } from './binary-data.config';
 import type { BinaryData } from './types';
-import { areConfigModes, binaryToBuffer } from './utils';
+import { binaryToBuffer } from './utils';
 import { InvalidManagerError } from '../errors/invalid-manager.error';
-import { InvalidModeError } from '../errors/invalid-mode.error';
 
 @Service()
 export class BinaryDataService {
@@ -28,7 +27,10 @@ export class BinaryDataService {
 
 	async init() {
 		const { config } = this;
-		if (!areConfigModes(config.availableModes)) throw new InvalidModeError();
+
+		if (config.mode === 'database' || config.availableModes.includes('database')) {
+			throw new UserError('Database mode is not implemented yet');
+		}
 
 		this.mode = config.mode === 'filesystem' ? 'filesystem-v2' : config.mode;
 

packages/core/src/binary-data/types.ts

@@ -1,5 +1,7 @@
 import type { Readable } from 'stream';
 
+import type { BINARY_DATA_MODES } from './binary-data.config';
+
 export namespace BinaryData {
 	type LegacyMode = 'filesystem';
 
@@ -8,7 +10,7 @@ export namespace BinaryData {
 	/**
 	 * Binary data mode selectable by user via env var config.
 	 */
-	export type ConfigMode = 'default' | 'filesystem' | 's3';
+	export type ConfigMode = (typeof BINARY_DATA_MODES)[number];
 
 	/**
 	 * Binary data mode used internally by binary data service. User-selected

packages/core/src/binary-data/utils.ts

@@ -4,13 +4,7 @@ import type { Readable } from 'node:stream';
 
 import type { BinaryData } from './types';
 
-export const CONFIG_MODES = ['default', 'filesystem', 's3'] as const;
-
-const STORED_MODES = ['filesystem', 'filesystem-v2', 's3'] as const;
-
-export function areConfigModes(modes: string[]): modes is BinaryData.ConfigMode[] {
-	return modes.every((m) => CONFIG_MODES.includes(m as BinaryData.ConfigMode));
-}
+const STORED_MODES = ['filesystem', 'filesystem-v2', 's3', 'database'] as const;
 
 export function isStoredMode(mode: string): mode is BinaryData.StoredMode {
 	return STORED_MODES.includes(mode as BinaryData.StoredMode);

packages/core/src/errors/index.ts

@@ -1,6 +1,5 @@
 export { FileNotFoundError } from './file-not-found.error';
 export { DisallowedFilepathError } from './disallowed-filepath.error';
-export { InvalidModeError } from './invalid-mode.error';
 export { InvalidManagerError } from './invalid-manager.error';
 export { InvalidExecutionMetadataError } from './invalid-execution-metadata.error';
 export { UnrecognizedCredentialTypeError } from './unrecognized-credential-type.error';

packages/core/src/errors/invalid-mode.error.ts

@@ -1,9 +0,0 @@
-import { ApplicationError } from '@n8n/errors';
-
-import { CONFIG_MODES } from '../binary-data/utils';
-
-export class InvalidModeError extends ApplicationError {
-	constructor() {
-		super(`Invalid binary data mode. Valid modes: ${CONFIG_MODES.join(', ')}`);
-	}
-}

packages/frontend/@n8n/stores/src/useRootStore.ts

@@ -31,7 +31,7 @@ export type RootStoreState = {
 	urlBaseWebhook: string;
 	urlBaseEditor: string;
 	instanceId: string;
-	binaryDataMode: 'default' | 'filesystem' | 's3';
+	binaryDataMode: 'default' | 'filesystem' | 's3' | 'database';
 };
 
 export const useRootStore = defineStore(STORES.ROOT, () => {

packages/frontend/editor-ui/src/app/composables/useDebugInfo.ts

@@ -20,7 +20,7 @@ type DebugInfo = {
 		error: WorkflowSettings.SaveDataExecution;
 		progress: boolean;
 		manual: boolean;
-		binaryMode: 'memory' | 'filesystem' | 's3';
+		binaryMode: 'memory' | 'filesystem' | 's3' | 'database';
 	};
 	pruning:
 		| {