diff --git a/packages/@n8n/api-types/src/frontend-settings.ts b/packages/@n8n/api-types/src/frontend-settings.ts index 1c5e8fab487..80863e0e702 100644 --- a/packages/@n8n/api-types/src/frontend-settings.ts +++ b/packages/@n8n/api-types/src/frontend-settings.ts @@ -95,7 +95,7 @@ export interface FrontendSettings { authCookie: { secure: boolean; }; - binaryDataMode: 'default' | 'filesystem' | 's3'; + binaryDataMode: 'default' | 'filesystem' | 's3' | 'database'; releaseChannel: 'stable' | 'beta' | 'nightly' | 'dev'; n8nMetadata?: { userId?: string; diff --git a/packages/@n8n/db/src/connection/__tests__/db-connection.test.ts b/packages/@n8n/db/src/connection/__tests__/db-connection.test.ts index 2addb2c7b29..9411c0bd6d1 100644 --- a/packages/@n8n/db/src/connection/__tests__/db-connection.test.ts +++ b/packages/@n8n/db/src/connection/__tests__/db-connection.test.ts @@ -3,7 +3,7 @@ import type { Logger } from '@n8n/backend-common'; import type { DatabaseConfig } from '@n8n/config'; import { DataSource, type DataSourceOptions } from '@n8n/typeorm'; import { mock, mockDeep } from 'jest-mock-extended'; -import type { ErrorReporter } from 'n8n-core'; +import type { BinaryDataConfig, ErrorReporter } from 'n8n-core'; import { DbConnectionTimeoutError } from 'n8n-workflow'; import * as migrationHelper from '../../migrations/migration-helpers'; @@ -24,6 +24,10 @@ describe('DbConnection', () => { const errorReporter = mock(); const databaseConfig = mock(); const logger = mock(); + const binaryDataConfig = mock({ + availableModes: ['filesystem'], + dbMaxFileSize: 512, + }); const dataSource = mockDeep({ options: { migrations } }); const connectionOptions = mockDeep(); const postgresOptions: DataSourceOptions = { @@ -42,7 +46,13 @@ describe('DbConnection', () => { connectionOptions.getOptions.mockReturnValue(postgresOptions); (DataSource as jest.Mock) = jest.fn().mockImplementation(() => dataSource); - dbConnection = new DbConnection(errorReporter, connectionOptions, databaseConfig, logger); + dbConnection = new DbConnection( + errorReporter, + connectionOptions, + databaseConfig, + logger, + binaryDataConfig, + ); }); describe('init', () => { @@ -196,6 +206,7 @@ describe('DbConnection', () => { pingIntervalSeconds: 1, }), logger, + binaryDataConfig, ); // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/packages/@n8n/db/src/connection/db-connection.ts b/packages/@n8n/db/src/connection/db-connection.ts index 3bebdd37b30..17824f0441b 100644 --- a/packages/@n8n/db/src/connection/db-connection.ts +++ b/packages/@n8n/db/src/connection/db-connection.ts @@ -4,7 +4,7 @@ import { Time } from '@n8n/constants'; import { Memoized } from '@n8n/decorators'; import { Container, Service } from '@n8n/di'; import { DataSource } from '@n8n/typeorm'; -import { ErrorReporter } from 'n8n-core'; +import { BinaryDataConfig, ErrorReporter } from 'n8n-core'; import { DbConnectionTimeoutError, ensureError, OperationalError } from 'n8n-workflow'; import { setTimeout as setTimeoutP } from 'timers/promises'; @@ -33,6 +33,7 @@ export class DbConnection { private readonly connectionOptions: DbConnectionOptions, private readonly databaseConfig: DatabaseConfig, private readonly logger: Logger, + private readonly binaryDataConfig: BinaryDataConfig, ) { this.dataSource = new DataSource(this.options); Container.set(DataSource, this.dataSource); @@ -62,6 +63,21 @@ export class DbConnection { throw error; } + if ( + (options.type === 'mysql' || options.type === 'mariadb') && + this.binaryDataConfig.availableModes.includes('database') + ) { + const maxAllowedPacket = this.binaryDataConfig.dbMaxFileSize * 1024 * 1024; + try { + await this.dataSource.query(`SET GLOBAL max_allowed_packet = ${maxAllowedPacket}`); + } catch { + this.logger.warn( + `Failed to set \`max_allowed_packet\` to ${maxAllowedPacket} bytes on your MySQL server. ` + + `Please set \`max_allowed_packet\` to at least ${this.binaryDataConfig.dbMaxFileSize} MiB in your MySQL server configuration.`, + ); + } + } + connectionState.connected = true; if (!inTest) this.scheduleNextPing(); } diff --git a/packages/core/src/binary-data/binary-data.config.ts b/packages/core/src/binary-data/binary-data.config.ts index 48d4391dba6..5ce6046ae6b 100644 --- a/packages/core/src/binary-data/binary-data.config.ts +++ b/packages/core/src/binary-data/binary-data.config.ts @@ -5,13 +5,19 @@ import { z } from 'zod'; import { InstanceSettings } from '@/instance-settings'; -const binaryDataModesSchema = z.enum(['default', 'filesystem', 's3']); +export const BINARY_DATA_MODES = ['default', 'filesystem', 's3', 'database'] as const; + +const binaryDataModesSchema = z.enum(BINARY_DATA_MODES); const availableModesSchema = z .string() .transform((value) => value.split(',')) .pipe(binaryDataModesSchema.array()); +const dbMaxFileSizeSchema = z + .number() + .max(1024, 'Binary data max file size in `database` mode cannot exceed 1024 MiB'); // because of Postgres BYTEA hard limit + @Config export class BinaryDataConfig { /** Available modes of binary data storage, as comma separated strings. */ @@ -33,6 +39,10 @@ export class BinaryDataConfig { @Env('N8N_BINARY_DATA_SIGNING_SECRET') signingSecret: string; + /** Maximum file size (in MiB) for binary data in `database` mode. **/ + @Env('N8N_BINARY_DATA_DATABASE_MAX_FILE_SIZE', dbMaxFileSizeSchema) + dbMaxFileSize: number = 512; + constructor({ encryptionKey, n8nFolder }: InstanceSettings) { this.localStoragePath = path.join(n8nFolder, 'binaryData'); this.signingSecret = createHash('sha256') diff --git a/packages/core/src/binary-data/binary-data.service.ts b/packages/core/src/binary-data/binary-data.service.ts index 229b8a65b04..176597625fc 100644 --- a/packages/core/src/binary-data/binary-data.service.ts +++ b/packages/core/src/binary-data/binary-data.service.ts @@ -1,7 +1,7 @@ import { Container, Service } from '@n8n/di'; import jwt from 'jsonwebtoken'; import type { StringValue as TimeUnitValue } from 'ms'; -import { BINARY_ENCODING, UnexpectedError } from 'n8n-workflow'; +import { BINARY_ENCODING, UnexpectedError, UserError } from 'n8n-workflow'; import type { INodeExecutionData, IBinaryData } from 'n8n-workflow'; import { readFile, stat } from 'node:fs/promises'; import prettyBytes from 'pretty-bytes'; @@ -11,9 +11,8 @@ import { ErrorReporter } from '@/errors'; import { BinaryDataConfig } from './binary-data.config'; import type { BinaryData } from './types'; -import { areConfigModes, binaryToBuffer } from './utils'; +import { binaryToBuffer } from './utils'; import { InvalidManagerError } from '../errors/invalid-manager.error'; -import { InvalidModeError } from '../errors/invalid-mode.error'; @Service() export class BinaryDataService { @@ -28,7 +27,10 @@ export class BinaryDataService { async init() { const { config } = this; - if (!areConfigModes(config.availableModes)) throw new InvalidModeError(); + + if (config.mode === 'database' || config.availableModes.includes('database')) { + throw new UserError('Database mode is not implemented yet'); + } this.mode = config.mode === 'filesystem' ? 'filesystem-v2' : config.mode; diff --git a/packages/core/src/binary-data/types.ts b/packages/core/src/binary-data/types.ts index 88657dc5d55..9f01e3315ad 100644 --- a/packages/core/src/binary-data/types.ts +++ b/packages/core/src/binary-data/types.ts @@ -1,5 +1,7 @@ import type { Readable } from 'stream'; +import type { BINARY_DATA_MODES } from './binary-data.config'; + export namespace BinaryData { type LegacyMode = 'filesystem'; @@ -8,7 +10,7 @@ export namespace BinaryData { /** * Binary data mode selectable by user via env var config. */ - export type ConfigMode = 'default' | 'filesystem' | 's3'; + export type ConfigMode = (typeof BINARY_DATA_MODES)[number]; /** * Binary data mode used internally by binary data service. User-selected diff --git a/packages/core/src/binary-data/utils.ts b/packages/core/src/binary-data/utils.ts index a7852e1fca1..9d5ebd7ff3e 100644 --- a/packages/core/src/binary-data/utils.ts +++ b/packages/core/src/binary-data/utils.ts @@ -4,13 +4,7 @@ import type { Readable } from 'node:stream'; import type { BinaryData } from './types'; -export const CONFIG_MODES = ['default', 'filesystem', 's3'] as const; - -const STORED_MODES = ['filesystem', 'filesystem-v2', 's3'] as const; - -export function areConfigModes(modes: string[]): modes is BinaryData.ConfigMode[] { - return modes.every((m) => CONFIG_MODES.includes(m as BinaryData.ConfigMode)); -} +const STORED_MODES = ['filesystem', 'filesystem-v2', 's3', 'database'] as const; export function isStoredMode(mode: string): mode is BinaryData.StoredMode { return STORED_MODES.includes(mode as BinaryData.StoredMode); diff --git a/packages/core/src/errors/index.ts b/packages/core/src/errors/index.ts index b6571dd3363..4072bc56790 100644 --- a/packages/core/src/errors/index.ts +++ b/packages/core/src/errors/index.ts @@ -1,6 +1,5 @@ export { FileNotFoundError } from './file-not-found.error'; export { DisallowedFilepathError } from './disallowed-filepath.error'; -export { InvalidModeError } from './invalid-mode.error'; export { InvalidManagerError } from './invalid-manager.error'; export { InvalidExecutionMetadataError } from './invalid-execution-metadata.error'; export { UnrecognizedCredentialTypeError } from './unrecognized-credential-type.error'; diff --git a/packages/core/src/errors/invalid-mode.error.ts b/packages/core/src/errors/invalid-mode.error.ts deleted file mode 100644 index 794f1ef4617..00000000000 --- a/packages/core/src/errors/invalid-mode.error.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { ApplicationError } from '@n8n/errors'; - -import { CONFIG_MODES } from '../binary-data/utils'; - -export class InvalidModeError extends ApplicationError { - constructor() { - super(`Invalid binary data mode. Valid modes: ${CONFIG_MODES.join(', ')}`); - } -} diff --git a/packages/frontend/@n8n/stores/src/useRootStore.ts b/packages/frontend/@n8n/stores/src/useRootStore.ts index e234290ba3f..c98739168cf 100644 --- a/packages/frontend/@n8n/stores/src/useRootStore.ts +++ b/packages/frontend/@n8n/stores/src/useRootStore.ts @@ -31,7 +31,7 @@ export type RootStoreState = { urlBaseWebhook: string; urlBaseEditor: string; instanceId: string; - binaryDataMode: 'default' | 'filesystem' | 's3'; + binaryDataMode: 'default' | 'filesystem' | 's3' | 'database'; }; export const useRootStore = defineStore(STORES.ROOT, () => { diff --git a/packages/frontend/editor-ui/src/app/composables/useDebugInfo.ts b/packages/frontend/editor-ui/src/app/composables/useDebugInfo.ts index 95e5d57adc0..d780236e3d7 100644 --- a/packages/frontend/editor-ui/src/app/composables/useDebugInfo.ts +++ b/packages/frontend/editor-ui/src/app/composables/useDebugInfo.ts @@ -20,7 +20,7 @@ type DebugInfo = { error: WorkflowSettings.SaveDataExecution; progress: boolean; manual: boolean; - binaryMode: 'memory' | 'filesystem' | 's3'; + binaryMode: 'memory' | 'filesystem' | 's3' | 'database'; }; pruning: | {