From 4e84c53a4708c2e70bd0d4d50449a46a28e3c199 Mon Sep 17 00:00:00 2001 From: Andreas Fitzek Date: Fri, 10 Oct 2025 11:32:01 +0200 Subject: [PATCH] chore(core): Migrate insights values to bigint to avoid overflow (#20343) --- .../backend-test-utils/MIGRATION_TESTING.md | 82 +++++ packages/@n8n/backend-test-utils/src/index.ts | 1 + .../src/migration-test-helpers.ts | 124 +++++++ ...59399811000-ChangeValueTypesForInsights.ts | 95 +++++ packages/@n8n/db/src/migrations/dsl/column.ts | 10 +- .../db/src/migrations/migration-helpers.ts | 5 + .../@n8n/db/src/migrations/mysqldb/index.ts | 2 + .../db/src/migrations/postgresdb/index.ts | 2 + .../@n8n/db/src/migrations/sqlite/index.ts | 2 + ...ights-bigint-migration.integration.test.ts | 343 ++++++++++++++++++ .../insights-by-period-migration.test.ts | 163 +++++++++ .../__tests__/insights-raw-migration.test.ts | 163 +++++++++ .../__tests__/migration-test-setup.ts | 53 +++ .../database/entities/insights-by-period.ts | 5 + .../database/entities/insights-raw.ts | 5 + .../migration/migration-test-helpers.test.ts | 69 ++++ 16 files changed, 1123 insertions(+), 1 deletion(-) create mode 100644 packages/@n8n/backend-test-utils/MIGRATION_TESTING.md create mode 100644 packages/@n8n/backend-test-utils/src/migration-test-helpers.ts create mode 100644 packages/@n8n/db/src/migrations/common/1759399811000-ChangeValueTypesForInsights.ts create mode 100644 packages/cli/src/modules/insights/__tests__/insights-bigint-migration.integration.test.ts create mode 100644 packages/cli/src/modules/insights/__tests__/insights-by-period-migration.test.ts create mode 100644 packages/cli/src/modules/insights/__tests__/insights-raw-migration.test.ts create mode 100644 packages/cli/src/modules/insights/__tests__/migration-test-setup.ts create mode 100644 packages/cli/test/migration/migration-test-helpers.test.ts diff --git a/packages/@n8n/backend-test-utils/MIGRATION_TESTING.md b/packages/@n8n/backend-test-utils/MIGRATION_TESTING.md new file mode 100644 index 00000000000..33083c31255 --- /dev/null +++ b/packages/@n8n/backend-test-utils/MIGRATION_TESTING.md @@ -0,0 +1,82 @@ +# Migration Testing Helpers + +This package provides utilities for testing database migrations by allowing you to stop before a specific migration, insert test data, and then run that migration. + +## API + +### `initDbUpToMigration(beforeMigrationName: string): Promise` + +Initializes the database and runs all migrations up to (but not including) the specified migration. + +**Parameters:** +- `beforeMigrationName`: The class name of the migration to stop before (e.g., `'AddUserRole1234567890'`) + +**Throws:** +- `UnexpectedError` if the migration is not found or database is not initialized + +### `runSingleMigration(migrationName: string): Promise` + +Runs a single migration by name. + +**Parameters:** +- `migrationName`: The class name of the migration to run (e.g., `'AddUserRole1234567890'`) + +**Throws:** +- `UnexpectedError` if the migration is not found or database is not initialized + +## Usage Example + +```typescript +import { Container } from '@n8n/di'; +import { DataSource } from '@n8n/typeorm'; +import { initDbUpToMigration, runSingleMigration } from '@n8n/backend-test-utils'; + +describe('AddUserRole1234567890 Migration', () => { + let dataSource: DataSource; + + beforeAll(async () => { + // Initialize database but stop BEFORE the migration we want to test + await initDbUpToMigration('AddUserRole1234567890'); + dataSource = Container.get(DataSource); + }); + + it('should add role column to users table', async () => { + // Insert test data in the OLD schema (before migration) + // You should not use Repositories, because these will break after schema changes + // over time. + await dataSource.query(` + INSERT INTO users (id, email, password) + VALUES (1, 'test@example.com', 'hashed_password') + `); + + // Run the migration + await runSingleMigration('AddUserRole1234567890'); + + // Verify the migration worked correctly + const users = await dataSource.query('SELECT * FROM users WHERE id = 1'); + expect(users[0].role).toBe('member'); // Default role was added + }); +}); +``` + +## How It Works + +1. **`initDbUpToMigration`**: + - Gets all available migrations from TypeORM DataSource + - Finds the target migration by name + - Temporarily replaces the migrations array with only migrations before the target + - Wraps and runs those migrations + - Restores the full migrations array + +2. **`runSingleMigration`**: + - Finds the specific migration by name + - Temporarily replaces the migrations array with only that migration + - Wraps and runs that single migration + - Restores the full migrations array + +## Important Notes + +- These functions must be used with an initialized database connection (after `dbConnection.init()`) +- Do NOT call `dbConnection.migrate()` before using these helpers - they replace that step +- Migration wrapping is idempotent - migrations won't be double-wrapped +- The full migrations array is always restored after operations complete (even on error) diff --git a/packages/@n8n/backend-test-utils/src/index.ts b/packages/@n8n/backend-test-utils/src/index.ts index 9b168308b18..b098a879bbb 100644 --- a/packages/@n8n/backend-test-utils/src/index.ts +++ b/packages/@n8n/backend-test-utils/src/index.ts @@ -10,3 +10,4 @@ export * as testModules from './test-modules'; export * from './db/workflows'; export * from './db/projects'; export * from './mocking'; +export * from './migration-test-helpers'; diff --git a/packages/@n8n/backend-test-utils/src/migration-test-helpers.ts b/packages/@n8n/backend-test-utils/src/migration-test-helpers.ts new file mode 100644 index 00000000000..9afbf82d0aa --- /dev/null +++ b/packages/@n8n/backend-test-utils/src/migration-test-helpers.ts @@ -0,0 +1,124 @@ +import { GlobalConfig } from '@n8n/config'; +import { type DatabaseType, DbConnection, type Migration } from '@n8n/db'; +import { Container } from '@n8n/di'; +import { DataSource, type QueryRunner } from '@n8n/typeorm'; +import { UnexpectedError } from 'n8n-workflow'; + +async function reinitializeDataConnection(): Promise { + const dbConnection = Container.get(DbConnection); + await dbConnection.close(); + await dbConnection.init(); +} + +/** + * Test migration context with database-specific helpers (similar to MigrationContext). + */ +export interface TestMigrationContext { + queryRunner: QueryRunner; + tablePrefix: string; + dbType: DatabaseType; + isMysql: boolean; + isSqlite: boolean; + isPostgres: boolean; + escape: { + columnName(name: string): string; + tableName(name: string): string; + indexName(name: string): string; + }; +} + +/** + * Create a test migration context with database-specific helpers. + * Provides the same utilities that migrations have access to. + */ +export function createTestMigrationContext(dataSource: DataSource): TestMigrationContext { + const globalConfig = Container.get(GlobalConfig); + const dbType = globalConfig.database.type; + const tablePrefix = globalConfig.database.tablePrefix; + const queryRunner = dataSource.createQueryRunner(); + + return { + queryRunner, + tablePrefix, + dbType, + isMysql: ['mariadb', 'mysqldb'].includes(dbType), + isSqlite: dbType === 'sqlite', + isPostgres: dbType === 'postgresdb', + escape: { + columnName: (name) => queryRunner.connection.driver.escape(name), + tableName: (name) => queryRunner.connection.driver.escape(`${tablePrefix}${name}`), + indexName: (name) => queryRunner.connection.driver.escape(`IDX_${tablePrefix}${name}`), + }, + }; +} + +/** + * Initialize database and run all migrations up to (but not including) the specified migration. + * Useful for testing data transformations by inserting test data before a migration runs. + * + * @param beforeMigrationName - The class name of the migration to stop before (e.g., 'AddUserRole1234567890') + * @throws {UnexpectedError} If the migration is not found or database is not initialized + */ +export async function initDbUpToMigration(beforeMigrationName: string): Promise { + const dataSource = Container.get(DataSource); + + if (!Array.isArray(dataSource.options.migrations)) { + throw new UnexpectedError('Database migrations are not an array'); + } + + const allMigrations = dataSource.options.migrations as Migration[]; + const targetIndex = allMigrations.findIndex((m) => m.name === beforeMigrationName); + + if (targetIndex === -1) { + throw new UnexpectedError(`Migration "${beforeMigrationName}" not found`); + } + + // Temporarily replace migrations array with subset + const migrationsToRun = allMigrations.slice(0, targetIndex); + (dataSource.options as { migrations: Migration[] }).migrations = migrationsToRun; + + try { + // Need to reinitialize the data source to rebuild the migrations + await reinitializeDataConnection(); + // Run migrations + await Container.get(DbConnection).migrate(); + } finally { + // Restore full migrations array + (dataSource.options as { migrations: Migration[] }).migrations = allMigrations; + // Need to reinitialize the data source to rebuild the migrations + await reinitializeDataConnection(); + } +} + +/** + * Run a single migration by name. + * Useful for testing a specific migration after inserting test data. + * + * @param migrationName - The class name of the migration to run (e.g., 'AddUserRole1234567890') + * @throws {UnexpectedError} If the migration is not found or database is not initialized + */ +export async function runSingleMigration(migrationName: string): Promise { + const dataSource = Container.get(DataSource); + + const allMigrations = dataSource.options.migrations as Migration[]; + const migration = allMigrations.find((m) => m.name === migrationName); + + if (!migration) { + throw new UnexpectedError(`Migration "${migrationName}" not found`); + } + + // Temporarily replace migrations array with only the target migration + (dataSource.options as { migrations: Migration[] }).migrations = [migration]; + + try { + // Need to reinitialize the data source to rebuild the migrations + await reinitializeDataConnection(); + // Run migrations + await Container.get(DbConnection).migrate(); + } finally { + // Restore full migrations array + (dataSource.options as { migrations: Migration[] }).migrations = allMigrations; + // Need to reinitialize the data source to rebuild the migrations + await reinitializeDataConnection(); + } +} diff --git a/packages/@n8n/db/src/migrations/common/1759399811000-ChangeValueTypesForInsights.ts b/packages/@n8n/db/src/migrations/common/1759399811000-ChangeValueTypesForInsights.ts new file mode 100644 index 00000000000..3bf3b474fc8 --- /dev/null +++ b/packages/@n8n/db/src/migrations/common/1759399811000-ChangeValueTypesForInsights.ts @@ -0,0 +1,95 @@ +import type { IrreversibleMigration, MigrationContext } from '../migration-types'; + +const INSIGHTS_RAW_TABLE_NAME = 'insights_raw'; +const INSIGHTS_RAW_TEMP_TABLE_NAME = 'temp_insights_raw'; +const INSIGHTS_BY_PERIOD_TABLE_NAME = 'insights_by_period'; +const INSIGHTS_BY_PERIOD_TEMP_TABLE_NAME = 'temp_insights_by_period'; +const INSIGHTS_METADATA_TABLE_NAME = 'insights_metadata'; +const VALUE_COLUMN_NAME = 'value'; + +export class ChangeValueTypesForInsights1759399811000 implements IrreversibleMigration { + async up({ + isSqlite, + isMysql, + isPostgres, + escape, + copyTable, + queryRunner, + schemaBuilder: { createTable, column, dropTable }, + }: MigrationContext) { + const insightsRawTable = escape.tableName(INSIGHTS_RAW_TABLE_NAME); + const insightsByPeriodTable = escape.tableName(INSIGHTS_BY_PERIOD_TABLE_NAME); + const valueColumnName = escape.columnName(VALUE_COLUMN_NAME); + + if (isSqlite) { + const tempInsightsByPeriodTable = escape.tableName(INSIGHTS_BY_PERIOD_TEMP_TABLE_NAME); + const tempInsightsRawTable = escape.tableName(INSIGHTS_RAW_TEMP_TABLE_NAME); + const typeComment = '0: time_saved_minutes, 1: runtime_milliseconds, 2: success, 3: failure'; + + // Create temporary raw table with new value type, copy data, remove the original table and rename the temporary table + await createTable(INSIGHTS_RAW_TEMP_TABLE_NAME) + .withColumns( + column('id').int.primary.autoGenerate2, + column('metaId').int.notNull, + column('type').int.notNull.comment(typeComment), + column('value').bigint.notNull, + column('timestamp').timestampTimezone(0).default('CURRENT_TIMESTAMP').notNull, + ) + .withForeignKey('metaId', { + tableName: INSIGHTS_METADATA_TABLE_NAME, + columnName: 'metaId', + onDelete: 'CASCADE', + }); + + // Copy data from the original table to the temporary table + await copyTable(INSIGHTS_RAW_TABLE_NAME, INSIGHTS_RAW_TEMP_TABLE_NAME); + + // drop the original table + await dropTable(INSIGHTS_RAW_TABLE_NAME); + + // rename the temporary table to the original table name + await queryRunner.query(`ALTER TABLE ${tempInsightsRawTable} RENAME TO ${insightsRawTable};`); + + await createTable(INSIGHTS_BY_PERIOD_TEMP_TABLE_NAME) + .withColumns( + column('id').int.primary.autoGenerate2, + column('metaId').int.notNull, + column('type').int.notNull.comment(typeComment), + column('value').bigint.notNull, + column('periodUnit').int.notNull.comment('0: hour, 1: day, 2: week'), + column('periodStart').default('CURRENT_TIMESTAMP').timestampTimezone(0), + ) + .withForeignKey('metaId', { + tableName: INSIGHTS_METADATA_TABLE_NAME, + columnName: 'metaId', + onDelete: 'CASCADE', + }) + .withIndexOn(['periodStart', 'type', 'periodUnit', 'metaId'], true); + + // Copy data from the original table to the temporary table + await copyTable(INSIGHTS_BY_PERIOD_TABLE_NAME, INSIGHTS_BY_PERIOD_TEMP_TABLE_NAME); + + // drop the original table + await dropTable(INSIGHTS_BY_PERIOD_TABLE_NAME); + + // rename the temporary table to the original table name + await queryRunner.query( + `ALTER TABLE ${tempInsightsByPeriodTable} RENAME TO ${insightsByPeriodTable};`, + ); + } else if (isMysql) { + await queryRunner.query( + `ALTER TABLE ${insightsRawTable} MODIFY COLUMN ${valueColumnName} BIGINT NOT NULL;`, + ); + await queryRunner.query( + `ALTER TABLE ${insightsByPeriodTable} MODIFY COLUMN ${valueColumnName} BIGINT NOT NULL;`, + ); + } else if (isPostgres) { + await queryRunner.query( + `ALTER TABLE ${insightsRawTable} ALTER COLUMN ${valueColumnName} TYPE BIGINT;`, + ); + await queryRunner.query( + `ALTER TABLE ${insightsByPeriodTable} ALTER COLUMN ${valueColumnName} TYPE BIGINT;`, + ); + } + } +} diff --git a/packages/@n8n/db/src/migrations/dsl/column.ts b/packages/@n8n/db/src/migrations/dsl/column.ts index 847dbb924a8..10ccc3e17e2 100644 --- a/packages/@n8n/db/src/migrations/dsl/column.ts +++ b/packages/@n8n/db/src/migrations/dsl/column.ts @@ -10,7 +10,8 @@ export class Column { | 'timestamptz' | 'timestamp' | 'uuid' - | 'double'; + | 'double' + | 'bigint'; private isGenerated = false; @@ -40,6 +41,11 @@ export class Column { return this; } + get bigint() { + this.type = 'bigint'; + return this; + } + get double() { this.type = 'double'; return this; @@ -176,6 +182,8 @@ export class Column { } else if (isSqlite) { options.type = 'real'; } + } else if (type === 'bigint') { + options.type = 'bigint'; } if ( diff --git a/packages/@n8n/db/src/migrations/migration-helpers.ts b/packages/@n8n/db/src/migrations/migration-helpers.ts index 1a8130a0d4d..ce4aeb497d4 100644 --- a/packages/@n8n/db/src/migrations/migration-helpers.ts +++ b/packages/@n8n/db/src/migrations/migration-helpers.ts @@ -178,6 +178,11 @@ const createContext = (queryRunner: QueryRunner, migration: Migration): Migratio }); export const wrapMigration = (migration: Migration) => { + const prototype = migration.prototype as unknown as { __n8n_wrapped?: boolean }; + if (prototype.__n8n_wrapped === true) { + return; + } + prototype.__n8n_wrapped = true; const { up, down } = migration.prototype; if (up) { Object.assign(migration.prototype, { diff --git a/packages/@n8n/db/src/migrations/mysqldb/index.ts b/packages/@n8n/db/src/migrations/mysqldb/index.ts index 86a8263d120..f024d4956c5 100644 --- a/packages/@n8n/db/src/migrations/mysqldb/index.ts +++ b/packages/@n8n/db/src/migrations/mysqldb/index.ts @@ -99,6 +99,7 @@ import { AddInputsOutputsToTestCaseExecution1752669793000 } from '../common/1752 import { CreateDataStoreTables1754475614601 } from '../common/1754475614601-CreateDataStoreTables'; import { ReplaceDataStoreTablesWithDataTables1754475614602 } from '../common/1754475614602-ReplaceDataStoreTablesWithDataTables'; import { AddTimestampsToRoleAndRoleIndexes1756906557570 } from '../common/1756906557570-AddTimestampsToRoleAndRoleIndexes'; +import { ChangeValueTypesForInsights1759399811000 } from '../common/1759399811000-ChangeValueTypesForInsights'; import type { Migration } from '../migration-types'; import { UpdateParentFolderIdColumn1740445074052 } from '../mysqldb/1740445074052-UpdateParentFolderIdColumn'; @@ -205,4 +206,5 @@ export const mysqlMigrations: Migration[] = [ AddTimestampsToRoleAndRoleIndexes1756906557570, AddProjectIdToVariableTable1758794506893, AddAudienceColumnToApiKeys1758731786132, + ChangeValueTypesForInsights1759399811000, ]; diff --git a/packages/@n8n/db/src/migrations/postgresdb/index.ts b/packages/@n8n/db/src/migrations/postgresdb/index.ts index d557a9c5df7..e157f681181 100644 --- a/packages/@n8n/db/src/migrations/postgresdb/index.ts +++ b/packages/@n8n/db/src/migrations/postgresdb/index.ts @@ -99,6 +99,7 @@ import { CreateDataStoreTables1754475614601 } from '../common/1754475614601-Crea import { ReplaceDataStoreTablesWithDataTables1754475614602 } from '../common/1754475614602-ReplaceDataStoreTablesWithDataTables'; import { AddTimestampsToRoleAndRoleIndexes1756906557570 } from '../common/1756906557570-AddTimestampsToRoleAndRoleIndexes'; import { AddAudienceColumnToApiKeys1758731786132 } from '../common/1758731786132-AddAudienceColumnToApiKey'; +import { ChangeValueTypesForInsights1759399811000 } from '../common/1759399811000-ChangeValueTypesForInsights'; import type { Migration } from '../migration-types'; export const postgresMigrations: Migration[] = [ @@ -203,4 +204,5 @@ export const postgresMigrations: Migration[] = [ AddTimestampsToRoleAndRoleIndexes1756906557570, AddProjectIdToVariableTable1758794506893, AddAudienceColumnToApiKeys1758731786132, + ChangeValueTypesForInsights1759399811000, ]; diff --git a/packages/@n8n/db/src/migrations/sqlite/index.ts b/packages/@n8n/db/src/migrations/sqlite/index.ts index e02222be28e..4aac2ef575e 100644 --- a/packages/@n8n/db/src/migrations/sqlite/index.ts +++ b/packages/@n8n/db/src/migrations/sqlite/index.ts @@ -94,6 +94,7 @@ import { AddInputsOutputsToTestCaseExecution1752669793000 } from '../common/1752 import { CreateDataStoreTables1754475614601 } from '../common/1754475614601-CreateDataStoreTables'; import { ReplaceDataStoreTablesWithDataTables1754475614602 } from '../common/1754475614602-ReplaceDataStoreTablesWithDataTables'; import { AddTimestampsToRoleAndRoleIndexes1756906557570 } from '../common/1756906557570-AddTimestampsToRoleAndRoleIndexes'; +import { ChangeValueTypesForInsights1759399811000 } from '../common/1759399811000-ChangeValueTypesForInsights'; import type { Migration } from '../migration-types'; import { LinkRoleToProjectRelationTable1753953244168 } from './../common/1753953244168-LinkRoleToProjectRelationTable'; import { AddProjectIdToVariableTable1758794506893 } from './1758794506893-AddProjectIdToVariableTable'; @@ -197,6 +198,7 @@ const sqliteMigrations: Migration[] = [ AddTimestampsToRoleAndRoleIndexes1756906557570, AddProjectIdToVariableTable1758794506893, AddAudienceColumnToApiKeys1758731786132, + ChangeValueTypesForInsights1759399811000, ]; export { sqliteMigrations }; diff --git a/packages/cli/src/modules/insights/__tests__/insights-bigint-migration.integration.test.ts b/packages/cli/src/modules/insights/__tests__/insights-bigint-migration.integration.test.ts new file mode 100644 index 00000000000..769b8f301f2 --- /dev/null +++ b/packages/cli/src/modules/insights/__tests__/insights-bigint-migration.integration.test.ts @@ -0,0 +1,343 @@ +import { createTeamProject, createWorkflow, testDb, testModules } from '@n8n/backend-test-utils'; +import { Container } from '@n8n/di'; +import { DateTime } from 'luxon'; + +import { InsightsRawRepository } from '@/modules/insights/database/repositories/insights-raw.repository'; + +import { + createRawInsightsEvent, + createRawInsightsEvents, +} from '../database/entities/__tests__/db-utils'; +import { InsightsByPeriodRepository } from '../database/repositories/insights-by-period.repository'; +import { InsightsCompactionService } from '../insights-compaction.service'; + +beforeAll(async () => { + await testModules.loadModules(['insights']); + await testDb.init(); +}); + +beforeEach(async () => { + await testDb.truncate([ + 'InsightsRaw', + 'InsightsByPeriod', + 'InsightsMetadata', + 'WorkflowEntity', + 'Project', + ]); +}); + +// Terminate DB once after all tests complete +afterAll(async () => { + await testDb.terminate(); +}); + +describe('BigInt migration validation', () => { + describe('Store value exceeding 32-bit integer maximum', () => { + test('should store and retrieve values larger than 2^31', async () => { + // ARRANGE + const insightsRawRepository = Container.get(InsightsRawRepository); + const project = await createTeamProject(); + const workflow = await createWorkflow({}, project); + + // Values exceeding 32-bit signed integer maximum (2,147,483,647) + const largeValue1 = 2_147_483_648; // 2^31 + const largeValue2 = 5_000_000_000; + + // ACT + const event1 = await createRawInsightsEvent(workflow, { + type: 'success', + value: largeValue1, + timestamp: DateTime.utc(), + }); + + const event2 = await createRawInsightsEvent(workflow, { + type: 'success', + value: largeValue2, + timestamp: DateTime.utc(), + }); + + // ASSERT + // Verify the events were stored with exact values (no overflow) + expect(event1.value).toBe(largeValue1); + expect(event2.value).toBe(largeValue2); + + // Verify retrieval from database returns exact values + const retrievedEvents = await insightsRawRepository.find(); + expect(retrievedEvents).toHaveLength(2); + + const retrieved1 = retrievedEvents.find((e) => e.id === event1.id); + const retrieved2 = retrievedEvents.find((e) => e.id === event2.id); + + expect(retrieved1?.value).toBe(largeValue1); + expect(retrieved2?.value).toBe(largeValue2); + }); + }); + + describe('Compaction sum exceeding 32-bit integer maximum', () => { + test('should correctly sum large values during compaction without overflow', async () => { + // ARRANGE + const insightsCompactionService = Container.get(InsightsCompactionService); + const insightsRawRepository = Container.get(InsightsRawRepository); + const insightsByPeriodRepository = Container.get(InsightsByPeriodRepository); + + const project = await createTeamProject(); + const workflow = await createWorkflow({}, project); + + // Create 3 events with large values that sum to exceed 32-bit max + // 1,800,000,000 * 3 = 5,400,000,000 (exceeds 2^31 - 1 = 2,147,483,647) + const eventValue = 1_800_000_000; + const expectedSum = 5_400_000_000; + + const timestamp = DateTime.utc().startOf('hour'); + + // Create 3 events in the same hour period for the same workflow + const events = [ + { type: 'success' as const, value: eventValue, timestamp }, + { type: 'success' as const, value: eventValue, timestamp: timestamp.plus({ minutes: 10 }) }, + { type: 'success' as const, value: eventValue, timestamp: timestamp.plus({ minutes: 20 }) }, + ]; + + await createRawInsightsEvents(workflow, events); + + // ACT + await insightsCompactionService.compactRawToHour(); + + // ASSERT + // Verify raw events are compacted (removed) + await expect(insightsRawRepository.count()).resolves.toBe(0); + + // Verify compacted event has correct sum (no overflow) + const compactedEvents = await insightsByPeriodRepository.find(); + expect(compactedEvents).toHaveLength(1); + expect(compactedEvents[0].value).toBe(expectedSum); + expect(compactedEvents[0].type).toBe('success'); + }); + }); + + describe('Maximum safe integer boundary validation', () => { + test('should handle Number.MAX_SAFE_INTEGER and arithmetic operations', async () => { + // ARRANGE + const insightsCompactionService = Container.get(InsightsCompactionService); + const insightsRawRepository = Container.get(InsightsRawRepository); + const insightsByPeriodRepository = Container.get(InsightsByPeriodRepository); + + const project = await createTeamProject(); + const workflow = await createWorkflow({}, project); + + // Maximum safe integer in JavaScript (2^53 - 1 = 9,007,199,254,740,991) + const maxSafeInteger = Number.MAX_SAFE_INTEGER; + + const timestamp = DateTime.utc().startOf('hour'); + + // ACT - Test storing MAX_SAFE_INTEGER + const event = await createRawInsightsEvent(workflow, { + type: 'runtime_ms', + value: maxSafeInteger, + timestamp, + }); + + // ASSERT - Verify exact storage and retrieval + expect(event.value).toBe(maxSafeInteger); + + const retrieved = await insightsRawRepository.findOne({ where: { id: event.id } }); + expect(retrieved?.value).toBe(maxSafeInteger); + + // Clean up for next test + await testDb.truncate(['InsightsRaw', 'InsightsByPeriod', 'InsightsMetadata']); + + // ACT - Test arithmetic with large values (50% of MAX_SAFE_INTEGER each) + const workflow2 = await createWorkflow({}, project); + const halfMaxSafe = Math.floor(Number.MAX_SAFE_INTEGER / 2); + + await createRawInsightsEvents(workflow2, [ + { type: 'time_saved_min', value: halfMaxSafe, timestamp }, + { type: 'time_saved_min', value: halfMaxSafe, timestamp: timestamp.plus({ minutes: 5 }) }, + ]); + + await insightsCompactionService.compactRawToHour(); + + // ASSERT - Verify compaction sum is correct + await expect(insightsRawRepository.count()).resolves.toBe(0); + const compacted = await insightsByPeriodRepository.find(); + expect(compacted).toHaveLength(1); + + // Sum should be close to MAX_SAFE_INTEGER (allowing for floor rounding) + expect(compacted[0].value).toBe(halfMaxSafe * 2); + expect(compacted[0].value).toBeLessThanOrEqual(Number.MAX_SAFE_INTEGER); + }); + + test('should demonstrate precision loss for values exceeding MAX_SAFE_INTEGER', async () => { + // ARRANGE + const insightsRawRepository = Container.get(InsightsRawRepository); + const project = await createTeamProject(); + const workflow = await createWorkflow({}, project); + + const timestamp = DateTime.utc(); + + // JavaScript Number type cannot safely represent integers beyond MAX_SAFE_INTEGER + // MAX_SAFE_INTEGER = 9,007,199,254,740,991 (2^53 - 1) + const maxSafeInteger = Number.MAX_SAFE_INTEGER; // 9,007,199,254,740,991 + + // Values BEYOND MAX_SAFE_INTEGER will experience precision loss in JavaScript + // Note: Database stores as bigint (no precision loss), but JS Number loses precision + // Beyond MAX_SAFE_INTEGER, consecutive integers cannot be represented uniquely + const unsafeValue1 = maxSafeInteger + 1; // Should be 9,007,199,254,740,992 + const unsafeValue2 = maxSafeInteger + 2; // Should be 9,007,199,254,740,993 + + // ASSERT - Demonstrate precision loss BEFORE storing + // Both values are NOT safe integers (precision cannot be guaranteed) + expect(Number.isSafeInteger(unsafeValue1)).toBe(false); + expect(Number.isSafeInteger(unsafeValue2)).toBe(false); + + // Critical demonstration: JavaScript rounds both values to the SAME number + // This proves precision loss - two different values become identical + expect(unsafeValue1).toBe(unsafeValue2); + expect(unsafeValue1).toBe(9007199254740992); // Both round to MAX_SAFE_INTEGER + 1 + + // ACT - Store values that exceed MAX_SAFE_INTEGER + const event1 = await createRawInsightsEvent(workflow, { + type: 'runtime_ms', + value: unsafeValue1, + timestamp, + }); + + const event2 = await createRawInsightsEvent(workflow, { + type: 'runtime_ms', + value: unsafeValue2, + timestamp: timestamp.plus({ seconds: 10 }), + }); + + // ASSERT - Stored values are identical (due to JS precision loss) + expect(event1.value).toBe(event2.value); // Both are 9007199254740992 + expect(event1.value).toBe(9007199254740992); + expect(event2.value).toBe(9007199254740992); + + // Retrieve from database - values remain identical due to JS Number conversion + const retrievedEvents = await insightsRawRepository.find({ order: { id: 'ASC' } }); + expect(retrievedEvents).toHaveLength(2); + + // Retrieved values are also identical (demonstrating persistent precision loss) + expect(retrievedEvents[0].value).toBe(retrievedEvents[1].value); + expect(retrievedEvents[0].value).toBe(9007199254740992); + expect(retrievedEvents[1].value).toBe(9007199254740992); + + // Both retrieved values are NOT safe integers + expect(Number.isSafeInteger(retrievedEvents[0].value)).toBe(false); + expect(Number.isSafeInteger(retrievedEvents[1].value)).toBe(false); + + // IMPORTANT: This test documents the current limitation. + // Two distinct values (MAX_SAFE_INTEGER + 1 and MAX_SAFE_INTEGER + 2) + // become indistinguishable due to JavaScript Number precision limits. + // + // To properly handle values > MAX_SAFE_INTEGER, we would need: + // 1. TypeORM transformer to convert bigint ↔ BigInt (not Number) + // 2. Application-level validation to reject values > MAX_SAFE_INTEGER + // 3. OR: Change entity type from 'number' to 'bigint' with proper transformers + }); + }); + + describe('Migration preserves existing small values', () => { + test('should correctly store and compact small integer values', async () => { + // ARRANGE + const insightsCompactionService = Container.get(InsightsCompactionService); + const insightsRawRepository = Container.get(InsightsRawRepository); + const insightsByPeriodRepository = Container.get(InsightsByPeriodRepository); + + const project = await createTeamProject(); + const workflow = await createWorkflow({}, project); + + const smallValue1 = 42; + const smallValue2 = 1_000_000; + const expectedSum = smallValue1 + smallValue2; + + const timestamp = DateTime.utc().startOf('hour'); + + // ACT + await createRawInsightsEvents(workflow, [ + { type: 'success', value: smallValue1, timestamp }, + { type: 'success', value: smallValue2, timestamp: timestamp.plus({ minutes: 15 }) }, + ]); + + // ASSERT - Verify retrieval of small values + const rawEvents = await insightsRawRepository.find({ order: { id: 'ASC' } }); + expect(rawEvents).toHaveLength(2); + expect(rawEvents[0].value).toBe(smallValue1); + expect(rawEvents[1].value).toBe(smallValue2); + + // ACT - Compact the events + await insightsCompactionService.compactRawToHour(); + + // ASSERT - Verify compaction sum is correct + await expect(insightsRawRepository.count()).resolves.toBe(0); + const compacted = await insightsByPeriodRepository.find(); + expect(compacted).toHaveLength(1); + expect(compacted[0].value).toBe(expectedSum); + }); + }); + + describe('Negative large values', () => { + test('should handle negative values exceeding 32-bit signed integer minimum', async () => { + // ARRANGE + const insightsRawRepository = Container.get(InsightsRawRepository); + const project = await createTeamProject(); + const workflow = await createWorkflow({}, project); + + // Values below 32-bit signed integer minimum (-2,147,483,648) + const negativeValue1 = -2_147_483_649; // Below 2^31 + const negativeValue2 = -5_000_000_000; + + // ACT + const event1 = await createRawInsightsEvent(workflow, { + type: 'time_saved_min', + value: negativeValue1, + timestamp: DateTime.utc(), + }); + + const event2 = await createRawInsightsEvent(workflow, { + type: 'time_saved_min', + value: negativeValue2, + timestamp: DateTime.utc(), + }); + + // ASSERT - Verify storage and retrieval of negative large values + expect(event1.value).toBe(negativeValue1); + expect(event2.value).toBe(negativeValue2); + + const retrievedEvents = await insightsRawRepository.find({ order: { id: 'ASC' } }); + expect(retrievedEvents).toHaveLength(2); + expect(retrievedEvents[0].value).toBe(negativeValue1); + expect(retrievedEvents[1].value).toBe(negativeValue2); + }); + + test('should correctly compact mixed positive and negative large values', async () => { + // ARRANGE + const insightsCompactionService = Container.get(InsightsCompactionService); + const insightsRawRepository = Container.get(InsightsRawRepository); + const insightsByPeriodRepository = Container.get(InsightsByPeriodRepository); + + const project = await createTeamProject(); + const workflow = await createWorkflow({}, project); + + const timestamp = DateTime.utc().startOf('hour'); + + // Mix of large positive and negative values + const positiveValue = 3_000_000_000; + const negativeValue = -2_500_000_000; + const expectedSum = positiveValue + negativeValue; // = 500,000,000 + + // ACT + await createRawInsightsEvents(workflow, [ + { type: 'runtime_ms', value: positiveValue, timestamp }, + { type: 'runtime_ms', value: negativeValue, timestamp: timestamp.plus({ minutes: 10 }) }, + ]); + + await insightsCompactionService.compactRawToHour(); + + // ASSERT + await expect(insightsRawRepository.count()).resolves.toBe(0); + const compacted = await insightsByPeriodRepository.find(); + expect(compacted).toHaveLength(1); + expect(compacted[0].value).toBe(expectedSum); + }); + }); +}); diff --git a/packages/cli/src/modules/insights/__tests__/insights-by-period-migration.test.ts b/packages/cli/src/modules/insights/__tests__/insights-by-period-migration.test.ts new file mode 100644 index 00000000000..344df4d64cb --- /dev/null +++ b/packages/cli/src/modules/insights/__tests__/insights-by-period-migration.test.ts @@ -0,0 +1,163 @@ +import { + createTestMigrationContext, + initDbUpToMigration, + runSingleMigration, + testModules, +} from '@n8n/backend-test-utils'; +import { DbConnection } from '@n8n/db'; +import { Container } from '@n8n/di'; +import { DataSource } from '@n8n/typeorm'; + +import { BOUNDARY_TEST_VALUES, insertPreMigrationPeriodData } from './migration-test-setup'; + +const MIGRATION_NAME = 'ChangeValueTypesForInsights1759399811000'; + +describe('ChangeValueTypesForInsights - insights_by_period table', () => { + let dataSource: DataSource; + + beforeAll(async () => { + await testModules.loadModules(['insights']); + + // Initialize DB connection without running migrations + const dbConnection = Container.get(DbConnection); + await dbConnection.init(); + + dataSource = Container.get(DataSource); + + // Run migrations up to (but not including) target migration + await initDbUpToMigration(MIGRATION_NAME); + }); + + afterAll(async () => { + const dbConnection = Container.get(DbConnection); + await dbConnection.close(); + }); + + describe('Schema Migration', () => { + it('should change value column from INT to BIGINT', async () => { + // Create migration context for schema queries + const context = createTestMigrationContext(dataSource); + + // Create prerequisite data for foreign keys using direct SQL + const projectTableName = context.escape.tableName('project'); + await context.queryRunner.query( + `INSERT INTO ${projectTableName} (id, name, type, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)`, + ['test-project-id', 'Test Project', 'personal', new Date(), new Date()], + ); + + const workflowTableName = context.escape.tableName('workflow_entity'); + await context.queryRunner.query( + `INSERT INTO ${workflowTableName} (id, name, active, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)`, + ['test-workflow-id', 'Test Workflow', false, new Date(), new Date()], + ); + + // Insert test metadata (required for foreign key) + const metaTableName = context.escape.tableName('insights_metadata'); + await context.queryRunner.query( + `INSERT INTO ${metaTableName} (workflowId, projectId, workflowName, projectName) VALUES (?, ?, ?, ?)`, + ['test-workflow-id', 'test-project-id', 'Test Workflow', 'Test Project'], + ); + const [metaRow] = await context.queryRunner.query( + `SELECT metaId FROM ${metaTableName} LIMIT 1`, + ); + const metaId = metaRow.metaId; + + // Insert test data in old INT schema + const testValues = [ + BOUNDARY_TEST_VALUES.zero, + BOUNDARY_TEST_VALUES.negativeOne, + BOUNDARY_TEST_VALUES.positiveOne, + BOUNDARY_TEST_VALUES.intMax, + BOUNDARY_TEST_VALUES.intMin, + ]; + await insertPreMigrationPeriodData(context, metaId, testValues); + + // Capture data before migration using SQL + const insightsByPeriodTableName = context.escape.tableName('insights_by_period'); + const beforeData = await context.queryRunner.query( + `SELECT id, metaId, value FROM ${insightsByPeriodTableName} ORDER BY id ASC`, + ); + expect(beforeData).toHaveLength(testValues.length); + + // Run the migration + await runSingleMigration(MIGRATION_NAME); + + // Release old query runner before creating new one + await context.queryRunner.release(); + + // Create fresh context after migration (dataSource is reinitialized) + const postMigrationContext = createTestMigrationContext(dataSource); + + // Verify schema change based on database type + if (postMigrationContext.isSqlite) { + const result = await postMigrationContext.queryRunner.query( + `PRAGMA table_info(${insightsByPeriodTableName})`, + ); + const valueColumn = result.find((col: { name: string }) => col.name === 'value'); + expect(valueColumn).toBeDefined(); + expect(valueColumn.type.toLowerCase()).toContain('bigint'); + } else if (postMigrationContext.isPostgres) { + const result = await postMigrationContext.queryRunner.query( + `SELECT data_type FROM information_schema.columns + WHERE table_name = ${insightsByPeriodTableName} AND column_name = 'value'`, + ); + expect(result[0].data_type).toBe('bigint'); + } else if (postMigrationContext.isMysql) { + const result = await postMigrationContext.queryRunner.query( + `SHOW COLUMNS FROM ${insightsByPeriodTableName} LIKE 'value'`, + ); + expect(result[0].Type.toLowerCase()).toContain('bigint'); + } + + // Verify data integrity after migration using SQL + const afterData = await postMigrationContext.queryRunner.query( + `SELECT id, metaId, value FROM ${insightsByPeriodTableName} ORDER BY id ASC`, + ); + expect(afterData).toHaveLength(beforeData.length); + + // Verify all values are preserved exactly + afterData.forEach( + (afterRow: { id: number; metaId: number; value: number }, index: number) => { + expect(afterRow.value).toBe(beforeData[index].value); + expect(afterRow.metaId).toBe(beforeData[index].metaId); + }, + ); + + // Cleanup + await postMigrationContext.queryRunner.release(); + }); + }); + + describe('Post-Migration Capacity', () => { + it('should accept values exceeding INT range', async () => { + const context = createTestMigrationContext(dataSource); + + // Get metaId from existing metadata + const metaTableName = context.escape.tableName('insights_metadata'); + const [metaRow] = await context.queryRunner.query( + `SELECT metaId FROM ${metaTableName} LIMIT 1`, + ); + const metaId = metaRow.metaId; + + // Insert value exceeding INT32 max + const beyondIntValue = BOUNDARY_TEST_VALUES.beyondInt; + const insightsByPeriodTableName = context.escape.tableName('insights_by_period'); + await context.queryRunner.query( + `INSERT INTO ${insightsByPeriodTableName} (metaId, type, value, periodUnit, periodStart) VALUES (?, ?, ?, ?, ?)`, + [metaId, 0, beyondIntValue, 0, new Date()], + ); + + // Verify retrieval using SQL + const [result] = await context.queryRunner.query( + `SELECT id, metaId, value FROM ${insightsByPeriodTableName} WHERE value = ?`, + [beyondIntValue], + ); + + expect(result).toBeDefined(); + expect(result.value).toBe(beyondIntValue); + + // Cleanup + await context.queryRunner.release(); + }); + }); +}); diff --git a/packages/cli/src/modules/insights/__tests__/insights-raw-migration.test.ts b/packages/cli/src/modules/insights/__tests__/insights-raw-migration.test.ts new file mode 100644 index 00000000000..257fa2cf1a8 --- /dev/null +++ b/packages/cli/src/modules/insights/__tests__/insights-raw-migration.test.ts @@ -0,0 +1,163 @@ +import { + createTestMigrationContext, + initDbUpToMigration, + runSingleMigration, + testModules, +} from '@n8n/backend-test-utils'; +import { DbConnection } from '@n8n/db'; +import { Container } from '@n8n/di'; +import { DataSource } from '@n8n/typeorm'; + +import { BOUNDARY_TEST_VALUES, insertPreMigrationRawData } from './migration-test-setup'; + +const MIGRATION_NAME = 'ChangeValueTypesForInsights1759399811000'; + +describe('ChangeValueTypesForInsights - insights_raw table', () => { + let dataSource: DataSource; + + beforeAll(async () => { + await testModules.loadModules(['insights']); + + // Initialize DB connection without running migrations + const dbConnection = Container.get(DbConnection); + await dbConnection.init(); + + dataSource = Container.get(DataSource); + + // Run migrations up to (but not including) target migration + await initDbUpToMigration(MIGRATION_NAME); + }); + + afterAll(async () => { + const dbConnection = Container.get(DbConnection); + await dbConnection.close(); + }); + + describe('Schema Migration', () => { + it('should change value column from INT to BIGINT', async () => { + // Create migration context for schema queries + const context = createTestMigrationContext(dataSource); + + // Create prerequisite data for foreign keys using direct SQL + const projectTableName = context.escape.tableName('project'); + await context.queryRunner.query( + `INSERT INTO ${projectTableName} (id, name, type, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)`, + ['test-project-id', 'Test Project', 'personal', new Date(), new Date()], + ); + + const workflowTableName = context.escape.tableName('workflow_entity'); + await context.queryRunner.query( + `INSERT INTO ${workflowTableName} (id, name, active, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)`, + ['test-workflow-id', 'Test Workflow', false, new Date(), new Date()], + ); + + // Insert test metadata (required for foreign key) + const metaTableName = context.escape.tableName('insights_metadata'); + await context.queryRunner.query( + `INSERT INTO ${metaTableName} (workflowId, projectId, workflowName, projectName) VALUES (?, ?, ?, ?)`, + ['test-workflow-id', 'test-project-id', 'Test Workflow', 'Test Project'], + ); + const [metaRow] = await context.queryRunner.query( + `SELECT metaId FROM ${metaTableName} LIMIT 1`, + ); + const metaId = metaRow.metaId; + + // Insert test data in old INT schema + const testValues = [ + BOUNDARY_TEST_VALUES.zero, + BOUNDARY_TEST_VALUES.negativeOne, + BOUNDARY_TEST_VALUES.positiveOne, + BOUNDARY_TEST_VALUES.intMax, + BOUNDARY_TEST_VALUES.intMin, + ]; + await insertPreMigrationRawData(context, metaId, testValues); + + // Capture data before migration using SQL + const insightsRawTableName = context.escape.tableName('insights_raw'); + const beforeData = await context.queryRunner.query( + `SELECT id, metaId, value FROM ${insightsRawTableName} ORDER BY id ASC`, + ); + expect(beforeData).toHaveLength(testValues.length); + + // Run the migration + await runSingleMigration(MIGRATION_NAME); + + // Release old query runner before creating new one + await context.queryRunner.release(); + + // Create fresh context after migration (dataSource is reinitialized) + const postMigrationContext = createTestMigrationContext(dataSource); + + // Verify schema change based on database type + if (postMigrationContext.isSqlite) { + const result = await postMigrationContext.queryRunner.query( + `PRAGMA table_info(${insightsRawTableName})`, + ); + const valueColumn = result.find((col: { name: string }) => col.name === 'value'); + expect(valueColumn).toBeDefined(); + expect(valueColumn.type.toLowerCase()).toContain('bigint'); + } else if (postMigrationContext.isPostgres) { + const result = await postMigrationContext.queryRunner.query( + `SELECT data_type FROM information_schema.columns + WHERE table_name = ${insightsRawTableName} AND column_name = 'value'`, + ); + expect(result[0].data_type).toBe('bigint'); + } else if (postMigrationContext.isMysql) { + const result = await postMigrationContext.queryRunner.query( + `SHOW COLUMNS FROM ${insightsRawTableName} LIKE 'value'`, + ); + expect(result[0].Type.toLowerCase()).toContain('bigint'); + } + + // Verify data integrity after migration using SQL + const afterData = await postMigrationContext.queryRunner.query( + `SELECT id, metaId, value FROM ${insightsRawTableName} ORDER BY id ASC`, + ); + expect(afterData).toHaveLength(beforeData.length); + + // Verify all values are preserved exactly + afterData.forEach( + (afterRow: { id: number; metaId: number; value: number }, index: number) => { + expect(afterRow.value).toBe(beforeData[index].value); + expect(afterRow.metaId).toBe(beforeData[index].metaId); + }, + ); + + // Cleanup + await postMigrationContext.queryRunner.release(); + }); + }); + + describe('Post-Migration Capacity', () => { + it('should accept values exceeding INT range', async () => { + const context = createTestMigrationContext(dataSource); + + // Get metaId from existing metadata + const metaTableName = context.escape.tableName('insights_metadata'); + const [metaRow] = await context.queryRunner.query( + `SELECT metaId FROM ${metaTableName} LIMIT 1`, + ); + const metaId = metaRow.metaId; + + // Insert value exceeding INT32 max + const beyondIntValue = BOUNDARY_TEST_VALUES.beyondInt; + const insightsRawTableName = context.escape.tableName('insights_raw'); + await context.queryRunner.query( + `INSERT INTO ${insightsRawTableName} (metaId, type, value, timestamp) VALUES (?, ?, ?, ?)`, + [metaId, 0, beyondIntValue, new Date()], + ); + + // Verify retrieval using SQL + const [result] = await context.queryRunner.query( + `SELECT id, metaId, value FROM ${insightsRawTableName} WHERE value = ?`, + [beyondIntValue], + ); + + expect(result).toBeDefined(); + expect(result.value).toBe(beyondIntValue); + + // Cleanup + await context.queryRunner.release(); + }); + }); +}); diff --git a/packages/cli/src/modules/insights/__tests__/migration-test-setup.ts b/packages/cli/src/modules/insights/__tests__/migration-test-setup.ts new file mode 100644 index 00000000000..394d48ead5a --- /dev/null +++ b/packages/cli/src/modules/insights/__tests__/migration-test-setup.ts @@ -0,0 +1,53 @@ +import type { TestMigrationContext } from '@n8n/backend-test-utils'; + +/** + * Test values for BIGINT migration validation. + * Covers INT32 boundaries and BIGINT capabilities. + */ +export const BOUNDARY_TEST_VALUES = { + zero: 0, + positiveOne: 1, + negativeOne: -1, + intMax: 2_147_483_647, // 2^31 - 1 (max signed 32-bit int) + intMin: -2_147_483_648, // -2^31 (min signed 32-bit int) + beyondInt: 2_147_483_648, // For post-migration testing (exceeds INT32) + bigintMax: 9_223_372_036_854_775_807n, // 2^63 - 1 (max signed 64-bit int) +} as const; + +/** + * Insert raw insights data using SQL (compatible with pre-migration INT schema). + * Uses database-specific helpers for table names and escaping. + */ +export async function insertPreMigrationRawData( + context: TestMigrationContext, + metaId: number, + testValues: number[], +): Promise { + const tableName = context.escape.tableName('insights_raw'); + for (const value of testValues) { + await context.queryRunner.query( + `INSERT INTO ${tableName} (metaId, type, value, timestamp) VALUES (?, ?, ?, ?)`, + [metaId, 0, value, new Date()], + ); + } +} + +/** + * Insert insights_by_period data using SQL (compatible with pre-migration INT schema). + * Uses database-specific helpers for table names and escaping. + */ +export async function insertPreMigrationPeriodData( + context: TestMigrationContext, + metaId: number, + testValues: number[], +): Promise { + const tableName = context.escape.tableName('insights_by_period'); + const baseDate = new Date('2025-01-01T00:00:00.000Z'); + for (let i = 0; i < testValues.length; i++) { + const periodStart = new Date(baseDate.getTime() + i * 3600000); // Add 1 hour per iteration + await context.queryRunner.query( + `INSERT INTO ${tableName} (metaId, type, value, periodUnit, periodStart) VALUES (?, ?, ?, ?, ?)`, + [metaId, 0, testValues[i], 0, periodStart.toISOString()], + ); + } +} diff --git a/packages/cli/src/modules/insights/database/entities/insights-by-period.ts b/packages/cli/src/modules/insights/database/entities/insights-by-period.ts index cef6aeefa8c..bf9c870c777 100644 --- a/packages/cli/src/modules/insights/database/entities/insights-by-period.ts +++ b/packages/cli/src/modules/insights/database/entities/insights-by-period.ts @@ -49,6 +49,11 @@ export class InsightsByPeriod extends BaseEntity { this.type_ = TypeToNumber[value]; } + /** + * Stored as BIGINT in database (see migration 1759399811000). + * JavaScript number type has precision limits at ±2^53-1 (9,007,199,254,740,991). + * Values exceeding Number.MAX_SAFE_INTEGER will lose precision. + */ @Column() value: number; diff --git a/packages/cli/src/modules/insights/database/entities/insights-raw.ts b/packages/cli/src/modules/insights/database/entities/insights-raw.ts index 57c3d30736b..34746394a30 100644 --- a/packages/cli/src/modules/insights/database/entities/insights-raw.ts +++ b/packages/cli/src/modules/insights/database/entities/insights-raw.ts @@ -38,6 +38,11 @@ export class InsightsRaw extends BaseEntity { this.type_ = TypeToNumber[value]; } + /** + * Stored as BIGINT in database (see migration 1759399811000). + * JavaScript number type has precision limits at ±2^53-1 (9,007,199,254,740,991). + * Values exceeding Number.MAX_SAFE_INTEGER will lose precision. + */ @Column() value: number; diff --git a/packages/cli/test/migration/migration-test-helpers.test.ts b/packages/cli/test/migration/migration-test-helpers.test.ts new file mode 100644 index 00000000000..c7468f0d602 --- /dev/null +++ b/packages/cli/test/migration/migration-test-helpers.test.ts @@ -0,0 +1,69 @@ +import { initDbUpToMigration, runSingleMigration } from '@n8n/backend-test-utils'; +import { DbConnection } from '@n8n/db'; +import { Container } from '@n8n/di'; +import { DataSource } from '@n8n/typeorm'; +import { UnexpectedError } from 'n8n-workflow'; + +describe('Migration Test Helpers', () => { + let dataSource: DataSource; + + beforeEach(async () => { + // Initialize connection without running migrations + const dbConnection = Container.get(DbConnection); + await dbConnection.init(); + dataSource = Container.get(DataSource); + }); + + afterEach(async () => { + const dbConnection = Container.get(DbConnection); + await dbConnection.close(); + }); + + describe('initDbUpToMigration', () => { + it('should throw error if migration not found', async () => { + await expect(initDbUpToMigration('NonExistentMigration')).rejects.toThrow( + new UnexpectedError('Migration "NonExistentMigration" not found'), + ); + }); + + it('should stop before specified migration', async () => { + const migrations = dataSource.options.migrations as Array<{ name: string }>; + expect(migrations.length).toBeGreaterThan(1); + + const secondMigrationName = migrations[1].name; + console.log('Running migrations up to ' + secondMigrationName); + await initDbUpToMigration(secondMigrationName); + console.log('Migrations executed up to ' + secondMigrationName); + + // Verify only first migration was executed + const executed = await dataSource.query('SELECT * FROM migrations ORDER BY timestamp'); + expect(executed).toHaveLength(1); + expect(executed[0].name).toBe(migrations[0].name); + }); + }); + + describe('runSingleMigration', () => { + it('should throw error if migration not found', async () => { + await expect(runSingleMigration('NonExistentMigration')).rejects.toThrow( + new UnexpectedError('Migration "NonExistentMigration" not found'), + ); + }); + + it('should run specific migration', async () => { + const migrations = dataSource.options.migrations as Array<{ name: string }>; + expect(migrations.length).toBeGreaterThan(1); + + const secondMigrationName = migrations[1].name; + console.log('Running migrations up to ' + secondMigrationName); + await initDbUpToMigration(secondMigrationName); + console.log('Migrations executed up to ' + secondMigrationName); + + await runSingleMigration(secondMigrationName); + + const executed = await dataSource.query('SELECT * FROM migrations ORDER BY timestamp'); + expect(executed).toHaveLength(2); + expect(executed[0].name).toBe(migrations[0].name); + expect(executed[1].name).toBe(secondMigrationName); + }); + }); +});