chore(core): Migrate insights values to bigint to avoid overflow (#20343)

This commit is contained in:
Andreas Fitzek
2025-10-10 11:32:01 +02:00
committed by GitHub
parent 596cdfec91
commit 4e84c53a47
16 changed files with 1123 additions and 1 deletions
@@ -0,0 +1,82 @@
# Migration Testing Helpers
This package provides utilities for testing database migrations by allowing you to stop before a specific migration, insert test data, and then run that migration.
## API
### `initDbUpToMigration(beforeMigrationName: string): Promise<void>`
Initializes the database and runs all migrations up to (but not including) the specified migration.
**Parameters:**
- `beforeMigrationName`: The class name of the migration to stop before (e.g., `'AddUserRole1234567890'`)
**Throws:**
- `UnexpectedError` if the migration is not found or database is not initialized
### `runSingleMigration(migrationName: string): Promise<void>`
Runs a single migration by name.
**Parameters:**
- `migrationName`: The class name of the migration to run (e.g., `'AddUserRole1234567890'`)
**Throws:**
- `UnexpectedError` if the migration is not found or database is not initialized
## Usage Example
```typescript
import { Container } from '@n8n/di';
import { DataSource } from '@n8n/typeorm';
import { initDbUpToMigration, runSingleMigration } from '@n8n/backend-test-utils';
describe('AddUserRole1234567890 Migration', () => {
let dataSource: DataSource;
beforeAll(async () => {
// Initialize database but stop BEFORE the migration we want to test
await initDbUpToMigration('AddUserRole1234567890');
dataSource = Container.get(DataSource);
});
it('should add role column to users table', async () => {
// Insert test data in the OLD schema (before migration)
// You should not use Repositories, because these will break after schema changes
// over time.
await dataSource.query(`
INSERT INTO users (id, email, password)
VALUES (1, 'test@example.com', 'hashed_password')
`);
// Run the migration
await runSingleMigration('AddUserRole1234567890');
// Verify the migration worked correctly
const users = await dataSource.query('SELECT * FROM users WHERE id = 1');
expect(users[0].role).toBe('member'); // Default role was added
});
});
```
## How It Works
1. **`initDbUpToMigration`**:
- Gets all available migrations from TypeORM DataSource
- Finds the target migration by name
- Temporarily replaces the migrations array with only migrations before the target
- Wraps and runs those migrations
- Restores the full migrations array
2. **`runSingleMigration`**:
- Finds the specific migration by name
- Temporarily replaces the migrations array with only that migration
- Wraps and runs that single migration
- Restores the full migrations array
## Important Notes
- These functions must be used with an initialized database connection (after `dbConnection.init()`)
- Do NOT call `dbConnection.migrate()` before using these helpers - they replace that step
- Migration wrapping is idempotent - migrations won't be double-wrapped
- The full migrations array is always restored after operations complete (even on error)
@@ -10,3 +10,4 @@ export * as testModules from './test-modules';
export * from './db/workflows';
export * from './db/projects';
export * from './mocking';
export * from './migration-test-helpers';
@@ -0,0 +1,124 @@
import { GlobalConfig } from '@n8n/config';
import { type DatabaseType, DbConnection, type Migration } from '@n8n/db';
import { Container } from '@n8n/di';
import { DataSource, type QueryRunner } from '@n8n/typeorm';
import { UnexpectedError } from 'n8n-workflow';
async function reinitializeDataConnection(): Promise<void> {
const dbConnection = Container.get(DbConnection);
await dbConnection.close();
await dbConnection.init();
}
/**
* Test migration context with database-specific helpers (similar to MigrationContext).
*/
export interface TestMigrationContext {
queryRunner: QueryRunner;
tablePrefix: string;
dbType: DatabaseType;
isMysql: boolean;
isSqlite: boolean;
isPostgres: boolean;
escape: {
columnName(name: string): string;
tableName(name: string): string;
indexName(name: string): string;
};
}
/**
* Create a test migration context with database-specific helpers.
* Provides the same utilities that migrations have access to.
*/
export function createTestMigrationContext(dataSource: DataSource): TestMigrationContext {
const globalConfig = Container.get(GlobalConfig);
const dbType = globalConfig.database.type;
const tablePrefix = globalConfig.database.tablePrefix;
const queryRunner = dataSource.createQueryRunner();
return {
queryRunner,
tablePrefix,
dbType,
isMysql: ['mariadb', 'mysqldb'].includes(dbType),
isSqlite: dbType === 'sqlite',
isPostgres: dbType === 'postgresdb',
escape: {
columnName: (name) => queryRunner.connection.driver.escape(name),
tableName: (name) => queryRunner.connection.driver.escape(`${tablePrefix}${name}`),
indexName: (name) => queryRunner.connection.driver.escape(`IDX_${tablePrefix}${name}`),
},
};
}
/**
* Initialize database and run all migrations up to (but not including) the specified migration.
* Useful for testing data transformations by inserting test data before a migration runs.
*
* @param beforeMigrationName - The class name of the migration to stop before (e.g., 'AddUserRole1234567890')
* @throws {UnexpectedError} If the migration is not found or database is not initialized
*/
export async function initDbUpToMigration(beforeMigrationName: string): Promise<void> {
const dataSource = Container.get(DataSource);
if (!Array.isArray(dataSource.options.migrations)) {
throw new UnexpectedError('Database migrations are not an array');
}
const allMigrations = dataSource.options.migrations as Migration[];
const targetIndex = allMigrations.findIndex((m) => m.name === beforeMigrationName);
if (targetIndex === -1) {
throw new UnexpectedError(`Migration "${beforeMigrationName}" not found`);
}
// Temporarily replace migrations array with subset
const migrationsToRun = allMigrations.slice(0, targetIndex);
(dataSource.options as { migrations: Migration[] }).migrations = migrationsToRun;
try {
// Need to reinitialize the data source to rebuild the migrations
await reinitializeDataConnection();
// Run migrations
await Container.get(DbConnection).migrate();
} finally {
// Restore full migrations array
(dataSource.options as { migrations: Migration[] }).migrations = allMigrations;
// Need to reinitialize the data source to rebuild the migrations
await reinitializeDataConnection();
}
}
/**
* Run a single migration by name.
* Useful for testing a specific migration after inserting test data.
*
* @param migrationName - The class name of the migration to run (e.g., 'AddUserRole1234567890')
* @throws {UnexpectedError} If the migration is not found or database is not initialized
*/
export async function runSingleMigration(migrationName: string): Promise<void> {
const dataSource = Container.get(DataSource);
const allMigrations = dataSource.options.migrations as Migration[];
const migration = allMigrations.find((m) => m.name === migrationName);
if (!migration) {
throw new UnexpectedError(`Migration "${migrationName}" not found`);
}
// Temporarily replace migrations array with only the target migration
(dataSource.options as { migrations: Migration[] }).migrations = [migration];
try {
// Need to reinitialize the data source to rebuild the migrations
await reinitializeDataConnection();
// Run migrations
await Container.get(DbConnection).migrate();
} finally {
// Restore full migrations array
(dataSource.options as { migrations: Migration[] }).migrations = allMigrations;
// Need to reinitialize the data source to rebuild the migrations
await reinitializeDataConnection();
}
}
@@ -0,0 +1,95 @@
import type { IrreversibleMigration, MigrationContext } from '../migration-types';
const INSIGHTS_RAW_TABLE_NAME = 'insights_raw';
const INSIGHTS_RAW_TEMP_TABLE_NAME = 'temp_insights_raw';
const INSIGHTS_BY_PERIOD_TABLE_NAME = 'insights_by_period';
const INSIGHTS_BY_PERIOD_TEMP_TABLE_NAME = 'temp_insights_by_period';
const INSIGHTS_METADATA_TABLE_NAME = 'insights_metadata';
const VALUE_COLUMN_NAME = 'value';
export class ChangeValueTypesForInsights1759399811000 implements IrreversibleMigration {
async up({
isSqlite,
isMysql,
isPostgres,
escape,
copyTable,
queryRunner,
schemaBuilder: { createTable, column, dropTable },
}: MigrationContext) {
const insightsRawTable = escape.tableName(INSIGHTS_RAW_TABLE_NAME);
const insightsByPeriodTable = escape.tableName(INSIGHTS_BY_PERIOD_TABLE_NAME);
const valueColumnName = escape.columnName(VALUE_COLUMN_NAME);
if (isSqlite) {
const tempInsightsByPeriodTable = escape.tableName(INSIGHTS_BY_PERIOD_TEMP_TABLE_NAME);
const tempInsightsRawTable = escape.tableName(INSIGHTS_RAW_TEMP_TABLE_NAME);
const typeComment = '0: time_saved_minutes, 1: runtime_milliseconds, 2: success, 3: failure';
// Create temporary raw table with new value type, copy data, remove the original table and rename the temporary table
await createTable(INSIGHTS_RAW_TEMP_TABLE_NAME)
.withColumns(
column('id').int.primary.autoGenerate2,
column('metaId').int.notNull,
column('type').int.notNull.comment(typeComment),
column('value').bigint.notNull,
column('timestamp').timestampTimezone(0).default('CURRENT_TIMESTAMP').notNull,
)
.withForeignKey('metaId', {
tableName: INSIGHTS_METADATA_TABLE_NAME,
columnName: 'metaId',
onDelete: 'CASCADE',
});
// Copy data from the original table to the temporary table
await copyTable(INSIGHTS_RAW_TABLE_NAME, INSIGHTS_RAW_TEMP_TABLE_NAME);
// drop the original table
await dropTable(INSIGHTS_RAW_TABLE_NAME);
// rename the temporary table to the original table name
await queryRunner.query(`ALTER TABLE ${tempInsightsRawTable} RENAME TO ${insightsRawTable};`);
await createTable(INSIGHTS_BY_PERIOD_TEMP_TABLE_NAME)
.withColumns(
column('id').int.primary.autoGenerate2,
column('metaId').int.notNull,
column('type').int.notNull.comment(typeComment),
column('value').bigint.notNull,
column('periodUnit').int.notNull.comment('0: hour, 1: day, 2: week'),
column('periodStart').default('CURRENT_TIMESTAMP').timestampTimezone(0),
)
.withForeignKey('metaId', {
tableName: INSIGHTS_METADATA_TABLE_NAME,
columnName: 'metaId',
onDelete: 'CASCADE',
})
.withIndexOn(['periodStart', 'type', 'periodUnit', 'metaId'], true);
// Copy data from the original table to the temporary table
await copyTable(INSIGHTS_BY_PERIOD_TABLE_NAME, INSIGHTS_BY_PERIOD_TEMP_TABLE_NAME);
// drop the original table
await dropTable(INSIGHTS_BY_PERIOD_TABLE_NAME);
// rename the temporary table to the original table name
await queryRunner.query(
`ALTER TABLE ${tempInsightsByPeriodTable} RENAME TO ${insightsByPeriodTable};`,
);
} else if (isMysql) {
await queryRunner.query(
`ALTER TABLE ${insightsRawTable} MODIFY COLUMN ${valueColumnName} BIGINT NOT NULL;`,
);
await queryRunner.query(
`ALTER TABLE ${insightsByPeriodTable} MODIFY COLUMN ${valueColumnName} BIGINT NOT NULL;`,
);
} else if (isPostgres) {
await queryRunner.query(
`ALTER TABLE ${insightsRawTable} ALTER COLUMN ${valueColumnName} TYPE BIGINT;`,
);
await queryRunner.query(
`ALTER TABLE ${insightsByPeriodTable} ALTER COLUMN ${valueColumnName} TYPE BIGINT;`,
);
}
}
}
@@ -10,7 +10,8 @@ export class Column {
| 'timestamptz'
| 'timestamp'
| 'uuid'
| 'double';
| 'double'
| 'bigint';
private isGenerated = false;
@@ -40,6 +41,11 @@ export class Column {
return this;
}
get bigint() {
this.type = 'bigint';
return this;
}
get double() {
this.type = 'double';
return this;
@@ -176,6 +182,8 @@ export class Column {
} else if (isSqlite) {
options.type = 'real';
}
} else if (type === 'bigint') {
options.type = 'bigint';
}
if (
@@ -178,6 +178,11 @@ const createContext = (queryRunner: QueryRunner, migration: Migration): Migratio
});
export const wrapMigration = (migration: Migration) => {
const prototype = migration.prototype as unknown as { __n8n_wrapped?: boolean };
if (prototype.__n8n_wrapped === true) {
return;
}
prototype.__n8n_wrapped = true;
const { up, down } = migration.prototype;
if (up) {
Object.assign(migration.prototype, {
@@ -99,6 +99,7 @@ import { AddInputsOutputsToTestCaseExecution1752669793000 } from '../common/1752
import { CreateDataStoreTables1754475614601 } from '../common/1754475614601-CreateDataStoreTables';
import { ReplaceDataStoreTablesWithDataTables1754475614602 } from '../common/1754475614602-ReplaceDataStoreTablesWithDataTables';
import { AddTimestampsToRoleAndRoleIndexes1756906557570 } from '../common/1756906557570-AddTimestampsToRoleAndRoleIndexes';
import { ChangeValueTypesForInsights1759399811000 } from '../common/1759399811000-ChangeValueTypesForInsights';
import type { Migration } from '../migration-types';
import { UpdateParentFolderIdColumn1740445074052 } from '../mysqldb/1740445074052-UpdateParentFolderIdColumn';
@@ -205,4 +206,5 @@ export const mysqlMigrations: Migration[] = [
AddTimestampsToRoleAndRoleIndexes1756906557570,
AddProjectIdToVariableTable1758794506893,
AddAudienceColumnToApiKeys1758731786132,
ChangeValueTypesForInsights1759399811000,
];
@@ -99,6 +99,7 @@ import { CreateDataStoreTables1754475614601 } from '../common/1754475614601-Crea
import { ReplaceDataStoreTablesWithDataTables1754475614602 } from '../common/1754475614602-ReplaceDataStoreTablesWithDataTables';
import { AddTimestampsToRoleAndRoleIndexes1756906557570 } from '../common/1756906557570-AddTimestampsToRoleAndRoleIndexes';
import { AddAudienceColumnToApiKeys1758731786132 } from '../common/1758731786132-AddAudienceColumnToApiKey';
import { ChangeValueTypesForInsights1759399811000 } from '../common/1759399811000-ChangeValueTypesForInsights';
import type { Migration } from '../migration-types';
export const postgresMigrations: Migration[] = [
@@ -203,4 +204,5 @@ export const postgresMigrations: Migration[] = [
AddTimestampsToRoleAndRoleIndexes1756906557570,
AddProjectIdToVariableTable1758794506893,
AddAudienceColumnToApiKeys1758731786132,
ChangeValueTypesForInsights1759399811000,
];
@@ -94,6 +94,7 @@ import { AddInputsOutputsToTestCaseExecution1752669793000 } from '../common/1752
import { CreateDataStoreTables1754475614601 } from '../common/1754475614601-CreateDataStoreTables';
import { ReplaceDataStoreTablesWithDataTables1754475614602 } from '../common/1754475614602-ReplaceDataStoreTablesWithDataTables';
import { AddTimestampsToRoleAndRoleIndexes1756906557570 } from '../common/1756906557570-AddTimestampsToRoleAndRoleIndexes';
import { ChangeValueTypesForInsights1759399811000 } from '../common/1759399811000-ChangeValueTypesForInsights';
import type { Migration } from '../migration-types';
import { LinkRoleToProjectRelationTable1753953244168 } from './../common/1753953244168-LinkRoleToProjectRelationTable';
import { AddProjectIdToVariableTable1758794506893 } from './1758794506893-AddProjectIdToVariableTable';
@@ -197,6 +198,7 @@ const sqliteMigrations: Migration[] = [
AddTimestampsToRoleAndRoleIndexes1756906557570,
AddProjectIdToVariableTable1758794506893,
AddAudienceColumnToApiKeys1758731786132,
ChangeValueTypesForInsights1759399811000,
];
export { sqliteMigrations };
@@ -0,0 +1,343 @@
import { createTeamProject, createWorkflow, testDb, testModules } from '@n8n/backend-test-utils';
import { Container } from '@n8n/di';
import { DateTime } from 'luxon';
import { InsightsRawRepository } from '@/modules/insights/database/repositories/insights-raw.repository';
import {
createRawInsightsEvent,
createRawInsightsEvents,
} from '../database/entities/__tests__/db-utils';
import { InsightsByPeriodRepository } from '../database/repositories/insights-by-period.repository';
import { InsightsCompactionService } from '../insights-compaction.service';
beforeAll(async () => {
await testModules.loadModules(['insights']);
await testDb.init();
});
beforeEach(async () => {
await testDb.truncate([
'InsightsRaw',
'InsightsByPeriod',
'InsightsMetadata',
'WorkflowEntity',
'Project',
]);
});
// Terminate DB once after all tests complete
afterAll(async () => {
await testDb.terminate();
});
describe('BigInt migration validation', () => {
describe('Store value exceeding 32-bit integer maximum', () => {
test('should store and retrieve values larger than 2^31', async () => {
// ARRANGE
const insightsRawRepository = Container.get(InsightsRawRepository);
const project = await createTeamProject();
const workflow = await createWorkflow({}, project);
// Values exceeding 32-bit signed integer maximum (2,147,483,647)
const largeValue1 = 2_147_483_648; // 2^31
const largeValue2 = 5_000_000_000;
// ACT
const event1 = await createRawInsightsEvent(workflow, {
type: 'success',
value: largeValue1,
timestamp: DateTime.utc(),
});
const event2 = await createRawInsightsEvent(workflow, {
type: 'success',
value: largeValue2,
timestamp: DateTime.utc(),
});
// ASSERT
// Verify the events were stored with exact values (no overflow)
expect(event1.value).toBe(largeValue1);
expect(event2.value).toBe(largeValue2);
// Verify retrieval from database returns exact values
const retrievedEvents = await insightsRawRepository.find();
expect(retrievedEvents).toHaveLength(2);
const retrieved1 = retrievedEvents.find((e) => e.id === event1.id);
const retrieved2 = retrievedEvents.find((e) => e.id === event2.id);
expect(retrieved1?.value).toBe(largeValue1);
expect(retrieved2?.value).toBe(largeValue2);
});
});
describe('Compaction sum exceeding 32-bit integer maximum', () => {
test('should correctly sum large values during compaction without overflow', async () => {
// ARRANGE
const insightsCompactionService = Container.get(InsightsCompactionService);
const insightsRawRepository = Container.get(InsightsRawRepository);
const insightsByPeriodRepository = Container.get(InsightsByPeriodRepository);
const project = await createTeamProject();
const workflow = await createWorkflow({}, project);
// Create 3 events with large values that sum to exceed 32-bit max
// 1,800,000,000 * 3 = 5,400,000,000 (exceeds 2^31 - 1 = 2,147,483,647)
const eventValue = 1_800_000_000;
const expectedSum = 5_400_000_000;
const timestamp = DateTime.utc().startOf('hour');
// Create 3 events in the same hour period for the same workflow
const events = [
{ type: 'success' as const, value: eventValue, timestamp },
{ type: 'success' as const, value: eventValue, timestamp: timestamp.plus({ minutes: 10 }) },
{ type: 'success' as const, value: eventValue, timestamp: timestamp.plus({ minutes: 20 }) },
];
await createRawInsightsEvents(workflow, events);
// ACT
await insightsCompactionService.compactRawToHour();
// ASSERT
// Verify raw events are compacted (removed)
await expect(insightsRawRepository.count()).resolves.toBe(0);
// Verify compacted event has correct sum (no overflow)
const compactedEvents = await insightsByPeriodRepository.find();
expect(compactedEvents).toHaveLength(1);
expect(compactedEvents[0].value).toBe(expectedSum);
expect(compactedEvents[0].type).toBe('success');
});
});
describe('Maximum safe integer boundary validation', () => {
test('should handle Number.MAX_SAFE_INTEGER and arithmetic operations', async () => {
// ARRANGE
const insightsCompactionService = Container.get(InsightsCompactionService);
const insightsRawRepository = Container.get(InsightsRawRepository);
const insightsByPeriodRepository = Container.get(InsightsByPeriodRepository);
const project = await createTeamProject();
const workflow = await createWorkflow({}, project);
// Maximum safe integer in JavaScript (2^53 - 1 = 9,007,199,254,740,991)
const maxSafeInteger = Number.MAX_SAFE_INTEGER;
const timestamp = DateTime.utc().startOf('hour');
// ACT - Test storing MAX_SAFE_INTEGER
const event = await createRawInsightsEvent(workflow, {
type: 'runtime_ms',
value: maxSafeInteger,
timestamp,
});
// ASSERT - Verify exact storage and retrieval
expect(event.value).toBe(maxSafeInteger);
const retrieved = await insightsRawRepository.findOne({ where: { id: event.id } });
expect(retrieved?.value).toBe(maxSafeInteger);
// Clean up for next test
await testDb.truncate(['InsightsRaw', 'InsightsByPeriod', 'InsightsMetadata']);
// ACT - Test arithmetic with large values (50% of MAX_SAFE_INTEGER each)
const workflow2 = await createWorkflow({}, project);
const halfMaxSafe = Math.floor(Number.MAX_SAFE_INTEGER / 2);
await createRawInsightsEvents(workflow2, [
{ type: 'time_saved_min', value: halfMaxSafe, timestamp },
{ type: 'time_saved_min', value: halfMaxSafe, timestamp: timestamp.plus({ minutes: 5 }) },
]);
await insightsCompactionService.compactRawToHour();
// ASSERT - Verify compaction sum is correct
await expect(insightsRawRepository.count()).resolves.toBe(0);
const compacted = await insightsByPeriodRepository.find();
expect(compacted).toHaveLength(1);
// Sum should be close to MAX_SAFE_INTEGER (allowing for floor rounding)
expect(compacted[0].value).toBe(halfMaxSafe * 2);
expect(compacted[0].value).toBeLessThanOrEqual(Number.MAX_SAFE_INTEGER);
});
test('should demonstrate precision loss for values exceeding MAX_SAFE_INTEGER', async () => {
// ARRANGE
const insightsRawRepository = Container.get(InsightsRawRepository);
const project = await createTeamProject();
const workflow = await createWorkflow({}, project);
const timestamp = DateTime.utc();
// JavaScript Number type cannot safely represent integers beyond MAX_SAFE_INTEGER
// MAX_SAFE_INTEGER = 9,007,199,254,740,991 (2^53 - 1)
const maxSafeInteger = Number.MAX_SAFE_INTEGER; // 9,007,199,254,740,991
// Values BEYOND MAX_SAFE_INTEGER will experience precision loss in JavaScript
// Note: Database stores as bigint (no precision loss), but JS Number loses precision
// Beyond MAX_SAFE_INTEGER, consecutive integers cannot be represented uniquely
const unsafeValue1 = maxSafeInteger + 1; // Should be 9,007,199,254,740,992
const unsafeValue2 = maxSafeInteger + 2; // Should be 9,007,199,254,740,993
// ASSERT - Demonstrate precision loss BEFORE storing
// Both values are NOT safe integers (precision cannot be guaranteed)
expect(Number.isSafeInteger(unsafeValue1)).toBe(false);
expect(Number.isSafeInteger(unsafeValue2)).toBe(false);
// Critical demonstration: JavaScript rounds both values to the SAME number
// This proves precision loss - two different values become identical
expect(unsafeValue1).toBe(unsafeValue2);
expect(unsafeValue1).toBe(9007199254740992); // Both round to MAX_SAFE_INTEGER + 1
// ACT - Store values that exceed MAX_SAFE_INTEGER
const event1 = await createRawInsightsEvent(workflow, {
type: 'runtime_ms',
value: unsafeValue1,
timestamp,
});
const event2 = await createRawInsightsEvent(workflow, {
type: 'runtime_ms',
value: unsafeValue2,
timestamp: timestamp.plus({ seconds: 10 }),
});
// ASSERT - Stored values are identical (due to JS precision loss)
expect(event1.value).toBe(event2.value); // Both are 9007199254740992
expect(event1.value).toBe(9007199254740992);
expect(event2.value).toBe(9007199254740992);
// Retrieve from database - values remain identical due to JS Number conversion
const retrievedEvents = await insightsRawRepository.find({ order: { id: 'ASC' } });
expect(retrievedEvents).toHaveLength(2);
// Retrieved values are also identical (demonstrating persistent precision loss)
expect(retrievedEvents[0].value).toBe(retrievedEvents[1].value);
expect(retrievedEvents[0].value).toBe(9007199254740992);
expect(retrievedEvents[1].value).toBe(9007199254740992);
// Both retrieved values are NOT safe integers
expect(Number.isSafeInteger(retrievedEvents[0].value)).toBe(false);
expect(Number.isSafeInteger(retrievedEvents[1].value)).toBe(false);
// IMPORTANT: This test documents the current limitation.
// Two distinct values (MAX_SAFE_INTEGER + 1 and MAX_SAFE_INTEGER + 2)
// become indistinguishable due to JavaScript Number precision limits.
//
// To properly handle values > MAX_SAFE_INTEGER, we would need:
// 1. TypeORM transformer to convert bigint ↔ BigInt (not Number)
// 2. Application-level validation to reject values > MAX_SAFE_INTEGER
// 3. OR: Change entity type from 'number' to 'bigint' with proper transformers
});
});
describe('Migration preserves existing small values', () => {
test('should correctly store and compact small integer values', async () => {
// ARRANGE
const insightsCompactionService = Container.get(InsightsCompactionService);
const insightsRawRepository = Container.get(InsightsRawRepository);
const insightsByPeriodRepository = Container.get(InsightsByPeriodRepository);
const project = await createTeamProject();
const workflow = await createWorkflow({}, project);
const smallValue1 = 42;
const smallValue2 = 1_000_000;
const expectedSum = smallValue1 + smallValue2;
const timestamp = DateTime.utc().startOf('hour');
// ACT
await createRawInsightsEvents(workflow, [
{ type: 'success', value: smallValue1, timestamp },
{ type: 'success', value: smallValue2, timestamp: timestamp.plus({ minutes: 15 }) },
]);
// ASSERT - Verify retrieval of small values
const rawEvents = await insightsRawRepository.find({ order: { id: 'ASC' } });
expect(rawEvents).toHaveLength(2);
expect(rawEvents[0].value).toBe(smallValue1);
expect(rawEvents[1].value).toBe(smallValue2);
// ACT - Compact the events
await insightsCompactionService.compactRawToHour();
// ASSERT - Verify compaction sum is correct
await expect(insightsRawRepository.count()).resolves.toBe(0);
const compacted = await insightsByPeriodRepository.find();
expect(compacted).toHaveLength(1);
expect(compacted[0].value).toBe(expectedSum);
});
});
describe('Negative large values', () => {
test('should handle negative values exceeding 32-bit signed integer minimum', async () => {
// ARRANGE
const insightsRawRepository = Container.get(InsightsRawRepository);
const project = await createTeamProject();
const workflow = await createWorkflow({}, project);
// Values below 32-bit signed integer minimum (-2,147,483,648)
const negativeValue1 = -2_147_483_649; // Below 2^31
const negativeValue2 = -5_000_000_000;
// ACT
const event1 = await createRawInsightsEvent(workflow, {
type: 'time_saved_min',
value: negativeValue1,
timestamp: DateTime.utc(),
});
const event2 = await createRawInsightsEvent(workflow, {
type: 'time_saved_min',
value: negativeValue2,
timestamp: DateTime.utc(),
});
// ASSERT - Verify storage and retrieval of negative large values
expect(event1.value).toBe(negativeValue1);
expect(event2.value).toBe(negativeValue2);
const retrievedEvents = await insightsRawRepository.find({ order: { id: 'ASC' } });
expect(retrievedEvents).toHaveLength(2);
expect(retrievedEvents[0].value).toBe(negativeValue1);
expect(retrievedEvents[1].value).toBe(negativeValue2);
});
test('should correctly compact mixed positive and negative large values', async () => {
// ARRANGE
const insightsCompactionService = Container.get(InsightsCompactionService);
const insightsRawRepository = Container.get(InsightsRawRepository);
const insightsByPeriodRepository = Container.get(InsightsByPeriodRepository);
const project = await createTeamProject();
const workflow = await createWorkflow({}, project);
const timestamp = DateTime.utc().startOf('hour');
// Mix of large positive and negative values
const positiveValue = 3_000_000_000;
const negativeValue = -2_500_000_000;
const expectedSum = positiveValue + negativeValue; // = 500,000,000
// ACT
await createRawInsightsEvents(workflow, [
{ type: 'runtime_ms', value: positiveValue, timestamp },
{ type: 'runtime_ms', value: negativeValue, timestamp: timestamp.plus({ minutes: 10 }) },
]);
await insightsCompactionService.compactRawToHour();
// ASSERT
await expect(insightsRawRepository.count()).resolves.toBe(0);
const compacted = await insightsByPeriodRepository.find();
expect(compacted).toHaveLength(1);
expect(compacted[0].value).toBe(expectedSum);
});
});
});
@@ -0,0 +1,163 @@
import {
createTestMigrationContext,
initDbUpToMigration,
runSingleMigration,
testModules,
} from '@n8n/backend-test-utils';
import { DbConnection } from '@n8n/db';
import { Container } from '@n8n/di';
import { DataSource } from '@n8n/typeorm';
import { BOUNDARY_TEST_VALUES, insertPreMigrationPeriodData } from './migration-test-setup';
const MIGRATION_NAME = 'ChangeValueTypesForInsights1759399811000';
describe('ChangeValueTypesForInsights - insights_by_period table', () => {
let dataSource: DataSource;
beforeAll(async () => {
await testModules.loadModules(['insights']);
// Initialize DB connection without running migrations
const dbConnection = Container.get(DbConnection);
await dbConnection.init();
dataSource = Container.get(DataSource);
// Run migrations up to (but not including) target migration
await initDbUpToMigration(MIGRATION_NAME);
});
afterAll(async () => {
const dbConnection = Container.get(DbConnection);
await dbConnection.close();
});
describe('Schema Migration', () => {
it('should change value column from INT to BIGINT', async () => {
// Create migration context for schema queries
const context = createTestMigrationContext(dataSource);
// Create prerequisite data for foreign keys using direct SQL
const projectTableName = context.escape.tableName('project');
await context.queryRunner.query(
`INSERT INTO ${projectTableName} (id, name, type, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)`,
['test-project-id', 'Test Project', 'personal', new Date(), new Date()],
);
const workflowTableName = context.escape.tableName('workflow_entity');
await context.queryRunner.query(
`INSERT INTO ${workflowTableName} (id, name, active, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)`,
['test-workflow-id', 'Test Workflow', false, new Date(), new Date()],
);
// Insert test metadata (required for foreign key)
const metaTableName = context.escape.tableName('insights_metadata');
await context.queryRunner.query(
`INSERT INTO ${metaTableName} (workflowId, projectId, workflowName, projectName) VALUES (?, ?, ?, ?)`,
['test-workflow-id', 'test-project-id', 'Test Workflow', 'Test Project'],
);
const [metaRow] = await context.queryRunner.query(
`SELECT metaId FROM ${metaTableName} LIMIT 1`,
);
const metaId = metaRow.metaId;
// Insert test data in old INT schema
const testValues = [
BOUNDARY_TEST_VALUES.zero,
BOUNDARY_TEST_VALUES.negativeOne,
BOUNDARY_TEST_VALUES.positiveOne,
BOUNDARY_TEST_VALUES.intMax,
BOUNDARY_TEST_VALUES.intMin,
];
await insertPreMigrationPeriodData(context, metaId, testValues);
// Capture data before migration using SQL
const insightsByPeriodTableName = context.escape.tableName('insights_by_period');
const beforeData = await context.queryRunner.query(
`SELECT id, metaId, value FROM ${insightsByPeriodTableName} ORDER BY id ASC`,
);
expect(beforeData).toHaveLength(testValues.length);
// Run the migration
await runSingleMigration(MIGRATION_NAME);
// Release old query runner before creating new one
await context.queryRunner.release();
// Create fresh context after migration (dataSource is reinitialized)
const postMigrationContext = createTestMigrationContext(dataSource);
// Verify schema change based on database type
if (postMigrationContext.isSqlite) {
const result = await postMigrationContext.queryRunner.query(
`PRAGMA table_info(${insightsByPeriodTableName})`,
);
const valueColumn = result.find((col: { name: string }) => col.name === 'value');
expect(valueColumn).toBeDefined();
expect(valueColumn.type.toLowerCase()).toContain('bigint');
} else if (postMigrationContext.isPostgres) {
const result = await postMigrationContext.queryRunner.query(
`SELECT data_type FROM information_schema.columns
WHERE table_name = ${insightsByPeriodTableName} AND column_name = 'value'`,
);
expect(result[0].data_type).toBe('bigint');
} else if (postMigrationContext.isMysql) {
const result = await postMigrationContext.queryRunner.query(
`SHOW COLUMNS FROM ${insightsByPeriodTableName} LIKE 'value'`,
);
expect(result[0].Type.toLowerCase()).toContain('bigint');
}
// Verify data integrity after migration using SQL
const afterData = await postMigrationContext.queryRunner.query(
`SELECT id, metaId, value FROM ${insightsByPeriodTableName} ORDER BY id ASC`,
);
expect(afterData).toHaveLength(beforeData.length);
// Verify all values are preserved exactly
afterData.forEach(
(afterRow: { id: number; metaId: number; value: number }, index: number) => {
expect(afterRow.value).toBe(beforeData[index].value);
expect(afterRow.metaId).toBe(beforeData[index].metaId);
},
);
// Cleanup
await postMigrationContext.queryRunner.release();
});
});
describe('Post-Migration Capacity', () => {
it('should accept values exceeding INT range', async () => {
const context = createTestMigrationContext(dataSource);
// Get metaId from existing metadata
const metaTableName = context.escape.tableName('insights_metadata');
const [metaRow] = await context.queryRunner.query(
`SELECT metaId FROM ${metaTableName} LIMIT 1`,
);
const metaId = metaRow.metaId;
// Insert value exceeding INT32 max
const beyondIntValue = BOUNDARY_TEST_VALUES.beyondInt;
const insightsByPeriodTableName = context.escape.tableName('insights_by_period');
await context.queryRunner.query(
`INSERT INTO ${insightsByPeriodTableName} (metaId, type, value, periodUnit, periodStart) VALUES (?, ?, ?, ?, ?)`,
[metaId, 0, beyondIntValue, 0, new Date()],
);
// Verify retrieval using SQL
const [result] = await context.queryRunner.query(
`SELECT id, metaId, value FROM ${insightsByPeriodTableName} WHERE value = ?`,
[beyondIntValue],
);
expect(result).toBeDefined();
expect(result.value).toBe(beyondIntValue);
// Cleanup
await context.queryRunner.release();
});
});
});
@@ -0,0 +1,163 @@
import {
createTestMigrationContext,
initDbUpToMigration,
runSingleMigration,
testModules,
} from '@n8n/backend-test-utils';
import { DbConnection } from '@n8n/db';
import { Container } from '@n8n/di';
import { DataSource } from '@n8n/typeorm';
import { BOUNDARY_TEST_VALUES, insertPreMigrationRawData } from './migration-test-setup';
const MIGRATION_NAME = 'ChangeValueTypesForInsights1759399811000';
describe('ChangeValueTypesForInsights - insights_raw table', () => {
let dataSource: DataSource;
beforeAll(async () => {
await testModules.loadModules(['insights']);
// Initialize DB connection without running migrations
const dbConnection = Container.get(DbConnection);
await dbConnection.init();
dataSource = Container.get(DataSource);
// Run migrations up to (but not including) target migration
await initDbUpToMigration(MIGRATION_NAME);
});
afterAll(async () => {
const dbConnection = Container.get(DbConnection);
await dbConnection.close();
});
describe('Schema Migration', () => {
it('should change value column from INT to BIGINT', async () => {
// Create migration context for schema queries
const context = createTestMigrationContext(dataSource);
// Create prerequisite data for foreign keys using direct SQL
const projectTableName = context.escape.tableName('project');
await context.queryRunner.query(
`INSERT INTO ${projectTableName} (id, name, type, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)`,
['test-project-id', 'Test Project', 'personal', new Date(), new Date()],
);
const workflowTableName = context.escape.tableName('workflow_entity');
await context.queryRunner.query(
`INSERT INTO ${workflowTableName} (id, name, active, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)`,
['test-workflow-id', 'Test Workflow', false, new Date(), new Date()],
);
// Insert test metadata (required for foreign key)
const metaTableName = context.escape.tableName('insights_metadata');
await context.queryRunner.query(
`INSERT INTO ${metaTableName} (workflowId, projectId, workflowName, projectName) VALUES (?, ?, ?, ?)`,
['test-workflow-id', 'test-project-id', 'Test Workflow', 'Test Project'],
);
const [metaRow] = await context.queryRunner.query(
`SELECT metaId FROM ${metaTableName} LIMIT 1`,
);
const metaId = metaRow.metaId;
// Insert test data in old INT schema
const testValues = [
BOUNDARY_TEST_VALUES.zero,
BOUNDARY_TEST_VALUES.negativeOne,
BOUNDARY_TEST_VALUES.positiveOne,
BOUNDARY_TEST_VALUES.intMax,
BOUNDARY_TEST_VALUES.intMin,
];
await insertPreMigrationRawData(context, metaId, testValues);
// Capture data before migration using SQL
const insightsRawTableName = context.escape.tableName('insights_raw');
const beforeData = await context.queryRunner.query(
`SELECT id, metaId, value FROM ${insightsRawTableName} ORDER BY id ASC`,
);
expect(beforeData).toHaveLength(testValues.length);
// Run the migration
await runSingleMigration(MIGRATION_NAME);
// Release old query runner before creating new one
await context.queryRunner.release();
// Create fresh context after migration (dataSource is reinitialized)
const postMigrationContext = createTestMigrationContext(dataSource);
// Verify schema change based on database type
if (postMigrationContext.isSqlite) {
const result = await postMigrationContext.queryRunner.query(
`PRAGMA table_info(${insightsRawTableName})`,
);
const valueColumn = result.find((col: { name: string }) => col.name === 'value');
expect(valueColumn).toBeDefined();
expect(valueColumn.type.toLowerCase()).toContain('bigint');
} else if (postMigrationContext.isPostgres) {
const result = await postMigrationContext.queryRunner.query(
`SELECT data_type FROM information_schema.columns
WHERE table_name = ${insightsRawTableName} AND column_name = 'value'`,
);
expect(result[0].data_type).toBe('bigint');
} else if (postMigrationContext.isMysql) {
const result = await postMigrationContext.queryRunner.query(
`SHOW COLUMNS FROM ${insightsRawTableName} LIKE 'value'`,
);
expect(result[0].Type.toLowerCase()).toContain('bigint');
}
// Verify data integrity after migration using SQL
const afterData = await postMigrationContext.queryRunner.query(
`SELECT id, metaId, value FROM ${insightsRawTableName} ORDER BY id ASC`,
);
expect(afterData).toHaveLength(beforeData.length);
// Verify all values are preserved exactly
afterData.forEach(
(afterRow: { id: number; metaId: number; value: number }, index: number) => {
expect(afterRow.value).toBe(beforeData[index].value);
expect(afterRow.metaId).toBe(beforeData[index].metaId);
},
);
// Cleanup
await postMigrationContext.queryRunner.release();
});
});
describe('Post-Migration Capacity', () => {
it('should accept values exceeding INT range', async () => {
const context = createTestMigrationContext(dataSource);
// Get metaId from existing metadata
const metaTableName = context.escape.tableName('insights_metadata');
const [metaRow] = await context.queryRunner.query(
`SELECT metaId FROM ${metaTableName} LIMIT 1`,
);
const metaId = metaRow.metaId;
// Insert value exceeding INT32 max
const beyondIntValue = BOUNDARY_TEST_VALUES.beyondInt;
const insightsRawTableName = context.escape.tableName('insights_raw');
await context.queryRunner.query(
`INSERT INTO ${insightsRawTableName} (metaId, type, value, timestamp) VALUES (?, ?, ?, ?)`,
[metaId, 0, beyondIntValue, new Date()],
);
// Verify retrieval using SQL
const [result] = await context.queryRunner.query(
`SELECT id, metaId, value FROM ${insightsRawTableName} WHERE value = ?`,
[beyondIntValue],
);
expect(result).toBeDefined();
expect(result.value).toBe(beyondIntValue);
// Cleanup
await context.queryRunner.release();
});
});
});
@@ -0,0 +1,53 @@
import type { TestMigrationContext } from '@n8n/backend-test-utils';
/**
* Test values for BIGINT migration validation.
* Covers INT32 boundaries and BIGINT capabilities.
*/
export const BOUNDARY_TEST_VALUES = {
zero: 0,
positiveOne: 1,
negativeOne: -1,
intMax: 2_147_483_647, // 2^31 - 1 (max signed 32-bit int)
intMin: -2_147_483_648, // -2^31 (min signed 32-bit int)
beyondInt: 2_147_483_648, // For post-migration testing (exceeds INT32)
bigintMax: 9_223_372_036_854_775_807n, // 2^63 - 1 (max signed 64-bit int)
} as const;
/**
* Insert raw insights data using SQL (compatible with pre-migration INT schema).
* Uses database-specific helpers for table names and escaping.
*/
export async function insertPreMigrationRawData(
context: TestMigrationContext,
metaId: number,
testValues: number[],
): Promise<void> {
const tableName = context.escape.tableName('insights_raw');
for (const value of testValues) {
await context.queryRunner.query(
`INSERT INTO ${tableName} (metaId, type, value, timestamp) VALUES (?, ?, ?, ?)`,
[metaId, 0, value, new Date()],
);
}
}
/**
* Insert insights_by_period data using SQL (compatible with pre-migration INT schema).
* Uses database-specific helpers for table names and escaping.
*/
export async function insertPreMigrationPeriodData(
context: TestMigrationContext,
metaId: number,
testValues: number[],
): Promise<void> {
const tableName = context.escape.tableName('insights_by_period');
const baseDate = new Date('2025-01-01T00:00:00.000Z');
for (let i = 0; i < testValues.length; i++) {
const periodStart = new Date(baseDate.getTime() + i * 3600000); // Add 1 hour per iteration
await context.queryRunner.query(
`INSERT INTO ${tableName} (metaId, type, value, periodUnit, periodStart) VALUES (?, ?, ?, ?, ?)`,
[metaId, 0, testValues[i], 0, periodStart.toISOString()],
);
}
}
@@ -49,6 +49,11 @@ export class InsightsByPeriod extends BaseEntity {
this.type_ = TypeToNumber[value];
}
/**
* Stored as BIGINT in database (see migration 1759399811000).
* JavaScript number type has precision limits at ±2^53-1 (9,007,199,254,740,991).
* Values exceeding Number.MAX_SAFE_INTEGER will lose precision.
*/
@Column()
value: number;
@@ -38,6 +38,11 @@ export class InsightsRaw extends BaseEntity {
this.type_ = TypeToNumber[value];
}
/**
* Stored as BIGINT in database (see migration 1759399811000).
* JavaScript number type has precision limits at ±2^53-1 (9,007,199,254,740,991).
* Values exceeding Number.MAX_SAFE_INTEGER will lose precision.
*/
@Column()
value: number;
@@ -0,0 +1,69 @@
import { initDbUpToMigration, runSingleMigration } from '@n8n/backend-test-utils';
import { DbConnection } from '@n8n/db';
import { Container } from '@n8n/di';
import { DataSource } from '@n8n/typeorm';
import { UnexpectedError } from 'n8n-workflow';
describe('Migration Test Helpers', () => {
let dataSource: DataSource;
beforeEach(async () => {
// Initialize connection without running migrations
const dbConnection = Container.get(DbConnection);
await dbConnection.init();
dataSource = Container.get(DataSource);
});
afterEach(async () => {
const dbConnection = Container.get(DbConnection);
await dbConnection.close();
});
describe('initDbUpToMigration', () => {
it('should throw error if migration not found', async () => {
await expect(initDbUpToMigration('NonExistentMigration')).rejects.toThrow(
new UnexpectedError('Migration "NonExistentMigration" not found'),
);
});
it('should stop before specified migration', async () => {
const migrations = dataSource.options.migrations as Array<{ name: string }>;
expect(migrations.length).toBeGreaterThan(1);
const secondMigrationName = migrations[1].name;
console.log('Running migrations up to ' + secondMigrationName);
await initDbUpToMigration(secondMigrationName);
console.log('Migrations executed up to ' + secondMigrationName);
// Verify only first migration was executed
const executed = await dataSource.query('SELECT * FROM migrations ORDER BY timestamp');
expect(executed).toHaveLength(1);
expect(executed[0].name).toBe(migrations[0].name);
});
});
describe('runSingleMigration', () => {
it('should throw error if migration not found', async () => {
await expect(runSingleMigration('NonExistentMigration')).rejects.toThrow(
new UnexpectedError('Migration "NonExistentMigration" not found'),
);
});
it('should run specific migration', async () => {
const migrations = dataSource.options.migrations as Array<{ name: string }>;
expect(migrations.length).toBeGreaterThan(1);
const secondMigrationName = migrations[1].name;
console.log('Running migrations up to ' + secondMigrationName);
await initDbUpToMigration(secondMigrationName);
console.log('Migrations executed up to ' + secondMigrationName);
await runSingleMigration(secondMigrationName);
const executed = await dataSource.query('SELECT * FROM migrations ORDER BY timestamp');
expect(executed).toHaveLength(2);
expect(executed[0].name).toBe(migrations[0].name);
expect(executed[1].name).toBe(secondMigrationName);
});
});
});