mirror of
https://github.com/n8n-io/n8n.git
synced 2026-09-24 23:22:38 +08:00
chore(core): Migrate insights values to bigint to avoid overflow (#20343)
This commit is contained in:
@@ -0,0 +1,82 @@
|
||||
# Migration Testing Helpers
|
||||
|
||||
This package provides utilities for testing database migrations by allowing you to stop before a specific migration, insert test data, and then run that migration.
|
||||
|
||||
## API
|
||||
|
||||
### `initDbUpToMigration(beforeMigrationName: string): Promise<void>`
|
||||
|
||||
Initializes the database and runs all migrations up to (but not including) the specified migration.
|
||||
|
||||
**Parameters:**
|
||||
- `beforeMigrationName`: The class name of the migration to stop before (e.g., `'AddUserRole1234567890'`)
|
||||
|
||||
**Throws:**
|
||||
- `UnexpectedError` if the migration is not found or database is not initialized
|
||||
|
||||
### `runSingleMigration(migrationName: string): Promise<void>`
|
||||
|
||||
Runs a single migration by name.
|
||||
|
||||
**Parameters:**
|
||||
- `migrationName`: The class name of the migration to run (e.g., `'AddUserRole1234567890'`)
|
||||
|
||||
**Throws:**
|
||||
- `UnexpectedError` if the migration is not found or database is not initialized
|
||||
|
||||
## Usage Example
|
||||
|
||||
```typescript
|
||||
import { Container } from '@n8n/di';
|
||||
import { DataSource } from '@n8n/typeorm';
|
||||
import { initDbUpToMigration, runSingleMigration } from '@n8n/backend-test-utils';
|
||||
|
||||
describe('AddUserRole1234567890 Migration', () => {
|
||||
let dataSource: DataSource;
|
||||
|
||||
beforeAll(async () => {
|
||||
// Initialize database but stop BEFORE the migration we want to test
|
||||
await initDbUpToMigration('AddUserRole1234567890');
|
||||
dataSource = Container.get(DataSource);
|
||||
});
|
||||
|
||||
it('should add role column to users table', async () => {
|
||||
// Insert test data in the OLD schema (before migration)
|
||||
// You should not use Repositories, because these will break after schema changes
|
||||
// over time.
|
||||
await dataSource.query(`
|
||||
INSERT INTO users (id, email, password)
|
||||
VALUES (1, 'test@example.com', 'hashed_password')
|
||||
`);
|
||||
|
||||
// Run the migration
|
||||
await runSingleMigration('AddUserRole1234567890');
|
||||
|
||||
// Verify the migration worked correctly
|
||||
const users = await dataSource.query('SELECT * FROM users WHERE id = 1');
|
||||
expect(users[0].role).toBe('member'); // Default role was added
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
1. **`initDbUpToMigration`**:
|
||||
- Gets all available migrations from TypeORM DataSource
|
||||
- Finds the target migration by name
|
||||
- Temporarily replaces the migrations array with only migrations before the target
|
||||
- Wraps and runs those migrations
|
||||
- Restores the full migrations array
|
||||
|
||||
2. **`runSingleMigration`**:
|
||||
- Finds the specific migration by name
|
||||
- Temporarily replaces the migrations array with only that migration
|
||||
- Wraps and runs that single migration
|
||||
- Restores the full migrations array
|
||||
|
||||
## Important Notes
|
||||
|
||||
- These functions must be used with an initialized database connection (after `dbConnection.init()`)
|
||||
- Do NOT call `dbConnection.migrate()` before using these helpers - they replace that step
|
||||
- Migration wrapping is idempotent - migrations won't be double-wrapped
|
||||
- The full migrations array is always restored after operations complete (even on error)
|
||||
@@ -10,3 +10,4 @@ export * as testModules from './test-modules';
|
||||
export * from './db/workflows';
|
||||
export * from './db/projects';
|
||||
export * from './mocking';
|
||||
export * from './migration-test-helpers';
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
import { GlobalConfig } from '@n8n/config';
|
||||
import { type DatabaseType, DbConnection, type Migration } from '@n8n/db';
|
||||
import { Container } from '@n8n/di';
|
||||
import { DataSource, type QueryRunner } from '@n8n/typeorm';
|
||||
import { UnexpectedError } from 'n8n-workflow';
|
||||
|
||||
async function reinitializeDataConnection(): Promise<void> {
|
||||
const dbConnection = Container.get(DbConnection);
|
||||
await dbConnection.close();
|
||||
await dbConnection.init();
|
||||
}
|
||||
|
||||
/**
|
||||
* Test migration context with database-specific helpers (similar to MigrationContext).
|
||||
*/
|
||||
export interface TestMigrationContext {
|
||||
queryRunner: QueryRunner;
|
||||
tablePrefix: string;
|
||||
dbType: DatabaseType;
|
||||
isMysql: boolean;
|
||||
isSqlite: boolean;
|
||||
isPostgres: boolean;
|
||||
escape: {
|
||||
columnName(name: string): string;
|
||||
tableName(name: string): string;
|
||||
indexName(name: string): string;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a test migration context with database-specific helpers.
|
||||
* Provides the same utilities that migrations have access to.
|
||||
*/
|
||||
export function createTestMigrationContext(dataSource: DataSource): TestMigrationContext {
|
||||
const globalConfig = Container.get(GlobalConfig);
|
||||
const dbType = globalConfig.database.type;
|
||||
const tablePrefix = globalConfig.database.tablePrefix;
|
||||
const queryRunner = dataSource.createQueryRunner();
|
||||
|
||||
return {
|
||||
queryRunner,
|
||||
tablePrefix,
|
||||
dbType,
|
||||
isMysql: ['mariadb', 'mysqldb'].includes(dbType),
|
||||
isSqlite: dbType === 'sqlite',
|
||||
isPostgres: dbType === 'postgresdb',
|
||||
escape: {
|
||||
columnName: (name) => queryRunner.connection.driver.escape(name),
|
||||
tableName: (name) => queryRunner.connection.driver.escape(`${tablePrefix}${name}`),
|
||||
indexName: (name) => queryRunner.connection.driver.escape(`IDX_${tablePrefix}${name}`),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize database and run all migrations up to (but not including) the specified migration.
|
||||
* Useful for testing data transformations by inserting test data before a migration runs.
|
||||
*
|
||||
* @param beforeMigrationName - The class name of the migration to stop before (e.g., 'AddUserRole1234567890')
|
||||
* @throws {UnexpectedError} If the migration is not found or database is not initialized
|
||||
*/
|
||||
export async function initDbUpToMigration(beforeMigrationName: string): Promise<void> {
|
||||
const dataSource = Container.get(DataSource);
|
||||
|
||||
if (!Array.isArray(dataSource.options.migrations)) {
|
||||
throw new UnexpectedError('Database migrations are not an array');
|
||||
}
|
||||
|
||||
const allMigrations = dataSource.options.migrations as Migration[];
|
||||
const targetIndex = allMigrations.findIndex((m) => m.name === beforeMigrationName);
|
||||
|
||||
if (targetIndex === -1) {
|
||||
throw new UnexpectedError(`Migration "${beforeMigrationName}" not found`);
|
||||
}
|
||||
|
||||
// Temporarily replace migrations array with subset
|
||||
const migrationsToRun = allMigrations.slice(0, targetIndex);
|
||||
(dataSource.options as { migrations: Migration[] }).migrations = migrationsToRun;
|
||||
|
||||
try {
|
||||
// Need to reinitialize the data source to rebuild the migrations
|
||||
await reinitializeDataConnection();
|
||||
// Run migrations
|
||||
await Container.get(DbConnection).migrate();
|
||||
} finally {
|
||||
// Restore full migrations array
|
||||
(dataSource.options as { migrations: Migration[] }).migrations = allMigrations;
|
||||
// Need to reinitialize the data source to rebuild the migrations
|
||||
await reinitializeDataConnection();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a single migration by name.
|
||||
* Useful for testing a specific migration after inserting test data.
|
||||
*
|
||||
* @param migrationName - The class name of the migration to run (e.g., 'AddUserRole1234567890')
|
||||
* @throws {UnexpectedError} If the migration is not found or database is not initialized
|
||||
*/
|
||||
export async function runSingleMigration(migrationName: string): Promise<void> {
|
||||
const dataSource = Container.get(DataSource);
|
||||
|
||||
const allMigrations = dataSource.options.migrations as Migration[];
|
||||
const migration = allMigrations.find((m) => m.name === migrationName);
|
||||
|
||||
if (!migration) {
|
||||
throw new UnexpectedError(`Migration "${migrationName}" not found`);
|
||||
}
|
||||
|
||||
// Temporarily replace migrations array with only the target migration
|
||||
(dataSource.options as { migrations: Migration[] }).migrations = [migration];
|
||||
|
||||
try {
|
||||
// Need to reinitialize the data source to rebuild the migrations
|
||||
await reinitializeDataConnection();
|
||||
// Run migrations
|
||||
await Container.get(DbConnection).migrate();
|
||||
} finally {
|
||||
// Restore full migrations array
|
||||
(dataSource.options as { migrations: Migration[] }).migrations = allMigrations;
|
||||
// Need to reinitialize the data source to rebuild the migrations
|
||||
await reinitializeDataConnection();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
import type { IrreversibleMigration, MigrationContext } from '../migration-types';
|
||||
|
||||
const INSIGHTS_RAW_TABLE_NAME = 'insights_raw';
|
||||
const INSIGHTS_RAW_TEMP_TABLE_NAME = 'temp_insights_raw';
|
||||
const INSIGHTS_BY_PERIOD_TABLE_NAME = 'insights_by_period';
|
||||
const INSIGHTS_BY_PERIOD_TEMP_TABLE_NAME = 'temp_insights_by_period';
|
||||
const INSIGHTS_METADATA_TABLE_NAME = 'insights_metadata';
|
||||
const VALUE_COLUMN_NAME = 'value';
|
||||
|
||||
export class ChangeValueTypesForInsights1759399811000 implements IrreversibleMigration {
|
||||
async up({
|
||||
isSqlite,
|
||||
isMysql,
|
||||
isPostgres,
|
||||
escape,
|
||||
copyTable,
|
||||
queryRunner,
|
||||
schemaBuilder: { createTable, column, dropTable },
|
||||
}: MigrationContext) {
|
||||
const insightsRawTable = escape.tableName(INSIGHTS_RAW_TABLE_NAME);
|
||||
const insightsByPeriodTable = escape.tableName(INSIGHTS_BY_PERIOD_TABLE_NAME);
|
||||
const valueColumnName = escape.columnName(VALUE_COLUMN_NAME);
|
||||
|
||||
if (isSqlite) {
|
||||
const tempInsightsByPeriodTable = escape.tableName(INSIGHTS_BY_PERIOD_TEMP_TABLE_NAME);
|
||||
const tempInsightsRawTable = escape.tableName(INSIGHTS_RAW_TEMP_TABLE_NAME);
|
||||
const typeComment = '0: time_saved_minutes, 1: runtime_milliseconds, 2: success, 3: failure';
|
||||
|
||||
// Create temporary raw table with new value type, copy data, remove the original table and rename the temporary table
|
||||
await createTable(INSIGHTS_RAW_TEMP_TABLE_NAME)
|
||||
.withColumns(
|
||||
column('id').int.primary.autoGenerate2,
|
||||
column('metaId').int.notNull,
|
||||
column('type').int.notNull.comment(typeComment),
|
||||
column('value').bigint.notNull,
|
||||
column('timestamp').timestampTimezone(0).default('CURRENT_TIMESTAMP').notNull,
|
||||
)
|
||||
.withForeignKey('metaId', {
|
||||
tableName: INSIGHTS_METADATA_TABLE_NAME,
|
||||
columnName: 'metaId',
|
||||
onDelete: 'CASCADE',
|
||||
});
|
||||
|
||||
// Copy data from the original table to the temporary table
|
||||
await copyTable(INSIGHTS_RAW_TABLE_NAME, INSIGHTS_RAW_TEMP_TABLE_NAME);
|
||||
|
||||
// drop the original table
|
||||
await dropTable(INSIGHTS_RAW_TABLE_NAME);
|
||||
|
||||
// rename the temporary table to the original table name
|
||||
await queryRunner.query(`ALTER TABLE ${tempInsightsRawTable} RENAME TO ${insightsRawTable};`);
|
||||
|
||||
await createTable(INSIGHTS_BY_PERIOD_TEMP_TABLE_NAME)
|
||||
.withColumns(
|
||||
column('id').int.primary.autoGenerate2,
|
||||
column('metaId').int.notNull,
|
||||
column('type').int.notNull.comment(typeComment),
|
||||
column('value').bigint.notNull,
|
||||
column('periodUnit').int.notNull.comment('0: hour, 1: day, 2: week'),
|
||||
column('periodStart').default('CURRENT_TIMESTAMP').timestampTimezone(0),
|
||||
)
|
||||
.withForeignKey('metaId', {
|
||||
tableName: INSIGHTS_METADATA_TABLE_NAME,
|
||||
columnName: 'metaId',
|
||||
onDelete: 'CASCADE',
|
||||
})
|
||||
.withIndexOn(['periodStart', 'type', 'periodUnit', 'metaId'], true);
|
||||
|
||||
// Copy data from the original table to the temporary table
|
||||
await copyTable(INSIGHTS_BY_PERIOD_TABLE_NAME, INSIGHTS_BY_PERIOD_TEMP_TABLE_NAME);
|
||||
|
||||
// drop the original table
|
||||
await dropTable(INSIGHTS_BY_PERIOD_TABLE_NAME);
|
||||
|
||||
// rename the temporary table to the original table name
|
||||
await queryRunner.query(
|
||||
`ALTER TABLE ${tempInsightsByPeriodTable} RENAME TO ${insightsByPeriodTable};`,
|
||||
);
|
||||
} else if (isMysql) {
|
||||
await queryRunner.query(
|
||||
`ALTER TABLE ${insightsRawTable} MODIFY COLUMN ${valueColumnName} BIGINT NOT NULL;`,
|
||||
);
|
||||
await queryRunner.query(
|
||||
`ALTER TABLE ${insightsByPeriodTable} MODIFY COLUMN ${valueColumnName} BIGINT NOT NULL;`,
|
||||
);
|
||||
} else if (isPostgres) {
|
||||
await queryRunner.query(
|
||||
`ALTER TABLE ${insightsRawTable} ALTER COLUMN ${valueColumnName} TYPE BIGINT;`,
|
||||
);
|
||||
await queryRunner.query(
|
||||
`ALTER TABLE ${insightsByPeriodTable} ALTER COLUMN ${valueColumnName} TYPE BIGINT;`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,7 +10,8 @@ export class Column {
|
||||
| 'timestamptz'
|
||||
| 'timestamp'
|
||||
| 'uuid'
|
||||
| 'double';
|
||||
| 'double'
|
||||
| 'bigint';
|
||||
|
||||
private isGenerated = false;
|
||||
|
||||
@@ -40,6 +41,11 @@ export class Column {
|
||||
return this;
|
||||
}
|
||||
|
||||
get bigint() {
|
||||
this.type = 'bigint';
|
||||
return this;
|
||||
}
|
||||
|
||||
get double() {
|
||||
this.type = 'double';
|
||||
return this;
|
||||
@@ -176,6 +182,8 @@ export class Column {
|
||||
} else if (isSqlite) {
|
||||
options.type = 'real';
|
||||
}
|
||||
} else if (type === 'bigint') {
|
||||
options.type = 'bigint';
|
||||
}
|
||||
|
||||
if (
|
||||
|
||||
@@ -178,6 +178,11 @@ const createContext = (queryRunner: QueryRunner, migration: Migration): Migratio
|
||||
});
|
||||
|
||||
export const wrapMigration = (migration: Migration) => {
|
||||
const prototype = migration.prototype as unknown as { __n8n_wrapped?: boolean };
|
||||
if (prototype.__n8n_wrapped === true) {
|
||||
return;
|
||||
}
|
||||
prototype.__n8n_wrapped = true;
|
||||
const { up, down } = migration.prototype;
|
||||
if (up) {
|
||||
Object.assign(migration.prototype, {
|
||||
|
||||
@@ -99,6 +99,7 @@ import { AddInputsOutputsToTestCaseExecution1752669793000 } from '../common/1752
|
||||
import { CreateDataStoreTables1754475614601 } from '../common/1754475614601-CreateDataStoreTables';
|
||||
import { ReplaceDataStoreTablesWithDataTables1754475614602 } from '../common/1754475614602-ReplaceDataStoreTablesWithDataTables';
|
||||
import { AddTimestampsToRoleAndRoleIndexes1756906557570 } from '../common/1756906557570-AddTimestampsToRoleAndRoleIndexes';
|
||||
import { ChangeValueTypesForInsights1759399811000 } from '../common/1759399811000-ChangeValueTypesForInsights';
|
||||
import type { Migration } from '../migration-types';
|
||||
import { UpdateParentFolderIdColumn1740445074052 } from '../mysqldb/1740445074052-UpdateParentFolderIdColumn';
|
||||
|
||||
@@ -205,4 +206,5 @@ export const mysqlMigrations: Migration[] = [
|
||||
AddTimestampsToRoleAndRoleIndexes1756906557570,
|
||||
AddProjectIdToVariableTable1758794506893,
|
||||
AddAudienceColumnToApiKeys1758731786132,
|
||||
ChangeValueTypesForInsights1759399811000,
|
||||
];
|
||||
|
||||
@@ -99,6 +99,7 @@ import { CreateDataStoreTables1754475614601 } from '../common/1754475614601-Crea
|
||||
import { ReplaceDataStoreTablesWithDataTables1754475614602 } from '../common/1754475614602-ReplaceDataStoreTablesWithDataTables';
|
||||
import { AddTimestampsToRoleAndRoleIndexes1756906557570 } from '../common/1756906557570-AddTimestampsToRoleAndRoleIndexes';
|
||||
import { AddAudienceColumnToApiKeys1758731786132 } from '../common/1758731786132-AddAudienceColumnToApiKey';
|
||||
import { ChangeValueTypesForInsights1759399811000 } from '../common/1759399811000-ChangeValueTypesForInsights';
|
||||
import type { Migration } from '../migration-types';
|
||||
|
||||
export const postgresMigrations: Migration[] = [
|
||||
@@ -203,4 +204,5 @@ export const postgresMigrations: Migration[] = [
|
||||
AddTimestampsToRoleAndRoleIndexes1756906557570,
|
||||
AddProjectIdToVariableTable1758794506893,
|
||||
AddAudienceColumnToApiKeys1758731786132,
|
||||
ChangeValueTypesForInsights1759399811000,
|
||||
];
|
||||
|
||||
@@ -94,6 +94,7 @@ import { AddInputsOutputsToTestCaseExecution1752669793000 } from '../common/1752
|
||||
import { CreateDataStoreTables1754475614601 } from '../common/1754475614601-CreateDataStoreTables';
|
||||
import { ReplaceDataStoreTablesWithDataTables1754475614602 } from '../common/1754475614602-ReplaceDataStoreTablesWithDataTables';
|
||||
import { AddTimestampsToRoleAndRoleIndexes1756906557570 } from '../common/1756906557570-AddTimestampsToRoleAndRoleIndexes';
|
||||
import { ChangeValueTypesForInsights1759399811000 } from '../common/1759399811000-ChangeValueTypesForInsights';
|
||||
import type { Migration } from '../migration-types';
|
||||
import { LinkRoleToProjectRelationTable1753953244168 } from './../common/1753953244168-LinkRoleToProjectRelationTable';
|
||||
import { AddProjectIdToVariableTable1758794506893 } from './1758794506893-AddProjectIdToVariableTable';
|
||||
@@ -197,6 +198,7 @@ const sqliteMigrations: Migration[] = [
|
||||
AddTimestampsToRoleAndRoleIndexes1756906557570,
|
||||
AddProjectIdToVariableTable1758794506893,
|
||||
AddAudienceColumnToApiKeys1758731786132,
|
||||
ChangeValueTypesForInsights1759399811000,
|
||||
];
|
||||
|
||||
export { sqliteMigrations };
|
||||
|
||||
+343
@@ -0,0 +1,343 @@
|
||||
import { createTeamProject, createWorkflow, testDb, testModules } from '@n8n/backend-test-utils';
|
||||
import { Container } from '@n8n/di';
|
||||
import { DateTime } from 'luxon';
|
||||
|
||||
import { InsightsRawRepository } from '@/modules/insights/database/repositories/insights-raw.repository';
|
||||
|
||||
import {
|
||||
createRawInsightsEvent,
|
||||
createRawInsightsEvents,
|
||||
} from '../database/entities/__tests__/db-utils';
|
||||
import { InsightsByPeriodRepository } from '../database/repositories/insights-by-period.repository';
|
||||
import { InsightsCompactionService } from '../insights-compaction.service';
|
||||
|
||||
beforeAll(async () => {
|
||||
await testModules.loadModules(['insights']);
|
||||
await testDb.init();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await testDb.truncate([
|
||||
'InsightsRaw',
|
||||
'InsightsByPeriod',
|
||||
'InsightsMetadata',
|
||||
'WorkflowEntity',
|
||||
'Project',
|
||||
]);
|
||||
});
|
||||
|
||||
// Terminate DB once after all tests complete
|
||||
afterAll(async () => {
|
||||
await testDb.terminate();
|
||||
});
|
||||
|
||||
describe('BigInt migration validation', () => {
|
||||
describe('Store value exceeding 32-bit integer maximum', () => {
|
||||
test('should store and retrieve values larger than 2^31', async () => {
|
||||
// ARRANGE
|
||||
const insightsRawRepository = Container.get(InsightsRawRepository);
|
||||
const project = await createTeamProject();
|
||||
const workflow = await createWorkflow({}, project);
|
||||
|
||||
// Values exceeding 32-bit signed integer maximum (2,147,483,647)
|
||||
const largeValue1 = 2_147_483_648; // 2^31
|
||||
const largeValue2 = 5_000_000_000;
|
||||
|
||||
// ACT
|
||||
const event1 = await createRawInsightsEvent(workflow, {
|
||||
type: 'success',
|
||||
value: largeValue1,
|
||||
timestamp: DateTime.utc(),
|
||||
});
|
||||
|
||||
const event2 = await createRawInsightsEvent(workflow, {
|
||||
type: 'success',
|
||||
value: largeValue2,
|
||||
timestamp: DateTime.utc(),
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
// Verify the events were stored with exact values (no overflow)
|
||||
expect(event1.value).toBe(largeValue1);
|
||||
expect(event2.value).toBe(largeValue2);
|
||||
|
||||
// Verify retrieval from database returns exact values
|
||||
const retrievedEvents = await insightsRawRepository.find();
|
||||
expect(retrievedEvents).toHaveLength(2);
|
||||
|
||||
const retrieved1 = retrievedEvents.find((e) => e.id === event1.id);
|
||||
const retrieved2 = retrievedEvents.find((e) => e.id === event2.id);
|
||||
|
||||
expect(retrieved1?.value).toBe(largeValue1);
|
||||
expect(retrieved2?.value).toBe(largeValue2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Compaction sum exceeding 32-bit integer maximum', () => {
|
||||
test('should correctly sum large values during compaction without overflow', async () => {
|
||||
// ARRANGE
|
||||
const insightsCompactionService = Container.get(InsightsCompactionService);
|
||||
const insightsRawRepository = Container.get(InsightsRawRepository);
|
||||
const insightsByPeriodRepository = Container.get(InsightsByPeriodRepository);
|
||||
|
||||
const project = await createTeamProject();
|
||||
const workflow = await createWorkflow({}, project);
|
||||
|
||||
// Create 3 events with large values that sum to exceed 32-bit max
|
||||
// 1,800,000,000 * 3 = 5,400,000,000 (exceeds 2^31 - 1 = 2,147,483,647)
|
||||
const eventValue = 1_800_000_000;
|
||||
const expectedSum = 5_400_000_000;
|
||||
|
||||
const timestamp = DateTime.utc().startOf('hour');
|
||||
|
||||
// Create 3 events in the same hour period for the same workflow
|
||||
const events = [
|
||||
{ type: 'success' as const, value: eventValue, timestamp },
|
||||
{ type: 'success' as const, value: eventValue, timestamp: timestamp.plus({ minutes: 10 }) },
|
||||
{ type: 'success' as const, value: eventValue, timestamp: timestamp.plus({ minutes: 20 }) },
|
||||
];
|
||||
|
||||
await createRawInsightsEvents(workflow, events);
|
||||
|
||||
// ACT
|
||||
await insightsCompactionService.compactRawToHour();
|
||||
|
||||
// ASSERT
|
||||
// Verify raw events are compacted (removed)
|
||||
await expect(insightsRawRepository.count()).resolves.toBe(0);
|
||||
|
||||
// Verify compacted event has correct sum (no overflow)
|
||||
const compactedEvents = await insightsByPeriodRepository.find();
|
||||
expect(compactedEvents).toHaveLength(1);
|
||||
expect(compactedEvents[0].value).toBe(expectedSum);
|
||||
expect(compactedEvents[0].type).toBe('success');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Maximum safe integer boundary validation', () => {
|
||||
test('should handle Number.MAX_SAFE_INTEGER and arithmetic operations', async () => {
|
||||
// ARRANGE
|
||||
const insightsCompactionService = Container.get(InsightsCompactionService);
|
||||
const insightsRawRepository = Container.get(InsightsRawRepository);
|
||||
const insightsByPeriodRepository = Container.get(InsightsByPeriodRepository);
|
||||
|
||||
const project = await createTeamProject();
|
||||
const workflow = await createWorkflow({}, project);
|
||||
|
||||
// Maximum safe integer in JavaScript (2^53 - 1 = 9,007,199,254,740,991)
|
||||
const maxSafeInteger = Number.MAX_SAFE_INTEGER;
|
||||
|
||||
const timestamp = DateTime.utc().startOf('hour');
|
||||
|
||||
// ACT - Test storing MAX_SAFE_INTEGER
|
||||
const event = await createRawInsightsEvent(workflow, {
|
||||
type: 'runtime_ms',
|
||||
value: maxSafeInteger,
|
||||
timestamp,
|
||||
});
|
||||
|
||||
// ASSERT - Verify exact storage and retrieval
|
||||
expect(event.value).toBe(maxSafeInteger);
|
||||
|
||||
const retrieved = await insightsRawRepository.findOne({ where: { id: event.id } });
|
||||
expect(retrieved?.value).toBe(maxSafeInteger);
|
||||
|
||||
// Clean up for next test
|
||||
await testDb.truncate(['InsightsRaw', 'InsightsByPeriod', 'InsightsMetadata']);
|
||||
|
||||
// ACT - Test arithmetic with large values (50% of MAX_SAFE_INTEGER each)
|
||||
const workflow2 = await createWorkflow({}, project);
|
||||
const halfMaxSafe = Math.floor(Number.MAX_SAFE_INTEGER / 2);
|
||||
|
||||
await createRawInsightsEvents(workflow2, [
|
||||
{ type: 'time_saved_min', value: halfMaxSafe, timestamp },
|
||||
{ type: 'time_saved_min', value: halfMaxSafe, timestamp: timestamp.plus({ minutes: 5 }) },
|
||||
]);
|
||||
|
||||
await insightsCompactionService.compactRawToHour();
|
||||
|
||||
// ASSERT - Verify compaction sum is correct
|
||||
await expect(insightsRawRepository.count()).resolves.toBe(0);
|
||||
const compacted = await insightsByPeriodRepository.find();
|
||||
expect(compacted).toHaveLength(1);
|
||||
|
||||
// Sum should be close to MAX_SAFE_INTEGER (allowing for floor rounding)
|
||||
expect(compacted[0].value).toBe(halfMaxSafe * 2);
|
||||
expect(compacted[0].value).toBeLessThanOrEqual(Number.MAX_SAFE_INTEGER);
|
||||
});
|
||||
|
||||
test('should demonstrate precision loss for values exceeding MAX_SAFE_INTEGER', async () => {
|
||||
// ARRANGE
|
||||
const insightsRawRepository = Container.get(InsightsRawRepository);
|
||||
const project = await createTeamProject();
|
||||
const workflow = await createWorkflow({}, project);
|
||||
|
||||
const timestamp = DateTime.utc();
|
||||
|
||||
// JavaScript Number type cannot safely represent integers beyond MAX_SAFE_INTEGER
|
||||
// MAX_SAFE_INTEGER = 9,007,199,254,740,991 (2^53 - 1)
|
||||
const maxSafeInteger = Number.MAX_SAFE_INTEGER; // 9,007,199,254,740,991
|
||||
|
||||
// Values BEYOND MAX_SAFE_INTEGER will experience precision loss in JavaScript
|
||||
// Note: Database stores as bigint (no precision loss), but JS Number loses precision
|
||||
// Beyond MAX_SAFE_INTEGER, consecutive integers cannot be represented uniquely
|
||||
const unsafeValue1 = maxSafeInteger + 1; // Should be 9,007,199,254,740,992
|
||||
const unsafeValue2 = maxSafeInteger + 2; // Should be 9,007,199,254,740,993
|
||||
|
||||
// ASSERT - Demonstrate precision loss BEFORE storing
|
||||
// Both values are NOT safe integers (precision cannot be guaranteed)
|
||||
expect(Number.isSafeInteger(unsafeValue1)).toBe(false);
|
||||
expect(Number.isSafeInteger(unsafeValue2)).toBe(false);
|
||||
|
||||
// Critical demonstration: JavaScript rounds both values to the SAME number
|
||||
// This proves precision loss - two different values become identical
|
||||
expect(unsafeValue1).toBe(unsafeValue2);
|
||||
expect(unsafeValue1).toBe(9007199254740992); // Both round to MAX_SAFE_INTEGER + 1
|
||||
|
||||
// ACT - Store values that exceed MAX_SAFE_INTEGER
|
||||
const event1 = await createRawInsightsEvent(workflow, {
|
||||
type: 'runtime_ms',
|
||||
value: unsafeValue1,
|
||||
timestamp,
|
||||
});
|
||||
|
||||
const event2 = await createRawInsightsEvent(workflow, {
|
||||
type: 'runtime_ms',
|
||||
value: unsafeValue2,
|
||||
timestamp: timestamp.plus({ seconds: 10 }),
|
||||
});
|
||||
|
||||
// ASSERT - Stored values are identical (due to JS precision loss)
|
||||
expect(event1.value).toBe(event2.value); // Both are 9007199254740992
|
||||
expect(event1.value).toBe(9007199254740992);
|
||||
expect(event2.value).toBe(9007199254740992);
|
||||
|
||||
// Retrieve from database - values remain identical due to JS Number conversion
|
||||
const retrievedEvents = await insightsRawRepository.find({ order: { id: 'ASC' } });
|
||||
expect(retrievedEvents).toHaveLength(2);
|
||||
|
||||
// Retrieved values are also identical (demonstrating persistent precision loss)
|
||||
expect(retrievedEvents[0].value).toBe(retrievedEvents[1].value);
|
||||
expect(retrievedEvents[0].value).toBe(9007199254740992);
|
||||
expect(retrievedEvents[1].value).toBe(9007199254740992);
|
||||
|
||||
// Both retrieved values are NOT safe integers
|
||||
expect(Number.isSafeInteger(retrievedEvents[0].value)).toBe(false);
|
||||
expect(Number.isSafeInteger(retrievedEvents[1].value)).toBe(false);
|
||||
|
||||
// IMPORTANT: This test documents the current limitation.
|
||||
// Two distinct values (MAX_SAFE_INTEGER + 1 and MAX_SAFE_INTEGER + 2)
|
||||
// become indistinguishable due to JavaScript Number precision limits.
|
||||
//
|
||||
// To properly handle values > MAX_SAFE_INTEGER, we would need:
|
||||
// 1. TypeORM transformer to convert bigint ↔ BigInt (not Number)
|
||||
// 2. Application-level validation to reject values > MAX_SAFE_INTEGER
|
||||
// 3. OR: Change entity type from 'number' to 'bigint' with proper transformers
|
||||
});
|
||||
});
|
||||
|
||||
describe('Migration preserves existing small values', () => {
|
||||
test('should correctly store and compact small integer values', async () => {
|
||||
// ARRANGE
|
||||
const insightsCompactionService = Container.get(InsightsCompactionService);
|
||||
const insightsRawRepository = Container.get(InsightsRawRepository);
|
||||
const insightsByPeriodRepository = Container.get(InsightsByPeriodRepository);
|
||||
|
||||
const project = await createTeamProject();
|
||||
const workflow = await createWorkflow({}, project);
|
||||
|
||||
const smallValue1 = 42;
|
||||
const smallValue2 = 1_000_000;
|
||||
const expectedSum = smallValue1 + smallValue2;
|
||||
|
||||
const timestamp = DateTime.utc().startOf('hour');
|
||||
|
||||
// ACT
|
||||
await createRawInsightsEvents(workflow, [
|
||||
{ type: 'success', value: smallValue1, timestamp },
|
||||
{ type: 'success', value: smallValue2, timestamp: timestamp.plus({ minutes: 15 }) },
|
||||
]);
|
||||
|
||||
// ASSERT - Verify retrieval of small values
|
||||
const rawEvents = await insightsRawRepository.find({ order: { id: 'ASC' } });
|
||||
expect(rawEvents).toHaveLength(2);
|
||||
expect(rawEvents[0].value).toBe(smallValue1);
|
||||
expect(rawEvents[1].value).toBe(smallValue2);
|
||||
|
||||
// ACT - Compact the events
|
||||
await insightsCompactionService.compactRawToHour();
|
||||
|
||||
// ASSERT - Verify compaction sum is correct
|
||||
await expect(insightsRawRepository.count()).resolves.toBe(0);
|
||||
const compacted = await insightsByPeriodRepository.find();
|
||||
expect(compacted).toHaveLength(1);
|
||||
expect(compacted[0].value).toBe(expectedSum);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Negative large values', () => {
|
||||
test('should handle negative values exceeding 32-bit signed integer minimum', async () => {
|
||||
// ARRANGE
|
||||
const insightsRawRepository = Container.get(InsightsRawRepository);
|
||||
const project = await createTeamProject();
|
||||
const workflow = await createWorkflow({}, project);
|
||||
|
||||
// Values below 32-bit signed integer minimum (-2,147,483,648)
|
||||
const negativeValue1 = -2_147_483_649; // Below 2^31
|
||||
const negativeValue2 = -5_000_000_000;
|
||||
|
||||
// ACT
|
||||
const event1 = await createRawInsightsEvent(workflow, {
|
||||
type: 'time_saved_min',
|
||||
value: negativeValue1,
|
||||
timestamp: DateTime.utc(),
|
||||
});
|
||||
|
||||
const event2 = await createRawInsightsEvent(workflow, {
|
||||
type: 'time_saved_min',
|
||||
value: negativeValue2,
|
||||
timestamp: DateTime.utc(),
|
||||
});
|
||||
|
||||
// ASSERT - Verify storage and retrieval of negative large values
|
||||
expect(event1.value).toBe(negativeValue1);
|
||||
expect(event2.value).toBe(negativeValue2);
|
||||
|
||||
const retrievedEvents = await insightsRawRepository.find({ order: { id: 'ASC' } });
|
||||
expect(retrievedEvents).toHaveLength(2);
|
||||
expect(retrievedEvents[0].value).toBe(negativeValue1);
|
||||
expect(retrievedEvents[1].value).toBe(negativeValue2);
|
||||
});
|
||||
|
||||
test('should correctly compact mixed positive and negative large values', async () => {
|
||||
// ARRANGE
|
||||
const insightsCompactionService = Container.get(InsightsCompactionService);
|
||||
const insightsRawRepository = Container.get(InsightsRawRepository);
|
||||
const insightsByPeriodRepository = Container.get(InsightsByPeriodRepository);
|
||||
|
||||
const project = await createTeamProject();
|
||||
const workflow = await createWorkflow({}, project);
|
||||
|
||||
const timestamp = DateTime.utc().startOf('hour');
|
||||
|
||||
// Mix of large positive and negative values
|
||||
const positiveValue = 3_000_000_000;
|
||||
const negativeValue = -2_500_000_000;
|
||||
const expectedSum = positiveValue + negativeValue; // = 500,000,000
|
||||
|
||||
// ACT
|
||||
await createRawInsightsEvents(workflow, [
|
||||
{ type: 'runtime_ms', value: positiveValue, timestamp },
|
||||
{ type: 'runtime_ms', value: negativeValue, timestamp: timestamp.plus({ minutes: 10 }) },
|
||||
]);
|
||||
|
||||
await insightsCompactionService.compactRawToHour();
|
||||
|
||||
// ASSERT
|
||||
await expect(insightsRawRepository.count()).resolves.toBe(0);
|
||||
const compacted = await insightsByPeriodRepository.find();
|
||||
expect(compacted).toHaveLength(1);
|
||||
expect(compacted[0].value).toBe(expectedSum);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,163 @@
|
||||
import {
|
||||
createTestMigrationContext,
|
||||
initDbUpToMigration,
|
||||
runSingleMigration,
|
||||
testModules,
|
||||
} from '@n8n/backend-test-utils';
|
||||
import { DbConnection } from '@n8n/db';
|
||||
import { Container } from '@n8n/di';
|
||||
import { DataSource } from '@n8n/typeorm';
|
||||
|
||||
import { BOUNDARY_TEST_VALUES, insertPreMigrationPeriodData } from './migration-test-setup';
|
||||
|
||||
const MIGRATION_NAME = 'ChangeValueTypesForInsights1759399811000';
|
||||
|
||||
describe('ChangeValueTypesForInsights - insights_by_period table', () => {
|
||||
let dataSource: DataSource;
|
||||
|
||||
beforeAll(async () => {
|
||||
await testModules.loadModules(['insights']);
|
||||
|
||||
// Initialize DB connection without running migrations
|
||||
const dbConnection = Container.get(DbConnection);
|
||||
await dbConnection.init();
|
||||
|
||||
dataSource = Container.get(DataSource);
|
||||
|
||||
// Run migrations up to (but not including) target migration
|
||||
await initDbUpToMigration(MIGRATION_NAME);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
const dbConnection = Container.get(DbConnection);
|
||||
await dbConnection.close();
|
||||
});
|
||||
|
||||
describe('Schema Migration', () => {
|
||||
it('should change value column from INT to BIGINT', async () => {
|
||||
// Create migration context for schema queries
|
||||
const context = createTestMigrationContext(dataSource);
|
||||
|
||||
// Create prerequisite data for foreign keys using direct SQL
|
||||
const projectTableName = context.escape.tableName('project');
|
||||
await context.queryRunner.query(
|
||||
`INSERT INTO ${projectTableName} (id, name, type, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)`,
|
||||
['test-project-id', 'Test Project', 'personal', new Date(), new Date()],
|
||||
);
|
||||
|
||||
const workflowTableName = context.escape.tableName('workflow_entity');
|
||||
await context.queryRunner.query(
|
||||
`INSERT INTO ${workflowTableName} (id, name, active, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)`,
|
||||
['test-workflow-id', 'Test Workflow', false, new Date(), new Date()],
|
||||
);
|
||||
|
||||
// Insert test metadata (required for foreign key)
|
||||
const metaTableName = context.escape.tableName('insights_metadata');
|
||||
await context.queryRunner.query(
|
||||
`INSERT INTO ${metaTableName} (workflowId, projectId, workflowName, projectName) VALUES (?, ?, ?, ?)`,
|
||||
['test-workflow-id', 'test-project-id', 'Test Workflow', 'Test Project'],
|
||||
);
|
||||
const [metaRow] = await context.queryRunner.query(
|
||||
`SELECT metaId FROM ${metaTableName} LIMIT 1`,
|
||||
);
|
||||
const metaId = metaRow.metaId;
|
||||
|
||||
// Insert test data in old INT schema
|
||||
const testValues = [
|
||||
BOUNDARY_TEST_VALUES.zero,
|
||||
BOUNDARY_TEST_VALUES.negativeOne,
|
||||
BOUNDARY_TEST_VALUES.positiveOne,
|
||||
BOUNDARY_TEST_VALUES.intMax,
|
||||
BOUNDARY_TEST_VALUES.intMin,
|
||||
];
|
||||
await insertPreMigrationPeriodData(context, metaId, testValues);
|
||||
|
||||
// Capture data before migration using SQL
|
||||
const insightsByPeriodTableName = context.escape.tableName('insights_by_period');
|
||||
const beforeData = await context.queryRunner.query(
|
||||
`SELECT id, metaId, value FROM ${insightsByPeriodTableName} ORDER BY id ASC`,
|
||||
);
|
||||
expect(beforeData).toHaveLength(testValues.length);
|
||||
|
||||
// Run the migration
|
||||
await runSingleMigration(MIGRATION_NAME);
|
||||
|
||||
// Release old query runner before creating new one
|
||||
await context.queryRunner.release();
|
||||
|
||||
// Create fresh context after migration (dataSource is reinitialized)
|
||||
const postMigrationContext = createTestMigrationContext(dataSource);
|
||||
|
||||
// Verify schema change based on database type
|
||||
if (postMigrationContext.isSqlite) {
|
||||
const result = await postMigrationContext.queryRunner.query(
|
||||
`PRAGMA table_info(${insightsByPeriodTableName})`,
|
||||
);
|
||||
const valueColumn = result.find((col: { name: string }) => col.name === 'value');
|
||||
expect(valueColumn).toBeDefined();
|
||||
expect(valueColumn.type.toLowerCase()).toContain('bigint');
|
||||
} else if (postMigrationContext.isPostgres) {
|
||||
const result = await postMigrationContext.queryRunner.query(
|
||||
`SELECT data_type FROM information_schema.columns
|
||||
WHERE table_name = ${insightsByPeriodTableName} AND column_name = 'value'`,
|
||||
);
|
||||
expect(result[0].data_type).toBe('bigint');
|
||||
} else if (postMigrationContext.isMysql) {
|
||||
const result = await postMigrationContext.queryRunner.query(
|
||||
`SHOW COLUMNS FROM ${insightsByPeriodTableName} LIKE 'value'`,
|
||||
);
|
||||
expect(result[0].Type.toLowerCase()).toContain('bigint');
|
||||
}
|
||||
|
||||
// Verify data integrity after migration using SQL
|
||||
const afterData = await postMigrationContext.queryRunner.query(
|
||||
`SELECT id, metaId, value FROM ${insightsByPeriodTableName} ORDER BY id ASC`,
|
||||
);
|
||||
expect(afterData).toHaveLength(beforeData.length);
|
||||
|
||||
// Verify all values are preserved exactly
|
||||
afterData.forEach(
|
||||
(afterRow: { id: number; metaId: number; value: number }, index: number) => {
|
||||
expect(afterRow.value).toBe(beforeData[index].value);
|
||||
expect(afterRow.metaId).toBe(beforeData[index].metaId);
|
||||
},
|
||||
);
|
||||
|
||||
// Cleanup
|
||||
await postMigrationContext.queryRunner.release();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Post-Migration Capacity', () => {
|
||||
it('should accept values exceeding INT range', async () => {
|
||||
const context = createTestMigrationContext(dataSource);
|
||||
|
||||
// Get metaId from existing metadata
|
||||
const metaTableName = context.escape.tableName('insights_metadata');
|
||||
const [metaRow] = await context.queryRunner.query(
|
||||
`SELECT metaId FROM ${metaTableName} LIMIT 1`,
|
||||
);
|
||||
const metaId = metaRow.metaId;
|
||||
|
||||
// Insert value exceeding INT32 max
|
||||
const beyondIntValue = BOUNDARY_TEST_VALUES.beyondInt;
|
||||
const insightsByPeriodTableName = context.escape.tableName('insights_by_period');
|
||||
await context.queryRunner.query(
|
||||
`INSERT INTO ${insightsByPeriodTableName} (metaId, type, value, periodUnit, periodStart) VALUES (?, ?, ?, ?, ?)`,
|
||||
[metaId, 0, beyondIntValue, 0, new Date()],
|
||||
);
|
||||
|
||||
// Verify retrieval using SQL
|
||||
const [result] = await context.queryRunner.query(
|
||||
`SELECT id, metaId, value FROM ${insightsByPeriodTableName} WHERE value = ?`,
|
||||
[beyondIntValue],
|
||||
);
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(result.value).toBe(beyondIntValue);
|
||||
|
||||
// Cleanup
|
||||
await context.queryRunner.release();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,163 @@
|
||||
import {
|
||||
createTestMigrationContext,
|
||||
initDbUpToMigration,
|
||||
runSingleMigration,
|
||||
testModules,
|
||||
} from '@n8n/backend-test-utils';
|
||||
import { DbConnection } from '@n8n/db';
|
||||
import { Container } from '@n8n/di';
|
||||
import { DataSource } from '@n8n/typeorm';
|
||||
|
||||
import { BOUNDARY_TEST_VALUES, insertPreMigrationRawData } from './migration-test-setup';
|
||||
|
||||
const MIGRATION_NAME = 'ChangeValueTypesForInsights1759399811000';
|
||||
|
||||
describe('ChangeValueTypesForInsights - insights_raw table', () => {
|
||||
let dataSource: DataSource;
|
||||
|
||||
beforeAll(async () => {
|
||||
await testModules.loadModules(['insights']);
|
||||
|
||||
// Initialize DB connection without running migrations
|
||||
const dbConnection = Container.get(DbConnection);
|
||||
await dbConnection.init();
|
||||
|
||||
dataSource = Container.get(DataSource);
|
||||
|
||||
// Run migrations up to (but not including) target migration
|
||||
await initDbUpToMigration(MIGRATION_NAME);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
const dbConnection = Container.get(DbConnection);
|
||||
await dbConnection.close();
|
||||
});
|
||||
|
||||
describe('Schema Migration', () => {
|
||||
it('should change value column from INT to BIGINT', async () => {
|
||||
// Create migration context for schema queries
|
||||
const context = createTestMigrationContext(dataSource);
|
||||
|
||||
// Create prerequisite data for foreign keys using direct SQL
|
||||
const projectTableName = context.escape.tableName('project');
|
||||
await context.queryRunner.query(
|
||||
`INSERT INTO ${projectTableName} (id, name, type, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)`,
|
||||
['test-project-id', 'Test Project', 'personal', new Date(), new Date()],
|
||||
);
|
||||
|
||||
const workflowTableName = context.escape.tableName('workflow_entity');
|
||||
await context.queryRunner.query(
|
||||
`INSERT INTO ${workflowTableName} (id, name, active, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)`,
|
||||
['test-workflow-id', 'Test Workflow', false, new Date(), new Date()],
|
||||
);
|
||||
|
||||
// Insert test metadata (required for foreign key)
|
||||
const metaTableName = context.escape.tableName('insights_metadata');
|
||||
await context.queryRunner.query(
|
||||
`INSERT INTO ${metaTableName} (workflowId, projectId, workflowName, projectName) VALUES (?, ?, ?, ?)`,
|
||||
['test-workflow-id', 'test-project-id', 'Test Workflow', 'Test Project'],
|
||||
);
|
||||
const [metaRow] = await context.queryRunner.query(
|
||||
`SELECT metaId FROM ${metaTableName} LIMIT 1`,
|
||||
);
|
||||
const metaId = metaRow.metaId;
|
||||
|
||||
// Insert test data in old INT schema
|
||||
const testValues = [
|
||||
BOUNDARY_TEST_VALUES.zero,
|
||||
BOUNDARY_TEST_VALUES.negativeOne,
|
||||
BOUNDARY_TEST_VALUES.positiveOne,
|
||||
BOUNDARY_TEST_VALUES.intMax,
|
||||
BOUNDARY_TEST_VALUES.intMin,
|
||||
];
|
||||
await insertPreMigrationRawData(context, metaId, testValues);
|
||||
|
||||
// Capture data before migration using SQL
|
||||
const insightsRawTableName = context.escape.tableName('insights_raw');
|
||||
const beforeData = await context.queryRunner.query(
|
||||
`SELECT id, metaId, value FROM ${insightsRawTableName} ORDER BY id ASC`,
|
||||
);
|
||||
expect(beforeData).toHaveLength(testValues.length);
|
||||
|
||||
// Run the migration
|
||||
await runSingleMigration(MIGRATION_NAME);
|
||||
|
||||
// Release old query runner before creating new one
|
||||
await context.queryRunner.release();
|
||||
|
||||
// Create fresh context after migration (dataSource is reinitialized)
|
||||
const postMigrationContext = createTestMigrationContext(dataSource);
|
||||
|
||||
// Verify schema change based on database type
|
||||
if (postMigrationContext.isSqlite) {
|
||||
const result = await postMigrationContext.queryRunner.query(
|
||||
`PRAGMA table_info(${insightsRawTableName})`,
|
||||
);
|
||||
const valueColumn = result.find((col: { name: string }) => col.name === 'value');
|
||||
expect(valueColumn).toBeDefined();
|
||||
expect(valueColumn.type.toLowerCase()).toContain('bigint');
|
||||
} else if (postMigrationContext.isPostgres) {
|
||||
const result = await postMigrationContext.queryRunner.query(
|
||||
`SELECT data_type FROM information_schema.columns
|
||||
WHERE table_name = ${insightsRawTableName} AND column_name = 'value'`,
|
||||
);
|
||||
expect(result[0].data_type).toBe('bigint');
|
||||
} else if (postMigrationContext.isMysql) {
|
||||
const result = await postMigrationContext.queryRunner.query(
|
||||
`SHOW COLUMNS FROM ${insightsRawTableName} LIKE 'value'`,
|
||||
);
|
||||
expect(result[0].Type.toLowerCase()).toContain('bigint');
|
||||
}
|
||||
|
||||
// Verify data integrity after migration using SQL
|
||||
const afterData = await postMigrationContext.queryRunner.query(
|
||||
`SELECT id, metaId, value FROM ${insightsRawTableName} ORDER BY id ASC`,
|
||||
);
|
||||
expect(afterData).toHaveLength(beforeData.length);
|
||||
|
||||
// Verify all values are preserved exactly
|
||||
afterData.forEach(
|
||||
(afterRow: { id: number; metaId: number; value: number }, index: number) => {
|
||||
expect(afterRow.value).toBe(beforeData[index].value);
|
||||
expect(afterRow.metaId).toBe(beforeData[index].metaId);
|
||||
},
|
||||
);
|
||||
|
||||
// Cleanup
|
||||
await postMigrationContext.queryRunner.release();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Post-Migration Capacity', () => {
|
||||
it('should accept values exceeding INT range', async () => {
|
||||
const context = createTestMigrationContext(dataSource);
|
||||
|
||||
// Get metaId from existing metadata
|
||||
const metaTableName = context.escape.tableName('insights_metadata');
|
||||
const [metaRow] = await context.queryRunner.query(
|
||||
`SELECT metaId FROM ${metaTableName} LIMIT 1`,
|
||||
);
|
||||
const metaId = metaRow.metaId;
|
||||
|
||||
// Insert value exceeding INT32 max
|
||||
const beyondIntValue = BOUNDARY_TEST_VALUES.beyondInt;
|
||||
const insightsRawTableName = context.escape.tableName('insights_raw');
|
||||
await context.queryRunner.query(
|
||||
`INSERT INTO ${insightsRawTableName} (metaId, type, value, timestamp) VALUES (?, ?, ?, ?)`,
|
||||
[metaId, 0, beyondIntValue, new Date()],
|
||||
);
|
||||
|
||||
// Verify retrieval using SQL
|
||||
const [result] = await context.queryRunner.query(
|
||||
`SELECT id, metaId, value FROM ${insightsRawTableName} WHERE value = ?`,
|
||||
[beyondIntValue],
|
||||
);
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(result.value).toBe(beyondIntValue);
|
||||
|
||||
// Cleanup
|
||||
await context.queryRunner.release();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,53 @@
|
||||
import type { TestMigrationContext } from '@n8n/backend-test-utils';
|
||||
|
||||
/**
|
||||
* Test values for BIGINT migration validation.
|
||||
* Covers INT32 boundaries and BIGINT capabilities.
|
||||
*/
|
||||
export const BOUNDARY_TEST_VALUES = {
|
||||
zero: 0,
|
||||
positiveOne: 1,
|
||||
negativeOne: -1,
|
||||
intMax: 2_147_483_647, // 2^31 - 1 (max signed 32-bit int)
|
||||
intMin: -2_147_483_648, // -2^31 (min signed 32-bit int)
|
||||
beyondInt: 2_147_483_648, // For post-migration testing (exceeds INT32)
|
||||
bigintMax: 9_223_372_036_854_775_807n, // 2^63 - 1 (max signed 64-bit int)
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Insert raw insights data using SQL (compatible with pre-migration INT schema).
|
||||
* Uses database-specific helpers for table names and escaping.
|
||||
*/
|
||||
export async function insertPreMigrationRawData(
|
||||
context: TestMigrationContext,
|
||||
metaId: number,
|
||||
testValues: number[],
|
||||
): Promise<void> {
|
||||
const tableName = context.escape.tableName('insights_raw');
|
||||
for (const value of testValues) {
|
||||
await context.queryRunner.query(
|
||||
`INSERT INTO ${tableName} (metaId, type, value, timestamp) VALUES (?, ?, ?, ?)`,
|
||||
[metaId, 0, value, new Date()],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert insights_by_period data using SQL (compatible with pre-migration INT schema).
|
||||
* Uses database-specific helpers for table names and escaping.
|
||||
*/
|
||||
export async function insertPreMigrationPeriodData(
|
||||
context: TestMigrationContext,
|
||||
metaId: number,
|
||||
testValues: number[],
|
||||
): Promise<void> {
|
||||
const tableName = context.escape.tableName('insights_by_period');
|
||||
const baseDate = new Date('2025-01-01T00:00:00.000Z');
|
||||
for (let i = 0; i < testValues.length; i++) {
|
||||
const periodStart = new Date(baseDate.getTime() + i * 3600000); // Add 1 hour per iteration
|
||||
await context.queryRunner.query(
|
||||
`INSERT INTO ${tableName} (metaId, type, value, periodUnit, periodStart) VALUES (?, ?, ?, ?, ?)`,
|
||||
[metaId, 0, testValues[i], 0, periodStart.toISOString()],
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,11 @@ export class InsightsByPeriod extends BaseEntity {
|
||||
this.type_ = TypeToNumber[value];
|
||||
}
|
||||
|
||||
/**
|
||||
* Stored as BIGINT in database (see migration 1759399811000).
|
||||
* JavaScript number type has precision limits at ±2^53-1 (9,007,199,254,740,991).
|
||||
* Values exceeding Number.MAX_SAFE_INTEGER will lose precision.
|
||||
*/
|
||||
@Column()
|
||||
value: number;
|
||||
|
||||
|
||||
@@ -38,6 +38,11 @@ export class InsightsRaw extends BaseEntity {
|
||||
this.type_ = TypeToNumber[value];
|
||||
}
|
||||
|
||||
/**
|
||||
* Stored as BIGINT in database (see migration 1759399811000).
|
||||
* JavaScript number type has precision limits at ±2^53-1 (9,007,199,254,740,991).
|
||||
* Values exceeding Number.MAX_SAFE_INTEGER will lose precision.
|
||||
*/
|
||||
@Column()
|
||||
value: number;
|
||||
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import { initDbUpToMigration, runSingleMigration } from '@n8n/backend-test-utils';
|
||||
import { DbConnection } from '@n8n/db';
|
||||
import { Container } from '@n8n/di';
|
||||
import { DataSource } from '@n8n/typeorm';
|
||||
import { UnexpectedError } from 'n8n-workflow';
|
||||
|
||||
describe('Migration Test Helpers', () => {
|
||||
let dataSource: DataSource;
|
||||
|
||||
beforeEach(async () => {
|
||||
// Initialize connection without running migrations
|
||||
const dbConnection = Container.get(DbConnection);
|
||||
await dbConnection.init();
|
||||
dataSource = Container.get(DataSource);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
const dbConnection = Container.get(DbConnection);
|
||||
await dbConnection.close();
|
||||
});
|
||||
|
||||
describe('initDbUpToMigration', () => {
|
||||
it('should throw error if migration not found', async () => {
|
||||
await expect(initDbUpToMigration('NonExistentMigration')).rejects.toThrow(
|
||||
new UnexpectedError('Migration "NonExistentMigration" not found'),
|
||||
);
|
||||
});
|
||||
|
||||
it('should stop before specified migration', async () => {
|
||||
const migrations = dataSource.options.migrations as Array<{ name: string }>;
|
||||
expect(migrations.length).toBeGreaterThan(1);
|
||||
|
||||
const secondMigrationName = migrations[1].name;
|
||||
console.log('Running migrations up to ' + secondMigrationName);
|
||||
await initDbUpToMigration(secondMigrationName);
|
||||
console.log('Migrations executed up to ' + secondMigrationName);
|
||||
|
||||
// Verify only first migration was executed
|
||||
const executed = await dataSource.query('SELECT * FROM migrations ORDER BY timestamp');
|
||||
expect(executed).toHaveLength(1);
|
||||
expect(executed[0].name).toBe(migrations[0].name);
|
||||
});
|
||||
});
|
||||
|
||||
describe('runSingleMigration', () => {
|
||||
it('should throw error if migration not found', async () => {
|
||||
await expect(runSingleMigration('NonExistentMigration')).rejects.toThrow(
|
||||
new UnexpectedError('Migration "NonExistentMigration" not found'),
|
||||
);
|
||||
});
|
||||
|
||||
it('should run specific migration', async () => {
|
||||
const migrations = dataSource.options.migrations as Array<{ name: string }>;
|
||||
expect(migrations.length).toBeGreaterThan(1);
|
||||
|
||||
const secondMigrationName = migrations[1].name;
|
||||
console.log('Running migrations up to ' + secondMigrationName);
|
||||
await initDbUpToMigration(secondMigrationName);
|
||||
console.log('Migrations executed up to ' + secondMigrationName);
|
||||
|
||||
await runSingleMigration(secondMigrationName);
|
||||
|
||||
const executed = await dataSource.query('SELECT * FROM migrations ORDER BY timestamp');
|
||||
expect(executed).toHaveLength(2);
|
||||
expect(executed[0].name).toBe(migrations[0].name);
|
||||
expect(executed[1].name).toBe(secondMigrationName);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user