diff --git a/packages/@n8n/db/src/index.ts b/packages/@n8n/db/src/index.ts index 586407cec7d..cf1d0bfbcb6 100644 --- a/packages/@n8n/db/src/index.ts +++ b/packages/@n8n/db/src/index.ts @@ -11,6 +11,7 @@ export { } from './entities/abstract-entity'; export { generateNanoId } from '@n8n/utils/generate-nano-id'; +export { chunkIds } from './utils/chunk-ids'; export { dbNowLiteral, dbNowPlusMsLiteral, parseDbTime } from './utils/dialect-time'; export { generateHostInstanceId } from './utils/generators'; export { isStringArray } from './utils/is-string-array'; diff --git a/packages/@n8n/db/src/repositories/__tests__/folder.repository.test.ts b/packages/@n8n/db/src/repositories/__tests__/folder.repository.test.ts new file mode 100644 index 00000000000..b3f3cca7d89 --- /dev/null +++ b/packages/@n8n/db/src/repositories/__tests__/folder.repository.test.ts @@ -0,0 +1,42 @@ +import { In } from '@n8n/typeorm'; + +import { Folder } from '../../entities'; +import { mockEntityManager } from '../../utils/test-utils/mock-entity-manager'; +import { FolderRepository } from '../folder.repository'; + +describe('FolderRepository', () => { + const entityManager = mockEntityManager(Folder); + const repository = new FolderRepository(entityManager.connection); + + beforeEach(() => { + vi.resetAllMocks(); + }); + + it('merges folders returned from different chunks', async () => { + const first = Object.assign(new Folder(), { id: 'first' }); + const last = Object.assign(new Folder(), { id: 'last' }); + entityManager.find.mockResolvedValueOnce([first]).mockResolvedValueOnce([last]); + const folderIds = Array.from({ length: 10_001 }, (_, index) => `folder-${index}`); + + const result = await repository.findManyByIds(folderIds); + + expect(entityManager.find).toHaveBeenCalledTimes(2); + expect(entityManager.find).toHaveBeenNthCalledWith(2, Folder, { + where: { id: In(['folder-10000']) }, + relations: { homeProject: true }, + }); + expect(result).toEqual([first, last]); + }); + + it('merges existing ids returned from different chunks', async () => { + entityManager.find + .mockResolvedValueOnce([{ id: 'first' }]) + .mockResolvedValueOnce([{ id: 'last' }]); + const folderIds = Array.from({ length: 10_001 }, (_, index) => `folder-${index}`); + + const result = await repository.findExistingIds(folderIds); + + expect(entityManager.find).toHaveBeenCalledTimes(2); + expect(result).toEqual(new Set(['first', 'last'])); + }); +}); diff --git a/packages/@n8n/db/src/repositories/__tests__/shared-workflow.repository.test.ts b/packages/@n8n/db/src/repositories/__tests__/shared-workflow.repository.test.ts index b878978b257..d699864f05a 100644 --- a/packages/@n8n/db/src/repositories/__tests__/shared-workflow.repository.test.ts +++ b/packages/@n8n/db/src/repositories/__tests__/shared-workflow.repository.test.ts @@ -96,5 +96,49 @@ describe('SharedWorkflowRepository', () => { expect(result).toEqual(new Map()); }); + + it('merges owner projects returned from different chunks', async () => { + const firstProject = mock({ id: 'first-project' }); + const lastProject = mock({ id: 'last-project' }); + entityManager.find + .mockResolvedValueOnce([ + mock({ workflowId: 'first', project: firstProject }), + ]) + .mockResolvedValueOnce([ + mock({ workflowId: 'last', project: lastProject }), + ]); + const workflowIds = Array.from({ length: 10_001 }, (_, index) => `workflow-${index}`); + + const result = await sharedWorkflowRepository.findOwnerProjectsByWorkflowIds(workflowIds); + + expect(entityManager.find).toHaveBeenCalledTimes(2); + expect(result).toEqual( + new Map([ + ['first', firstProject], + ['last', lastProject], + ]), + ); + }); + }); + + describe('findByWorkflowIds', () => { + it('merges owner rows returned from different chunks', async () => { + const first = mock({ workflowId: 'first' }); + const last = mock({ workflowId: 'last' }); + entityManager.find.mockResolvedValueOnce([first]).mockResolvedValueOnce([last]); + const workflowIds = Array.from({ length: 10_001 }, (_, index) => `workflow-${index}`); + + const result = await sharedWorkflowRepository.findByWorkflowIds(workflowIds); + + expect(entityManager.find).toHaveBeenCalledTimes(2); + expect(entityManager.find).toHaveBeenNthCalledWith(2, SharedWorkflow, { + where: { + role: 'workflow:owner', + workflowId: In(['workflow-10000']), + }, + relations: { project: { projectRelations: { user: true, role: true } } }, + }); + expect(result).toEqual([first, last]); + }); }); }); diff --git a/packages/@n8n/db/src/repositories/__tests__/workflow.repository.test.ts b/packages/@n8n/db/src/repositories/__tests__/workflow.repository.test.ts index 87cc5ac9c35..536e8b6471c 100644 --- a/packages/@n8n/db/src/repositories/__tests__/workflow.repository.test.ts +++ b/packages/@n8n/db/src/repositories/__tests__/workflow.repository.test.ts @@ -553,6 +553,43 @@ describe('WorkflowRepository', () => { expect(findSpy).toHaveBeenCalledTimes(1); expect(findSpy).toHaveBeenCalledWith({ where: { id: In(workflowIds) } }); }); + + it('merges workflows returned from different chunks', async () => { + const first = Object.assign(new WorkflowEntity(), { id: 'first' }); + const last = Object.assign(new WorkflowEntity(), { id: 'last' }); + const findSpy = vi + .spyOn(workflowRepository, 'find') + .mockResolvedValueOnce([first]) + .mockResolvedValueOnce([last]); + const workflowIds = Array.from({ length: 10_001 }, (_, index) => `workflow-${index}`); + + const result = await workflowRepository.findByIds(workflowIds, { fields: ['name'] }); + + expect(findSpy).toHaveBeenCalledTimes(2); + expect(findSpy).toHaveBeenNthCalledWith(2, { + where: { id: In(['workflow-10000']) }, + select: ['id', 'name'], + }); + expect(result).toEqual([first, last]); + }); + }); + + describe('findPreExistingWorkflows', () => { + it('merges workflows returned from different chunks', async () => { + const first = Object.assign(new WorkflowEntity(), { id: 'first' }); + const last = Object.assign(new WorkflowEntity(), { id: 'last' }); + queryBuilder.getMany.mockResolvedValueOnce([first]).mockResolvedValueOnce([last]); + const workflowIds = Array.from({ length: 10_001 }, (_, index) => `workflow-${index}`); + + const result = await workflowRepository.findPreExistingWorkflows(workflowIds); + + expect(queryBuilder.getMany).toHaveBeenCalledTimes(2); + expect(result).toEqual([first, last]); + expect(queryBuilder.where.mock.calls[1]).toEqual([ + 'workflow.id IN (:...workflowIds)', + { workflowIds: ['workflow-10000'] }, + ]); + }); }); describe('getPublishedPersonalWorkflowsCount', () => { diff --git a/packages/@n8n/db/src/repositories/folder.repository.ts b/packages/@n8n/db/src/repositories/folder.repository.ts index 0c046adace0..21c3ffc8f73 100644 --- a/packages/@n8n/db/src/repositories/folder.repository.ts +++ b/packages/@n8n/db/src/repositories/folder.repository.ts @@ -1,10 +1,11 @@ import { Service } from '@n8n/di'; import type { EntityManager, SelectQueryBuilder } from '@n8n/typeorm'; -import { DataSource, Repository } from '@n8n/typeorm'; +import { DataSource, In, Repository } from '@n8n/typeorm'; import { PROJECT_ROOT } from 'n8n-workflow'; import { Folder, FolderTagMapping, TagEntity } from '../entities'; import type { FolderWithWorkflowAndSubFolderCountAndPath, ListQuery } from '../entities/types-db'; +import { chunkIds } from '../utils/chunk-ids'; import { parseListQuerySortBy } from '../utils/list-query-sort'; @Service() @@ -13,6 +14,31 @@ export class FolderRepository extends Repository { super(Folder, dataSource.manager); } + async findExistingIds(folderIds: string[]): Promise> { + const ids = new Set(); + + for (const chunk of chunkIds(folderIds)) { + const found = await this.find({ select: { id: true }, where: { id: In(chunk) } }); + for (const { id } of found) ids.add(id); + } + + return ids; + } + + async findManyByIds(folderIds: string[]): Promise { + const folders = new Map(); + + for (const chunk of chunkIds(folderIds)) { + const found = await this.find({ + where: { id: In(chunk) }, + relations: { homeProject: true }, + }); + for (const folder of found) folders.set(folder.id, folder); + } + + return [...folders.values()]; + } + async getManyAndCount(options: ListQuery.Options = {}) { const query = this.getManyQuery(options); return (await query.getManyAndCount()) as unknown as [ @@ -391,28 +417,35 @@ export class FolderRepository extends Repository { async getAllFolderIdsInSubtrees(parentFolderIds: string[]): Promise { if (parentFolderIds.length === 0) return []; - // Base case: the direct children of any requested parent. - const baseQuery = this.createQueryBuilder('f') - .select('f.id', 'id') - .where('f.parentFolderId IN (:...parentFolderIds)', { parentFolderIds }); + // Subtrees are independent, so each chunk resolves in its own recursive + // query and the ids are merged — a folder reachable from two chunks lands once. + const ids = new Set(); - // Recursive case: descendants of folders already in the tree. - const recursiveQuery = this.createQueryBuilder('child') - .select('child.id', 'id') - .innerJoin('folder_tree', 'parent', 'child.parentFolderId = parent.id'); + for (const chunk of chunkIds(parentFolderIds)) { + // Base case: the direct children of any requested parent. + const baseQuery = this.createQueryBuilder('f') + .select('f.id', 'id') + .where('f.parentFolderId IN (:...parentFolderIds)', { parentFolderIds: chunk }); - const query = this.createQueryBuilder() - .addCommonTableExpression( - `${baseQuery.getQuery()} UNION ALL ${recursiveQuery.getQuery()}`, - 'folder_tree', - { recursive: true }, - ) - .select('DISTINCT tree.id', 'id') - .from('folder_tree', 'tree') - .setParameters(baseQuery.getParameters()); + // Recursive case: descendants of folders already in the tree. + const recursiveQuery = this.createQueryBuilder('child') + .select('child.id', 'id') + .innerJoin('folder_tree', 'parent', 'child.parentFolderId = parent.id'); - const result = await query.getRawMany<{ id: string }>(); - return result.map((row) => row.id); + const query = this.createQueryBuilder() + .addCommonTableExpression( + `${baseQuery.getQuery()} UNION ALL ${recursiveQuery.getQuery()}`, + 'folder_tree', + { recursive: true }, + ) + .select('DISTINCT tree.id', 'id') + .from('folder_tree', 'tree') + .setParameters(baseQuery.getParameters()); + + for (const row of await query.getRawMany<{ id: string }>()) ids.add(row.id); + } + + return [...ids]; } async getFolderPathsToRoot(folderIds: string[]): Promise> { diff --git a/packages/@n8n/db/src/repositories/shared-workflow.repository.ts b/packages/@n8n/db/src/repositories/shared-workflow.repository.ts index 4b5e074f07e..5e2acf71e19 100644 --- a/packages/@n8n/db/src/repositories/shared-workflow.repository.ts +++ b/packages/@n8n/db/src/repositories/shared-workflow.repository.ts @@ -17,6 +17,7 @@ import { BaseRepository } from './base-repository'; import type { User } from '../entities'; import { Project, ProjectRelation, SharedWorkflow } from '../entities'; import { type OperationContext, TransactionRunner } from '../services/transaction'; +import { chunkIds } from '../utils/chunk-ids'; @Service() export class SharedWorkflowRepository extends BaseRepository { @@ -35,21 +36,34 @@ export class SharedWorkflowRepository extends BaseRepository { } async findByWorkflowIds(workflowIds: string[]) { - return await this.find({ - where: { - role: 'workflow:owner', - workflowId: In(workflowIds), - }, - relations: { project: { projectRelations: { user: true, role: true } } }, - }); + const rows = new Map(); + + for (const chunk of chunkIds(workflowIds)) { + const found = await this.find({ + where: { + role: 'workflow:owner', + workflowId: In(chunk), + }, + relations: { project: { projectRelations: { user: true, role: true } } }, + }); + for (const row of found) rows.set(row.workflowId, row); + } + + return [...rows.values()]; } /** Owner project of each workflow, keyed by workflow id. */ async findOwnerProjectsByWorkflowIds(workflowIds: string[]): Promise> { - const ownerRows = await this.find({ - where: { workflowId: In(workflowIds), role: 'workflow:owner' }, - relations: { project: true }, - }); + const ownerRows: SharedWorkflow[] = []; + + for (const chunk of chunkIds(workflowIds)) { + ownerRows.push( + ...(await this.find({ + where: { workflowId: In(chunk), role: 'workflow:owner' }, + relations: { project: true }, + })), + ); + } return new Map(ownerRows.map(({ workflowId, project }) => [workflowId, project])); } diff --git a/packages/@n8n/db/src/repositories/workflow.repository.ts b/packages/@n8n/db/src/repositories/workflow.repository.ts index 862b519831a..6473ea11d1e 100644 --- a/packages/@n8n/db/src/repositories/workflow.repository.ts +++ b/packages/@n8n/db/src/repositories/workflow.repository.ts @@ -33,6 +33,7 @@ import type { } from '../entities/types-db'; import { type OperationContext, TransactionRunner } from '../services/transaction'; import { applyWorkflowBooleanSettingFilter } from '../utils/apply-workflow-boolean-setting-filter'; +import { chunkIds } from '../utils/chunk-ids'; import { isStringArray } from '../utils/is-string-array'; import { parseListQuerySortBy } from '../utils/list-query-sort'; import { TimedQuery } from '../utils/timed-query'; @@ -239,13 +240,20 @@ export class WorkflowRepository extends BaseRepository { return []; } - const options: FindManyOptions = { - where: { id: In(workflowIds) }, - }; + const workflows = new Map(); + for (const chunk of chunkIds(workflowIds)) { + const options: FindManyOptions = { + where: { id: In(chunk) }, + }; - if (fields?.length) options.select = fields as FindOptionsSelect; + if (fields?.length) { + options.select = [...new Set(['id', ...fields])] as FindOptionsSelect; + } - return await this.find(options); + for (const workflow of await this.find(options)) workflows.set(workflow.id, workflow); + } + + return [...workflows.values()]; } async findManyByAgentToolReferences( @@ -292,12 +300,21 @@ export class WorkflowRepository extends BaseRepository { return []; } - return await this.createQueryBuilder('workflow') - .select(['workflow.id', 'workflow.name', 'workflow.isArchived']) - .leftJoin('workflow.shared', 'shared', 'shared.role = :role', { role: 'workflow:owner' }) - .addSelect(['shared.workflowId', 'shared.projectId', 'shared.role']) - .where('workflow.id IN (:...workflowIds)', { workflowIds }) - .getMany(); + const found = new Map(); + + for (const chunk of chunkIds(workflowIds)) { + const workflows = await this.createQueryBuilder('workflow') + .select(['workflow.id', 'workflow.name', 'workflow.isArchived']) + .leftJoin('workflow.shared', 'shared', 'shared.role = :role', { + role: 'workflow:owner', + }) + .addSelect(['shared.workflowId', 'shared.projectId', 'shared.role']) + .where('workflow.id IN (:...workflowIds)', { workflowIds: chunk }) + .getMany(); + for (const workflow of workflows) found.set(workflow.id, workflow); + } + + return [...found.values()]; } async getActiveTriggerCount() { diff --git a/packages/@n8n/db/src/utils/__tests__/chunk-ids.test.ts b/packages/@n8n/db/src/utils/__tests__/chunk-ids.test.ts new file mode 100644 index 00000000000..aad34599cb3 --- /dev/null +++ b/packages/@n8n/db/src/utils/__tests__/chunk-ids.test.ts @@ -0,0 +1,19 @@ +import { chunkIds } from '../chunk-ids'; + +const SQLITE_MAX_BIND_PARAMETERS = 32_766; +const MAX_BINDS_PER_ID = 3; +const FIXED_BIND_PARAMETERS = 6; + +describe('chunkIds', () => { + it('keeps query batches within the SQLite bind-parameter limit', () => { + const ids = Array.from({ length: 100_000 }, (_, index) => `id-${index}`); + const chunks = chunkIds(ids); + + expect(chunks.flat()).toEqual(ids); + for (const chunk of chunks) { + expect(chunk.length * MAX_BINDS_PER_ID + FIXED_BIND_PARAMETERS).toBeLessThanOrEqual( + SQLITE_MAX_BIND_PARAMETERS, + ); + } + }); +}); diff --git a/packages/@n8n/db/src/utils/chunk-ids.ts b/packages/@n8n/db/src/utils/chunk-ids.ts new file mode 100644 index 00000000000..82bcc6c2cf5 --- /dev/null +++ b/packages/@n8n/db/src/utils/chunk-ids.ts @@ -0,0 +1,8 @@ +import chunk from 'lodash/chunk'; + +/** Safe for three binds per ID: SQLite allows 32,766 binds; PostgreSQL, 65,535. */ +const ID_QUERY_BATCH_SIZE = 10_000; + +export function chunkIds(ids: T[]): T[][] { + return chunk(ids, ID_QUERY_BATCH_SIZE); +} diff --git a/packages/cli/src/services/__tests__/folder-finder.service.test.ts b/packages/cli/src/services/__tests__/folder-finder.service.test.ts index a89f379d2e2..d605f3d2554 100644 --- a/packages/cli/src/services/__tests__/folder-finder.service.test.ts +++ b/packages/cli/src/services/__tests__/folder-finder.service.test.ts @@ -23,6 +23,7 @@ function makeFolder(overrides: Partial = {}): Folder { function makeFinder(found: Folder[]) { const folderRepository = mock(); folderRepository.find.mockResolvedValue(found); + folderRepository.findExistingIds.mockResolvedValue(new Set(found.map(({ id }) => id))); const roleService = mock(); roleService.rolesWithScope.mockResolvedValue(['project:admin', 'project:editor']); const finder = new FolderFinderService(folderRepository, roleService); @@ -48,6 +49,26 @@ describe('FolderFinderService', () => { expect(result).toEqual(folders); }); + it('merges distinct folders returned from different chunks', async () => { + const firstFolder = makeFolder({ id: 'first' }); + const lastFolder = makeFolder({ id: 'last' }); + const { finder, folderRepository } = makeFinder([]); + folderRepository.find.mockResolvedValueOnce([firstFolder]).mockResolvedValueOnce([lastFolder]); + const folderIds = [ + ...Array.from({ length: 10_000 }, (_, index) => `folder-${index}`), + lastFolder.id, + ]; + + const result = await finder.findFoldersByIdsForUser(folderIds, nonGlobalUser, ['folder:read']); + + expect(folderRepository.find).toHaveBeenCalledTimes(2); + expect(result).toEqual([firstFolder, lastFolder]); + const secondChunk = folderRepository.find.mock.calls[1][0]?.where as unknown as { + id: { value: string[] }; + }; + expect(secondChunk.id.value).toEqual([lastFolder.id]); + }); + it('filters by the requested project scope for non-global users', async () => { const { finder, folderRepository, roleService } = makeFinder([makeFolder()]); @@ -181,15 +202,16 @@ describe('FolderFinderService', () => { const result = await finder.findExistingFolderIds([]); expect(result.size).toBe(0); - expect(folderRepository.find.mock.calls).toHaveLength(0); + expect(folderRepository.findExistingIds).not.toHaveBeenCalled(); }); it('returns the ids that exist in the database, unscoped by access', async () => { - const { finder } = makeFinder([makeFolder({ id: 'fld-1' })]); + const { finder, folderRepository } = makeFinder([makeFolder({ id: 'fld-1' })]); const result = await finder.findExistingFolderIds(['fld-1', 'fld-missing']); expect(result).toEqual(new Set(['fld-1'])); + expect(folderRepository.findExistingIds).toHaveBeenCalledWith(['fld-1', 'fld-missing']); }); }); diff --git a/packages/cli/src/services/folder-finder.service.ts b/packages/cli/src/services/folder-finder.service.ts index 473de721f1f..12bf45c09a4 100644 --- a/packages/cli/src/services/folder-finder.service.ts +++ b/packages/cli/src/services/folder-finder.service.ts @@ -1,5 +1,5 @@ import type { Folder, User } from '@n8n/db'; -import { FolderRepository } from '@n8n/db'; +import { chunkIds, FolderRepository } from '@n8n/db'; import { Service } from '@n8n/di'; import { hasGlobalScope, type Scope } from '@n8n/permissions'; import type { FindOptionsWhere } from '@n8n/typeorm'; @@ -22,11 +22,8 @@ export class FolderFinderService { async findExistingFolderIds(folderIds: string[]): Promise> { if (folderIds.length === 0) return new Set(); - const folders = await this.folderRepository.find({ - select: { id: true }, - where: { id: In(folderIds) }, - }); - return new Set(folders.map(({ id }) => id)); + + return await this.folderRepository.findExistingIds(folderIds); } /** @@ -74,9 +71,14 @@ export class FolderFinderService { const accessWhere = await this.buildFolderReadWhere(user, scopes); - return await this.folderRepository.find({ - where: { id: In(folderIds), ...accessWhere }, - }); + const folders = new Map(); + for (const chunk of chunkIds(folderIds)) { + const found = await this.folderRepository.find({ + where: { id: In(chunk), ...accessWhere }, + }); + for (const folder of found) folders.set(folder.id, folder); + } + return [...folders.values()]; } /** diff --git a/packages/cli/src/services/folder.service.ts b/packages/cli/src/services/folder.service.ts index 0cf0c27bdce..469f38674cc 100644 --- a/packages/cli/src/services/folder.service.ts +++ b/packages/cli/src/services/folder.service.ts @@ -6,7 +6,7 @@ import type { } from '@n8n/db'; import { Folder, FolderTagMappingRepository, FolderRepository, WorkflowRepository } from '@n8n/db'; import { Service } from '@n8n/di'; -import { In, type EntityManager } from '@n8n/typeorm'; +import type { EntityManager } from '@n8n/typeorm'; import { UserError, PROJECT_ROOT } from 'n8n-workflow'; import { FolderNotFoundError } from '@/errors/folder-not-found.error'; @@ -55,11 +55,7 @@ export class FolderService { } async getFoldersByIds(folderIds: string[]): Promise { - if (folderIds.length === 0) return []; - return await this.folderRepository.find({ - where: { id: In(folderIds) }, - relations: { homeProject: true }, - }); + return await this.folderRepository.findManyByIds(folderIds); } /** Every folder a project holds, flat, with the parent each one sits under (`null` at the root). */ diff --git a/packages/cli/src/workflows/__tests__/workflow-finder.service.test.ts b/packages/cli/src/workflows/__tests__/workflow-finder.service.test.ts index 975c31c9331..af5691c7916 100644 --- a/packages/cli/src/workflows/__tests__/workflow-finder.service.test.ts +++ b/packages/cli/src/workflows/__tests__/workflow-finder.service.test.ts @@ -86,16 +86,42 @@ describe('WorkflowFinderService', () => { const result = await service.findExistingWorkflowIds([]); expect(result.size).toBe(0); - expect(workflowRepository.find).not.toHaveBeenCalled(); + expect(workflowRepository.findByIds).not.toHaveBeenCalled(); }); it('returns the ids that exist in the database, unscoped by access', async () => { const { service, workflowRepository } = makeService(); - workflowRepository.find.mockResolvedValue([{ id: 'wf-1' }] as never); + workflowRepository.findByIds.mockResolvedValue([{ id: 'wf-1' }] as never); const result = await service.findExistingWorkflowIds(['wf-1', 'wf-missing']); expect(result).toEqual(new Set(['wf-1'])); + expect(workflowRepository.findByIds).toHaveBeenCalledWith(['wf-1', 'wf-missing'], { + fields: ['id'], + }); + }); + }); + + describe('findOwnedWorkflowsBySourceWorkflowIds', () => { + it('merges workflows returned from different chunks', async () => { + const { service, sharedWorkflowRepository } = makeService(); + sharedWorkflowRepository.find + .mockResolvedValueOnce([{ workflow: { id: 'first' } }] as never) + .mockResolvedValueOnce([{ workflow: { id: 'last' } }] as never); + const sourceWorkflowIds = Array.from({ length: 10_001 }, (_, index) => `source-${index}`); + + const result = await service.findOwnedWorkflowsBySourceWorkflowIds( + 'project-1', + sourceWorkflowIds, + ); + + expect(sharedWorkflowRepository.find).toHaveBeenCalledTimes(2); + expect(result.map(({ id }) => id)).toEqual(['first', 'last']); + const secondChunkWhere = sharedWorkflowRepository.find.mock.calls[1][0]?.where as Array<{ + workflow: { id?: { value: string[] }; sourceWorkflowId?: { value: string[] } }; + }>; + expect(secondChunkWhere[0].workflow.sourceWorkflowId?.value).toEqual(['source-10000']); + expect(secondChunkWhere[1].workflow.id?.value).toEqual(['source-10000']); }); }); diff --git a/packages/cli/src/workflows/workflow-finder.service.ts b/packages/cli/src/workflows/workflow-finder.service.ts index ce6f49e980b..6c0f559549a 100644 --- a/packages/cli/src/workflows/workflow-finder.service.ts +++ b/packages/cli/src/workflows/workflow-finder.service.ts @@ -1,5 +1,5 @@ import type { SharedWorkflow, User, WorkflowEntity, ListQuery } from '@n8n/db'; -import { SharedWorkflowRepository, FolderRepository, WorkflowRepository } from '@n8n/db'; +import { SharedWorkflowRepository, FolderRepository, WorkflowRepository, chunkIds } from '@n8n/db'; import { Service } from '@n8n/di'; import { hasGlobalScope, type Scope } from '@n8n/permissions'; import type { EntityManager, FindOptionsWhere } from '@n8n/typeorm'; @@ -161,19 +161,22 @@ export class WorkflowFinderService { ): Promise> { if (workflowIds.length === 0) return new Set(); const where = await this.findAllWhere(user, scopes); - const sharedWorkflows = await this.sharedWorkflowRepository.find({ - select: { workflowId: true }, - where: { ...where, workflowId: In(workflowIds) }, - }); - return new Set(sharedWorkflows.map((sw) => sw.workflowId)); + + const found = new Set(); + for (const chunk of chunkIds(workflowIds)) { + const sharedWorkflows = await this.sharedWorkflowRepository.find({ + select: { workflowId: true }, + where: { ...where, workflowId: In(chunk) }, + }); + for (const sw of sharedWorkflows) found.add(sw.workflowId); + } + return found; } async findExistingWorkflowIds(workflowIds: string[]): Promise> { if (workflowIds.length === 0) return new Set(); - const workflows = await this.workflowRepository.find({ - select: { id: true }, - where: { id: In(workflowIds) }, - }); + + const workflows = await this.workflowRepository.findByIds(workflowIds, { fields: ['id'] }); return new Set(workflows.map(({ id }) => id)); } @@ -190,48 +193,56 @@ export class WorkflowFinderService { if (workflowIds.length === 0) return []; const where = await this.findAllWhere(user, scopes); - const sharedWorkflows = await this.sharedWorkflowRepository.find({ - where: { ...where, workflowId: In(workflowIds) }, - relations: { - workflow: { - parentFolder: options.includeParentFolder, - tags: options.includeTags, - activeVersion: options.includeActiveVersion, - }, - }, - }); // A workflow may appear via several share paths (project membership + // direct share); dedupe so callers see one entity per id. const seen = new Set(); const workflows: WorkflowEntity[] = []; - for (const { workflow } of sharedWorkflows) { - if (seen.has(workflow.id)) continue; - seen.add(workflow.id); - workflows.push(workflow); + + for (const chunk of chunkIds(workflowIds)) { + const sharedWorkflows = await this.sharedWorkflowRepository.find({ + where: { ...where, workflowId: In(chunk) }, + relations: { + workflow: { + parentFolder: options.includeParentFolder, + tags: options.includeTags, + activeVersion: options.includeActiveVersion, + }, + }, + }); + + for (const { workflow } of sharedWorkflows) { + if (seen.has(workflow.id)) continue; + seen.add(workflow.id); + workflows.push(workflow); + } } + return workflows; } async findWorkflowIdsByFolder(folderIds: string[]): Promise> { if (folderIds.length === 0) return new Map(); - const rows = await this.sharedWorkflowRepository.find({ - where: { workflow: { parentFolder: In(folderIds) } }, - relations: { workflow: { parentFolder: true } }, - select: { workflowId: true, workflow: { id: true, parentFolder: { id: true } } }, - }); - const byFolder = new Map(); const seen = new Set(); - for (const { workflow } of rows) { - const folderId = workflow.parentFolder?.id; - // A workflow may appear via several share rows; dedupe so it lands once. - if (!folderId || seen.has(workflow.id)) continue; - seen.add(workflow.id); - const list = byFolder.get(folderId) ?? []; - list.push(workflow.id); - byFolder.set(folderId, list); + + for (const chunk of chunkIds(folderIds)) { + const rows = await this.sharedWorkflowRepository.find({ + where: { workflow: { parentFolder: In(chunk) } }, + relations: { workflow: { parentFolder: true } }, + select: { workflowId: true, workflow: { id: true, parentFolder: { id: true } } }, + }); + + for (const { workflow } of rows) { + const folderId = workflow.parentFolder?.id; + // A workflow may appear via several share rows; dedupe so it lands once. + if (!folderId || seen.has(workflow.id)) continue; + seen.add(workflow.id); + const list = byFolder.get(folderId) ?? []; + list.push(workflow.id); + byFolder.set(folderId, list); + } } return byFolder; @@ -307,27 +318,37 @@ export class WorkflowFinderService { activeVersion: options.includeActiveVersion, parentFolder: options.includeParentFolder, }; - const sharedWorkflows = await this.sharedWorkflowRepository.find({ - where: [ - { - projectId, - role: 'workflow:owner', - workflow: { sourceWorkflowId: In(sourceWorkflowIds), isArchived: false }, - }, - { - projectId, - role: 'workflow:owner', - workflow: { - id: In(sourceWorkflowIds), - sourceWorkflowId: IsNull(), - isArchived: false, - }, - }, - ], - relations: { workflow: workflowRelations }, - }); - return sharedWorkflows.map(({ workflow }) => workflow); + const workflows = new Map(); + + // The two branches below each expand the id list, so this costs two bind + // parameters per id — half the ids fit in one statement compared with a + // single-branch lookup. + for (const chunk of chunkIds(sourceWorkflowIds)) { + const sharedWorkflows = await this.sharedWorkflowRepository.find({ + where: [ + { + projectId, + role: 'workflow:owner', + workflow: { sourceWorkflowId: In(chunk), isArchived: false }, + }, + { + projectId, + role: 'workflow:owner', + workflow: { + id: In(chunk), + sourceWorkflowId: IsNull(), + isArchived: false, + }, + }, + ], + relations: { workflow: workflowRelations }, + }); + + for (const { workflow } of sharedWorkflows) workflows.set(workflow.id, workflow); + } + + return [...workflows.values()]; } async hasProjectScopeForUser(user: User, scopes: Scope[], projectId: string) { diff --git a/packages/cli/test/integration/database/repositories/folder.repository.test.ts b/packages/cli/test/integration/database/repositories/folder.repository.test.ts new file mode 100644 index 00000000000..3efcdad8344 --- /dev/null +++ b/packages/cli/test/integration/database/repositories/folder.repository.test.ts @@ -0,0 +1,26 @@ +import { testDb } from '@n8n/backend-test-utils'; +import { FolderRepository } from '@n8n/db'; +import { Container } from '@n8n/di'; + +const SQLITE_MAX_BIND_PARAMETERS = 32_766; + +describe('FolderRepository', () => { + beforeAll(async () => { + await testDb.init(); + }); + + afterAll(async () => { + await testDb.terminate(); + }); + + it('resolves subtree ids beyond the SQLite bind-parameter limit', async () => { + const folderIds = Array.from( + { length: SQLITE_MAX_BIND_PARAMETERS + 1 }, + (_, index) => `folder-${index}`, + ); + + await expect( + Container.get(FolderRepository).getAllFolderIdsInSubtrees(folderIds), + ).resolves.toEqual([]); + }); +});