perf(core): Chunk package ID lookups (#36923)

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Nour Alhadi Mahmoud
2026-08-26 10:29:30 +00:00
committed by GitHub
parent a53a75d193
commit 98e51043a8
15 changed files with 427 additions and 119 deletions
+1
View File
@@ -11,6 +11,7 @@ export {
} from './entities/abstract-entity';
export { generateNanoId } from '@n8n/utils/generate-nano-id';
export { chunkIds } from './utils/chunk-ids';
export { dbNowLiteral, dbNowPlusMsLiteral, parseDbTime } from './utils/dialect-time';
export { generateHostInstanceId } from './utils/generators';
export { isStringArray } from './utils/is-string-array';
@@ -0,0 +1,42 @@
import { In } from '@n8n/typeorm';
import { Folder } from '../../entities';
import { mockEntityManager } from '../../utils/test-utils/mock-entity-manager';
import { FolderRepository } from '../folder.repository';
describe('FolderRepository', () => {
const entityManager = mockEntityManager(Folder);
const repository = new FolderRepository(entityManager.connection);
beforeEach(() => {
vi.resetAllMocks();
});
it('merges folders returned from different chunks', async () => {
const first = Object.assign(new Folder(), { id: 'first' });
const last = Object.assign(new Folder(), { id: 'last' });
entityManager.find.mockResolvedValueOnce([first]).mockResolvedValueOnce([last]);
const folderIds = Array.from({ length: 10_001 }, (_, index) => `folder-${index}`);
const result = await repository.findManyByIds(folderIds);
expect(entityManager.find).toHaveBeenCalledTimes(2);
expect(entityManager.find).toHaveBeenNthCalledWith(2, Folder, {
where: { id: In(['folder-10000']) },
relations: { homeProject: true },
});
expect(result).toEqual([first, last]);
});
it('merges existing ids returned from different chunks', async () => {
entityManager.find
.mockResolvedValueOnce([{ id: 'first' }])
.mockResolvedValueOnce([{ id: 'last' }]);
const folderIds = Array.from({ length: 10_001 }, (_, index) => `folder-${index}`);
const result = await repository.findExistingIds(folderIds);
expect(entityManager.find).toHaveBeenCalledTimes(2);
expect(result).toEqual(new Set(['first', 'last']));
});
});
@@ -96,5 +96,49 @@ describe('SharedWorkflowRepository', () => {
expect(result).toEqual(new Map());
});
it('merges owner projects returned from different chunks', async () => {
const firstProject = mock<Project>({ id: 'first-project' });
const lastProject = mock<Project>({ id: 'last-project' });
entityManager.find
.mockResolvedValueOnce([
mock<SharedWorkflow>({ workflowId: 'first', project: firstProject }),
])
.mockResolvedValueOnce([
mock<SharedWorkflow>({ workflowId: 'last', project: lastProject }),
]);
const workflowIds = Array.from({ length: 10_001 }, (_, index) => `workflow-${index}`);
const result = await sharedWorkflowRepository.findOwnerProjectsByWorkflowIds(workflowIds);
expect(entityManager.find).toHaveBeenCalledTimes(2);
expect(result).toEqual(
new Map([
['first', firstProject],
['last', lastProject],
]),
);
});
});
describe('findByWorkflowIds', () => {
it('merges owner rows returned from different chunks', async () => {
const first = mock<SharedWorkflow>({ workflowId: 'first' });
const last = mock<SharedWorkflow>({ workflowId: 'last' });
entityManager.find.mockResolvedValueOnce([first]).mockResolvedValueOnce([last]);
const workflowIds = Array.from({ length: 10_001 }, (_, index) => `workflow-${index}`);
const result = await sharedWorkflowRepository.findByWorkflowIds(workflowIds);
expect(entityManager.find).toHaveBeenCalledTimes(2);
expect(entityManager.find).toHaveBeenNthCalledWith(2, SharedWorkflow, {
where: {
role: 'workflow:owner',
workflowId: In(['workflow-10000']),
},
relations: { project: { projectRelations: { user: true, role: true } } },
});
expect(result).toEqual([first, last]);
});
});
});
@@ -553,6 +553,43 @@ describe('WorkflowRepository', () => {
expect(findSpy).toHaveBeenCalledTimes(1);
expect(findSpy).toHaveBeenCalledWith({ where: { id: In(workflowIds) } });
});
it('merges workflows returned from different chunks', async () => {
const first = Object.assign(new WorkflowEntity(), { id: 'first' });
const last = Object.assign(new WorkflowEntity(), { id: 'last' });
const findSpy = vi
.spyOn(workflowRepository, 'find')
.mockResolvedValueOnce([first])
.mockResolvedValueOnce([last]);
const workflowIds = Array.from({ length: 10_001 }, (_, index) => `workflow-${index}`);
const result = await workflowRepository.findByIds(workflowIds, { fields: ['name'] });
expect(findSpy).toHaveBeenCalledTimes(2);
expect(findSpy).toHaveBeenNthCalledWith(2, {
where: { id: In(['workflow-10000']) },
select: ['id', 'name'],
});
expect(result).toEqual([first, last]);
});
});
describe('findPreExistingWorkflows', () => {
it('merges workflows returned from different chunks', async () => {
const first = Object.assign(new WorkflowEntity(), { id: 'first' });
const last = Object.assign(new WorkflowEntity(), { id: 'last' });
queryBuilder.getMany.mockResolvedValueOnce([first]).mockResolvedValueOnce([last]);
const workflowIds = Array.from({ length: 10_001 }, (_, index) => `workflow-${index}`);
const result = await workflowRepository.findPreExistingWorkflows(workflowIds);
expect(queryBuilder.getMany).toHaveBeenCalledTimes(2);
expect(result).toEqual([first, last]);
expect(queryBuilder.where.mock.calls[1]).toEqual([
'workflow.id IN (:...workflowIds)',
{ workflowIds: ['workflow-10000'] },
]);
});
});
describe('getPublishedPersonalWorkflowsCount', () => {
@@ -1,10 +1,11 @@
import { Service } from '@n8n/di';
import type { EntityManager, SelectQueryBuilder } from '@n8n/typeorm';
import { DataSource, Repository } from '@n8n/typeorm';
import { DataSource, In, Repository } from '@n8n/typeorm';
import { PROJECT_ROOT } from 'n8n-workflow';
import { Folder, FolderTagMapping, TagEntity } from '../entities';
import type { FolderWithWorkflowAndSubFolderCountAndPath, ListQuery } from '../entities/types-db';
import { chunkIds } from '../utils/chunk-ids';
import { parseListQuerySortBy } from '../utils/list-query-sort';
@Service()
@@ -13,6 +14,31 @@ export class FolderRepository extends Repository<Folder> {
super(Folder, dataSource.manager);
}
async findExistingIds(folderIds: string[]): Promise<Set<string>> {
const ids = new Set<string>();
for (const chunk of chunkIds(folderIds)) {
const found = await this.find({ select: { id: true }, where: { id: In(chunk) } });
for (const { id } of found) ids.add(id);
}
return ids;
}
async findManyByIds(folderIds: string[]): Promise<Folder[]> {
const folders = new Map<string, Folder>();
for (const chunk of chunkIds(folderIds)) {
const found = await this.find({
where: { id: In(chunk) },
relations: { homeProject: true },
});
for (const folder of found) folders.set(folder.id, folder);
}
return [...folders.values()];
}
async getManyAndCount(options: ListQuery.Options = {}) {
const query = this.getManyQuery(options);
return (await query.getManyAndCount()) as unknown as [
@@ -391,28 +417,35 @@ export class FolderRepository extends Repository<Folder> {
async getAllFolderIdsInSubtrees(parentFolderIds: string[]): Promise<string[]> {
if (parentFolderIds.length === 0) return [];
// Base case: the direct children of any requested parent.
const baseQuery = this.createQueryBuilder('f')
.select('f.id', 'id')
.where('f.parentFolderId IN (:...parentFolderIds)', { parentFolderIds });
// Subtrees are independent, so each chunk resolves in its own recursive
// query and the ids are merged — a folder reachable from two chunks lands once.
const ids = new Set<string>();
// Recursive case: descendants of folders already in the tree.
const recursiveQuery = this.createQueryBuilder('child')
.select('child.id', 'id')
.innerJoin('folder_tree', 'parent', 'child.parentFolderId = parent.id');
for (const chunk of chunkIds(parentFolderIds)) {
// Base case: the direct children of any requested parent.
const baseQuery = this.createQueryBuilder('f')
.select('f.id', 'id')
.where('f.parentFolderId IN (:...parentFolderIds)', { parentFolderIds: chunk });
const query = this.createQueryBuilder()
.addCommonTableExpression(
`${baseQuery.getQuery()} UNION ALL ${recursiveQuery.getQuery()}`,
'folder_tree',
{ recursive: true },
)
.select('DISTINCT tree.id', 'id')
.from('folder_tree', 'tree')
.setParameters(baseQuery.getParameters());
// Recursive case: descendants of folders already in the tree.
const recursiveQuery = this.createQueryBuilder('child')
.select('child.id', 'id')
.innerJoin('folder_tree', 'parent', 'child.parentFolderId = parent.id');
const result = await query.getRawMany<{ id: string }>();
return result.map((row) => row.id);
const query = this.createQueryBuilder()
.addCommonTableExpression(
`${baseQuery.getQuery()} UNION ALL ${recursiveQuery.getQuery()}`,
'folder_tree',
{ recursive: true },
)
.select('DISTINCT tree.id', 'id')
.from('folder_tree', 'tree')
.setParameters(baseQuery.getParameters());
for (const row of await query.getRawMany<{ id: string }>()) ids.add(row.id);
}
return [...ids];
}
async getFolderPathsToRoot(folderIds: string[]): Promise<Map<string, string[]>> {
@@ -17,6 +17,7 @@ import { BaseRepository } from './base-repository';
import type { User } from '../entities';
import { Project, ProjectRelation, SharedWorkflow } from '../entities';
import { type OperationContext, TransactionRunner } from '../services/transaction';
import { chunkIds } from '../utils/chunk-ids';
@Service()
export class SharedWorkflowRepository extends BaseRepository<SharedWorkflow> {
@@ -35,21 +36,34 @@ export class SharedWorkflowRepository extends BaseRepository<SharedWorkflow> {
}
async findByWorkflowIds(workflowIds: string[]) {
return await this.find({
where: {
role: 'workflow:owner',
workflowId: In(workflowIds),
},
relations: { project: { projectRelations: { user: true, role: true } } },
});
const rows = new Map<string, SharedWorkflow>();
for (const chunk of chunkIds(workflowIds)) {
const found = await this.find({
where: {
role: 'workflow:owner',
workflowId: In(chunk),
},
relations: { project: { projectRelations: { user: true, role: true } } },
});
for (const row of found) rows.set(row.workflowId, row);
}
return [...rows.values()];
}
/** Owner project of each workflow, keyed by workflow id. */
async findOwnerProjectsByWorkflowIds(workflowIds: string[]): Promise<Map<string, Project>> {
const ownerRows = await this.find({
where: { workflowId: In(workflowIds), role: 'workflow:owner' },
relations: { project: true },
});
const ownerRows: SharedWorkflow[] = [];
for (const chunk of chunkIds(workflowIds)) {
ownerRows.push(
...(await this.find({
where: { workflowId: In(chunk), role: 'workflow:owner' },
relations: { project: true },
})),
);
}
return new Map(ownerRows.map(({ workflowId, project }) => [workflowId, project]));
}
@@ -33,6 +33,7 @@ import type {
} from '../entities/types-db';
import { type OperationContext, TransactionRunner } from '../services/transaction';
import { applyWorkflowBooleanSettingFilter } from '../utils/apply-workflow-boolean-setting-filter';
import { chunkIds } from '../utils/chunk-ids';
import { isStringArray } from '../utils/is-string-array';
import { parseListQuerySortBy } from '../utils/list-query-sort';
import { TimedQuery } from '../utils/timed-query';
@@ -239,13 +240,20 @@ export class WorkflowRepository extends BaseRepository<WorkflowEntity> {
return [];
}
const options: FindManyOptions<WorkflowEntity> = {
where: { id: In(workflowIds) },
};
const workflows = new Map<string, WorkflowEntity>();
for (const chunk of chunkIds(workflowIds)) {
const options: FindManyOptions<WorkflowEntity> = {
where: { id: In(chunk) },
};
if (fields?.length) options.select = fields as FindOptionsSelect<WorkflowEntity>;
if (fields?.length) {
options.select = [...new Set(['id', ...fields])] as FindOptionsSelect<WorkflowEntity>;
}
return await this.find(options);
for (const workflow of await this.find(options)) workflows.set(workflow.id, workflow);
}
return [...workflows.values()];
}
async findManyByAgentToolReferences(
@@ -292,12 +300,21 @@ export class WorkflowRepository extends BaseRepository<WorkflowEntity> {
return [];
}
return await this.createQueryBuilder('workflow')
.select(['workflow.id', 'workflow.name', 'workflow.isArchived'])
.leftJoin('workflow.shared', 'shared', 'shared.role = :role', { role: 'workflow:owner' })
.addSelect(['shared.workflowId', 'shared.projectId', 'shared.role'])
.where('workflow.id IN (:...workflowIds)', { workflowIds })
.getMany();
const found = new Map<string, WorkflowEntity>();
for (const chunk of chunkIds(workflowIds)) {
const workflows = await this.createQueryBuilder('workflow')
.select(['workflow.id', 'workflow.name', 'workflow.isArchived'])
.leftJoin('workflow.shared', 'shared', 'shared.role = :role', {
role: 'workflow:owner',
})
.addSelect(['shared.workflowId', 'shared.projectId', 'shared.role'])
.where('workflow.id IN (:...workflowIds)', { workflowIds: chunk })
.getMany();
for (const workflow of workflows) found.set(workflow.id, workflow);
}
return [...found.values()];
}
async getActiveTriggerCount() {
@@ -0,0 +1,19 @@
import { chunkIds } from '../chunk-ids';
const SQLITE_MAX_BIND_PARAMETERS = 32_766;
const MAX_BINDS_PER_ID = 3;
const FIXED_BIND_PARAMETERS = 6;
describe('chunkIds', () => {
it('keeps query batches within the SQLite bind-parameter limit', () => {
const ids = Array.from({ length: 100_000 }, (_, index) => `id-${index}`);
const chunks = chunkIds(ids);
expect(chunks.flat()).toEqual(ids);
for (const chunk of chunks) {
expect(chunk.length * MAX_BINDS_PER_ID + FIXED_BIND_PARAMETERS).toBeLessThanOrEqual(
SQLITE_MAX_BIND_PARAMETERS,
);
}
});
});
+8
View File
@@ -0,0 +1,8 @@
import chunk from 'lodash/chunk';
/** Safe for three binds per ID: SQLite allows 32,766 binds; PostgreSQL, 65,535. */
const ID_QUERY_BATCH_SIZE = 10_000;
export function chunkIds<T>(ids: T[]): T[][] {
return chunk(ids, ID_QUERY_BATCH_SIZE);
}
@@ -23,6 +23,7 @@ function makeFolder(overrides: Partial<Folder> = {}): Folder {
function makeFinder(found: Folder[]) {
const folderRepository = mock<FolderRepository>();
folderRepository.find.mockResolvedValue(found);
folderRepository.findExistingIds.mockResolvedValue(new Set(found.map(({ id }) => id)));
const roleService = mock<RoleService>();
roleService.rolesWithScope.mockResolvedValue(['project:admin', 'project:editor']);
const finder = new FolderFinderService(folderRepository, roleService);
@@ -48,6 +49,26 @@ describe('FolderFinderService', () => {
expect(result).toEqual(folders);
});
it('merges distinct folders returned from different chunks', async () => {
const firstFolder = makeFolder({ id: 'first' });
const lastFolder = makeFolder({ id: 'last' });
const { finder, folderRepository } = makeFinder([]);
folderRepository.find.mockResolvedValueOnce([firstFolder]).mockResolvedValueOnce([lastFolder]);
const folderIds = [
...Array.from({ length: 10_000 }, (_, index) => `folder-${index}`),
lastFolder.id,
];
const result = await finder.findFoldersByIdsForUser(folderIds, nonGlobalUser, ['folder:read']);
expect(folderRepository.find).toHaveBeenCalledTimes(2);
expect(result).toEqual([firstFolder, lastFolder]);
const secondChunk = folderRepository.find.mock.calls[1][0]?.where as unknown as {
id: { value: string[] };
};
expect(secondChunk.id.value).toEqual([lastFolder.id]);
});
it('filters by the requested project scope for non-global users', async () => {
const { finder, folderRepository, roleService } = makeFinder([makeFolder()]);
@@ -181,15 +202,16 @@ describe('FolderFinderService', () => {
const result = await finder.findExistingFolderIds([]);
expect(result.size).toBe(0);
expect(folderRepository.find.mock.calls).toHaveLength(0);
expect(folderRepository.findExistingIds).not.toHaveBeenCalled();
});
it('returns the ids that exist in the database, unscoped by access', async () => {
const { finder } = makeFinder([makeFolder({ id: 'fld-1' })]);
const { finder, folderRepository } = makeFinder([makeFolder({ id: 'fld-1' })]);
const result = await finder.findExistingFolderIds(['fld-1', 'fld-missing']);
expect(result).toEqual(new Set(['fld-1']));
expect(folderRepository.findExistingIds).toHaveBeenCalledWith(['fld-1', 'fld-missing']);
});
});
@@ -1,5 +1,5 @@
import type { Folder, User } from '@n8n/db';
import { FolderRepository } from '@n8n/db';
import { chunkIds, FolderRepository } from '@n8n/db';
import { Service } from '@n8n/di';
import { hasGlobalScope, type Scope } from '@n8n/permissions';
import type { FindOptionsWhere } from '@n8n/typeorm';
@@ -22,11 +22,8 @@ export class FolderFinderService {
async findExistingFolderIds(folderIds: string[]): Promise<Set<string>> {
if (folderIds.length === 0) return new Set();
const folders = await this.folderRepository.find({
select: { id: true },
where: { id: In(folderIds) },
});
return new Set(folders.map(({ id }) => id));
return await this.folderRepository.findExistingIds(folderIds);
}
/**
@@ -74,9 +71,14 @@ export class FolderFinderService {
const accessWhere = await this.buildFolderReadWhere(user, scopes);
return await this.folderRepository.find({
where: { id: In(folderIds), ...accessWhere },
});
const folders = new Map<string, Folder>();
for (const chunk of chunkIds(folderIds)) {
const found = await this.folderRepository.find({
where: { id: In(chunk), ...accessWhere },
});
for (const folder of found) folders.set(folder.id, folder);
}
return [...folders.values()];
}
/**
+2 -6
View File
@@ -6,7 +6,7 @@ import type {
} from '@n8n/db';
import { Folder, FolderTagMappingRepository, FolderRepository, WorkflowRepository } from '@n8n/db';
import { Service } from '@n8n/di';
import { In, type EntityManager } from '@n8n/typeorm';
import type { EntityManager } from '@n8n/typeorm';
import { UserError, PROJECT_ROOT } from 'n8n-workflow';
import { FolderNotFoundError } from '@/errors/folder-not-found.error';
@@ -55,11 +55,7 @@ export class FolderService {
}
async getFoldersByIds(folderIds: string[]): Promise<Folder[]> {
if (folderIds.length === 0) return [];
return await this.folderRepository.find({
where: { id: In(folderIds) },
relations: { homeProject: true },
});
return await this.folderRepository.findManyByIds(folderIds);
}
/** Every folder a project holds, flat, with the parent each one sits under (`null` at the root). */
@@ -86,16 +86,42 @@ describe('WorkflowFinderService', () => {
const result = await service.findExistingWorkflowIds([]);
expect(result.size).toBe(0);
expect(workflowRepository.find).not.toHaveBeenCalled();
expect(workflowRepository.findByIds).not.toHaveBeenCalled();
});
it('returns the ids that exist in the database, unscoped by access', async () => {
const { service, workflowRepository } = makeService();
workflowRepository.find.mockResolvedValue([{ id: 'wf-1' }] as never);
workflowRepository.findByIds.mockResolvedValue([{ id: 'wf-1' }] as never);
const result = await service.findExistingWorkflowIds(['wf-1', 'wf-missing']);
expect(result).toEqual(new Set(['wf-1']));
expect(workflowRepository.findByIds).toHaveBeenCalledWith(['wf-1', 'wf-missing'], {
fields: ['id'],
});
});
});
describe('findOwnedWorkflowsBySourceWorkflowIds', () => {
it('merges workflows returned from different chunks', async () => {
const { service, sharedWorkflowRepository } = makeService();
sharedWorkflowRepository.find
.mockResolvedValueOnce([{ workflow: { id: 'first' } }] as never)
.mockResolvedValueOnce([{ workflow: { id: 'last' } }] as never);
const sourceWorkflowIds = Array.from({ length: 10_001 }, (_, index) => `source-${index}`);
const result = await service.findOwnedWorkflowsBySourceWorkflowIds(
'project-1',
sourceWorkflowIds,
);
expect(sharedWorkflowRepository.find).toHaveBeenCalledTimes(2);
expect(result.map(({ id }) => id)).toEqual(['first', 'last']);
const secondChunkWhere = sharedWorkflowRepository.find.mock.calls[1][0]?.where as Array<{
workflow: { id?: { value: string[] }; sourceWorkflowId?: { value: string[] } };
}>;
expect(secondChunkWhere[0].workflow.sourceWorkflowId?.value).toEqual(['source-10000']);
expect(secondChunkWhere[1].workflow.id?.value).toEqual(['source-10000']);
});
});
@@ -1,5 +1,5 @@
import type { SharedWorkflow, User, WorkflowEntity, ListQuery } from '@n8n/db';
import { SharedWorkflowRepository, FolderRepository, WorkflowRepository } from '@n8n/db';
import { SharedWorkflowRepository, FolderRepository, WorkflowRepository, chunkIds } from '@n8n/db';
import { Service } from '@n8n/di';
import { hasGlobalScope, type Scope } from '@n8n/permissions';
import type { EntityManager, FindOptionsWhere } from '@n8n/typeorm';
@@ -161,19 +161,22 @@ export class WorkflowFinderService {
): Promise<Set<string>> {
if (workflowIds.length === 0) return new Set();
const where = await this.findAllWhere(user, scopes);
const sharedWorkflows = await this.sharedWorkflowRepository.find({
select: { workflowId: true },
where: { ...where, workflowId: In(workflowIds) },
});
return new Set(sharedWorkflows.map((sw) => sw.workflowId));
const found = new Set<string>();
for (const chunk of chunkIds(workflowIds)) {
const sharedWorkflows = await this.sharedWorkflowRepository.find({
select: { workflowId: true },
where: { ...where, workflowId: In(chunk) },
});
for (const sw of sharedWorkflows) found.add(sw.workflowId);
}
return found;
}
async findExistingWorkflowIds(workflowIds: string[]): Promise<Set<string>> {
if (workflowIds.length === 0) return new Set();
const workflows = await this.workflowRepository.find({
select: { id: true },
where: { id: In(workflowIds) },
});
const workflows = await this.workflowRepository.findByIds(workflowIds, { fields: ['id'] });
return new Set(workflows.map(({ id }) => id));
}
@@ -190,48 +193,56 @@ export class WorkflowFinderService {
if (workflowIds.length === 0) return [];
const where = await this.findAllWhere(user, scopes);
const sharedWorkflows = await this.sharedWorkflowRepository.find({
where: { ...where, workflowId: In(workflowIds) },
relations: {
workflow: {
parentFolder: options.includeParentFolder,
tags: options.includeTags,
activeVersion: options.includeActiveVersion,
},
},
});
// A workflow may appear via several share paths (project membership +
// direct share); dedupe so callers see one entity per id.
const seen = new Set<string>();
const workflows: WorkflowEntity[] = [];
for (const { workflow } of sharedWorkflows) {
if (seen.has(workflow.id)) continue;
seen.add(workflow.id);
workflows.push(workflow);
for (const chunk of chunkIds(workflowIds)) {
const sharedWorkflows = await this.sharedWorkflowRepository.find({
where: { ...where, workflowId: In(chunk) },
relations: {
workflow: {
parentFolder: options.includeParentFolder,
tags: options.includeTags,
activeVersion: options.includeActiveVersion,
},
},
});
for (const { workflow } of sharedWorkflows) {
if (seen.has(workflow.id)) continue;
seen.add(workflow.id);
workflows.push(workflow);
}
}
return workflows;
}
async findWorkflowIdsByFolder(folderIds: string[]): Promise<Map<string, string[]>> {
if (folderIds.length === 0) return new Map();
const rows = await this.sharedWorkflowRepository.find({
where: { workflow: { parentFolder: In(folderIds) } },
relations: { workflow: { parentFolder: true } },
select: { workflowId: true, workflow: { id: true, parentFolder: { id: true } } },
});
const byFolder = new Map<string, string[]>();
const seen = new Set<string>();
for (const { workflow } of rows) {
const folderId = workflow.parentFolder?.id;
// A workflow may appear via several share rows; dedupe so it lands once.
if (!folderId || seen.has(workflow.id)) continue;
seen.add(workflow.id);
const list = byFolder.get(folderId) ?? [];
list.push(workflow.id);
byFolder.set(folderId, list);
for (const chunk of chunkIds(folderIds)) {
const rows = await this.sharedWorkflowRepository.find({
where: { workflow: { parentFolder: In(chunk) } },
relations: { workflow: { parentFolder: true } },
select: { workflowId: true, workflow: { id: true, parentFolder: { id: true } } },
});
for (const { workflow } of rows) {
const folderId = workflow.parentFolder?.id;
// A workflow may appear via several share rows; dedupe so it lands once.
if (!folderId || seen.has(workflow.id)) continue;
seen.add(workflow.id);
const list = byFolder.get(folderId) ?? [];
list.push(workflow.id);
byFolder.set(folderId, list);
}
}
return byFolder;
@@ -307,27 +318,37 @@ export class WorkflowFinderService {
activeVersion: options.includeActiveVersion,
parentFolder: options.includeParentFolder,
};
const sharedWorkflows = await this.sharedWorkflowRepository.find({
where: [
{
projectId,
role: 'workflow:owner',
workflow: { sourceWorkflowId: In(sourceWorkflowIds), isArchived: false },
},
{
projectId,
role: 'workflow:owner',
workflow: {
id: In(sourceWorkflowIds),
sourceWorkflowId: IsNull(),
isArchived: false,
},
},
],
relations: { workflow: workflowRelations },
});
return sharedWorkflows.map(({ workflow }) => workflow);
const workflows = new Map<string, WorkflowEntity>();
// The two branches below each expand the id list, so this costs two bind
// parameters per id — half the ids fit in one statement compared with a
// single-branch lookup.
for (const chunk of chunkIds(sourceWorkflowIds)) {
const sharedWorkflows = await this.sharedWorkflowRepository.find({
where: [
{
projectId,
role: 'workflow:owner',
workflow: { sourceWorkflowId: In(chunk), isArchived: false },
},
{
projectId,
role: 'workflow:owner',
workflow: {
id: In(chunk),
sourceWorkflowId: IsNull(),
isArchived: false,
},
},
],
relations: { workflow: workflowRelations },
});
for (const { workflow } of sharedWorkflows) workflows.set(workflow.id, workflow);
}
return [...workflows.values()];
}
async hasProjectScopeForUser(user: User, scopes: Scope[], projectId: string) {
@@ -0,0 +1,26 @@
import { testDb } from '@n8n/backend-test-utils';
import { FolderRepository } from '@n8n/db';
import { Container } from '@n8n/di';
const SQLITE_MAX_BIND_PARAMETERS = 32_766;
describe('FolderRepository', () => {
beforeAll(async () => {
await testDb.init();
});
afterAll(async () => {
await testDb.terminate();
});
it('resolves subtree ids beyond the SQLite bind-parameter limit', async () => {
const folderIds = Array.from(
{ length: SQLITE_MAX_BIND_PARAMETERS + 1 },
(_, index) => `folder-${index}`,
);
await expect(
Container.get(FolderRepository).getAllFolderIdsInSubtrees(folderIds),
).resolves.toEqual([]);
});
});