mirror of
https://github.com/n8n-io/n8n.git
synced 2026-08-28 17:22:01 +08:00
perf(core): Chunk package ID lookups (#36923)
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
committed by
GitHub
parent
a53a75d193
commit
98e51043a8
@@ -11,6 +11,7 @@ export {
|
||||
} from './entities/abstract-entity';
|
||||
|
||||
export { generateNanoId } from '@n8n/utils/generate-nano-id';
|
||||
export { chunkIds } from './utils/chunk-ids';
|
||||
export { dbNowLiteral, dbNowPlusMsLiteral, parseDbTime } from './utils/dialect-time';
|
||||
export { generateHostInstanceId } from './utils/generators';
|
||||
export { isStringArray } from './utils/is-string-array';
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
import { In } from '@n8n/typeorm';
|
||||
|
||||
import { Folder } from '../../entities';
|
||||
import { mockEntityManager } from '../../utils/test-utils/mock-entity-manager';
|
||||
import { FolderRepository } from '../folder.repository';
|
||||
|
||||
describe('FolderRepository', () => {
|
||||
const entityManager = mockEntityManager(Folder);
|
||||
const repository = new FolderRepository(entityManager.connection);
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
});
|
||||
|
||||
it('merges folders returned from different chunks', async () => {
|
||||
const first = Object.assign(new Folder(), { id: 'first' });
|
||||
const last = Object.assign(new Folder(), { id: 'last' });
|
||||
entityManager.find.mockResolvedValueOnce([first]).mockResolvedValueOnce([last]);
|
||||
const folderIds = Array.from({ length: 10_001 }, (_, index) => `folder-${index}`);
|
||||
|
||||
const result = await repository.findManyByIds(folderIds);
|
||||
|
||||
expect(entityManager.find).toHaveBeenCalledTimes(2);
|
||||
expect(entityManager.find).toHaveBeenNthCalledWith(2, Folder, {
|
||||
where: { id: In(['folder-10000']) },
|
||||
relations: { homeProject: true },
|
||||
});
|
||||
expect(result).toEqual([first, last]);
|
||||
});
|
||||
|
||||
it('merges existing ids returned from different chunks', async () => {
|
||||
entityManager.find
|
||||
.mockResolvedValueOnce([{ id: 'first' }])
|
||||
.mockResolvedValueOnce([{ id: 'last' }]);
|
||||
const folderIds = Array.from({ length: 10_001 }, (_, index) => `folder-${index}`);
|
||||
|
||||
const result = await repository.findExistingIds(folderIds);
|
||||
|
||||
expect(entityManager.find).toHaveBeenCalledTimes(2);
|
||||
expect(result).toEqual(new Set(['first', 'last']));
|
||||
});
|
||||
});
|
||||
@@ -96,5 +96,49 @@ describe('SharedWorkflowRepository', () => {
|
||||
|
||||
expect(result).toEqual(new Map());
|
||||
});
|
||||
|
||||
it('merges owner projects returned from different chunks', async () => {
|
||||
const firstProject = mock<Project>({ id: 'first-project' });
|
||||
const lastProject = mock<Project>({ id: 'last-project' });
|
||||
entityManager.find
|
||||
.mockResolvedValueOnce([
|
||||
mock<SharedWorkflow>({ workflowId: 'first', project: firstProject }),
|
||||
])
|
||||
.mockResolvedValueOnce([
|
||||
mock<SharedWorkflow>({ workflowId: 'last', project: lastProject }),
|
||||
]);
|
||||
const workflowIds = Array.from({ length: 10_001 }, (_, index) => `workflow-${index}`);
|
||||
|
||||
const result = await sharedWorkflowRepository.findOwnerProjectsByWorkflowIds(workflowIds);
|
||||
|
||||
expect(entityManager.find).toHaveBeenCalledTimes(2);
|
||||
expect(result).toEqual(
|
||||
new Map([
|
||||
['first', firstProject],
|
||||
['last', lastProject],
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('findByWorkflowIds', () => {
|
||||
it('merges owner rows returned from different chunks', async () => {
|
||||
const first = mock<SharedWorkflow>({ workflowId: 'first' });
|
||||
const last = mock<SharedWorkflow>({ workflowId: 'last' });
|
||||
entityManager.find.mockResolvedValueOnce([first]).mockResolvedValueOnce([last]);
|
||||
const workflowIds = Array.from({ length: 10_001 }, (_, index) => `workflow-${index}`);
|
||||
|
||||
const result = await sharedWorkflowRepository.findByWorkflowIds(workflowIds);
|
||||
|
||||
expect(entityManager.find).toHaveBeenCalledTimes(2);
|
||||
expect(entityManager.find).toHaveBeenNthCalledWith(2, SharedWorkflow, {
|
||||
where: {
|
||||
role: 'workflow:owner',
|
||||
workflowId: In(['workflow-10000']),
|
||||
},
|
||||
relations: { project: { projectRelations: { user: true, role: true } } },
|
||||
});
|
||||
expect(result).toEqual([first, last]);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -553,6 +553,43 @@ describe('WorkflowRepository', () => {
|
||||
expect(findSpy).toHaveBeenCalledTimes(1);
|
||||
expect(findSpy).toHaveBeenCalledWith({ where: { id: In(workflowIds) } });
|
||||
});
|
||||
|
||||
it('merges workflows returned from different chunks', async () => {
|
||||
const first = Object.assign(new WorkflowEntity(), { id: 'first' });
|
||||
const last = Object.assign(new WorkflowEntity(), { id: 'last' });
|
||||
const findSpy = vi
|
||||
.spyOn(workflowRepository, 'find')
|
||||
.mockResolvedValueOnce([first])
|
||||
.mockResolvedValueOnce([last]);
|
||||
const workflowIds = Array.from({ length: 10_001 }, (_, index) => `workflow-${index}`);
|
||||
|
||||
const result = await workflowRepository.findByIds(workflowIds, { fields: ['name'] });
|
||||
|
||||
expect(findSpy).toHaveBeenCalledTimes(2);
|
||||
expect(findSpy).toHaveBeenNthCalledWith(2, {
|
||||
where: { id: In(['workflow-10000']) },
|
||||
select: ['id', 'name'],
|
||||
});
|
||||
expect(result).toEqual([first, last]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('findPreExistingWorkflows', () => {
|
||||
it('merges workflows returned from different chunks', async () => {
|
||||
const first = Object.assign(new WorkflowEntity(), { id: 'first' });
|
||||
const last = Object.assign(new WorkflowEntity(), { id: 'last' });
|
||||
queryBuilder.getMany.mockResolvedValueOnce([first]).mockResolvedValueOnce([last]);
|
||||
const workflowIds = Array.from({ length: 10_001 }, (_, index) => `workflow-${index}`);
|
||||
|
||||
const result = await workflowRepository.findPreExistingWorkflows(workflowIds);
|
||||
|
||||
expect(queryBuilder.getMany).toHaveBeenCalledTimes(2);
|
||||
expect(result).toEqual([first, last]);
|
||||
expect(queryBuilder.where.mock.calls[1]).toEqual([
|
||||
'workflow.id IN (:...workflowIds)',
|
||||
{ workflowIds: ['workflow-10000'] },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getPublishedPersonalWorkflowsCount', () => {
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import { Service } from '@n8n/di';
|
||||
import type { EntityManager, SelectQueryBuilder } from '@n8n/typeorm';
|
||||
import { DataSource, Repository } from '@n8n/typeorm';
|
||||
import { DataSource, In, Repository } from '@n8n/typeorm';
|
||||
import { PROJECT_ROOT } from 'n8n-workflow';
|
||||
|
||||
import { Folder, FolderTagMapping, TagEntity } from '../entities';
|
||||
import type { FolderWithWorkflowAndSubFolderCountAndPath, ListQuery } from '../entities/types-db';
|
||||
import { chunkIds } from '../utils/chunk-ids';
|
||||
import { parseListQuerySortBy } from '../utils/list-query-sort';
|
||||
|
||||
@Service()
|
||||
@@ -13,6 +14,31 @@ export class FolderRepository extends Repository<Folder> {
|
||||
super(Folder, dataSource.manager);
|
||||
}
|
||||
|
||||
async findExistingIds(folderIds: string[]): Promise<Set<string>> {
|
||||
const ids = new Set<string>();
|
||||
|
||||
for (const chunk of chunkIds(folderIds)) {
|
||||
const found = await this.find({ select: { id: true }, where: { id: In(chunk) } });
|
||||
for (const { id } of found) ids.add(id);
|
||||
}
|
||||
|
||||
return ids;
|
||||
}
|
||||
|
||||
async findManyByIds(folderIds: string[]): Promise<Folder[]> {
|
||||
const folders = new Map<string, Folder>();
|
||||
|
||||
for (const chunk of chunkIds(folderIds)) {
|
||||
const found = await this.find({
|
||||
where: { id: In(chunk) },
|
||||
relations: { homeProject: true },
|
||||
});
|
||||
for (const folder of found) folders.set(folder.id, folder);
|
||||
}
|
||||
|
||||
return [...folders.values()];
|
||||
}
|
||||
|
||||
async getManyAndCount(options: ListQuery.Options = {}) {
|
||||
const query = this.getManyQuery(options);
|
||||
return (await query.getManyAndCount()) as unknown as [
|
||||
@@ -391,28 +417,35 @@ export class FolderRepository extends Repository<Folder> {
|
||||
async getAllFolderIdsInSubtrees(parentFolderIds: string[]): Promise<string[]> {
|
||||
if (parentFolderIds.length === 0) return [];
|
||||
|
||||
// Base case: the direct children of any requested parent.
|
||||
const baseQuery = this.createQueryBuilder('f')
|
||||
.select('f.id', 'id')
|
||||
.where('f.parentFolderId IN (:...parentFolderIds)', { parentFolderIds });
|
||||
// Subtrees are independent, so each chunk resolves in its own recursive
|
||||
// query and the ids are merged — a folder reachable from two chunks lands once.
|
||||
const ids = new Set<string>();
|
||||
|
||||
// Recursive case: descendants of folders already in the tree.
|
||||
const recursiveQuery = this.createQueryBuilder('child')
|
||||
.select('child.id', 'id')
|
||||
.innerJoin('folder_tree', 'parent', 'child.parentFolderId = parent.id');
|
||||
for (const chunk of chunkIds(parentFolderIds)) {
|
||||
// Base case: the direct children of any requested parent.
|
||||
const baseQuery = this.createQueryBuilder('f')
|
||||
.select('f.id', 'id')
|
||||
.where('f.parentFolderId IN (:...parentFolderIds)', { parentFolderIds: chunk });
|
||||
|
||||
const query = this.createQueryBuilder()
|
||||
.addCommonTableExpression(
|
||||
`${baseQuery.getQuery()} UNION ALL ${recursiveQuery.getQuery()}`,
|
||||
'folder_tree',
|
||||
{ recursive: true },
|
||||
)
|
||||
.select('DISTINCT tree.id', 'id')
|
||||
.from('folder_tree', 'tree')
|
||||
.setParameters(baseQuery.getParameters());
|
||||
// Recursive case: descendants of folders already in the tree.
|
||||
const recursiveQuery = this.createQueryBuilder('child')
|
||||
.select('child.id', 'id')
|
||||
.innerJoin('folder_tree', 'parent', 'child.parentFolderId = parent.id');
|
||||
|
||||
const result = await query.getRawMany<{ id: string }>();
|
||||
return result.map((row) => row.id);
|
||||
const query = this.createQueryBuilder()
|
||||
.addCommonTableExpression(
|
||||
`${baseQuery.getQuery()} UNION ALL ${recursiveQuery.getQuery()}`,
|
||||
'folder_tree',
|
||||
{ recursive: true },
|
||||
)
|
||||
.select('DISTINCT tree.id', 'id')
|
||||
.from('folder_tree', 'tree')
|
||||
.setParameters(baseQuery.getParameters());
|
||||
|
||||
for (const row of await query.getRawMany<{ id: string }>()) ids.add(row.id);
|
||||
}
|
||||
|
||||
return [...ids];
|
||||
}
|
||||
|
||||
async getFolderPathsToRoot(folderIds: string[]): Promise<Map<string, string[]>> {
|
||||
|
||||
@@ -17,6 +17,7 @@ import { BaseRepository } from './base-repository';
|
||||
import type { User } from '../entities';
|
||||
import { Project, ProjectRelation, SharedWorkflow } from '../entities';
|
||||
import { type OperationContext, TransactionRunner } from '../services/transaction';
|
||||
import { chunkIds } from '../utils/chunk-ids';
|
||||
|
||||
@Service()
|
||||
export class SharedWorkflowRepository extends BaseRepository<SharedWorkflow> {
|
||||
@@ -35,21 +36,34 @@ export class SharedWorkflowRepository extends BaseRepository<SharedWorkflow> {
|
||||
}
|
||||
|
||||
async findByWorkflowIds(workflowIds: string[]) {
|
||||
return await this.find({
|
||||
where: {
|
||||
role: 'workflow:owner',
|
||||
workflowId: In(workflowIds),
|
||||
},
|
||||
relations: { project: { projectRelations: { user: true, role: true } } },
|
||||
});
|
||||
const rows = new Map<string, SharedWorkflow>();
|
||||
|
||||
for (const chunk of chunkIds(workflowIds)) {
|
||||
const found = await this.find({
|
||||
where: {
|
||||
role: 'workflow:owner',
|
||||
workflowId: In(chunk),
|
||||
},
|
||||
relations: { project: { projectRelations: { user: true, role: true } } },
|
||||
});
|
||||
for (const row of found) rows.set(row.workflowId, row);
|
||||
}
|
||||
|
||||
return [...rows.values()];
|
||||
}
|
||||
|
||||
/** Owner project of each workflow, keyed by workflow id. */
|
||||
async findOwnerProjectsByWorkflowIds(workflowIds: string[]): Promise<Map<string, Project>> {
|
||||
const ownerRows = await this.find({
|
||||
where: { workflowId: In(workflowIds), role: 'workflow:owner' },
|
||||
relations: { project: true },
|
||||
});
|
||||
const ownerRows: SharedWorkflow[] = [];
|
||||
|
||||
for (const chunk of chunkIds(workflowIds)) {
|
||||
ownerRows.push(
|
||||
...(await this.find({
|
||||
where: { workflowId: In(chunk), role: 'workflow:owner' },
|
||||
relations: { project: true },
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
return new Map(ownerRows.map(({ workflowId, project }) => [workflowId, project]));
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ import type {
|
||||
} from '../entities/types-db';
|
||||
import { type OperationContext, TransactionRunner } from '../services/transaction';
|
||||
import { applyWorkflowBooleanSettingFilter } from '../utils/apply-workflow-boolean-setting-filter';
|
||||
import { chunkIds } from '../utils/chunk-ids';
|
||||
import { isStringArray } from '../utils/is-string-array';
|
||||
import { parseListQuerySortBy } from '../utils/list-query-sort';
|
||||
import { TimedQuery } from '../utils/timed-query';
|
||||
@@ -239,13 +240,20 @@ export class WorkflowRepository extends BaseRepository<WorkflowEntity> {
|
||||
return [];
|
||||
}
|
||||
|
||||
const options: FindManyOptions<WorkflowEntity> = {
|
||||
where: { id: In(workflowIds) },
|
||||
};
|
||||
const workflows = new Map<string, WorkflowEntity>();
|
||||
for (const chunk of chunkIds(workflowIds)) {
|
||||
const options: FindManyOptions<WorkflowEntity> = {
|
||||
where: { id: In(chunk) },
|
||||
};
|
||||
|
||||
if (fields?.length) options.select = fields as FindOptionsSelect<WorkflowEntity>;
|
||||
if (fields?.length) {
|
||||
options.select = [...new Set(['id', ...fields])] as FindOptionsSelect<WorkflowEntity>;
|
||||
}
|
||||
|
||||
return await this.find(options);
|
||||
for (const workflow of await this.find(options)) workflows.set(workflow.id, workflow);
|
||||
}
|
||||
|
||||
return [...workflows.values()];
|
||||
}
|
||||
|
||||
async findManyByAgentToolReferences(
|
||||
@@ -292,12 +300,21 @@ export class WorkflowRepository extends BaseRepository<WorkflowEntity> {
|
||||
return [];
|
||||
}
|
||||
|
||||
return await this.createQueryBuilder('workflow')
|
||||
.select(['workflow.id', 'workflow.name', 'workflow.isArchived'])
|
||||
.leftJoin('workflow.shared', 'shared', 'shared.role = :role', { role: 'workflow:owner' })
|
||||
.addSelect(['shared.workflowId', 'shared.projectId', 'shared.role'])
|
||||
.where('workflow.id IN (:...workflowIds)', { workflowIds })
|
||||
.getMany();
|
||||
const found = new Map<string, WorkflowEntity>();
|
||||
|
||||
for (const chunk of chunkIds(workflowIds)) {
|
||||
const workflows = await this.createQueryBuilder('workflow')
|
||||
.select(['workflow.id', 'workflow.name', 'workflow.isArchived'])
|
||||
.leftJoin('workflow.shared', 'shared', 'shared.role = :role', {
|
||||
role: 'workflow:owner',
|
||||
})
|
||||
.addSelect(['shared.workflowId', 'shared.projectId', 'shared.role'])
|
||||
.where('workflow.id IN (:...workflowIds)', { workflowIds: chunk })
|
||||
.getMany();
|
||||
for (const workflow of workflows) found.set(workflow.id, workflow);
|
||||
}
|
||||
|
||||
return [...found.values()];
|
||||
}
|
||||
|
||||
async getActiveTriggerCount() {
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import { chunkIds } from '../chunk-ids';
|
||||
|
||||
const SQLITE_MAX_BIND_PARAMETERS = 32_766;
|
||||
const MAX_BINDS_PER_ID = 3;
|
||||
const FIXED_BIND_PARAMETERS = 6;
|
||||
|
||||
describe('chunkIds', () => {
|
||||
it('keeps query batches within the SQLite bind-parameter limit', () => {
|
||||
const ids = Array.from({ length: 100_000 }, (_, index) => `id-${index}`);
|
||||
const chunks = chunkIds(ids);
|
||||
|
||||
expect(chunks.flat()).toEqual(ids);
|
||||
for (const chunk of chunks) {
|
||||
expect(chunk.length * MAX_BINDS_PER_ID + FIXED_BIND_PARAMETERS).toBeLessThanOrEqual(
|
||||
SQLITE_MAX_BIND_PARAMETERS,
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,8 @@
|
||||
import chunk from 'lodash/chunk';
|
||||
|
||||
/** Safe for three binds per ID: SQLite allows 32,766 binds; PostgreSQL, 65,535. */
|
||||
const ID_QUERY_BATCH_SIZE = 10_000;
|
||||
|
||||
export function chunkIds<T>(ids: T[]): T[][] {
|
||||
return chunk(ids, ID_QUERY_BATCH_SIZE);
|
||||
}
|
||||
@@ -23,6 +23,7 @@ function makeFolder(overrides: Partial<Folder> = {}): Folder {
|
||||
function makeFinder(found: Folder[]) {
|
||||
const folderRepository = mock<FolderRepository>();
|
||||
folderRepository.find.mockResolvedValue(found);
|
||||
folderRepository.findExistingIds.mockResolvedValue(new Set(found.map(({ id }) => id)));
|
||||
const roleService = mock<RoleService>();
|
||||
roleService.rolesWithScope.mockResolvedValue(['project:admin', 'project:editor']);
|
||||
const finder = new FolderFinderService(folderRepository, roleService);
|
||||
@@ -48,6 +49,26 @@ describe('FolderFinderService', () => {
|
||||
expect(result).toEqual(folders);
|
||||
});
|
||||
|
||||
it('merges distinct folders returned from different chunks', async () => {
|
||||
const firstFolder = makeFolder({ id: 'first' });
|
||||
const lastFolder = makeFolder({ id: 'last' });
|
||||
const { finder, folderRepository } = makeFinder([]);
|
||||
folderRepository.find.mockResolvedValueOnce([firstFolder]).mockResolvedValueOnce([lastFolder]);
|
||||
const folderIds = [
|
||||
...Array.from({ length: 10_000 }, (_, index) => `folder-${index}`),
|
||||
lastFolder.id,
|
||||
];
|
||||
|
||||
const result = await finder.findFoldersByIdsForUser(folderIds, nonGlobalUser, ['folder:read']);
|
||||
|
||||
expect(folderRepository.find).toHaveBeenCalledTimes(2);
|
||||
expect(result).toEqual([firstFolder, lastFolder]);
|
||||
const secondChunk = folderRepository.find.mock.calls[1][0]?.where as unknown as {
|
||||
id: { value: string[] };
|
||||
};
|
||||
expect(secondChunk.id.value).toEqual([lastFolder.id]);
|
||||
});
|
||||
|
||||
it('filters by the requested project scope for non-global users', async () => {
|
||||
const { finder, folderRepository, roleService } = makeFinder([makeFolder()]);
|
||||
|
||||
@@ -181,15 +202,16 @@ describe('FolderFinderService', () => {
|
||||
const result = await finder.findExistingFolderIds([]);
|
||||
|
||||
expect(result.size).toBe(0);
|
||||
expect(folderRepository.find.mock.calls).toHaveLength(0);
|
||||
expect(folderRepository.findExistingIds).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns the ids that exist in the database, unscoped by access', async () => {
|
||||
const { finder } = makeFinder([makeFolder({ id: 'fld-1' })]);
|
||||
const { finder, folderRepository } = makeFinder([makeFolder({ id: 'fld-1' })]);
|
||||
|
||||
const result = await finder.findExistingFolderIds(['fld-1', 'fld-missing']);
|
||||
|
||||
expect(result).toEqual(new Set(['fld-1']));
|
||||
expect(folderRepository.findExistingIds).toHaveBeenCalledWith(['fld-1', 'fld-missing']);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { Folder, User } from '@n8n/db';
|
||||
import { FolderRepository } from '@n8n/db';
|
||||
import { chunkIds, FolderRepository } from '@n8n/db';
|
||||
import { Service } from '@n8n/di';
|
||||
import { hasGlobalScope, type Scope } from '@n8n/permissions';
|
||||
import type { FindOptionsWhere } from '@n8n/typeorm';
|
||||
@@ -22,11 +22,8 @@ export class FolderFinderService {
|
||||
|
||||
async findExistingFolderIds(folderIds: string[]): Promise<Set<string>> {
|
||||
if (folderIds.length === 0) return new Set();
|
||||
const folders = await this.folderRepository.find({
|
||||
select: { id: true },
|
||||
where: { id: In(folderIds) },
|
||||
});
|
||||
return new Set(folders.map(({ id }) => id));
|
||||
|
||||
return await this.folderRepository.findExistingIds(folderIds);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -74,9 +71,14 @@ export class FolderFinderService {
|
||||
|
||||
const accessWhere = await this.buildFolderReadWhere(user, scopes);
|
||||
|
||||
return await this.folderRepository.find({
|
||||
where: { id: In(folderIds), ...accessWhere },
|
||||
});
|
||||
const folders = new Map<string, Folder>();
|
||||
for (const chunk of chunkIds(folderIds)) {
|
||||
const found = await this.folderRepository.find({
|
||||
where: { id: In(chunk), ...accessWhere },
|
||||
});
|
||||
for (const folder of found) folders.set(folder.id, folder);
|
||||
}
|
||||
return [...folders.values()];
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -6,7 +6,7 @@ import type {
|
||||
} from '@n8n/db';
|
||||
import { Folder, FolderTagMappingRepository, FolderRepository, WorkflowRepository } from '@n8n/db';
|
||||
import { Service } from '@n8n/di';
|
||||
import { In, type EntityManager } from '@n8n/typeorm';
|
||||
import type { EntityManager } from '@n8n/typeorm';
|
||||
import { UserError, PROJECT_ROOT } from 'n8n-workflow';
|
||||
|
||||
import { FolderNotFoundError } from '@/errors/folder-not-found.error';
|
||||
@@ -55,11 +55,7 @@ export class FolderService {
|
||||
}
|
||||
|
||||
async getFoldersByIds(folderIds: string[]): Promise<Folder[]> {
|
||||
if (folderIds.length === 0) return [];
|
||||
return await this.folderRepository.find({
|
||||
where: { id: In(folderIds) },
|
||||
relations: { homeProject: true },
|
||||
});
|
||||
return await this.folderRepository.findManyByIds(folderIds);
|
||||
}
|
||||
|
||||
/** Every folder a project holds, flat, with the parent each one sits under (`null` at the root). */
|
||||
|
||||
@@ -86,16 +86,42 @@ describe('WorkflowFinderService', () => {
|
||||
const result = await service.findExistingWorkflowIds([]);
|
||||
|
||||
expect(result.size).toBe(0);
|
||||
expect(workflowRepository.find).not.toHaveBeenCalled();
|
||||
expect(workflowRepository.findByIds).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns the ids that exist in the database, unscoped by access', async () => {
|
||||
const { service, workflowRepository } = makeService();
|
||||
workflowRepository.find.mockResolvedValue([{ id: 'wf-1' }] as never);
|
||||
workflowRepository.findByIds.mockResolvedValue([{ id: 'wf-1' }] as never);
|
||||
|
||||
const result = await service.findExistingWorkflowIds(['wf-1', 'wf-missing']);
|
||||
|
||||
expect(result).toEqual(new Set(['wf-1']));
|
||||
expect(workflowRepository.findByIds).toHaveBeenCalledWith(['wf-1', 'wf-missing'], {
|
||||
fields: ['id'],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('findOwnedWorkflowsBySourceWorkflowIds', () => {
|
||||
it('merges workflows returned from different chunks', async () => {
|
||||
const { service, sharedWorkflowRepository } = makeService();
|
||||
sharedWorkflowRepository.find
|
||||
.mockResolvedValueOnce([{ workflow: { id: 'first' } }] as never)
|
||||
.mockResolvedValueOnce([{ workflow: { id: 'last' } }] as never);
|
||||
const sourceWorkflowIds = Array.from({ length: 10_001 }, (_, index) => `source-${index}`);
|
||||
|
||||
const result = await service.findOwnedWorkflowsBySourceWorkflowIds(
|
||||
'project-1',
|
||||
sourceWorkflowIds,
|
||||
);
|
||||
|
||||
expect(sharedWorkflowRepository.find).toHaveBeenCalledTimes(2);
|
||||
expect(result.map(({ id }) => id)).toEqual(['first', 'last']);
|
||||
const secondChunkWhere = sharedWorkflowRepository.find.mock.calls[1][0]?.where as Array<{
|
||||
workflow: { id?: { value: string[] }; sourceWorkflowId?: { value: string[] } };
|
||||
}>;
|
||||
expect(secondChunkWhere[0].workflow.sourceWorkflowId?.value).toEqual(['source-10000']);
|
||||
expect(secondChunkWhere[1].workflow.id?.value).toEqual(['source-10000']);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { SharedWorkflow, User, WorkflowEntity, ListQuery } from '@n8n/db';
|
||||
import { SharedWorkflowRepository, FolderRepository, WorkflowRepository } from '@n8n/db';
|
||||
import { SharedWorkflowRepository, FolderRepository, WorkflowRepository, chunkIds } from '@n8n/db';
|
||||
import { Service } from '@n8n/di';
|
||||
import { hasGlobalScope, type Scope } from '@n8n/permissions';
|
||||
import type { EntityManager, FindOptionsWhere } from '@n8n/typeorm';
|
||||
@@ -161,19 +161,22 @@ export class WorkflowFinderService {
|
||||
): Promise<Set<string>> {
|
||||
if (workflowIds.length === 0) return new Set();
|
||||
const where = await this.findAllWhere(user, scopes);
|
||||
const sharedWorkflows = await this.sharedWorkflowRepository.find({
|
||||
select: { workflowId: true },
|
||||
where: { ...where, workflowId: In(workflowIds) },
|
||||
});
|
||||
return new Set(sharedWorkflows.map((sw) => sw.workflowId));
|
||||
|
||||
const found = new Set<string>();
|
||||
for (const chunk of chunkIds(workflowIds)) {
|
||||
const sharedWorkflows = await this.sharedWorkflowRepository.find({
|
||||
select: { workflowId: true },
|
||||
where: { ...where, workflowId: In(chunk) },
|
||||
});
|
||||
for (const sw of sharedWorkflows) found.add(sw.workflowId);
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
async findExistingWorkflowIds(workflowIds: string[]): Promise<Set<string>> {
|
||||
if (workflowIds.length === 0) return new Set();
|
||||
const workflows = await this.workflowRepository.find({
|
||||
select: { id: true },
|
||||
where: { id: In(workflowIds) },
|
||||
});
|
||||
|
||||
const workflows = await this.workflowRepository.findByIds(workflowIds, { fields: ['id'] });
|
||||
return new Set(workflows.map(({ id }) => id));
|
||||
}
|
||||
|
||||
@@ -190,48 +193,56 @@ export class WorkflowFinderService {
|
||||
if (workflowIds.length === 0) return [];
|
||||
|
||||
const where = await this.findAllWhere(user, scopes);
|
||||
const sharedWorkflows = await this.sharedWorkflowRepository.find({
|
||||
where: { ...where, workflowId: In(workflowIds) },
|
||||
relations: {
|
||||
workflow: {
|
||||
parentFolder: options.includeParentFolder,
|
||||
tags: options.includeTags,
|
||||
activeVersion: options.includeActiveVersion,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// A workflow may appear via several share paths (project membership +
|
||||
// direct share); dedupe so callers see one entity per id.
|
||||
const seen = new Set<string>();
|
||||
const workflows: WorkflowEntity[] = [];
|
||||
for (const { workflow } of sharedWorkflows) {
|
||||
if (seen.has(workflow.id)) continue;
|
||||
seen.add(workflow.id);
|
||||
workflows.push(workflow);
|
||||
|
||||
for (const chunk of chunkIds(workflowIds)) {
|
||||
const sharedWorkflows = await this.sharedWorkflowRepository.find({
|
||||
where: { ...where, workflowId: In(chunk) },
|
||||
relations: {
|
||||
workflow: {
|
||||
parentFolder: options.includeParentFolder,
|
||||
tags: options.includeTags,
|
||||
activeVersion: options.includeActiveVersion,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
for (const { workflow } of sharedWorkflows) {
|
||||
if (seen.has(workflow.id)) continue;
|
||||
seen.add(workflow.id);
|
||||
workflows.push(workflow);
|
||||
}
|
||||
}
|
||||
|
||||
return workflows;
|
||||
}
|
||||
|
||||
async findWorkflowIdsByFolder(folderIds: string[]): Promise<Map<string, string[]>> {
|
||||
if (folderIds.length === 0) return new Map();
|
||||
|
||||
const rows = await this.sharedWorkflowRepository.find({
|
||||
where: { workflow: { parentFolder: In(folderIds) } },
|
||||
relations: { workflow: { parentFolder: true } },
|
||||
select: { workflowId: true, workflow: { id: true, parentFolder: { id: true } } },
|
||||
});
|
||||
|
||||
const byFolder = new Map<string, string[]>();
|
||||
const seen = new Set<string>();
|
||||
for (const { workflow } of rows) {
|
||||
const folderId = workflow.parentFolder?.id;
|
||||
// A workflow may appear via several share rows; dedupe so it lands once.
|
||||
if (!folderId || seen.has(workflow.id)) continue;
|
||||
seen.add(workflow.id);
|
||||
const list = byFolder.get(folderId) ?? [];
|
||||
list.push(workflow.id);
|
||||
byFolder.set(folderId, list);
|
||||
|
||||
for (const chunk of chunkIds(folderIds)) {
|
||||
const rows = await this.sharedWorkflowRepository.find({
|
||||
where: { workflow: { parentFolder: In(chunk) } },
|
||||
relations: { workflow: { parentFolder: true } },
|
||||
select: { workflowId: true, workflow: { id: true, parentFolder: { id: true } } },
|
||||
});
|
||||
|
||||
for (const { workflow } of rows) {
|
||||
const folderId = workflow.parentFolder?.id;
|
||||
// A workflow may appear via several share rows; dedupe so it lands once.
|
||||
if (!folderId || seen.has(workflow.id)) continue;
|
||||
seen.add(workflow.id);
|
||||
const list = byFolder.get(folderId) ?? [];
|
||||
list.push(workflow.id);
|
||||
byFolder.set(folderId, list);
|
||||
}
|
||||
}
|
||||
|
||||
return byFolder;
|
||||
@@ -307,27 +318,37 @@ export class WorkflowFinderService {
|
||||
activeVersion: options.includeActiveVersion,
|
||||
parentFolder: options.includeParentFolder,
|
||||
};
|
||||
const sharedWorkflows = await this.sharedWorkflowRepository.find({
|
||||
where: [
|
||||
{
|
||||
projectId,
|
||||
role: 'workflow:owner',
|
||||
workflow: { sourceWorkflowId: In(sourceWorkflowIds), isArchived: false },
|
||||
},
|
||||
{
|
||||
projectId,
|
||||
role: 'workflow:owner',
|
||||
workflow: {
|
||||
id: In(sourceWorkflowIds),
|
||||
sourceWorkflowId: IsNull(),
|
||||
isArchived: false,
|
||||
},
|
||||
},
|
||||
],
|
||||
relations: { workflow: workflowRelations },
|
||||
});
|
||||
|
||||
return sharedWorkflows.map(({ workflow }) => workflow);
|
||||
const workflows = new Map<string, WorkflowEntity>();
|
||||
|
||||
// The two branches below each expand the id list, so this costs two bind
|
||||
// parameters per id — half the ids fit in one statement compared with a
|
||||
// single-branch lookup.
|
||||
for (const chunk of chunkIds(sourceWorkflowIds)) {
|
||||
const sharedWorkflows = await this.sharedWorkflowRepository.find({
|
||||
where: [
|
||||
{
|
||||
projectId,
|
||||
role: 'workflow:owner',
|
||||
workflow: { sourceWorkflowId: In(chunk), isArchived: false },
|
||||
},
|
||||
{
|
||||
projectId,
|
||||
role: 'workflow:owner',
|
||||
workflow: {
|
||||
id: In(chunk),
|
||||
sourceWorkflowId: IsNull(),
|
||||
isArchived: false,
|
||||
},
|
||||
},
|
||||
],
|
||||
relations: { workflow: workflowRelations },
|
||||
});
|
||||
|
||||
for (const { workflow } of sharedWorkflows) workflows.set(workflow.id, workflow);
|
||||
}
|
||||
|
||||
return [...workflows.values()];
|
||||
}
|
||||
|
||||
async hasProjectScopeForUser(user: User, scopes: Scope[], projectId: string) {
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import { testDb } from '@n8n/backend-test-utils';
|
||||
import { FolderRepository } from '@n8n/db';
|
||||
import { Container } from '@n8n/di';
|
||||
|
||||
const SQLITE_MAX_BIND_PARAMETERS = 32_766;
|
||||
|
||||
describe('FolderRepository', () => {
|
||||
beforeAll(async () => {
|
||||
await testDb.init();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await testDb.terminate();
|
||||
});
|
||||
|
||||
it('resolves subtree ids beyond the SQLite bind-parameter limit', async () => {
|
||||
const folderIds = Array.from(
|
||||
{ length: SQLITE_MAX_BIND_PARAMETERS + 1 },
|
||||
(_, index) => `folder-${index}`,
|
||||
);
|
||||
|
||||
await expect(
|
||||
Container.get(FolderRepository).getAllFolderIdsInSubtrees(folderIds),
|
||||
).resolves.toEqual([]);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user