From e93deef366ce2aa27cfb43364f65b21c6eb8f443 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ir=C3=A9n=C3=A9e?= Date: Mon, 31 Aug 2026 19:19:53 +0000 Subject: [PATCH] feat(API): Add git connections pull to import projects from working copy (no-changelog) (#36770) Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: Nour Alhadi Mahmoud Co-authored-by: Cursor --- .../git-connections/git-connections.dto.ts | 44 ++++ packages/@n8n/api-types/src/dto/index.ts | 1 + .../@n8n/cli/src/__tests__/client.test.ts | 31 +++ packages/@n8n/cli/src/client.ts | 38 ++++ .../cli/src/commands/git-connections/pull.ts | 24 +++ packages/@n8n/cli/src/index.ts | 2 + .../scope-information.test.ts.snap | 1 + packages/@n8n/permissions/src/constants.ee.ts | 4 +- .../src/public-api-permissions.ee.ts | 1 + .../src/roles/scopes/global-scopes.ee.ts | 1 + .../@n8n/permissions/src/scope-information.ts | 5 + .../__tests__/credentials.service.test.ts | 34 +++ .../src/credentials/credentials.service.ts | 13 +- ...ion-project.repository.integration.test.ts | 114 ++++++++++ .../__tests__/git-connections.service.test.ts | 129 +++++++++++ .../git-connection-project.repository.ts | 28 ++- .../git-connections.service.ts | 127 ++++++++++- .../import-package.integration.test.ts | 6 +- ...project-from-directory.integration.test.ts | 198 +++++++++++++++++ .../n8n-packages/engine/import-result.ts | 3 + .../n8n-packages/engine/import-telemetry.ts | 11 +- .../engine/project-package-importer.ts | 26 +-- .../engine/workflow-package-importer.ts | 46 ++-- .../__tests__/credential-importer.test.ts | 63 +++++- .../credential/credential-importer.ts | 2 + .../directory-package-reader.test.ts | 158 ++++++++++++++ .../io/directory/directory-package-reader.ts | 177 +++++++++++++++ .../n8n-packages/n8n-packages.service.ts | 54 +++++ .../n8n-packages/n8n-packages.types.ts | 16 +- .../git-connections.public.controller.ts | 22 ++ .../pullGitConnectionProjects.generated.yml | 202 ++++++++++++++++++ .../spec/paths/n8n-packages.import.yml | 13 ++ .../v1/openapi.decorator-routes.generated.yml | 3 + .../public-api/git-connections.test.ts | 59 +++++ .../public-api/n8n-packages-import.test.ts | 4 + .../nodes/N8n/n8n-api-coverage.json | 3 + 36 files changed, 1600 insertions(+), 63 deletions(-) create mode 100644 packages/@n8n/cli/src/commands/git-connections/pull.ts create mode 100644 packages/cli/src/modules/git-connections.ee/__tests__/git-connection-project.repository.integration.test.ts create mode 100644 packages/cli/src/modules/n8n-packages/__tests__/import-project-from-directory.integration.test.ts create mode 100644 packages/cli/src/modules/n8n-packages/io/__tests__/directory-package-reader.test.ts create mode 100644 packages/cli/src/modules/n8n-packages/io/directory/directory-package-reader.ts create mode 100644 packages/cli/src/public-api/v1/handlers/git-connections/spec/paths/pullGitConnectionProjects.generated.yml diff --git a/packages/@n8n/api-types/src/dto/git-connections/git-connections.dto.ts b/packages/@n8n/api-types/src/dto/git-connections/git-connections.dto.ts index 5eb4a57e3b2..122b0305157 100644 --- a/packages/@n8n/api-types/src/dto/git-connections/git-connections.dto.ts +++ b/packages/@n8n/api-types/src/dto/git-connections/git-connections.dto.ts @@ -84,6 +84,50 @@ export const gitConnectionPushResultSchema = z.object({ export class GitConnectionPushResultDto extends Z.class(gitConnectionPushResultSchema.shape) {} +const count = () => z.number().int().nonnegative(); + +export const gitConnectionImportCountsSchema = z.object({ + projects: z.object({ created: count(), updated: count(), skipped: count() }), + folders: z.object({ created: count(), skipped: count(), removed: count() }), + workflows: z.object({ + created: count(), + updated: count(), + skipped: count(), + archived: count(), + deleted: count(), + publishing: z.object({ + published: count(), + unpublished: count(), + unchanged: count(), + blocked: count(), + failed: count(), + }), + }), + credentials: z.object({ matched: count(), stubbed: count() }), + dataTables: z.object({ matched: count(), created: count() }), + variables: z.object({ + matched: count(), + created: count(), + updated: count(), + stubbed: count(), + missing: count(), + }), + tags: z.object({ + matched: count(), + created: count(), + renamed: count(), + reconciled: count(), + skipped: count(), + }), +}); + +export const gitConnectionPullResultSchema = z.object({ + connectionId: z.string(), + counts: gitConnectionImportCountsSchema, +}); + +export class GitConnectionPullResultDto extends Z.class(gitConnectionPullResultSchema.shape) {} + export const gitConnectionSummarySchema = gitConnectionPublicSchema.omit({ publicKey: true }); export class GitConnectionListPublicDto extends Z.class({ diff --git a/packages/@n8n/api-types/src/dto/index.ts b/packages/@n8n/api-types/src/dto/index.ts index 18d9f614169..41767611179 100644 --- a/packages/@n8n/api-types/src/dto/index.ts +++ b/packages/@n8n/api-types/src/dto/index.ts @@ -105,6 +105,7 @@ export { GitConnectionProjectListPublicDto, GitConnectionProjectPublicDto, GitConnectionPublicDto, + GitConnectionPullResultDto, GitConnectionPushResultDto, ListGitConnectionsQueryDto, UpdateGitConnectionDto, diff --git a/packages/@n8n/cli/src/__tests__/client.test.ts b/packages/@n8n/cli/src/__tests__/client.test.ts index 68c352fe3d1..92d060cfe8d 100644 --- a/packages/@n8n/cli/src/__tests__/client.test.ts +++ b/packages/@n8n/cli/src/__tests__/client.test.ts @@ -58,6 +58,37 @@ describe('N8nClient packages', () => { }); }); + describe('pullGitConnectionProjects', () => { + it('posts to the pull endpoint and returns the import counts', async () => { + const response = { + connectionId: 'connection-id', + counts: { + projects: { created: 1, updated: 0, skipped: 0 }, + folders: { created: 0, skipped: 0, removed: 0 }, + workflows: { + created: 2, + updated: 0, + skipped: 0, + archived: 0, + deleted: 0, + publishing: { published: 2, unpublished: 0, unchanged: 0, blocked: 0, failed: 0 }, + }, + credentials: { matched: 0, stubbed: 1 }, + dataTables: { matched: 0, created: 0 }, + variables: { matched: 0, created: 0, updated: 0, stubbed: 0, missing: 0 }, + tags: { matched: 0, created: 0, renamed: 0, reconciled: 0, skipped: 0 }, + }, + }; + fetchMock.mockResolvedValue(jsonResponse(200, response)); + + await expect(client.pullGitConnectionProjects('connection-id')).resolves.toEqual(response); + + const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit]; + expect(url).toBe('https://n8n.example.com/api/v1/git-connections/connection-id/pull'); + expect(init.method).toBe('POST'); + }); + }); + describe('exportPackage', () => { it('posts the workflow IDs as JSON and returns the archive bytes', async () => { fetchMock.mockResolvedValue(binaryResponse(200, new Uint8Array([1, 2, 3]))); diff --git a/packages/@n8n/cli/src/client.ts b/packages/@n8n/cli/src/client.ts index d814748d8ed..a0c0cceef00 100644 --- a/packages/@n8n/cli/src/client.ts +++ b/packages/@n8n/cli/src/client.ts @@ -66,6 +66,40 @@ export type PushGitConnectionResult = { counts: ExportPackageCounts; }; +export interface ImportPackageCounts { + projects: { created: number; updated: number; skipped: number }; + folders: { created: number; skipped: number; removed: number }; + workflows: { + created: number; + updated: number; + skipped: number; + archived: number; + deleted: number; + publishing: { + published: number; + unpublished: number; + unchanged: number; + blocked: number; + failed: number; + }; + }; + credentials: { matched: number; stubbed: number }; + dataTables: { matched: number; created: number }; + variables: { + matched: number; + created: number; + updated: number; + stubbed: number; + missing: number; + }; + tags: { matched: number; created: number; renamed: number; reconciled: number; skipped: number }; +} + +export type PullGitConnectionResult = { + connectionId: string; + counts: ImportPackageCounts; +}; + export class ApiError extends Error { constructor( readonly statusCode: number, @@ -270,6 +304,10 @@ export class N8nClient { return await this.del(`/git-connections/${id}/projects/${projectId}`); } + async pullGitConnectionProjects(id: string) { + return await this.post(`/git-connections/${id}/pull`); + } + // ─── Workflows ───────────────────────────────────────────────── async listWorkflows(query: Record = {}, limit?: number) { diff --git a/packages/@n8n/cli/src/commands/git-connections/pull.ts b/packages/@n8n/cli/src/commands/git-connections/pull.ts new file mode 100644 index 00000000000..4bebac48b5e --- /dev/null +++ b/packages/@n8n/cli/src/commands/git-connections/pull.ts @@ -0,0 +1,24 @@ +import { Args } from '@oclif/core'; + +import { BaseCommand } from '../../base-command'; + +export default class GitConnectionsPull extends BaseCommand { + static override description = + 'Import all projects from a Git connection working copy into the instance, overwriting to match it (work in progress; does not pull from the remote)'; + static override args = { + id: Args.string({ description: 'ID of the Git connection', required: true }), + }; + static override flags = { ...BaseCommand.baseFlags }; + + async run() { + const { args, flags } = await this.parse(GitConnectionsPull); + await this.execute(async () => { + const result = await this.getClient(flags).pullGitConnectionProjects(args.id); + this.succeed( + `Projects imported from the local working copy for Git connection ${args.id}. This work-in-progress command imported whatever the last clone produced; it did not pull from the remote.`, + flags, + result, + ); + }); + } +} diff --git a/packages/@n8n/cli/src/index.ts b/packages/@n8n/cli/src/index.ts index afd030f4db3..ddd167e44f2 100644 --- a/packages/@n8n/cli/src/index.ts +++ b/packages/@n8n/cli/src/index.ts @@ -30,6 +30,7 @@ import GitConnectionsDisconnect from './commands/git-connections/disconnect'; import GitConnectionsGet from './commands/git-connections/get'; import GitConnectionsList from './commands/git-connections/list'; import GitConnectionsListProjects from './commands/git-connections/list-projects'; +import GitConnectionsPull from './commands/git-connections/pull'; import GitConnectionsPush from './commands/git-connections/push'; import GitConnectionsRemoveProject from './commands/git-connections/remove-project'; import GitConnectionsUpdate from './commands/git-connections/update'; @@ -115,6 +116,7 @@ export const commands = { 'tag:delete': TagDelete, 'git-connections:push': GitConnectionsPush, + 'git-connections:pull': GitConnectionsPull, 'project:list': ProjectList, 'project:get': ProjectGet, diff --git a/packages/@n8n/permissions/src/__tests__/__snapshots__/scope-information.test.ts.snap b/packages/@n8n/permissions/src/__tests__/__snapshots__/scope-information.test.ts.snap index 4b4584c1971..93947180273 100644 --- a/packages/@n8n/permissions/src/__tests__/__snapshots__/scope-information.test.ts.snap +++ b/packages/@n8n/permissions/src/__tests__/__snapshots__/scope-information.test.ts.snap @@ -77,6 +77,7 @@ exports[`Scope Information > ensure scopes are defined correctly 1`] = ` "gitConnection:clone", "gitConnection:push", "gitConnection:manageProjects", + "gitConnection:pull", "tag:create", "tag:read", "tag:update", diff --git a/packages/@n8n/permissions/src/constants.ee.ts b/packages/@n8n/permissions/src/constants.ee.ts index f170eff65f8..a279b4f03fa 100644 --- a/packages/@n8n/permissions/src/constants.ee.ts +++ b/packages/@n8n/permissions/src/constants.ee.ts @@ -34,7 +34,7 @@ export const RESOURCES = { securityAudit: ['generate'] as const, securitySettings: ['manage'] as const, sourceControl: ['pull', 'push', 'manage'] as const, - gitConnection: [...DEFAULT_OPERATIONS, 'clone', 'push', 'manageProjects'] as const, + gitConnection: [...DEFAULT_OPERATIONS, 'clone', 'push', 'manageProjects', 'pull'] as const, tag: [...DEFAULT_OPERATIONS] as const, user: [ 'resetPassword', @@ -108,7 +108,7 @@ export const API_KEY_RESOURCES = { credential: ['create', 'read', 'update', 'move', 'delete', 'list'] as const, eventBusDestination: ['test', 'create', 'read', 'update', 'delete', 'list'] as const, sourceControl: ['pull'] as const, - gitConnection: [...DEFAULT_OPERATIONS, 'clone', 'push', 'manageProjects'] as const, + gitConnection: [...DEFAULT_OPERATIONS, 'clone', 'push', 'manageProjects', 'pull'] as const, workflowTags: ['update', 'list'] as const, executionTags: ['update', 'list'] as const, communityPackage: ['install', 'uninstall', 'update', 'list'] as const, diff --git a/packages/@n8n/permissions/src/public-api-permissions.ee.ts b/packages/@n8n/permissions/src/public-api-permissions.ee.ts index 0f1452ccc6b..acc1d5d3ae9 100644 --- a/packages/@n8n/permissions/src/public-api-permissions.ee.ts +++ b/packages/@n8n/permissions/src/public-api-permissions.ee.ts @@ -21,6 +21,7 @@ export const OWNER_API_KEY_SCOPES: ApiKeyScope[] = [ 'gitConnection:clone', 'gitConnection:push', 'gitConnection:manageProjects', + 'gitConnection:pull', 'securityAudit:generate', 'securitySettings:manage', 'saml:manage', diff --git a/packages/@n8n/permissions/src/roles/scopes/global-scopes.ee.ts b/packages/@n8n/permissions/src/roles/scopes/global-scopes.ee.ts index 38510147ea4..5fcceb3b522 100644 --- a/packages/@n8n/permissions/src/roles/scopes/global-scopes.ee.ts +++ b/packages/@n8n/permissions/src/roles/scopes/global-scopes.ee.ts @@ -67,6 +67,7 @@ export const GLOBAL_OWNER_SCOPES: Scope[] = [ 'gitConnection:clone', 'gitConnection:push', 'gitConnection:manageProjects', + 'gitConnection:pull', 'tag:create', 'tag:read', 'tag:update', diff --git a/packages/@n8n/permissions/src/scope-information.ts b/packages/@n8n/permissions/src/scope-information.ts index b5302ebeade..f89627ac46b 100644 --- a/packages/@n8n/permissions/src/scope-information.ts +++ b/packages/@n8n/permissions/src/scope-information.ts @@ -165,4 +165,9 @@ export const scopeInformation: Partial> = { displayName: 'Manage Git Connection Projects', description: 'Allows adding projects to and removing projects from a Git connection.', }, + 'gitConnection:pull': { + displayName: 'Pull Git Connection', + description: + 'Allows importing all projects from a Git connection working copy into the instance.', + }, }; diff --git a/packages/cli/src/credentials/__tests__/credentials.service.test.ts b/packages/cli/src/credentials/__tests__/credentials.service.test.ts index 648cf760e64..5c9f6190ad7 100644 --- a/packages/cli/src/credentials/__tests__/credentials.service.test.ts +++ b/packages/cli/src/credentials/__tests__/credentials.service.test.ts @@ -35,6 +35,7 @@ import type { InstanceCredentialUseRegistry } from '@/credentials/instance-crede import * as validation from '@/credentials/validation'; import type { CredentialsHelper } from '@/credentials-helper'; import { CredentialNotFoundError } from '@/errors/credential-not-found.error'; +import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { ForbiddenError } from '@/errors/response-errors/forbidden.error'; import type { EventService } from '@/events/event.service'; import type { ExternalHooks } from '@/external-hooks'; @@ -3248,6 +3249,39 @@ describe('CredentialsService', () => { }); }); + it('mints a fresh id when none is supplied', async () => { + const createEncryptedDataSpy = vi.spyOn(service, 'createEncryptedData'); + mockTransactionManager({ credentialId: 'stub-cred-id' }); + + await service.createStubCredential(stubOpts, ownerUser); + + expect(createEncryptedDataSpy).toHaveBeenCalledWith(expect.objectContaining({ id: null })); + }); + + it('reuses a supplied id so id-based matching resolves the stub on a later import', async () => { + const createEncryptedDataSpy = vi.spyOn(service, 'createEncryptedData'); + credentialsRepository.existsBy.mockResolvedValue(false); + mockTransactionManager({ credentialId: 'cred-source' }); + + await service.createStubCredential({ ...stubOpts, id: 'cred-source' }, ownerUser); + + expect(createEncryptedDataSpy).toHaveBeenCalledWith( + expect.objectContaining({ id: 'cred-source' }), + ); + }); + + it('rejects a supplied id that already belongs to another credential (no upsert)', async () => { + const createEncryptedDataSpy = vi.spyOn(service, 'createEncryptedData'); + credentialsRepository.existsBy.mockResolvedValue(true); + + await expect( + service.createStubCredential({ ...stubOpts, id: 'cred-existing' }, ownerUser), + ).rejects.toThrow(BadRequestError); + + expect(credentialsRepository.existsBy).toHaveBeenCalledWith({ id: 'cred-existing' }); + expect(createEncryptedDataSpy).not.toHaveBeenCalled(); + }); + it('rejects when user lacks credential:create on the target project', async () => { projectService.getProjectWithScope.mockResolvedValue(null); // @ts-expect-error - Mocking manager for testing diff --git a/packages/cli/src/credentials/credentials.service.ts b/packages/cli/src/credentials/credentials.service.ts index c24af10ab44..1fbc4b11009 100644 --- a/packages/cli/src/credentials/credentials.service.ts +++ b/packages/cli/src/credentials/credentials.service.ts @@ -1741,14 +1741,21 @@ export class CredentialsService { /** * Creates an empty credential placeholder for package import. Skips field * validation so every known type can be stubbed; {@link save} still enforces - * `credential:create` on the target project. + * `credential:create` on the target project. A supplied `id` preserves source identity. */ async createStubCredential( - opts: { name: string; type: string; projectId: string }, + opts: { id?: string; name: string; type: string; projectId: string }, user: User, ): Promise { + // `save` upserts by id, so reject a taken id rather than overwriting that credential. + if (opts.id !== undefined && (await this.credentialsRepository.existsBy({ id: opts.id }))) { + throw new BadRequestError( + `Cannot create credential stub: a credential with id "${opts.id}" already exists`, + ); + } + const encryptedCredential = await this.createEncryptedData({ - id: null, + id: opts.id ?? null, name: opts.name, type: opts.type, data: {}, diff --git a/packages/cli/src/modules/git-connections.ee/__tests__/git-connection-project.repository.integration.test.ts b/packages/cli/src/modules/git-connections.ee/__tests__/git-connection-project.repository.integration.test.ts new file mode 100644 index 00000000000..c8242b87ea5 --- /dev/null +++ b/packages/cli/src/modules/git-connections.ee/__tests__/git-connection-project.repository.integration.test.ts @@ -0,0 +1,114 @@ +import { createTeamProject, testDb, testModules } from '@n8n/backend-test-utils'; +import { Container } from '@n8n/di'; + +import { GitConnectionProjectRepository } from '../database/repositories/git-connection-project.repository'; +import { GitConnectionRepository } from '../database/repositories/git-connection.repository'; + +let connectionRepository: GitConnectionRepository; +let projectLinkRepository: GitConnectionProjectRepository; + +async function createConnection(name = 'conn') { + return await connectionRepository.save( + connectionRepository.create({ + name, + repositoryUrl: 'https://github.com/o/r.git', + branchName: 'main', + connectionType: 'https', + publicKey: null, + encryptedPrivateKey: null, + encryptedUsername: 'enc:user', + encryptedPassword: 'enc:pass', + keyGeneratorType: null, + baseCommit: null, + }), + ); +} + +async function link(gitConnectionId: string, projectId: string) { + await projectLinkRepository.insert({ gitConnectionId, projectId }); +} + +beforeAll(async () => { + await testModules.loadModules(['git-connections']); + await testDb.init(); + + connectionRepository = Container.get(GitConnectionRepository); + projectLinkRepository = Container.get(GitConnectionProjectRepository); +}); + +beforeEach(async () => { + // Delete links before connections to satisfy the foreign key. + await projectLinkRepository.delete({}); + await connectionRepository.delete({}); +}); + +afterAll(async () => { + await testDb.terminate(); +}); + +describe('GitConnectionProjectRepository.syncConnectionProjects (integration)', () => { + it('links projects that were not previously linked', async () => { + const connection = await createConnection(); + const [a, b] = [await createTeamProject(), await createTeamProject()]; + + await projectLinkRepository.syncConnectionProjects(connection.id, [a.id, b.id]); + + expect(await projectLinkRepository.findProjectIdsByConnection(connection.id)).toEqual( + [a.id, b.id].sort(), + ); + }); + + it('prunes links for projects no longer in the working copy', async () => { + const connection = await createConnection(); + const [a, b, c] = [ + await createTeamProject(), + await createTeamProject(), + await createTeamProject(), + ]; + await link(connection.id, a.id); + await link(connection.id, b.id); + await link(connection.id, c.id); + + await projectLinkRepository.syncConnectionProjects(connection.id, [a.id, b.id]); + + expect(await projectLinkRepository.findProjectIdsByConnection(connection.id)).toEqual( + [a.id, b.id].sort(), + ); + }); + + it('moves a project already linked to another connection', async () => { + const source = await createConnection('source'); + const target = await createConnection('target'); + const project = await createTeamProject(); + await link(source.id, project.id); + + await projectLinkRepository.syncConnectionProjects(target.id, [project.id]); + + expect(await projectLinkRepository.findProjectIdsByConnection(source.id)).toEqual([]); + expect(await projectLinkRepository.findProjectIdsByConnection(target.id)).toEqual([project.id]); + }); + + it('is idempotent when the imported set is unchanged', async () => { + const connection = await createConnection(); + const [a, b] = [await createTeamProject(), await createTeamProject()]; + + await projectLinkRepository.syncConnectionProjects(connection.id, [a.id, b.id]); + await projectLinkRepository.syncConnectionProjects(connection.id, [a.id, b.id]); + + expect(await projectLinkRepository.findProjectIdsByConnection(connection.id)).toEqual( + [a.id, b.id].sort(), + ); + }); + + it('leaves existing links untouched for an empty import', async () => { + const connection = await createConnection(); + const project = await createTeamProject(); + await link(connection.id, project.id); + + await projectLinkRepository.syncConnectionProjects(connection.id, []); + + expect(await projectLinkRepository.findProjectIdsByConnection(connection.id)).toEqual([ + project.id, + ]); + }); +}); diff --git a/packages/cli/src/modules/git-connections.ee/__tests__/git-connections.service.test.ts b/packages/cli/src/modules/git-connections.ee/__tests__/git-connections.service.test.ts index 8ebb56156e7..338143af86e 100644 --- a/packages/cli/src/modules/git-connections.ee/__tests__/git-connections.service.test.ts +++ b/packages/cli/src/modules/git-connections.ee/__tests__/git-connections.service.test.ts @@ -530,4 +530,133 @@ describe('GitConnectionsService (credential state machine)', () => { }); }); }); + + describe('pull import', () => { + let n8nFolder: string; + let importService: GitConnectionsService; + let exportFolder: string; + const actor = mock({ id: 'actor' }); + + const importResult = () => + ({ + package: { sourceN8nVersion: '1.0.0', sourceId: 'src', exportedAt: 'now' }, + projects: [ + { status: 'created', localId: 'p1' }, + { status: 'updated', localId: 'p2' }, + ], + folders: [{ status: 'created' }, { status: 'skipped' }, { status: 'created' }], + workflows: [ + { status: 'created', publishing: { state: 'published' } }, + { status: 'created', publishing: { state: 'blocked', blockedReason: 'stub-credential' } }, + { status: 'updated', publishing: { state: 'unchanged' } }, + ], + removedWorkflows: [ + { deletion: 'archived' }, + { deletion: 'deleted' }, + { deletion: 'deleted' }, + ], + removedFolders: [{}, {}], + bindings: { workflows: {}, credentials: {} }, + credentials: { matched: ['c1'], stubbed: ['c2', 'c3'] }, + dataTables: { matched: 1, created: 2 }, + variables: { matched: ['v1'], created: ['v2'], updated: ['v3'], stubbed: [], missing: [] }, + tags: { matched: [], created: ['t1'], renamed: ['t2'], reconciled: [], skipped: [] }, + }) as unknown as Awaited>; + + beforeEach(async () => { + n8nFolder = await mkdtemp(path.join(tmpdir(), 'n8n-git-connection-import-')); + exportFolder = path.join(n8nFolder, 'git-connections', '1', 'repository', 'n8n-export'); + importService = new GitConnectionsService( + repository, + gitConnectionProjectRepository, + projectRepository, + gitService, + n8nPackagesService, + cipher, + mock({ n8nFolder }), + logger, + ); + repository.findOneBy.mockResolvedValue(sshEntity()); + n8nPackagesService.importPackageFromDirectory.mockResolvedValue(importResult()); + }); + + afterEach(async () => { + await rm(n8nFolder, { recursive: true, force: true }); + }); + + it('imports from the n8n-export subfolder with the overwrite policy and maps counts by status', async () => { + await mkdir(exportFolder, { recursive: true }); + + const result = await importService.pull('1', actor); + + expect(n8nPackagesService.importPackageFromDirectory).toHaveBeenCalledWith( + { + user: actor, + projectConflictPolicy: 'overwrite', + workflowConflictPolicy: 'new-version', + workflowIdPolicy: 'source', + workflowPublishingPolicy: 'match-source', + missingNodeTypeMode: 'fail', + credentialMatchingMode: 'id-only', + credentialMissingMode: 'create-stub', + folderConflictPolicy: 'overwrite', + overwriteDeletionPolicy: 'hard-delete', + dataTableMatchingMode: 'by-id', + dataTableMissingMode: 'create', + dataTableSchemaConflictPolicy: 'keep-existing', + variableMissingMode: 'create-with-value', + variableConflictPolicy: 'overwrite', + tagMissingMode: 'create', + tagConflictPolicy: 'rename', + }, + { sourceDir: exportFolder }, + ); + expect(result).toEqual({ + connectionId: '1', + counts: { + projects: { created: 1, updated: 1, skipped: 0 }, + folders: { created: 2, skipped: 1, removed: 2 }, + workflows: { + created: 2, + updated: 1, + skipped: 0, + archived: 1, + deleted: 2, + publishing: { published: 1, unpublished: 0, unchanged: 1, blocked: 1, failed: 0 }, + }, + credentials: { matched: 1, stubbed: 2 }, + dataTables: { matched: 1, created: 2 }, + variables: { matched: 1, created: 1, updated: 1, stubbed: 0, missing: 0 }, + tags: { matched: 0, created: 1, renamed: 1, reconciled: 0, skipped: 0 }, + }, + }); + }); + + it('reconciles the connection links to every imported project', async () => { + await mkdir(exportFolder, { recursive: true }); + + await importService.pull('1', actor); + + expect(gitConnectionProjectRepository.syncConnectionProjects).toHaveBeenCalledWith('1', [ + 'p1', + 'p2', + ]); + }); + + it('fails with a clear error when there is no exported working copy', async () => { + await expect(importService.pull('1', actor)).rejects.toThrow( + 'no exported working copy to import', + ); + expect(n8nPackagesService.importPackageFromDirectory).not.toHaveBeenCalled(); + }); + + it('does not touch the filesystem for a missing connection', async () => { + repository.findOneBy.mockResolvedValueOnce(null); + + await expect(importService.pull('missing', actor)).rejects.toThrow( + 'Git connection not found', + ); + expect(n8nPackagesService.importPackageFromDirectory).not.toHaveBeenCalled(); + }); + }); }); diff --git a/packages/cli/src/modules/git-connections.ee/database/repositories/git-connection-project.repository.ts b/packages/cli/src/modules/git-connections.ee/database/repositories/git-connection-project.repository.ts index 42e21413e65..427a662b76e 100644 --- a/packages/cli/src/modules/git-connections.ee/database/repositories/git-connection-project.repository.ts +++ b/packages/cli/src/modules/git-connections.ee/database/repositories/git-connection-project.repository.ts @@ -1,12 +1,13 @@ +import { BaseRepository, type OperationContext, TransactionRunner } from '@n8n/db'; import { Service } from '@n8n/di'; -import { DataSource, Repository } from '@n8n/typeorm'; +import { DataSource, In, Not } from '@n8n/typeorm'; import { GitConnectionProject } from '../entities/git-connection-project.entity'; @Service() -export class GitConnectionProjectRepository extends Repository { - constructor(dataSource: DataSource) { - super(GitConnectionProject, dataSource.manager); +export class GitConnectionProjectRepository extends BaseRepository { + constructor(dataSource: DataSource, transactionRunner: TransactionRunner) { + super(GitConnectionProject, dataSource.manager, transactionRunner); } async findByProjectId(projectId: string): Promise { @@ -45,4 +46,23 @@ export class GitConnectionProjectRepository extends Repository row.projectId); } + + async syncConnectionProjects( + gitConnectionId: string, + projectIds: string[], + ctx: OperationContext = {}, + ) { + if (projectIds.length === 0) return; + await this.runInTransaction(ctx, async (trx) => { + await trx.delete(GitConnectionProject, { + gitConnectionId, + projectId: Not(In(projectIds)), + }); + await trx.upsert( + GitConnectionProject, + projectIds.map((projectId) => ({ projectId, gitConnectionId })), + ['projectId'], + ); + }); + } } diff --git a/packages/cli/src/modules/git-connections.ee/git-connections.service.ts b/packages/cli/src/modules/git-connections.ee/git-connections.service.ts index 3f35dcd978f..0d90f01c8cb 100644 --- a/packages/cli/src/modules/git-connections.ee/git-connections.service.ts +++ b/packages/cli/src/modules/git-connections.ee/git-connections.service.ts @@ -3,6 +3,7 @@ import { GitConnectionProjectListPublicDto, GitConnectionProjectPublicDto, type GitConnectionPublicDto, + type GitConnectionPullResultDto, type GitConnectionPushResultDto, type UpdateGitConnectionDto, } from '@n8n/api-types'; @@ -11,7 +12,7 @@ import type { User } from '@n8n/db'; import { ProjectRepository } from '@n8n/db'; import { Service } from '@n8n/di'; import { Cipher, InstanceSettings } from 'n8n-core'; -import { mkdir, mkdtemp, rename, rm } from 'node:fs/promises'; +import { mkdir, mkdtemp, rename, rm, stat } from 'node:fs/promises'; import path from 'node:path'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; @@ -20,8 +21,23 @@ import { ForbiddenError } from '@/errors/response-errors/forbidden.error'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; import { N8nPackagesService } from '@/modules/n8n-packages/n8n-packages.service'; import { + DataTableMissingMode, + DataTableSchemaConflictPolicy, + FolderConflictPolicy, + MissingNodeTypeMode, MissingWorkflowDependencyPolicy, + OverwriteDeletionPolicy, + ProjectConflictPolicy, + TagConflictPolicy, + TagMissingMode, + VariableConflictPolicy, + VariableMissingMode, + WorkflowConflictPolicy, + WorkflowIdPolicy, + WorkflowPublishingPolicy, WorkflowVersionPolicy, + type ImportRequest, + type ImportResult, } from '@/modules/n8n-packages/n8n-packages.types'; import { userHasScopes } from '@/permissions.ee/check-access'; @@ -44,6 +60,26 @@ type ManageProjectLinkOptions = { projectId: string; }; +// Pull treats the working copy as source of truth; callers cannot override this policy. +const IMPORT_POLICY: Omit = { + projectConflictPolicy: ProjectConflictPolicy.Overwrite, + workflowConflictPolicy: WorkflowConflictPolicy.NewVersion, + workflowIdPolicy: WorkflowIdPolicy.Source, + workflowPublishingPolicy: WorkflowPublishingPolicy.MatchSource, + missingNodeTypeMode: MissingNodeTypeMode.Fail, + credentialMatchingMode: 'id-only', + credentialMissingMode: 'create-stub', + folderConflictPolicy: FolderConflictPolicy.Overwrite, + overwriteDeletionPolicy: OverwriteDeletionPolicy.HardDelete, + dataTableMatchingMode: 'by-id', + dataTableMissingMode: DataTableMissingMode.Create, + dataTableSchemaConflictPolicy: DataTableSchemaConflictPolicy.KeepExisting, + variableMissingMode: VariableMissingMode.CreateWithValue, + variableConflictPolicy: VariableConflictPolicy.Overwrite, + tagMissingMode: TagMissingMode.Create, + tagConflictPolicy: TagConflictPolicy.Rename, +}; + @Service() export class GitConnectionsService { constructor( @@ -153,13 +189,6 @@ export class GitConnectionsService { } async push(connectionId: string, actor: User): Promise { - return await this.exportProjectsToRepository(connectionId, actor); - } - - private async exportProjectsToRepository( - connectionId: string, - actor: User, - ): Promise { // Validates the connection exists (throws NotFound otherwise) before any export work. await this.getEntity(connectionId); // The instance connection covers every team project; personal projects are @@ -279,6 +308,88 @@ export class GitConnectionsService { }); } + async pull(connectionId: string, actor: User): Promise { + await this.getEntity(connectionId); + const importFolder = path.join(this.rootFolder(connectionId), 'repository', EXPORT_SUBFOLDER); + + if (!(await this.exportedWorkingCopyExists(importFolder))) { + throw new BadRequestError( + 'This Git connection has no exported working copy to import. Connect it and push projects first.', + ); + } + + this.logger.info('Importing projects from Git connection repository', { connectionId }); + + const result = await this.n8nPackagesService.importPackageFromDirectory( + { user: actor, ...IMPORT_POLICY }, + { sourceDir: importFolder }, + ); + + // Keep links aligned so later pushes preserve the pulled project set. + await this.gitConnectionProjectRepository.syncConnectionProjects( + connectionId, + result.projects.map((project) => project.localId), + ); + + return { connectionId, counts: this.toImportCounts(result) }; + } + + private async exportedWorkingCopyExists(folder: string): Promise { + try { + return (await stat(folder)).isDirectory(); + } catch { + return false; + } + } + + private toImportCounts(result: ImportResult): GitConnectionPullResultDto['counts'] { + const tally = (rows: Array<{ status: S }>, statuses: readonly S[]) => { + const counts = Object.fromEntries(statuses.map((status) => [status, 0])) as Record; + for (const { status } of rows) counts[status] += 1; + return counts; + }; + + return { + projects: tally(result.projects, ['created', 'updated', 'skipped'] as const), + folders: { + ...tally(result.folders, ['created', 'skipped'] as const), + removed: result.removedFolders.length, + }, + workflows: { + ...tally(result.workflows, ['created', 'updated', 'skipped'] as const), + archived: result.removedWorkflows.filter(({ deletion }) => deletion === 'archived').length, + deleted: result.removedWorkflows.filter(({ deletion }) => deletion === 'deleted').length, + // Publishing happens after writes, so failures are reported without failing the pull. + publishing: tally( + result.workflows.map(({ publishing }) => ({ status: publishing.state })), + ['published', 'unpublished', 'unchanged', 'blocked', 'failed'] as const, + ), + }, + credentials: { + matched: result.credentials.matched.length, + stubbed: result.credentials.stubbed.length, + }, + dataTables: { + matched: result.dataTables.matched, + created: result.dataTables.created, + }, + variables: { + matched: result.variables.matched.length, + created: result.variables.created.length, + updated: result.variables.updated.length, + stubbed: result.variables.stubbed.length, + missing: result.variables.missing.length, + }, + tags: { + matched: result.tags.matched.length, + created: result.tags.created.length, + renamed: result.tags.renamed.length, + reconciled: result.tags.reconciled.length, + skipped: result.tags.skipped.length, + }, + }; + } + private async applyNewAuthentication( connection: GitConnection, input: Pick< diff --git a/packages/cli/src/modules/n8n-packages/__tests__/import-package.integration.test.ts b/packages/cli/src/modules/n8n-packages/__tests__/import-package.integration.test.ts index 7257304159f..907e9aff201 100644 --- a/packages/cli/src/modules/n8n-packages/__tests__/import-package.integration.test.ts +++ b/packages/cli/src/modules/n8n-packages/__tests__/import-package.integration.test.ts @@ -1242,8 +1242,7 @@ describe('Package import event emission', () => { ); expect(importedPayload.credentialIds.matched).toHaveLength(2); expect(importedPayload.credentialIds.created).toHaveLength(1); - expect(importedPayload.credentialIds.created[0]).toEqual(expect.any(String)); - expect(importedPayload.credentialIds.created[0]).not.toBe('missing-cred'); + expect(importedPayload.credentialIds.created[0]).toBe('missing-cred'); expect(importedPayload.credentialIds.updated).toEqual([]); expect(importedPayload.counts).toEqual({ workflows: { @@ -2088,8 +2087,7 @@ describe('credential-missing-mode: create-stub', () => { }); expect(result.credentials).toEqual({ matched: [], stubbed: ['missing-cred'] }); - expect(result.bindings.credentials['missing-cred']).toEqual(expect.any(String)); - expect(result.bindings.credentials['missing-cred']).not.toBe('missing-cred'); + expect(result.bindings.credentials['missing-cred']).toBe('missing-cred'); const workflow = await Container.get(WorkflowRepository).findOneOrFail({ where: { name: 'Stubbed cred workflow' }, diff --git a/packages/cli/src/modules/n8n-packages/__tests__/import-project-from-directory.integration.test.ts b/packages/cli/src/modules/n8n-packages/__tests__/import-project-from-directory.integration.test.ts new file mode 100644 index 00000000000..dbe06a02c71 --- /dev/null +++ b/packages/cli/src/modules/n8n-packages/__tests__/import-project-from-directory.integration.test.ts @@ -0,0 +1,198 @@ +import { LicenseState } from '@n8n/backend-common'; +import { + createTeamProject, + createWorkflow, + mockInstance, + testDb, + testModules, +} from '@n8n/backend-test-utils'; +import type { User } from '@n8n/db'; +import { ProjectRepository, WorkflowRepository } from '@n8n/db'; +import { Container } from '@n8n/di'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { ActiveWorkflowManager } from '@/active-workflow-manager'; +import { EventService } from '@/events/event.service'; +import { createOwner } from '@test-integration/db/users'; +import { LicenseMocker } from '@test-integration/license'; + +import { PackageImportConfig } from '../n8n-packages.config'; +import { N8nPackagesService } from '../n8n-packages.service'; +import type { ImportRequest } from '../n8n-packages.types'; + +const licenseMocker = new LicenseMocker(); + +mockInstance(ActiveWorkflowManager); + +let service: N8nPackagesService; +let owner: User; +let sourceDir: string; + +const importPolicy: Omit = { + projectConflictPolicy: 'overwrite', + workflowConflictPolicy: 'new-version', + workflowIdPolicy: 'source', + workflowPublishingPolicy: 'match-source', + missingNodeTypeMode: 'fail', + credentialMatchingMode: 'id-only', + credentialMissingMode: 'create-stub', + folderConflictPolicy: 'overwrite', + overwriteDeletionPolicy: 'hard-delete', + dataTableMatchingMode: 'by-id', + dataTableMissingMode: 'create', + dataTableSchemaConflictPolicy: 'keep-existing', + variableMissingMode: 'create-with-value', + variableConflictPolicy: 'overwrite', + tagMissingMode: 'create', + tagConflictPolicy: 'rename', +}; + +beforeAll(async () => { + await testModules.loadModules(['n8n-packages']); + await testDb.init(); + service = Container.get(N8nPackagesService); + licenseMocker.mockLicenseState(Container.get(LicenseState)); + licenseMocker.setDefaults({ + features: ['feat:projectRole:admin', 'feat:folders'], + quotas: { 'quota:maxTeamProjects': 100 }, + }); +}); + +afterAll(async () => { + await testDb.terminate(); +}); + +beforeEach(async () => { + await testDb.truncate([ + 'Folder', + 'WorkflowEntity', + 'SharedWorkflow', + 'ProjectRelation', + 'Project', + ]); + licenseMocker.reset(); + owner = await createOwner(); + sourceDir = await mkdtemp(path.join(tmpdir(), 'n8n-import-dir-')); +}); + +afterEach(async () => { + await rm(sourceDir, { recursive: true, force: true }); +}); + +describe('importPackageFromDirectory', () => { + it('imports the projects an exported directory contains', async () => { + const project = await createTeamProject('Alpha Project', owner); + await createWorkflow({ name: 'WF One', nodes: [], connections: {} }, project); + await service.exportPackageToDirectory( + { user: owner, projectIds: [project.id] }, + { targetDir: sourceDir }, + ); + + // Force the create path. + await testDb.truncate(['WorkflowEntity', 'SharedWorkflow', 'ProjectRelation', 'Project']); + + const result = await service.importPackageFromDirectory( + { user: owner, ...importPolicy }, + { sourceDir }, + ); + + expect(result.projects).toHaveLength(1); + expect(result.projects[0]).toMatchObject({ name: 'Alpha Project', status: 'created' }); + expect(result.workflows.map((w) => w.name)).toEqual(['WF One']); + }); + + it('overwrites an existing project and workflow to match the directory', async () => { + const project = await createTeamProject('Alpha Project', owner); + const workflow = await createWorkflow({ name: 'WF One', nodes: [], connections: {} }, project); + await service.exportPackageToDirectory( + { user: owner, projectIds: [project.id] }, + { targetDir: sourceDir }, + ); + + // Drift existing rows to exercise overwrite. + await Container.get(ProjectRepository).update(project.id, { name: 'Alpha Project (edited)' }); + await Container.get(WorkflowRepository).update(workflow.id, { name: 'WF One (edited)' }); + + const result = await service.importPackageFromDirectory( + { user: owner, ...importPolicy }, + { sourceDir }, + ); + + expect(result.projects).toHaveLength(1); + expect(result.projects[0]).toMatchObject({ + localId: project.id, + name: 'Alpha Project', + status: 'updated', + }); + expect(result.workflows).toHaveLength(1); + expect(result.workflows[0]).toMatchObject({ + localId: workflow.id, + name: 'WF One', + status: 'updated', + }); + + expect(await Container.get(ProjectRepository).count({ where: { type: 'team' } })).toBe(1); + expect((await Container.get(ProjectRepository).findOneBy({ id: project.id }))?.name).toBe( + 'Alpha Project', + ); + expect(await Container.get(WorkflowRepository).count()).toBe(1); + expect((await Container.get(WorkflowRepository).findOneBy({ id: workflow.id }))?.name).toBe( + 'WF One', + ); + }); + + it('does not emit the user package-import event', async () => { + const project = await createTeamProject('Alpha Project', owner); + await createWorkflow({ name: 'WF One', nodes: [], connections: {} }, project); + await service.exportPackageToDirectory( + { user: owner, projectIds: [project.id] }, + { targetDir: sourceDir }, + ); + await testDb.truncate(['WorkflowEntity', 'SharedWorkflow', 'ProjectRelation', 'Project']); + + const emitSpy = vi.spyOn(Container.get(EventService), 'emit'); + + await service.importPackageFromDirectory({ user: owner, ...importPolicy }, { sourceDir }); + + expect(emitSpy).not.toHaveBeenCalledWith('n8n-package-imported', expect.anything()); + emitSpy.mockRestore(); + }); + + it('rejects a working copy that exceeds the package-wide entry limit', async () => { + const project = await createTeamProject('Alpha Project', owner); + await createWorkflow({ name: 'WF One', nodes: [], connections: {} }, project); + await service.exportPackageToDirectory( + { user: owner, projectIds: [project.id] }, + { targetDir: sourceDir }, + ); + await testDb.truncate(['WorkflowEntity', 'SharedWorkflow', 'ProjectRelation', 'Project']); + + const config = Container.get(PackageImportConfig); + const originalMaxEntries = config.maxEntries; + config.maxEntries = 1; + try { + await expect( + service.importPackageFromDirectory({ user: owner, ...importPolicy }, { sourceDir }), + ).rejects.toThrow('too many entries'); + } finally { + config.maxEntries = originalMaxEntries; + } + }); + + it('is a no-op for a working copy with no projects', async () => { + await service.exportPackageToDirectory( + { user: owner, projectIds: [] }, + { targetDir: sourceDir }, + ); + + const result = await service.importPackageFromDirectory( + { user: owner, ...importPolicy }, + { sourceDir }, + ); + + expect(result.projects).toHaveLength(0); + expect(result.workflows).toHaveLength(0); + }); +}); diff --git a/packages/cli/src/modules/n8n-packages/engine/import-result.ts b/packages/cli/src/modules/n8n-packages/engine/import-result.ts index a08156a48e6..67a9ad97530 100644 --- a/packages/cli/src/modules/n8n-packages/engine/import-result.ts +++ b/packages/cli/src/modules/n8n-packages/engine/import-result.ts @@ -11,6 +11,7 @@ import type { RemovedWorkflowSummary, ImportBindingMap, ImportCredentialSummary, + ImportDataTableSummary, ImportedFolderSummary, ImportedProjectSummary, ImportedWorkflowSummary, @@ -68,6 +69,7 @@ export function buildImportResult(input: { projects: ImportedProjectSummary[]; bindings: PackageImportBindings; credentials: ImportCredentialSummary; + dataTables: ImportDataTableSummary; variables: ImportVariableSummary; tags: ImportTagSummary; }): ImportResult { @@ -80,6 +82,7 @@ export function buildImportResult(input: { projects: input.projects, bindings: serializeBindings(input.bindings), credentials: input.credentials, + dataTables: input.dataTables, variables: input.variables, tags: input.tags, }; diff --git a/packages/cli/src/modules/n8n-packages/engine/import-telemetry.ts b/packages/cli/src/modules/n8n-packages/engine/import-telemetry.ts index d52d08b7a4b..1bfe7805cfe 100644 --- a/packages/cli/src/modules/n8n-packages/engine/import-telemetry.ts +++ b/packages/cli/src/modules/n8n-packages/engine/import-telemetry.ts @@ -6,7 +6,11 @@ import type { TagImportPlan, TagImportRequest } from '../entities/tag/tag.types' import type { VariableImportRequest } from '../entities/variable/variable.types'; import type { PersistedWorkflowOutcome } from '../entities/workflow/workflow-import.types'; import { VariableParentPolicy } from '../n8n-packages.types'; -import type { ImportContext, ResolvedImportPackageRequest } from '../n8n-packages.types'; +import type { + ImportContext, + ResolvedImportPackageRequest, + ImportResult, +} from '../n8n-packages.types'; import type { ImportContentResult } from './import-orchestrator'; import { reconcileVariableSummary } from './import-result'; import type { PackageManifest } from '../spec/manifest.schema'; @@ -21,6 +25,11 @@ export interface PackageImportScope { tagRequest: TagImportRequest; } +export interface ImportOutcome { + result: ImportResult; + scopes: PackageImportScope[]; +} + export function emitPackageImportedEvent( eventService: EventService, params: { diff --git a/packages/cli/src/modules/n8n-packages/engine/project-package-importer.ts b/packages/cli/src/modules/n8n-packages/engine/project-package-importer.ts index a4cd688b621..6db7de88c2f 100644 --- a/packages/cli/src/modules/n8n-packages/engine/project-package-importer.ts +++ b/packages/cli/src/modules/n8n-packages/engine/project-package-importer.ts @@ -2,7 +2,6 @@ import { LicenseState } from '@n8n/backend-common'; import { Service } from '@n8n/di'; import { ForbiddenError } from '@/errors/response-errors/forbidden.error'; -import { EventService } from '@/events/event.service'; import type { CredentialBindingRequest } from '../entities/credential/credential.types'; import { removesUnpackagedWorkflows } from '../entities/folder/folder-conflict-policy'; @@ -20,8 +19,7 @@ import type { ImportBindingMap, ImportedFolderSummary, ImportedWorkflowSummary, - ResolvedImportPackageRequest, - ImportResult, + ResolvedImportRequest, ImportTagSummary, PackageImportBindings, } from '../n8n-packages.types'; @@ -45,7 +43,7 @@ import { toTagSummary, unionTagSummaries, } from './import-result'; -import { emitPackageImportedEvent, type PackageImportScope } from './import-telemetry'; +import type { ImportOutcome, PackageImportScope } from './import-telemetry'; import { N8nPackageParser } from './n8n-package-parser'; import type { ManifestEntry, PackageManifest } from '../spec/manifest.schema'; import type { SerializedVariable } from '../spec/serialized/variable.schema'; @@ -57,15 +55,14 @@ export class ProjectPackageImporter { private readonly projectImporter: ProjectImporter, private readonly importOrchestrator: ImportOrchestrator, private readonly workflowPublisher: WorkflowPublisher, - private readonly eventService: EventService, private readonly licenseState: LicenseState, ) {} async import( - request: ResolvedImportPackageRequest, + request: ResolvedImportRequest, reader: PackageReader, manifest: PackageManifest, - ): Promise { + ): Promise { this.assertAdequatePermissions(request, manifest); const projects = await this.packageParser.getProjects(reader); @@ -163,6 +160,8 @@ export class ProjectPackageImporter { const scopedBindings: PackageImportBindings[] = []; const matched: string[] = []; const stubbed: string[] = []; + let dataTablesMatched = 0; + let dataTablesCreated = 0; const variablesMatched: string[] = []; const variablesMissing: string[] = []; const variablesCreated: string[] = []; @@ -182,6 +181,8 @@ export class ProjectPackageImporter { scopedBindings.push(content.bindings); matched.push(...content.credentialResult.matched); stubbed.push(...content.credentialResult.stubbed); + dataTablesMatched += content.dataTablePlan.matchedCount; + dataTablesCreated += content.dataTablePlan.creations.length; variablesMatched.push(...content.variablePlan.matched); variablesMissing.push(...content.variablePlan.missing.map(({ name }) => name)); variablesCreated.push(...content.variableResult.created); @@ -199,9 +200,7 @@ export class ProjectPackageImporter { }); } - emitPackageImportedEvent(this.eventService, { request, manifest, scopes }); - - return buildImportResult({ + const result = buildImportResult({ package: toPackageSummary(manifest), workflows, removedWorkflows, @@ -210,6 +209,7 @@ export class ProjectPackageImporter { projects: projectSummaries, bindings: mergeBindings(...scopedBindings), credentials: { matched, stubbed }, + dataTables: { matched: dataTablesMatched, created: dataTablesCreated }, variables: reconcileVariableSummary({ matched: variablesMatched, missing: variablesMissing, @@ -220,10 +220,12 @@ export class ProjectPackageImporter { }), tags: unionTagSummaries(tagSummaries), }); + + return { result, scopes }; } private async buildImportContextForProject( - request: ResolvedImportPackageRequest, + request: ResolvedImportRequest, reader: PackageReader, manifest: PackageManifest, project: ManifestEntry, @@ -295,7 +297,7 @@ export class ProjectPackageImporter { } private assertAdequatePermissions( - request: ResolvedImportPackageRequest, + request: ResolvedImportRequest, manifest: PackageManifest, ): void { // A project package can create new projects or update matched ones (by source id), so require both — diff --git a/packages/cli/src/modules/n8n-packages/engine/workflow-package-importer.ts b/packages/cli/src/modules/n8n-packages/engine/workflow-package-importer.ts index 4dfa26d5142..3eeecf957e0 100644 --- a/packages/cli/src/modules/n8n-packages/engine/workflow-package-importer.ts +++ b/packages/cli/src/modules/n8n-packages/engine/workflow-package-importer.ts @@ -6,7 +6,6 @@ import { UserError } from 'n8n-workflow'; import { ForbiddenError } from '@/errors/response-errors/forbidden.error'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; -import { EventService } from '@/events/event.service'; import { FolderService } from '@/services/folder.service'; import { ProjectService } from '@/services/project.service.ee'; @@ -17,11 +16,7 @@ import type { VariableImportRequest } from '../entities/variable/variable.types' import { WorkflowPublisher } from '../entities/workflow/workflow-publisher'; import type { PackageReader } from '../io/package-reader'; import { VariableParentPolicy } from '../n8n-packages.types'; -import type { - ImportContext, - ImportResult, - ResolvedImportPackageRequest, -} from '../n8n-packages.types'; +import type { ImportContext, ResolvedImportRequest } from '../n8n-packages.types'; import { assertPackageImportApiKeyScopes, assertTagWritesAllowed } from './import-gates'; import { ImportOrchestrator } from './import-orchestrator'; import { @@ -32,7 +27,7 @@ import { toTagSummary, toVariableSummary, } from './import-result'; -import { emitPackageImportedEvent } from './import-telemetry'; +import type { ImportOutcome, PackageImportScope } from './import-telemetry'; import { N8nPackageParser } from './n8n-package-parser'; import { needsBundledVariableValues, placeByPolicy } from './package-layout'; import type { PackageManifest } from '../spec/manifest.schema'; @@ -49,15 +44,14 @@ export class WorkflowPackageImporter { private readonly workflowPublisher: WorkflowPublisher, private readonly projectService: ProjectService, private readonly folderService: FolderService, - private readonly eventService: EventService, private readonly licenseState: LicenseState, ) {} async import( - request: ResolvedImportPackageRequest, + request: ResolvedImportRequest, reader: PackageReader, manifest: PackageManifest, - ): Promise { + ): Promise { const folders = await this.packageParser.getFolders(reader); if (folders.length > 0) { this.assertFoldersLicensed(); @@ -144,22 +138,18 @@ export class WorkflowPackageImporter { subWorkflowRequirements: plan.input.subWorkflowRequirements, }); - emitPackageImportedEvent(this.eventService, { - request, - manifest, - scopes: [ - { - context, - imported: content, - credentialRequest, - dataTableRequest, - variableRequest, - tagRequest, - }, - ], - }); + const scopes: PackageImportScope[] = [ + { + context, + imported: content, + credentialRequest, + dataTableRequest, + variableRequest, + tagRequest, + }, + ]; - return buildImportResult({ + const result = buildImportResult({ package: toPackageSummary(manifest), workflows: toImportedWorkflowSummaries( content.workflowOutcomes, @@ -176,9 +166,15 @@ export class WorkflowPackageImporter { matched: content.credentialResult.matched, stubbed: content.credentialResult.stubbed, }, + dataTables: { + matched: content.dataTablePlan.matchedCount, + created: content.dataTablePlan.creations.length, + }, variables: toVariableSummary(content.variablePlan, content.variableResult), tags: toTagSummary(content.tagPlan), }); + + return { result, scopes }; } private assertFoldersLicensed(): void { diff --git a/packages/cli/src/modules/n8n-packages/entities/credential/__tests__/credential-importer.test.ts b/packages/cli/src/modules/n8n-packages/entities/credential/__tests__/credential-importer.test.ts index 89fd8e744c4..14f07053a14 100644 --- a/packages/cli/src/modules/n8n-packages/entities/credential/__tests__/credential-importer.test.ts +++ b/packages/cli/src/modules/n8n-packages/entities/credential/__tests__/credential-importer.test.ts @@ -37,10 +37,11 @@ describe('CredentialImporter', () => { options: { credentialBindings?: CredentialBindingRequest['credentialBindings']; missingMode?: CredentialBindingRequest['missingMode']; + matchingMode?: CredentialBindingRequest['matchingMode']; } = {}, ): CredentialBindingRequest => ({ requirements, - matchingMode: 'id-only', + matchingMode: options.matchingMode ?? 'id-only', missingMode: options.missingMode ?? 'must-preexist', credentialBindings: options.credentialBindings, }); @@ -227,6 +228,7 @@ describe('CredentialImporter', () => { expect(credentialsService.createStubCredential).toHaveBeenCalledTimes(1); expect(credentialsService.createStubCredential).toHaveBeenCalledWith( { + id: 'missing-cred', name: 'Missing GitHub', type: 'githubApi', projectId: 'project-target', @@ -281,6 +283,7 @@ describe('CredentialImporter', () => { expect(credentialsService.createStubCredential).toHaveBeenCalledWith( { + id: 'orphan-not-in-requirements', name: 'Package GitHub', type: 'githubApi', projectId: 'project-target', @@ -290,6 +293,64 @@ describe('CredentialImporter', () => { expect(result.stubbed).toEqual(['orphan-not-in-requirements']); }); + it('apply mints a fresh id (no reused source id) under name-and-type matching', async () => { + credentialsService.createStubCredential.mockResolvedValue({ id: 'stub-1' } as never); + + const missingCredential = packageCredential({ id: 'missing-cred', name: 'Missing GitHub' }); + const request = bindingRequest([missingCredential], { + missingMode: 'create-stub', + matchingMode: 'name-and-type', + }); + + await importer.apply(context, request, { + successes: new Map(), + failures: [notFoundFailure(missingCredential)], + }); + + expect(credentialsService.createStubCredential).toHaveBeenCalledWith( + { + id: undefined, + name: 'Missing GitHub', + type: 'githubApi', + projectId: 'project-target', + }, + user, + ); + }); + + it('reuses the source id so a repeated pull resolves the stub instead of restubbing', async () => { + const missingCredential = packageCredential({ id: 'cred-source', name: 'Source GitHub' }); + + credentialsService.createStubCredential.mockResolvedValue({ id: 'cred-source' } as never); + const firstRequest = bindingRequest([missingCredential], { missingMode: 'create-stub' }); + const firstResult = await importer.apply(context, firstRequest, { + successes: new Map(), + failures: [notFoundFailure(missingCredential)], + }); + + expect(credentialsService.createStubCredential).toHaveBeenCalledWith( + expect.objectContaining({ id: 'cred-source' }), + user, + ); + expect(firstResult.bindings).toEqual(new Map([['cred-source', 'cred-source']])); + + vi.clearAllMocks(); + credentialTypes.recognizes.mockReturnValue(true); + credentialsService.getCredentialsAUserCanUseInAWorkflow.mockResolvedValue([ + usable('cred-source'), + ]); + + const resolution = await importer.plan(context, firstRequest); + const secondResult = await importer.apply(context, firstRequest, resolution); + + expect(credentialsService.createStubCredential).not.toHaveBeenCalled(); + expect(secondResult).toEqual({ + bindings: new Map([['cred-source', 'cred-source']]), + matched: ['cred-source'], + stubbed: [], + }); + }); + it('apply rejects when stub creation lacks credential:create', async () => { credentialsService.createStubCredential.mockRejectedValue( new ForbiddenError( diff --git a/packages/cli/src/modules/n8n-packages/entities/credential/credential-importer.ts b/packages/cli/src/modules/n8n-packages/entities/credential/credential-importer.ts index 671c13b747b..1fbdc7b0dd3 100644 --- a/packages/cli/src/modules/n8n-packages/entities/credential/credential-importer.ts +++ b/packages/cli/src/modules/n8n-packages/entities/credential/credential-importer.ts @@ -81,6 +81,8 @@ export class CredentialImporter { const stubCredential = await this.credentialsService.createStubCredential( { + // Preserve source identity only when matching by id. + id: request.matchingMode === 'id-only' ? sourceId : undefined, name: name ?? sourceId, type, projectId: context.projectId, diff --git a/packages/cli/src/modules/n8n-packages/io/__tests__/directory-package-reader.test.ts b/packages/cli/src/modules/n8n-packages/io/__tests__/directory-package-reader.test.ts new file mode 100644 index 00000000000..cbf72067803 --- /dev/null +++ b/packages/cli/src/modules/n8n-packages/io/__tests__/directory-package-reader.test.ts @@ -0,0 +1,158 @@ +import { mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { BadRequestError } from '@/errors/response-errors/bad-request.error'; + +import { DirectoryPackageReader } from '../directory/directory-package-reader'; + +const limits = { + maxUncompressedBytes: 1024 * 1024, + maxEntryBytes: 1024, + maxEntries: 100, + maxPathLength: 1024, +}; + +describe('DirectoryPackageReader', () => { + let baseDir: string; + + beforeEach(async () => { + baseDir = await mkdtemp(path.join(tmpdir(), 'n8n-dir-reader-')); + }); + + afterEach(async () => { + await rm(baseDir, { recursive: true, force: true }); + }); + + const reader = () => new DirectoryPackageReader(baseDir, limits); + + it('reads and parses the manifest', async () => { + await writeFile(path.join(baseDir, 'manifest.json'), '{"packageFormatVersion":"1"}'); + + expect(await reader().readManifest()).toEqual({ packageFormatVersion: '1' }); + }); + + it('throws a clear error when the manifest is missing', async () => { + await expect(reader().readManifest()).rejects.toThrow('Package is missing manifest.json'); + }); + + it('throws when the manifest is not valid JSON', async () => { + await writeFile(path.join(baseDir, 'manifest.json'), 'not-json'); + + await expect(reader().readManifest()).rejects.toThrow('Package manifest is not valid JSON'); + }); + + it('reads a nested file by its posix-relative path', async () => { + await mkdir(path.join(baseDir, 'projects', 'alpha'), { recursive: true }); + await writeFile(path.join(baseDir, 'projects', 'alpha', 'project.json'), '{"id":"alpha"}'); + + const content = await reader().readFile('projects/alpha/project.json'); + + expect(content.toString('utf-8')).toBe('{"id":"alpha"}'); + }); + + it('throws when a requested entry does not exist', async () => { + await expect(reader().readFile('projects/missing/project.json')).rejects.toThrow( + 'Package does not contain entry: projects/missing/project.json', + ); + }); + + it('rejects a path that escapes the package root', async () => { + await expect(reader().readFile('../secrets.json')).rejects.toThrow(BadRequestError); + }); + + it('rejects a path with disallowed characters', async () => { + await expect(reader().readFile('projects/al pha/project.json')).rejects.toThrow( + 'contains disallowed characters', + ); + }); + + it('rejects a single file that exceeds the per-entry size limit', async () => { + await writeFile(path.join(baseDir, 'big.json'), Buffer.alloc(limits.maxEntryBytes + 1, 0x61)); + + await expect(reader().readFile('big.json')).rejects.toThrow( + 'exceeds the maximum allowed uncompressed size per entry', + ); + }); + + it.skipIf(process.platform === 'win32')('rejects a symbolic-link file entry', async () => { + await writeFile(path.join(baseDir, 'target.json'), '{}'); + await symlink('target.json', path.join(baseDir, 'linked.json')); + + await expect(reader().readFile('linked.json')).rejects.toThrow('disallowed entry type'); + }); + + it.skipIf(process.platform === 'win32')( + 'rejects an entry below a symbolic-link directory', + async () => { + await mkdir(path.join(baseDir, 'targets', 'alpha'), { recursive: true }); + await writeFile(path.join(baseDir, 'targets', 'alpha', 'project.json'), '{}'); + await mkdir(path.join(baseDir, 'projects')); + await symlink( + path.join(baseDir, 'targets', 'alpha'), + path.join(baseDir, 'projects', 'alpha'), + ); + + await expect(reader().readFile('projects/alpha/project.json')).rejects.toThrow( + 'disallowed entry type', + ); + }, + ); + + it('lists every regular file as a posix-relative path', async () => { + await mkdir(path.join(baseDir, 'projects', 'alpha'), { recursive: true }); + await writeFile(path.join(baseDir, 'manifest.json'), '{}'); + await writeFile(path.join(baseDir, 'projects', 'alpha', 'project.json'), '{}'); + + expect((await reader().listEntries()).sort()).toEqual( + ['manifest.json', 'projects/alpha/project.json'].sort(), + ); + }); + + it('fails when the tree has more entries than allowed', async () => { + const tight = new DirectoryPackageReader(baseDir, { ...limits, maxEntries: 1 }); + await writeFile(path.join(baseDir, 'a.json'), '{}'); + await writeFile(path.join(baseDir, 'b.json'), '{}'); + + await expect(tight.listEntries()).rejects.toThrow('too many entries'); + }); + + it.skipIf(process.platform === 'win32')( + 'rejects symbolic links during whole-tree validation', + async () => { + await writeFile(path.join(baseDir, 'target.json'), '{}'); + await symlink('target.json', path.join(baseDir, 'linked.json')); + + await expect(reader().listEntries()).rejects.toThrow('disallowed entry type'); + }, + ); + + it.skipIf(process.platform === 'win32')('rejects a symbolic-link package root', async () => { + const actualBaseDir = path.join(baseDir, 'actual'); + const linkedBaseDir = path.join(baseDir, 'linked'); + await mkdir(actualBaseDir); + await symlink(actualBaseDir, linkedBaseDir); + + const linkedReader = new DirectoryPackageReader(linkedBaseDir, limits); + + await expect(linkedReader.listEntries()).rejects.toThrow('disallowed entry type'); + }); + + it('fails when a single entry exceeds the per-entry size limit', async () => { + await writeFile(path.join(baseDir, 'big.json'), Buffer.alloc(limits.maxEntryBytes + 1, 0x61)); + + await expect(reader().listEntries()).rejects.toThrow( + 'exceeds the maximum allowed uncompressed size per entry', + ); + }); + + it('fails when the tree exceeds the total uncompressed size limit', async () => { + const tight = new DirectoryPackageReader(baseDir, { ...limits, maxUncompressedBytes: 1024 }); + await writeFile(path.join(baseDir, 'a.json'), Buffer.alloc(600, 0x61)); + await writeFile(path.join(baseDir, 'b.json'), Buffer.alloc(600, 0x61)); + + await expect(tight.listEntries()).rejects.toThrow( + 'exceeds the maximum allowed uncompressed size', + ); + }); +}); diff --git a/packages/cli/src/modules/n8n-packages/io/directory/directory-package-reader.ts b/packages/cli/src/modules/n8n-packages/io/directory/directory-package-reader.ts new file mode 100644 index 00000000000..49b4e5e54d1 --- /dev/null +++ b/packages/cli/src/modules/n8n-packages/io/directory/directory-package-reader.ts @@ -0,0 +1,177 @@ +import { jsonParse } from 'n8n-workflow'; +import type { Stats } from 'node:fs'; +import { lstat, readdir, readFile } from 'node:fs/promises'; +import path from 'node:path'; + +import { BadRequestError } from '@/errors/response-errors/bad-request.error'; + +import type { PackageManifest } from '../../spec/manifest.schema'; +import type { PackageReader } from '../package-reader'; + +const MANIFEST_PATH = 'manifest.json'; +const ALLOWED_PATH_CHARS = /^[a-zA-Z0-9._/-]+$/; + +export interface DirectoryReaderLimits { + maxUncompressedBytes: number; + maxEntryBytes: number; + maxEntries: number; + maxPathLength: number; +} + +export class DirectoryPackageReader implements PackageReader { + constructor( + private readonly baseDir: string, + private readonly limits: DirectoryReaderLimits, + ) {} + + async readManifest(): Promise { + let raw: Buffer; + try { + raw = await this.readWithinBase(MANIFEST_PATH); + } catch { + throw new BadRequestError('Package is missing manifest.json'); + } + try { + return jsonParse(raw.toString('utf-8')); + } catch { + throw new BadRequestError('Package manifest is not valid JSON'); + } + } + + async readFile(entryPath: string): Promise { + try { + return await this.readWithinBase(entryPath); + } catch (error) { + if (error instanceof BadRequestError) throw error; + throw new BadRequestError(`Package does not contain entry: ${entryPath}`); + } + } + + async listEntries(): Promise { + const entries: string[] = []; + let totalBytes = 0; + await this.walk(this.baseDir, (relativePath, size) => { + if (entries.length + 1 > this.limits.maxEntries) { + throw new BadRequestError('Package contains too many entries'); + } + if (relativePath.length > this.limits.maxPathLength) { + throw new BadRequestError('Package entry path exceeds the maximum allowed length'); + } + if (size > this.limits.maxEntryBytes) { + throw new BadRequestError( + `Package entry "${relativePath}" exceeds the maximum allowed uncompressed size per entry`, + ); + } + totalBytes += size; + if (totalBytes > this.limits.maxUncompressedBytes) { + throw new BadRequestError('Package exceeds the maximum allowed uncompressed size'); + } + entries.push(relativePath); + }); + return entries; + } + + private async readWithinBase(entryPath: string): Promise { + const safePath = this.validateEntryPath(entryPath); + const absolutePath = this.resolveWithin(safePath); + + const stats = await this.lstatWithoutSymlinks(absolutePath, safePath); + if (!stats.isFile()) { + throw new BadRequestError(`Package entry is not a file: ${entryPath}`); + } + if (stats.size > this.limits.maxEntryBytes) { + throw new BadRequestError( + `Package entry "${safePath}" exceeds the maximum allowed uncompressed size per entry`, + ); + } + return await readFile(absolutePath); + } + + private async lstatWithoutSymlinks(absolutePath: string, entryPath: string): Promise { + let currentPath = this.baseDir; + let stats = await lstat(currentPath); + if (stats.isSymbolicLink()) { + throw new BadRequestError('Package root has a disallowed entry type'); + } + + const relativePath = path.relative(this.baseDir, absolutePath); + for (const component of relativePath.split(path.sep)) { + currentPath = path.join(currentPath, component); + stats = await lstat(currentPath); + if (stats.isSymbolicLink()) { + throw new BadRequestError(`Package contains a disallowed entry type for "${entryPath}"`); + } + } + + return stats; + } + + /** Keep path validation aligned with TarPackageReader. */ + private validateEntryPath(rawPath: string): string { + const trimmed = rawPath.endsWith('/') ? rawPath.slice(0, -1) : rawPath; + + if (trimmed.length === 0) { + throw new BadRequestError('Package contains an entry with an empty path'); + } + if (trimmed.length > this.limits.maxPathLength) { + throw new BadRequestError('Package entry path exceeds the maximum allowed length'); + } + if (trimmed.startsWith('/')) { + throw new BadRequestError(`Package entry path "${trimmed}" must be relative`); + } + if (!ALLOWED_PATH_CHARS.test(trimmed)) { + throw new BadRequestError(`Package entry path "${trimmed}" contains disallowed characters`); + } + + const normalized = path.posix.normalize(trimmed); + if ( + normalized === '..' || + normalized.startsWith('../') || + normalized.includes('/../') || + normalized.endsWith('/..') + ) { + throw new BadRequestError( + `Package entry path "${trimmed}" attempts to escape the package root`, + ); + } + + return normalized; + } + + private resolveWithin(safePath: string): string { + const destination = path.resolve(this.baseDir, safePath); + const relative = path.relative(this.baseDir, destination); + if (relative === '' || relative.startsWith('..') || path.isAbsolute(relative)) { + throw new BadRequestError( + `Package entry path "${safePath}" attempts to escape the package root`, + ); + } + return destination; + } + + private async walk( + dir: string, + visit: (relativePath: string, size: number) => void, + ): Promise { + if ((await lstat(dir)).isSymbolicLink()) { + const relativeDirPath = path.relative(this.baseDir, dir).split(path.sep).join('/') || '.'; + throw new BadRequestError( + `Package contains a disallowed entry type for "${relativeDirPath}"`, + ); + } + const dirents = await readdir(dir, { withFileTypes: true }); + for (const dirent of dirents) { + const absolutePath = path.join(dir, dirent.name); + const relativePath = path.relative(this.baseDir, absolutePath).split(path.sep).join('/'); + const stats = await lstat(absolutePath); + if (stats.isSymbolicLink()) { + throw new BadRequestError(`Package contains a disallowed entry type for "${relativePath}"`); + } + if (stats.isDirectory()) { + await this.walk(absolutePath, visit); + } else if (stats.isFile()) { + visit(relativePath, stats.size); + } + } + } +} diff --git a/packages/cli/src/modules/n8n-packages/n8n-packages.service.ts b/packages/cli/src/modules/n8n-packages/n8n-packages.service.ts index 34bfe4806ee..17b69a30c76 100644 --- a/packages/cli/src/modules/n8n-packages/n8n-packages.service.ts +++ b/packages/cli/src/modules/n8n-packages/n8n-packages.service.ts @@ -7,6 +7,8 @@ import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { ForbiddenError } from '@/errors/response-errors/forbidden.error'; import { EventService } from '@/events/event.service'; +import { buildImportResult, toPackageSummary } from './engine/import-result'; +import { emitPackageImportedEvent, type ImportOutcome } from './engine/import-telemetry'; import { N8nPackageParser } from './engine/n8n-package-parser'; import { ProjectPackageImporter } from './engine/project-package-importer'; import { WorkflowPackageImporter } from './engine/workflow-package-importer'; @@ -31,7 +33,9 @@ import { import { WorkflowDependencyResolver } from './entities/workflow/workflow-dependency-resolver'; import { WorkflowRequirementExporter } from './entities/workflow/workflow-requirement.exporter'; import { WorkflowExporter } from './entities/workflow/workflow.exporter'; +import { DirectoryPackageReader } from './io/directory/directory-package-reader'; import { DirectoryPackageWriter } from './io/directory/directory-package-writer'; +import type { PackageReader } from './io/package-reader'; import type { PackageWriter } from './io/package-writer'; import { TarPackageReader } from './io/tar/tar-package-reader'; import { TarPackageWriter } from './io/tar/tar-package-writer'; @@ -45,7 +49,10 @@ import { type ExportPackageResult, type ExportPackageSummary, type ImportPackageRequest, + type ImportRequest, type ImportResult, + type ResolvedImportPackageRequest, + createBindings, } from './n8n-packages.types'; import { FORMAT_VERSION } from './spec/constants'; import { @@ -351,6 +358,37 @@ export class N8nPackagesService { async importPackage(request: ImportPackageRequest): Promise { const reader = new TarPackageReader(request.packageBuffer, this.packageImportConfig); const manifest = await this.packageParser.getManifest(reader); + const { result, scopes } = await this.dispatchImport(request, reader, manifest); + + const resolvedRequest: ResolvedImportPackageRequest = { + ...request, + folderConflictPolicy: resolveFolderConflictPolicy( + request, + isProjectPackage(manifest) ? 'project' : 'workflow', + ), + }; + emitPackageImportedEvent(this.eventService, { request: resolvedRequest, manifest, scopes }); + + return result; + } + + async importPackageFromDirectory( + request: ImportRequest, + source: { sourceDir: string }, + ): Promise { + const reader = new DirectoryPackageReader(source.sourceDir, this.packageImportConfig); + await reader.listEntries(); + const manifest = await this.packageParser.getManifest(reader); + if (!isProjectPackage(manifest)) return emptyImportResult(manifest); + const { result } = await this.dispatchImport(request, reader, manifest); + return result; + } + + private async dispatchImport( + request: ImportRequest, + reader: PackageReader, + manifest: PackageManifest, + ): Promise { if (isProjectPackage(manifest)) { if (request.variableParentPolicy !== undefined) { throw new BadRequestError( @@ -415,3 +453,19 @@ export class N8nPackagesService { function isProjectPackage(manifest: PackageManifest): boolean { return (manifest.projects?.length ?? 0) > 0; } + +function emptyImportResult(manifest: PackageManifest): ImportResult { + return buildImportResult({ + package: toPackageSummary(manifest), + workflows: [], + removedWorkflows: [], + removedFolders: [], + folders: [], + projects: [], + bindings: createBindings(), + credentials: { matched: [], stubbed: [] }, + dataTables: { matched: 0, created: 0 }, + variables: { matched: [], created: [], stubbed: [], updated: [], missing: [] }, + tags: { matched: [], created: [], renamed: [], reconciled: [], skipped: [] }, + }); +} diff --git a/packages/cli/src/modules/n8n-packages/n8n-packages.types.ts b/packages/cli/src/modules/n8n-packages/n8n-packages.types.ts index 0209b07cd07..e751d3ed22b 100644 --- a/packages/cli/src/modules/n8n-packages/n8n-packages.types.ts +++ b/packages/cli/src/modules/n8n-packages/n8n-packages.types.ts @@ -230,11 +230,10 @@ export interface ExportPackageRequest { credentialExportPolicy?: CredentialExportPolicy; } -export type ImportPackageRequest = { +export type ImportRequest = { user: User; projectId?: string; folderId?: string; - packageBuffer: Buffer; bindings?: Partial; apiKeyScopes?: string[]; } & ImportCredentialProperties & @@ -245,6 +244,10 @@ export type ImportPackageRequest = { ImportVariableProperties & ImportTagProperties; +export type ImportPackageRequest = ImportRequest & { + packageBuffer: Buffer; +}; + export type ImportCredentialProperties = { credentialMatchingMode: CredentialMatchingMode; credentialMissingMode: CredentialMissingMode; @@ -267,7 +270,8 @@ export type ResolvedImportFolderProperties = ImportFolderProperties & { folderConflictPolicy: FolderConflictPolicy; }; -/** An import request every importer can read without re-deriving what the caller omitted. */ +export type ResolvedImportRequest = ImportRequest & ResolvedImportFolderProperties; + export type ResolvedImportPackageRequest = ImportPackageRequest & ResolvedImportFolderProperties; export type ImportFolderProperties = { @@ -590,6 +594,11 @@ export interface ImportVariableSummary { updated: string[]; } +export interface ImportDataTableSummary { + matched: number; + created: number; +} + /** Tag names (not ids), grouped by how the import resolved them. */ export interface ImportTagSummary { matched: string[]; @@ -611,6 +620,7 @@ export interface ImportResult { projects: ImportedProjectSummary[]; bindings: SerializedBindings; credentials: ImportCredentialSummary; + dataTables: ImportDataTableSummary; variables: ImportVariableSummary; tags: ImportTagSummary; } diff --git a/packages/cli/src/public-api/v1/controllers/git-connections.public.controller.ts b/packages/cli/src/public-api/v1/controllers/git-connections.public.controller.ts index 9f0bed919f5..4e2fe63aca9 100644 --- a/packages/cli/src/public-api/v1/controllers/git-connections.public.controller.ts +++ b/packages/cli/src/public-api/v1/controllers/git-connections.public.controller.ts @@ -5,6 +5,7 @@ import { GitConnectionProjectListPublicDto, GitConnectionProjectPublicDto, GitConnectionPublicDto, + GitConnectionPullResultDto, GitConnectionPushResultDto, ListGitConnectionsQueryDto, MAX_ITEMS_PER_PAGE, @@ -286,4 +287,25 @@ export class GitConnectionsPublicController { projectId, }); } + + @Post('/:id/pull') + @Licensed(LICENSE_FEATURES.GIT_CONNECTIONS) + @ApiKeyScope('gitConnection:pull') + @GlobalScope('gitConnection:pull') + @ApiSummary('Import all projects from a Git connection working copy') + @ApiDescription( + 'Work in progress. Imports all projects from the local repository working copy into the instance, overwriting to match it. It does not pull from the remote yet, so it imports whatever the last clone produced.', + ) + @ApiTags(tags) + @ApiResponse(200, GitConnectionPullResultDto) + @ApiErrorResponse(400) + @ApiErrorResponse(404) + @ApiErrorResponse(503) + async pullGitConnectionProjects( + req: AuthenticatedRequest, + _res: Response, + @Param('id') id: string, + ): Promise { + return await (await this.gitConnectionsService()).pull(id, req.user); + } } diff --git a/packages/cli/src/public-api/v1/handlers/git-connections/spec/paths/pullGitConnectionProjects.generated.yml b/packages/cli/src/public-api/v1/handlers/git-connections/spec/paths/pullGitConnectionProjects.generated.yml new file mode 100644 index 00000000000..2c40654f67e --- /dev/null +++ b/packages/cli/src/public-api/v1/handlers/git-connections/spec/paths/pullGitConnectionProjects.generated.yml @@ -0,0 +1,202 @@ +operationId: pullGitConnectionProjects +tags: + - GitConnections +summary: Import all projects from a Git connection working copy +description: Work in progress. Imports all projects from the local repository working copy into the instance, overwriting to match it. It does not pull from the remote yet, so it imports whatever the last clone produced. +x-required-scope: gitConnection:pull +x-eov-operation-id: unreachable +x-eov-operation-handler: v1/handlers/decorator-routed.handler +x-decorator-routed: true +parameters: + - schema: + type: string + required: true + name: id + in: path +responses: + '200': + description: Operation successful. + content: + application/json: + schema: + type: object + properties: + connectionId: + type: string + counts: + type: object + properties: + projects: + type: object + properties: + created: + type: integer + minimum: 0 + updated: + type: integer + minimum: 0 + skipped: + type: integer + minimum: 0 + required: + - created + - updated + - skipped + folders: + type: object + properties: + created: + type: integer + minimum: 0 + skipped: + type: integer + minimum: 0 + removed: + type: integer + minimum: 0 + required: + - created + - skipped + - removed + workflows: + type: object + properties: + created: + type: integer + minimum: 0 + updated: + type: integer + minimum: 0 + skipped: + type: integer + minimum: 0 + archived: + type: integer + minimum: 0 + deleted: + type: integer + minimum: 0 + publishing: + type: object + properties: + published: + type: integer + minimum: 0 + unpublished: + type: integer + minimum: 0 + unchanged: + type: integer + minimum: 0 + blocked: + type: integer + minimum: 0 + failed: + type: integer + minimum: 0 + required: + - published + - unpublished + - unchanged + - blocked + - failed + required: + - created + - updated + - skipped + - archived + - deleted + - publishing + credentials: + type: object + properties: + matched: + type: integer + minimum: 0 + stubbed: + type: integer + minimum: 0 + required: + - matched + - stubbed + dataTables: + type: object + properties: + matched: + type: integer + minimum: 0 + created: + type: integer + minimum: 0 + required: + - matched + - created + variables: + type: object + properties: + matched: + type: integer + minimum: 0 + created: + type: integer + minimum: 0 + updated: + type: integer + minimum: 0 + stubbed: + type: integer + minimum: 0 + missing: + type: integer + minimum: 0 + required: + - matched + - created + - updated + - stubbed + - missing + tags: + type: object + properties: + matched: + type: integer + minimum: 0 + created: + type: integer + minimum: 0 + renamed: + type: integer + minimum: 0 + reconciled: + type: integer + minimum: 0 + skipped: + type: integer + minimum: 0 + required: + - matched + - created + - renamed + - reconciled + - skipped + required: + - projects + - folders + - workflows + - credentials + - dataTables + - variables + - tags + required: + - connectionId + - counts + '400': + $ref: ../../../../shared/spec/responses/badRequest.yml + '401': + $ref: ../../../../shared/spec/responses/unauthorized.yml + '403': + $ref: ../../../../shared/spec/responses/forbidden.yml + '404': + $ref: ../../../../shared/spec/responses/notFound.yml + '503': + $ref: ../../../../shared/spec/responses/serviceUnavailable.yml diff --git a/packages/cli/src/public-api/v1/handlers/n8n-packages/spec/paths/n8n-packages.import.yml b/packages/cli/src/public-api/v1/handlers/n8n-packages/spec/paths/n8n-packages.import.yml index 122996468fe..26032ad5d65 100644 --- a/packages/cli/src/public-api/v1/handlers/n8n-packages/spec/paths/n8n-packages.import.yml +++ b/packages/cli/src/public-api/v1/handlers/n8n-packages/spec/paths/n8n-packages.import.yml @@ -376,6 +376,7 @@ post: - projects - bindings - credentials + - dataTables - variables - tags properties: @@ -631,6 +632,18 @@ post: description: > Source credential ids for which empty placeholder credentials were created in the target project. + dataTables: + type: object + required: + - matched + - created + properties: + matched: + type: integer + minimum: 0 + created: + type: integer + minimum: 0 variables: type: object description: > diff --git a/packages/cli/src/public-api/v1/openapi.decorator-routes.generated.yml b/packages/cli/src/public-api/v1/openapi.decorator-routes.generated.yml index d74ca5e5175..33446ad8970 100644 --- a/packages/cli/src/public-api/v1/openapi.decorator-routes.generated.yml +++ b/packages/cli/src/public-api/v1/openapi.decorator-routes.generated.yml @@ -32,6 +32,9 @@ paths: $ref: ./handlers/git-connections/spec/paths/addProjectToGitConnection.generated.yml delete: $ref: ./handlers/git-connections/spec/paths/removeProjectFromGitConnection.generated.yml + /git-connections/{id}/pull: + post: + $ref: ./handlers/git-connections/spec/paths/pullGitConnectionProjects.generated.yml /role-mapping-rules: get: $ref: ./handlers/role-mapping-rules/spec/paths/getRoleMappingRules.generated.yml diff --git a/packages/cli/test/integration/public-api/git-connections.test.ts b/packages/cli/test/integration/public-api/git-connections.test.ts index 718bbfa3f30..8ccf8c2b8a1 100644 --- a/packages/cli/test/integration/public-api/git-connections.test.ts +++ b/packages/cli/test/integration/public-api/git-connections.test.ts @@ -313,4 +313,63 @@ describe('Git connections in Public API', () => { ).toBeNull(); }); }); + async function createConnection(agent: ReturnType) { + const response = await agent.post('/git-connections').send({ + name: 'Deployments', + repositoryUrl: 'https://example.com/org/repo.git', + branchName: 'main', + connectionType: 'https', + username: 'git-user', + password: 'secret', + }); + return response.body.id as string; + } + + it('pushes then pulls the working copy back, reporting imported counts', async () => { + const agent = testServer.publicApiAgentFor(owner); + const id = await createConnection(agent); + const teamProjectCount = (await Container.get(ProjectRepository).findTeamProjectIds()).length; + + const pushResponse = await agent.post(`/git-connections/${id}/push`); + expect(pushResponse.status, JSON.stringify(pushResponse.body)).toBe(200); + + const pullResponse = await agent.post(`/git-connections/${id}/pull`); + expect(pullResponse.status, JSON.stringify(pullResponse.body)).toBe(200); + expect(pullResponse.body).toEqual({ + connectionId: id, + counts: { + projects: { created: 0, updated: teamProjectCount, skipped: 0 }, + folders: { created: 0, skipped: 0, removed: 0 }, + workflows: { + created: 0, + updated: 0, + skipped: 0, + archived: 0, + deleted: 0, + publishing: { published: 0, unpublished: 0, unchanged: 0, blocked: 0, failed: 0 }, + }, + credentials: { matched: 0, stubbed: 0 }, + dataTables: { matched: 0, created: 0 }, + variables: { matched: 0, created: 0, updated: 0, stubbed: 0, missing: 0 }, + tags: { matched: 0, created: 0, renamed: 0, reconciled: 0, skipped: 0 }, + }, + }); + }); + + it('rejects a pull with a clear error when there is no working copy', async () => { + const agent = testServer.publicApiAgentFor(owner); + const id = await createConnection(agent); + + const response = await agent.post(`/git-connections/${id}/pull`); + expect(response.status).toBe(400); + expect(response.body.message).toContain('no exported working copy'); + }); + + it('rejects a pull from a key without the gitConnection:pull scope', async () => { + const unscopedOwner = await createOwnerWithApiKey({ scopes: ['tag:list'] }); + const response = await testServer + .publicApiAgentFor(unscopedOwner) + .post('/git-connections/some-id/pull'); + expect(response.status).toBe(403); + }); }); diff --git a/packages/cli/test/integration/public-api/n8n-packages-import.test.ts b/packages/cli/test/integration/public-api/n8n-packages-import.test.ts index 5aa61e0a8a3..4d1221a456d 100644 --- a/packages/cli/test/integration/public-api/n8n-packages-import.test.ts +++ b/packages/cli/test/integration/public-api/n8n-packages-import.test.ts @@ -258,6 +258,10 @@ describe('POST /n8n-packages/import', () => { matched: [], stubbed: [], }, + dataTables: { + matched: 0, + created: 0, + }, variables: { matched: [], missing: [], diff --git a/packages/nodes-base/nodes/N8n/n8n-api-coverage.json b/packages/nodes-base/nodes/N8n/n8n-api-coverage.json index 345f8a8d697..4de2207a42c 100644 --- a/packages/nodes-base/nodes/N8n/n8n-api-coverage.json +++ b/packages/nodes-base/nodes/N8n/n8n-api-coverage.json @@ -244,6 +244,9 @@ "POST /git-connections/{id}/push": { "status": "gap" }, + "POST /git-connections/{id}/pull": { + "status": "gap" + }, "GET /settings/security-policy": { "status": "gap" },