mirror of
https://github.com/n8n-io/n8n.git
synced 2026-10-30 19:35:36 +08:00
feat(Compression Node): Add tar and tar.gz support (#32547)
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
900c0eae4f
commit
d595631d60
+1
-1
@@ -148,7 +148,7 @@
|
||||
"@smithy/config-resolver": ">=4.4.0",
|
||||
"@rudderstack/rudder-sdk-node@<=3.0.0": "3.0.0",
|
||||
"diff": "8.0.3",
|
||||
"tar": "^7.5.16",
|
||||
"tar": "catalog:",
|
||||
"ajv@6": "6.14.0",
|
||||
"ajv@7": "8.18.0",
|
||||
"ajv@8": "8.18.0",
|
||||
|
||||
@@ -241,7 +241,7 @@
|
||||
"sshpk": "1.18.0",
|
||||
"sucrase": "3.35.0",
|
||||
"swagger-ui-express": "5.0.1",
|
||||
"tar": "^7.5.11",
|
||||
"tar": "catalog:",
|
||||
"turndown": "catalog:",
|
||||
"undici": "catalog:undici-v7",
|
||||
"uuid": "catalog:",
|
||||
|
||||
@@ -11,14 +11,40 @@ import {
|
||||
type INodeType,
|
||||
type INodeTypeDescription,
|
||||
} from 'n8n-workflow';
|
||||
import { buffer } from 'node:stream/consumers';
|
||||
import { promisify } from 'util';
|
||||
|
||||
import { boundedGunzip } from './decompress/BoundedGunzip';
|
||||
import { boundedUntar } from './decompress/BoundedUntar';
|
||||
import { boundedUnzip } from './decompress/BoundedUnzip';
|
||||
|
||||
const gzip = promisify(fflate.gzip);
|
||||
const zip = promisify(fflate.zip);
|
||||
|
||||
type TarInputFile = { fileName: string; data: Buffer };
|
||||
|
||||
/**
|
||||
* Bundle the given files into a tar archive, optionally gzip-compressed
|
||||
* (.tar.gz/.tgz). The archive is built entirely in memory.
|
||||
*/
|
||||
async function createTar(files: TarInputFile[], gzipOutput: boolean): Promise<Buffer> {
|
||||
// Lazy-load tar since it is only needed on this compress path.
|
||||
const { Header, Pack, ReadEntry } = await import('tar');
|
||||
|
||||
const pack = new Pack(gzipOutput ? { gzip: true } : {});
|
||||
|
||||
for (const { fileName, data } of files) {
|
||||
const entry = new ReadEntry(
|
||||
new Header({ path: fileName, size: data.length, mode: 0o644, type: 'File' }),
|
||||
);
|
||||
entry.end(data);
|
||||
pack.write(entry);
|
||||
}
|
||||
pack.end();
|
||||
|
||||
return await buffer(pack);
|
||||
}
|
||||
|
||||
const ALREADY_COMPRESSED = [
|
||||
'7z',
|
||||
'aifc',
|
||||
@@ -126,6 +152,14 @@ export class Compression implements INodeType {
|
||||
name: 'Gzip',
|
||||
value: 'gzip',
|
||||
},
|
||||
{
|
||||
name: 'Tar',
|
||||
value: 'tar',
|
||||
},
|
||||
{
|
||||
name: 'Tar (Gzip)',
|
||||
value: 'targz',
|
||||
},
|
||||
{
|
||||
name: 'Zip',
|
||||
value: 'zip',
|
||||
@@ -149,6 +183,14 @@ export class Compression implements INodeType {
|
||||
name: 'Gzip',
|
||||
value: 'gzip',
|
||||
},
|
||||
{
|
||||
name: 'Tar',
|
||||
value: 'tar',
|
||||
},
|
||||
{
|
||||
name: 'Tar (Gzip)',
|
||||
value: 'targz',
|
||||
},
|
||||
{
|
||||
name: 'Zip',
|
||||
value: 'zip',
|
||||
@@ -174,7 +216,7 @@ export class Compression implements INodeType {
|
||||
displayOptions: {
|
||||
show: {
|
||||
operation: ['compress'],
|
||||
outputFormat: ['zip'],
|
||||
outputFormat: ['zip', 'tar', 'targz'],
|
||||
},
|
||||
},
|
||||
description: 'Name of the output file',
|
||||
@@ -186,7 +228,7 @@ export class Compression implements INodeType {
|
||||
default: 'data',
|
||||
displayOptions: {
|
||||
show: {
|
||||
outputFormat: ['zip'],
|
||||
outputFormat: ['zip', 'tar', 'targz'],
|
||||
operation: ['compress'],
|
||||
},
|
||||
},
|
||||
@@ -289,12 +331,16 @@ export class Compression implements INodeType {
|
||||
);
|
||||
}
|
||||
|
||||
if (fileExtension === 'zip') {
|
||||
const files = await boundedUnzip(
|
||||
binaryDataBuffer,
|
||||
maxDecompressedSize,
|
||||
maxZipEntries,
|
||||
);
|
||||
// .tar.gz keeps the gz extension, so detect it from the file name
|
||||
const isTar =
|
||||
fileExtension === 'tar' ||
|
||||
fileExtension === 'tgz' ||
|
||||
(binaryData.fileName?.toLowerCase().endsWith('.tar.gz') ?? false);
|
||||
|
||||
if (fileExtension === 'zip' || isTar) {
|
||||
const files = isTar
|
||||
? await boundedUntar(binaryDataBuffer, maxDecompressedSize, maxZipEntries)
|
||||
: await boundedUnzip(binaryDataBuffer, maxDecompressedSize, maxZipEntries);
|
||||
|
||||
for (const key of Object.keys(files)) {
|
||||
// when files are compressed using MACOSX for some reason they are duplicated under __MACOSX
|
||||
@@ -337,6 +383,15 @@ export class Compression implements INodeType {
|
||||
binaryObject[propertyName].fileName = `${fileName}.${fileExtension}`;
|
||||
binaryObject[propertyName].fileExtension = fileExtension;
|
||||
binaryObject[propertyName].mimeType = mimeType as string;
|
||||
} else {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
`Unsupported archive format ".${fileExtension}" for binary data ${binaryPropertyName}`,
|
||||
{
|
||||
description:
|
||||
'The Decompress operation supports the following formats: zip, gzip, tar, tar.gz and tgz',
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -365,6 +420,7 @@ export class Compression implements INodeType {
|
||||
const outputFormat = this.getNodeParameter('outputFormat', 0) as string;
|
||||
|
||||
const zipData: fflate.Zippable = {};
|
||||
const tarFiles: TarInputFile[] = [];
|
||||
const binaryObject: IBinaryKeyData = {};
|
||||
|
||||
for (const [index, binaryPropertyName] of binaryPropertyNames.entries()) {
|
||||
@@ -378,6 +434,11 @@ export class Compression implements INodeType {
|
||||
level: ALREADY_COMPRESSED.includes(binaryData.fileExtension as string) ? 0 : 6,
|
||||
},
|
||||
];
|
||||
} else if (outputFormat === 'tar' || outputFormat === 'targz') {
|
||||
tarFiles.push({
|
||||
fileName: binaryData.fileName as string,
|
||||
data: binaryDataBuffer,
|
||||
});
|
||||
} else if (outputFormat === 'gzip') {
|
||||
let outputPrefix;
|
||||
let fileName;
|
||||
@@ -439,6 +500,30 @@ export class Compression implements INodeType {
|
||||
});
|
||||
}
|
||||
|
||||
if (outputFormat === 'tar' || outputFormat === 'targz') {
|
||||
let tarOptionsIndex = 0;
|
||||
if (nodeVersion > 1) {
|
||||
tarOptionsIndex = i;
|
||||
}
|
||||
const fileName = this.getNodeParameter('fileName', tarOptionsIndex) as string;
|
||||
const binaryPropertyOutput = this.getNodeParameter(
|
||||
'binaryPropertyOutput',
|
||||
tarOptionsIndex,
|
||||
);
|
||||
const buffer = await createTar(tarFiles, outputFormat === 'targz');
|
||||
const data = await this.helpers.prepareBinaryData(buffer, fileName);
|
||||
|
||||
returnData.push({
|
||||
json: items[i].json,
|
||||
binary: {
|
||||
[binaryPropertyOutput]: data,
|
||||
},
|
||||
pairedItem: {
|
||||
item: i,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (outputFormat === 'gzip') {
|
||||
returnData.push({
|
||||
json: items[i].json,
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
import { ensureError, isSafeObjectProperty, setSafeObjectProperty, UserError } from 'n8n-workflow';
|
||||
import path from 'node:path';
|
||||
import type { ReadEntry } from 'tar';
|
||||
|
||||
import { DecompressedSizeExceededError } from './DecompressedSizeExceededError';
|
||||
|
||||
/**
|
||||
* Whether an entry path is absolute or escapes the archive root via `..`.
|
||||
* Such entries are skipped rather than surfaced.
|
||||
*/
|
||||
function escapesRoot(entryPath: string): boolean {
|
||||
const trimmed = entryPath.endsWith('/') ? entryPath.slice(0, -1) : entryPath;
|
||||
if (trimmed.startsWith('/')) return true;
|
||||
|
||||
const normalized = path.posix.normalize(trimmed);
|
||||
return (
|
||||
normalized === '..' ||
|
||||
normalized.startsWith('../') ||
|
||||
normalized.includes('/../') ||
|
||||
normalized.endsWith('/..')
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Decompress a tar archive (optionally gzip-compressed, e.g. .tar.gz/.tgz) with
|
||||
* upper bounds on total output size and number of entries.
|
||||
*
|
||||
* Only regular files are returned; directories, symbolic links and other special
|
||||
* entry types are skipped. The size limit is enforced from each entry's declared
|
||||
* header size, before its body is read, so an oversized archive is rejected
|
||||
* without buffering its contents. Gzip compression is detected automatically.
|
||||
*/
|
||||
export async function boundedUntar(
|
||||
data: Buffer,
|
||||
maxOutputSize: number,
|
||||
maxEntries: number,
|
||||
): Promise<Record<string, Buffer>> {
|
||||
// Lazy-load tar since it is only needed on this decompress path.
|
||||
const { Parser } = await import('tar');
|
||||
|
||||
return await new Promise<Record<string, Buffer>>((resolve, reject) => {
|
||||
const result: Record<string, Buffer> = {};
|
||||
let entryCount = 0;
|
||||
let totalSize = 0;
|
||||
let settled = false;
|
||||
|
||||
const fail = (error: Error): void => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
try {
|
||||
parser.abort(error);
|
||||
} catch {
|
||||
// abort can throw if the parser is already torn down; ignore and reject below
|
||||
}
|
||||
reject(error);
|
||||
};
|
||||
|
||||
const parser = new Parser();
|
||||
|
||||
parser.on('entry', (entry: ReadEntry) => {
|
||||
if (settled) {
|
||||
entry.resume();
|
||||
return;
|
||||
}
|
||||
|
||||
// Only surface regular files, mirroring how the zip path skips directories.
|
||||
if (entry.type !== 'File' && entry.type !== 'OldFile' && entry.type !== 'ContiguousFile') {
|
||||
entry.resume();
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip entries that are absolute or reach outside the archive root.
|
||||
if (escapesRoot(entry.path)) {
|
||||
entry.resume();
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip entries whose path would write to an unsafe object key (e.g. __proto__).
|
||||
if (!isSafeObjectProperty(entry.path)) {
|
||||
entry.resume();
|
||||
return;
|
||||
}
|
||||
|
||||
if (++entryCount > maxEntries) {
|
||||
fail(new UserError(`The archive contains more than ${maxEntries} entries`));
|
||||
entry.resume();
|
||||
return;
|
||||
}
|
||||
|
||||
totalSize += entry.size;
|
||||
if (totalSize > maxOutputSize) {
|
||||
fail(new DecompressedSizeExceededError(maxOutputSize));
|
||||
entry.resume();
|
||||
return;
|
||||
}
|
||||
|
||||
const chunks: Buffer[] = [];
|
||||
entry.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
entry.on('end', () => {
|
||||
if (!settled) setSafeObjectProperty(result, entry.path, Buffer.concat(chunks));
|
||||
});
|
||||
entry.on('error', (error: unknown) => fail(ensureError(error)));
|
||||
entry.resume();
|
||||
});
|
||||
|
||||
parser.on('error', (error: Error) => fail(ensureError(error)));
|
||||
parser.on('end', () => {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
resolve(result);
|
||||
}
|
||||
});
|
||||
|
||||
parser.end(data);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
import * as fflate from 'fflate';
|
||||
|
||||
import { boundedUntar } from '../../decompress/BoundedUntar';
|
||||
|
||||
/**
|
||||
* Builds a minimal ustar header block (512 bytes) for a single entry.
|
||||
* typeflag: '0' regular file, '5' directory, '2' symbolic link.
|
||||
*/
|
||||
function tarHeader(name: string, size: number, typeflag = '0', linkname = ''): Buffer {
|
||||
const header = Buffer.alloc(512);
|
||||
header.write(name, 0, 100, 'utf-8');
|
||||
header.write('0000644\0', 100, 'ascii'); // mode
|
||||
header.write('0000000\0', 108, 'ascii'); // uid
|
||||
header.write('0000000\0', 116, 'ascii'); // gid
|
||||
header.write(size.toString(8).padStart(11, '0') + '\0', 124, 'ascii'); // size
|
||||
header.write('00000000000\0', 136, 'ascii'); // mtime
|
||||
header.write(' ', 148, 8, 'ascii'); // checksum placeholder (8 spaces)
|
||||
header.write(typeflag, 156, 'ascii');
|
||||
if (linkname) header.write(linkname, 157, 100, 'utf-8');
|
||||
header.write('ustar\0', 257, 'ascii'); // magic
|
||||
header.write('00', 263, 'ascii'); // version
|
||||
|
||||
let sum = 0;
|
||||
for (let i = 0; i < 512; i++) sum += header[i];
|
||||
header.write(sum.toString(8).padStart(6, '0') + '\0 ', 148, 8, 'ascii');
|
||||
|
||||
return header;
|
||||
}
|
||||
|
||||
type TarEntry = { content?: Buffer | string; typeflag?: string; linkname?: string };
|
||||
|
||||
function createTar(entries: Record<string, TarEntry>): Buffer {
|
||||
const blocks: Buffer[] = [];
|
||||
for (const [name, entry] of Object.entries(entries)) {
|
||||
const data = Buffer.isBuffer(entry.content) ? entry.content : Buffer.from(entry.content ?? '');
|
||||
blocks.push(tarHeader(name, data.length, entry.typeflag ?? '0', entry.linkname));
|
||||
if (data.length > 0) {
|
||||
blocks.push(data);
|
||||
const remainder = data.length % 512;
|
||||
if (remainder !== 0) blocks.push(Buffer.alloc(512 - remainder));
|
||||
}
|
||||
}
|
||||
// archive terminates with two zero-filled blocks
|
||||
blocks.push(Buffer.alloc(1024));
|
||||
return Buffer.concat(blocks);
|
||||
}
|
||||
|
||||
describe('boundedUntar', () => {
|
||||
it('should extract files from a plain tar within the size limit', async () => {
|
||||
const archive = createTar({
|
||||
'a.txt': { content: 'hello' },
|
||||
'sub/b.txt': { content: 'world!!' },
|
||||
});
|
||||
|
||||
const result = await boundedUntar(archive, 1024, 100);
|
||||
|
||||
expect(Object.keys(result).sort()).toEqual(['a.txt', 'sub/b.txt']);
|
||||
expect(result['a.txt'].toString()).toBe('hello');
|
||||
expect(result['sub/b.txt'].toString()).toBe('world!!');
|
||||
});
|
||||
|
||||
it('should auto-detect and extract a gzip-compressed tar', async () => {
|
||||
const archive = Buffer.from(fflate.gzipSync(createTar({ 'a.txt': { content: 'hello' } })));
|
||||
|
||||
const result = await boundedUntar(archive, 1024, 100);
|
||||
|
||||
expect(Object.keys(result)).toEqual(['a.txt']);
|
||||
expect(result['a.txt'].toString()).toBe('hello');
|
||||
});
|
||||
|
||||
it('should reject when total decompressed output exceeds the size limit', async () => {
|
||||
const archive = createTar({
|
||||
'a.bin': { content: Buffer.alloc(1024) },
|
||||
'b.bin': { content: Buffer.alloc(1024) },
|
||||
});
|
||||
|
||||
await expect(boundedUntar(archive, 1500, 100)).rejects.toThrow(
|
||||
'The decompressed output exceeds the maximum allowed size of 0 MB',
|
||||
);
|
||||
});
|
||||
|
||||
it('should reject a single entry whose declared size exceeds the limit', async () => {
|
||||
const archive = createTar({ 'bomb.bin': { content: Buffer.alloc(4096) } });
|
||||
|
||||
await expect(boundedUntar(archive, 1024, 100)).rejects.toThrow(
|
||||
'The decompressed output exceeds the maximum allowed size of 0 MB',
|
||||
);
|
||||
});
|
||||
|
||||
it('should reject when entry count exceeds the limit', async () => {
|
||||
const entries: Record<string, TarEntry> = {};
|
||||
for (let i = 0; i < 5; i++) {
|
||||
entries[`file${i}.txt`] = { content: 'x' };
|
||||
}
|
||||
const archive = createTar(entries);
|
||||
|
||||
await expect(boundedUntar(archive, 1024 * 1024, 3)).rejects.toThrow(
|
||||
'The archive contains more than 3 entries',
|
||||
);
|
||||
});
|
||||
|
||||
it('should skip directory entries', async () => {
|
||||
const archive = createTar({
|
||||
'folder/': { typeflag: '5' },
|
||||
'folder/file.txt': { content: 'data' },
|
||||
});
|
||||
|
||||
const result = await boundedUntar(archive, 1024, 100);
|
||||
|
||||
expect(Object.keys(result)).toEqual(['folder/file.txt']);
|
||||
expect(result['folder/']).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should skip symbolic link entries', async () => {
|
||||
const archive = createTar({
|
||||
'real.txt': { content: 'data' },
|
||||
'link.txt': { typeflag: '2', linkname: 'real.txt' },
|
||||
});
|
||||
|
||||
const result = await boundedUntar(archive, 1024, 100);
|
||||
|
||||
expect(Object.keys(result)).toEqual(['real.txt']);
|
||||
expect(result['link.txt']).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should skip entries whose path escapes the archive root', async () => {
|
||||
const archive = createTar({
|
||||
'safe.txt': { content: 'ok' },
|
||||
'../evil.txt': { content: 'bad' },
|
||||
'/abs.txt': { content: 'bad' },
|
||||
'nested/../../escape.txt': { content: 'bad' },
|
||||
});
|
||||
|
||||
const result = await boundedUntar(archive, 1024, 100);
|
||||
|
||||
expect(Object.keys(result)).toEqual(['safe.txt']);
|
||||
});
|
||||
|
||||
it('should handle an empty tar archive', async () => {
|
||||
const archive = createTar({});
|
||||
|
||||
const result = await boundedUntar(archive, 1024, 100);
|
||||
|
||||
expect(Object.keys(result)).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,107 @@
|
||||
import { mockDeep } from 'vitest-mock-extended';
|
||||
import type { IExecuteFunctions, INode, IBinaryData } from 'n8n-workflow';
|
||||
import type { Mocked } from 'vitest';
|
||||
|
||||
import { Compression } from '../../Compression.node';
|
||||
import { boundedUntar } from '../../decompress/BoundedUntar';
|
||||
|
||||
describe('Compression Node - Compress Operation (tar)', () => {
|
||||
let compression: Compression;
|
||||
let mockExecuteFunctions: Mocked<IExecuteFunctions>;
|
||||
// the archive buffer handed to prepareBinaryData by the node
|
||||
let outputArchive: Buffer;
|
||||
|
||||
const mockNode: INode = {
|
||||
id: 'test-node',
|
||||
name: 'Compression',
|
||||
type: 'n8n-nodes-base.compression',
|
||||
typeVersion: 1.1,
|
||||
position: [0, 0],
|
||||
parameters: {},
|
||||
};
|
||||
|
||||
const setParams = (outputFormat: string, binaryPropertyName = 'data') => {
|
||||
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
|
||||
const params: Record<string, string> = {
|
||||
operation: 'compress',
|
||||
binaryPropertyName,
|
||||
outputFormat,
|
||||
fileName: 'archive.tar',
|
||||
binaryPropertyOutput: 'data',
|
||||
};
|
||||
return params[paramName];
|
||||
});
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
compression = new Compression();
|
||||
mockExecuteFunctions = mockDeep<IExecuteFunctions>();
|
||||
vi.clearAllMocks();
|
||||
|
||||
mockExecuteFunctions.getNode.mockReturnValue(mockNode);
|
||||
mockExecuteFunctions.getInputData.mockReturnValue([{ json: { test: 'data' } }]);
|
||||
mockExecuteFunctions.continueOnFail.mockReturnValue(false);
|
||||
|
||||
// each binary field maps to a distinct file name
|
||||
vi.mocked(mockExecuteFunctions.helpers.assertBinaryData).mockImplementation(
|
||||
(_i, propertyName) =>
|
||||
({
|
||||
data: 'base64data',
|
||||
mimeType: 'text/plain',
|
||||
fileName: propertyName === 'data2' ? 'b.txt' : 'a.txt',
|
||||
fileExtension: 'txt',
|
||||
}) as IBinaryData,
|
||||
);
|
||||
vi.mocked(mockExecuteFunctions.helpers.getBinaryDataBuffer).mockResolvedValue(
|
||||
Buffer.from('hello'),
|
||||
);
|
||||
vi.mocked(mockExecuteFunctions.helpers.prepareBinaryData).mockImplementation(
|
||||
async (buffer, fileName) => {
|
||||
outputArchive = buffer as Buffer;
|
||||
return {
|
||||
data: buffer.toString('base64'),
|
||||
mimeType: 'application/x-tar',
|
||||
fileName: fileName ?? 'archive.tar',
|
||||
fileExtension: 'tar',
|
||||
} as IBinaryData;
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.resetAllMocks();
|
||||
});
|
||||
|
||||
it('should compress to an uncompressed tar that round-trips', async () => {
|
||||
setParams('tar');
|
||||
|
||||
const result = await compression.execute.call(mockExecuteFunctions);
|
||||
|
||||
// not gzip-compressed
|
||||
expect(outputArchive[0] === 0x1f && outputArchive[1] === 0x8b).toBe(false);
|
||||
const extracted = await boundedUntar(outputArchive, 1024 * 1024, 100);
|
||||
expect(extracted['a.txt'].toString()).toBe('hello');
|
||||
expect(result[0][0].binary?.data).toBeDefined();
|
||||
});
|
||||
|
||||
it('should compress to a gzip-compressed tar that round-trips', async () => {
|
||||
setParams('targz');
|
||||
|
||||
await compression.execute.call(mockExecuteFunctions);
|
||||
|
||||
// gzip magic bytes
|
||||
expect(outputArchive[0]).toBe(0x1f);
|
||||
expect(outputArchive[1]).toBe(0x8b);
|
||||
const extracted = await boundedUntar(outputArchive, 1024 * 1024, 100);
|
||||
expect(extracted['a.txt'].toString()).toBe('hello');
|
||||
});
|
||||
|
||||
it('should bundle multiple input fields into a single tar archive', async () => {
|
||||
setParams('tar', 'data1,data2');
|
||||
|
||||
await compression.execute.call(mockExecuteFunctions);
|
||||
|
||||
const extracted = await boundedUntar(outputArchive, 1024 * 1024, 100);
|
||||
expect(Object.keys(extracted).sort()).toEqual(['a.txt', 'b.txt']);
|
||||
});
|
||||
});
|
||||
@@ -4,10 +4,12 @@ import { NodeOperationError } from 'n8n-workflow';
|
||||
|
||||
import { Compression } from '../../Compression.node';
|
||||
import { boundedGunzip } from '../../decompress/BoundedGunzip';
|
||||
import { boundedUntar } from '../../decompress/BoundedUntar';
|
||||
import { boundedUnzip } from '../../decompress/BoundedUnzip';
|
||||
import type { Mocked } from 'vitest';
|
||||
|
||||
vi.mock('../../decompress/BoundedGunzip');
|
||||
vi.mock('../../decompress/BoundedUntar');
|
||||
vi.mock('../../decompress/BoundedUnzip');
|
||||
|
||||
const mockBoundedUnzip = (data: Record<string, Buffer>, error?: Error) => {
|
||||
@@ -18,6 +20,14 @@ const mockBoundedUnzip = (data: Record<string, Buffer>, error?: Error) => {
|
||||
}
|
||||
};
|
||||
|
||||
const mockBoundedUntar = (data: Record<string, Buffer>, error?: Error) => {
|
||||
if (error) {
|
||||
vi.mocked(boundedUntar).mockRejectedValue(error);
|
||||
} else {
|
||||
vi.mocked(boundedUntar).mockResolvedValue(data);
|
||||
}
|
||||
};
|
||||
|
||||
const mockBoundedGunzip = (data: Buffer, error?: Error) => {
|
||||
if (error) {
|
||||
vi.mocked(boundedGunzip).mockRejectedValue(error);
|
||||
@@ -338,7 +348,111 @@ describe('Compression Node - Decompress Operation', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('Tar Decompression', () => {
|
||||
const tarContents: Record<string, Buffer> = {
|
||||
'a.txt': Buffer.from([72, 101, 108, 108, 111]),
|
||||
'sub/b.txt': Buffer.from([87, 111, 114, 108, 100]),
|
||||
};
|
||||
|
||||
const prepareBinaryDataPassthrough = () => {
|
||||
vi.mocked(mockExecuteFunctions.helpers.prepareBinaryData).mockImplementation(
|
||||
async (buffer, fileName) =>
|
||||
({
|
||||
data: buffer.toString('base64'),
|
||||
mimeType: 'text/plain',
|
||||
fileName: fileName ?? 'file',
|
||||
fileExtension: 'txt',
|
||||
}) as IBinaryData,
|
||||
);
|
||||
};
|
||||
|
||||
it('should decompress a tar file successfully', async () => {
|
||||
const mockBinaryData: IBinaryData = {
|
||||
data: 'base64data',
|
||||
mimeType: 'application/x-tar',
|
||||
fileName: 'test.tar',
|
||||
fileExtension: 'tar',
|
||||
};
|
||||
|
||||
vi.mocked(mockExecuteFunctions.helpers.assertBinaryData).mockReturnValue(mockBinaryData);
|
||||
vi.mocked(mockExecuteFunctions.helpers.getBinaryDataBuffer).mockResolvedValue(
|
||||
Buffer.from('mock tar data'),
|
||||
);
|
||||
mockBoundedUntar(tarContents);
|
||||
prepareBinaryDataPassthrough();
|
||||
|
||||
const result = await compression.execute.call(mockExecuteFunctions);
|
||||
|
||||
expect(result[0][0].binary?.file_0).toBeDefined();
|
||||
expect(result[0][0].binary?.file_1).toBeDefined();
|
||||
expect(boundedUntar).toHaveBeenCalledTimes(1);
|
||||
expect(boundedUnzip).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should route a .tar.gz file to tar extraction, not gzip', async () => {
|
||||
const mockBinaryData: IBinaryData = {
|
||||
data: 'base64data',
|
||||
mimeType: 'application/gzip',
|
||||
fileName: 'archive.tar.gz',
|
||||
fileExtension: 'gz',
|
||||
};
|
||||
|
||||
vi.mocked(mockExecuteFunctions.helpers.assertBinaryData).mockReturnValue(mockBinaryData);
|
||||
vi.mocked(mockExecuteFunctions.helpers.getBinaryDataBuffer).mockResolvedValue(
|
||||
Buffer.from('mock targz data'),
|
||||
);
|
||||
mockBoundedUntar(tarContents);
|
||||
prepareBinaryDataPassthrough();
|
||||
|
||||
const result = await compression.execute.call(mockExecuteFunctions);
|
||||
|
||||
expect(result[0][0].binary?.file_0).toBeDefined();
|
||||
expect(boundedUntar).toHaveBeenCalledTimes(1);
|
||||
expect(boundedGunzip).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should decompress a .tgz file via tar extraction', async () => {
|
||||
const mockBinaryData: IBinaryData = {
|
||||
data: 'base64data',
|
||||
mimeType: 'application/gzip',
|
||||
fileName: 'archive.tgz',
|
||||
fileExtension: 'tgz',
|
||||
};
|
||||
|
||||
vi.mocked(mockExecuteFunctions.helpers.assertBinaryData).mockReturnValue(mockBinaryData);
|
||||
vi.mocked(mockExecuteFunctions.helpers.getBinaryDataBuffer).mockResolvedValue(
|
||||
Buffer.from('mock tgz data'),
|
||||
);
|
||||
mockBoundedUntar(tarContents);
|
||||
prepareBinaryDataPassthrough();
|
||||
|
||||
const result = await compression.execute.call(mockExecuteFunctions);
|
||||
|
||||
expect(result[0][0].binary?.file_0).toBeDefined();
|
||||
expect(boundedUntar).toHaveBeenCalledTimes(1);
|
||||
expect(boundedGunzip).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Error Handling', () => {
|
||||
it('should throw NodeOperationError for an unsupported archive format', async () => {
|
||||
const mockBinaryData: IBinaryData = {
|
||||
data: 'base64data',
|
||||
mimeType: 'application/x-rar-compressed',
|
||||
fileName: 'archive.rar',
|
||||
fileExtension: 'rar',
|
||||
};
|
||||
|
||||
vi.mocked(mockExecuteFunctions.helpers.assertBinaryData).mockReturnValue(mockBinaryData);
|
||||
vi.mocked(mockExecuteFunctions.helpers.getBinaryDataBuffer).mockResolvedValue(
|
||||
Buffer.from('mock rar data'),
|
||||
);
|
||||
|
||||
await expect(compression.execute.call(mockExecuteFunctions)).rejects.toThrow(
|
||||
NodeOperationError,
|
||||
);
|
||||
});
|
||||
|
||||
it('should throw error when file extension is not found', async () => {
|
||||
const mockBinaryData: IBinaryData = {
|
||||
data: 'base64data',
|
||||
|
||||
@@ -977,6 +977,7 @@
|
||||
"simple-git": "catalog:",
|
||||
"snowflake-sdk": "2.1.0",
|
||||
"ssh2-sftp-client": "12.1.0",
|
||||
"tar": "catalog:",
|
||||
"tmp-promise": "3.0.3",
|
||||
"ts-ics": "1.2.2",
|
||||
"undici": "catalog:undici-v6",
|
||||
|
||||
Generated
+3
@@ -5210,6 +5210,9 @@ importers:
|
||||
ssh2-sftp-client:
|
||||
specifier: 12.1.0
|
||||
version: 12.1.0
|
||||
tar:
|
||||
specifier: ^7.5.16
|
||||
version: 7.5.16
|
||||
tmp-promise:
|
||||
specifier: 3.0.3
|
||||
version: 3.0.3
|
||||
|
||||
@@ -150,6 +150,7 @@ catalog:
|
||||
simple-git: 3.36.0
|
||||
source-map-support: 0.5.21
|
||||
stream-json: 1.9.1
|
||||
tar: ^7.5.16
|
||||
testcontainers: ^11.13.0
|
||||
ts-morph: ^27.0.2
|
||||
tsdown: ^0.16.5
|
||||
|
||||
Reference in New Issue
Block a user