feat(Compression Node): Add tar and tar.gz support (#32547)

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garrit Franke
2026-06-23 08:52:31 +00:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 900c0eae4f
commit d595631d60
10 changed files with 583 additions and 10 deletions
+1 -1
View File
@@ -148,7 +148,7 @@
"@smithy/config-resolver": ">=4.4.0",
"@rudderstack/rudder-sdk-node@<=3.0.0": "3.0.0",
"diff": "8.0.3",
"tar": "^7.5.16",
"tar": "catalog:",
"ajv@6": "6.14.0",
"ajv@7": "8.18.0",
"ajv@8": "8.18.0",
+1 -1
View File
@@ -241,7 +241,7 @@
"sshpk": "1.18.0",
"sucrase": "3.35.0",
"swagger-ui-express": "5.0.1",
"tar": "^7.5.11",
"tar": "catalog:",
"turndown": "catalog:",
"undici": "catalog:undici-v7",
"uuid": "catalog:",
@@ -11,14 +11,40 @@ import {
type INodeType,
type INodeTypeDescription,
} from 'n8n-workflow';
import { buffer } from 'node:stream/consumers';
import { promisify } from 'util';
import { boundedGunzip } from './decompress/BoundedGunzip';
import { boundedUntar } from './decompress/BoundedUntar';
import { boundedUnzip } from './decompress/BoundedUnzip';
const gzip = promisify(fflate.gzip);
const zip = promisify(fflate.zip);
type TarInputFile = { fileName: string; data: Buffer };
/**
* Bundle the given files into a tar archive, optionally gzip-compressed
* (.tar.gz/.tgz). The archive is built entirely in memory.
*/
async function createTar(files: TarInputFile[], gzipOutput: boolean): Promise<Buffer> {
// Lazy-load tar since it is only needed on this compress path.
const { Header, Pack, ReadEntry } = await import('tar');
const pack = new Pack(gzipOutput ? { gzip: true } : {});
for (const { fileName, data } of files) {
const entry = new ReadEntry(
new Header({ path: fileName, size: data.length, mode: 0o644, type: 'File' }),
);
entry.end(data);
pack.write(entry);
}
pack.end();
return await buffer(pack);
}
const ALREADY_COMPRESSED = [
'7z',
'aifc',
@@ -126,6 +152,14 @@ export class Compression implements INodeType {
name: 'Gzip',
value: 'gzip',
},
{
name: 'Tar',
value: 'tar',
},
{
name: 'Tar (Gzip)',
value: 'targz',
},
{
name: 'Zip',
value: 'zip',
@@ -149,6 +183,14 @@ export class Compression implements INodeType {
name: 'Gzip',
value: 'gzip',
},
{
name: 'Tar',
value: 'tar',
},
{
name: 'Tar (Gzip)',
value: 'targz',
},
{
name: 'Zip',
value: 'zip',
@@ -174,7 +216,7 @@ export class Compression implements INodeType {
displayOptions: {
show: {
operation: ['compress'],
outputFormat: ['zip'],
outputFormat: ['zip', 'tar', 'targz'],
},
},
description: 'Name of the output file',
@@ -186,7 +228,7 @@ export class Compression implements INodeType {
default: 'data',
displayOptions: {
show: {
outputFormat: ['zip'],
outputFormat: ['zip', 'tar', 'targz'],
operation: ['compress'],
},
},
@@ -289,12 +331,16 @@ export class Compression implements INodeType {
);
}
if (fileExtension === 'zip') {
const files = await boundedUnzip(
binaryDataBuffer,
maxDecompressedSize,
maxZipEntries,
);
// .tar.gz keeps the gz extension, so detect it from the file name
const isTar =
fileExtension === 'tar' ||
fileExtension === 'tgz' ||
(binaryData.fileName?.toLowerCase().endsWith('.tar.gz') ?? false);
if (fileExtension === 'zip' || isTar) {
const files = isTar
? await boundedUntar(binaryDataBuffer, maxDecompressedSize, maxZipEntries)
: await boundedUnzip(binaryDataBuffer, maxDecompressedSize, maxZipEntries);
for (const key of Object.keys(files)) {
// when files are compressed using MACOSX for some reason they are duplicated under __MACOSX
@@ -337,6 +383,15 @@ export class Compression implements INodeType {
binaryObject[propertyName].fileName = `${fileName}.${fileExtension}`;
binaryObject[propertyName].fileExtension = fileExtension;
binaryObject[propertyName].mimeType = mimeType as string;
} else {
throw new NodeOperationError(
this.getNode(),
`Unsupported archive format ".${fileExtension}" for binary data ${binaryPropertyName}`,
{
description:
'The Decompress operation supports the following formats: zip, gzip, tar, tar.gz and tgz',
},
);
}
}
@@ -365,6 +420,7 @@ export class Compression implements INodeType {
const outputFormat = this.getNodeParameter('outputFormat', 0) as string;
const zipData: fflate.Zippable = {};
const tarFiles: TarInputFile[] = [];
const binaryObject: IBinaryKeyData = {};
for (const [index, binaryPropertyName] of binaryPropertyNames.entries()) {
@@ -378,6 +434,11 @@ export class Compression implements INodeType {
level: ALREADY_COMPRESSED.includes(binaryData.fileExtension as string) ? 0 : 6,
},
];
} else if (outputFormat === 'tar' || outputFormat === 'targz') {
tarFiles.push({
fileName: binaryData.fileName as string,
data: binaryDataBuffer,
});
} else if (outputFormat === 'gzip') {
let outputPrefix;
let fileName;
@@ -439,6 +500,30 @@ export class Compression implements INodeType {
});
}
if (outputFormat === 'tar' || outputFormat === 'targz') {
let tarOptionsIndex = 0;
if (nodeVersion > 1) {
tarOptionsIndex = i;
}
const fileName = this.getNodeParameter('fileName', tarOptionsIndex) as string;
const binaryPropertyOutput = this.getNodeParameter(
'binaryPropertyOutput',
tarOptionsIndex,
);
const buffer = await createTar(tarFiles, outputFormat === 'targz');
const data = await this.helpers.prepareBinaryData(buffer, fileName);
returnData.push({
json: items[i].json,
binary: {
[binaryPropertyOutput]: data,
},
pairedItem: {
item: i,
},
});
}
if (outputFormat === 'gzip') {
returnData.push({
json: items[i].json,
@@ -0,0 +1,116 @@
import { ensureError, isSafeObjectProperty, setSafeObjectProperty, UserError } from 'n8n-workflow';
import path from 'node:path';
import type { ReadEntry } from 'tar';
import { DecompressedSizeExceededError } from './DecompressedSizeExceededError';
/**
* Whether an entry path is absolute or escapes the archive root via `..`.
* Such entries are skipped rather than surfaced.
*/
function escapesRoot(entryPath: string): boolean {
const trimmed = entryPath.endsWith('/') ? entryPath.slice(0, -1) : entryPath;
if (trimmed.startsWith('/')) return true;
const normalized = path.posix.normalize(trimmed);
return (
normalized === '..' ||
normalized.startsWith('../') ||
normalized.includes('/../') ||
normalized.endsWith('/..')
);
}
/**
* Decompress a tar archive (optionally gzip-compressed, e.g. .tar.gz/.tgz) with
* upper bounds on total output size and number of entries.
*
* Only regular files are returned; directories, symbolic links and other special
* entry types are skipped. The size limit is enforced from each entry's declared
* header size, before its body is read, so an oversized archive is rejected
* without buffering its contents. Gzip compression is detected automatically.
*/
export async function boundedUntar(
data: Buffer,
maxOutputSize: number,
maxEntries: number,
): Promise<Record<string, Buffer>> {
// Lazy-load tar since it is only needed on this decompress path.
const { Parser } = await import('tar');
return await new Promise<Record<string, Buffer>>((resolve, reject) => {
const result: Record<string, Buffer> = {};
let entryCount = 0;
let totalSize = 0;
let settled = false;
const fail = (error: Error): void => {
if (settled) return;
settled = true;
try {
parser.abort(error);
} catch {
// abort can throw if the parser is already torn down; ignore and reject below
}
reject(error);
};
const parser = new Parser();
parser.on('entry', (entry: ReadEntry) => {
if (settled) {
entry.resume();
return;
}
// Only surface regular files, mirroring how the zip path skips directories.
if (entry.type !== 'File' && entry.type !== 'OldFile' && entry.type !== 'ContiguousFile') {
entry.resume();
return;
}
// Skip entries that are absolute or reach outside the archive root.
if (escapesRoot(entry.path)) {
entry.resume();
return;
}
// Skip entries whose path would write to an unsafe object key (e.g. __proto__).
if (!isSafeObjectProperty(entry.path)) {
entry.resume();
return;
}
if (++entryCount > maxEntries) {
fail(new UserError(`The archive contains more than ${maxEntries} entries`));
entry.resume();
return;
}
totalSize += entry.size;
if (totalSize > maxOutputSize) {
fail(new DecompressedSizeExceededError(maxOutputSize));
entry.resume();
return;
}
const chunks: Buffer[] = [];
entry.on('data', (chunk: Buffer) => chunks.push(chunk));
entry.on('end', () => {
if (!settled) setSafeObjectProperty(result, entry.path, Buffer.concat(chunks));
});
entry.on('error', (error: unknown) => fail(ensureError(error)));
entry.resume();
});
parser.on('error', (error: Error) => fail(ensureError(error)));
parser.on('end', () => {
if (!settled) {
settled = true;
resolve(result);
}
});
parser.end(data);
});
}
@@ -0,0 +1,146 @@
import * as fflate from 'fflate';
import { boundedUntar } from '../../decompress/BoundedUntar';
/**
* Builds a minimal ustar header block (512 bytes) for a single entry.
* typeflag: '0' regular file, '5' directory, '2' symbolic link.
*/
function tarHeader(name: string, size: number, typeflag = '0', linkname = ''): Buffer {
const header = Buffer.alloc(512);
header.write(name, 0, 100, 'utf-8');
header.write('0000644\0', 100, 'ascii'); // mode
header.write('0000000\0', 108, 'ascii'); // uid
header.write('0000000\0', 116, 'ascii'); // gid
header.write(size.toString(8).padStart(11, '0') + '\0', 124, 'ascii'); // size
header.write('00000000000\0', 136, 'ascii'); // mtime
header.write(' ', 148, 8, 'ascii'); // checksum placeholder (8 spaces)
header.write(typeflag, 156, 'ascii');
if (linkname) header.write(linkname, 157, 100, 'utf-8');
header.write('ustar\0', 257, 'ascii'); // magic
header.write('00', 263, 'ascii'); // version
let sum = 0;
for (let i = 0; i < 512; i++) sum += header[i];
header.write(sum.toString(8).padStart(6, '0') + '\0 ', 148, 8, 'ascii');
return header;
}
type TarEntry = { content?: Buffer | string; typeflag?: string; linkname?: string };
function createTar(entries: Record<string, TarEntry>): Buffer {
const blocks: Buffer[] = [];
for (const [name, entry] of Object.entries(entries)) {
const data = Buffer.isBuffer(entry.content) ? entry.content : Buffer.from(entry.content ?? '');
blocks.push(tarHeader(name, data.length, entry.typeflag ?? '0', entry.linkname));
if (data.length > 0) {
blocks.push(data);
const remainder = data.length % 512;
if (remainder !== 0) blocks.push(Buffer.alloc(512 - remainder));
}
}
// archive terminates with two zero-filled blocks
blocks.push(Buffer.alloc(1024));
return Buffer.concat(blocks);
}
describe('boundedUntar', () => {
it('should extract files from a plain tar within the size limit', async () => {
const archive = createTar({
'a.txt': { content: 'hello' },
'sub/b.txt': { content: 'world!!' },
});
const result = await boundedUntar(archive, 1024, 100);
expect(Object.keys(result).sort()).toEqual(['a.txt', 'sub/b.txt']);
expect(result['a.txt'].toString()).toBe('hello');
expect(result['sub/b.txt'].toString()).toBe('world!!');
});
it('should auto-detect and extract a gzip-compressed tar', async () => {
const archive = Buffer.from(fflate.gzipSync(createTar({ 'a.txt': { content: 'hello' } })));
const result = await boundedUntar(archive, 1024, 100);
expect(Object.keys(result)).toEqual(['a.txt']);
expect(result['a.txt'].toString()).toBe('hello');
});
it('should reject when total decompressed output exceeds the size limit', async () => {
const archive = createTar({
'a.bin': { content: Buffer.alloc(1024) },
'b.bin': { content: Buffer.alloc(1024) },
});
await expect(boundedUntar(archive, 1500, 100)).rejects.toThrow(
'The decompressed output exceeds the maximum allowed size of 0 MB',
);
});
it('should reject a single entry whose declared size exceeds the limit', async () => {
const archive = createTar({ 'bomb.bin': { content: Buffer.alloc(4096) } });
await expect(boundedUntar(archive, 1024, 100)).rejects.toThrow(
'The decompressed output exceeds the maximum allowed size of 0 MB',
);
});
it('should reject when entry count exceeds the limit', async () => {
const entries: Record<string, TarEntry> = {};
for (let i = 0; i < 5; i++) {
entries[`file${i}.txt`] = { content: 'x' };
}
const archive = createTar(entries);
await expect(boundedUntar(archive, 1024 * 1024, 3)).rejects.toThrow(
'The archive contains more than 3 entries',
);
});
it('should skip directory entries', async () => {
const archive = createTar({
'folder/': { typeflag: '5' },
'folder/file.txt': { content: 'data' },
});
const result = await boundedUntar(archive, 1024, 100);
expect(Object.keys(result)).toEqual(['folder/file.txt']);
expect(result['folder/']).toBeUndefined();
});
it('should skip symbolic link entries', async () => {
const archive = createTar({
'real.txt': { content: 'data' },
'link.txt': { typeflag: '2', linkname: 'real.txt' },
});
const result = await boundedUntar(archive, 1024, 100);
expect(Object.keys(result)).toEqual(['real.txt']);
expect(result['link.txt']).toBeUndefined();
});
it('should skip entries whose path escapes the archive root', async () => {
const archive = createTar({
'safe.txt': { content: 'ok' },
'../evil.txt': { content: 'bad' },
'/abs.txt': { content: 'bad' },
'nested/../../escape.txt': { content: 'bad' },
});
const result = await boundedUntar(archive, 1024, 100);
expect(Object.keys(result)).toEqual(['safe.txt']);
});
it('should handle an empty tar archive', async () => {
const archive = createTar({});
const result = await boundedUntar(archive, 1024, 100);
expect(Object.keys(result)).toHaveLength(0);
});
});
@@ -0,0 +1,107 @@
import { mockDeep } from 'vitest-mock-extended';
import type { IExecuteFunctions, INode, IBinaryData } from 'n8n-workflow';
import type { Mocked } from 'vitest';
import { Compression } from '../../Compression.node';
import { boundedUntar } from '../../decompress/BoundedUntar';
describe('Compression Node - Compress Operation (tar)', () => {
let compression: Compression;
let mockExecuteFunctions: Mocked<IExecuteFunctions>;
// the archive buffer handed to prepareBinaryData by the node
let outputArchive: Buffer;
const mockNode: INode = {
id: 'test-node',
name: 'Compression',
type: 'n8n-nodes-base.compression',
typeVersion: 1.1,
position: [0, 0],
parameters: {},
};
const setParams = (outputFormat: string, binaryPropertyName = 'data') => {
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
const params: Record<string, string> = {
operation: 'compress',
binaryPropertyName,
outputFormat,
fileName: 'archive.tar',
binaryPropertyOutput: 'data',
};
return params[paramName];
});
};
beforeEach(() => {
compression = new Compression();
mockExecuteFunctions = mockDeep<IExecuteFunctions>();
vi.clearAllMocks();
mockExecuteFunctions.getNode.mockReturnValue(mockNode);
mockExecuteFunctions.getInputData.mockReturnValue([{ json: { test: 'data' } }]);
mockExecuteFunctions.continueOnFail.mockReturnValue(false);
// each binary field maps to a distinct file name
vi.mocked(mockExecuteFunctions.helpers.assertBinaryData).mockImplementation(
(_i, propertyName) =>
({
data: 'base64data',
mimeType: 'text/plain',
fileName: propertyName === 'data2' ? 'b.txt' : 'a.txt',
fileExtension: 'txt',
}) as IBinaryData,
);
vi.mocked(mockExecuteFunctions.helpers.getBinaryDataBuffer).mockResolvedValue(
Buffer.from('hello'),
);
vi.mocked(mockExecuteFunctions.helpers.prepareBinaryData).mockImplementation(
async (buffer, fileName) => {
outputArchive = buffer as Buffer;
return {
data: buffer.toString('base64'),
mimeType: 'application/x-tar',
fileName: fileName ?? 'archive.tar',
fileExtension: 'tar',
} as IBinaryData;
},
);
});
afterEach(() => {
vi.resetAllMocks();
});
it('should compress to an uncompressed tar that round-trips', async () => {
setParams('tar');
const result = await compression.execute.call(mockExecuteFunctions);
// not gzip-compressed
expect(outputArchive[0] === 0x1f && outputArchive[1] === 0x8b).toBe(false);
const extracted = await boundedUntar(outputArchive, 1024 * 1024, 100);
expect(extracted['a.txt'].toString()).toBe('hello');
expect(result[0][0].binary?.data).toBeDefined();
});
it('should compress to a gzip-compressed tar that round-trips', async () => {
setParams('targz');
await compression.execute.call(mockExecuteFunctions);
// gzip magic bytes
expect(outputArchive[0]).toBe(0x1f);
expect(outputArchive[1]).toBe(0x8b);
const extracted = await boundedUntar(outputArchive, 1024 * 1024, 100);
expect(extracted['a.txt'].toString()).toBe('hello');
});
it('should bundle multiple input fields into a single tar archive', async () => {
setParams('tar', 'data1,data2');
await compression.execute.call(mockExecuteFunctions);
const extracted = await boundedUntar(outputArchive, 1024 * 1024, 100);
expect(Object.keys(extracted).sort()).toEqual(['a.txt', 'b.txt']);
});
});
@@ -4,10 +4,12 @@ import { NodeOperationError } from 'n8n-workflow';
import { Compression } from '../../Compression.node';
import { boundedGunzip } from '../../decompress/BoundedGunzip';
import { boundedUntar } from '../../decompress/BoundedUntar';
import { boundedUnzip } from '../../decompress/BoundedUnzip';
import type { Mocked } from 'vitest';
vi.mock('../../decompress/BoundedGunzip');
vi.mock('../../decompress/BoundedUntar');
vi.mock('../../decompress/BoundedUnzip');
const mockBoundedUnzip = (data: Record<string, Buffer>, error?: Error) => {
@@ -18,6 +20,14 @@ const mockBoundedUnzip = (data: Record<string, Buffer>, error?: Error) => {
}
};
const mockBoundedUntar = (data: Record<string, Buffer>, error?: Error) => {
if (error) {
vi.mocked(boundedUntar).mockRejectedValue(error);
} else {
vi.mocked(boundedUntar).mockResolvedValue(data);
}
};
const mockBoundedGunzip = (data: Buffer, error?: Error) => {
if (error) {
vi.mocked(boundedGunzip).mockRejectedValue(error);
@@ -338,7 +348,111 @@ describe('Compression Node - Decompress Operation', () => {
});
});
describe('Tar Decompression', () => {
const tarContents: Record<string, Buffer> = {
'a.txt': Buffer.from([72, 101, 108, 108, 111]),
'sub/b.txt': Buffer.from([87, 111, 114, 108, 100]),
};
const prepareBinaryDataPassthrough = () => {
vi.mocked(mockExecuteFunctions.helpers.prepareBinaryData).mockImplementation(
async (buffer, fileName) =>
({
data: buffer.toString('base64'),
mimeType: 'text/plain',
fileName: fileName ?? 'file',
fileExtension: 'txt',
}) as IBinaryData,
);
};
it('should decompress a tar file successfully', async () => {
const mockBinaryData: IBinaryData = {
data: 'base64data',
mimeType: 'application/x-tar',
fileName: 'test.tar',
fileExtension: 'tar',
};
vi.mocked(mockExecuteFunctions.helpers.assertBinaryData).mockReturnValue(mockBinaryData);
vi.mocked(mockExecuteFunctions.helpers.getBinaryDataBuffer).mockResolvedValue(
Buffer.from('mock tar data'),
);
mockBoundedUntar(tarContents);
prepareBinaryDataPassthrough();
const result = await compression.execute.call(mockExecuteFunctions);
expect(result[0][0].binary?.file_0).toBeDefined();
expect(result[0][0].binary?.file_1).toBeDefined();
expect(boundedUntar).toHaveBeenCalledTimes(1);
expect(boundedUnzip).not.toHaveBeenCalled();
});
it('should route a .tar.gz file to tar extraction, not gzip', async () => {
const mockBinaryData: IBinaryData = {
data: 'base64data',
mimeType: 'application/gzip',
fileName: 'archive.tar.gz',
fileExtension: 'gz',
};
vi.mocked(mockExecuteFunctions.helpers.assertBinaryData).mockReturnValue(mockBinaryData);
vi.mocked(mockExecuteFunctions.helpers.getBinaryDataBuffer).mockResolvedValue(
Buffer.from('mock targz data'),
);
mockBoundedUntar(tarContents);
prepareBinaryDataPassthrough();
const result = await compression.execute.call(mockExecuteFunctions);
expect(result[0][0].binary?.file_0).toBeDefined();
expect(boundedUntar).toHaveBeenCalledTimes(1);
expect(boundedGunzip).not.toHaveBeenCalled();
});
it('should decompress a .tgz file via tar extraction', async () => {
const mockBinaryData: IBinaryData = {
data: 'base64data',
mimeType: 'application/gzip',
fileName: 'archive.tgz',
fileExtension: 'tgz',
};
vi.mocked(mockExecuteFunctions.helpers.assertBinaryData).mockReturnValue(mockBinaryData);
vi.mocked(mockExecuteFunctions.helpers.getBinaryDataBuffer).mockResolvedValue(
Buffer.from('mock tgz data'),
);
mockBoundedUntar(tarContents);
prepareBinaryDataPassthrough();
const result = await compression.execute.call(mockExecuteFunctions);
expect(result[0][0].binary?.file_0).toBeDefined();
expect(boundedUntar).toHaveBeenCalledTimes(1);
expect(boundedGunzip).not.toHaveBeenCalled();
});
});
describe('Error Handling', () => {
it('should throw NodeOperationError for an unsupported archive format', async () => {
const mockBinaryData: IBinaryData = {
data: 'base64data',
mimeType: 'application/x-rar-compressed',
fileName: 'archive.rar',
fileExtension: 'rar',
};
vi.mocked(mockExecuteFunctions.helpers.assertBinaryData).mockReturnValue(mockBinaryData);
vi.mocked(mockExecuteFunctions.helpers.getBinaryDataBuffer).mockResolvedValue(
Buffer.from('mock rar data'),
);
await expect(compression.execute.call(mockExecuteFunctions)).rejects.toThrow(
NodeOperationError,
);
});
it('should throw error when file extension is not found', async () => {
const mockBinaryData: IBinaryData = {
data: 'base64data',
+1
View File
@@ -977,6 +977,7 @@
"simple-git": "catalog:",
"snowflake-sdk": "2.1.0",
"ssh2-sftp-client": "12.1.0",
"tar": "catalog:",
"tmp-promise": "3.0.3",
"ts-ics": "1.2.2",
"undici": "catalog:undici-v6",
+3
View File
@@ -5210,6 +5210,9 @@ importers:
ssh2-sftp-client:
specifier: 12.1.0
version: 12.1.0
tar:
specifier: ^7.5.16
version: 7.5.16
tmp-promise:
specifier: 3.0.3
version: 3.0.3
+1
View File
@@ -150,6 +150,7 @@ catalog:
simple-git: 3.36.0
source-map-support: 0.5.21
stream-json: 1.9.1
tar: ^7.5.16
testcontainers: ^11.13.0
ts-morph: ^27.0.2
tsdown: ^0.16.5