feat(Google Gemini Node): Ability to create a file store and upload files to it (#22988)

This commit is contained in:
Dawid Myslak
2025-12-15 13:59:20 +01:00
committed by GitHub
parent 2a6e49925e
commit c4d6502b05
13 changed files with 1061 additions and 36 deletions
@@ -12,9 +12,9 @@ export const description: INodeProperties[] = [
noDataExpression: true,
options: [
{
name: 'Upload File',
name: 'Upload Media File',
value: 'upload',
action: 'Upload a file',
action: 'Upload a media file',
description: 'Upload a file to the Google Gemini API for later use',
},
],
@@ -0,0 +1,39 @@
import type { IExecuteFunctions, INodeExecutionData, INodeProperties } from 'n8n-workflow';
import { updateDisplayOptions } from 'n8n-workflow';
import { createFileSearchStore } from '../../helpers/utils';
export const properties: INodeProperties[] = [
{
displayName: 'Display Name',
name: 'displayName',
type: 'string',
placeholder: 'e.g. My File Search Store',
description: 'A human-readable name for the File Search store',
default: '',
required: true,
},
];
const displayOptions = {
show: {
operation: ['createStore'],
resource: ['fileSearch'],
},
};
export const description = updateDisplayOptions(displayOptions, properties);
export async function execute(this: IExecuteFunctions, i: number): Promise<INodeExecutionData[]> {
const displayName = this.getNodeParameter('displayName', i, '') as string;
const response = await createFileSearchStore.call(this, displayName);
return [
{
json: response,
pairedItem: {
item: i,
},
},
];
}
@@ -0,0 +1,48 @@
import type { IExecuteFunctions, INodeExecutionData, INodeProperties } from 'n8n-workflow';
import { updateDisplayOptions } from 'n8n-workflow';
import { deleteFileSearchStore } from '../../helpers/utils';
export const properties: INodeProperties[] = [
{
displayName: 'File Search Store Name',
name: 'fileSearchStoreName',
type: 'string',
placeholder: 'e.g. fileSearchStores/abc123',
description: 'The full name of the File Search store to delete (format: fileSearchStores/...)',
default: '',
required: true,
},
{
displayName: 'Force Delete',
name: 'force',
type: 'boolean',
description:
'Whether to delete related Documents and objects. If false, deletion will fail if the store contains any Documents.',
default: false,
},
];
const displayOptions = {
show: {
operation: ['deleteStore'],
resource: ['fileSearch'],
},
};
export const description = updateDisplayOptions(displayOptions, properties);
export async function execute(this: IExecuteFunctions, i: number): Promise<INodeExecutionData[]> {
const fileSearchStoreName = this.getNodeParameter('fileSearchStoreName', i, '') as string;
const force = this.getNodeParameter('force', i, false) as boolean | undefined;
const response = await deleteFileSearchStore.call(this, fileSearchStoreName, force);
return [
{
json: response,
pairedItem: {
item: i,
},
},
];
}
@@ -0,0 +1,54 @@
import type { INodeProperties } from 'n8n-workflow';
import * as createStore from './createStore.operation';
import * as deleteStore from './deleteStore.operation';
import * as listStores from './listStores.operation';
import * as uploadToStore from './uploadToStore.operation';
export { createStore, deleteStore, listStores, uploadToStore };
export const description: INodeProperties[] = [
{
displayName: 'Operation',
name: 'operation',
type: 'options',
noDataExpression: true,
options: [
{
name: 'Create File Search Store',
value: 'createStore',
action: 'Create a File Search store',
description: 'Create a new File Search store for RAG (Retrieval Augmented Generation)',
},
{
name: 'Delete File Search Store',
value: 'deleteStore',
action: 'Delete a File Search store',
description: 'Delete a File Search store',
},
{
name: 'List File Search Stores',
value: 'listStores',
action: 'List all File Search stores',
description: 'List all File Search stores owned by the user',
},
{
name: 'Upload to File Search Store',
value: 'uploadToStore',
action: 'Upload a file to a File Search store',
description:
'Upload a file to a File Search store for RAG (Retrieval Augmented Generation)',
},
],
default: 'createStore',
displayOptions: {
show: {
resource: ['fileSearch'],
},
},
},
...createStore.description,
...deleteStore.description,
...listStores.description,
...uploadToStore.description,
];
@@ -0,0 +1,50 @@
import type { IExecuteFunctions, INodeExecutionData, INodeProperties } from 'n8n-workflow';
import { updateDisplayOptions } from 'n8n-workflow';
import { listFileSearchStores } from '../../helpers/utils';
export const properties: INodeProperties[] = [
{
displayName: 'Page Size',
name: 'pageSize',
type: 'number',
description: 'Maximum number of File Search stores to return per page (max 20)',
default: 10,
typeOptions: {
minValue: 1,
maxValue: 20,
},
},
{
displayName: 'Page Token',
name: 'pageToken',
// eslint-disable-next-line -- pageToken is a pagination token, not a password
type: 'string',
description: 'Token from a previous page to retrieve the next page of results',
default: '',
},
];
const displayOptions = {
show: {
operation: ['listStores'],
resource: ['fileSearch'],
},
};
export const description = updateDisplayOptions(displayOptions, properties);
export async function execute(this: IExecuteFunctions, i: number): Promise<INodeExecutionData[]> {
const pageSize = this.getNodeParameter('pageSize', i) as number | undefined;
const pageToken = this.getNodeParameter('pageToken', i, '') as string | undefined;
const response = await listFileSearchStores.call(this, pageSize, pageToken);
return [
{
json: response,
pairedItem: {
item: i,
},
},
];
}
@@ -0,0 +1,107 @@
import type { IExecuteFunctions, INodeExecutionData, INodeProperties } from 'n8n-workflow';
import { updateDisplayOptions } from 'n8n-workflow';
import { uploadToFileSearchStore } from '../../helpers/utils';
export const properties: INodeProperties[] = [
{
displayName: 'File Search Store Name',
name: 'fileSearchStoreName',
type: 'string',
placeholder: 'e.g. fileSearchStores/abc123',
description:
'The full name of the File Search store to upload to (format: fileSearchStores/...)',
default: '',
required: true,
},
{
displayName: 'File Display Name',
name: 'displayName',
type: 'string',
placeholder: 'e.g. My Document',
description: 'A human-readable name for the file (will be visible in citations)',
default: '',
required: true,
},
{
displayName: 'Input Type',
name: 'inputType',
type: 'options',
default: 'url',
options: [
{
name: 'File URL',
value: 'url',
},
{
name: 'Binary File',
value: 'binary',
},
],
},
{
displayName: 'URL',
name: 'fileUrl',
type: 'string',
placeholder: 'e.g. https://example.com/file.pdf',
description: 'URL of the file to upload',
default: '',
displayOptions: {
show: {
inputType: ['url'],
},
},
},
{
displayName: 'Input Data Field Name',
name: 'binaryPropertyName',
type: 'string',
default: 'data',
placeholder: 'e.g. data',
hint: 'The name of the input field containing the binary file data to be processed',
description: 'Name of the binary property which contains the file',
displayOptions: {
show: {
inputType: ['binary'],
},
},
},
];
const displayOptions = {
show: {
operation: ['uploadToStore'],
resource: ['fileSearch'],
},
};
export const description = updateDisplayOptions(displayOptions, properties);
export async function execute(this: IExecuteFunctions, i: number): Promise<INodeExecutionData[]> {
const fileSearchStoreName = this.getNodeParameter('fileSearchStoreName', i, '') as string;
const displayName = this.getNodeParameter('displayName', i, '') as string;
const inputType = this.getNodeParameter('inputType', i, 'url') as string;
let fileUrl: string | undefined;
if (inputType === 'url') {
fileUrl = this.getNodeParameter('fileUrl', i, '') as string;
}
const response = await uploadToFileSearchStore.call(
this,
i,
fileSearchStoreName,
displayName,
fileUrl,
'application/octet-stream',
);
return [
{
json: response ?? {},
pairedItem: {
item: i,
},
},
];
}
@@ -7,6 +7,7 @@ type NodeMap = {
audio: 'transcribe' | 'analyze';
document: 'analyze';
file: 'upload';
fileSearch: 'createStore' | 'deleteStore' | 'listStores' | 'uploadToStore';
};
export type GoogleGeminiType = AllEntities<NodeMap>;
@@ -4,6 +4,7 @@ import type { IExecuteFunctions } from 'n8n-workflow';
import * as audio from './audio';
import * as document from './document';
import * as file from './file';
import * as fileSearch from './fileSearch';
import * as image from './image';
import { router } from './router';
import * as text from './text';
@@ -14,6 +15,10 @@ describe('Google Gemini router', () => {
const mockAudio = jest.spyOn(audio.analyze, 'execute');
const mockDocument = jest.spyOn(document.analyze, 'execute');
const mockFile = jest.spyOn(file.upload, 'execute');
const mockFileSearchCreateStore = jest.spyOn(fileSearch.createStore, 'execute');
const mockFileSearchDeleteStore = jest.spyOn(fileSearch.deleteStore, 'execute');
const mockFileSearchListStores = jest.spyOn(fileSearch.listStores, 'execute');
const mockFileSearchUploadToStore = jest.spyOn(fileSearch.uploadToStore, 'execute');
const mockImage = jest.spyOn(image.analyze, 'execute');
const mockText = jest.spyOn(text.message, 'execute');
const mockVideo = jest.spyOn(video.analyze, 'execute');
@@ -21,6 +26,10 @@ describe('Google Gemini router', () => {
[mockAudio, 'audio', 'analyze'],
[mockDocument, 'document', 'analyze'],
[mockFile, 'file', 'upload'],
[mockFileSearchCreateStore, 'fileSearch', 'createStore'],
[mockFileSearchDeleteStore, 'fileSearch', 'deleteStore'],
[mockFileSearchListStores, 'fileSearch', 'listStores'],
[mockFileSearchUploadToStore, 'fileSearch', 'uploadToStore'],
[mockImage, 'image', 'analyze'],
[mockText, 'text', 'message'],
[mockVideo, 'video', 'analyze'],
@@ -3,6 +3,7 @@ import { NodeOperationError, type IExecuteFunctions, type INodeExecutionData } f
import * as audio from './audio';
import * as document from './document';
import * as file from './file';
import * as fileSearch from './fileSearch';
import * as image from './image';
import type { GoogleGeminiType } from './node.type';
import * as text from './text';
@@ -31,6 +32,9 @@ export async function router(this: IExecuteFunctions) {
case 'file':
execute = file[googleGeminiTypeData.operation].execute;
break;
case 'fileSearch':
execute = fileSearch[googleGeminiTypeData.operation].execute;
break;
case 'image':
execute = image[googleGeminiTypeData.operation].execute;
break;
@@ -4,6 +4,7 @@ import { NodeConnectionTypes, type INodeTypeDescription } from 'n8n-workflow';
import * as audio from './audio';
import * as document from './document';
import * as file from './file';
import * as fileSearch from './fileSearch';
import * as image from './image';
import * as text from './text';
import * as video from './video';
@@ -69,13 +70,17 @@ export const versionDescription: INodeTypeDescription = {
value: 'document',
},
{
name: 'File',
value: 'file',
name: 'File Search',
value: 'fileSearch',
},
{
name: 'Image',
value: 'image',
},
{
name: 'Media File',
value: 'file',
},
{
name: 'Text',
value: 'text',
@@ -90,6 +95,7 @@ export const versionDescription: INodeTypeDescription = {
...audio.description,
...document.description,
...file.description,
...fileSearch.description,
...image.description,
...text.description,
...video.description,
@@ -104,6 +104,17 @@ export interface VeoResponse {
};
}
/**
* File Search operation interface for long-running upload operations
* Based on: https://ai.google.dev/api/file-search/file-search-stores#method:-media.uploadtofilesearchstore
*/
export interface FileSearchOperation {
name: string;
done: boolean;
error?: { message: string };
response?: IDataObject;
}
/**
* User configuration for built-in tools in the node parameters
*/
@@ -3,7 +3,15 @@ import { mockDeep } from 'jest-mock-extended';
import type { IBinaryData, IExecuteFunctions } from 'n8n-workflow';
import { NodeOperationError } from 'n8n-workflow';
import { downloadFile, uploadFile, transferFile } from './utils';
import {
createFileSearchStore,
deleteFileSearchStore,
downloadFile,
listFileSearchStores,
transferFile,
uploadFile,
uploadToFileSearchStore,
} from './utils';
import * as transport from '../transport';
jest.mock('axios');
@@ -436,9 +444,13 @@ describe('GoogleGemini -> utils', () => {
await expect(
transferFile.call(mockExecuteFunctions, 0, undefined, 'application/octet-stream'),
).rejects.toThrow(
new NodeOperationError(mockExecuteFunctions.getNode(), 'Binary property name is required', {
description: 'Error uploading file',
}),
new NodeOperationError(
mockExecuteFunctions.getNode(),
'Binary property name or download URL is required',
{
description: 'Error uploading file',
},
),
);
});
@@ -532,4 +544,528 @@ describe('GoogleGemini -> utils', () => {
);
});
});
describe('createFileSearchStore', () => {
it('should create a file search store', async () => {
const displayName = 'My File Search Store';
const mockResponse = {
name: 'fileSearchStores/abc123',
displayName: 'My File Search Store',
};
apiRequestMock.mockResolvedValue(mockResponse);
const result = await createFileSearchStore.call(mockExecuteFunctions, displayName);
expect(result).toEqual(mockResponse);
expect(apiRequestMock).toHaveBeenCalledWith('POST', '/v1beta/fileSearchStores', {
body: { displayName },
});
});
});
describe('uploadToFileSearchStore', () => {
it('should upload file from URL to file search store', async () => {
const fileSearchStoreName = 'fileSearchStores/abc123';
const displayName = 'test-file.pdf';
const mockStream = {
pipe: jest.fn(),
on: jest.fn(),
} as any;
mockedAxios.get.mockResolvedValue({
data: mockStream,
headers: {
'content-type': 'application/pdf; charset=utf-8',
},
});
apiRequestMock
.mockResolvedValueOnce({
headers: {
'x-goog-upload-url': 'https://upload.googleapis.com/upload/123',
},
})
.mockResolvedValueOnce({
name: 'operations/op123',
done: false,
})
.mockResolvedValueOnce({
name: 'operations/op123',
done: true,
response: {
name: 'fileSearchStores/abc123/files/file123',
},
});
mockExecuteFunctions.helpers.httpRequest.mockResolvedValueOnce({
body: {
name: 'operations/op123',
},
});
jest.spyOn(global, 'setTimeout').mockImplementation((callback: any) => {
callback();
return {} as any;
});
const result = await uploadToFileSearchStore.call(
mockExecuteFunctions,
0,
fileSearchStoreName,
displayName,
'https://example.com/file.pdf',
);
expect(result).toEqual({
name: 'fileSearchStores/abc123/files/file123',
});
expect(mockedAxios.get).toHaveBeenCalledWith('https://example.com/file.pdf', {
params: undefined,
responseType: 'stream',
});
expect(apiRequestMock).toHaveBeenCalledWith(
'POST',
`/upload/v1beta/${fileSearchStoreName}:uploadToFileSearchStore`,
{
headers: {
'X-Goog-Upload-Protocol': 'resumable',
'X-Goog-Upload-Command': 'start',
'X-Goog-Upload-Header-Content-Type': 'application/pdf',
'Content-Type': 'application/json',
},
body: { displayName, mimeType: 'application/pdf' },
option: { returnFullResponse: true },
},
);
expect(apiRequestMock).toHaveBeenCalledWith('GET', '/v1beta/operations/op123');
});
it('should upload file from binary data (buffer) to file search store', async () => {
const fileSearchStoreName = 'fileSearchStores/abc123';
const displayName = 'test-file.pdf';
const mockBinaryData: IBinaryData = {
mimeType: 'application/pdf',
fileName: 'test.pdf',
fileSize: '1024',
fileExtension: 'pdf',
data: 'test',
};
mockExecuteFunctions.getNodeParameter.mockReturnValue('data');
mockExecuteFunctions.helpers.assertBinaryData.mockReturnValue(mockBinaryData);
mockExecuteFunctions.helpers.getBinaryDataBuffer.mockResolvedValue(Buffer.from('test'));
apiRequestMock
.mockResolvedValueOnce({
headers: {
'x-goog-upload-url': 'https://upload.googleapis.com/upload/123',
},
})
.mockResolvedValueOnce({
name: 'operations/op123',
done: true,
response: {
name: 'fileSearchStores/abc123/files/file123',
},
});
mockExecuteFunctions.helpers.httpRequest.mockResolvedValueOnce({
body: {
name: 'operations/op123',
},
});
const result = await uploadToFileSearchStore.call(
mockExecuteFunctions,
0,
fileSearchStoreName,
displayName,
);
expect(result).toEqual({
name: 'fileSearchStores/abc123/files/file123',
});
expect(mockExecuteFunctions.helpers.assertBinaryData).toHaveBeenCalledWith(0, 'data');
expect(mockExecuteFunctions.helpers.getBinaryDataBuffer).toHaveBeenCalledWith(0, 'data');
});
it('should upload file from binary data (stream) to file search store', async () => {
const fileSearchStoreName = 'fileSearchStores/abc123';
const displayName = 'test-file.pdf';
const mockBinaryData: IBinaryData = {
id: 'binary-123',
mimeType: 'application/pdf',
fileName: 'test.pdf',
fileSize: '1024',
fileExtension: 'pdf',
data: 'test',
};
const mockStream = {
pipe: jest.fn(),
on: jest.fn(),
} as any;
mockExecuteFunctions.getNodeParameter.mockReturnValue('data');
mockExecuteFunctions.helpers.assertBinaryData.mockReturnValue(mockBinaryData);
mockExecuteFunctions.helpers.getBinaryStream.mockResolvedValue(mockStream);
apiRequestMock
.mockResolvedValueOnce({
headers: {
'x-goog-upload-url': 'https://upload.googleapis.com/upload/123',
},
})
.mockResolvedValueOnce({
name: 'operations/op123',
done: true,
response: {
name: 'fileSearchStores/abc123/files/file123',
},
});
mockExecuteFunctions.helpers.httpRequest.mockResolvedValueOnce({
body: {
name: 'operations/op123',
},
});
const result = await uploadToFileSearchStore.call(
mockExecuteFunctions,
0,
fileSearchStoreName,
displayName,
);
expect(result).toEqual({
name: 'fileSearchStores/abc123/files/file123',
});
expect(mockExecuteFunctions.helpers.getBinaryStream).toHaveBeenCalledWith(
'binary-123',
262144,
);
});
it('should poll operation until done', async () => {
const fileSearchStoreName = 'fileSearchStores/abc123';
const displayName = 'test-file.pdf';
const mockStream = {
pipe: jest.fn(),
on: jest.fn(),
} as any;
mockedAxios.get.mockResolvedValue({
data: mockStream,
headers: {
'content-type': 'application/pdf',
},
});
apiRequestMock
.mockResolvedValueOnce({
headers: {
'x-goog-upload-url': 'https://upload.googleapis.com/upload/123',
},
})
.mockResolvedValueOnce({
name: 'operations/op123',
done: false,
})
.mockResolvedValueOnce({
name: 'operations/op123',
done: false,
})
.mockResolvedValueOnce({
name: 'operations/op123',
done: true,
response: {
name: 'fileSearchStores/abc123/files/file123',
},
});
mockExecuteFunctions.helpers.httpRequest.mockResolvedValueOnce({
body: {
name: 'operations/op123',
},
});
jest.spyOn(global, 'setTimeout').mockImplementation((callback: any) => {
callback();
return {} as any;
});
const result = await uploadToFileSearchStore.call(
mockExecuteFunctions,
0,
fileSearchStoreName,
displayName,
'https://example.com/file.pdf',
);
expect(result).toEqual({
name: 'fileSearchStores/abc123/files/file123',
});
expect(apiRequestMock).toHaveBeenCalledTimes(4); // 1 upload init + 3 operation polls
});
it('should throw error when operation fails', async () => {
const fileSearchStoreName = 'fileSearchStores/abc123';
const displayName = 'test-file.pdf';
const mockStream = {
pipe: jest.fn(),
on: jest.fn(),
} as any;
mockedAxios.get.mockResolvedValue({
data: mockStream,
headers: {
'content-type': 'application/pdf',
},
});
apiRequestMock
.mockResolvedValueOnce({
headers: {
'x-goog-upload-url': 'https://upload.googleapis.com/upload/123',
},
})
.mockResolvedValueOnce({
name: 'operations/op123',
done: false,
})
.mockResolvedValueOnce({
name: 'operations/op123',
done: true,
error: { message: 'Upload failed' },
});
mockExecuteFunctions.helpers.httpRequest.mockResolvedValueOnce({
body: {
name: 'operations/op123',
},
});
jest.spyOn(global, 'setTimeout').mockImplementation((callback: any) => {
callback();
return {} as any;
});
mockExecuteFunctions.getNode.mockReturnValue({ name: 'Google Gemini' } as any);
await expect(
uploadToFileSearchStore.call(
mockExecuteFunctions,
0,
fileSearchStoreName,
displayName,
'https://example.com/file.pdf',
),
).rejects.toThrow(
new NodeOperationError(mockExecuteFunctions.getNode(), 'Upload failed', {
description: 'Error uploading file to File Search store',
}),
);
});
it('should throw error when binary property name is missing', async () => {
const fileSearchStoreName = 'fileSearchStores/abc123';
const displayName = 'test-file.pdf';
mockExecuteFunctions.getNodeParameter.mockReturnValue('');
mockExecuteFunctions.getNode.mockReturnValue({ name: 'Google Gemini' } as any);
await expect(
uploadToFileSearchStore.call(mockExecuteFunctions, 0, fileSearchStoreName, displayName),
).rejects.toThrow(
new NodeOperationError(
mockExecuteFunctions.getNode(),
'Binary property name or download URL is required',
{
description: 'Error uploading file',
},
),
);
});
it('should return undefined when response is missing', async () => {
const fileSearchStoreName = 'fileSearchStores/abc123';
const displayName = 'test-file.pdf';
const mockStream = {
pipe: jest.fn(),
on: jest.fn(),
} as any;
mockedAxios.get.mockResolvedValue({
data: mockStream,
headers: {
'content-type': 'application/pdf',
},
});
apiRequestMock
.mockResolvedValueOnce({
headers: {
'x-goog-upload-url': 'https://upload.googleapis.com/upload/123',
},
})
.mockResolvedValueOnce({
name: 'operations/op123',
done: true,
});
mockExecuteFunctions.helpers.httpRequest.mockResolvedValueOnce({
body: {
name: 'operations/op123',
},
});
const result = await uploadToFileSearchStore.call(
mockExecuteFunctions,
0,
fileSearchStoreName,
displayName,
'https://example.com/file.pdf',
);
expect(result).toBeUndefined();
});
});
describe('listFileSearchStores', () => {
it('should list file search stores without pagination', async () => {
const mockResponse = {
fileSearchStores: [
{
name: 'fileSearchStores/store1',
displayName: 'Store 1',
},
{
name: 'fileSearchStores/store2',
displayName: 'Store 2',
},
],
};
apiRequestMock.mockResolvedValue(mockResponse);
const result = await listFileSearchStores.call(mockExecuteFunctions);
expect(result).toEqual(mockResponse);
expect(apiRequestMock).toHaveBeenCalledWith('GET', '/v1beta/fileSearchStores', {
qs: {},
});
});
it('should list file search stores with pageSize', async () => {
const mockResponse = {
fileSearchStores: [
{
name: 'fileSearchStores/store1',
displayName: 'Store 1',
},
],
};
apiRequestMock.mockResolvedValue(mockResponse);
const result = await listFileSearchStores.call(mockExecuteFunctions, 20);
expect(result).toEqual(mockResponse);
expect(apiRequestMock).toHaveBeenCalledWith('GET', '/v1beta/fileSearchStores', {
qs: { pageSize: 20 },
});
});
it('should list file search stores with pageToken', async () => {
const mockResponse = {
fileSearchStores: [
{
name: 'fileSearchStores/store3',
displayName: 'Store 3',
},
],
nextPageToken: 'token123',
};
apiRequestMock.mockResolvedValue(mockResponse);
const result = await listFileSearchStores.call(mockExecuteFunctions, undefined, 'token123');
expect(result).toEqual(mockResponse);
expect(apiRequestMock).toHaveBeenCalledWith('GET', '/v1beta/fileSearchStores', {
qs: { pageToken: 'token123' },
});
});
it('should list file search stores with both pageSize and pageToken', async () => {
const mockResponse = {
fileSearchStores: [
{
name: 'fileSearchStores/store1',
displayName: 'Store 1',
},
],
};
apiRequestMock.mockResolvedValue(mockResponse);
const result = await listFileSearchStores.call(mockExecuteFunctions, 10, 'token123');
expect(result).toEqual(mockResponse);
expect(apiRequestMock).toHaveBeenCalledWith('GET', '/v1beta/fileSearchStores', {
qs: { pageSize: 10, pageToken: 'token123' },
});
});
});
describe('deleteFileSearchStore', () => {
it('should delete file search store without force', async () => {
const name = 'fileSearchStores/abc123';
const mockResponse = {};
apiRequestMock.mockResolvedValue(mockResponse);
const result = await deleteFileSearchStore.call(mockExecuteFunctions, name);
expect(result).toEqual(mockResponse);
expect(apiRequestMock).toHaveBeenCalledWith('DELETE', `/v1beta/${name}`, {
qs: {},
});
});
it('should delete file search store with force', async () => {
const name = 'fileSearchStores/abc123';
const mockResponse = {};
apiRequestMock.mockResolvedValue(mockResponse);
const result = await deleteFileSearchStore.call(mockExecuteFunctions, name, true);
expect(result).toEqual(mockResponse);
expect(apiRequestMock).toHaveBeenCalledWith('DELETE', `/v1beta/${name}`, {
qs: { force: true },
});
});
it('should delete file search store with force false', async () => {
const name = 'fileSearchStores/abc123';
const mockResponse = {};
apiRequestMock.mockResolvedValue(mockResponse);
const result = await deleteFileSearchStore.call(mockExecuteFunctions, name, false);
expect(result).toEqual(mockResponse);
expect(apiRequestMock).toHaveBeenCalledWith('DELETE', `/v1beta/${name}`, {
qs: { force: false },
});
});
});
});
@@ -1,10 +1,13 @@
import axios from 'axios';
import type { IDataObject, IExecuteFunctions } from 'n8n-workflow';
import { NodeOperationError } from 'n8n-workflow';
import { Readable } from 'node:stream';
import type Stream from 'node:stream';
import type { FileSearchOperation } from './interfaces';
import { apiRequest } from '../transport';
import axios from 'axios';
import type Stream from 'node:stream';
const OPERATION_CHECK_INTERVAL = 1000;
interface File {
name: string;
@@ -14,6 +17,22 @@ interface File {
error?: { message: string };
}
interface FileStreamData {
stream: Stream;
mimeType: string;
}
interface FileBufferData {
buffer: Buffer;
mimeType: string;
}
interface UploadStreamConfig {
endpoint: string;
mimeType: string;
body?: IDataObject;
}
const CHUNK_SIZE = 256 * 1024;
export async function downloadFile(
@@ -67,7 +86,7 @@ export async function uploadFile(this: IExecuteFunctions, fileContent: Buffer, m
})) as { file: File };
while (uploadResponse.file.state !== 'ACTIVE' && uploadResponse.file.state !== 'FAILED') {
await new Promise((resolve) => setTimeout(resolve, 1000));
await new Promise((resolve) => setTimeout(resolve, OPERATION_CHECK_INTERVAL));
uploadResponse.file = (await apiRequest.call(
this,
'GET',
@@ -88,48 +107,69 @@ export async function uploadFile(this: IExecuteFunctions, fileContent: Buffer, m
return { fileUri: uploadResponse.file.uri, mimeType: uploadResponse.file.mimeType };
}
export async function transferFile(
async function getFileStreamFromUrlOrBinary(
this: IExecuteFunctions,
i: number,
downloadUrl?: string,
fallbackMimeType?: string,
qs?: IDataObject,
) {
let stream: Stream;
let mimeType: string;
): Promise<FileStreamData | FileBufferData> {
if (downloadUrl) {
const downloadResponse = await axios.get(downloadUrl, {
params: qs,
responseType: 'stream',
});
mimeType = downloadResponse.headers['content-type']?.split(';')?.[0] ?? fallbackMimeType;
stream = downloadResponse.data;
} else {
const binaryPropertyName = this.getNodeParameter('binaryPropertyName', i, 'data');
if (!binaryPropertyName) {
throw new NodeOperationError(this.getNode(), 'Binary property name is required', {
description: 'Error uploading file',
});
}
const binaryData = this.helpers.assertBinaryData(i, binaryPropertyName);
if (!binaryData.id) {
const buffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName);
return await uploadFile.call(this, buffer, binaryData.mimeType);
} else {
stream = await this.helpers.getBinaryStream(binaryData.id, CHUNK_SIZE);
mimeType = binaryData.mimeType;
}
const contentType = downloadResponse.headers['content-type'] as string | undefined;
const mimeType = contentType?.split(';')?.[0] ?? fallbackMimeType ?? 'application/octet-stream';
return {
stream: downloadResponse.data as Stream,
mimeType,
};
}
const uploadInitResponse = (await apiRequest.call(this, 'POST', '/upload/v1beta/files', {
const binaryPropertyName = this.getNodeParameter('binaryPropertyName', i, 'data');
if (!binaryPropertyName) {
throw new NodeOperationError(
this.getNode(),
'Binary property name or download URL is required',
{
description: 'Error uploading file',
},
);
}
const binaryData = this.helpers.assertBinaryData(i, binaryPropertyName);
if (!binaryData.id) {
const buffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName);
return {
buffer,
mimeType: binaryData.mimeType,
};
}
return {
stream: await this.helpers.getBinaryStream(binaryData.id, CHUNK_SIZE),
mimeType: binaryData.mimeType,
};
}
async function uploadStream(
this: IExecuteFunctions,
stream: Stream,
config: UploadStreamConfig,
): Promise<{ body: IDataObject }> {
const { endpoint, mimeType, body } = config;
const uploadInitResponse = (await apiRequest.call(this, 'POST', endpoint, {
headers: {
'X-Goog-Upload-Protocol': 'resumable',
'X-Goog-Upload-Command': 'start',
'X-Goog-Upload-Header-Content-Type': mimeType,
'Content-Type': 'application/json',
},
body,
option: { returnFullResponse: true },
})) as { headers: IDataObject };
@@ -138,7 +178,7 @@ export async function transferFile(
throw new NodeOperationError(this.getNode(), 'Failed to get upload URL');
}
const uploadResponse = (await this.helpers.httpRequest({
return (await this.helpers.httpRequest({
method: 'POST',
url: uploadUrl,
headers: {
@@ -148,12 +188,38 @@ export async function transferFile(
},
body: stream,
returnFullResponse: true,
})) as { body: IDataObject };
}
export async function transferFile(
this: IExecuteFunctions,
i: number,
downloadUrl?: string,
fallbackMimeType?: string,
qs?: IDataObject,
) {
const fileData = await getFileStreamFromUrlOrBinary.call(
this,
i,
downloadUrl,
fallbackMimeType,
qs,
);
if ('buffer' in fileData) {
return await uploadFile.call(this, fileData.buffer, fileData.mimeType);
}
const { stream, mimeType } = fileData;
const uploadResponse = (await uploadStream.call(this, stream, {
endpoint: '/upload/v1beta/files',
mimeType,
})) as { body: { file: File } };
let file = uploadResponse.body.file;
while (file.state !== 'ACTIVE' && file.state !== 'FAILED') {
await new Promise((resolve) => setTimeout(resolve, 1000));
await new Promise((resolve) => setTimeout(resolve, OPERATION_CHECK_INTERVAL));
file = (await apiRequest.call(this, 'GET', `/v1beta/${file.name}`)) as File;
}
@@ -165,3 +231,97 @@ export async function transferFile(
return { fileUri: file.uri, mimeType: file.mimeType };
}
export async function createFileSearchStore(this: IExecuteFunctions, displayName: string) {
return (await apiRequest.call(this, 'POST', '/v1beta/fileSearchStores', {
body: { displayName },
})) as IDataObject;
}
export async function uploadToFileSearchStore(
this: IExecuteFunctions,
i: number,
fileSearchStoreName: string,
displayName: string,
downloadUrl?: string,
fallbackMimeType?: string,
qs?: IDataObject,
) {
const fileData = await getFileStreamFromUrlOrBinary.call(
this,
i,
downloadUrl,
fallbackMimeType,
qs,
);
let stream: Stream;
let mimeType: string;
if ('buffer' in fileData) {
stream = Readable.from(fileData.buffer);
mimeType = fileData.mimeType;
} else {
stream = fileData.stream;
mimeType = fileData.mimeType;
}
const uploadResponse = (await uploadStream.call(this, stream, {
endpoint: `/upload/v1beta/${fileSearchStoreName}:uploadToFileSearchStore`,
mimeType,
body: { displayName, mimeType },
})) as { body: { name: string } };
const operationName = uploadResponse.body.name;
let operation = (await apiRequest.call(
this,
'GET',
`/v1beta/${operationName}`,
)) as FileSearchOperation;
while (!operation.done) {
await new Promise((resolve) => setTimeout(resolve, OPERATION_CHECK_INTERVAL));
operation = (await apiRequest.call(
this,
'GET',
`/v1beta/${operationName}`,
)) as FileSearchOperation;
}
if (operation.error) {
throw new NodeOperationError(this.getNode(), operation.error.message ?? 'Unknown error', {
description: 'Error uploading file to File Search store',
});
}
return operation.response;
}
export async function listFileSearchStores(
this: IExecuteFunctions,
pageSize?: number,
pageToken?: string,
) {
const qs: IDataObject = {};
if (pageSize !== undefined) {
qs.pageSize = pageSize;
}
if (pageToken) {
qs.pageToken = pageToken;
}
return (await apiRequest.call(this, 'GET', '/v1beta/fileSearchStores', { qs })) as IDataObject;
}
export async function deleteFileSearchStore(
this: IExecuteFunctions,
name: string,
force?: boolean,
) {
const qs: IDataObject = {};
if (force !== undefined) {
qs.force = force;
}
return (await apiRequest.call(this, 'DELETE', `/v1beta/${name}`, { qs })) as IDataObject;
}