diff --git a/docs/generated/postgres-schema/README.md b/docs/generated/postgres-schema/README.md index 88d31b873e4..7542cba72b1 100644 --- a/docs/generated/postgres-schema/README.md +++ b/docs/generated/postgres-schema/README.md @@ -16,7 +16,7 @@ Auto-generated from the PostgreSQL migrations in @n8n/db. Do not edit by hand. | [public.agent_eval_run](public.agent_eval_run.md) | 14 | | BASE TABLE | | [public.agent_execution](public.agent_execution.md) | 19 | | BASE TABLE | | [public.agent_execution_threads](public.agent_execution_threads.md) | 17 | | BASE TABLE | -| [public.agent_files](public.agent_files.md) | 8 | | BASE TABLE | +| [public.agent_files](public.agent_files.md) | 10 | | BASE TABLE | | [public.agent_history](public.agent_history.md) | 9 | | BASE TABLE | | [public.agent_task_definition](public.agent_task_definition.md) | 7 | | BASE TABLE | | [public.agent_task_run_lock](public.agent_task_run_lock.md) | 6 | | BASE TABLE | @@ -447,6 +447,8 @@ erDiagram integer fileSizeBytes varchar_16_ id varchar_255_ mimeType + text storageKey + varchar_2_ storedAt timestamp_3__with_time_zone updatedAt } "public.agent_history" { diff --git a/docs/generated/postgres-schema/public.agent_files.md b/docs/generated/postgres-schema/public.agent_files.md index c6610397810..0d9c8c55674 100644 --- a/docs/generated/postgres-schema/public.agent_files.md +++ b/docs/generated/postgres-schema/public.agent_files.md @@ -5,27 +5,31 @@ | Name | Type | Default | Nullable | Children | Parents | Comment | | ---- | ---- | ------- | -------- | -------- | ------- | ------- | | agentId | varchar(36) | | false | | [public.agents](public.agents.md) | Agent that owns this uploaded file | -| binaryDataId | text | | false | | | Opaque BinaryDataService reference (mode-prefixed, e.g. "filesystem-v2:\"); not an FK to binary_data, which only has rows in DB storage mode | +| binaryDataId | text | | true | | | Opaque BinaryDataService reference (mode-prefixed, e.g. "filesystem-v2:\"); not an FK to binary_data, which only has rows in DB storage mode | | createdAt | timestamp(3) with time zone | CURRENT_TIMESTAMP(3) | false | | | | | fileName | varchar(255) | | false | | | | | fileSizeBytes | integer | | false | | | Uploaded file size in bytes | | id | varchar(16) | | false | | | Application-generated n8n nano ID | | mimeType | varchar(255) | | false | | | | +| storageKey | text | | false | | | Key addressing the bytes within storedAt: a binary_data.fileId for db, a byte-store key otherwise. Not a foreign key | +| storedAt | varchar(2) | 'db'::character varying | false | | | Where the file bytes live: 'db' (binary_data table), or a blob-storage backend ('fs', 's3', 'az') | | updatedAt | timestamp(3) with time zone | CURRENT_TIMESTAMP(3) | false | | | | ## Constraints | Name | Type | Definition | | ---- | ---- | ---------- | +| CHK_agent_files_storedAt | CHECK | CHECK ((("storedAt")::text = ANY ((ARRAY['db'::character varying, 'fs'::character varying, 's3'::character varying, 'az'::character varying])::text[]))) | | FK_aca4514cb500494b64356c2e164 | FOREIGN KEY | FOREIGN KEY ("agentId") REFERENCES agents(id) ON DELETE CASCADE | | PK_692920e59217af7d124cd95106f | PRIMARY KEY | PRIMARY KEY (id) | | agent_files_agentId_not_null | n | NOT NULL "agentId" | -| agent_files_binaryDataId_not_null | n | NOT NULL "binaryDataId" | | agent_files_createdAt_not_null | n | NOT NULL "createdAt" | | agent_files_fileName_not_null | n | NOT NULL "fileName" | | agent_files_fileSizeBytes_not_null | n | NOT NULL "fileSizeBytes" | | agent_files_id_not_null | n | NOT NULL id | | agent_files_mimeType_not_null | n | NOT NULL "mimeType" | +| agent_files_storageKey_not_null | n | NOT NULL "storageKey" | +| agent_files_storedAt_not_null | n | NOT NULL "storedAt" | | agent_files_updatedAt_not_null | n | NOT NULL "updatedAt" | ## Indexes @@ -52,6 +56,8 @@ erDiagram integer fileSizeBytes varchar_16_ id varchar_255_ mimeType + text storageKey + varchar_2_ storedAt timestamp_3__with_time_zone updatedAt } "public.agents" { diff --git a/docs/generated/postgres-schema/public.agents.md b/docs/generated/postgres-schema/public.agents.md index 957edeb1c18..3e49fbeefad 100644 --- a/docs/generated/postgres-schema/public.agents.md +++ b/docs/generated/postgres-schema/public.agents.md @@ -145,6 +145,8 @@ erDiagram integer fileSizeBytes varchar_16_ id varchar_255_ mimeType + text storageKey + varchar_2_ storedAt timestamp_3__with_time_zone updatedAt } "public.agent_task_definition" { diff --git a/docs/generated/sqlite-schema/README.md b/docs/generated/sqlite-schema/README.md index 056cd791652..2a472176eed 100644 --- a/docs/generated/sqlite-schema/README.md +++ b/docs/generated/sqlite-schema/README.md @@ -16,7 +16,7 @@ Auto-generated from the SQLite migrations in @n8n/db. Do not edit by hand. | [agent_eval_run](agent_eval_run.md) | 14 | | table | | [agent_execution](agent_execution.md) | 19 | | table | | [agent_execution_threads](agent_execution_threads.md) | 17 | | table | -| [agent_files](agent_files.md) | 8 | | table | +| [agent_files](agent_files.md) | 10 | | table | | [agent_history](agent_history.md) | 9 | | table | | [agent_task_definition](agent_task_definition.md) | 7 | | table | | [agent_task_run_lock](agent_task_run_lock.md) | 6 | | table | @@ -434,6 +434,8 @@ erDiagram INTEGER fileSizeBytes varchar_16_ id PK varchar_255_ mimeType + TEXT storageKey + varchar_2_ storedAt datetime_3_ updatedAt } "agent_history" { diff --git a/docs/generated/sqlite-schema/agent_files.md b/docs/generated/sqlite-schema/agent_files.md index cb67a888cec..2b8d767063b 100644 --- a/docs/generated/sqlite-schema/agent_files.md +++ b/docs/generated/sqlite-schema/agent_files.md @@ -6,7 +6,7 @@ Table Definition ```sql -CREATE TABLE "agent_files" ("id" varchar(16) PRIMARY KEY NOT NULL, "agentId" varchar(36) NOT NULL, "binaryDataId" text NOT NULL, "fileName" varchar(255) NOT NULL, "mimeType" varchar(255) NOT NULL, "fileSizeBytes" integer NOT NULL, "createdAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), "updatedAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), CONSTRAINT "FK_aca4514cb500494b64356c2e164" FOREIGN KEY ("agentId") REFERENCES "agents" ("id") ON DELETE CASCADE) +CREATE TABLE "agent_files" ("id" varchar(16) PRIMARY KEY NOT NULL, "agentId" varchar(36) NOT NULL, "binaryDataId" text, "fileName" varchar(255) NOT NULL, "mimeType" varchar(255) NOT NULL, "fileSizeBytes" integer NOT NULL, "createdAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), "updatedAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), "storedAt" varchar(2) NOT NULL DEFAULT ('db'), "storageKey" text NOT NULL, CONSTRAINT "CHK_agent_files_storedAt" CHECK ((("storedAt" IN ('db', 'fs', 's3', 'az')))), CONSTRAINT "FK_aca4514cb500494b64356c2e164" FOREIGN KEY ("agentId") REFERENCES "agents" ("id") ON DELETE CASCADE ON UPDATE NO ACTION) ``` @@ -16,18 +16,21 @@ CREATE TABLE "agent_files" ("id" varchar(16) PRIMARY KEY NOT NULL, "agentId" var | Name | Type | Default | Nullable | Children | Parents | Comment | | ---- | ---- | ------- | -------- | -------- | ------- | ------- | | agentId | varchar(36) | | false | | [agents](agents.md) | | -| binaryDataId | TEXT | | false | | | | +| binaryDataId | TEXT | | true | | | | | createdAt | datetime(3) | STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') | false | | | | | fileName | varchar(255) | | false | | | | | fileSizeBytes | INTEGER | | false | | | | | id | varchar(16) | | false | | | | | mimeType | varchar(255) | | false | | | | +| storageKey | TEXT | | false | | | | +| storedAt | varchar(2) | 'db' | false | | | | | updatedAt | datetime(3) | STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') | false | | | | ## Constraints | Name | Type | Definition | | ---- | ---- | ---------- | +| - | CHECK | CHECK ((("storedAt" IN ('db', 'fs', 's3', 'az')))) | | - (Foreign key ID: 0) | FOREIGN KEY | FOREIGN KEY (agentId) REFERENCES agents (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE | | id | PRIMARY KEY | PRIMARY KEY (id) | | sqlite_autoindex_agent_files_1 | PRIMARY KEY | PRIMARY KEY (id) | @@ -56,6 +59,8 @@ erDiagram INTEGER fileSizeBytes varchar_16_ id PK varchar_255_ mimeType + TEXT storageKey + varchar_2_ storedAt datetime_3_ updatedAt } "agents" { diff --git a/docs/generated/sqlite-schema/agents.md b/docs/generated/sqlite-schema/agents.md index 338cdea363e..db5609decfe 100644 --- a/docs/generated/sqlite-schema/agents.md +++ b/docs/generated/sqlite-schema/agents.md @@ -148,6 +148,8 @@ erDiagram INTEGER fileSizeBytes varchar_16_ id PK varchar_255_ mimeType + TEXT storageKey + varchar_2_ storedAt datetime_3_ updatedAt } "agent_task_definition" { diff --git a/packages/@n8n/blob-storage/src/byte-store-registry.ts b/packages/@n8n/blob-storage/src/byte-store-registry.ts new file mode 100644 index 00000000000..cf760ecd94a --- /dev/null +++ b/packages/@n8n/blob-storage/src/byte-store-registry.ts @@ -0,0 +1,42 @@ +import { UnexpectedError } from 'n8n-workflow'; + +import type { ByteStore, StorageLocation } from './types'; + +/** + * The byte store backing each storage location. `fs` is always present; `s3` + * and `az` are registered at startup only when their client initialized, so a + * configured-but-unreachable backend is absent rather than broken. + */ +export class ByteStoreRegistry { + private readonly byteStores = new Map(); + + constructor(byteStores: Partial>) { + for (const [loc, store] of Object.entries(byteStores)) { + if (store) this.byteStores.set(loc as StorageLocation, store); + } + } + + register(loc: StorageLocation, store: ByteStore) { + this.byteStores.set(loc, store); + } + + has(loc: StorageLocation) { + return this.byteStores.has(loc); + } + + /** `undefined` when the location is not configured. */ + find(loc: StorageLocation) { + return this.byteStores.get(loc); + } + + /** Throws when the location is not configured. */ + get(loc: StorageLocation): ByteStore { + const store = this.byteStores.get(loc); + + if (!store) { + throw new UnexpectedError(`Byte store for location "${loc}" is not configured.`); + } + + return store; + } +} diff --git a/packages/@n8n/blob-storage/src/index.ts b/packages/@n8n/blob-storage/src/index.ts index 319e7826b84..8128e06c450 100644 --- a/packages/@n8n/blob-storage/src/index.ts +++ b/packages/@n8n/blob-storage/src/index.ts @@ -8,6 +8,7 @@ export type { Stored, JsonStoreOptions, } from './types'; +export { ByteStoreRegistry } from './byte-store-registry'; export { FsByteStore, type FsByteStoreOptions } from './fs-byte-store'; export { S3ByteStore } from './s3-byte-store.ee'; export { AzureByteStore } from './azure-byte-store.ee'; diff --git a/packages/@n8n/blob-storage/src/json-store.ts b/packages/@n8n/blob-storage/src/json-store.ts index 1d910ef377b..a20899f203f 100644 --- a/packages/@n8n/blob-storage/src/json-store.ts +++ b/packages/@n8n/blob-storage/src/json-store.ts @@ -1,6 +1,7 @@ import chunk from 'lodash/chunk'; import { jsonParse, jsonStringify, UnexpectedError } from 'n8n-workflow'; +import { ByteStoreRegistry } from './byte-store-registry'; import { SkippedEntryDeletionError } from './skipped-entry-deletion.error'; import type { ByteStore, JsonEntry, JsonStoreOptions, StorageLocation, Stored } from './types'; @@ -13,16 +14,14 @@ const MAX_READ_CONCURRENCY = 50; * The `version` key is reserved for the schema version: writes reject payloads that contain it. */ export class JsonStore { - private readonly byteStores = new Map(); + private readonly byteStores: ByteStoreRegistry; constructor(private readonly options: JsonStoreOptions) { - for (const [loc, store] of Object.entries(options.byteStores)) { - if (store) this.byteStores.set(loc as StorageLocation, store); - } + this.byteStores = new ByteStoreRegistry(options.byteStores); } registerByteStore(loc: StorageLocation, store: ByteStore) { - this.byteStores.set(loc, store); + this.byteStores.register(loc, store); } hasLocation(loc: StorageLocation) { @@ -30,7 +29,7 @@ export class JsonStore { } async write(ref: Ref, payload: Payload, loc: StorageLocation) { - const store = this.getByteStore(loc); + const store = this.byteStores.get(loc); if ('version' in payload) { throw new UnexpectedError('Payload must not contain the reserved `version` key.'); } @@ -46,7 +45,7 @@ export class JsonStore { } async read(ref: Ref, loc: StorageLocation) { - const store = this.getByteStore(loc); + const store = this.byteStores.get(loc); const key = this.options.key(ref); const bytes = await store.read(key); if (!bytes) return null; @@ -59,7 +58,7 @@ export class JsonStore { await Promise.all( [...this.groupByLocation(refs)].map(async ([loc, group]) => { - const store = this.getByteStore(loc); + const store = this.byteStores.get(loc); for (const batch of chunk(group, MAX_READ_CONCURRENCY)) { const results = await Promise.all( batch.map(async (ref) => await this.tryRead(ref, store)), @@ -79,7 +78,7 @@ export class JsonStore { await Promise.all( [...this.groupByLocation(refs)].map(async ([loc, group]) => { - const store = this.byteStores.get(loc); + const store = this.byteStores.find(loc); if (!store) { this.options.reportError(new SkippedEntryDeletionError(loc, group.length)); return; @@ -127,14 +126,6 @@ export class JsonStore { } } - private getByteStore(loc: StorageLocation): ByteStore { - const store = this.byteStores.get(loc); - if (!store) { - throw new UnexpectedError(`JSON store for location "${loc}" is not configured.`); - } - return store; - } - private markCorrupt(error: E): E { Object.defineProperty(error, CORRUPT_ENTRY, { value: true }); return error; diff --git a/packages/@n8n/db/src/migrations/common/1785186578138-AddAgentFileStorageColumns.ts b/packages/@n8n/db/src/migrations/common/1785186578138-AddAgentFileStorageColumns.ts new file mode 100644 index 00000000000..6caae96bbde --- /dev/null +++ b/packages/@n8n/db/src/migrations/common/1785186578138-AddAgentFileStorageColumns.ts @@ -0,0 +1,122 @@ +import type { MigrationContext, ReversibleMigration } from '../migration-types'; + +/** BinaryDataService reference prefix paired with the location holding the bytes. */ +const PREFIX_LOCATIONS = [ + ['filesystem:', 'fs'], + ['filesystem-v2:', 'fs'], + ['s3:', 's3'], + ['azure:', 'az'], + ['database:', 'db'], +] as const; + +/** + * Adds the storage location (`storedAt`) and key (`storageKey`) that address a + * knowledge file's bytes, backfilled from BinaryDataService's opaque + * `binaryDataId`. + * + * `binaryDataId` stays, only relaxed to nullable, so the previous release keeps + * resolving pre-existing rows during a rolling deploy and a rollback still has + * the original references. A follow-up migration drops it once this release has + * been observed in production. + * + * No bytes move: BinaryDataService wrote them through the same fs/s3/azure byte + * stores the new agent knowledge file store uses, so stripping the prefix off + * `binaryDataId` yields a key that resolves as-is. For `db` rows the key is the + * `binary_data.fileId` holding the bytes. + * + * `fs` rows resolve only when `N8N_BINARY_DATA_STORAGE_PATH` was left unset: + * the new store roots keys at `N8N_STORAGE_PATH` (`~/.n8n/storage`), which is + * also BinaryDataService's fs root unless that deprecated var pointed it + * elsewhere. Where the two diverge, the bytes stay where they are and the file + * reads as missing. + */ +export class AddAgentFileStorageColumns1785186578138 implements ReversibleMigration { + async up(ctx: MigrationContext) { + const { + schemaBuilder: { addColumns, addNotNull, column, dropNotNull }, + } = ctx; + + await addColumns( + 'agent_files', + [ + column('storedAt') + .varchar(2) + .notNull.default("'db'") + .withEnumCheck(['db', 'fs', 's3', 'az']) + .comment( + "Where the file bytes live: 'db' (binary_data table), or a blob-storage backend ('fs', 's3', 'az')", + ), + column('storageKey').text.comment( + 'Key addressing the bytes within storedAt: a binary_data.fileId for db, a byte-store key otherwise. Not a foreign key', + ), + ], + { recreatesOnSqlite: true }, + ); + + await this.backfillFromBinaryDataIds(ctx); + + await addNotNull('agent_files', 'storageKey', { recreatesOnSqlite: true }); + // New rows leave `binaryDataId` null; the previous release still needs the + // column to exist to read the rows it wrote. + await dropNotNull('agent_files', 'binaryDataId', { recreatesOnSqlite: true }); + } + + async down(ctx: MigrationContext) { + const { + schemaBuilder: { addNotNull, dropColumns, dropEnumCheck }, + } = ctx; + + await this.restoreMissingBinaryDataIds(ctx); + await addNotNull('agent_files', 'binaryDataId', { recreatesOnSqlite: true }); + + // Must precede the column drop: SQLite carries the CHECK over to the + // rebuilt table, where it would reference a column that no longer exists. + await dropEnumCheck('agent_files', 'storedAt', { recreatesOnSqlite: true }); + await dropColumns('agent_files', ['storedAt', 'storageKey'], { recreatesOnSqlite: true }); + } + + /** Splits each `binaryDataId` into the location and key that address its bytes. */ + private async backfillFromBinaryDataIds({ escape, runQuery }: MigrationContext) { + const agentFiles = escape.tableName('agent_files'); + const binaryDataId = escape.columnName('binaryDataId'); + const storedAt = escape.columnName('storedAt'); + const storageKey = escape.columnName('storageKey'); + + for (const [prefix, location] of PREFIX_LOCATIONS) { + // The offset is derived from the prefix, not from row data. + await runQuery( + `UPDATE ${agentFiles} SET ${storedAt} = :location, ${storageKey} = SUBSTR(${binaryDataId}, ${prefix.length + 1}) WHERE ${binaryDataId} LIKE :pattern`, + { location, pattern: `${prefix}%` }, + ); + } + + // Every reference BinaryDataService wrote carries one of the prefixes + // above. Keep an unrecognized one whole so the row survives the NOT NULL + // below, parked on 'fs' where a key that addresses nothing simply reads as + // missing — 'db' would instead point at a binary_data.fileId that is not + // one. The original reference stays in binaryDataId either way. + await runQuery( + `UPDATE ${agentFiles} SET ${storedAt} = 'fs', ${storageKey} = ${binaryDataId} WHERE ${storageKey} IS NULL`, + ); + } + + /** + * Rebuilds the prefixed reference for rows written after `up()`. Rows that + * predate it kept theirs, so their opaque ids round-trip untouched. + */ + private async restoreMissingBinaryDataIds({ escape, runQuery }: MigrationContext) { + const agentFiles = escape.tableName('agent_files'); + const binaryDataId = escape.columnName('binaryDataId'); + const storedAt = escape.columnName('storedAt'); + const storageKey = escape.columnName('storageKey'); + + // `filesystem` rows come back as `filesystem-v2`; both resolve to the same + // BinaryDataService manager, so the reconstructed id stays readable. + await runQuery( + `UPDATE ${agentFiles} SET ${binaryDataId} = CASE ${storedAt} ` + + "WHEN 'db' THEN 'database:' WHEN 'fs' THEN 'filesystem-v2:' " + + "WHEN 's3' THEN 's3:' WHEN 'az' THEN 'azure:' END " + + `|| ${storageKey} WHERE ${binaryDataId} IS NULL`, + ); + } +} diff --git a/packages/@n8n/db/src/migrations/postgresdb/index.ts b/packages/@n8n/db/src/migrations/postgresdb/index.ts index 2409fda64fb..5f361ee11cc 100644 --- a/packages/@n8n/db/src/migrations/postgresdb/index.ts +++ b/packages/@n8n/db/src/migrations/postgresdb/index.ts @@ -232,6 +232,7 @@ import { AddInstanceCredentials1784815940111 } from '../common/1784815940111-Add import { CreateAgentEvalTables1784815940112 } from '../common/1784815940112-CreateAgentEvalTables'; import { AddAvailableInMcpToAgents1784897791636 } from '../common/1784897791636-AddAvailableInMcpToAgents'; import { AddIsFirstPartyToOAuthClients1785162364001 } from '../common/1785162364001-AddIsFirstPartyToOAuthClients'; +import { AddAgentFileStorageColumns1785186578138 } from '../common/1785186578138-AddAgentFileStorageColumns'; import type { Migration } from '../migration-types'; export const postgresMigrations: Migration[] = [ @@ -469,4 +470,5 @@ export const postgresMigrations: Migration[] = [ AddAvailableInMcpToAgents1784897791636, ChangeInstalledNodeVersionType1785162364000, AddIsFirstPartyToOAuthClients1785162364001, + AddAgentFileStorageColumns1785186578138, ]; diff --git a/packages/@n8n/db/src/migrations/sqlite/index.ts b/packages/@n8n/db/src/migrations/sqlite/index.ts index d4bb95768af..96343221273 100644 --- a/packages/@n8n/db/src/migrations/sqlite/index.ts +++ b/packages/@n8n/db/src/migrations/sqlite/index.ts @@ -224,6 +224,7 @@ import { CreateWorkflowReviewRequestTables1784000000052 } from '../common/178400 import { AddStoredAtToAgentExecution1784815940110 } from '../common/1784815940110-AddStoredAtToAgentExecution'; import { AddInstanceCredentials1784815940111 } from '../common/1784815940111-AddInstanceCredentials'; import { CreateAgentEvalTables1784815940112 } from '../common/1784815940112-CreateAgentEvalTables'; +import { AddAgentFileStorageColumns1785186578138 } from '../common/1785186578138-AddAgentFileStorageColumns'; const sqliteMigrations: Migration[] = [ InitialMigration1588102412422, @@ -451,6 +452,7 @@ const sqliteMigrations: Migration[] = [ AddAvailableInMcpToAgents1784897791636, ChangeInstalledNodeVersionType1785162364000, AddIsFirstPartyToOAuthClients1785162364001, + AddAgentFileStorageColumns1785186578138, ]; export { sqliteMigrations }; diff --git a/packages/@n8n/db/src/repositories/binary-data.repository.ts b/packages/@n8n/db/src/repositories/binary-data.repository.ts index 3d22da97ec4..2582219409e 100644 --- a/packages/@n8n/db/src/repositories/binary-data.repository.ts +++ b/packages/@n8n/db/src/repositories/binary-data.repository.ts @@ -1,6 +1,6 @@ import { DatabaseConfig } from '@n8n/config'; import { Service } from '@n8n/di'; -import { DataSource, Repository } from '@n8n/typeorm'; +import { DataSource, In, Repository } from '@n8n/typeorm'; import { BinaryDataFile } from '../entities'; import { dbType } from '../entities/abstract-entity'; @@ -14,6 +14,19 @@ export class BinaryDataRepository extends Repository { super(BinaryDataFile, dataSource.manager); } + /** Content bytes for a stored file, or null when the row is gone. */ + async findContentByFileId(fileId: string): Promise { + const file = await this.findOne({ where: { fileId }, select: ['data'] }); + + return file?.data ?? null; + } + + async deleteByFileIds(fileIds: string[]): Promise { + if (fileIds.length === 0) return; + + await this.delete({ fileId: In(fileIds) }); + } + async copyStoredFile( sourceFileId: string, targetFileId: string, diff --git a/packages/cli/src/executions/execution-data/__tests__/execution-data-json-store.test.ts b/packages/cli/src/executions/execution-data/__tests__/execution-data-json-store.test.ts index 2ee3a966003..982045c25c6 100644 --- a/packages/cli/src/executions/execution-data/__tests__/execution-data-json-store.test.ts +++ b/packages/cli/src/executions/execution-data/__tests__/execution-data-json-store.test.ts @@ -1,23 +1,21 @@ /* eslint-disable @typescript-eslint/unbound-method */ /* eslint-disable n8n-local-rules/no-uncaught-json-parse */ -import type { ErrorReporter } from 'n8n-core'; +import type { ErrorReporter, FsByteStoreService } from 'n8n-core'; import { mock } from 'vitest-mock-extended'; -import type { ExecutionDataFsByteStore } from '../execution-data-json-store'; - import { CorruptedExecutionDataError } from '../corrupted-execution-data.error'; import { ExecutionDataJsonStore } from '../execution-data-json-store'; import { ExecutionDataWriteError } from '../execution-data-write.error'; import { createExecutionRef } from '../types'; import { executionId, payload, ref, workflowId } from './mocks'; -let fsByteStore: ReturnType>; +let fsByteStore: ReturnType>; let errorReporter: ReturnType>; let store: ExecutionDataJsonStore; beforeEach(() => { - fsByteStore = mock(); + fsByteStore = mock(); errorReporter = mock(); store = new ExecutionDataJsonStore(fsByteStore, errorReporter); }); diff --git a/packages/cli/src/executions/execution-data/execution-data-json-store.ts b/packages/cli/src/executions/execution-data/execution-data-json-store.ts index 15d346a63f5..a2c221ba3d7 100644 --- a/packages/cli/src/executions/execution-data/execution-data-json-store.ts +++ b/packages/cli/src/executions/execution-data/execution-data-json-store.ts @@ -1,29 +1,19 @@ -import { FsByteStore, JsonStore } from '@n8n/blob-storage'; +import { JsonStore } from '@n8n/blob-storage'; import { Service } from '@n8n/di'; -import { ErrorReporter, StorageConfig } from 'n8n-core'; +import { ErrorReporter, FsByteStoreService } from 'n8n-core'; import { EXECUTION_DATA_BUNDLE_FILENAME, EXECUTION_DATA_BUNDLE_VERSION } from './constants'; import { CorruptedExecutionDataError } from './corrupted-execution-data.error'; import { ExecutionDataWriteError } from './execution-data-write.error'; import type { ExecutionDataPayload, ExecutionRef } from './types'; -@Service() -export class ExecutionDataFsByteStore extends FsByteStore { - constructor(storageConfig: StorageConfig, errorReporter: ErrorReporter) { - super({ - storagePath: storageConfig.storagePath, - reportError: (error) => errorReporter.error(error), - }); - } -} - /** * Stores execution data bundles as JSON blobs. The `fs` backend is always * available, but `s3` and `az` are registered at startup only if configured. */ @Service() export class ExecutionDataJsonStore extends JsonStore { - constructor(fsByteStore: ExecutionDataFsByteStore, errorReporter: ErrorReporter) { + constructor(fsByteStore: FsByteStoreService, errorReporter: ErrorReporter) { super({ byteStores: { fs: fsByteStore }, version: EXECUTION_DATA_BUNDLE_VERSION, diff --git a/packages/cli/src/modules/agents/__tests__/agent-knowledge-commands.test.ts b/packages/cli/src/modules/agents/__tests__/agent-knowledge-commands.test.ts index 2b66c8a341e..f8b6c895e6b 100644 --- a/packages/cli/src/modules/agents/__tests__/agent-knowledge-commands.test.ts +++ b/packages/cli/src/modules/agents/__tests__/agent-knowledge-commands.test.ts @@ -18,7 +18,8 @@ import { const mobyDickFile: AgentKnowledgeFileReference = { file: 'moby-dick.txt', fileId: 'file-1', - binaryDataId: 'filesystem-v2:agents/agent-1/knowledge-files/file-1/binary_data/uuid', + storedAt: 'fs', + storageKey: 'agents/agent-1/knowledge-files/file-1/content', displayName: 'moby-dick.txt', mimeType: 'text/plain', fileSizeBytes: 123, diff --git a/packages/cli/src/modules/agents/__tests__/agent-knowledge-file-store.test.ts b/packages/cli/src/modules/agents/__tests__/agent-knowledge-file-store.test.ts new file mode 100644 index 00000000000..cf38bc82d49 --- /dev/null +++ b/packages/cli/src/modules/agents/__tests__/agent-knowledge-file-store.test.ts @@ -0,0 +1,164 @@ +import { FsByteStore, SkippedEntryDeletionError } from '@n8n/blob-storage'; +import type { BinaryDataRepository } from '@n8n/db'; +import type { ErrorReporter, StorageConfig } from 'n8n-core'; +import { UnexpectedError } from 'n8n-workflow'; +import { mkdtemp, readdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Readable } from 'node:stream'; +import { mock } from 'vitest-mock-extended'; + +import { AgentKnowledgeFileStore } from '../agent-knowledge-file-store'; + +vi.unmock('node:fs'); +vi.unmock('node:fs/promises'); + +const LEGACY_KEY = + 'agents/agent-1/knowledge-files/file-1/binary_data/2f1b3c4d-5e6f-4a7b-8c9d-0e1f2a3b4c5d'; + +describe('AgentKnowledgeFileStore', () => { + let storagePath: string; + let binaryDataRepository: ReturnType>; + let errorReporter: ReturnType>; + let fsByteStore: FsByteStore; + let store: AgentKnowledgeFileStore; + + function makeStore(modeTag: 'db' | 'fs' | 's3' | 'az'): AgentKnowledgeFileStore { + return new AgentKnowledgeFileStore( + fsByteStore, + { storagePath, modeTag } as StorageConfig, + binaryDataRepository, + errorReporter, + ); + } + + beforeAll(async () => { + storagePath = await mkdtemp(join(tmpdir(), 'n8n-agent-knowledge-file-store-')); + }); + + beforeEach(async () => { + for (const entry of await readdir(storagePath)) { + await rm(join(storagePath, entry), { recursive: true, force: true }); + } + binaryDataRepository = mock(); + errorReporter = mock(); + fsByteStore = new FsByteStore({ storagePath, reportError: () => {} }); + store = makeStore('fs'); + }); + + afterAll(async () => { + await rm(storagePath, { recursive: true, force: true }); + }); + + it('writes to and reads from binary_data when modeTag is db', async () => { + store = makeStore('db'); + const body = Buffer.from('knowledge-bytes', 'utf-8'); + + const stored = await store.write( + { agentId: 'agent-1', fileId: 'file-1' }, + Readable.from(body), + { fileName: 'notes.txt', mimeType: 'text/plain' }, + ); + + expect(stored.storedAt).toBe('db'); + expect(binaryDataRepository.insert).toHaveBeenCalledWith({ + fileId: stored.storageKey, + sourceType: 'agent_file', + sourceId: 'file-1', + data: body, + mimeType: 'text/plain', + fileName: 'notes.txt', + fileSize: body.length, + }); + + binaryDataRepository.findContentByFileId.mockResolvedValue(body); + await expect(store.readAsBuffer(stored)).resolves.toEqual(body); + expect(binaryDataRepository.findContentByFileId).toHaveBeenCalledWith(stored.storageKey); + }); + + it('reads a blob stored under the legacy BinaryDataService key', async () => { + const body = Buffer.from('legacy-bytes', 'utf-8'); + await fsByteStore.write(LEGACY_KEY, body); + + await expect(store.readAsBuffer({ storedAt: 'fs', storageKey: LEGACY_KEY })).resolves.toEqual( + body, + ); + }); + + it('writes to a registered s3 location when modeTag is s3', async () => { + store = makeStore('s3'); + const s3Store = new FsByteStore({ + storagePath: join(storagePath, 's3-root'), + reportError: () => {}, + }); + store.registerByteStore('s3', s3Store); + + const body = Buffer.from('s3-bytes', 'utf-8'); + + const stored = await store.write({ agentId: 'agent-1', fileId: 'file-2' }, body, { + mimeType: 'text/plain', + }); + + expect(stored.storedAt).toBe('s3'); + await expect(store.readAsBuffer(stored)).resolves.toEqual(body); + }); + + it('throws when writing to a location that has no byte store', async () => { + store = makeStore('s3'); + + await expect( + store.write({ agentId: 'agent-1', fileId: 'file-3' }, Buffer.from('bytes', 'utf-8'), { + mimeType: 'text/plain', + }), + ).rejects.toThrow(UnexpectedError); + }); + + it('returns null for a missing blob', async () => { + await expect( + store.readAsBuffer({ + storedAt: 'fs', + storageKey: 'agents/a/knowledge-files/missing/content', + }), + ).resolves.toBeNull(); + }); + + it('throws when reading from a location that has no byte store', async () => { + await expect( + store.readAsBuffer({ storedAt: 's3', storageKey: 'agents/a/knowledge-files/f/content' }), + ).rejects.toThrow(UnexpectedError); + }); + + it('deletes registered blobs and reports unregistered locations without throwing', async () => { + const body = Buffer.from('delete-me', 'utf-8'); + const kept = await store.write({ agentId: 'agent-1', fileId: 'keep' }, body, { + mimeType: 'text/plain', + }); + + await expect( + store.delete([ + kept, + { storedAt: 'db', storageKey: '9c1f4b7a-2d3e-4f5a-8b6c-7d8e9f0a1b2c' }, + { storedAt: 's3', storageKey: 'agents/agent-1/knowledge-files/orphan/content' }, + ]), + ).resolves.toBeUndefined(); + + await expect(store.readAsBuffer(kept)).resolves.toBeNull(); + expect(binaryDataRepository.deleteByFileIds).toHaveBeenCalledWith([ + '9c1f4b7a-2d3e-4f5a-8b6c-7d8e9f0a1b2c', + ]); + expect(errorReporter.error).toHaveBeenCalledWith(expect.any(SkippedEntryDeletionError)); + }); + + it('deletes the companion metadata entry of a legacy filesystem key', async () => { + await fsByteStore.write(LEGACY_KEY, Buffer.from('legacy-bytes', 'utf-8')); + await fsByteStore.write( + `${LEGACY_KEY}.metadata`, + Buffer.from('{"fileName":"notes.txt"}', 'utf-8'), + ); + + await store.delete([{ storedAt: 'fs', storageKey: LEGACY_KEY }]); + + await expect(fsByteStore.read(LEGACY_KEY)).resolves.toBeNull(); + await expect(fsByteStore.read(`${LEGACY_KEY}.metadata`)).resolves.toBeNull(); + }); +}); diff --git a/packages/cli/src/modules/agents/__tests__/agent-knowledge-sandbox.service.test.ts b/packages/cli/src/modules/agents/__tests__/agent-knowledge-sandbox.service.test.ts index f68a26232dc..1087878b91d 100644 --- a/packages/cli/src/modules/agents/__tests__/agent-knowledge-sandbox.service.test.ts +++ b/packages/cli/src/modules/agents/__tests__/agent-knowledge-sandbox.service.test.ts @@ -3,10 +3,11 @@ import type { Logger } from '@n8n/backend-common'; import type { AgentsConfig } from '@n8n/config'; import type { AiAssistantClient } from '@n8n_io/ai-assistant-sdk'; import { mock } from 'vitest-mock-extended'; -import type { BinaryDataService, InstanceSettings } from 'n8n-core'; +import type { InstanceSettings } from 'n8n-core'; import type { AiService } from '../../../services/ai.service'; +import type { AgentKnowledgeFileStore } from '../agent-knowledge-file-store'; import type { Agent } from '../entities/agent.entity'; import type { AgentFile } from '../entities/agent-file.entity'; import { KNOWLEDGE_MIRROR_FILES_DIR } from '../agent-knowledge-storage'; @@ -117,10 +118,10 @@ function makeAgentRepository(): ReturnType> { return repository; } -function makeBinaryDataService(): Mocked { - const service = mock(); - service.getAsBuffer.mockResolvedValue(Buffer.from('mock file content')); - return service; +function makeKnowledgeFileStore(): Mocked { + const store = mock(); + store.readAsBuffer.mockResolvedValue(Buffer.from('mock file content')); + return store; } function makeService( @@ -130,7 +131,7 @@ function makeService( instanceSettings: InstanceSettings = mock({ instanceId }), agentFileRepository: AgentFileRepository = mock(), agentRepository: AgentRepository = makeAgentRepository(), - binaryDataService: BinaryDataService = makeBinaryDataService(), + agentKnowledgeFileStore: AgentKnowledgeFileStore = makeKnowledgeFileStore(), ): AgentKnowledgeSandboxService { return new AgentKnowledgeSandboxService( { @@ -149,7 +150,7 @@ function makeService( instanceSettings, agentFileRepository, agentRepository, - binaryDataService, + agentKnowledgeFileStore, ); } @@ -159,7 +160,8 @@ function makeAgentFile(overrides: Partial = {}): AgentFile { return { id, agentId, - binaryDataId: `filesystem-v2:agents/${agentId}/knowledge-files/${id}/binary_data/uuid`, + storedAt: 'fs', + storageKey: `agents/${agentId}/knowledge-files/${id}/content`, fileName, mimeType: 'text/plain', fileSizeBytes: 100, @@ -565,7 +567,7 @@ describe('AgentKnowledgeSandboxService', () => { ]); const agentRepository = makeAgentRepository(); agentRepository.existsBy.mockResolvedValue(true); - const binaryDataService = makeBinaryDataService(); + const agentKnowledgeFileStore = makeKnowledgeFileStore(); const service = makeService( {}, mock(), @@ -573,30 +575,30 @@ describe('AgentKnowledgeSandboxService', () => { mock({ instanceId }), agentFileRepository, agentRepository, - binaryDataService, + agentKnowledgeFileStore, ); await service.searchKnowledge(projectId, agentId, { pattern: 'foo' }); let commands = sandbox.process.executeCommand.mock.calls.map(([command]) => command); expect(commands.filter(isManifestReadCommand)).toHaveLength(1); expect(commands.filter(isMirrorSyncCommand)).toHaveLength(1); - expect(binaryDataService.getAsBuffer).toHaveBeenCalledTimes(2); + expect(agentKnowledgeFileStore.readAsBuffer).toHaveBeenCalledTimes(2); expect(sandbox.fs.uploadFiles).toHaveBeenCalledTimes(1); manifestState = 'doc1.txt\ndoc2.txt\n'; sandbox.process.executeCommand.mockClear(); - binaryDataService.getAsBuffer.mockClear(); + agentKnowledgeFileStore.readAsBuffer.mockClear(); sandbox.fs.uploadFiles.mockClear(); await service.searchKnowledge(projectId, agentId, { pattern: 'bar' }); commands = sandbox.process.executeCommand.mock.calls.map(([command]) => command); expect(commands.filter(isManifestReadCommand)).toHaveLength(0); expect(commands.filter(isMirrorSyncCommand)).toHaveLength(0); expect(commands).toHaveLength(1); - expect(binaryDataService.getAsBuffer).not.toHaveBeenCalled(); + expect(agentKnowledgeFileStore.readAsBuffer).not.toHaveBeenCalled(); expect(sandbox.fs.uploadFiles).not.toHaveBeenCalled(); sandbox.process.executeCommand.mockClear(); - binaryDataService.getAsBuffer.mockClear(); + agentKnowledgeFileStore.readAsBuffer.mockClear(); sandbox.fs.uploadFiles.mockClear(); agentFileRepository.findByAgentId.mockResolvedValue([ makeMirrorFile('file-1', 'doc1.txt'), @@ -611,7 +613,7 @@ describe('AgentKnowledgeSandboxService', () => { // Only the newly-added name should be fetched and staged for move — // the manifest rewrite (which always lists every expected name) is // a separate, later part of the finalize command. - expect(binaryDataService.getAsBuffer).toHaveBeenCalledTimes(1); + expect(agentKnowledgeFileStore.readAsBuffer).toHaveBeenCalledTimes(1); expect(syncCommands[0]).toContain('.tmp-doc3.txt'); expect(syncCommands[0]).not.toContain('.tmp-doc1.txt'); expect(syncCommands[0]).not.toContain('.tmp-doc2.txt'); @@ -632,8 +634,10 @@ describe('AgentKnowledgeSandboxService', () => { ]); const agentRepository = makeAgentRepository(); agentRepository.existsBy.mockResolvedValue(true); - const binaryDataService = makeBinaryDataService(); - binaryDataService.getAsBuffer.mockResolvedValue(Buffer.alloc(MIRROR_UPLOAD_BATCH_BYTES)); + const agentKnowledgeFileStore = makeKnowledgeFileStore(); + agentKnowledgeFileStore.readAsBuffer.mockResolvedValue( + Buffer.alloc(MIRROR_UPLOAD_BATCH_BYTES), + ); const service = makeService( {}, mock(), @@ -641,7 +645,7 @@ describe('AgentKnowledgeSandboxService', () => { mock({ instanceId }), agentFileRepository, agentRepository, - binaryDataService, + agentKnowledgeFileStore, ); await service.searchKnowledge(projectId, agentId, { pattern: 'foo' }); @@ -709,7 +713,7 @@ describe('AgentKnowledgeSandboxService', () => { ); }); - it('skips a file that fails to load from binary data storage and retries it next sync', async () => { + it('skips a file that fails to load from the knowledge file store and retries it next sync', async () => { const sandbox = makeSandbox('started'); sandbox.process.executeCommand.mockResolvedValue({ exitCode: 0, @@ -720,8 +724,8 @@ describe('AgentKnowledgeSandboxService', () => { agentFileRepository.findByAgentId.mockResolvedValue([makeMirrorFile('file-1', 'doc1.txt')]); const agentRepository = makeAgentRepository(); agentRepository.existsBy.mockResolvedValue(true); - const binaryDataService = makeBinaryDataService(); - binaryDataService.getAsBuffer.mockRejectedValueOnce(new Error('missing on disk')); + const agentKnowledgeFileStore = makeKnowledgeFileStore(); + agentKnowledgeFileStore.readAsBuffer.mockRejectedValueOnce(new Error('missing on disk')); const logger = mock(); const service = makeService( {}, @@ -730,7 +734,7 @@ describe('AgentKnowledgeSandboxService', () => { mock({ instanceId }), agentFileRepository, agentRepository, - binaryDataService, + agentKnowledgeFileStore, ); await expect( @@ -745,12 +749,12 @@ describe('AgentKnowledgeSandboxService', () => { // The failed file was left out of the cached manifest, so the next // sync's expected hash mismatches and it retries — this time - // `getAsBuffer` succeeds (the mock's default resolves). + // `readAsBuffer` succeeds (the mock's default resolves). await expect( service.searchKnowledge(projectId, agentId, { pattern: 'bar' }), ).resolves.toBeDefined(); - expect(binaryDataService.getAsBuffer).toHaveBeenCalledTimes(2); + expect(agentKnowledgeFileStore.readAsBuffer).toHaveBeenCalledTimes(2); expect(sandbox.fs.uploadFiles).toHaveBeenCalledTimes(1); }); }); diff --git a/packages/cli/src/modules/agents/__tests__/agent-knowledge.service.test.ts b/packages/cli/src/modules/agents/__tests__/agent-knowledge.service.test.ts index e70396d8dfd..c97895e9de2 100644 --- a/packages/cli/src/modules/agents/__tests__/agent-knowledge.service.test.ts +++ b/packages/cli/src/modules/agents/__tests__/agent-knowledge.service.test.ts @@ -7,8 +7,8 @@ import { Readable } from 'node:stream'; import { MAX_AGENT_KNOWLEDGE_BASE_SIZE_BYTES } from '@n8n/api-types'; import type { Logger } from '@n8n/backend-common'; -import type { BinaryDataConfig, BinaryDataService } from 'n8n-core'; +import type { AgentKnowledgeFileStore } from '../agent-knowledge-file-store'; import { AgentKnowledgeService } from '../agent-knowledge.service'; import type { AgentKnowledgeSandboxService } from '../agent-knowledge-sandbox.service'; import type { AgentFile } from '../entities/agent-file.entity'; @@ -28,10 +28,9 @@ vi.mock('@n8n/ai-utilities', () => ({ })); /** - * Real `BinaryDataService` managers fully consume the stream before - * resolving/rejecting; mock implementations must too, or an unconsumed - * `createReadStream` lazily opens after the test's temp-file cleanup runs - * and throws an unhandled ENOENT. + * The knowledge file store writes streams by consuming them; mock + * implementations must too, or an unconsumed `createReadStream` lazily opens + * after the test's temp-file cleanup runs and throws an unhandled ENOENT. */ async function drainIfStream(content: unknown): Promise { if (content instanceof Readable) { @@ -65,7 +64,8 @@ function makeAgentFile(overrides: Partial = {}): AgentFile { return { id: 'file-1', agentId, - binaryDataId: 'filesystem-v2:agents/agent-1/knowledge-files/file-1/binary_data/uuid', + storedAt: 'fs', + storageKey: `agents/${agentId}/knowledge-files/${overrides.id ?? 'file-1'}/content`, fileName: 'first.txt', mimeType: 'text/plain', fileSizeBytes: 4, @@ -84,7 +84,8 @@ class InMemoryAgentFileRepository { return { id: input.id ?? 'generated-id', agentId: input.agentId ?? agentId, - binaryDataId: input.binaryDataId ?? '', + storedAt: input.storedAt ?? 'fs', + storageKey: input.storageKey ?? '', fileName: input.fileName ?? '', mimeType: input.mimeType ?? '', fileSizeBytes: input.fileSizeBytes ?? 0, @@ -134,39 +135,36 @@ describe('AgentKnowledgeService', () => { let agentRepository: Mocked; let agentFileRepository: InMemoryAgentFileRepository; let agentKnowledgeSandboxService: Mocked; - let binaryDataService: Mocked; - let binaryDataConfig: BinaryDataConfig; + let agentKnowledgeFileStore: Mocked; let logger: Mocked; let service: AgentKnowledgeService; - let storeSequence: number; beforeEach(() => { vi.clearAllMocks(); - storeSequence = 0; agentRepository = mock(); agentFileRepository = new InMemoryAgentFileRepository(); agentKnowledgeSandboxService = mock(); - binaryDataService = mock(); - binaryDataService.store.mockImplementation(async (_location, content, binaryData) => { - storeSequence += 1; + agentKnowledgeFileStore = mock(); + agentKnowledgeFileStore.write.mockImplementation(async (ref, content) => { await drainIfStream(content); - return { ...binaryData, id: `filesystem-v2:mock-${storeSequence}` }; + return { + storedAt: 'fs', + storageKey: `agents/${ref.agentId}/knowledge-files/${ref.fileId}/content`, + }; }); - binaryDataService.deleteManyByBinaryDataId.mockResolvedValue(undefined); - binaryDataConfig = { mode: 'filesystem' } as BinaryDataConfig; + agentKnowledgeFileStore.delete.mockResolvedValue(undefined); logger = mock(); service = new AgentKnowledgeService( agentRepository, agentFileRepository as unknown as AgentFileRepository, agentKnowledgeSandboxService, - binaryDataService, - binaryDataConfig, + agentKnowledgeFileStore, logger, ); loadMock.mockResolvedValue([{ pageContent: 'extracted pdf text' }]); }); - it('stores text and PDF files via BinaryDataService, creates DB rows, and cleans temp files', async () => { + it('stores text and PDF files via the knowledge file store, creates DB rows, and cleans temp files', async () => { // activeVersionId: null — unpublished agents may upload knowledge files agentRepository.findByIdAndProjectId.mockResolvedValue({ id: agentId, @@ -211,16 +209,9 @@ describe('AgentKnowledgeService', () => { }), ]); - expect(binaryDataService.store).toHaveBeenCalledTimes(2); - const [textCall, pdfCall] = binaryDataService.store.mock.calls; - expect(textCall[0]).toEqual( - expect.objectContaining({ - type: 'custom', - pathSegments: ['agents', agentId, 'knowledge-files', expect.any(String)], - sourceType: 'agent_file', - sourceId: expect.any(String), - }), - ); + expect(agentKnowledgeFileStore.write).toHaveBeenCalledTimes(2); + const [textCall, pdfCall] = agentKnowledgeFileStore.write.mock.calls; + expect(textCall[0]).toEqual({ agentId, fileId: expect.any(String) }); expect(textCall[1]).toBeInstanceOf(Readable); expect(textCall[2]).toEqual( expect.objectContaining({ mimeType: 'text/plain', fileName: 'notes.txt' }), @@ -231,8 +222,8 @@ describe('AgentKnowledgeService', () => { ); const [storedTextFile, storedPdfFile] = agentFileRepository.all(); - expect(storedTextFile.binaryDataId).toBe('filesystem-v2:mock-1'); - expect(storedPdfFile.binaryDataId).toBe('filesystem-v2:mock-2'); + expect(storedTextFile.storedAt).toBe('fs'); + expect(storedPdfFile.storedAt).toBe('fs'); await expect(access(textFilePath)).rejects.toThrow(); await expect(access(pdfFilePath)).rejects.toThrow(); @@ -246,7 +237,7 @@ describe('AgentKnowledgeService', () => { ).rejects.toThrow('Invalid knowledge file name'); expect(agentFileRepository.all()).toEqual([]); - expect(binaryDataService.store).not.toHaveBeenCalled(); + expect(agentKnowledgeFileStore.write).not.toHaveBeenCalled(); }); it('cleans up already-stored files when a later upload in the batch fails', async () => { @@ -256,13 +247,16 @@ describe('AgentKnowledgeService', () => { const secondPath = path.join(tempDirectory, 'second.txt'); await writeFile(firstPath, 'hello'); await writeFile(secondPath, 'world'); - binaryDataService.store + agentKnowledgeFileStore.write .mockReset() - .mockImplementationOnce(async (_location, content, binaryData) => { + .mockImplementationOnce(async (ref, content) => { await drainIfStream(content); - return { ...binaryData, id: 'filesystem-v2:mock-1' }; + return { + storedAt: 'fs', + storageKey: `agents/${ref.agentId}/knowledge-files/${ref.fileId}/content`, + }; }) - .mockImplementationOnce(async (_location, content) => { + .mockImplementationOnce(async (_ref, content) => { await drainIfStream(content); throw new Error('store failed'); }); @@ -280,48 +274,11 @@ describe('AgentKnowledgeService', () => { ).rejects.toThrow('store failed'); expect(agentFileRepository.all()).toEqual([]); - expect(binaryDataService.deleteManyByBinaryDataId).toHaveBeenCalledWith([ - 'filesystem-v2:mock-1', + expect(agentKnowledgeFileStore.delete).toHaveBeenCalledWith([ + expect.objectContaining({ storedAt: 'fs', storageKey: expect.any(String) }), ]); }); - it('rejects uploads when the binary data service has no persisted storage mode', async () => { - agentRepository.findByIdAndProjectId.mockResolvedValue({ id: agentId, projectId } as never); - const tempDirectory = await mkdtemp(path.join(tmpdir(), 'agent-knowledge-upload-')); - const filePath = path.join(tempDirectory, 'notes.txt'); - await writeFile(filePath, 'hello'); - binaryDataService.store.mockImplementationOnce(async (_location, content, binaryData) => { - await drainIfStream(content); - return { ...binaryData, id: undefined }; - }); - - await expect( - service.uploadFiles(agentId, projectId, [ - makeMulterFile({ originalname: 'notes.txt', path: filePath, size: 5, buffer: undefined }), - ]), - ).rejects.toThrow('Agent knowledge base requires a persisted binary data storage mode'); - expect(agentFileRepository.all()).toEqual([]); - }); - - it('rejects uploads up front when binary data mode is in-memory', async () => { - agentRepository.findByIdAndProjectId.mockResolvedValue({ id: agentId, projectId } as never); - binaryDataConfig = { mode: 'default' } as BinaryDataConfig; - service = new AgentKnowledgeService( - agentRepository, - agentFileRepository as unknown as AgentFileRepository, - agentKnowledgeSandboxService, - binaryDataService, - binaryDataConfig, - logger, - ); - - await expect( - service.uploadFiles(agentId, projectId, [makeMulterFile({ originalname: 'notes.txt' })]), - ).rejects.toThrow('Agent knowledge base requires a persisted binary data storage mode'); - expect(binaryDataService.store).not.toHaveBeenCalled(); - expect(agentFileRepository.all()).toEqual([]); - }); - it('allows uploads that bring the knowledge base exactly to the size limit', async () => { agentRepository.findByIdAndProjectId.mockResolvedValue({ id: agentId, projectId } as never); await agentFileRepository.save( @@ -352,7 +309,7 @@ describe('AgentKnowledgeService', () => { }), ]); expect(agentFileRepository.all()).toHaveLength(2); - expect(binaryDataService.store).toHaveBeenCalledTimes(1); + expect(agentKnowledgeFileStore.write).toHaveBeenCalledTimes(1); }); it('rejects uploads that would exceed the knowledge base size limit', async () => { @@ -374,7 +331,7 @@ describe('AgentKnowledgeService', () => { ]), ).rejects.toThrow('Knowledge base limit reached'); expect(agentFileRepository.all()).toHaveLength(1); - expect(binaryDataService.store).not.toHaveBeenCalled(); + expect(agentKnowledgeFileStore.write).not.toHaveBeenCalled(); }); it('rejects uploads when existing knowledge files already exceed the size limit', async () => { @@ -396,48 +353,42 @@ describe('AgentKnowledgeService', () => { ]), ).rejects.toThrow('Knowledge base limit reached'); expect(agentFileRepository.all()).toHaveLength(1); - expect(binaryDataService.store).not.toHaveBeenCalled(); + expect(agentKnowledgeFileStore.write).not.toHaveBeenCalled(); }); - it('deletes the DB row and its binary data', async () => { + it('deletes the DB row and its blob', async () => { agentRepository.findByIdAndProjectId.mockResolvedValue({ id: agentId, projectId } as never); - await agentFileRepository.save( - makeAgentFile({ id: 'file-1', binaryDataId: 'filesystem-v2:file-1.txt' }), - ); + await agentFileRepository.save(makeAgentFile({ id: 'file-1', storedAt: 'fs' })); await expect(service.deleteFile(agentId, projectId, 'file-1')).resolves.toBeUndefined(); expect(agentFileRepository.all()).toEqual([]); - expect(binaryDataService.deleteManyByBinaryDataId).toHaveBeenCalledWith([ - 'filesystem-v2:file-1.txt', + expect(agentKnowledgeFileStore.delete).toHaveBeenCalledWith([ + { storedAt: 'fs', storageKey: `agents/${agentId}/knowledge-files/file-1/content` }, ]); }); - it('logs binary data deletion failures without restoring the DB row', async () => { + it('logs blob deletion failures without restoring the DB row', async () => { agentRepository.findByIdAndProjectId.mockResolvedValue({ id: agentId, projectId } as never); - await agentFileRepository.save( - makeAgentFile({ id: 'file-1', binaryDataId: 'filesystem-v2:file-1.txt' }), - ); - binaryDataService.deleteManyByBinaryDataId.mockRejectedValueOnce(new Error('delete failed')); + await agentFileRepository.save(makeAgentFile({ id: 'file-1', storedAt: 'fs' })); + agentKnowledgeFileStore.delete.mockRejectedValueOnce(new Error('delete failed')); await expect(service.deleteFile(agentId, projectId, 'file-1')).resolves.toBeUndefined(); expect(agentFileRepository.all()).toEqual([]); - expect(logger.warn).toHaveBeenCalledWith('Failed to delete knowledge file binary data', { + expect(logger.warn).toHaveBeenCalledWith('Failed to delete knowledge file blob', { agentId, fileId: 'file-1', error: 'delete failed', }); }); - it('deletes all agent files and their binary data', async () => { - await agentFileRepository.save( - makeAgentFile({ id: 'file-1', binaryDataId: 'filesystem-v2:file-1.txt' }), - ); + it('deletes all agent files and their blobs', async () => { + await agentFileRepository.save(makeAgentFile({ id: 'file-1', storedAt: 'fs' })); await agentFileRepository.save( makeAgentFile({ id: 'file-2', - binaryDataId: 'filesystem-v2:file-2.md', + storedAt: 's3', fileName: 'guide.md', mimeType: 'text/markdown', }), @@ -446,11 +397,10 @@ describe('AgentKnowledgeService', () => { await expect(service.deleteAllFilesForAgent(projectId, agentId)).resolves.toBeUndefined(); expect(agentFileRepository.all()).toEqual([]); - expect(binaryDataService.deleteManyByBinaryDataId).toHaveBeenCalledWith([ - 'filesystem-v2:file-1.txt', - 'filesystem-v2:file-2.md', + expect(agentKnowledgeFileStore.delete).toHaveBeenCalledWith([ + { storedAt: 'fs', storageKey: `agents/${agentId}/knowledge-files/file-1/content` }, + { storedAt: 's3', storageKey: `agents/${agentId}/knowledge-files/file-2/content` }, ]); - expect(binaryDataService.deleteMany).not.toHaveBeenCalled(); }); it('delegates warmup to the sandbox service for an unpublished agent', async () => { diff --git a/packages/cli/src/modules/agents/agent-knowledge-file-store.ts b/packages/cli/src/modules/agents/agent-knowledge-file-store.ts new file mode 100644 index 00000000000..7bcd2380346 --- /dev/null +++ b/packages/cli/src/modules/agents/agent-knowledge-file-store.ts @@ -0,0 +1,135 @@ +import { binaryToBuffer } from '@n8n/backend-network'; +import { + ByteStoreRegistry, + SkippedEntryDeletionError, + type ByteStore, + type PreWriteBlobMetadata, + type StorageLocation, +} from '@n8n/blob-storage'; +import { BinaryDataRepository, type ExecutionDataStorageLocation } from '@n8n/db'; +import { Service } from '@n8n/di'; +import { ErrorReporter, FsByteStoreService, StorageConfig } from 'n8n-core'; +import type { Readable } from 'node:stream'; +import { v4 as uuid } from 'uuid'; + +/** Where a knowledge file's bytes live, as recorded on its `agent_files` row. */ +export type StoredAgentKnowledgeFile = { + storedAt: ExecutionDataStorageLocation; + storageKey: string; +}; + +/** + * Marks a key written by the former BinaryDataService layout. On `fs` those + * carry a companion `.metadata` entry, which has to go with the content. + */ +const LEGACY_KEY_SEGMENT = '/binary_data/'; + +/** + * Stores agent knowledge file bytes wherever the execution data storage mode + * points: in `db` mode the bytes go into the `binary_data` table keyed by a + * uuid, otherwise into a byte store keyed by a path. The `fs` byte store is + * always available; `s3` and `az` are registered at module init when configured. + * + * Keys are generated on write and persisted, so files written by the former + * BinaryDataService layout keep resolving under their original key. + */ +@Service() +export class AgentKnowledgeFileStore { + private readonly byteStores: ByteStoreRegistry; + + constructor( + fsByteStore: FsByteStoreService, + private readonly storageConfig: StorageConfig, + private readonly binaryDataRepository: BinaryDataRepository, + private readonly errorReporter: ErrorReporter, + ) { + this.byteStores = new ByteStoreRegistry({ fs: fsByteStore }); + } + + registerByteStore(loc: StorageLocation, store: ByteStore) { + this.byteStores.register(loc, store); + } + + async write( + ref: { agentId: string; fileId: string }, + body: Buffer | Readable, + metadata: PreWriteBlobMetadata, + ): Promise { + const storedAt = this.storageConfig.modeTag; + + if (storedAt === 'db') { + const buffer = await binaryToBuffer(body); + const storageKey = uuid(); + await this.binaryDataRepository.insert({ + fileId: storageKey, + sourceType: 'agent_file', + sourceId: ref.fileId, + data: buffer, + mimeType: metadata.mimeType ?? null, + fileName: metadata.fileName ?? null, + fileSize: buffer.length, + }); + return { storedAt, storageKey }; + } + + const storageKey = this.newKeyFor(ref); + await this.byteStores.get(storedAt).write(storageKey, body, metadata); + return { storedAt, storageKey }; + } + + async readAsBuffer(file: StoredAgentKnowledgeFile): Promise { + if (file.storedAt === 'db') { + return await this.binaryDataRepository.findContentByFileId(file.storageKey); + } + + return await this.byteStores.get(file.storedAt).read(file.storageKey); + } + + async delete(files: StoredAgentKnowledgeFile[]): Promise { + if (files.length === 0) return; + + const groups = new Map(); + for (const file of files) { + const group = groups.get(file.storedAt) ?? []; + group.push(file); + groups.set(file.storedAt, group); + } + + await Promise.all( + [...groups].map(async ([loc, group]) => { + if (loc === 'db') { + await this.binaryDataRepository.deleteByFileIds(group.map((file) => file.storageKey)); + return; + } + + const store = this.byteStores.find(loc); + if (!store) { + this.errorReporter.error(new SkippedEntryDeletionError(loc, group.length)); + return; + } + await store.delete(this.keysToDelete(loc, group)); + }), + ); + } + + private keysToDelete(loc: StorageLocation, files: StoredAgentKnowledgeFile[]): string[] { + // s3 and az keep metadata on the object itself, so only fs has a companion. + if (loc !== 'fs') return files.map((file) => file.storageKey); + + return files.flatMap((file) => + file.storageKey.includes(LEGACY_KEY_SEGMENT) + ? [file.storageKey, `${file.storageKey}.metadata`] + : [file.storageKey], + ); + } + + private newKeyFor(ref: { agentId: string; fileId: string }) { + return [ + 'agents', + encodeURIComponent(ref.agentId), + 'knowledge-files', + encodeURIComponent(ref.fileId), + 'content', + ].join('/'); + } +} diff --git a/packages/cli/src/modules/agents/agent-knowledge-retrieval.ts b/packages/cli/src/modules/agents/agent-knowledge-retrieval.ts index f9288f59f34..11e845aca97 100644 --- a/packages/cli/src/modules/agents/agent-knowledge-retrieval.ts +++ b/packages/cli/src/modules/agents/agent-knowledge-retrieval.ts @@ -1,6 +1,8 @@ -import { BadRequestError } from '@/errors/response-errors/bad-request.error'; +import type { ExecutionDataStorageLocation } from '@n8n/db'; import { z } from 'zod'; +import { BadRequestError } from '@/errors/response-errors/bad-request.error'; + import { hasControlCharacter } from './agent-knowledge-storage'; const MAX_SEARCH_PATTERN_LENGTH = 500; @@ -186,7 +188,8 @@ export type ReadKnowledgeRequest = z.infer; export interface AgentKnowledgeFileReference { file: string; fileId: string; - binaryDataId: string; + storedAt: ExecutionDataStorageLocation; + storageKey: string; displayName: string; mimeType: string; fileSizeBytes: number; diff --git a/packages/cli/src/modules/agents/agent-knowledge-sandbox.service.ts b/packages/cli/src/modules/agents/agent-knowledge-sandbox.service.ts index 0ac940c138f..81dcc4aab85 100644 --- a/packages/cli/src/modules/agents/agent-knowledge-sandbox.service.ts +++ b/packages/cli/src/modules/agents/agent-knowledge-sandbox.service.ts @@ -5,7 +5,7 @@ import { Logger } from '@n8n/backend-common'; import { AgentsConfig } from '@n8n/config'; import { Time } from '@n8n/constants'; import { Service } from '@n8n/di'; -import { BinaryDataService, InstanceSettings } from 'n8n-core'; +import { InstanceSettings } from 'n8n-core'; import { OperationalError } from 'n8n-workflow'; import { nanoid } from 'nanoid'; import { createHash } from 'node:crypto'; @@ -30,6 +30,7 @@ import { parseRipgrepFilesOutput, parseRipgrepOutput, } from './agent-knowledge-commands'; +import { AgentKnowledgeFileStore } from './agent-knowledge-file-store'; import { isAgentKnowledgeBaseEnabled } from './agent-knowledge-gate'; import { assertValidKnowledgeFilePath, @@ -216,7 +217,7 @@ export class AgentKnowledgeSandboxService { private readonly instanceSettings: InstanceSettings, private readonly agentFileRepository: AgentFileRepository, private readonly agentRepository: AgentRepository, - private readonly binaryDataService: BinaryDataService, + private readonly agentKnowledgeFileStore: AgentKnowledgeFileStore, ) {} async warmSandbox(projectId: string, agentId: string): Promise { @@ -497,8 +498,8 @@ export class AgentKnowledgeSandboxService { const filesByName = new Map(files.map((file) => [file.file, file])); const copiedNames = await this.uploadMirrorFiles(sandbox, toCopy, filesByName, sandboxName); - // Files that failed to load from BinaryDataService are left out of the - // manifest, so the next sync attempt retries them as `toCopy` again. + // Files that failed to load from the knowledge file store are left out of + // the manifest, so the next sync attempt retries them as `toCopy` again. const finalManifestNames = expectedNames.filter( (name) => copiedNames.has(name) || presentSet.has(name), ); @@ -521,11 +522,11 @@ export class AgentKnowledgeSandboxService { } /** - * Fetches each `names` entry from BinaryDataService and uploads it to the - * sandbox mirror under a `.tmp-` prefix; `buildMirrorFinalizeCommand` moves - * it into place so a concurrent search never sees a partially-written file. - * Uploads flush in `MIRROR_UPLOAD_BATCH_BYTES`-sized batches so the whole - * knowledge base (up to 1.5 GB) is never held in memory at once. + * Fetches each `names` entry from the knowledge file store and uploads it to + * the sandbox mirror under a `.tmp-` prefix; `buildMirrorFinalizeCommand` + * moves it into place so a concurrent search never sees a partially-written + * file. Uploads flush in `MIRROR_UPLOAD_BATCH_BYTES`-sized batches so the + * whole knowledge base (up to 1.5 GB) is never held in memory at once. * Returns the subset of `names` that were fetched and uploaded successfully. */ private async uploadMirrorFiles( @@ -557,11 +558,18 @@ export class AgentKnowledgeSandboxService { if (!file) continue; try { - const buffer = await this.binaryDataService.getAsBuffer({ - id: file.binaryDataId, - data: '', - mimeType: file.mimeType, + const buffer = await this.agentKnowledgeFileStore.readAsBuffer({ + storedAt: file.storedAt, + storageKey: file.storageKey, }); + if (!buffer) { + this.logger.warn('Failed to load agent knowledge file for mirror sync', { + sandboxName, + file: name, + error: 'not found', + }); + continue; + } batch.push({ source: buffer, destination: `${KNOWLEDGE_MIRROR_FILES_DIR}/.tmp-${name}` }); batchNames.push(name); batchBytes += buffer.length; @@ -639,7 +647,8 @@ export class AgentKnowledgeSandboxService { return files.map((file) => ({ file: storageFileNameForOriginalFileName(file.fileName), fileId: file.id, - binaryDataId: file.binaryDataId, + storedAt: file.storedAt, + storageKey: file.storageKey, displayName: file.fileName, mimeType: file.mimeType, fileSizeBytes: file.fileSizeBytes, diff --git a/packages/cli/src/modules/agents/agent-knowledge-storage.ts b/packages/cli/src/modules/agents/agent-knowledge-storage.ts index d0a162aaa9c..2b9a0be306b 100644 --- a/packages/cli/src/modules/agents/agent-knowledge-storage.ts +++ b/packages/cli/src/modules/agents/agent-knowledge-storage.ts @@ -1,22 +1,16 @@ import type { AgentFileDto } from '@n8n/api-types'; -import type { SourceType } from '@n8n/db'; -import { FileLocation } from 'n8n-core'; import path from 'node:path'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import type { AgentFile } from './entities/agent-file.entity'; -// Local sandbox disk mirroring the DB-backed knowledge files, so repeated -// reads/searches avoid re-fetching from BinaryDataService each time. +// Local sandbox disk mirroring the persisted knowledge files, so repeated +// reads/searches avoid re-fetching from the knowledge file store each time. export const KNOWLEDGE_MIRROR_DIR = '/home/daytona/knowledge-mirror'; export const KNOWLEDGE_MIRROR_FILES_DIR = `${KNOWLEDGE_MIRROR_DIR}/files`; export const KNOWLEDGE_MIRROR_MANIFEST = `${KNOWLEDGE_MIRROR_DIR}/manifest`; -// Typed against `SourceType` so a drift from the `binary_data` schema enum -// (see `packages/@n8n/db/src/entities/binary-data-file.ts`) is a compile error. -const AGENT_FILE_SOURCE_TYPE: SourceType = 'agent_file'; - export function hasControlCharacter(value: string): boolean { for (const character of value) { if (character.charCodeAt(0) < 32) { @@ -33,15 +27,6 @@ function sanitizePathCharacter(character: string): string { return character; } -/** One directory per file, so deleting a single file never touches others. */ -export function buildKnowledgeFileLocation(agentId: string, fileId: string) { - return FileLocation.ofCustom({ - pathSegments: ['agents', agentId, 'knowledge-files', fileId], - sourceType: AGENT_FILE_SOURCE_TYPE, - sourceId: fileId, - }); -} - export function assertKnowledgePathSegment(segment: string, label: string): void { if ( !segment || diff --git a/packages/cli/src/modules/agents/agent-knowledge.service.ts b/packages/cli/src/modules/agents/agent-knowledge.service.ts index 60c5dd979f0..a9694e519f7 100644 --- a/packages/cli/src/modules/agents/agent-knowledge.service.ts +++ b/packages/cli/src/modules/agents/agent-knowledge.service.ts @@ -8,8 +8,6 @@ import { Logger } from '@n8n/backend-common'; import { Service } from '@n8n/di'; import { QueryFailedError } from '@n8n/typeorm'; import { generateNanoId } from '@n8n/utils/generate-nano-id'; -import { BinaryDataConfig, BinaryDataService } from 'n8n-core'; -import { OperationalError, type IBinaryData } from 'n8n-workflow'; import { createReadStream } from 'node:fs'; import { unlink } from 'node:fs/promises'; import path from 'node:path'; @@ -18,10 +16,10 @@ import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; import { - buildKnowledgeFileLocation, - storageFileNameForOriginalFileName, - toAgentFileDto, -} from './agent-knowledge-storage'; + AgentKnowledgeFileStore, + type StoredAgentKnowledgeFile, +} from './agent-knowledge-file-store'; +import { storageFileNameForOriginalFileName, toAgentFileDto } from './agent-knowledge-storage'; import { AgentKnowledgeSandboxService } from './agent-knowledge-sandbox.service'; import type { AgentFile } from './entities/agent-file.entity'; import { AgentFileRepository } from './repositories/agent-file.repository'; @@ -51,8 +49,7 @@ export class AgentKnowledgeService { private readonly agentRepository: AgentRepository, private readonly agentFileRepository: AgentFileRepository, private readonly agentKnowledgeSandboxService: AgentKnowledgeSandboxService, - private readonly binaryDataService: BinaryDataService, - private readonly binaryDataConfig: BinaryDataConfig, + private readonly agentKnowledgeFileStore: AgentKnowledgeFileStore, private readonly logger: Logger, ) {} @@ -63,11 +60,6 @@ export class AgentKnowledgeService { ): Promise { try { await this.ensureAgentBelongsToProject(agentId, projectId); - if (this.binaryDataConfig.mode === 'default') { - throw new OperationalError( - 'Agent knowledge base requires a persisted binary data storage mode', - ); - } this.validateUploadMetadata(files); await this.validateUploadBatch(agentId, files); @@ -110,13 +102,15 @@ export class AgentKnowledgeService { } await this.agentFileRepository.delete({ id: fileId, agentId }); - await this.binaryDataService.deleteManyByBinaryDataId([file.binaryDataId]).catch((error) => { - this.logger.warn('Failed to delete knowledge file binary data', { - agentId, - fileId: file.id, - error: error instanceof Error ? error.message : error, + await this.agentKnowledgeFileStore + .delete([{ storedAt: file.storedAt, storageKey: file.storageKey }]) + .catch((error) => { + this.logger.warn('Failed to delete knowledge file blob', { + agentId, + fileId: file.id, + error: error instanceof Error ? error.message : error, + }); }); - }); this.agentKnowledgeSandboxService.invalidateMirror(projectId, agentId); this.agentKnowledgeSandboxService.prewarmMirrorInBackground(projectId, agentId); } @@ -125,10 +119,15 @@ export class AgentKnowledgeService { const files = await this.agentFileRepository.findByAgentId(agentId); await this.agentFileRepository.delete({ agentId }); if (files.length > 0) { - await this.binaryDataService - .deleteManyByBinaryDataId(files.map((file) => file.binaryDataId)) + await this.agentKnowledgeFileStore + .delete( + files.map((file) => ({ + storedAt: file.storedAt, + storageKey: file.storageKey, + })), + ) .catch((error) => { - this.logger.warn('Failed to delete knowledge files binary data', { + this.logger.warn('Failed to delete knowledge file blobs', { agentId, error: error instanceof Error ? error.message : error, }); @@ -142,32 +141,21 @@ export class AgentKnowledgeService { await this.agentKnowledgeSandboxService.destroySandbox(projectId, agentId); } - /** Stores the file's bytes via BinaryDataService, then reserves its DB row. */ + /** Stores the file's bytes via AgentKnowledgeFileStore, then reserves its DB row. */ private async storeAgentFile(agentId: string, file: Express.Multer.File): Promise { const fileId = generateNanoId(); const storageFileName = storageFileNameForOriginalFileName(file.originalname); const content = await this.prepareUploadContent(file); - const binaryData: IBinaryData = { - data: '', - mimeType: file.mimetype, + const stored = await this.agentKnowledgeFileStore.write({ agentId, fileId }, content, { fileName: storageFileName, - }; - const stored = await this.binaryDataService.store( - buildKnowledgeFileLocation(agentId, fileId), - content, - binaryData, - ); - if (!stored.id) { - throw new OperationalError( - 'Agent knowledge base requires a persisted binary data storage mode', - ); - } + mimeType: file.mimetype, + }); try { - return await this.saveAgentFile(agentId, fileId, file, stored.id); + return await this.saveAgentFile(agentId, fileId, file, stored); } catch (error) { - await this.binaryDataService.deleteManyByBinaryDataId([stored.id]).catch(() => {}); + await this.agentKnowledgeFileStore.delete([stored]).catch(() => {}); if (isUniqueConstraintError(error)) { throw this.duplicateFileNameError(file.originalname); } @@ -179,12 +167,13 @@ export class AgentKnowledgeService { agentId: string, fileId: string, file: Express.Multer.File, - binaryDataId: string, + stored: StoredAgentKnowledgeFile, ): Promise { const agentFile = this.agentFileRepository.create({ id: fileId, agentId, - binaryDataId, + storedAt: stored.storedAt, + storageKey: stored.storageKey, fileName: file.originalname, mimeType: file.mimetype, fileSizeBytes: file.size, @@ -209,7 +198,9 @@ export class AgentKnowledgeService { private async cleanupUploadedFiles(files: AgentFile[]): Promise { for (const file of files) { - await this.binaryDataService.deleteManyByBinaryDataId([file.binaryDataId]).catch(() => {}); + await this.agentKnowledgeFileStore + .delete([{ storedAt: file.storedAt, storageKey: file.storageKey }]) + .catch(() => {}); await this.agentFileRepository.delete({ id: file.id, agentId: file.agentId }).catch(() => {}); } } diff --git a/packages/cli/src/modules/agents/agents.module.ts b/packages/cli/src/modules/agents/agents.module.ts index 10a87e37550..fb833a012aa 100644 --- a/packages/cli/src/modules/agents/agents.module.ts +++ b/packages/cli/src/modules/agents/agents.module.ts @@ -34,32 +34,23 @@ export class AgentsModule implements ModuleInterface { const { AgentExecutionService } = await import('./agent-execution.service.js'); Container.get(AgentExecutionService); - // Register blob backends for agent execution logs. The fs backend is always - // available; s3/az reuse the clients base-command already initialized - // (initBinaryDataService runs before module init in all commands) and are - // only registered when that init succeeded — mirroring ExecutionDataJsonStore, - // so a configured-but-unreachable backend degrades instead of throwing. - const { AzureByteStore, S3ByteStore } = await import('@n8n/blob-storage'); + // Register blob backends for agent execution logs and knowledge files. + // The fs backend is always available; s3/az reuse the clients base-command + // already initialized (initBinaryDataService runs before module init in + // all commands), which exits the process when the configured execution + // data backend cannot be reached — so a live process in s3/az mode always + // has that location registered here. const { AgentExecutionLogStore } = await import('./execution-log/agent-execution-log-store.js'); + const { AgentKnowledgeFileStore } = await import('./agent-knowledge-file-store.js'); const { ExecutionDataJsonStore } = await import( '@/executions/execution-data/execution-data-json-store.js' ); - const agentExecutionLogStore = Container.get(AgentExecutionLogStore); - const executionDataJsonStore = Container.get(ExecutionDataJsonStore); - if (executionDataJsonStore.hasLocation('s3')) { - const { ObjectStoreService } = await import('@n8n/blob-storage/object-store'); - agentExecutionLogStore.registerByteStore( - 's3', - new S3ByteStore(Container.get(ObjectStoreService)), - ); - } - if (executionDataJsonStore.hasLocation('az')) { - const { AzureBlobService } = await import('@n8n/blob-storage/azure-blob'); - agentExecutionLogStore.registerByteStore( - 'az', - new AzureByteStore(Container.get(AzureBlobService)), - ); - } + const { registerAgentBlobByteStores } = await import('./register-blob-byte-stores.js'); + await registerAgentBlobByteStores({ + executionDataJsonStore: Container.get(ExecutionDataJsonStore), + agentExecutionLogStore: Container.get(AgentExecutionLogStore), + agentKnowledgeFileStore: Container.get(AgentKnowledgeFileStore), + }); const { AgentRuntimeCacheService } = await import('./agent-runtime-cache.service.js'); Container.get(AgentRuntimeCacheService); diff --git a/packages/cli/src/modules/agents/entities/agent-file.entity.ts b/packages/cli/src/modules/agents/entities/agent-file.entity.ts index 1c1548f0a38..4a07e705ff9 100644 --- a/packages/cli/src/modules/agents/entities/agent-file.entity.ts +++ b/packages/cli/src/modules/agents/entities/agent-file.entity.ts @@ -1,12 +1,17 @@ -import { WithTimestampsAndStringId } from '@n8n/db'; +import { WithTimestampsAndStringId, type ExecutionDataStorageLocation } from '@n8n/db'; import { Column, Entity, Index, JoinColumn, ManyToOne, type Relation } from '@n8n/typeorm'; import { Agent } from './agent.entity'; +/** + * The table still carries a nullable `binaryDataId` and its unique index, left + * behind by `AddAgentFileStorageColumns` so the previous release can read the + * rows it wrote. It is deliberately absent here — new rows leave it null — and + * a follow-up migration drops it. + */ @Entity({ name: 'agent_files' }) @Index(['agentId', 'createdAt']) @Index(['agentId', 'fileName'], { unique: true }) -@Index(['agentId', 'binaryDataId'], { unique: true }) export class AgentFile extends WithTimestampsAndStringId { @ManyToOne(() => Agent, { onDelete: 'CASCADE' }) @JoinColumn({ name: 'agentId' }) @@ -16,17 +21,25 @@ export class AgentFile extends WithTimestampsAndStringId { agentId: string; /** - * BinaryDataService id (e.g. `filesystem-v2:agents//knowledge-files//binary_data/`). - * Not a DB FK — see `BinaryDataService` in `n8n-core` for how the bytes are resolved from this id. + * Where the file bytes live: 'db' (the `binary_data` table) or a blob-storage + * backend ('fs', 's3', 'az'). Bytes are resolved via AgentKnowledgeFileStore. + */ + @Column({ type: 'varchar', length: 2, nullable: false, default: 'db' }) + storedAt: ExecutionDataStorageLocation; + + /** + * Key addressing the bytes within `storedAt`: a `binary_data.fileId` for + * 'db', a byte-store key otherwise. Persisted rather than derived, so files + * written under the former BinaryDataService layout keep resolving after the + * storage migration. */ @Column({ type: 'text' }) - binaryDataId: string; + storageKey: string; // fileName/mimeType/fileSizeBytes are intentionally denormalized rather than - // joined from binary_data: (1) binaryDataId is an opaque storage reference, - // not an FK; (2) we keep the original user-facing values, which differ from - // the stored binary for converted uploads (a PDF is stored as extracted - // text with a different byte size). + // joined from storage metadata: we keep the original user-facing values, + // which differ from the stored bytes for converted uploads (a PDF is stored + // as extracted text with a different byte size). @Column({ type: 'varchar', length: 255 }) fileName: string; diff --git a/packages/cli/src/modules/agents/execution-log/__tests__/agent-execution-log-store.test.ts b/packages/cli/src/modules/agents/execution-log/__tests__/agent-execution-log-store.test.ts index 0158a7687fb..498b7117e4c 100644 --- a/packages/cli/src/modules/agents/execution-log/__tests__/agent-execution-log-store.test.ts +++ b/packages/cli/src/modules/agents/execution-log/__tests__/agent-execution-log-store.test.ts @@ -1,14 +1,11 @@ +import type { ErrorReporter, FsByteStoreService } from 'n8n-core'; import { mock } from 'vitest-mock-extended'; -import type { ErrorReporter } from 'n8n-core'; -import { - AgentExecutionLogStore, - type AgentExecutionLogFsByteStore, -} from '../agent-execution-log-store'; +import { AgentExecutionLogStore } from '../agent-execution-log-store'; describe('AgentExecutionLogStore', () => { it('keeps a threadId carrying path separators within the agent prefix', async () => { - const fsByteStore = mock(); + const fsByteStore = mock(); const store = new AgentExecutionLogStore(fsByteStore, mock()); await store.write( diff --git a/packages/cli/src/modules/agents/execution-log/agent-execution-log-store.ts b/packages/cli/src/modules/agents/execution-log/agent-execution-log-store.ts index 9f5d7c30b7e..bbe49d014d1 100644 --- a/packages/cli/src/modules/agents/execution-log/agent-execution-log-store.ts +++ b/packages/cli/src/modules/agents/execution-log/agent-execution-log-store.ts @@ -1,6 +1,6 @@ -import { FsByteStore, JsonStore } from '@n8n/blob-storage'; +import { JsonStore } from '@n8n/blob-storage'; import { Service } from '@n8n/di'; -import { ErrorReporter, StorageConfig } from 'n8n-core'; +import { ErrorReporter, FsByteStoreService } from 'n8n-core'; import { AgentExecutionLogWriteError } from './agent-execution-log-write.error'; import { CorruptedAgentExecutionLogError } from './corrupted-agent-execution-log.error'; @@ -11,16 +11,6 @@ export type AgentExecutionLogPayload = { timeline: TimelineEvent[] }; const AGENT_EXECUTION_LOG_VERSION = 1; -@Service() -export class AgentExecutionLogFsByteStore extends FsByteStore { - constructor(storageConfig: StorageConfig, errorReporter: ErrorReporter) { - super({ - storagePath: storageConfig.storagePath, - reportError: (error) => errorReporter.error(error), - }); - } -} - /** * Stores agent execution timelines as JSON blobs. The `fs` backend is always * available, but `s3` and `az` are registered at module init only if configured. @@ -30,7 +20,7 @@ export class AgentExecutionLogStore extends JsonStore< AgentExecutionLogRef, AgentExecutionLogPayload > { - constructor(fsByteStore: AgentExecutionLogFsByteStore, errorReporter: ErrorReporter) { + constructor(fsByteStore: FsByteStoreService, errorReporter: ErrorReporter) { super({ byteStores: { fs: fsByteStore }, version: AGENT_EXECUTION_LOG_VERSION, diff --git a/packages/cli/src/modules/agents/register-blob-byte-stores.ts b/packages/cli/src/modules/agents/register-blob-byte-stores.ts new file mode 100644 index 00000000000..ca2c3430cbd --- /dev/null +++ b/packages/cli/src/modules/agents/register-blob-byte-stores.ts @@ -0,0 +1,34 @@ +import { AzureByteStore, S3ByteStore } from '@n8n/blob-storage'; +import { Container } from '@n8n/di'; + +import type { ExecutionDataJsonStore } from '@/executions/execution-data/execution-data-json-store'; + +import type { AgentKnowledgeFileStore } from './agent-knowledge-file-store'; +import type { AgentExecutionLogStore } from './execution-log/agent-execution-log-store'; + +/** + * Registers s3/az byte stores on the agent execution-log and knowledge-file + * stores when ExecutionDataJsonStore already has those locations configured + * (i.e. base-command init succeeded). The fs backend is always available on + * both stores via their constructors. + */ +export async function registerAgentBlobByteStores(stores: { + executionDataJsonStore: ExecutionDataJsonStore; + agentExecutionLogStore: AgentExecutionLogStore; + agentKnowledgeFileStore: AgentKnowledgeFileStore; +}): Promise { + const { executionDataJsonStore, agentExecutionLogStore, agentKnowledgeFileStore } = stores; + + if (executionDataJsonStore.hasLocation('s3')) { + const { ObjectStoreService } = await import('@n8n/blob-storage/object-store'); + const s3Store = new S3ByteStore(Container.get(ObjectStoreService)); + agentExecutionLogStore.registerByteStore('s3', s3Store); + agentKnowledgeFileStore.registerByteStore('s3', s3Store); + } + if (executionDataJsonStore.hasLocation('az')) { + const { AzureBlobService } = await import('@n8n/blob-storage/azure-blob'); + const azStore = new AzureByteStore(Container.get(AzureBlobService)); + agentExecutionLogStore.registerByteStore('az', azStore); + agentKnowledgeFileStore.registerByteStore('az', azStore); + } +} diff --git a/packages/cli/test/migration/1785186578138-add-agent-file-storage-columns.test.ts b/packages/cli/test/migration/1785186578138-add-agent-file-storage-columns.test.ts new file mode 100644 index 00000000000..2b13a135d97 --- /dev/null +++ b/packages/cli/test/migration/1785186578138-add-agent-file-storage-columns.test.ts @@ -0,0 +1,288 @@ +import { + createTestMigrationContext, + initDbUpToMigration, + runSingleMigration, + undoLastSingleMigration, + type TestMigrationContext, +} from '@n8n/backend-test-utils'; +import { DbConnection } from '@n8n/db'; +import { Container } from '@n8n/di'; +import { DataSource } from '@n8n/typeorm'; + +const MIGRATION_NAME = 'AddAgentFileStorageColumns1785186578138'; + +const FS_KEY = + 'agents/agent-1/knowledge-files/file-fs/binary_data/11111111-1111-1111-1111-111111111111'; +const LEGACY_FS_KEY = + 'agents/agent-1/knowledge-files/file-fs-v1/binary_data/55555555-5555-5555-5555-555555555555'; +const S3_KEY = + 'agents/agent-1/knowledge-files/file-s3/binary_data/22222222-2222-2222-2222-222222222222'; +const DB_FILE_ID = '33333333-3333-3333-3333-333333333333'; +const EXECUTION_FILE_ID = '44444444-4444-4444-4444-444444444444'; +const NEW_ROW_KEY = 'agents/agent-1/knowledge-files/file-new/content'; + +type ColumnInfo = { name: string; nullable: boolean }; + +async function columnInfo(context: TestMigrationContext, table: string): Promise { + if (context.isSqlite) { + const rows = await context.runQuery>( + `PRAGMA table_info(${context.escape.tableName(table)})`, + ); + return rows.map((row) => ({ name: row.name, nullable: Number(row.notnull) === 0 })); + } + const rows = await context.runQuery>( + 'SELECT column_name, is_nullable FROM information_schema.columns WHERE table_name = :name', + { name: `${context.tablePrefix}${table}` }, + ); + return rows.map((row) => ({ name: row.column_name, nullable: row.is_nullable === 'YES' })); +} + +describe('AddAgentFileStorageColumns Migration', () => { + let dataSource: DataSource; + + beforeAll(async () => { + const dbConnection = Container.get(DbConnection); + await dbConnection.init(); + + dataSource = Container.get(DataSource); + + const context = createTestMigrationContext(dataSource); + await context.queryRunner.clearDatabase(); + await context.queryRunner.release(); + + await initDbUpToMigration(MIGRATION_NAME); + }); + + afterAll(async () => { + const dbConnection = Container.get(DbConnection); + await dbConnection.close(); + }); + + async function insertProject(context: TestMigrationContext, id: string): Promise { + const table = context.escape.tableName('project'); + const now = new Date(); + await context.runQuery( + `INSERT INTO ${table} ("id", "name", "type", "customTelemetryTags", "createdAt", "updatedAt") VALUES (:id, :name, :type, :tags, :createdAt, :updatedAt)`, + { id, name: 'Test Project', type: 'team', tags: '[]', createdAt: now, updatedAt: now }, + ); + } + + async function insertAgent( + context: TestMigrationContext, + data: { id: string; projectId: string }, + ): Promise { + const table = context.escape.tableName('agents'); + const now = new Date(); + await context.runQuery( + `INSERT INTO ${table} ("id", "name", "projectId", "integrations", "tools", "skills", "createdAt", "updatedAt") VALUES (:id, :name, :projectId, :integrations, :tools, :skills, :createdAt, :updatedAt)`, + { + id: data.id, + name: 'Test Agent', + projectId: data.projectId, + integrations: '[]', + tools: '{}', + skills: '{}', + createdAt: now, + updatedAt: now, + }, + ); + } + + async function insertAgentFile( + context: TestMigrationContext, + data: { id: string; binaryDataId: string }, + ): Promise { + const table = context.escape.tableName('agent_files'); + const now = new Date(); + await context.runQuery( + `INSERT INTO ${table} ("id", "agentId", "binaryDataId", "fileName", "mimeType", "fileSizeBytes", "createdAt", "updatedAt") VALUES (:id, :agentId, :binaryDataId, :fileName, :mimeType, :fileSizeBytes, :createdAt, :updatedAt)`, + { + id: data.id, + agentId: 'agent-1', + binaryDataId: data.binaryDataId, + fileName: `${data.id}.txt`, + mimeType: 'text/plain', + fileSizeBytes: 11, + createdAt: now, + updatedAt: now, + }, + ); + } + + /** Inserts the way the post-migration code does: storage columns, no `binaryDataId`. */ + async function insertMigratedAgentFile( + context: TestMigrationContext, + data: { id: string; storedAt: string; storageKey: string }, + ): Promise { + const table = context.escape.tableName('agent_files'); + const now = new Date(); + await context.runQuery( + `INSERT INTO ${table} ("id", "agentId", "storedAt", "storageKey", "fileName", "mimeType", "fileSizeBytes", "createdAt", "updatedAt") VALUES (:id, :agentId, :storedAt, :storageKey, :fileName, :mimeType, :fileSizeBytes, :createdAt, :updatedAt)`, + { + id: data.id, + agentId: 'agent-1', + storedAt: data.storedAt, + storageKey: data.storageKey, + fileName: `${data.id}.txt`, + mimeType: 'text/plain', + fileSizeBytes: 11, + createdAt: now, + updatedAt: now, + }, + ); + } + + async function insertBinaryData( + context: TestMigrationContext, + data: { fileId: string; sourceType: string; sourceId: string }, + ): Promise { + const table = context.escape.tableName('binary_data'); + const now = new Date(); + await context.runQuery( + `INSERT INTO ${table} ("fileId", "sourceType", "sourceId", "data", "mimeType", "fileName", "fileSize", "createdAt", "updatedAt") VALUES (:fileId, :sourceType, :sourceId, :data, :mimeType, :fileName, :fileSize, :createdAt, :updatedAt)`, + { + fileId: data.fileId, + sourceType: data.sourceType, + sourceId: data.sourceId, + data: Buffer.from('bytes'), + mimeType: 'text/plain', + fileName: 'notes.txt', + fileSize: 5, + createdAt: now, + updatedAt: now, + }, + ); + } + + it('backfills storedAt plus storageKey from every binaryDataId prefix', async () => { + const seedContext = createTestMigrationContext(dataSource); + try { + await insertProject(seedContext, 'project-1'); + await insertAgent(seedContext, { id: 'agent-1', projectId: 'project-1' }); + + await insertAgentFile(seedContext, { + id: 'file-fs', + binaryDataId: `filesystem-v2:${FS_KEY}`, + }); + await insertAgentFile(seedContext, { + id: 'file-fs-v1', + binaryDataId: `filesystem:${LEGACY_FS_KEY}`, + }); + await insertAgentFile(seedContext, { id: 'file-s3', binaryDataId: `s3:${S3_KEY}` }); + await insertAgentFile(seedContext, { id: 'file-db', binaryDataId: `database:${DB_FILE_ID}` }); + await insertAgentFile(seedContext, { id: 'file-bad', binaryDataId: 'garbage' }); + + await insertBinaryData(seedContext, { + fileId: DB_FILE_ID, + sourceType: 'agent_file', + sourceId: 'file-db', + }); + await insertBinaryData(seedContext, { + fileId: EXECUTION_FILE_ID, + sourceType: 'execution', + sourceId: 'exec-1', + }); + } finally { + await seedContext.queryRunner.release(); + } + + await runSingleMigration(MIGRATION_NAME); + + const context = createTestMigrationContext(dataSource); + try { + const agentFiles = context.escape.tableName('agent_files'); + const binaryData = context.escape.tableName('binary_data'); + + const files = await context.runQuery< + Array<{ id: string; storedAt: string; storageKey: string }> + >(`SELECT "id", "storedAt", "storageKey" FROM ${agentFiles} ORDER BY "id"`); + // `file-bad` carries an unrecognized reference: kept whole rather than dropped. + expect(files).toEqual([ + { id: 'file-bad', storedAt: 'fs', storageKey: 'garbage' }, + { id: 'file-db', storedAt: 'db', storageKey: DB_FILE_ID }, + { id: 'file-fs', storedAt: 'fs', storageKey: FS_KEY }, + { id: 'file-fs-v1', storedAt: 'fs', storageKey: LEGACY_FS_KEY }, + { id: 'file-s3', storedAt: 's3', storageKey: S3_KEY }, + ]); + + // The database-mode file keeps its bytes, addressed by storageKey. + const binaryRows = await context.runQuery>( + `SELECT "fileId" FROM ${binaryData} ORDER BY "fileId"`, + ); + expect(binaryRows).toEqual([{ fileId: DB_FILE_ID }, { fileId: EXECUTION_FILE_ID }]); + } finally { + await context.queryRunner.release(); + } + }); + + it('leaves binaryDataId in place as nullable so the previous release keeps reading it', async () => { + const context = createTestMigrationContext(dataSource); + try { + const agentFiles = context.escape.tableName('agent_files'); + + expect(await columnInfo(context, 'agent_files')).toContainEqual({ + name: 'binaryDataId', + nullable: true, + }); + + const files = await context.runQuery>( + `SELECT "id", "binaryDataId" FROM ${agentFiles} ORDER BY "id"`, + ); + expect(files).toEqual([ + { id: 'file-bad', binaryDataId: 'garbage' }, + { id: 'file-db', binaryDataId: `database:${DB_FILE_ID}` }, + { id: 'file-fs', binaryDataId: `filesystem-v2:${FS_KEY}` }, + { id: 'file-fs-v1', binaryDataId: `filesystem:${LEGACY_FS_KEY}` }, + { id: 'file-s3', binaryDataId: `s3:${S3_KEY}` }, + ]); + } finally { + await context.queryRunner.release(); + } + }); + + // Declared last: the revert undoes the schema the tests above assert on. + describe('down', () => { + it('rebuilds binaryDataId only for rows written after the migration', async () => { + const seedContext = createTestMigrationContext(dataSource); + try { + await insertMigratedAgentFile(seedContext, { + id: 'file-new', + storedAt: 'fs', + storageKey: NEW_ROW_KEY, + }); + } finally { + await seedContext.queryRunner.release(); + } + + await undoLastSingleMigration(); + + const context = createTestMigrationContext(dataSource); + try { + const agentFiles = context.escape.tableName('agent_files'); + + const files = await context.runQuery>( + `SELECT "id", "binaryDataId" FROM ${agentFiles} ORDER BY "id"`, + ); + // Pre-existing rows kept their original opaque reference — including the + // unrecognized one — so the revert is lossless for them. + expect(files).toEqual([ + { id: 'file-bad', binaryDataId: 'garbage' }, + { id: 'file-db', binaryDataId: `database:${DB_FILE_ID}` }, + { id: 'file-fs', binaryDataId: `filesystem-v2:${FS_KEY}` }, + { id: 'file-fs-v1', binaryDataId: `filesystem:${LEGACY_FS_KEY}` }, + { id: 'file-new', binaryDataId: `filesystem-v2:${NEW_ROW_KEY}` }, + { id: 'file-s3', binaryDataId: `s3:${S3_KEY}` }, + ]); + + const columns = (await columnInfo(context, 'agent_files')).map((column) => column.name); + expect(columns).not.toContain('storedAt'); + expect(columns).not.toContain('storageKey'); + } finally { + await context.queryRunner.release(); + } + + // The revert leaves a state up() applies cleanly to again. + await runSingleMigration(MIGRATION_NAME); + }); + }); +}); diff --git a/packages/core/src/fs-byte-store.service.ts b/packages/core/src/fs-byte-store.service.ts new file mode 100644 index 00000000000..b69dcd5c156 --- /dev/null +++ b/packages/core/src/fs-byte-store.service.ts @@ -0,0 +1,20 @@ +import { FsByteStore } from '@n8n/blob-storage'; +import { Service } from '@n8n/di'; + +import { ErrorReporter } from '@/errors'; +import { StorageConfig } from '@/storage.config'; + +/** + * The `fs` byte store rooted at `N8N_STORAGE_PATH`, shared by every domain that + * persists blobs there. Needed because `FsByteStore` takes plain options and + * carries no DI decorators, so injecting one requires a registered subclass. + */ +@Service() +export class FsByteStoreService extends FsByteStore { + constructor(storageConfig: StorageConfig, errorReporter: ErrorReporter) { + super({ + storagePath: storageConfig.storagePath, + reportError: (error) => errorReporter.error(error), + }); + } +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 835ff3d884f..80d600141fc 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -3,6 +3,7 @@ import * as NodeExecuteFunctions from './node-execute-functions'; export * from './binary-data'; export * from './constants'; export { StorageConfig } from './storage.config'; +export { FsByteStoreService } from './fs-byte-store.service'; export * from './credentials'; export * from './data-deduplication-service'; export * from './encryption';