diff --git a/docs/generated/postgres-schema/README.md b/docs/generated/postgres-schema/README.md
index d89b61b6676..2eec9e7a69f 100644
--- a/docs/generated/postgres-schema/README.md
+++ b/docs/generated/postgres-schema/README.md
@@ -10,6 +10,10 @@ Auto-generated from the PostgreSQL migrations in @n8n/db. Do not edit by hand.
| ---- | ------- | ------- | ---- |
| [public.agent_chat_subscriptions](public.agent_chat_subscriptions.md) | 6 | | BASE TABLE |
| [public.agent_checkpoints](public.agent_checkpoints.md) | 6 | | BASE TABLE |
+| [public.agent_eval_dataset](public.agent_eval_dataset.md) | 10 | | BASE TABLE |
+| [public.agent_eval_rating](public.agent_eval_rating.md) | 8 | | BASE TABLE |
+| [public.agent_eval_result](public.agent_eval_result.md) | 15 | | BASE TABLE |
+| [public.agent_eval_run](public.agent_eval_run.md) | 14 | | BASE TABLE |
| [public.agent_execution](public.agent_execution.md) | 19 | | BASE TABLE |
| [public.agent_execution_threads](public.agent_execution_threads.md) | 17 | | BASE TABLE |
| [public.agent_files](public.agent_files.md) | 8 | | BASE TABLE |
@@ -151,6 +155,13 @@ erDiagram
"public.agent_chat_subscriptions" }o--|| "public.agents" : "FOREIGN KEY (#quot;agentId#quot;) REFERENCES agents(id) ON DELETE CASCADE"
"public.agent_checkpoints" }o--o| "public.agents" : "FOREIGN KEY (#quot;agentId#quot;) REFERENCES agents(id) ON DELETE CASCADE"
+"public.agent_eval_dataset" }o--o| "public.user" : "FOREIGN KEY (#quot;createdById#quot;) REFERENCES #quot;user#quot;(id) ON DELETE SET NULL"
+"public.agent_eval_dataset" }o--|| "public.agents" : "FOREIGN KEY (#quot;agentId#quot;) REFERENCES agents(id) ON DELETE CASCADE"
+"public.agent_eval_rating" }o--o| "public.user" : "FOREIGN KEY (#quot;ratedById#quot;) REFERENCES #quot;user#quot;(id) ON DELETE SET NULL"
+"public.agent_eval_rating" }o--|| "public.agent_eval_result" : "FOREIGN KEY (#quot;resultId#quot;) REFERENCES agent_eval_result(id) ON DELETE CASCADE"
+"public.agent_eval_result" }o--|| "public.agent_eval_run" : "FOREIGN KEY (#quot;runId#quot;) REFERENCES agent_eval_run(id) ON DELETE CASCADE"
+"public.agent_eval_run" }o--o| "public.user" : "FOREIGN KEY (#quot;createdById#quot;) REFERENCES #quot;user#quot;(id) ON DELETE SET NULL"
+"public.agent_eval_run" }o--|| "public.agent_eval_dataset" : "FOREIGN KEY (#quot;datasetId#quot;) REFERENCES agent_eval_dataset(id) ON DELETE CASCADE"
"public.agent_execution" }o--|| "public.agent_execution_threads" : "FOREIGN KEY (#quot;threadId#quot;) REFERENCES agent_execution_threads(id) ON DELETE CASCADE"
"public.agent_execution_threads" }o--|| "public.project" : "FOREIGN KEY (#quot;projectId#quot;) REFERENCES project(id) ON DELETE CASCADE"
"public.agent_execution_threads" }o--|| "public.agents" : "FOREIGN KEY (#quot;agentId#quot;) REFERENCES agents(id) ON DELETE CASCADE"
@@ -333,6 +344,61 @@ erDiagram
text state
timestamp_3__with_time_zone updatedAt
}
+"public.agent_eval_dataset" {
+ varchar_36_ agentId FK
+ json columnMapping
+ timestamp_3__with_time_zone createdAt
+ uuid createdById FK
+ json datasetRef
+ varchar_32_ datasetSource
+ text description
+ varchar_36_ id
+ varchar_128_ name
+ timestamp_3__with_time_zone updatedAt
+}
+"public.agent_eval_rating" {
+ text comment
+ json correction
+ timestamp_3__with_time_zone createdAt
+ varchar_36_ id
+ uuid ratedById FK
+ varchar_36_ resultId FK
+ timestamp_3__with_time_zone updatedAt
+ varchar_8_ vote
+}
+"public.agent_eval_result" {
+ timestamp_3__with_time_zone completedAt
+ timestamp_3__with_time_zone createdAt
+ varchar_255_ errorCode
+ json errorDetails
+ varchar_36_ id
+ json input
+ json metrics
+ json output
+ timestamp_3__with_time_zone runAt
+ varchar_36_ runId FK
+ integer runIndex
+ varchar_255_ sourceRowId
+ varchar status
+ json toolCalls
+ timestamp_3__with_time_zone updatedAt
+}
+"public.agent_eval_run" {
+ varchar_36_ agentVersionId
+ boolean cancelRequested
+ timestamp_3__with_time_zone completedAt
+ timestamp_3__with_time_zone createdAt
+ uuid createdById FK
+ varchar_36_ datasetId FK
+ varchar_255_ errorCode
+ json errorDetails
+ varchar_36_ id
+ json metrics
+ timestamp_3__with_time_zone runAt
+ varchar_255_ runningInstanceId
+ varchar status
+ timestamp_3__with_time_zone updatedAt
+}
"public.agent_execution" {
integer completionTokens
double_precision cost
diff --git a/docs/generated/postgres-schema/public.agent_eval_dataset.md b/docs/generated/postgres-schema/public.agent_eval_dataset.md
new file mode 100644
index 00000000000..7b19f5e9a26
--- /dev/null
+++ b/docs/generated/postgres-schema/public.agent_eval_dataset.md
@@ -0,0 +1,112 @@
+# public.agent_eval_dataset
+
+## Columns
+
+| Name | Type | Default | Nullable | Children | Parents | Comment |
+| ---- | ---- | ------- | -------- | -------- | ------- | ------- |
+| agentId | varchar(36) | | false | | [public.agents](public.agents.md) | |
+| columnMapping | json | | true | | | Maps dataset columns onto input / expectedOutput / criteria roles |
+| createdAt | timestamp(3) with time zone | CURRENT_TIMESTAMP(3) | false | | | |
+| createdById | uuid | | true | | [public.user](public.user.md) | |
+| datasetRef | json | | false | | | Pointer into the dataset backend (e.g. { dataTableId }); shape varies by datasetSource |
+| datasetSource | varchar(32) | | false | | | Dataset backend the cases are read from |
+| description | text | | true | | | |
+| id | varchar(36) | | false | [public.agent_eval_run](public.agent_eval_run.md) | | |
+| name | varchar(128) | | false | | | |
+| updatedAt | timestamp(3) with time zone | CURRENT_TIMESTAMP(3) | false | | | |
+
+## Constraints
+
+| Name | Type | Definition |
+| ---- | ---- | ---------- |
+| CHK_agent_eval_dataset_datasetSource | CHECK | CHECK ((("datasetSource")::text = ANY ((ARRAY['data_table'::character varying, 'google_sheets'::character varying])::text[]))) |
+| FK_9d3a6fd750f7746de453cf4d5ed | FOREIGN KEY | FOREIGN KEY ("agentId") REFERENCES agents(id) ON DELETE CASCADE |
+| FK_eda9e5fa5558d3f16bc2f6c21b6 | FOREIGN KEY | FOREIGN KEY ("createdById") REFERENCES "user"(id) ON DELETE SET NULL |
+| PK_ac887efbb3580a577f1442cfa89 | PRIMARY KEY | PRIMARY KEY (id) |
+| agent_eval_dataset_agentId_not_null | n | NOT NULL "agentId" |
+| agent_eval_dataset_createdAt_not_null | n | NOT NULL "createdAt" |
+| agent_eval_dataset_datasetRef_not_null | n | NOT NULL "datasetRef" |
+| agent_eval_dataset_datasetSource_not_null | n | NOT NULL "datasetSource" |
+| agent_eval_dataset_id_not_null | n | NOT NULL id |
+| agent_eval_dataset_name_not_null | n | NOT NULL name |
+| agent_eval_dataset_updatedAt_not_null | n | NOT NULL "updatedAt" |
+
+## Indexes
+
+| Name | Definition |
+| ---- | ---------- |
+| IDX_9d3a6fd750f7746de453cf4d5e | CREATE INDEX "IDX_9d3a6fd750f7746de453cf4d5e" ON public.agent_eval_dataset USING btree ("agentId") |
+| PK_ac887efbb3580a577f1442cfa89 | CREATE UNIQUE INDEX "PK_ac887efbb3580a577f1442cfa89" ON public.agent_eval_dataset USING btree (id) |
+
+## Relations
+
+```mermaid
+erDiagram
+
+"public.agent_eval_dataset" }o--|| "public.agents" : "FOREIGN KEY (#quot;agentId#quot;) REFERENCES agents(id) ON DELETE CASCADE"
+"public.agent_eval_dataset" }o--o| "public.user" : "FOREIGN KEY (#quot;createdById#quot;) REFERENCES #quot;user#quot;(id) ON DELETE SET NULL"
+"public.agent_eval_run" }o--|| "public.agent_eval_dataset" : "FOREIGN KEY (#quot;datasetId#quot;) REFERENCES agent_eval_dataset(id) ON DELETE CASCADE"
+
+"public.agent_eval_dataset" {
+ varchar_36_ agentId FK
+ json columnMapping
+ timestamp_3__with_time_zone createdAt
+ uuid createdById FK
+ json datasetRef
+ varchar_32_ datasetSource
+ text description
+ varchar_36_ id
+ varchar_128_ name
+ timestamp_3__with_time_zone updatedAt
+}
+"public.agents" {
+ varchar_36_ activeVersionId FK
+ timestamp_3__with_time_zone createdAt
+ varchar_36_ id
+ json integrations
+ varchar_128_ name
+ varchar_255_ projectId FK
+ json schema
+ json skills
+ json tools
+ timestamp_3__with_time_zone updatedAt
+ varchar_36_ versionId
+}
+"public.user" {
+ timestamp_3__with_time_zone createdAt
+ boolean disabled
+ varchar_255_ email
+ varchar_32_ firstName
+ uuid id
+ date lastActiveAt
+ varchar_32_ lastName
+ boolean mfaEnabled
+ text mfaRecoveryCodes
+ text mfaSecret
+ varchar_255_ password
+ json personalizationAnswers
+ varchar_128_ roleSlug FK
+ json settings
+ timestamp_3__with_time_zone updatedAt
+}
+"public.agent_eval_run" {
+ varchar_36_ agentVersionId
+ boolean cancelRequested
+ timestamp_3__with_time_zone completedAt
+ timestamp_3__with_time_zone createdAt
+ uuid createdById FK
+ varchar_36_ datasetId FK
+ varchar_255_ errorCode
+ json errorDetails
+ varchar_36_ id
+ json metrics
+ timestamp_3__with_time_zone runAt
+ varchar_255_ runningInstanceId
+ varchar status
+ timestamp_3__with_time_zone updatedAt
+}
+```
+
+---
+
+> Generated by [tbls](https://github.com/k1LoW/tbls)
diff --git a/docs/generated/postgres-schema/public.agent_eval_rating.md b/docs/generated/postgres-schema/public.agent_eval_rating.md
new file mode 100644
index 00000000000..45ad23a27a5
--- /dev/null
+++ b/docs/generated/postgres-schema/public.agent_eval_rating.md
@@ -0,0 +1,93 @@
+# public.agent_eval_rating
+
+## Columns
+
+| Name | Type | Default | Nullable | Children | Parents | Comment |
+| ---- | ---- | ------- | -------- | -------- | ------- | ------- |
+| comment | text | | true | | | |
+| correction | json | | true | | | Optional corrected/edited output supplied by the rater |
+| createdAt | timestamp(3) with time zone | CURRENT_TIMESTAMP(3) | false | | | |
+| id | varchar(36) | | false | | | |
+| ratedById | uuid | | true | | [public.user](public.user.md) | |
+| resultId | varchar(36) | | false | | [public.agent_eval_result](public.agent_eval_result.md) | |
+| updatedAt | timestamp(3) with time zone | CURRENT_TIMESTAMP(3) | false | | | |
+| vote | varchar(8) | | false | | | Human feedback direction |
+
+## Constraints
+
+| Name | Type | Definition |
+| ---- | ---- | ---------- |
+| CHK_agent_eval_rating_vote | CHECK | CHECK (((vote)::text = ANY ((ARRAY['up'::character varying, 'down'::character varying])::text[]))) |
+| FK_9cadae6591c64498f1b58a2cef3 | FOREIGN KEY | FOREIGN KEY ("resultId") REFERENCES agent_eval_result(id) ON DELETE CASCADE |
+| FK_e06a7408573a3e152e673977d2c | FOREIGN KEY | FOREIGN KEY ("ratedById") REFERENCES "user"(id) ON DELETE SET NULL |
+| PK_04b7709a435e0c07520ceb37393 | PRIMARY KEY | PRIMARY KEY (id) |
+| agent_eval_rating_createdAt_not_null | n | NOT NULL "createdAt" |
+| agent_eval_rating_id_not_null | n | NOT NULL id |
+| agent_eval_rating_resultId_not_null | n | NOT NULL "resultId" |
+| agent_eval_rating_updatedAt_not_null | n | NOT NULL "updatedAt" |
+| agent_eval_rating_vote_not_null | n | NOT NULL vote |
+
+## Indexes
+
+| Name | Definition |
+| ---- | ---------- |
+| IDX_9cadae6591c64498f1b58a2cef | CREATE INDEX "IDX_9cadae6591c64498f1b58a2cef" ON public.agent_eval_rating USING btree ("resultId") |
+| PK_04b7709a435e0c07520ceb37393 | CREATE UNIQUE INDEX "PK_04b7709a435e0c07520ceb37393" ON public.agent_eval_rating USING btree (id) |
+
+## Relations
+
+```mermaid
+erDiagram
+
+"public.agent_eval_rating" }o--o| "public.user" : "FOREIGN KEY (#quot;ratedById#quot;) REFERENCES #quot;user#quot;(id) ON DELETE SET NULL"
+"public.agent_eval_rating" }o--|| "public.agent_eval_result" : "FOREIGN KEY (#quot;resultId#quot;) REFERENCES agent_eval_result(id) ON DELETE CASCADE"
+
+"public.agent_eval_rating" {
+ text comment
+ json correction
+ timestamp_3__with_time_zone createdAt
+ varchar_36_ id
+ uuid ratedById FK
+ varchar_36_ resultId FK
+ timestamp_3__with_time_zone updatedAt
+ varchar_8_ vote
+}
+"public.user" {
+ timestamp_3__with_time_zone createdAt
+ boolean disabled
+ varchar_255_ email
+ varchar_32_ firstName
+ uuid id
+ date lastActiveAt
+ varchar_32_ lastName
+ boolean mfaEnabled
+ text mfaRecoveryCodes
+ text mfaSecret
+ varchar_255_ password
+ json personalizationAnswers
+ varchar_128_ roleSlug FK
+ json settings
+ timestamp_3__with_time_zone updatedAt
+}
+"public.agent_eval_result" {
+ timestamp_3__with_time_zone completedAt
+ timestamp_3__with_time_zone createdAt
+ varchar_255_ errorCode
+ json errorDetails
+ varchar_36_ id
+ json input
+ json metrics
+ json output
+ timestamp_3__with_time_zone runAt
+ varchar_36_ runId FK
+ integer runIndex
+ varchar_255_ sourceRowId
+ varchar status
+ json toolCalls
+ timestamp_3__with_time_zone updatedAt
+}
+```
+
+---
+
+> Generated by [tbls](https://github.com/k1LoW/tbls)
diff --git a/docs/generated/postgres-schema/public.agent_eval_result.md b/docs/generated/postgres-schema/public.agent_eval_result.md
new file mode 100644
index 00000000000..380c57c7853
--- /dev/null
+++ b/docs/generated/postgres-schema/public.agent_eval_result.md
@@ -0,0 +1,98 @@
+# public.agent_eval_result
+
+## Columns
+
+| Name | Type | Default | Nullable | Children | Parents | Comment |
+| ---- | ---- | ------- | -------- | -------- | ------- | ------- |
+| completedAt | timestamp(3) with time zone | | true | | | |
+| createdAt | timestamp(3) with time zone | CURRENT_TIMESTAMP(3) | false | | | |
+| errorCode | varchar(255) | | true | | | |
+| errorDetails | json | | true | | | |
+| id | varchar(36) | | false | [public.agent_eval_rating](public.agent_eval_rating.md) | | |
+| input | json | | true | | | Snapshot of the case input actually run (row may later change) |
+| metrics | json | | true | | | Per-case judge scores |
+| output | json | | true | | | Agent output for this case |
+| runAt | timestamp(3) with time zone | | true | | | |
+| runId | varchar(36) | | false | | [public.agent_eval_run](public.agent_eval_run.md) | |
+| runIndex | integer | | true | | | Order of this case within the run |
+| sourceRowId | varchar(255) | | true | | | Origin dataset row id; loose pointer, rows are external and mutable |
+| status | varchar | | false | | | Per-case lifecycle |
+| toolCalls | json | | true | | | Tool-call timeline captured during the run |
+| updatedAt | timestamp(3) with time zone | CURRENT_TIMESTAMP(3) | false | | | |
+
+## Constraints
+
+| Name | Type | Definition |
+| ---- | ---- | ---------- |
+| CHK_agent_eval_result_status | CHECK | CHECK (((status)::text = ANY ((ARRAY['new'::character varying, 'running'::character varying, 'success'::character varying, 'error'::character varying, 'cancelled'::character varying])::text[]))) |
+| FK_40a2d1248a6d984f442de2fac1b | FOREIGN KEY | FOREIGN KEY ("runId") REFERENCES agent_eval_run(id) ON DELETE CASCADE |
+| PK_d40a22dc56dc200313c1f0e41ac | PRIMARY KEY | PRIMARY KEY (id) |
+| agent_eval_result_createdAt_not_null | n | NOT NULL "createdAt" |
+| agent_eval_result_id_not_null | n | NOT NULL id |
+| agent_eval_result_runId_not_null | n | NOT NULL "runId" |
+| agent_eval_result_status_not_null | n | NOT NULL status |
+| agent_eval_result_updatedAt_not_null | n | NOT NULL "updatedAt" |
+
+## Indexes
+
+| Name | Definition |
+| ---- | ---------- |
+| IDX_40a2d1248a6d984f442de2fac1 | CREATE INDEX "IDX_40a2d1248a6d984f442de2fac1" ON public.agent_eval_result USING btree ("runId") |
+| PK_d40a22dc56dc200313c1f0e41ac | CREATE UNIQUE INDEX "PK_d40a22dc56dc200313c1f0e41ac" ON public.agent_eval_result USING btree (id) |
+
+## Relations
+
+```mermaid
+erDiagram
+
+"public.agent_eval_rating" }o--|| "public.agent_eval_result" : "FOREIGN KEY (#quot;resultId#quot;) REFERENCES agent_eval_result(id) ON DELETE CASCADE"
+"public.agent_eval_result" }o--|| "public.agent_eval_run" : "FOREIGN KEY (#quot;runId#quot;) REFERENCES agent_eval_run(id) ON DELETE CASCADE"
+
+"public.agent_eval_result" {
+ timestamp_3__with_time_zone completedAt
+ timestamp_3__with_time_zone createdAt
+ varchar_255_ errorCode
+ json errorDetails
+ varchar_36_ id
+ json input
+ json metrics
+ json output
+ timestamp_3__with_time_zone runAt
+ varchar_36_ runId FK
+ integer runIndex
+ varchar_255_ sourceRowId
+ varchar status
+ json toolCalls
+ timestamp_3__with_time_zone updatedAt
+}
+"public.agent_eval_rating" {
+ text comment
+ json correction
+ timestamp_3__with_time_zone createdAt
+ varchar_36_ id
+ uuid ratedById FK
+ varchar_36_ resultId FK
+ timestamp_3__with_time_zone updatedAt
+ varchar_8_ vote
+}
+"public.agent_eval_run" {
+ varchar_36_ agentVersionId
+ boolean cancelRequested
+ timestamp_3__with_time_zone completedAt
+ timestamp_3__with_time_zone createdAt
+ uuid createdById FK
+ varchar_36_ datasetId FK
+ varchar_255_ errorCode
+ json errorDetails
+ varchar_36_ id
+ json metrics
+ timestamp_3__with_time_zone runAt
+ varchar_255_ runningInstanceId
+ varchar status
+ timestamp_3__with_time_zone updatedAt
+}
+```
+
+---
+
+> Generated by [tbls](https://github.com/k1LoW/tbls)
diff --git a/docs/generated/postgres-schema/public.agent_eval_run.md b/docs/generated/postgres-schema/public.agent_eval_run.md
new file mode 100644
index 00000000000..ad1d7426567
--- /dev/null
+++ b/docs/generated/postgres-schema/public.agent_eval_run.md
@@ -0,0 +1,119 @@
+# public.agent_eval_run
+
+## Columns
+
+| Name | Type | Default | Nullable | Children | Parents | Comment |
+| ---- | ---- | ------- | -------- | -------- | ------- | ------- |
+| agentVersionId | varchar(36) | | true | | | Published agent version under test (agent_history.versionId); loose pointer, no FK so runs survive history pruning. The agent itself comes from the dataset. |
+| cancelRequested | boolean | false | false | | | Fallback cancellation flag polled by the running main |
+| completedAt | timestamp(3) with time zone | | true | | | |
+| createdAt | timestamp(3) with time zone | CURRENT_TIMESTAMP(3) | false | | | |
+| createdById | uuid | | true | | [public.user](public.user.md) | |
+| datasetId | varchar(36) | | false | | [public.agent_eval_dataset](public.agent_eval_dataset.md) | |
+| errorCode | varchar(255) | | true | | | |
+| errorDetails | json | | true | | | |
+| id | varchar(36) | | false | [public.agent_eval_result](public.agent_eval_result.md) | | |
+| metrics | json | | true | | | Aggregated run-level scores |
+| runAt | timestamp(3) with time zone | | true | | | |
+| runningInstanceId | varchar(255) | | true | | | Main instance executing this run; used to coordinate cancellation |
+| status | varchar | | false | | | Run lifecycle |
+| updatedAt | timestamp(3) with time zone | CURRENT_TIMESTAMP(3) | false | | | |
+
+## Constraints
+
+| Name | Type | Definition |
+| ---- | ---- | ---------- |
+| CHK_agent_eval_run_status | CHECK | CHECK (((status)::text = ANY ((ARRAY['new'::character varying, 'running'::character varying, 'completed'::character varying, 'error'::character varying, 'cancelled'::character varying])::text[]))) |
+| FK_39ca447735d378e365f4227abff | FOREIGN KEY | FOREIGN KEY ("datasetId") REFERENCES agent_eval_dataset(id) ON DELETE CASCADE |
+| FK_54ee897f442cc7393a6d6165334 | FOREIGN KEY | FOREIGN KEY ("createdById") REFERENCES "user"(id) ON DELETE SET NULL |
+| PK_280832ca3f1b43663cc0e25e77f | PRIMARY KEY | PRIMARY KEY (id) |
+| agent_eval_run_cancelRequested_not_null | n | NOT NULL "cancelRequested" |
+| agent_eval_run_createdAt_not_null | n | NOT NULL "createdAt" |
+| agent_eval_run_datasetId_not_null | n | NOT NULL "datasetId" |
+| agent_eval_run_id_not_null | n | NOT NULL id |
+| agent_eval_run_status_not_null | n | NOT NULL status |
+| agent_eval_run_updatedAt_not_null | n | NOT NULL "updatedAt" |
+
+## Indexes
+
+| Name | Definition |
+| ---- | ---------- |
+| IDX_39ca447735d378e365f4227abf | CREATE INDEX "IDX_39ca447735d378e365f4227abf" ON public.agent_eval_run USING btree ("datasetId") |
+| PK_280832ca3f1b43663cc0e25e77f | CREATE UNIQUE INDEX "PK_280832ca3f1b43663cc0e25e77f" ON public.agent_eval_run USING btree (id) |
+
+## Relations
+
+```mermaid
+erDiagram
+
+"public.agent_eval_run" }o--o| "public.user" : "FOREIGN KEY (#quot;createdById#quot;) REFERENCES #quot;user#quot;(id) ON DELETE SET NULL"
+"public.agent_eval_run" }o--|| "public.agent_eval_dataset" : "FOREIGN KEY (#quot;datasetId#quot;) REFERENCES agent_eval_dataset(id) ON DELETE CASCADE"
+"public.agent_eval_result" }o--|| "public.agent_eval_run" : "FOREIGN KEY (#quot;runId#quot;) REFERENCES agent_eval_run(id) ON DELETE CASCADE"
+
+"public.agent_eval_run" {
+ varchar_36_ agentVersionId
+ boolean cancelRequested
+ timestamp_3__with_time_zone completedAt
+ timestamp_3__with_time_zone createdAt
+ uuid createdById FK
+ varchar_36_ datasetId FK
+ varchar_255_ errorCode
+ json errorDetails
+ varchar_36_ id
+ json metrics
+ timestamp_3__with_time_zone runAt
+ varchar_255_ runningInstanceId
+ varchar status
+ timestamp_3__with_time_zone updatedAt
+}
+"public.user" {
+ timestamp_3__with_time_zone createdAt
+ boolean disabled
+ varchar_255_ email
+ varchar_32_ firstName
+ uuid id
+ date lastActiveAt
+ varchar_32_ lastName
+ boolean mfaEnabled
+ text mfaRecoveryCodes
+ text mfaSecret
+ varchar_255_ password
+ json personalizationAnswers
+ varchar_128_ roleSlug FK
+ json settings
+ timestamp_3__with_time_zone updatedAt
+}
+"public.agent_eval_dataset" {
+ varchar_36_ agentId FK
+ json columnMapping
+ timestamp_3__with_time_zone createdAt
+ uuid createdById FK
+ json datasetRef
+ varchar_32_ datasetSource
+ text description
+ varchar_36_ id
+ varchar_128_ name
+ timestamp_3__with_time_zone updatedAt
+}
+"public.agent_eval_result" {
+ timestamp_3__with_time_zone completedAt
+ timestamp_3__with_time_zone createdAt
+ varchar_255_ errorCode
+ json errorDetails
+ varchar_36_ id
+ json input
+ json metrics
+ json output
+ timestamp_3__with_time_zone runAt
+ varchar_36_ runId FK
+ integer runIndex
+ varchar_255_ sourceRowId
+ varchar status
+ json toolCalls
+ timestamp_3__with_time_zone updatedAt
+}
+```
+
+---
+
+> Generated by [tbls](https://github.com/k1LoW/tbls)
diff --git a/docs/generated/postgres-schema/public.agents.md b/docs/generated/postgres-schema/public.agents.md
index 7dad93feb26..358b5a4b071 100644
--- a/docs/generated/postgres-schema/public.agents.md
+++ b/docs/generated/postgres-schema/public.agents.md
@@ -6,7 +6,7 @@
| ---- | ---- | ------- | -------- | -------- | ------- | ------- |
| activeVersionId | varchar(36) | | true | | [public.agent_history](public.agent_history.md) | |
| createdAt | timestamp(3) with time zone | CURRENT_TIMESTAMP(3) | false | | | |
-| id | varchar(36) | | false | [public.agent_chat_subscriptions](public.agent_chat_subscriptions.md) [public.agent_checkpoints](public.agent_checkpoints.md) [public.agent_execution_threads](public.agent_execution_threads.md) [public.agent_files](public.agent_files.md) [public.agent_history](public.agent_history.md) [public.agent_task_definition](public.agent_task_definition.md) [public.agent_task_run_lock](public.agent_task_run_lock.md) [public.agents_memory_entries](public.agents_memory_entries.md) [public.agents_memory_entry_cursors](public.agents_memory_entry_cursors.md) [public.agents_memory_entry_locks](public.agents_memory_entry_locks.md) [public.agents_memory_entry_sources](public.agents_memory_entry_sources.md) [public.agents_observation_cursors](public.agents_observation_cursors.md) [public.agents_observation_locks](public.agents_observation_locks.md) [public.agents_observations](public.agents_observations.md) | | |
+| id | varchar(36) | | false | [public.agent_chat_subscriptions](public.agent_chat_subscriptions.md) [public.agent_checkpoints](public.agent_checkpoints.md) [public.agent_eval_dataset](public.agent_eval_dataset.md) [public.agent_execution_threads](public.agent_execution_threads.md) [public.agent_files](public.agent_files.md) [public.agent_history](public.agent_history.md) [public.agent_task_definition](public.agent_task_definition.md) [public.agent_task_run_lock](public.agent_task_run_lock.md) [public.agents_memory_entries](public.agents_memory_entries.md) [public.agents_memory_entry_cursors](public.agents_memory_entry_cursors.md) [public.agents_memory_entry_locks](public.agents_memory_entry_locks.md) [public.agents_memory_entry_sources](public.agents_memory_entry_sources.md) [public.agents_observation_cursors](public.agents_observation_cursors.md) [public.agents_observation_locks](public.agents_observation_locks.md) [public.agents_observations](public.agents_observations.md) | | |
| integrations | json | '[]'::json | false | | | |
| name | varchar(128) | | false | | | |
| projectId | varchar(255) | | false | | [public.project](public.project.md) | |
@@ -48,6 +48,7 @@ erDiagram
"public.agents" }o--o| "public.agent_history" : "FOREIGN KEY (#quot;activeVersionId#quot;) REFERENCES agent_history(#quot;versionId#quot;) ON DELETE SET NULL"
"public.agent_chat_subscriptions" }o--|| "public.agents" : "FOREIGN KEY (#quot;agentId#quot;) REFERENCES agents(id) ON DELETE CASCADE"
"public.agent_checkpoints" }o--o| "public.agents" : "FOREIGN KEY (#quot;agentId#quot;) REFERENCES agents(id) ON DELETE CASCADE"
+"public.agent_eval_dataset" }o--|| "public.agents" : "FOREIGN KEY (#quot;agentId#quot;) REFERENCES agents(id) ON DELETE CASCADE"
"public.agent_execution_threads" }o--|| "public.agents" : "FOREIGN KEY (#quot;agentId#quot;) REFERENCES agents(id) ON DELETE CASCADE"
"public.agent_files" }o--|| "public.agents" : "FOREIGN KEY (#quot;agentId#quot;) REFERENCES agents(id) ON DELETE CASCADE"
"public.agent_history" }o--|| "public.agents" : "FOREIGN KEY (#quot;agentId#quot;) REFERENCES agents(id) ON DELETE CASCADE"
@@ -102,6 +103,18 @@ erDiagram
text state
timestamp_3__with_time_zone updatedAt
}
+"public.agent_eval_dataset" {
+ varchar_36_ agentId FK
+ json columnMapping
+ timestamp_3__with_time_zone createdAt
+ uuid createdById FK
+ json datasetRef
+ varchar_32_ datasetSource
+ text description
+ varchar_36_ id
+ varchar_128_ name
+ timestamp_3__with_time_zone updatedAt
+}
"public.agent_execution_threads" {
varchar_36_ agentId FK
varchar_255_ agentName
diff --git a/docs/generated/postgres-schema/public.user.md b/docs/generated/postgres-schema/public.user.md
index a6ccbd9785b..9e08c32acc6 100644
--- a/docs/generated/postgres-schema/public.user.md
+++ b/docs/generated/postgres-schema/public.user.md
@@ -8,7 +8,7 @@
| disabled | boolean | false | false | | | |
| email | varchar(255) | | true | | | |
| firstName | varchar(32) | | true | | | |
-| id | uuid | gen_random_uuid() | false | [public.agent_history](public.agent_history.md) [public.auth_identity](public.auth_identity.md) [public.chat_hub_agents](public.chat_hub_agents.md) [public.chat_hub_sessions](public.chat_hub_sessions.md) [public.chat_hub_tools](public.chat_hub_tools.md) [public.dynamic_credential_user_entry](public.dynamic_credential_user_entry.md) [public.evaluation_collection](public.evaluation_collection.md) [public.instance_ai_mcp_registry_connections](public.instance_ai_mcp_registry_connections.md) [public.instance_ai_pending_confirmations](public.instance_ai_pending_confirmations.md) [public.instance_ai_thread_grants](public.instance_ai_thread_grants.md) [public.oauth_access_tokens](public.oauth_access_tokens.md) [public.oauth_authorization_codes](public.oauth_authorization_codes.md) [public.oauth_refresh_tokens](public.oauth_refresh_tokens.md) [public.oauth_user_consents](public.oauth_user_consents.md) [public.project](public.project.md) [public.project_relation](public.project_relation.md) [public.user_api_keys](public.user_api_keys.md) [public.user_favorites](public.user_favorites.md) [public.workflow_builder_session](public.workflow_builder_session.md) [public.workflow_publish_history](public.workflow_publish_history.md) [public.workflow_review_request](public.workflow_review_request.md) [public.workflow_review_request_authors](public.workflow_review_request_authors.md) [public.workflow_review_request_reviewers](public.workflow_review_request_reviewers.md) | | |
+| id | uuid | gen_random_uuid() | false | [public.agent_eval_dataset](public.agent_eval_dataset.md) [public.agent_eval_rating](public.agent_eval_rating.md) [public.agent_eval_run](public.agent_eval_run.md) [public.agent_history](public.agent_history.md) [public.auth_identity](public.auth_identity.md) [public.chat_hub_agents](public.chat_hub_agents.md) [public.chat_hub_sessions](public.chat_hub_sessions.md) [public.chat_hub_tools](public.chat_hub_tools.md) [public.dynamic_credential_user_entry](public.dynamic_credential_user_entry.md) [public.evaluation_collection](public.evaluation_collection.md) [public.instance_ai_mcp_registry_connections](public.instance_ai_mcp_registry_connections.md) [public.instance_ai_pending_confirmations](public.instance_ai_pending_confirmations.md) [public.instance_ai_thread_grants](public.instance_ai_thread_grants.md) [public.oauth_access_tokens](public.oauth_access_tokens.md) [public.oauth_authorization_codes](public.oauth_authorization_codes.md) [public.oauth_refresh_tokens](public.oauth_refresh_tokens.md) [public.oauth_user_consents](public.oauth_user_consents.md) [public.project](public.project.md) [public.project_relation](public.project_relation.md) [public.user_api_keys](public.user_api_keys.md) [public.user_favorites](public.user_favorites.md) [public.workflow_builder_session](public.workflow_builder_session.md) [public.workflow_publish_history](public.workflow_publish_history.md) [public.workflow_review_request](public.workflow_review_request.md) [public.workflow_review_request_authors](public.workflow_review_request_authors.md) [public.workflow_review_request_reviewers](public.workflow_review_request_reviewers.md) | | |
| lastActiveAt | date | | true | | | |
| lastName | varchar(32) | | true | | | |
| mfaEnabled | boolean | false | false | | | |
@@ -47,6 +47,9 @@
```mermaid
erDiagram
+"public.agent_eval_dataset" }o--o| "public.user" : "FOREIGN KEY (#quot;createdById#quot;) REFERENCES #quot;user#quot;(id) ON DELETE SET NULL"
+"public.agent_eval_rating" }o--o| "public.user" : "FOREIGN KEY (#quot;ratedById#quot;) REFERENCES #quot;user#quot;(id) ON DELETE SET NULL"
+"public.agent_eval_run" }o--o| "public.user" : "FOREIGN KEY (#quot;createdById#quot;) REFERENCES #quot;user#quot;(id) ON DELETE SET NULL"
"public.agent_history" }o--o| "public.user" : "FOREIGN KEY (#quot;publishedById#quot;) REFERENCES #quot;user#quot;(id) ON DELETE SET NULL"
"public.auth_identity" }o--o| "public.user" : "FOREIGN KEY (#quot;userId#quot;) REFERENCES #quot;user#quot;(id)"
"public.chat_hub_agents" }o--|| "public.user" : "FOREIGN KEY (#quot;ownerId#quot;) REFERENCES #quot;user#quot;(id) ON DELETE CASCADE"
@@ -91,6 +94,44 @@ erDiagram
json settings
timestamp_3__with_time_zone updatedAt
}
+"public.agent_eval_dataset" {
+ varchar_36_ agentId FK
+ json columnMapping
+ timestamp_3__with_time_zone createdAt
+ uuid createdById FK
+ json datasetRef
+ varchar_32_ datasetSource
+ text description
+ varchar_36_ id
+ varchar_128_ name
+ timestamp_3__with_time_zone updatedAt
+}
+"public.agent_eval_rating" {
+ text comment
+ json correction
+ timestamp_3__with_time_zone createdAt
+ varchar_36_ id
+ uuid ratedById FK
+ varchar_36_ resultId FK
+ timestamp_3__with_time_zone updatedAt
+ varchar_8_ vote
+}
+"public.agent_eval_run" {
+ varchar_36_ agentVersionId
+ boolean cancelRequested
+ timestamp_3__with_time_zone completedAt
+ timestamp_3__with_time_zone createdAt
+ uuid createdById FK
+ varchar_36_ datasetId FK
+ varchar_255_ errorCode
+ json errorDetails
+ varchar_36_ id
+ json metrics
+ timestamp_3__with_time_zone runAt
+ varchar_255_ runningInstanceId
+ varchar status
+ timestamp_3__with_time_zone updatedAt
+}
"public.agent_history" {
varchar_36_ agentId FK
varchar_255_ author
diff --git a/docs/generated/sqlite-schema/README.md b/docs/generated/sqlite-schema/README.md
index 55742aad538..0f5cae0de59 100644
--- a/docs/generated/sqlite-schema/README.md
+++ b/docs/generated/sqlite-schema/README.md
@@ -10,6 +10,10 @@ Auto-generated from the SQLite migrations in @n8n/db. Do not edit by hand.
| ---- | ------- | ------- | ---- |
| [agent_chat_subscriptions](agent_chat_subscriptions.md) | 6 | | table |
| [agent_checkpoints](agent_checkpoints.md) | 6 | | table |
+| [agent_eval_dataset](agent_eval_dataset.md) | 10 | | table |
+| [agent_eval_rating](agent_eval_rating.md) | 8 | | table |
+| [agent_eval_result](agent_eval_result.md) | 15 | | table |
+| [agent_eval_run](agent_eval_run.md) | 14 | | table |
| [agent_execution](agent_execution.md) | 19 | | table |
| [agent_execution_threads](agent_execution_threads.md) | 17 | | table |
| [agent_files](agent_files.md) | 8 | | table |
@@ -134,6 +138,13 @@ erDiagram
"agent_chat_subscriptions" |o--|| "agents" : "FOREIGN KEY (agentId) REFERENCES agents (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
"agent_checkpoints" }o--o| "agents" : "FOREIGN KEY (agentId) REFERENCES agents (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
+"agent_eval_dataset" }o--o| "user" : "FOREIGN KEY (createdById) REFERENCES user (id) ON UPDATE NO ACTION ON DELETE SET NULL MATCH NONE"
+"agent_eval_dataset" }o--|| "agents" : "FOREIGN KEY (agentId) REFERENCES agents (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
+"agent_eval_rating" }o--o| "user" : "FOREIGN KEY (ratedById) REFERENCES user (id) ON UPDATE NO ACTION ON DELETE SET NULL MATCH NONE"
+"agent_eval_rating" }o--|| "agent_eval_result" : "FOREIGN KEY (resultId) REFERENCES agent_eval_result (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
+"agent_eval_result" }o--|| "agent_eval_run" : "FOREIGN KEY (runId) REFERENCES agent_eval_run (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
+"agent_eval_run" }o--o| "user" : "FOREIGN KEY (createdById) REFERENCES user (id) ON UPDATE NO ACTION ON DELETE SET NULL MATCH NONE"
+"agent_eval_run" }o--|| "agent_eval_dataset" : "FOREIGN KEY (datasetId) REFERENCES agent_eval_dataset (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
"agent_execution" }o--|| "agent_execution_threads" : "FOREIGN KEY (threadId) REFERENCES agent_execution_threads (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
"agent_execution_threads" }o--o| "agent_history" : "FOREIGN KEY (taskVersionId) REFERENCES agent_history (versionId) ON UPDATE NO ACTION ON DELETE SET NULL MATCH NONE"
"agent_execution_threads" }o--|| "agents" : "FOREIGN KEY (agentId) REFERENCES agents (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
@@ -320,6 +331,61 @@ erDiagram
TEXT state
datetime_3_ updatedAt
}
+"agent_eval_dataset" {
+ varchar_36_ agentId FK
+ TEXT columnMapping
+ datetime_3_ createdAt
+ varchar createdById FK
+ TEXT datasetRef
+ varchar_32_ datasetSource
+ TEXT description
+ varchar_36_ id PK
+ varchar_128_ name
+ datetime_3_ updatedAt
+}
+"agent_eval_rating" {
+ TEXT comment
+ TEXT correction
+ datetime_3_ createdAt
+ varchar_36_ id PK
+ varchar ratedById FK
+ varchar_36_ resultId FK
+ datetime_3_ updatedAt
+ varchar_8_ vote
+}
+"agent_eval_result" {
+ datetime_3_ completedAt
+ datetime_3_ createdAt
+ varchar_255_ errorCode
+ TEXT errorDetails
+ varchar_36_ id PK
+ TEXT input
+ TEXT metrics
+ TEXT output
+ datetime_3_ runAt
+ varchar_36_ runId FK
+ INTEGER runIndex
+ varchar_255_ sourceRowId
+ varchar status
+ TEXT toolCalls
+ datetime_3_ updatedAt
+}
+"agent_eval_run" {
+ varchar_36_ agentVersionId
+ boolean cancelRequested
+ datetime_3_ completedAt
+ datetime_3_ createdAt
+ varchar createdById FK
+ varchar_36_ datasetId FK
+ varchar_255_ errorCode
+ TEXT errorDetails
+ varchar_36_ id PK
+ TEXT metrics
+ datetime_3_ runAt
+ varchar_255_ runningInstanceId
+ varchar status
+ datetime_3_ updatedAt
+}
"agent_execution" {
INTEGER completionTokens
REAL cost
diff --git a/docs/generated/sqlite-schema/agent_eval_dataset.md b/docs/generated/sqlite-schema/agent_eval_dataset.md
new file mode 100644
index 00000000000..3fb21fe79f8
--- /dev/null
+++ b/docs/generated/sqlite-schema/agent_eval_dataset.md
@@ -0,0 +1,117 @@
+# agent_eval_dataset
+
+## Description
+
+
+Table Definition
+
+```sql
+CREATE TABLE "agent_eval_dataset" ("id" varchar(36) PRIMARY KEY NOT NULL, "name" varchar(128) NOT NULL, "description" text, "agentId" varchar(36) NOT NULL, "datasetSource" varchar(32) NOT NULL, "datasetRef" text NOT NULL, "columnMapping" text, "createdById" varchar, "createdAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), "updatedAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), CONSTRAINT "CHK_agent_eval_dataset_datasetSource" CHECK ("datasetSource" IN ('data_table', 'google_sheets')), CONSTRAINT "FK_9d3a6fd750f7746de453cf4d5ed" FOREIGN KEY ("agentId") REFERENCES "agents" ("id") ON DELETE CASCADE, CONSTRAINT "FK_eda9e5fa5558d3f16bc2f6c21b6" FOREIGN KEY ("createdById") REFERENCES "user" ("id") ON DELETE SET NULL)
+```
+
+
+
+## Columns
+
+| Name | Type | Default | Nullable | Children | Parents | Comment |
+| ---- | ---- | ------- | -------- | -------- | ------- | ------- |
+| agentId | varchar(36) | | false | | [agents](agents.md) | |
+| columnMapping | TEXT | | true | | | |
+| createdAt | datetime(3) | STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') | false | | | |
+| createdById | varchar | | true | | [user](user.md) | |
+| datasetRef | TEXT | | false | | | |
+| datasetSource | varchar(32) | | false | | | |
+| description | TEXT | | true | | | |
+| id | varchar(36) | | false | [agent_eval_run](agent_eval_run.md) | | |
+| name | varchar(128) | | false | | | |
+| updatedAt | datetime(3) | STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') | false | | | |
+
+## Constraints
+
+| Name | Type | Definition |
+| ---- | ---- | ---------- |
+| - | CHECK | CHECK ("datasetSource" IN ('data_table', 'google_sheets')) |
+| - (Foreign key ID: 0) | FOREIGN KEY | FOREIGN KEY (createdById) REFERENCES user (id) ON UPDATE NO ACTION ON DELETE SET NULL MATCH NONE |
+| - (Foreign key ID: 1) | FOREIGN KEY | FOREIGN KEY (agentId) REFERENCES agents (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE |
+| id | PRIMARY KEY | PRIMARY KEY (id) |
+| sqlite_autoindex_agent_eval_dataset_1 | PRIMARY KEY | PRIMARY KEY (id) |
+
+## Indexes
+
+| Name | Definition |
+| ---- | ---------- |
+| IDX_9d3a6fd750f7746de453cf4d5e | CREATE INDEX "IDX_9d3a6fd750f7746de453cf4d5e" ON "agent_eval_dataset" ("agentId") |
+| sqlite_autoindex_agent_eval_dataset_1 | PRIMARY KEY (id) |
+
+## Relations
+
+```mermaid
+erDiagram
+
+"agent_eval_dataset" }o--|| "agents" : "FOREIGN KEY (agentId) REFERENCES agents (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
+"agent_eval_dataset" }o--o| "user" : "FOREIGN KEY (createdById) REFERENCES user (id) ON UPDATE NO ACTION ON DELETE SET NULL MATCH NONE"
+"agent_eval_run" }o--|| "agent_eval_dataset" : "FOREIGN KEY (datasetId) REFERENCES agent_eval_dataset (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
+
+"agent_eval_dataset" {
+ varchar_36_ agentId FK
+ TEXT columnMapping
+ datetime_3_ createdAt
+ varchar createdById FK
+ TEXT datasetRef
+ varchar_32_ datasetSource
+ TEXT description
+ varchar_36_ id PK
+ varchar_128_ name
+ datetime_3_ updatedAt
+}
+"agents" {
+ varchar_36_ activeVersionId FK
+ datetime_3_ createdAt
+ varchar_36_ id PK
+ TEXT integrations
+ varchar_128_ name
+ varchar_255_ projectId FK
+ TEXT schema
+ TEXT skills
+ TEXT tools
+ datetime_3_ updatedAt
+ varchar_36_ versionId
+}
+"user" {
+ datetime_3_ createdAt
+ boolean disabled
+ varchar_255_ email
+ varchar_32_ firstName
+ varchar id PK
+ date lastActiveAt
+ varchar_32_ lastName
+ boolean mfaEnabled
+ TEXT mfaRecoveryCodes
+ TEXT mfaSecret
+ varchar password
+ TEXT personalizationAnswers
+ varchar_128_ roleSlug FK
+ TEXT settings
+ datetime_3_ updatedAt
+}
+"agent_eval_run" {
+ varchar_36_ agentVersionId
+ boolean cancelRequested
+ datetime_3_ completedAt
+ datetime_3_ createdAt
+ varchar createdById FK
+ varchar_36_ datasetId FK
+ varchar_255_ errorCode
+ TEXT errorDetails
+ varchar_36_ id PK
+ TEXT metrics
+ datetime_3_ runAt
+ varchar_255_ runningInstanceId
+ varchar status
+ datetime_3_ updatedAt
+}
+```
+
+---
+
+> Generated by [tbls](https://github.com/k1LoW/tbls)
diff --git a/docs/generated/sqlite-schema/agent_eval_rating.md b/docs/generated/sqlite-schema/agent_eval_rating.md
new file mode 100644
index 00000000000..786de848e46
--- /dev/null
+++ b/docs/generated/sqlite-schema/agent_eval_rating.md
@@ -0,0 +1,100 @@
+# agent_eval_rating
+
+## Description
+
+
+Table Definition
+
+```sql
+CREATE TABLE "agent_eval_rating" ("id" varchar(36) PRIMARY KEY NOT NULL, "resultId" varchar(36) NOT NULL, "vote" varchar(8) NOT NULL, "comment" text, "correction" text, "ratedById" varchar, "createdAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), "updatedAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), CONSTRAINT "CHK_agent_eval_rating_vote" CHECK ("vote" IN ('up', 'down')), CONSTRAINT "FK_9cadae6591c64498f1b58a2cef3" FOREIGN KEY ("resultId") REFERENCES "agent_eval_result" ("id") ON DELETE CASCADE, CONSTRAINT "FK_e06a7408573a3e152e673977d2c" FOREIGN KEY ("ratedById") REFERENCES "user" ("id") ON DELETE SET NULL)
+```
+
+
+
+## Columns
+
+| Name | Type | Default | Nullable | Children | Parents | Comment |
+| ---- | ---- | ------- | -------- | -------- | ------- | ------- |
+| comment | TEXT | | true | | | |
+| correction | TEXT | | true | | | |
+| createdAt | datetime(3) | STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') | false | | | |
+| id | varchar(36) | | false | | | |
+| ratedById | varchar | | true | | [user](user.md) | |
+| resultId | varchar(36) | | false | | [agent_eval_result](agent_eval_result.md) | |
+| updatedAt | datetime(3) | STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') | false | | | |
+| vote | varchar(8) | | false | | | |
+
+## Constraints
+
+| Name | Type | Definition |
+| ---- | ---- | ---------- |
+| - | CHECK | CHECK ("vote" IN ('up', 'down')) |
+| - (Foreign key ID: 0) | FOREIGN KEY | FOREIGN KEY (ratedById) REFERENCES user (id) ON UPDATE NO ACTION ON DELETE SET NULL MATCH NONE |
+| - (Foreign key ID: 1) | FOREIGN KEY | FOREIGN KEY (resultId) REFERENCES agent_eval_result (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE |
+| id | PRIMARY KEY | PRIMARY KEY (id) |
+| sqlite_autoindex_agent_eval_rating_1 | PRIMARY KEY | PRIMARY KEY (id) |
+
+## Indexes
+
+| Name | Definition |
+| ---- | ---------- |
+| IDX_9cadae6591c64498f1b58a2cef | CREATE INDEX "IDX_9cadae6591c64498f1b58a2cef" ON "agent_eval_rating" ("resultId") |
+| sqlite_autoindex_agent_eval_rating_1 | PRIMARY KEY (id) |
+
+## Relations
+
+```mermaid
+erDiagram
+
+"agent_eval_rating" }o--o| "user" : "FOREIGN KEY (ratedById) REFERENCES user (id) ON UPDATE NO ACTION ON DELETE SET NULL MATCH NONE"
+"agent_eval_rating" }o--|| "agent_eval_result" : "FOREIGN KEY (resultId) REFERENCES agent_eval_result (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
+
+"agent_eval_rating" {
+ TEXT comment
+ TEXT correction
+ datetime_3_ createdAt
+ varchar_36_ id PK
+ varchar ratedById FK
+ varchar_36_ resultId FK
+ datetime_3_ updatedAt
+ varchar_8_ vote
+}
+"user" {
+ datetime_3_ createdAt
+ boolean disabled
+ varchar_255_ email
+ varchar_32_ firstName
+ varchar id PK
+ date lastActiveAt
+ varchar_32_ lastName
+ boolean mfaEnabled
+ TEXT mfaRecoveryCodes
+ TEXT mfaSecret
+ varchar password
+ TEXT personalizationAnswers
+ varchar_128_ roleSlug FK
+ TEXT settings
+ datetime_3_ updatedAt
+}
+"agent_eval_result" {
+ datetime_3_ completedAt
+ datetime_3_ createdAt
+ varchar_255_ errorCode
+ TEXT errorDetails
+ varchar_36_ id PK
+ TEXT input
+ TEXT metrics
+ TEXT output
+ datetime_3_ runAt
+ varchar_36_ runId FK
+ INTEGER runIndex
+ varchar_255_ sourceRowId
+ varchar status
+ TEXT toolCalls
+ datetime_3_ updatedAt
+}
+```
+
+---
+
+> Generated by [tbls](https://github.com/k1LoW/tbls)
diff --git a/docs/generated/sqlite-schema/agent_eval_result.md b/docs/generated/sqlite-schema/agent_eval_result.md
new file mode 100644
index 00000000000..b183de9fa3a
--- /dev/null
+++ b/docs/generated/sqlite-schema/agent_eval_result.md
@@ -0,0 +1,105 @@
+# agent_eval_result
+
+## Description
+
+
+Table Definition
+
+```sql
+CREATE TABLE "agent_eval_result" ("id" varchar(36) PRIMARY KEY NOT NULL, "runId" varchar(36) NOT NULL, "sourceRowId" varchar(255), "runIndex" integer, "status" varchar NOT NULL, "input" text, "output" text, "toolCalls" text, "metrics" text, "runAt" datetime(3), "completedAt" datetime(3), "errorCode" varchar(255), "errorDetails" text, "createdAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), "updatedAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), CONSTRAINT "CHK_agent_eval_result_status" CHECK ("status" IN ('new', 'running', 'success', 'error', 'cancelled')), CONSTRAINT "FK_40a2d1248a6d984f442de2fac1b" FOREIGN KEY ("runId") REFERENCES "agent_eval_run" ("id") ON DELETE CASCADE)
+```
+
+
+
+## Columns
+
+| Name | Type | Default | Nullable | Children | Parents | Comment |
+| ---- | ---- | ------- | -------- | -------- | ------- | ------- |
+| completedAt | datetime(3) | | true | | | |
+| createdAt | datetime(3) | STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') | false | | | |
+| errorCode | varchar(255) | | true | | | |
+| errorDetails | TEXT | | true | | | |
+| id | varchar(36) | | false | [agent_eval_rating](agent_eval_rating.md) | | |
+| input | TEXT | | true | | | |
+| metrics | TEXT | | true | | | |
+| output | TEXT | | true | | | |
+| runAt | datetime(3) | | true | | | |
+| runId | varchar(36) | | false | | [agent_eval_run](agent_eval_run.md) | |
+| runIndex | INTEGER | | true | | | |
+| sourceRowId | varchar(255) | | true | | | |
+| status | varchar | | false | | | |
+| toolCalls | TEXT | | true | | | |
+| updatedAt | datetime(3) | STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') | false | | | |
+
+## Constraints
+
+| Name | Type | Definition |
+| ---- | ---- | ---------- |
+| - | CHECK | CHECK ("status" IN ('new', 'running', 'success', 'error', 'cancelled')) |
+| - (Foreign key ID: 0) | FOREIGN KEY | FOREIGN KEY (runId) REFERENCES agent_eval_run (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE |
+| id | PRIMARY KEY | PRIMARY KEY (id) |
+| sqlite_autoindex_agent_eval_result_1 | PRIMARY KEY | PRIMARY KEY (id) |
+
+## Indexes
+
+| Name | Definition |
+| ---- | ---------- |
+| IDX_40a2d1248a6d984f442de2fac1 | CREATE INDEX "IDX_40a2d1248a6d984f442de2fac1" ON "agent_eval_result" ("runId") |
+| sqlite_autoindex_agent_eval_result_1 | PRIMARY KEY (id) |
+
+## Relations
+
+```mermaid
+erDiagram
+
+"agent_eval_rating" }o--|| "agent_eval_result" : "FOREIGN KEY (resultId) REFERENCES agent_eval_result (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
+"agent_eval_result" }o--|| "agent_eval_run" : "FOREIGN KEY (runId) REFERENCES agent_eval_run (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
+
+"agent_eval_result" {
+ datetime_3_ completedAt
+ datetime_3_ createdAt
+ varchar_255_ errorCode
+ TEXT errorDetails
+ varchar_36_ id PK
+ TEXT input
+ TEXT metrics
+ TEXT output
+ datetime_3_ runAt
+ varchar_36_ runId FK
+ INTEGER runIndex
+ varchar_255_ sourceRowId
+ varchar status
+ TEXT toolCalls
+ datetime_3_ updatedAt
+}
+"agent_eval_rating" {
+ TEXT comment
+ TEXT correction
+ datetime_3_ createdAt
+ varchar_36_ id PK
+ varchar ratedById FK
+ varchar_36_ resultId FK
+ datetime_3_ updatedAt
+ varchar_8_ vote
+}
+"agent_eval_run" {
+ varchar_36_ agentVersionId
+ boolean cancelRequested
+ datetime_3_ completedAt
+ datetime_3_ createdAt
+ varchar createdById FK
+ varchar_36_ datasetId FK
+ varchar_255_ errorCode
+ TEXT errorDetails
+ varchar_36_ id PK
+ TEXT metrics
+ datetime_3_ runAt
+ varchar_255_ runningInstanceId
+ varchar status
+ datetime_3_ updatedAt
+}
+```
+
+---
+
+> Generated by [tbls](https://github.com/k1LoW/tbls)
diff --git a/docs/generated/sqlite-schema/agent_eval_run.md b/docs/generated/sqlite-schema/agent_eval_run.md
new file mode 100644
index 00000000000..242d916fa55
--- /dev/null
+++ b/docs/generated/sqlite-schema/agent_eval_run.md
@@ -0,0 +1,125 @@
+# agent_eval_run
+
+## Description
+
+
+Table Definition
+
+```sql
+CREATE TABLE "agent_eval_run" ("id" varchar(36) PRIMARY KEY NOT NULL, "datasetId" varchar(36) NOT NULL, "agentVersionId" varchar(36), "status" varchar NOT NULL, "runAt" datetime(3), "completedAt" datetime(3), "metrics" text, "errorCode" varchar(255), "errorDetails" text, "runningInstanceId" varchar(255), "cancelRequested" boolean NOT NULL DEFAULT (false), "createdById" varchar, "createdAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), "updatedAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), CONSTRAINT "CHK_agent_eval_run_status" CHECK ("status" IN ('new', 'running', 'completed', 'error', 'cancelled')), CONSTRAINT "FK_39ca447735d378e365f4227abff" FOREIGN KEY ("datasetId") REFERENCES "agent_eval_dataset" ("id") ON DELETE CASCADE, CONSTRAINT "FK_54ee897f442cc7393a6d6165334" FOREIGN KEY ("createdById") REFERENCES "user" ("id") ON DELETE SET NULL)
+```
+
+
+
+## Columns
+
+| Name | Type | Default | Nullable | Children | Parents | Comment |
+| ---- | ---- | ------- | -------- | -------- | ------- | ------- |
+| agentVersionId | varchar(36) | | true | | | |
+| cancelRequested | boolean | false | false | | | |
+| completedAt | datetime(3) | | true | | | |
+| createdAt | datetime(3) | STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') | false | | | |
+| createdById | varchar | | true | | [user](user.md) | |
+| datasetId | varchar(36) | | false | | [agent_eval_dataset](agent_eval_dataset.md) | |
+| errorCode | varchar(255) | | true | | | |
+| errorDetails | TEXT | | true | | | |
+| id | varchar(36) | | false | [agent_eval_result](agent_eval_result.md) | | |
+| metrics | TEXT | | true | | | |
+| runAt | datetime(3) | | true | | | |
+| runningInstanceId | varchar(255) | | true | | | |
+| status | varchar | | false | | | |
+| updatedAt | datetime(3) | STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') | false | | | |
+
+## Constraints
+
+| Name | Type | Definition |
+| ---- | ---- | ---------- |
+| - | CHECK | CHECK ("status" IN ('new', 'running', 'completed', 'error', 'cancelled')) |
+| - (Foreign key ID: 0) | FOREIGN KEY | FOREIGN KEY (createdById) REFERENCES user (id) ON UPDATE NO ACTION ON DELETE SET NULL MATCH NONE |
+| - (Foreign key ID: 1) | FOREIGN KEY | FOREIGN KEY (datasetId) REFERENCES agent_eval_dataset (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE |
+| id | PRIMARY KEY | PRIMARY KEY (id) |
+| sqlite_autoindex_agent_eval_run_1 | PRIMARY KEY | PRIMARY KEY (id) |
+
+## Indexes
+
+| Name | Definition |
+| ---- | ---------- |
+| IDX_39ca447735d378e365f4227abf | CREATE INDEX "IDX_39ca447735d378e365f4227abf" ON "agent_eval_run" ("datasetId") |
+| sqlite_autoindex_agent_eval_run_1 | PRIMARY KEY (id) |
+
+## Relations
+
+```mermaid
+erDiagram
+
+"agent_eval_run" }o--o| "user" : "FOREIGN KEY (createdById) REFERENCES user (id) ON UPDATE NO ACTION ON DELETE SET NULL MATCH NONE"
+"agent_eval_run" }o--|| "agent_eval_dataset" : "FOREIGN KEY (datasetId) REFERENCES agent_eval_dataset (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
+"agent_eval_result" }o--|| "agent_eval_run" : "FOREIGN KEY (runId) REFERENCES agent_eval_run (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
+
+"agent_eval_run" {
+ varchar_36_ agentVersionId
+ boolean cancelRequested
+ datetime_3_ completedAt
+ datetime_3_ createdAt
+ varchar createdById FK
+ varchar_36_ datasetId FK
+ varchar_255_ errorCode
+ TEXT errorDetails
+ varchar_36_ id PK
+ TEXT metrics
+ datetime_3_ runAt
+ varchar_255_ runningInstanceId
+ varchar status
+ datetime_3_ updatedAt
+}
+"user" {
+ datetime_3_ createdAt
+ boolean disabled
+ varchar_255_ email
+ varchar_32_ firstName
+ varchar id PK
+ date lastActiveAt
+ varchar_32_ lastName
+ boolean mfaEnabled
+ TEXT mfaRecoveryCodes
+ TEXT mfaSecret
+ varchar password
+ TEXT personalizationAnswers
+ varchar_128_ roleSlug FK
+ TEXT settings
+ datetime_3_ updatedAt
+}
+"agent_eval_dataset" {
+ varchar_36_ agentId FK
+ TEXT columnMapping
+ datetime_3_ createdAt
+ varchar createdById FK
+ TEXT datasetRef
+ varchar_32_ datasetSource
+ TEXT description
+ varchar_36_ id PK
+ varchar_128_ name
+ datetime_3_ updatedAt
+}
+"agent_eval_result" {
+ datetime_3_ completedAt
+ datetime_3_ createdAt
+ varchar_255_ errorCode
+ TEXT errorDetails
+ varchar_36_ id PK
+ TEXT input
+ TEXT metrics
+ TEXT output
+ datetime_3_ runAt
+ varchar_36_ runId FK
+ INTEGER runIndex
+ varchar_255_ sourceRowId
+ varchar status
+ TEXT toolCalls
+ datetime_3_ updatedAt
+}
+```
+
+---
+
+> Generated by [tbls](https://github.com/k1LoW/tbls)
diff --git a/docs/generated/sqlite-schema/agents.md b/docs/generated/sqlite-schema/agents.md
index c22b4bb70bd..d925895eee7 100644
--- a/docs/generated/sqlite-schema/agents.md
+++ b/docs/generated/sqlite-schema/agents.md
@@ -17,7 +17,7 @@ CREATE TABLE "agents" ("id" varchar(36) PRIMARY KEY NOT NULL, "name" varchar(128
| ---- | ---- | ------- | -------- | -------- | ------- | ------- |
| activeVersionId | varchar(36) | | true | | [agent_history](agent_history.md) | |
| createdAt | datetime(3) | STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') | false | | | |
-| id | varchar(36) | | false | [agent_chat_subscriptions](agent_chat_subscriptions.md) [agent_checkpoints](agent_checkpoints.md) [agent_execution_threads](agent_execution_threads.md) [agent_files](agent_files.md) [agent_history](agent_history.md) [agent_task_definition](agent_task_definition.md) [agent_task_run_lock](agent_task_run_lock.md) [agents_memory_entries](agents_memory_entries.md) [agents_memory_entry_cursors](agents_memory_entry_cursors.md) [agents_memory_entry_locks](agents_memory_entry_locks.md) [agents_memory_entry_sources](agents_memory_entry_sources.md) [agents_observation_cursors](agents_observation_cursors.md) [agents_observation_locks](agents_observation_locks.md) [agents_observations](agents_observations.md) | | |
+| id | varchar(36) | | false | [agent_chat_subscriptions](agent_chat_subscriptions.md) [agent_checkpoints](agent_checkpoints.md) [agent_eval_dataset](agent_eval_dataset.md) [agent_execution_threads](agent_execution_threads.md) [agent_files](agent_files.md) [agent_history](agent_history.md) [agent_task_definition](agent_task_definition.md) [agent_task_run_lock](agent_task_run_lock.md) [agents_memory_entries](agents_memory_entries.md) [agents_memory_entry_cursors](agents_memory_entry_cursors.md) [agents_memory_entry_locks](agents_memory_entry_locks.md) [agents_memory_entry_sources](agents_memory_entry_sources.md) [agents_observation_cursors](agents_observation_cursors.md) [agents_observation_locks](agents_observation_locks.md) [agents_observations](agents_observations.md) | | |
| integrations | TEXT | '[]' | false | | | |
| name | varchar(128) | | false | | | |
| projectId | varchar(255) | | false | | [project](project.md) | |
@@ -52,6 +52,7 @@ erDiagram
"agents" }o--o| "agent_history" : "FOREIGN KEY (activeVersionId) REFERENCES agent_history (versionId) ON UPDATE NO ACTION ON DELETE SET NULL MATCH NONE"
"agent_chat_subscriptions" |o--|| "agents" : "FOREIGN KEY (agentId) REFERENCES agents (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
"agent_checkpoints" }o--o| "agents" : "FOREIGN KEY (agentId) REFERENCES agents (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
+"agent_eval_dataset" }o--|| "agents" : "FOREIGN KEY (agentId) REFERENCES agents (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
"agent_execution_threads" }o--|| "agents" : "FOREIGN KEY (agentId) REFERENCES agents (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
"agent_files" }o--|| "agents" : "FOREIGN KEY (agentId) REFERENCES agents (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
"agent_history" }o--|| "agents" : "FOREIGN KEY (agentId) REFERENCES agents (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
@@ -106,6 +107,18 @@ erDiagram
TEXT state
datetime_3_ updatedAt
}
+"agent_eval_dataset" {
+ varchar_36_ agentId FK
+ TEXT columnMapping
+ datetime_3_ createdAt
+ varchar createdById FK
+ TEXT datasetRef
+ varchar_32_ datasetSource
+ TEXT description
+ varchar_36_ id PK
+ varchar_128_ name
+ datetime_3_ updatedAt
+}
"agent_execution_threads" {
varchar_36_ agentId FK
varchar_255_ agentName
diff --git a/docs/generated/sqlite-schema/user.md b/docs/generated/sqlite-schema/user.md
index 70fa8cba416..ee5555072c9 100644
--- a/docs/generated/sqlite-schema/user.md
+++ b/docs/generated/sqlite-schema/user.md
@@ -19,7 +19,7 @@ CREATE TABLE "user" ("id" varchar PRIMARY KEY, "email" varchar(255), "firstName"
| disabled | boolean | FALSE | false | | | |
| email | varchar(255) | | true | | | |
| firstName | varchar(32) | | true | | | |
-| id | varchar | | true | [agent_history](agent_history.md) [auth_identity](auth_identity.md) [chat_hub_agents](chat_hub_agents.md) [chat_hub_sessions](chat_hub_sessions.md) [chat_hub_tools](chat_hub_tools.md) [dynamic_credential_user_entry](dynamic_credential_user_entry.md) [evaluation_collection](evaluation_collection.md) [instance_ai_mcp_registry_connections](instance_ai_mcp_registry_connections.md) [instance_ai_pending_confirmations](instance_ai_pending_confirmations.md) [instance_ai_thread_grants](instance_ai_thread_grants.md) [oauth_access_tokens](oauth_access_tokens.md) [oauth_authorization_codes](oauth_authorization_codes.md) [oauth_refresh_tokens](oauth_refresh_tokens.md) [oauth_user_consents](oauth_user_consents.md) [project](project.md) [project_relation](project_relation.md) [user_api_keys](user_api_keys.md) [user_favorites](user_favorites.md) [workflow_builder_session](workflow_builder_session.md) [workflow_publish_history](workflow_publish_history.md) [workflow_review_request](workflow_review_request.md) [workflow_review_request_authors](workflow_review_request_authors.md) [workflow_review_request_reviewers](workflow_review_request_reviewers.md) | | |
+| id | varchar | | true | [agent_eval_dataset](agent_eval_dataset.md) [agent_eval_rating](agent_eval_rating.md) [agent_eval_run](agent_eval_run.md) [agent_history](agent_history.md) [auth_identity](auth_identity.md) [chat_hub_agents](chat_hub_agents.md) [chat_hub_sessions](chat_hub_sessions.md) [chat_hub_tools](chat_hub_tools.md) [dynamic_credential_user_entry](dynamic_credential_user_entry.md) [evaluation_collection](evaluation_collection.md) [instance_ai_mcp_registry_connections](instance_ai_mcp_registry_connections.md) [instance_ai_pending_confirmations](instance_ai_pending_confirmations.md) [instance_ai_thread_grants](instance_ai_thread_grants.md) [oauth_access_tokens](oauth_access_tokens.md) [oauth_authorization_codes](oauth_authorization_codes.md) [oauth_refresh_tokens](oauth_refresh_tokens.md) [oauth_user_consents](oauth_user_consents.md) [project](project.md) [project_relation](project_relation.md) [user_api_keys](user_api_keys.md) [user_favorites](user_favorites.md) [workflow_builder_session](workflow_builder_session.md) [workflow_publish_history](workflow_publish_history.md) [workflow_review_request](workflow_review_request.md) [workflow_review_request_authors](workflow_review_request_authors.md) [workflow_review_request_reviewers](workflow_review_request_reviewers.md) | | |
| lastActiveAt | date | | true | | | |
| lastName | varchar(32) | | true | | | |
| mfaEnabled | boolean | FALSE | false | | | |
@@ -53,6 +53,9 @@ CREATE TABLE "user" ("id" varchar PRIMARY KEY, "email" varchar(255), "firstName"
```mermaid
erDiagram
+"agent_eval_dataset" }o--o| "user" : "FOREIGN KEY (createdById) REFERENCES user (id) ON UPDATE NO ACTION ON DELETE SET NULL MATCH NONE"
+"agent_eval_rating" }o--o| "user" : "FOREIGN KEY (ratedById) REFERENCES user (id) ON UPDATE NO ACTION ON DELETE SET NULL MATCH NONE"
+"agent_eval_run" }o--o| "user" : "FOREIGN KEY (createdById) REFERENCES user (id) ON UPDATE NO ACTION ON DELETE SET NULL MATCH NONE"
"agent_history" }o--o| "user" : "FOREIGN KEY (publishedById) REFERENCES user (id) ON UPDATE NO ACTION ON DELETE SET NULL MATCH NONE"
"auth_identity" }o--o| "user" : "FOREIGN KEY (userId) REFERENCES user (id) ON UPDATE NO ACTION ON DELETE NO ACTION MATCH NONE"
"chat_hub_agents" }o--|| "user" : "FOREIGN KEY (ownerId) REFERENCES user (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE"
@@ -97,6 +100,44 @@ erDiagram
TEXT settings
datetime_3_ updatedAt
}
+"agent_eval_dataset" {
+ varchar_36_ agentId FK
+ TEXT columnMapping
+ datetime_3_ createdAt
+ varchar createdById FK
+ TEXT datasetRef
+ varchar_32_ datasetSource
+ TEXT description
+ varchar_36_ id PK
+ varchar_128_ name
+ datetime_3_ updatedAt
+}
+"agent_eval_rating" {
+ TEXT comment
+ TEXT correction
+ datetime_3_ createdAt
+ varchar_36_ id PK
+ varchar ratedById FK
+ varchar_36_ resultId FK
+ datetime_3_ updatedAt
+ varchar_8_ vote
+}
+"agent_eval_run" {
+ varchar_36_ agentVersionId
+ boolean cancelRequested
+ datetime_3_ completedAt
+ datetime_3_ createdAt
+ varchar createdById FK
+ varchar_36_ datasetId FK
+ varchar_255_ errorCode
+ TEXT errorDetails
+ varchar_36_ id PK
+ TEXT metrics
+ datetime_3_ runAt
+ varchar_255_ runningInstanceId
+ varchar status
+ datetime_3_ updatedAt
+}
"agent_history" {
varchar_36_ agentId FK
varchar_255_ author
diff --git a/packages/@n8n/db/src/entities/agent-eval-dataset.ee.ts b/packages/@n8n/db/src/entities/agent-eval-dataset.ee.ts
new file mode 100644
index 00000000000..1df5411bf7f
--- /dev/null
+++ b/packages/@n8n/db/src/entities/agent-eval-dataset.ee.ts
@@ -0,0 +1,57 @@
+import type { DatasetRef } from '@n8n/api-types';
+import { Column, Entity, Index } from '@n8n/typeorm';
+
+import { JsonColumn, WithTimestampsAndStringId } from './abstract-entity';
+
+/**
+ * Maps the roles an agent eval needs onto columns of the referenced dataset
+ * (a Data Table or Google Sheet). Only `input` is required; a run without a
+ * `expectedOutput`/`criteria` column simply has no reference answer or
+ * per-case check to judge against.
+ */
+export interface AgentEvalColumnMapping {
+ input: string;
+ expectedOutput?: string;
+ criteria?: string;
+}
+
+/**
+ * A named, reusable eval setup for a single agent: which dataset to run and how
+ * its columns map onto input / reference answer / "what to check".
+ *
+ * The dataset itself is **not** stored here — like {@link EvaluationConfig}, this
+ * carries a polymorphic pointer (`datasetSource` + `datasetRef`) at an existing
+ * Data Table (or Google Sheet); the rows are the cases. Reusing the shipped
+ * `DatasetRef` shape gives Data Table and Google Sheets sources for free.
+ *
+ * **TypeORM relations:** `agentId` and `createdById` are plain FK columns *without*
+ * `@ManyToOne` decorators — the target entities (`agents`, `user`) aren't part of
+ * the `@n8n/db` entity registry, and declaring the relations would extend the
+ * entity-relation type graph. The FK constraints are preserved by the migration's
+ * raw SQL. Only the agent-eval subgraph (`dataset → run → result → rating`) uses
+ * relation decorators.
+ */
+@Entity({ name: 'agent_eval_dataset' })
+export class AgentEvalDataset extends WithTimestampsAndStringId {
+ @Column('varchar', { length: 128 })
+ name: string;
+
+ @Column('text', { nullable: true })
+ description: string | null;
+
+ @Index()
+ @Column('varchar', { length: 36 })
+ agentId: string;
+
+ @Column('varchar', { length: 32 })
+ datasetSource: DatasetRef['datasetSource'];
+
+ @JsonColumn()
+ datasetRef: DatasetRef['datasetRef'];
+
+ @JsonColumn({ nullable: true })
+ columnMapping: AgentEvalColumnMapping | null;
+
+ @Column({ type: 'uuid', nullable: true })
+ createdById: string | null;
+}
diff --git a/packages/@n8n/db/src/entities/agent-eval-rating.ee.ts b/packages/@n8n/db/src/entities/agent-eval-rating.ee.ts
new file mode 100644
index 00000000000..9642dd8befe
--- /dev/null
+++ b/packages/@n8n/db/src/entities/agent-eval-rating.ee.ts
@@ -0,0 +1,35 @@
+import { Column, Entity, Index, ManyToOne } from '@n8n/typeorm';
+import type { JsonObject } from 'n8n-workflow';
+
+import { JsonColumn, WithTimestampsAndStringId } from './abstract-entity';
+import { AgentEvalResult } from './agent-eval-result.ee';
+
+export type AgentEvalVote = 'up' | 'down';
+
+/**
+ * A human's rating of an {@link AgentEvalResult}: a 👍/👎 vote plus an optional
+ * correction (an edited "should have been" output) and free-text comment.
+ * Multiple ratings per result are allowed (per-user history); `ratedById` is a
+ * plain FK column (see {@link AgentEvalDataset} for the no-decorator rationale).
+ */
+@Entity({ name: 'agent_eval_rating' })
+export class AgentEvalRating extends WithTimestampsAndStringId {
+ @Index()
+ @Column('varchar', { length: 36 })
+ resultId: string;
+
+ @ManyToOne('AgentEvalResult', { onDelete: 'CASCADE' })
+ result: AgentEvalResult;
+
+ @Column('varchar', { length: 8 })
+ vote: AgentEvalVote;
+
+ @Column('text', { nullable: true })
+ comment: string | null;
+
+ @JsonColumn({ nullable: true })
+ correction: JsonObject | null;
+
+ @Column({ type: 'uuid', nullable: true })
+ ratedById: string | null;
+}
diff --git a/packages/@n8n/db/src/entities/agent-eval-result.ee.ts b/packages/@n8n/db/src/entities/agent-eval-result.ee.ts
new file mode 100644
index 00000000000..424966bed3f
--- /dev/null
+++ b/packages/@n8n/db/src/entities/agent-eval-result.ee.ts
@@ -0,0 +1,67 @@
+import { Column, Entity, Index, ManyToOne, OneToMany } from '@n8n/typeorm';
+import type { IDataObject, JsonObject } from 'n8n-workflow';
+
+import { DateTimeColumn, JsonColumn, WithTimestampsAndStringId } from './abstract-entity';
+import type { AgentEvalRating } from './agent-eval-rating.ee';
+import { AgentEvalRun } from './agent-eval-run.ee';
+
+export type AgentEvalResultStatus = 'new' | 'running' | 'success' | 'error' | 'cancelled';
+
+/**
+ * Per-case outcome of an {@link AgentEvalRun}: the agent's output, tool-call
+ * timeline, and metrics for one dataset row.
+ *
+ * Results outlive the dataset they came from (Data Table rows can be edited or
+ * deleted), so the case is **snapshotted** into `input` and the origin row is
+ * tracked loosely via `sourceRowId` with no FK.
+ */
+@Entity({ name: 'agent_eval_result' })
+export class AgentEvalResult extends WithTimestampsAndStringId {
+ @Index()
+ @Column('varchar', { length: 36 })
+ runId: string;
+
+ @ManyToOne('AgentEvalRun', { onDelete: 'CASCADE' })
+ run: AgentEvalRun;
+
+ /** ID of the source dataset row (Data Table / sheet); loose, no FK. */
+ @Column('varchar', { length: 255, nullable: true })
+ sourceRowId: string | null;
+
+ /**
+ * Sequential index of this case within its run, set when the run is seeded.
+ * Orders pending/running cases before `runAt` is populated.
+ */
+ @Column('integer', { nullable: true })
+ runIndex: number | null;
+
+ @Column('varchar')
+ status: AgentEvalResultStatus;
+
+ @JsonColumn({ nullable: true })
+ input: JsonObject | null;
+
+ @JsonColumn({ nullable: true })
+ output: JsonObject | null;
+
+ @JsonColumn({ nullable: true })
+ toolCalls: JsonObject | null;
+
+ @JsonColumn({ nullable: true })
+ metrics: IDataObject | null;
+
+ @DateTimeColumn({ nullable: true })
+ runAt: Date | null;
+
+ @DateTimeColumn({ nullable: true })
+ completedAt: Date | null;
+
+ @Column('varchar', { length: 255, nullable: true })
+ errorCode: string | null;
+
+ @JsonColumn({ nullable: true })
+ errorDetails: IDataObject | null;
+
+ @OneToMany('AgentEvalRating', 'result')
+ ratings: AgentEvalRating[];
+}
diff --git a/packages/@n8n/db/src/entities/agent-eval-run.ee.ts b/packages/@n8n/db/src/entities/agent-eval-run.ee.ts
new file mode 100644
index 00000000000..d8f6c49b1be
--- /dev/null
+++ b/packages/@n8n/db/src/entities/agent-eval-run.ee.ts
@@ -0,0 +1,73 @@
+import { Column, Entity, Index, ManyToOne, OneToMany } from '@n8n/typeorm';
+import type { IDataObject } from 'n8n-workflow';
+
+import { DateTimeColumn, JsonColumn, WithTimestampsAndStringId } from './abstract-entity';
+import { AgentEvalDataset } from './agent-eval-dataset.ee';
+import type { AgentEvalResult } from './agent-eval-result.ee';
+
+export type AgentEvalRunStatus = 'new' | 'running' | 'completed' | 'error' | 'cancelled';
+
+/**
+ * One execution of an {@link AgentEvalDataset} against a specific agent version.
+ * The agent under test is the dataset's agent — it is not duplicated here, so a
+ * run can never disagree with its dataset about which agent it targets. Stores
+ * run-level status, aggregated metrics, and the cross-main cancellation
+ * coordination fields (`runningInstanceId` + `cancelRequested`) proven on
+ * {@link TestRun}.
+ */
+@Entity({ name: 'agent_eval_run' })
+export class AgentEvalRun extends WithTimestampsAndStringId {
+ @Index()
+ @Column('varchar', { length: 36 })
+ datasetId: string;
+
+ @ManyToOne('AgentEvalDataset', { onDelete: 'CASCADE' })
+ dataset: AgentEvalDataset;
+
+ /**
+ * The published agent version under test — a loose pointer to an
+ * `agent_history` row, with no FK so a run survives history pruning (mirrors
+ * `TestRun.workflowVersionId`). Not enforced at the DB level: callers must
+ * pin a version of the dataset's own agent.
+ */
+ @Column('varchar', { length: 36, nullable: true })
+ agentVersionId: string | null;
+
+ @Column('varchar')
+ status: AgentEvalRunStatus;
+
+ @DateTimeColumn({ nullable: true })
+ runAt: Date | null;
+
+ @DateTimeColumn({ nullable: true })
+ completedAt: Date | null;
+
+ @JsonColumn({ nullable: true })
+ metrics: IDataObject | null;
+
+ @Column('varchar', { length: 255, nullable: true })
+ errorCode: string | null;
+
+ @JsonColumn({ nullable: true })
+ errorDetails: IDataObject | null;
+
+ /**
+ * ID of the instance running this eval. Used to coordinate cancellation
+ * across multiple main instances. See {@link TestRun.runningInstanceId}.
+ */
+ @Column('varchar', { length: 255, nullable: true })
+ runningInstanceId: string | null;
+
+ /**
+ * Fallback cancellation flag when the running instance can't be reached via
+ * pub/sub. See {@link TestRun.cancelRequested}.
+ */
+ @Column('boolean', { default: false })
+ cancelRequested: boolean;
+
+ @Column({ type: 'uuid', nullable: true })
+ createdById: string | null;
+
+ @OneToMany('AgentEvalResult', 'run')
+ results: AgentEvalResult[];
+}
diff --git a/packages/@n8n/db/src/entities/index.ts b/packages/@n8n/db/src/entities/index.ts
index 12495a5a58a..eda21f1914e 100644
--- a/packages/@n8n/db/src/entities/index.ts
+++ b/packages/@n8n/db/src/entities/index.ts
@@ -1,3 +1,7 @@
+import { AgentEvalDataset, type AgentEvalColumnMapping } from './agent-eval-dataset.ee';
+import { AgentEvalRating, type AgentEvalVote } from './agent-eval-rating.ee';
+import { AgentEvalResult, type AgentEvalResultStatus } from './agent-eval-result.ee';
+import { AgentEvalRun, type AgentEvalRunStatus } from './agent-eval-run.ee';
import { AiBuilderTemporaryWorkflow } from './ai-builder-temporary-workflow';
import { AnnotationTagEntity } from './annotation-tag-entity.ee';
import { AnnotationTagMapping } from './annotation-tag-mapping.ee';
@@ -76,6 +80,14 @@ import { WorkflowStatistics } from './workflow-statistics';
import { WorkflowTagMapping } from './workflow-tag-mapping';
export {
+ AgentEvalDataset,
+ type AgentEvalColumnMapping,
+ AgentEvalRun,
+ type AgentEvalRunStatus,
+ AgentEvalResult,
+ type AgentEvalResultStatus,
+ AgentEvalRating,
+ type AgentEvalVote,
InvalidAuthToken,
AiBuilderTemporaryWorkflow,
ProcessedData,
@@ -149,6 +161,10 @@ export {
};
export const entities = {
+ AgentEvalDataset,
+ AgentEvalRun,
+ AgentEvalResult,
+ AgentEvalRating,
InvalidAuthToken,
AiBuilderTemporaryWorkflow,
ProcessedData,
diff --git a/packages/@n8n/db/src/migrations/common/1784815940112-CreateAgentEvalTables.ts b/packages/@n8n/db/src/migrations/common/1784815940112-CreateAgentEvalTables.ts
new file mode 100644
index 00000000000..73dbc3dd6d2
--- /dev/null
+++ b/packages/@n8n/db/src/migrations/common/1784815940112-CreateAgentEvalTables.ts
@@ -0,0 +1,148 @@
+import type { MigrationContext, ReversibleMigration } from '../migration-types';
+
+const DATASET_TABLE = 'agent_eval_dataset';
+const RUN_TABLE = 'agent_eval_run';
+const RESULT_TABLE = 'agent_eval_result';
+const RATING_TABLE = 'agent_eval_rating';
+const AGENTS_TABLE = 'agents';
+const USER_TABLE = 'user';
+
+const RUN_STATUSES = ['new', 'running', 'completed', 'error', 'cancelled'];
+const RESULT_STATUSES = ['new', 'running', 'success', 'error', 'cancelled'];
+
+/**
+ * Persistence layer for agent evals: a dataset points (via `datasetSource` +
+ * `datasetRef`) at an existing Data Table / sheet, a run executes it against an
+ * agent version, a result captures per-case output, and a rating records human
+ * feedback. `agentVersionId` is a loose pointer (no FK) so runs survive history
+ * pruning, mirroring `TestRun.workflowVersionId`.
+ */
+export class CreateAgentEvalTables1784815940112 implements ReversibleMigration {
+ async up({ schemaBuilder: { createTable, column } }: MigrationContext) {
+ await createTable(DATASET_TABLE)
+ .withColumns(
+ column('id').varchar(36).primary.notNull,
+ column('name').varchar(128).notNull,
+ column('description').text,
+ column('agentId').varchar(36).notNull,
+ column('datasetSource')
+ .varchar(32)
+ .notNull.withEnumCheck(['data_table', 'google_sheets'])
+ .comment('Dataset backend the cases are read from'),
+ column('datasetRef').json.notNull.comment(
+ 'Pointer into the dataset backend (e.g. { dataTableId }); shape varies by datasetSource',
+ ),
+ column('columnMapping').json.comment(
+ 'Maps dataset columns onto input / expectedOutput / criteria roles',
+ ),
+ column('createdById').uuid,
+ )
+ .withIndexOn('agentId')
+ .withForeignKey('agentId', {
+ tableName: AGENTS_TABLE,
+ columnName: 'id',
+ onDelete: 'CASCADE',
+ })
+ .withForeignKey('createdById', {
+ tableName: USER_TABLE,
+ columnName: 'id',
+ onDelete: 'SET NULL',
+ }).withTimestamps;
+
+ await createTable(RUN_TABLE)
+ .withColumns(
+ column('id').varchar(36).primary.notNull,
+ column('datasetId').varchar(36).notNull,
+ column('agentVersionId')
+ .varchar(36)
+ .comment(
+ 'Published agent version under test (agent_history.versionId); loose pointer, no FK so runs survive history pruning. The agent itself comes from the dataset.',
+ ),
+ column('status').varchar().notNull.withEnumCheck(RUN_STATUSES).comment('Run lifecycle'),
+ column('runAt').timestampTimezone(),
+ column('completedAt').timestampTimezone(),
+ column('metrics').json.comment('Aggregated run-level scores'),
+ column('errorCode').varchar(255),
+ column('errorDetails').json,
+ column('runningInstanceId')
+ .varchar(255)
+ .comment('Main instance executing this run; used to coordinate cancellation'),
+ column('cancelRequested')
+ .bool.default(false)
+ .notNull.comment('Fallback cancellation flag polled by the running main'),
+ column('createdById').uuid,
+ )
+ .withIndexOn('datasetId')
+ .withForeignKey('datasetId', {
+ tableName: DATASET_TABLE,
+ columnName: 'id',
+ onDelete: 'CASCADE',
+ })
+ .withForeignKey('createdById', {
+ tableName: USER_TABLE,
+ columnName: 'id',
+ onDelete: 'SET NULL',
+ }).withTimestamps;
+
+ await createTable(RESULT_TABLE)
+ .withColumns(
+ column('id').varchar(36).primary.notNull,
+ column('runId').varchar(36).notNull,
+ column('sourceRowId')
+ .varchar(255)
+ .comment('Origin dataset row id; loose pointer, rows are external and mutable'),
+ column('runIndex').int.comment('Order of this case within the run'),
+ column('status')
+ .varchar()
+ .notNull.withEnumCheck(RESULT_STATUSES)
+ .comment('Per-case lifecycle'),
+ column('input').json.comment(
+ 'Snapshot of the case input actually run (row may later change)',
+ ),
+ column('output').json.comment('Agent output for this case'),
+ column('toolCalls').json.comment('Tool-call timeline captured during the run'),
+ column('metrics').json.comment('Per-case judge scores'),
+ column('runAt').timestampTimezone(),
+ column('completedAt').timestampTimezone(),
+ column('errorCode').varchar(255),
+ column('errorDetails').json,
+ )
+ .withIndexOn('runId')
+ .withForeignKey('runId', {
+ tableName: RUN_TABLE,
+ columnName: 'id',
+ onDelete: 'CASCADE',
+ }).withTimestamps;
+
+ await createTable(RATING_TABLE)
+ .withColumns(
+ column('id').varchar(36).primary.notNull,
+ column('resultId').varchar(36).notNull,
+ column('vote')
+ .varchar(8)
+ .notNull.withEnumCheck(['up', 'down'])
+ .comment('Human feedback direction'),
+ column('comment').text,
+ column('correction').json.comment('Optional corrected/edited output supplied by the rater'),
+ column('ratedById').uuid,
+ )
+ .withIndexOn('resultId')
+ .withForeignKey('resultId', {
+ tableName: RESULT_TABLE,
+ columnName: 'id',
+ onDelete: 'CASCADE',
+ })
+ .withForeignKey('ratedById', {
+ tableName: USER_TABLE,
+ columnName: 'id',
+ onDelete: 'SET NULL',
+ }).withTimestamps;
+ }
+
+ async down({ schemaBuilder: { dropTable } }: MigrationContext) {
+ await dropTable(RATING_TABLE);
+ await dropTable(RESULT_TABLE);
+ await dropTable(RUN_TABLE);
+ await dropTable(DATASET_TABLE);
+ }
+}
diff --git a/packages/@n8n/db/src/migrations/postgresdb/index.ts b/packages/@n8n/db/src/migrations/postgresdb/index.ts
index 000794dc2af..fcdd15e889a 100644
--- a/packages/@n8n/db/src/migrations/postgresdb/index.ts
+++ b/packages/@n8n/db/src/migrations/postgresdb/index.ts
@@ -228,6 +228,7 @@ import { BackfillInstanceAiEventLog1784000000051 } from '../common/1784000000051
import { CreateWorkflowReviewRequestTables1784000000052 } from '../common/1784000000052-CreateWorkflowReviewRequestTables';
import { AddStoredAtToAgentExecution1784815940110 } from '../common/1784815940110-AddStoredAtToAgentExecution';
import { AddInstanceCredentials1784815940111 } from '../common/1784815940111-AddInstanceCredentials';
+import { CreateAgentEvalTables1784815940112 } from '../common/1784815940112-CreateAgentEvalTables';
import type { Migration } from '../migration-types';
export const postgresMigrations: Migration[] = [
@@ -461,4 +462,5 @@ export const postgresMigrations: Migration[] = [
CreateWorkflowReviewRequestTables1784000000052,
AddStoredAtToAgentExecution1784815940110,
AddInstanceCredentials1784815940111,
+ CreateAgentEvalTables1784815940112,
];
diff --git a/packages/@n8n/db/src/migrations/sqlite/index.ts b/packages/@n8n/db/src/migrations/sqlite/index.ts
index 56d9b497e49..3679349e891 100644
--- a/packages/@n8n/db/src/migrations/sqlite/index.ts
+++ b/packages/@n8n/db/src/migrations/sqlite/index.ts
@@ -220,6 +220,7 @@ import { BackfillInstanceAiEventLog1784000000051 } from '../common/1784000000051
import { CreateWorkflowReviewRequestTables1784000000052 } from '../common/1784000000052-CreateWorkflowReviewRequestTables';
import { AddStoredAtToAgentExecution1784815940110 } from '../common/1784815940110-AddStoredAtToAgentExecution';
import { AddInstanceCredentials1784815940111 } from '../common/1784815940111-AddInstanceCredentials';
+import { CreateAgentEvalTables1784815940112 } from '../common/1784815940112-CreateAgentEvalTables';
const sqliteMigrations: Migration[] = [
InitialMigration1588102412422,
@@ -443,6 +444,7 @@ const sqliteMigrations: Migration[] = [
CreateWorkflowReviewRequestTables1784000000052,
AddStoredAtToAgentExecution1784815940110,
AddInstanceCredentials1784815940111,
+ CreateAgentEvalTables1784815940112,
];
export { sqliteMigrations };
diff --git a/packages/@n8n/db/src/repositories/__tests__/agent-eval-dataset.repository.test.ts b/packages/@n8n/db/src/repositories/__tests__/agent-eval-dataset.repository.test.ts
new file mode 100644
index 00000000000..2ebbd6b7891
--- /dev/null
+++ b/packages/@n8n/db/src/repositories/__tests__/agent-eval-dataset.repository.test.ts
@@ -0,0 +1,80 @@
+import { Container } from '@n8n/di';
+import type { Mock } from 'vitest';
+
+import { AgentEvalDataset } from '../../entities/agent-eval-dataset.ee';
+import { mockEntityManager } from '../../utils/test-utils/mock-entity-manager';
+import { AgentEvalDatasetRepository } from '../agent-eval-dataset.repository.ee';
+
+describe('AgentEvalDatasetRepository', () => {
+ const entityManager = mockEntityManager(AgentEvalDataset);
+ const repo = Container.get(AgentEvalDatasetRepository);
+
+ beforeEach(() => {
+ vi.resetAllMocks();
+ });
+
+ describe('createDataset', () => {
+ it('defaults optional fields to null', async () => {
+ (entityManager.create as Mock).mockImplementation(
+ (_target: unknown, entityLike: unknown) => entityLike as AgentEvalDataset,
+ );
+ entityManager.save.mockImplementationOnce(async (_target, entity) => entity);
+
+ await repo.createDataset({
+ name: 'D',
+ agentId: 'agent-1',
+ datasetSource: 'data_table',
+ datasetRef: { dataTableId: 'dt-1' },
+ });
+
+ const saved = entityManager.save.mock.calls[0]?.[1];
+ expect(saved).toMatchObject({
+ name: 'D',
+ agentId: 'agent-1',
+ datasetSource: 'data_table',
+ datasetRef: { dataTableId: 'dt-1' },
+ description: null,
+ columnMapping: null,
+ createdById: null,
+ });
+ });
+
+ it('persists provided optional fields', async () => {
+ (entityManager.create as Mock).mockImplementation(
+ (_target: unknown, entityLike: unknown) => entityLike as AgentEvalDataset,
+ );
+ entityManager.save.mockImplementationOnce(async (_target, entity) => entity);
+
+ await repo.createDataset({
+ name: 'D',
+ agentId: 'agent-1',
+ datasetSource: 'data_table',
+ datasetRef: { dataTableId: 'dt-1' },
+ description: 'desc',
+ columnMapping: { input: 'q', expectedOutput: 'a', criteria: 'c' },
+ createdById: 'user-1',
+ });
+
+ const saved = entityManager.save.mock.calls[0]?.[1];
+ expect(saved).toMatchObject({
+ description: 'desc',
+ columnMapping: { input: 'q', expectedOutput: 'a', criteria: 'c' },
+ createdById: 'user-1',
+ });
+ });
+ });
+
+ describe('findByAgentId', () => {
+ it('scopes the lookup to agentId, newest first', async () => {
+ entityManager.find.mockResolvedValueOnce([]);
+
+ await repo.findByAgentId('agent-1');
+
+ const callArgs = entityManager.find.mock.calls[0];
+ expect(callArgs?.[1]).toEqual({
+ where: { agentId: 'agent-1' },
+ order: { createdAt: 'DESC' },
+ });
+ });
+ });
+});
diff --git a/packages/@n8n/db/src/repositories/__tests__/agent-eval-rating.repository.test.ts b/packages/@n8n/db/src/repositories/__tests__/agent-eval-rating.repository.test.ts
new file mode 100644
index 00000000000..cba415b0ffd
--- /dev/null
+++ b/packages/@n8n/db/src/repositories/__tests__/agent-eval-rating.repository.test.ts
@@ -0,0 +1,69 @@
+import { Container } from '@n8n/di';
+import type { Mock } from 'vitest';
+
+import { AgentEvalRating } from '../../entities/agent-eval-rating.ee';
+import { mockEntityManager } from '../../utils/test-utils/mock-entity-manager';
+import { AgentEvalRatingRepository } from '../agent-eval-rating.repository.ee';
+
+describe('AgentEvalRatingRepository', () => {
+ const entityManager = mockEntityManager(AgentEvalRating);
+ const repo = Container.get(AgentEvalRatingRepository);
+
+ beforeEach(() => {
+ vi.resetAllMocks();
+ });
+
+ describe('createRating', () => {
+ it('defaults comment/correction/ratedById to null', async () => {
+ (entityManager.create as Mock).mockImplementation(
+ (_target: unknown, entityLike: unknown) => entityLike as AgentEvalRating,
+ );
+ entityManager.save.mockImplementationOnce(async (_target, entity) => entity);
+
+ await repo.createRating({ resultId: 'res-1', vote: 'up' });
+
+ const saved = entityManager.save.mock.calls[0]?.[1];
+ expect(saved).toMatchObject({
+ resultId: 'res-1',
+ vote: 'up',
+ comment: null,
+ correction: null,
+ ratedById: null,
+ });
+ });
+
+ it('persists a downvote with a correction and author', async () => {
+ (entityManager.create as Mock).mockImplementation(
+ (_target: unknown, entityLike: unknown) => entityLike as AgentEvalRating,
+ );
+ entityManager.save.mockImplementationOnce(async (_target, entity) => entity);
+
+ await repo.createRating({
+ resultId: 'res-1',
+ vote: 'down',
+ comment: 'wrong tone',
+ correction: { answer: 'better' },
+ ratedById: 'user-1',
+ });
+
+ const saved = entityManager.save.mock.calls[0]?.[1];
+ expect(saved).toMatchObject({
+ vote: 'down',
+ comment: 'wrong tone',
+ correction: { answer: 'better' },
+ ratedById: 'user-1',
+ });
+ });
+ });
+
+ describe('findByResultId', () => {
+ it('scopes to resultId, newest first', async () => {
+ entityManager.find.mockResolvedValueOnce([]);
+
+ await repo.findByResultId('res-1');
+
+ const callArgs = entityManager.find.mock.calls[0];
+ expect(callArgs?.[1]).toEqual({ where: { resultId: 'res-1' }, order: { createdAt: 'DESC' } });
+ });
+ });
+});
diff --git a/packages/@n8n/db/src/repositories/__tests__/agent-eval-result.repository.test.ts b/packages/@n8n/db/src/repositories/__tests__/agent-eval-result.repository.test.ts
new file mode 100644
index 00000000000..44bb6af81f5
--- /dev/null
+++ b/packages/@n8n/db/src/repositories/__tests__/agent-eval-result.repository.test.ts
@@ -0,0 +1,102 @@
+import { Container } from '@n8n/di';
+import type { Mock } from 'vitest';
+
+import { AgentEvalResult } from '../../entities/agent-eval-result.ee';
+import { mockEntityManager } from '../../utils/test-utils/mock-entity-manager';
+import { AgentEvalResultRepository } from '../agent-eval-result.repository.ee';
+
+describe('AgentEvalResultRepository', () => {
+ const entityManager = mockEntityManager(AgentEvalResult);
+ const repo = Container.get(AgentEvalResultRepository);
+
+ beforeEach(() => {
+ vi.resetAllMocks();
+ });
+
+ describe('seedResults', () => {
+ it('creates one pending result per case, preserving order index', async () => {
+ (entityManager.create as Mock).mockImplementation(
+ (_target: unknown, entityLike: unknown) => entityLike as AgentEvalResult,
+ );
+ entityManager.save.mockImplementationOnce(async (_target, entities) => entities);
+
+ await repo.seedResults([
+ { runId: 'run-1', sourceRowId: 'row-a', runIndex: 0, input: { q: '1' } },
+ { runId: 'run-1', sourceRowId: 'row-b', runIndex: 1, input: { q: '2' } },
+ ]);
+
+ const saved = entityManager.save.mock.calls[0]?.[1] as AgentEvalResult[];
+ expect(saved).toHaveLength(2);
+ expect(saved[0]).toMatchObject({
+ status: 'new',
+ runId: 'run-1',
+ sourceRowId: 'row-a',
+ runIndex: 0,
+ input: { q: '1' },
+ });
+ expect(saved[1]).toMatchObject({ runIndex: 1, sourceRowId: 'row-b' });
+ });
+
+ it('defaults sourceRowId/input to null and runIndex to the seed position', async () => {
+ (entityManager.create as Mock).mockImplementation(
+ (_target: unknown, entityLike: unknown) => entityLike as AgentEvalResult,
+ );
+ entityManager.save.mockImplementationOnce(async (_target, entities) => entities);
+
+ await repo.seedResults([{ runId: 'run-1' }, { runId: 'run-1' }]);
+
+ const saved = entityManager.save.mock.calls[0]?.[1] as AgentEvalResult[];
+ expect(saved[0]).toMatchObject({
+ status: 'new',
+ sourceRowId: null,
+ runIndex: 0,
+ input: null,
+ });
+ // runIndex falls back to the array position, so order is stable cross-DB.
+ expect(saved[1]).toMatchObject({ runIndex: 1 });
+ });
+ });
+
+ describe('markAsCompleted', () => {
+ it('stores the agent output and defaults toolCalls/metrics to null', async () => {
+ entityManager.update.mockResolvedValueOnce({ affected: 1, generatedMaps: [], raw: [] });
+
+ await repo.markAsCompleted('res-1', { output: { answer: '42' } });
+
+ const callArgs = entityManager.update.mock.calls[0];
+ expect(callArgs?.[1]).toBe('res-1');
+ expect(callArgs?.[2]).toMatchObject({
+ status: 'success',
+ output: { answer: '42' },
+ toolCalls: null,
+ metrics: null,
+ });
+ });
+ });
+
+ describe('markAsError', () => {
+ it('stores the error code and details', async () => {
+ entityManager.update.mockResolvedValueOnce({ affected: 1, generatedMaps: [], raw: [] });
+
+ await repo.markAsError('res-1', 'AGENT_THREW', { message: 'boom' });
+
+ const callArgs = entityManager.update.mock.calls[0];
+ expect(callArgs?.[2]).toMatchObject({
+ status: 'error',
+ errorCode: 'AGENT_THREW',
+ errorDetails: { message: 'boom' },
+ });
+ });
+ });
+
+ describe('findByRunId', () => {
+ it('scopes to runId ordered by runIndex ascending', async () => {
+ entityManager.find.mockResolvedValueOnce([]);
+
+ await repo.findByRunId('run-1');
+
+ const callArgs = entityManager.find.mock.calls[0];
+ expect(callArgs?.[1]).toEqual({ where: { runId: 'run-1' }, order: { runIndex: 'ASC' } });
+ });
+ });
+});
diff --git a/packages/@n8n/db/src/repositories/__tests__/agent-eval-run.repository.test.ts b/packages/@n8n/db/src/repositories/__tests__/agent-eval-run.repository.test.ts
new file mode 100644
index 00000000000..5e6ba3c0a72
--- /dev/null
+++ b/packages/@n8n/db/src/repositories/__tests__/agent-eval-run.repository.test.ts
@@ -0,0 +1,117 @@
+import { Container } from '@n8n/di';
+import type { Mock } from 'vitest';
+
+import { AgentEvalRun } from '../../entities/agent-eval-run.ee';
+import { mockEntityManager } from '../../utils/test-utils/mock-entity-manager';
+import { AgentEvalRunRepository } from '../agent-eval-run.repository.ee';
+
+describe('AgentEvalRunRepository', () => {
+ const entityManager = mockEntityManager(AgentEvalRun);
+ const repo = Container.get(AgentEvalRunRepository);
+
+ beforeEach(() => {
+ vi.resetAllMocks();
+ });
+
+ describe('createRun', () => {
+ it('starts a run as "new" with cancellation cleared', async () => {
+ (entityManager.create as Mock).mockImplementation(
+ (_target: unknown, entityLike: unknown) => entityLike as AgentEvalRun,
+ );
+ entityManager.save.mockImplementationOnce(async (_target, entity) => entity);
+
+ await repo.createRun({ datasetId: 'ds-1', agentVersionId: 'v-1' });
+
+ const saved = entityManager.save.mock.calls[0]?.[1];
+ expect(saved).toMatchObject({
+ status: 'new',
+ datasetId: 'ds-1',
+ agentVersionId: 'v-1',
+ createdById: null,
+ cancelRequested: false,
+ });
+ });
+
+ it('defaults agentVersionId to null when unpinned', async () => {
+ (entityManager.create as Mock).mockImplementation(
+ (_target: unknown, entityLike: unknown) => entityLike as AgentEvalRun,
+ );
+ entityManager.save.mockImplementationOnce(async (_target, entity) => entity);
+
+ await repo.createRun({ datasetId: 'ds-1' });
+
+ const saved = entityManager.save.mock.calls[0]?.[1] as AgentEvalRun;
+ expect(saved.agentVersionId).toBeNull();
+ });
+ });
+
+ describe('markAsRunning', () => {
+ it('records the running instance for cross-main cancellation', async () => {
+ entityManager.update.mockResolvedValueOnce({ affected: 1, generatedMaps: [], raw: [] });
+
+ await repo.markAsRunning('run-1', 'main-2');
+
+ const callArgs = entityManager.update.mock.calls[0];
+ expect(callArgs?.[1]).toBe('run-1');
+ expect(callArgs?.[2]).toMatchObject({ status: 'running', runningInstanceId: 'main-2' });
+ expect((callArgs?.[2] as { runAt: Date }).runAt).toBeInstanceOf(Date);
+ });
+
+ it('nulls the running instance when none is given', async () => {
+ entityManager.update.mockResolvedValueOnce({ affected: 1, generatedMaps: [], raw: [] });
+
+ await repo.markAsRunning('run-1');
+
+ const callArgs = entityManager.update.mock.calls[0];
+ expect((callArgs?.[2] as { runningInstanceId: string | null }).runningInstanceId).toBeNull();
+ });
+ });
+
+ describe('markAsError', () => {
+ it('stores the error code and details', async () => {
+ entityManager.update.mockResolvedValueOnce({ affected: 1, generatedMaps: [], raw: [] });
+
+ await repo.markAsError('run-1', 'RUNNER_FAILED', { message: 'boom' });
+
+ const callArgs = entityManager.update.mock.calls[0];
+ expect(callArgs?.[2]).toMatchObject({
+ status: 'error',
+ errorCode: 'RUNNER_FAILED',
+ errorDetails: { message: 'boom' },
+ runningInstanceId: null,
+ });
+ });
+
+ it('clears the running instance so finished runs leave no stale pointer', async () => {
+ entityManager.update.mockResolvedValueOnce({ affected: 1, generatedMaps: [], raw: [] });
+
+ await repo.markAsCompleted('run-1', { score: 1 });
+
+ const callArgs = entityManager.update.mock.calls[0];
+ expect(callArgs?.[2]).toMatchObject({ status: 'completed', runningInstanceId: null });
+ });
+ });
+
+ describe('requestCancellation', () => {
+ it('sets the cancel flag as a pub/sub fallback', async () => {
+ entityManager.update.mockResolvedValueOnce({ affected: 1, generatedMaps: [], raw: [] });
+
+ await repo.requestCancellation('run-1');
+
+ const callArgs = entityManager.update.mock.calls[0];
+ expect(callArgs?.[1]).toBe('run-1');
+ expect(callArgs?.[2]).toEqual({ cancelRequested: true });
+ });
+ });
+
+ describe('findByDatasetId', () => {
+ it('scopes to datasetId, newest first', async () => {
+ entityManager.find.mockResolvedValueOnce([]);
+
+ await repo.findByDatasetId('ds-1');
+
+ const callArgs = entityManager.find.mock.calls[0];
+ expect(callArgs?.[1]).toEqual({ where: { datasetId: 'ds-1' }, order: { createdAt: 'DESC' } });
+ });
+ });
+});
diff --git a/packages/@n8n/db/src/repositories/agent-eval-dataset.repository.ee.ts b/packages/@n8n/db/src/repositories/agent-eval-dataset.repository.ee.ts
new file mode 100644
index 00000000000..9cfb0d42f7e
--- /dev/null
+++ b/packages/@n8n/db/src/repositories/agent-eval-dataset.repository.ee.ts
@@ -0,0 +1,41 @@
+import type { DatasetRef } from '@n8n/api-types';
+import { Service } from '@n8n/di';
+import { DataSource, Repository } from '@n8n/typeorm';
+
+import { AgentEvalDataset } from '../entities';
+import type { AgentEvalColumnMapping } from '../entities/agent-eval-dataset.ee';
+
+type CreateAgentEvalDatasetAttrs = {
+ name: string;
+ agentId: string;
+ datasetSource: DatasetRef['datasetSource'];
+ datasetRef: DatasetRef['datasetRef'];
+ description?: string | null;
+ columnMapping?: AgentEvalColumnMapping | null;
+ createdById?: string | null;
+};
+
+@Service()
+export class AgentEvalDatasetRepository extends Repository {
+ constructor(dataSource: DataSource) {
+ super(AgentEvalDataset, dataSource.manager);
+ }
+
+ async createDataset(attrs: CreateAgentEvalDatasetAttrs): Promise {
+ const dataset = this.create({
+ name: attrs.name,
+ agentId: attrs.agentId,
+ datasetSource: attrs.datasetSource,
+ datasetRef: attrs.datasetRef,
+ description: attrs.description ?? null,
+ columnMapping: attrs.columnMapping ?? null,
+ createdById: attrs.createdById ?? null,
+ });
+
+ return await this.save(dataset);
+ }
+
+ async findByAgentId(agentId: string): Promise {
+ return await this.find({ where: { agentId }, order: { createdAt: 'DESC' } });
+ }
+}
diff --git a/packages/@n8n/db/src/repositories/agent-eval-rating.repository.ee.ts b/packages/@n8n/db/src/repositories/agent-eval-rating.repository.ee.ts
new file mode 100644
index 00000000000..1ce4ffb521b
--- /dev/null
+++ b/packages/@n8n/db/src/repositories/agent-eval-rating.repository.ee.ts
@@ -0,0 +1,37 @@
+import { Service } from '@n8n/di';
+import { DataSource, Repository } from '@n8n/typeorm';
+import type { JsonObject } from 'n8n-workflow';
+
+import { AgentEvalRating } from '../entities';
+import type { AgentEvalVote } from '../entities/agent-eval-rating.ee';
+
+type CreateAgentEvalRatingAttrs = {
+ resultId: string;
+ vote: AgentEvalVote;
+ comment?: string | null;
+ correction?: JsonObject | null;
+ ratedById?: string | null;
+};
+
+@Service()
+export class AgentEvalRatingRepository extends Repository {
+ constructor(dataSource: DataSource) {
+ super(AgentEvalRating, dataSource.manager);
+ }
+
+ async createRating(attrs: CreateAgentEvalRatingAttrs): Promise {
+ const rating = this.create({
+ resultId: attrs.resultId,
+ vote: attrs.vote,
+ comment: attrs.comment ?? null,
+ correction: attrs.correction ?? null,
+ ratedById: attrs.ratedById ?? null,
+ });
+
+ return await this.save(rating);
+ }
+
+ async findByResultId(resultId: string): Promise {
+ return await this.find({ where: { resultId }, order: { createdAt: 'DESC' } });
+ }
+}
diff --git a/packages/@n8n/db/src/repositories/agent-eval-result.repository.ee.ts b/packages/@n8n/db/src/repositories/agent-eval-result.repository.ee.ts
new file mode 100644
index 00000000000..f48ba22f30f
--- /dev/null
+++ b/packages/@n8n/db/src/repositories/agent-eval-result.repository.ee.ts
@@ -0,0 +1,70 @@
+import { Service } from '@n8n/di';
+import { DataSource, Repository } from '@n8n/typeorm';
+import type { IDataObject, JsonObject } from 'n8n-workflow';
+
+import { AgentEvalResult } from '../entities';
+
+type CreateAgentEvalResultAttrs = {
+ runId: string;
+ sourceRowId?: string | null;
+ runIndex?: number | null;
+ input?: JsonObject | null;
+};
+
+@Service()
+export class AgentEvalResultRepository extends Repository {
+ constructor(dataSource: DataSource) {
+ super(AgentEvalResult, dataSource.manager);
+ }
+
+ /**
+ * Seed one pending result per dataset row at run start, so the run detail
+ * view can list cases before each is executed.
+ */
+ async seedResults(cases: CreateAgentEvalResultAttrs[]): Promise {
+ const results = cases.map((c, index) =>
+ this.create({
+ status: 'new',
+ runId: c.runId,
+ sourceRowId: c.sourceRowId ?? null,
+ // Fall back to the seed position so `findByRunId` (orders by
+ // runIndex ASC) returns a stable order on every database. A null
+ // runIndex would sort first on SQLite but last on Postgres.
+ runIndex: c.runIndex ?? index,
+ input: c.input ?? null,
+ }),
+ );
+
+ return await this.save(results);
+ }
+
+ async markAsCompleted(
+ id: string,
+ attrs: {
+ output: JsonObject | null;
+ toolCalls?: JsonObject | null;
+ metrics?: IDataObject | null;
+ },
+ ) {
+ return await this.update(id, {
+ status: 'success',
+ completedAt: new Date(),
+ output: attrs.output,
+ toolCalls: attrs.toolCalls ?? null,
+ metrics: attrs.metrics ?? null,
+ });
+ }
+
+ async markAsError(id: string, errorCode: string, errorDetails?: IDataObject | null) {
+ return await this.update(id, {
+ status: 'error',
+ completedAt: new Date(),
+ errorCode,
+ errorDetails: errorDetails ?? null,
+ });
+ }
+
+ async findByRunId(runId: string): Promise {
+ return await this.find({ where: { runId }, order: { runIndex: 'ASC' } });
+ }
+}
diff --git a/packages/@n8n/db/src/repositories/agent-eval-run.repository.ee.ts b/packages/@n8n/db/src/repositories/agent-eval-run.repository.ee.ts
new file mode 100644
index 00000000000..f459ae0bf8b
--- /dev/null
+++ b/packages/@n8n/db/src/repositories/agent-eval-run.repository.ee.ts
@@ -0,0 +1,77 @@
+import { Service } from '@n8n/di';
+import { DataSource, Repository } from '@n8n/typeorm';
+import type { IDataObject } from 'n8n-workflow';
+
+import { AgentEvalRun } from '../entities';
+
+type CreateAgentEvalRunAttrs = {
+ datasetId: string;
+ agentVersionId?: string | null;
+ createdById?: string | null;
+};
+
+@Service()
+export class AgentEvalRunRepository extends Repository {
+ constructor(dataSource: DataSource) {
+ super(AgentEvalRun, dataSource.manager);
+ }
+
+ async createRun(attrs: CreateAgentEvalRunAttrs): Promise {
+ const run = this.create({
+ status: 'new',
+ datasetId: attrs.datasetId,
+ agentVersionId: attrs.agentVersionId ?? null,
+ createdById: attrs.createdById ?? null,
+ cancelRequested: false,
+ });
+
+ return await this.save(run);
+ }
+
+ async markAsRunning(id: string, instanceId?: string) {
+ return await this.update(id, {
+ status: 'running',
+ runAt: new Date(),
+ runningInstanceId: instanceId ?? null,
+ });
+ }
+
+ async markAsCompleted(id: string, metrics: IDataObject | null) {
+ return await this.update(id, {
+ status: 'completed',
+ completedAt: new Date(),
+ metrics,
+ runningInstanceId: null,
+ });
+ }
+
+ async markAsError(id: string, errorCode: string, errorDetails?: IDataObject | null) {
+ return await this.update(id, {
+ status: 'error',
+ completedAt: new Date(),
+ errorCode,
+ errorDetails: errorDetails ?? null,
+ runningInstanceId: null,
+ });
+ }
+
+ async markAsCancelled(id: string) {
+ return await this.update(id, {
+ status: 'cancelled',
+ completedAt: new Date(),
+ runningInstanceId: null,
+ });
+ }
+
+ /**
+ * Fallback cancellation signal when the running instance can't be reached
+ * via pub/sub — the running main polls this flag. Mirrors `TestRun`.
+ */
+ async requestCancellation(id: string) {
+ return await this.update(id, { cancelRequested: true });
+ }
+
+ async findByDatasetId(datasetId: string): Promise {
+ return await this.find({ where: { datasetId }, order: { createdAt: 'DESC' } });
+ }
+}
diff --git a/packages/@n8n/db/src/repositories/index.ts b/packages/@n8n/db/src/repositories/index.ts
index f4692947439..c19bbe148f3 100644
--- a/packages/@n8n/db/src/repositories/index.ts
+++ b/packages/@n8n/db/src/repositories/index.ts
@@ -1,3 +1,7 @@
+export { AgentEvalDatasetRepository } from './agent-eval-dataset.repository.ee';
+export { AgentEvalRunRepository } from './agent-eval-run.repository.ee';
+export { AgentEvalResultRepository } from './agent-eval-result.repository.ee';
+export { AgentEvalRatingRepository } from './agent-eval-rating.repository.ee';
export { AnnotationTagMappingRepository } from './annotation-tag-mapping.repository.ee';
export { AnnotationTagRepository } from './annotation-tag.repository.ee';
export { AiBuilderTemporaryWorkflowRepository } from './ai-builder-temporary-workflow.repository';
diff --git a/packages/cli/test/migration/1784815940112-create-agent-eval-tables.test.ts b/packages/cli/test/migration/1784815940112-create-agent-eval-tables.test.ts
new file mode 100644
index 00000000000..92a6aa5f2a1
--- /dev/null
+++ b/packages/cli/test/migration/1784815940112-create-agent-eval-tables.test.ts
@@ -0,0 +1,247 @@
+import {
+ createTestMigrationContext,
+ initDbUpToMigration,
+ runSingleMigration,
+ undoLastSingleMigration,
+ type TestMigrationContext,
+} from '@n8n/backend-test-utils';
+import { DbConnection } from '@n8n/db';
+import { Container } from '@n8n/di';
+import { DataSource } from '@n8n/typeorm';
+import { randomUUID } from 'node:crypto';
+
+const MIGRATION_NAME = 'CreateAgentEvalTables1784815940112';
+
+const AGENT_EVAL_TABLES = [
+ 'agent_eval_dataset',
+ 'agent_eval_run',
+ 'agent_eval_result',
+ 'agent_eval_rating',
+];
+
+async function tableExists(context: TestMigrationContext, table: string): Promise {
+ const name = `${context.tablePrefix}${table}`;
+ if (context.isSqlite) {
+ const rows = await context.runQuery>(
+ "SELECT name FROM sqlite_master WHERE type = 'table' AND name = :name",
+ { name },
+ );
+ return rows.length > 0;
+ }
+ const rows = await context.runQuery>(
+ 'SELECT tablename FROM pg_tables WHERE tablename = :name',
+ { name },
+ );
+ return rows.length > 0;
+}
+
+describe('CreateAgentEvalTables Migration', () => {
+ let dataSource: DataSource;
+
+ beforeAll(async () => {
+ const dbConnection = Container.get(DbConnection);
+ await dbConnection.init();
+
+ dataSource = Container.get(DataSource);
+
+ const context = createTestMigrationContext(dataSource);
+ await context.queryRunner.clearDatabase();
+ await context.queryRunner.release();
+
+ await initDbUpToMigration(MIGRATION_NAME);
+ });
+
+ afterAll(async () => {
+ const dbConnection = Container.get(DbConnection);
+ await dbConnection.close();
+ });
+
+ async function insertProject(context: TestMigrationContext, id: string): Promise {
+ const table = context.escape.tableName('project');
+ const now = new Date();
+ await context.runQuery(
+ `INSERT INTO ${table} ("id", "name", "type", "customTelemetryTags", "createdAt", "updatedAt") VALUES (:id, :name, :type, :tags, :createdAt, :updatedAt)`,
+ { id, name: 'Test Project', type: 'team', tags: '[]', createdAt: now, updatedAt: now },
+ );
+ }
+
+ async function insertAgent(
+ context: TestMigrationContext,
+ data: { id: string; projectId: string },
+ ): Promise {
+ const table = context.escape.tableName('agents');
+ const now = new Date();
+ await context.runQuery(
+ `INSERT INTO ${table} ("id", "name", "projectId", "integrations", "tools", "skills", "createdAt", "updatedAt") VALUES (:id, :name, :projectId, :integrations, :tools, :skills, :createdAt, :updatedAt)`,
+ {
+ id: data.id,
+ name: 'Test Agent',
+ projectId: data.projectId,
+ integrations: '[]',
+ tools: '{}',
+ skills: '{}',
+ createdAt: now,
+ updatedAt: now,
+ },
+ );
+ }
+
+ async function insertDataset(
+ context: TestMigrationContext,
+ data: { id: string; agentId: string },
+ ): Promise {
+ const table = context.escape.tableName('agent_eval_dataset');
+ const now = new Date();
+ await context.runQuery(
+ `INSERT INTO ${table} ("id", "name", "agentId", "datasetSource", "datasetRef", "createdAt", "updatedAt") VALUES (:id, :name, :agentId, :datasetSource, :datasetRef, :createdAt, :updatedAt)`,
+ {
+ id: data.id,
+ name: 'Test Dataset',
+ agentId: data.agentId,
+ datasetSource: 'data_table',
+ datasetRef: JSON.stringify({ dataTableId: 'dt-1' }),
+ createdAt: now,
+ updatedAt: now,
+ },
+ );
+ }
+
+ async function insertRun(
+ context: TestMigrationContext,
+ data: { id: string; datasetId: string },
+ ): Promise {
+ const table = context.escape.tableName('agent_eval_run');
+ const now = new Date();
+ await context.runQuery(
+ `INSERT INTO ${table} ("id", "datasetId", "status", "cancelRequested", "createdAt", "updatedAt") VALUES (:id, :datasetId, :status, :cancelRequested, :createdAt, :updatedAt)`,
+ {
+ id: data.id,
+ datasetId: data.datasetId,
+ status: 'new',
+ cancelRequested: false,
+ createdAt: now,
+ updatedAt: now,
+ },
+ );
+ }
+
+ async function insertResult(
+ context: TestMigrationContext,
+ data: { id: string; runId: string },
+ ): Promise {
+ const table = context.escape.tableName('agent_eval_result');
+ const now = new Date();
+ await context.runQuery(
+ `INSERT INTO ${table} ("id", "runId", "status", "createdAt", "updatedAt") VALUES (:id, :runId, :status, :createdAt, :updatedAt)`,
+ { id: data.id, runId: data.runId, status: 'new', createdAt: now, updatedAt: now },
+ );
+ }
+
+ async function insertRating(
+ context: TestMigrationContext,
+ data: { id: string; resultId: string },
+ ): Promise {
+ const table = context.escape.tableName('agent_eval_rating');
+ const now = new Date();
+ await context.runQuery(
+ `INSERT INTO ${table} ("id", "resultId", "vote", "createdAt", "updatedAt") VALUES (:id, :resultId, :vote, :createdAt, :updatedAt)`,
+ { id: data.id, resultId: data.resultId, vote: 'up', createdAt: now, updatedAt: now },
+ );
+ }
+
+ describe('up', () => {
+ beforeAll(async () => {
+ await runSingleMigration(MIGRATION_NAME);
+ });
+
+ it('creates all four agent-eval tables', async () => {
+ const context = createTestMigrationContext(dataSource);
+ for (const table of AGENT_EVAL_TABLES) {
+ expect(await tableExists(context, table)).toBe(true);
+ }
+ await context.queryRunner.release();
+ });
+
+ it('persists a full dataset → run → result → rating chain through the FKs', async () => {
+ const context = createTestMigrationContext(dataSource);
+ const projectId = randomUUID();
+ const agentId = randomUUID();
+ const datasetId = randomUUID();
+ const runId = randomUUID();
+ const resultId = randomUUID();
+ const ratingId = randomUUID();
+
+ await insertProject(context, projectId);
+ await insertAgent(context, { id: agentId, projectId });
+ await insertDataset(context, { id: datasetId, agentId });
+ await insertRun(context, { id: runId, datasetId });
+ await insertResult(context, { id: resultId, runId });
+ await insertRating(context, { id: ratingId, resultId });
+
+ const runTable = context.escape.tableName('agent_eval_run');
+ const runs = await context.runQuery>(
+ `SELECT "id", "datasetId" FROM ${runTable} WHERE "id" = :id`,
+ { id: runId },
+ );
+ expect(runs).toHaveLength(1);
+ expect(runs[0].datasetId).toBe(datasetId);
+
+ await context.queryRunner.release();
+ });
+
+ it('cascades deletes from dataset down to results and ratings', async () => {
+ const context = createTestMigrationContext(dataSource);
+ const projectId = randomUUID();
+ const agentId = randomUUID();
+ const datasetId = randomUUID();
+ const runId = randomUUID();
+ const resultId = randomUUID();
+ const ratingId = randomUUID();
+
+ await insertProject(context, projectId);
+ await insertAgent(context, { id: agentId, projectId });
+ await insertDataset(context, { id: datasetId, agentId });
+ await insertRun(context, { id: runId, datasetId });
+ await insertResult(context, { id: resultId, runId });
+ await insertRating(context, { id: ratingId, resultId });
+
+ const datasetTable = context.escape.tableName('agent_eval_dataset');
+ await context.runQuery(`DELETE FROM ${datasetTable} WHERE "id" = :id`, { id: datasetId });
+
+ // The whole chain below the dataset must be gone, not just the leaf.
+ const runTable = context.escape.tableName('agent_eval_run');
+ const resultTable = context.escape.tableName('agent_eval_result');
+ const ratingTable = context.escape.tableName('agent_eval_rating');
+
+ const runs = await context.runQuery>(
+ `SELECT "id" FROM ${runTable} WHERE "id" = :id`,
+ { id: runId },
+ );
+ const results = await context.runQuery>(
+ `SELECT "id" FROM ${resultTable} WHERE "id" = :id`,
+ { id: resultId },
+ );
+ const ratings = await context.runQuery>(
+ `SELECT "id" FROM ${ratingTable} WHERE "id" = :id`,
+ { id: ratingId },
+ );
+ expect(runs).toHaveLength(0);
+ expect(results).toHaveLength(0);
+ expect(ratings).toHaveLength(0);
+
+ await context.queryRunner.release();
+ });
+ });
+
+ describe('down', () => {
+ it('drops all four agent-eval tables', async () => {
+ await undoLastSingleMigration();
+
+ const context = createTestMigrationContext(dataSource);
+ for (const table of AGENT_EVAL_TABLES) {
+ expect(await tableExists(context, table)).toBe(false);
+ }
+ await context.queryRunner.release();
+ });
+ });
+});