test: Add Instance AI workflow evals CI pipeline (no-changelog) (#28366)

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

.github/workflows/ci-pull-requests.yml

@@ -27,6 +27,7 @@ jobs:
       db: ${{ fromJSON(steps.ci-filter.outputs.results).db == true }}
       performance: ${{ fromJSON(steps.ci-filter.outputs.results).performance == true }}
       e2e_performance: ${{ fromJSON(steps.ci-filter.outputs.results)['e2e-performance'] == true }}
+      instance_ai_workflow_eval: ${{ fromJSON(steps.ci-filter.outputs.results)['instance-ai-workflow-eval'] == true }}
       commit_sha: ${{ steps.commit-sha.outputs.sha }}
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -70,6 +71,12 @@ jobs:
               packages/testing/playwright/utils/performance-helper.ts
               packages/testing/containers/**
               .github/workflows/test-e2e-performance-reusable.yml
+            instance-ai-workflow-eval:
+              packages/@n8n/instance-ai/src/**
+              packages/@n8n/instance-ai/evaluations/**
+              packages/cli/src/modules/instance-ai/**
+              packages/core/src/execution-engine/eval-mock-helpers.ts
+              .github/workflows/test-evals-instance-ai*.yml
             db:
               packages/cli/src/databases/**
               packages/cli/src/modules/*/database/**
@@ -196,6 +203,18 @@ jobs:
       ref: ${{ needs.install-and-build.outputs.commit_sha }}
     secrets: inherit
 
+  instance-ai-workflow-evals:
+    name: Instance AI Workflow Evals
+    needs: install-and-build
+    if: >-
+      needs.install-and-build.outputs.instance_ai_workflow_eval == 'true' &&
+      github.repository == 'n8n-io/n8n' &&
+      (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork)
+    uses: ./.github/workflows/test-evals-instance-ai.yml
+    with:
+      branch: ${{ needs.install-and-build.outputs.commit_sha }}
+    secrets: inherit
+
   # This job is required by GitHub branch protection rules.
   # PRs cannot be merged unless this job passes.
   required-checks:

.github/workflows/test-evals-instance-ai.yml

@@ -0,0 +1,141 @@
+name: 'Test: Instance AI Exec Evals'
+
+on:
+  workflow_call:
+    inputs:
+      branch:
+        description: 'GitHub branch to test'
+        required: false
+        type: string
+        default: 'master'
+      filter:
+        description: 'Filter test cases by name (e.g. "contact-form")'
+        required: false
+        type: string
+        default: ''
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'GitHub branch to test'
+        required: false
+        default: 'master'
+      filter:
+        description: 'Filter test cases by name (e.g. "contact-form")'
+        required: false
+        default: ''
+
+jobs:
+  run-evals:
+    name: 'Run Evals'
+    runs-on: blacksmith-4vcpu-ubuntu-2204
+    timeout-minutes: 45
+    permissions:
+      contents: read
+      pull-requests: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ inputs.branch || github.ref }}
+          fetch-depth: 1
+
+      - name: Setup Environment
+        uses: ./.github/actions/setup-nodejs
+        with:
+          build-command: 'pnpm build'
+
+      - name: Build Docker image
+        run: pnpm build:docker
+        env:
+          INCLUDE_TEST_CONTROLLER: 'true'
+
+      - name: Start n8n container
+        run: |
+          docker run -d --name n8n-eval \
+            -e E2E_TESTS=true \
+            -e N8N_ENABLED_MODULES=instance-ai \
+            -e N8N_AI_ENABLED=true \
+            -e N8N_INSTANCE_AI_MODEL_API_KEY=${{ secrets.EVALS_ANTHROPIC_KEY }} \
+            -e N8N_LICENSE_ACTIVATION_KEY=${{ secrets.N8N_LICENSE_ACTIVATION_KEY }} \
+            -e N8N_LICENSE_CERT=${{ secrets.N8N_LICENSE_CERT }} \
+            -e N8N_ENCRYPTION_KEY=${{ secrets.N8N_ENCRYPTION_KEY }} \
+            -p 5678:5678 \
+            n8nio/n8n:local
+          echo "Waiting for n8n to be ready..."
+          for i in $(seq 1 60); do
+            if curl -s http://localhost:5678/healthz/readiness -o /dev/null -w "%{http_code}" | grep -q 200; then
+              echo "n8n ready after ${i}s"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "::error::n8n failed to start within 60s"
+          docker logs n8n-eval --tail 30
+          exit 1
+
+      - name: Create test user
+        run: |
+          curl -sf -X POST http://localhost:5678/rest/e2e/reset \
+            -H "Content-Type: application/json" \
+            -d '{
+              "owner":{"email":"nathan@n8n.io","password":"PlaywrightTest123","firstName":"Eval","lastName":"Owner"},
+              "admin":{"email":"admin@n8n.io","password":"PlaywrightTest123","firstName":"Admin","lastName":"User"},
+              "members":[],
+              "chat":{"email":"chat@n8n.io","password":"PlaywrightTest123","firstName":"Chat","lastName":"User"}
+            }'
+
+      - name: Run Instance AI Evals
+        continue-on-error: true
+        working-directory: packages/@n8n/instance-ai
+        run: >-
+          pnpm eval:instance-ai
+          --base-url http://localhost:5678
+          --verbose
+          ${{ inputs.filter && format('--filter "{0}"', inputs.filter) || '' }}
+        env:
+          N8N_INSTANCE_AI_MODEL_API_KEY: ${{ secrets.EVALS_ANTHROPIC_KEY }}
+
+      - name: Stop n8n container
+        if: ${{ always() }}
+        run: docker stop n8n-eval && docker rm n8n-eval || true
+
+      - name: Post eval results to PR
+        if: ${{ always() && github.event_name == 'pull_request' }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          RESULTS_FILE="packages/@n8n/instance-ai/eval-results.json"
+          if [ ! -f "$RESULTS_FILE" ]; then
+            echo "No eval results file found"
+            exit 0
+          fi
+
+          # Build the full comment body with jq
+          jq -r '
+            "### Instance AI Workflow Eval Results\n\n" +
+            "**\(.summary.built)/\(.summary.testCases) built | \(.summary.scenariosPassed)/\(.summary.scenariosTotal) passed (\(.summary.passRate * 100 | floor)%)**\n\n" +
+            "| Workflow | Build | Passed |\n|---|---|---|\n" +
+            ([.testCases[] | "| \(.name) | \(if .built then "✅" else "❌" end) | \([.scenarios[] | select(.passed)] | length)/\(.scenarios | length) |"] | join("\n")) +
+            "\n\n<details><summary>Failure details</summary>\n\n" +
+            ([.testCases[].scenarios[] | select(.passed == false) | "**\(.name)** \(if .failureCategory then "[\(.failureCategory)]" else "" end)\n> \(.reasoning | .[0:200])\n"] | join("\n")) +
+            "\n</details>"
+          ' "$RESULTS_FILE" > /tmp/eval-comment.md
+
+          # Find and update existing eval comment, or create new one
+          COMMENT_ID=$(gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \
+            --jq '.[] | select(.body | startswith("### Instance AI Workflow Eval")) | .id' | tail -1)
+
+          if [ -n "$COMMENT_ID" ]; then
+            gh api "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" -X PATCH -F body=@/tmp/eval-comment.md
+          else
+            gh pr comment "${{ github.event.pull_request.number }}" --body-file /tmp/eval-comment.md
+          fi
+
+      - name: Upload Results
+        if: ${{ always() }}
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: instance-ai-workflow-eval-results
+          path: packages/@n8n/instance-ai/eval-results.json
+          retention-days: 14