feat(ai-builder): Re-run Instance AI evals against the PR head on demand (no-changelog) (#33148)

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.github/actions/load-n8n-docker/action.yml

@@ -18,6 +18,10 @@ inputs:
     description: 'Variant of the image to load. Must match what build-n8n-docker used (standard or coverage).'
     required: false
     default: 'standard'
+  cache-sha:
+    description: 'SHA for the image cache key. Empty = github.sha. Override when testing a ref other than the triggering commit (e.g. a PR head resolved at dispatch time) so the restore matches the code under test instead of a stale image (cache miss => rebuild from the checkout).'
+    required: false
+    default: ''
 
 runs:
   using: 'composite'
@@ -26,7 +30,7 @@ runs:
       id: restore
       uses: actions/cache/restore@640a1c2554105b57832a23eea0b4672fc7a790d5 # v4.2.3
       with:
-        key: n8n-docker-image-${{ inputs.build-variant }}-${{ github.sha }}
+        key: n8n-docker-image-${{ inputs.build-variant }}-${{ inputs.cache-sha || github.sha }}
         path: /tmp/n8n-image.tar.zst
 
     - name: Load n8n and runners images into docker
@@ -39,7 +43,7 @@ runs:
       shell: bash
       env:
         BUILD_VARIANT: ${{ inputs.build-variant }}
-        GITHUB_SHA: ${{ github.sha }}
+        GITHUB_SHA: ${{ inputs.cache-sha || github.sha }}
       run: |
         echo "::warning::Cache miss for n8n-docker-image-$BUILD_VARIANT-$GITHUB_SHA (SHA $GITHUB_SHA); falling back to rebuild via build-n8n-docker."
 

.github/workflows/ci-instance-ai-evals.yml

@@ -12,14 +12,18 @@ on:
       - 'packages/core/src/execution-engine/eval-mock-helpers.ts'
   workflow_dispatch:
     inputs:
+      pr:
+        description: 'PR number to re-run against its latest commit. Resolves the PR head at dispatch time and posts results back; takes precedence over `branch`. Leave empty to use `branch`.'
+        required: false
+        default: ''
       branch:
-        description: 'GitHub branch to test'
+        description: 'GitHub branch to test (ignored when `pr` is set)'
         required: false
         default: 'master'
       tier:
-        description: 'Test-case dataset to run (e.g. `pr`, `full`)'
+        description: 'Test-case dataset to run (e.g. `pr`, `full`). Empty => `pr` for a PR, `full` otherwise.'
         required: false
-        default: 'full'
+        default: ''
       sandbox-provider:
         description: 'Sandbox provider (n8n-sandbox or daytona)'
         required: false
@@ -34,25 +38,107 @@ on:
         default: ''
 
 concurrency:
-  group: instance-ai-evals-${{ github.ref }}
+  # Key on the PR number for dispatched re-runs (github.ref would collapse all
+  # dispatches from the default branch into one group and cross-cancel them).
+  group: instance-ai-evals-${{ inputs.pr || github.ref }}
   cancel-in-progress: true
 
 jobs:
-  run-evals:
-    name: Instance AI Workflow Evals
-    # Skip drafts; allow workflow_dispatch (no pull_request payload).
+  # Resolves the ref/SHA/PR to test. A `uses:` caller job can't run steps, so
+  # the `gh pr view` lookup for `-f pr=<n>` dispatches lives in its own job and
+  # feeds run-evals via outputs. Also the draft/repo gate: if this is skipped,
+  # run-evals (needs: resolve) is skipped too.
+  resolve:
+    name: Resolve eval target
     if: >-
       github.repository == 'n8n-io/n8n' &&
       (github.event_name != 'pull_request' || github.event.pull_request.draft == false)
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+    outputs:
+      branch: ${{ steps.resolve.outputs.branch }}
+      cache_sha: ${{ steps.resolve.outputs.cache_sha }}
+      revision_sha: ${{ steps.resolve.outputs.revision_sha }}
+      head_ref: ${{ steps.resolve.outputs.head_ref }}
+      pr_number: ${{ steps.resolve.outputs.pr_number }}
+      tier: ${{ steps.resolve.outputs.tier }}
+    steps:
+      - name: Resolve target ref
+        id: resolve
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          EVENT_NAME: ${{ github.event_name }}
+          GH_SHA: ${{ github.sha }}
+          INPUT_PR: ${{ inputs.pr }}
+          INPUT_BRANCH: ${{ inputs.branch }}
+          INPUT_TIER: ${{ inputs.tier }}
+          EVENT_PR_NUMBER: ${{ github.event.pull_request.number }}
+          EVENT_HEAD_REF: ${{ github.event.pull_request.head.ref }}
+        run: |
+          set -euo pipefail
+          if [ -n "$INPUT_PR" ]; then
+            # Dispatch against a PR number: test the latest push (auto PR-open
+            # runs go stale). Prefer the merge ref (refs/pull/N/merge) so we test
+            # the merged state like PR-open does. The merge ref updates async
+            # after a push - use it only when its parents include the current
+            # head (i.e. it reflects the latest push); otherwise fall back to the
+            # raw head. Either way a dispatched run can't restore the prepare-docker
+            # image cache (it's scoped to refs/pull/N/merge), so load-n8n-docker
+            # rebuilds from the checkout.
+            pr_json=$(gh api "repos/$REPO/pulls/$INPUT_PR")
+            head_sha=$(echo "$pr_json" | jq -r '.head.sha')
+            head_ref=$(echo "$pr_json" | jq -r '.head.ref')
+            merge_sha=$(echo "$pr_json" | jq -r '.merge_commit_sha // empty')
+            tested_sha="$head_sha"
+            if [ -n "$merge_sha" ] && \
+               gh api "repos/$REPO/commits/$merge_sha" --jq '.parents[].sha' 2>/dev/null \
+                 | grep -qx "$head_sha"; then
+              tested_sha="$merge_sha"
+              echo "PR #$INPUT_PR: testing merge ref $merge_sha (fresh; merged state, image rebuilt)"
+            else
+              echo "PR #$INPUT_PR: testing head $head_sha (merge ref stale/unavailable; image rebuilt)"
+            fi
+            branch="$tested_sha"; cache_sha="$tested_sha"; revision_sha="$tested_sha"
+            pr_number="$INPUT_PR"; tier="${INPUT_TIER:-pr}"
+          elif [ "$EVENT_NAME" = "pull_request" ]; then
+            # PR open/reopen/ready: unchanged - test the merge commit, which also
+            # matches the SHA-keyed prepare-docker image cache.
+            branch="$GH_SHA"; cache_sha="$GH_SHA"; revision_sha="$GH_SHA"
+            head_ref="$EVENT_HEAD_REF"; pr_number="$EVENT_PR_NUMBER"; tier="${INPUT_TIER:-pr}"
+          else
+            # Dispatch with an explicit branch (defaults to 'master'). Key the
+            # image cache and LangSmith tags on the ref under test, not
+            # github.sha: github.sha is the workflow's own ref (master when
+            # dispatched there), so keying on it would load master's prebuilt
+            # image even when testing another branch. A branch name misses the
+            # SHA-keyed cache, so load-n8n-docker rebuilds from the checkout.
+            branch="${INPUT_BRANCH:-$GH_SHA}"; cache_sha="$branch"; revision_sha="$branch"
+            head_ref="$branch"; pr_number=""; tier="${INPUT_TIER:-full}"
+          fi
+          {
+            echo "branch=$branch"
+            echo "cache_sha=$cache_sha"
+            echo "revision_sha=$revision_sha"
+            echo "head_ref=$head_ref"
+            echo "pr_number=$pr_number"
+            echo "tier=$tier"
+          } >> "$GITHUB_OUTPUT"
+
+  run-evals:
+    name: Instance AI Workflow Evals
+    needs: resolve
     uses: ./.github/workflows/test-evals-instance-ai.yml
     with:
-      # On PR events github.sha is the merge commit, so the eval tests the
-      # merged state (like e2e) and the checkout matches the key under which
-      # prepare-docker publishes the docker image cache.
-      branch: ${{ inputs.branch || github.sha }}
-      # PR events default to `pr`; dispatch defaults to `full`.
-      tier: ${{ inputs.tier || 'pr' }}
+      branch: ${{ needs.resolve.outputs.branch }}
+      tier: ${{ needs.resolve.outputs.tier }}
       sandbox-provider: ${{ inputs.sandbox-provider || 'n8n-sandbox' }}
       iterations: ${{ inputs.iterations }}
       experiment-name: ${{ inputs.experiment-name }}
+      pr-number: ${{ needs.resolve.outputs.pr_number }}
+      cache-sha: ${{ needs.resolve.outputs.cache_sha }}
+      revision-sha: ${{ needs.resolve.outputs.revision_sha }}
+      head-ref: ${{ needs.resolve.outputs.head_ref }}
     secrets: inherit

.github/workflows/test-evals-instance-ai.yml

@@ -33,6 +33,26 @@ on:
         required: false
         type: string
         default: ''
+      pr-number:
+        description: 'PR number to post results back to. Empty = derive from the pull_request event.'
+        required: false
+        type: string
+        default: ''
+      cache-sha:
+        description: 'SHA for the docker image cache key. Empty = github.sha. Set to the tested commit on dispatch re-runs so the image matches the code under test.'
+        required: false
+        type: string
+        default: ''
+      revision-sha:
+        description: 'Commit SHA under test, for LangSmith revision tagging. Empty = github.sha.'
+        required: false
+        type: string
+        default: ''
+      head-ref:
+        description: 'Branch name under test, for LangSmith branch tagging. Empty = derive from context.'
+        required: false
+        type: string
+        default: ''
   workflow_dispatch:
     inputs:
       branch:
@@ -90,6 +110,8 @@ jobs:
       # via build-n8n-docker.
       - name: Load n8n Docker image
         uses: ./.github/actions/load-n8n-docker
+        with:
+          cache-sha: ${{ inputs.cache-sha }}
 
       - name: Start sandbox service
         if: ${{ inputs.sandbox-provider == 'n8n-sandbox' }}
@@ -224,12 +246,13 @@ jobs:
           LANGSMITH_TRACING: 'true'
           LANGSMITH_ENDPOINT: ${{ secrets.EVALS_LANGSMITH_ENDPOINT }}
           LANGSMITH_API_KEY: ${{ secrets.EVALS_LANGSMITH_API_KEY }}
-          LANGSMITH_REVISION_ID: ${{ github.sha }}
-          LANGSMITH_BRANCH: ${{ github.event.pull_request.head.ref || github.head_ref || github.ref_name }}
+          LANGSMITH_REVISION_ID: ${{ inputs.revision-sha || github.sha }}
+          LANGSMITH_BRANCH: ${{ inputs.head-ref || github.event.pull_request.head.ref || github.head_ref || github.ref_name }}
           FILTER: ${{ inputs.filter }}
           TIER: ${{ inputs.tier }}
           ITERATIONS: ${{ inputs.iterations }}
           EXPERIMENT_NAME: ${{ inputs.experiment-name }}
+          EVAL_PR_NUMBER: ${{ inputs.pr-number || github.event.pull_request.number }}
         run: |
           IFS=',' read -ra PORTS <<< "$LANE_PORTS"
           URLS=()
@@ -320,9 +343,11 @@ jobs:
           docker network rm n8n-eval-net 2>/dev/null || true
 
       - name: Post eval results to PR
-        if: ${{ always() && github.event.pull_request.number }}
+        if: ${{ always() && (inputs.pr-number || github.event.pull_request.number) }}
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ inputs.pr-number || github.event.pull_request.number }}
         run: |
           # The eval CLI writes the full PR comment as eval-pr-comment.md
           # (see comparison/format.ts:formatComparisonMarkdown). It includes
@@ -336,13 +361,13 @@ jobs:
           cp "$COMMENT_FILE" /tmp/eval-comment.md
 
           # Find and update existing eval comment, or create new one
-          COMMENT_ID=$(gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \
+          COMMENT_ID=$(gh api "repos/$REPO/issues/$PR_NUMBER/comments" \
             --jq '.[] | select(.body | startswith("### Instance AI Workflow Eval")) | .id' | tail -1)
 
           if [ -n "$COMMENT_ID" ]; then
-            gh api "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" -X PATCH -F body=@/tmp/eval-comment.md
+            gh api "repos/$REPO/issues/comments/${COMMENT_ID}" -X PATCH -F body=@/tmp/eval-comment.md
           else
-            gh pr comment "${{ github.event.pull_request.number }}" --body-file /tmp/eval-comment.md
+            gh pr comment "$PR_NUMBER" --body-file /tmp/eval-comment.md
           fi
 
       - name: Upload Results