feat(ai-builder): Re-run Instance AI evals against the PR head on demand (no-changelog) (#33148)

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-09-01 15:47:41 +08:00 · 2026-06-29 07:00:43 +01:00
parent 0c7a61d54a
commit c3741ce12a
7 changed files with 218 additions and 39 deletions
@@ -18,6 +18,10 @@ inputs:
    description: 'Variant of the image to load. Must match what build-n8n-docker used (standard or coverage).'
    required: false
    default: 'standard'
+  cache-sha:
+    description: 'SHA for the image cache key. Empty = github.sha. Override when testing a ref other than the triggering commit (e.g. a PR head resolved at dispatch time) so the restore matches the code under test instead of a stale image (cache miss => rebuild from the checkout).'
+    required: false
+    default: ''

 runs:
  using: 'composite'
@@ -26,7 +30,7 @@ runs:
      id: restore
      uses: actions/cache/restore@640a1c2554105b57832a23eea0b4672fc7a790d5 # v4.2.3
      with:
-        key: n8n-docker-image-${{ inputs.build-variant }}-${{ github.sha }}
+        key: n8n-docker-image-${{ inputs.build-variant }}-${{ inputs.cache-sha || github.sha }}
        path: /tmp/n8n-image.tar.zst

    - name: Load n8n and runners images into docker
@@ -39,7 +43,7 @@ runs:
      shell: bash
      env:
        BUILD_VARIANT: ${{ inputs.build-variant }}
-        GITHUB_SHA: ${{ github.sha }}
+        GITHUB_SHA: ${{ inputs.cache-sha || github.sha }}
      run: |
        echo "::warning::Cache miss for n8n-docker-image-$BUILD_VARIANT-$GITHUB_SHA (SHA $GITHUB_SHA); falling back to rebuild via build-n8n-docker."

@@ -12,14 +12,18 @@ on:
      - 'packages/core/src/execution-engine/eval-mock-helpers.ts'
  workflow_dispatch:
    inputs:
+      pr:
+        description: 'PR number to re-run against its latest commit. Resolves the PR head at dispatch time and posts results back; takes precedence over `branch`. Leave empty to use `branch`.'
+        required: false
+        default: ''
      branch:
-        description: 'GitHub branch to test'
+        description: 'GitHub branch to test (ignored when `pr` is set)'
        required: false
        default: 'master'
      tier:
-        description: 'Test-case dataset to run (e.g. `pr`, `full`)'
+        description: 'Test-case dataset to run (e.g. `pr`, `full`). Empty => `pr` for a PR, `full` otherwise.'
        required: false
-        default: 'full'
+        default: ''
      sandbox-provider:
        description: 'Sandbox provider (n8n-sandbox or daytona)'
        required: false
@@ -34,25 +38,107 @@ on:
        default: ''

 concurrency:
-  group: instance-ai-evals-${{ github.ref }}
+  # Key on the PR number for dispatched re-runs (github.ref would collapse all
+  # dispatches from the default branch into one group and cross-cancel them).
+  group: instance-ai-evals-${{ inputs.pr || github.ref }}
  cancel-in-progress: true

 jobs:
-  run-evals:
-    name: Instance AI Workflow Evals
-    # Skip drafts; allow workflow_dispatch (no pull_request payload).
+  # Resolves the ref/SHA/PR to test. A `uses:` caller job can't run steps, so
+  # the `gh pr view` lookup for `-f pr=<n>` dispatches lives in its own job and
+  # feeds run-evals via outputs. Also the draft/repo gate: if this is skipped,
+  # run-evals (needs: resolve) is skipped too.
+  resolve:
+    name: Resolve eval target
    if: >-
      github.repository == 'n8n-io/n8n' &&
      (github.event_name != 'pull_request' || github.event.pull_request.draft == false)
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+    outputs:
+      branch: ${{ steps.resolve.outputs.branch }}
+      cache_sha: ${{ steps.resolve.outputs.cache_sha }}
+      revision_sha: ${{ steps.resolve.outputs.revision_sha }}
+      head_ref: ${{ steps.resolve.outputs.head_ref }}
+      pr_number: ${{ steps.resolve.outputs.pr_number }}
+      tier: ${{ steps.resolve.outputs.tier }}
+    steps:
+      - name: Resolve target ref
+        id: resolve
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          EVENT_NAME: ${{ github.event_name }}
+          GH_SHA: ${{ github.sha }}
+          INPUT_PR: ${{ inputs.pr }}
+          INPUT_BRANCH: ${{ inputs.branch }}
+          INPUT_TIER: ${{ inputs.tier }}
+          EVENT_PR_NUMBER: ${{ github.event.pull_request.number }}
+          EVENT_HEAD_REF: ${{ github.event.pull_request.head.ref }}
+        run: |
+          set -euo pipefail
+          if [ -n "$INPUT_PR" ]; then
+            # Dispatch against a PR number: test the latest push (auto PR-open
+            # runs go stale). Prefer the merge ref (refs/pull/N/merge) so we test
+            # the merged state like PR-open does. The merge ref updates async
+            # after a push - use it only when its parents include the current
+            # head (i.e. it reflects the latest push); otherwise fall back to the
+            # raw head. Either way a dispatched run can't restore the prepare-docker
+            # image cache (it's scoped to refs/pull/N/merge), so load-n8n-docker
+            # rebuilds from the checkout.
+            pr_json=$(gh api "repos/$REPO/pulls/$INPUT_PR")
+            head_sha=$(echo "$pr_json" | jq -r '.head.sha')
+            head_ref=$(echo "$pr_json" | jq -r '.head.ref')
+            merge_sha=$(echo "$pr_json" | jq -r '.merge_commit_sha // empty')
+            tested_sha="$head_sha"
+            if [ -n "$merge_sha" ] && \
+               gh api "repos/$REPO/commits/$merge_sha" --jq '.parents[].sha' 2>/dev/null \
+                 | grep -qx "$head_sha"; then
+              tested_sha="$merge_sha"
+              echo "PR #$INPUT_PR: testing merge ref $merge_sha (fresh; merged state, image rebuilt)"
+            else
+              echo "PR #$INPUT_PR: testing head $head_sha (merge ref stale/unavailable; image rebuilt)"
+            fi
+            branch="$tested_sha"; cache_sha="$tested_sha"; revision_sha="$tested_sha"
+            pr_number="$INPUT_PR"; tier="${INPUT_TIER:-pr}"
+          elif [ "$EVENT_NAME" = "pull_request" ]; then
+            # PR open/reopen/ready: unchanged - test the merge commit, which also
+            # matches the SHA-keyed prepare-docker image cache.
+            branch="$GH_SHA"; cache_sha="$GH_SHA"; revision_sha="$GH_SHA"
+            head_ref="$EVENT_HEAD_REF"; pr_number="$EVENT_PR_NUMBER"; tier="${INPUT_TIER:-pr}"
+          else
+            # Dispatch with an explicit branch (defaults to 'master'). Key the
+            # image cache and LangSmith tags on the ref under test, not
+            # github.sha: github.sha is the workflow's own ref (master when
+            # dispatched there), so keying on it would load master's prebuilt
+            # image even when testing another branch. A branch name misses the
+            # SHA-keyed cache, so load-n8n-docker rebuilds from the checkout.
+            branch="${INPUT_BRANCH:-$GH_SHA}"; cache_sha="$branch"; revision_sha="$branch"
+            head_ref="$branch"; pr_number=""; tier="${INPUT_TIER:-full}"
+          fi
+          {
+            echo "branch=$branch"
+            echo "cache_sha=$cache_sha"
+            echo "revision_sha=$revision_sha"
+            echo "head_ref=$head_ref"
+            echo "pr_number=$pr_number"
+            echo "tier=$tier"
+          } >> "$GITHUB_OUTPUT"
+
+  run-evals:
+    name: Instance AI Workflow Evals
+    needs: resolve
    uses: ./.github/workflows/test-evals-instance-ai.yml
    with:
-      # On PR events github.sha is the merge commit, so the eval tests the
-      # merged state (like e2e) and the checkout matches the key under which
-      # prepare-docker publishes the docker image cache.
-      branch: ${{ inputs.branch || github.sha }}
-      # PR events default to `pr`; dispatch defaults to `full`.
-      tier: ${{ inputs.tier || 'pr' }}
+      branch: ${{ needs.resolve.outputs.branch }}
+      tier: ${{ needs.resolve.outputs.tier }}
      sandbox-provider: ${{ inputs.sandbox-provider || 'n8n-sandbox' }}
      iterations: ${{ inputs.iterations }}
      experiment-name: ${{ inputs.experiment-name }}
+      pr-number: ${{ needs.resolve.outputs.pr_number }}
+      cache-sha: ${{ needs.resolve.outputs.cache_sha }}
+      revision-sha: ${{ needs.resolve.outputs.revision_sha }}
+      head-ref: ${{ needs.resolve.outputs.head_ref }}
    secrets: inherit
@@ -33,6 +33,26 @@ on:
        required: false
        type: string
        default: ''
+      pr-number:
+        description: 'PR number to post results back to. Empty = derive from the pull_request event.'
+        required: false
+        type: string
+        default: ''
+      cache-sha:
+        description: 'SHA for the docker image cache key. Empty = github.sha. Set to the tested commit on dispatch re-runs so the image matches the code under test.'
+        required: false
+        type: string
+        default: ''
+      revision-sha:
+        description: 'Commit SHA under test, for LangSmith revision tagging. Empty = github.sha.'
+        required: false
+        type: string
+        default: ''
+      head-ref:
+        description: 'Branch name under test, for LangSmith branch tagging. Empty = derive from context.'
+        required: false
+        type: string
+        default: ''
  workflow_dispatch:
    inputs:
      branch:
@@ -90,6 +110,8 @@ jobs:
      # via build-n8n-docker.
      - name: Load n8n Docker image
        uses: ./.github/actions/load-n8n-docker
+        with:
+          cache-sha: ${{ inputs.cache-sha }}

      - name: Start sandbox service
        if: ${{ inputs.sandbox-provider == 'n8n-sandbox' }}
@@ -224,12 +246,13 @@ jobs:
          LANGSMITH_TRACING: 'true'
          LANGSMITH_ENDPOINT: ${{ secrets.EVALS_LANGSMITH_ENDPOINT }}
          LANGSMITH_API_KEY: ${{ secrets.EVALS_LANGSMITH_API_KEY }}
-          LANGSMITH_REVISION_ID: ${{ github.sha }}
-          LANGSMITH_BRANCH: ${{ github.event.pull_request.head.ref || github.head_ref || github.ref_name }}
+          LANGSMITH_REVISION_ID: ${{ inputs.revision-sha || github.sha }}
+          LANGSMITH_BRANCH: ${{ inputs.head-ref || github.event.pull_request.head.ref || github.head_ref || github.ref_name }}
          FILTER: ${{ inputs.filter }}
          TIER: ${{ inputs.tier }}
          ITERATIONS: ${{ inputs.iterations }}
          EXPERIMENT_NAME: ${{ inputs.experiment-name }}
+          EVAL_PR_NUMBER: ${{ inputs.pr-number || github.event.pull_request.number }}
        run: |
          IFS=',' read -ra PORTS <<< "$LANE_PORTS"
          URLS=()
@@ -320,9 +343,11 @@ jobs:
          docker network rm n8n-eval-net 2>/dev/null || true

      - name: Post eval results to PR
-        if: ${{ always() && github.event.pull_request.number }}
+        if: ${{ always() && (inputs.pr-number || github.event.pull_request.number) }}
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ inputs.pr-number || github.event.pull_request.number }}
        run: |
          # The eval CLI writes the full PR comment as eval-pr-comment.md
          # (see comparison/format.ts:formatComparisonMarkdown). It includes
@@ -336,13 +361,13 @@ jobs:
          cp "$COMMENT_FILE" /tmp/eval-comment.md

          # Find and update existing eval comment, or create new one
-          COMMENT_ID=$(gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \
+          COMMENT_ID=$(gh api "repos/$REPO/issues/$PR_NUMBER/comments" \
            --jq '.[] | select(.body | startswith("### Instance AI Workflow Eval")) | .id' | tail -1)

          if [ -n "$COMMENT_ID" ]; then
-            gh api "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" -X PATCH -F body=@/tmp/eval-comment.md
+            gh api "repos/$REPO/issues/comments/${COMMENT_ID}" -X PATCH -F body=@/tmp/eval-comment.md
          else
-            gh pr comment "${{ github.event.pull_request.number }}" --body-file /tmp/eval-comment.md
+            gh pr comment "$PR_NUMBER" --body-file /tmp/eval-comment.md
          fi

      - name: Upload Results
@@ -259,6 +259,16 @@ When `LANGSMITH_API_KEY` is set, every eval run automatically compares its resul

 The CI PR-comment step uses `eval-pr-comment.md` as the entire comment body (no jq assembly in the workflow). The console output uses a separate aligned-text formatter — same data, no markdown noise in the terminal.

+### Re-running on a PR
+
+Evals auto-run when a PR is **opened / reopened / marked ready** (path-filtered), but **not on new pushes** — `synchronize` is intentionally off (full runs are expensive). To exercise the latest push, re-run against the PR head on demand:
+
+```bash
+gh workflow run ci-instance-ai-evals.yml -f pr=<number>
+```
+
+…or use the **Run workflow** button on the **CI: Instance AI Evals** workflow and set `pr=<number>`. A `resolve` job looks up the PR's current head at dispatch time (preferring the merge ref when it reflects the latest push, so it tests the merged state like a PR-open run; otherwise it uses the head), runs the eval against it, and posts results back to the PR. A dispatched run rebuilds the docker image either way — the prebuilt image cache is scoped to `refs/pull/<n>/merge`, which a dispatch can't restore. GitHub's built-in "Re-run jobs" instead replays the original PR-open commit, so use the dispatch above. Each eval PR comment also embeds this `gh workflow run -f pr=<n>` line.
+
 ### Refreshing the baseline

 There is no auto-refresh — refresh explicitly when you want a new reference point, ideally with high N for low noise:
@@ -192,6 +192,32 @@ describe('formatComparisonMarkdown', () => {
 		expect(md).not.toContain('abc12345');
 	});

+	it('renders a self-seeded re-run command and button when a rerun hint is given', () => {
+		const md = formatComparisonMarkdown(
+			evalFixture,
+			{ kind: 'no_baseline' },
+			{
+				rerun: {
+					prNumber: '4242',
+					dispatchUrl: 'https://github.com/n8n-io/n8n/actions/workflows/ci-instance-ai-evals.yml',
+				},
+			},
+		);
+		expect(md).toContain('does not re-run on new commits');
+		expect(md).toContain('gh workflow run ci-instance-ai-evals.yml -f pr=4242');
+		expect(md).toContain(
+			'[Run workflow button](https://github.com/n8n-io/n8n/actions/workflows/ci-instance-ai-evals.yml)',
+		);
+		expect(md).toContain('**pr** = `4242`');
+	});
+
+	it('falls back to a generic re-run instruction when no rerun hint is given', () => {
+		const md = formatComparisonMarkdown(evalFixture, { kind: 'no_baseline' });
+		expect(md).toContain('does not re-run on new commits');
+		expect(md).toContain('dispatching the **CI: Instance AI Evals** workflow');
+		expect(md).not.toContain('gh workflow run');
+	});
+
 	it('renders the Workflow checks table when at least one run has outcomes', () => {
 		const withChecks = evaluation({
 			totalRuns: 2,
@@ -39,7 +39,11 @@ import {
 	type ScenarioCounts,
 } from '../comparison/compare';
 import { fetchBaselineBucket, findLatestBaseline } from '../comparison/fetch-baseline';
-import { formatComparisonMarkdown, formatComparisonTerminal } from '../comparison/format';
+import {
+	formatComparisonMarkdown,
+	formatComparisonTerminal,
+	type RerunHint,
+} from '../comparison/format';
 import { evaluateGate, isGatedTier, type GateResult } from '../comparison/gate';
 import { cleanupCredentials } from '../credentials/seeder';
 import { loadWorkflowTestCasesWithFiles } from '../data/workflows';
@@ -214,7 +218,7 @@ async function main(): Promise<void> {
 			outcome,
 			commitSha,
 			slugByTestCase,
-			ciRunUrl(),
+			ciRerunHint(),
 			gate,
 		);
 		console.log(`Results:    ${jsonPath}`);
@@ -1126,11 +1130,15 @@ function computePassRatePerIter(evaluation: MultiRunEvaluation): string {
 	return rates.join(' / ');
 }

-// GitHub Actions run URL (for the PR comment's re-run link); undefined outside CI.
-function ciRunUrl(): string | undefined {
-	const { GITHUB_SERVER_URL, GITHUB_REPOSITORY, GITHUB_RUN_ID } = process.env;
-	if (!GITHUB_SERVER_URL || !GITHUB_REPOSITORY || !GITHUB_RUN_ID) return undefined;
-	return `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}`;
+// Re-run hint for the PR comment: a self-seeded dispatch against the PR head.
+// Undefined outside CI or when not associated with a PR (EVAL_PR_NUMBER unset).
+function ciRerunHint(): RerunHint | undefined {
+	const { GITHUB_SERVER_URL, GITHUB_REPOSITORY, EVAL_PR_NUMBER } = process.env;
+	if (!GITHUB_SERVER_URL || !GITHUB_REPOSITORY || !EVAL_PR_NUMBER) return undefined;
+	return {
+		prNumber: EVAL_PR_NUMBER,
+		dispatchUrl: `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/workflows/ci-instance-ai-evals.yml`,
+	};
 }

 function writeEvalResults(
@@ -1141,7 +1149,7 @@ function writeEvalResults(
 	outcome: ComparisonOutcome | undefined,
 	commitSha: string | undefined,
 	slugByTestCase: Map<WorkflowTestCase, string> | undefined,
-	runUrl: string | undefined,
+	rerun: RerunHint | undefined,
 	gate: GateResult | undefined,
 ): { jsonPath: string; prCommentPath: string } {
 	const { totalRuns, testCases } = evaluation;
@@ -1228,7 +1236,7 @@ function writeEvalResults(
 	const prCommentPath = join(targetDir, 'eval-pr-comment.md');
 	writeFileSync(
 		prCommentPath,
-		formatComparisonMarkdown(evaluation, outcome, { commitSha, slugByTestCase, runUrl, gate }),
+		formatComparisonMarkdown(evaluation, outcome, { commitSha, slugByTestCase, rerun, gate }),
 	);

 	return { jsonPath, prCommentPath };
@@ -37,11 +37,20 @@ import type {
 } from '../types';
 import { caseDisplayPrompt } from '../utils/conversation-text';

+/** How to re-run this eval against a PR's latest commit (PR-open runs go stale
+ *  on new pushes); drives the comment's re-run instructions. */
+export interface RerunHint {
+	/** PR number to dispatch against (`-f pr=<n>`). */
+	prNumber: string;
+	/** GitHub Actions "Run workflow" page for the eval dispatch workflow. */
+	dispatchUrl: string;
+}
+
 interface FormatOptions {
 	/** Optional commit SHA for the terminal heading. Truncated to 8 chars. */
 	commitSha?: string;
-	/** GitHub Actions run URL; when set, the comment leads with a re-run link. */
-	runUrl?: string;
+	/** When set, the comment shows how to re-run against the PR's latest commit. */
+	rerun?: RerunHint;
 	/** Maps each test-case reference to its file slug. When provided, the
 	 *  per-scenario failure breakdown looks up failed runs by
 	 *  `${fileSlug}/${scenarioName}` — deterministic across collisions like
@@ -68,7 +77,7 @@ export function formatComparisonMarkdown(

 	lines.push(formatHeading());
 	lines.push('');
-	lines.push(renderRerunCallout(options.runUrl));
+	lines.push(renderRerunCallout(options.rerun));
 	lines.push('');
 	if (gate) {
 		lines.push(formatGateAlertMarkdown(gate));
@@ -269,14 +278,25 @@ function formatTerminalGateSummary(gate: GateResult): string[] {
 	return lines;
 }

-// Evals fire on PR open/ready, not on push — lead with a one-click re-run prompt.
-function renderRerunCallout(runUrl?: string): string {
-	const cta = runUrl
-		? `[▶ Re-run this eval](${runUrl}) (then **Re-run jobs**)`
-		: 'Re-run it from the **Checks** tab (**Re-run jobs**)';
+// Evals fire on PR open/ready, not on push. The re-run is a self-seeded
+// dispatch keyed by PR number — it resolves the PR head at run time, so it
+// always tests the latest push (unlike GitHub's "Re-run jobs", which replays
+// the stale PR-open commit).
+function renderRerunCallout(rerun?: RerunHint): string {
+	if (!rerun) {
+		return [
+			'> [!IMPORTANT]',
+			"> **This eval does not re-run on new commits.** Re-run it against the PR's latest commit by dispatching the **CI: Instance AI Evals** workflow with your PR number.",
+		].join('\n');
+	}
 	return [
 		'> [!IMPORTANT]',
-		`> **This eval does not re-run on new commits** — ${cta} when you're ready to merge.`,
+		'> **This eval does not re-run on new commits.** To test your latest push, re-run it against the PR head:',
+		'>',
+		'> ```bash',
+		`> gh workflow run ci-instance-ai-evals.yml -f pr=${rerun.prNumber}`,
+		'> ```',
+		`> …or use the [Run workflow button](${rerun.dispatchUrl}) and set **pr** = \`${rerun.prNumber}\`.`,
 	].join('\n');
 }