From ee46e64d50980002b6afd783c585a65626f17f0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Braulio=20Gonz=C3=A1lez=20Valido?= Date: Wed, 22 Jul 2026 09:18:41 +0100 Subject: [PATCH] chore(ai-builder): Typecheck the eval harness and pin its external contracts (no-changelog) (#34673) Co-authored-by: Claude Fable 5 --- packages/@n8n/instance-ai/eslint.config.mjs | 11 +- .../artifact-config-eval-handler.test.ts | 9 +- .../artifact-workflow-handler.test.ts | 2 +- .../__tests__/comparison-format.test.ts | 5 +- .../__tests__/comparison-gate.test.ts | 2 +- .../__tests__/conversation-seed.test.ts | 1 + .../__tests__/data-discovery.test.ts | 1 + .../__tests__/data-workflows-schema.test.ts | 6 +- .../__tests__/data-workflows.test.ts | 2 +- .../eval-results-dispatcher-contract.test.ts | 167 ++++++++++++++++++ .../evaluations/__tests__/fixtures.ts | 14 +- .../__tests__/partial-results-abort.test.ts | 1 + .../__tests__/runner-prebuilt.test.ts | 1 + .../__tests__/subagent-runner.test.ts | 2 +- .../evaluations/__tests__/summary.test.ts | 1 + .../__tests__/verification-artifact.test.ts | 1 + .../workflow-checks-applicability.test.ts | 1 + .../__tests__/workflow-discovery.test.ts | 1 + .../__tests__/write-partial-results.test.ts | 1 + .../checks/code-node-no-http-requests.test.ts | 2 + .../checks/error-routes-consistent.test.ts | 3 + .../google-sheets-rlc-default-mode.test.ts | 24 +-- ...p-generic-auth-type-matches-prompt.test.ts | 2 + .../memory-session-key-expression.test.ts | 1 + .../evaluations/cli/langtracer-push.ts | 9 +- .../evaluations/clients/n8n-client.ts | 2 + .../__tests__/graders-llm.test.ts | 12 +- .../evaluations/computer-use/graders/index.ts | 2 +- .../evaluations/computer-use/types.ts | 2 + .../__tests__/expected-tools-invoked.test.ts | 1 + .../instance-ai/evaluations/discovery/cli.ts | 4 +- .../evaluations/discovery/runner.ts | 16 +- .../harness/in-memory-event-bus.ts | 4 +- .../evaluations/harness/langsmith-seed.ts | 12 +- .../evaluations/harness/normalize-workflow.ts | 5 +- .../evaluations/harness/stub-services.ts | 13 ++ .../evaluations/langtracer/config.ts | 8 +- .../instance-ai/evaluations/tsconfig.json | 19 +- .../evaluations/utils/user-proxy/agent.ts | 4 +- packages/@n8n/instance-ai/package.json | 2 +- 40 files changed, 318 insertions(+), 58 deletions(-) create mode 100644 packages/@n8n/instance-ai/evaluations/__tests__/eval-results-dispatcher-contract.test.ts diff --git a/packages/@n8n/instance-ai/eslint.config.mjs b/packages/@n8n/instance-ai/eslint.config.mjs index e0557d2eb12..1d0558b9c5a 100644 --- a/packages/@n8n/instance-ai/eslint.config.mjs +++ b/packages/@n8n/instance-ai/eslint.config.mjs @@ -22,7 +22,16 @@ const restrictedLazyRuntimeImports = [ export default defineConfig( baseConfig, { - ignores: ['scripts/**/*.cjs', 'skills/**/*.mjs'], + ignores: [ + 'scripts/**/*.cjs', + 'skills/**/*.mjs', + // Local eval scratch output — never linted, never committed. + '.data/**', + 'evaluations/.data/**', + // Deep-imports ai-workflow-builder.ee's evaluations source, so it sits outside + // evaluations/tsconfig.json (see its exclude) and the eslint project service. + 'evaluations/cli/pairwise.ts', + ], }, { rules: { diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/artifact-config-eval-handler.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/artifact-config-eval-handler.test.ts index 71ef29fd55e..799a5843ceb 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/artifact-config-eval-handler.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/artifact-config-eval-handler.test.ts @@ -24,7 +24,14 @@ function googleSheetsConfig(workflowId: string): EvaluationConfigDto { id: 'metric-2', name: 'Correctness', type: 'llm_judge', - config: { preset: 'correctness' }, + config: { + preset: 'correctness', + provider: '@n8n/n8n-nodes-langchain.lmChatOpenAi', + credentialId: 'cred-judge', + model: 'gpt-4.1', + outputType: 'numeric', + inputs: { actualAnswer: '={{ $json.actual }}', expectedAnswer: '={{ $json.expected }}' }, + }, }, ], datasetSource: 'google_sheets', diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/artifact-workflow-handler.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/artifact-workflow-handler.test.ts index 17a7f2d834b..2880679335b 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/artifact-workflow-handler.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/artifact-workflow-handler.test.ts @@ -54,7 +54,7 @@ describe('workflowHandler', () => { const expectedIds = extractWorkflowIdsFromMessages(messages); expect(expectedIds.length).toBeGreaterThan(0); - const refs = workflowHandler.discover({ messages }); + const refs = workflowHandler.discover({ messages, artifactRefs: [] }); expect(refs.map((r) => r.id)).toEqual(expectedIds); for (const ref of refs) { expect(ref.type).toBe('workflow'); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/comparison-format.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/comparison-format.test.ts index 2d465d47d47..7cefb2cfccc 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/comparison-format.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/comparison-format.test.ts @@ -67,6 +67,7 @@ function evaluation( conversation: [{ role: 'user', text: tc.userText ?? 'Test workflow prompt' }], complexity: 'medium' as const, tags: [], + datasets: ['full'], executionScenarios: (tc.scenarios ?? []).map((sa) => ({ name: sa.name, description: '', @@ -76,7 +77,7 @@ function evaluation( } as WorkflowTestCase; const buildSuccessCount = tc.buildSuccessCount ?? totalRuns; const scenarios = (tc.scenarios ?? []).map((sa) => ({ - scenario: testCase.executionScenarios.find((sc) => sc.name === sa.name)!, + scenario: testCase.executionScenarios!.find((sc) => sc.name === sa.name)!, evaluatedCount: sa.passes.length, passCount: sa.passCount, passRate: totalRuns > 0 ? sa.passCount / totalRuns : 0, @@ -84,7 +85,7 @@ function evaluation( passHatK: new Array(totalRuns).fill(sa.passCount === totalRuns ? 1 : 0) as number[], runs: sa.passes.map( (passed): ExecutionScenarioResult => ({ - scenario: testCase.executionScenarios.find((sc) => sc.name === sa.name)!, + scenario: testCase.executionScenarios!.find((sc) => sc.name === sa.name)!, success: passed, score: passed ? 1 : 0, reasoning: sa.reasoning ?? '', diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/comparison-gate.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/comparison-gate.test.ts index 424422dd2e0..772f5c5a287 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/comparison-gate.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/comparison-gate.test.ts @@ -42,7 +42,7 @@ function makeEval(totalRuns: number, cases: CaseSpec[]) { const scenarioAggs = (c.scenarios ?? []).map((sa) => { const evaluated = sa.passes.filter((p) => p !== 'incomplete'); const passCount = evaluated.filter((p) => p).length; - const scenario = testCase.executionScenarios.find((x) => x.name === sa.name)!; + const scenario = testCase.executionScenarios!.find((x) => x.name === sa.name)!; return { scenario, evaluatedCount: evaluated.length, diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/conversation-seed.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/conversation-seed.test.ts index 486eed6a846..8550b0e2ab4 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/conversation-seed.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/conversation-seed.test.ts @@ -42,6 +42,7 @@ function makeSeed(): ConversationSeed { }, ], workflows: [{ id: WF_ID, name: 'Daily digest', nodes: [], connections: {} }], + dataTables: [], }; } diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/data-discovery.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/data-discovery.test.ts index 7d1fdb487cb..8e139fb98f3 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/data-discovery.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/data-discovery.test.ts @@ -51,6 +51,7 @@ describe('loadDiscoveryTestCasesWithFiles', () => { workflowIds: [], executionIds: [], dataTableIds: [], + artifactRefs: [], finalText: '', toolCalls: [], agentActivities: [], diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts index 3692e482d7a..3f6446c51a6 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts @@ -37,7 +37,7 @@ describe('EvalTestCaseSchema', () => { it('accepts a minimal valid fixture', () => { const parsed = EvalTestCaseSchema.parse(validFixture()); expect(parsed.executionScenarios).toHaveLength(1); - expect(parsed.conversation[0].role).toBe('user'); + expect(parsed.conversation![0].role).toBe('user'); }); it('rejects an empty conversation', () => { @@ -49,7 +49,7 @@ describe('EvalTestCaseSchema', () => { ...validFixture(), conversation: [{ role: 'user', text: ['line 1', 'line 2'] }], }); - expect(parsed.conversation[0].text).toBe('line 1\nline 2'); + expect(parsed.conversation![0].text).toBe('line 1\nline 2'); }); it('rejects 0 execution scenarios AND 0 expectations (a case must assert something)', () => { @@ -251,7 +251,7 @@ describe('EvalTestCaseSchema', () => { requires: 'mock-server', } as (typeof fixture.executionScenarios)[number]; const parsed = EvalTestCaseSchema.parse(fixture); - expect(parsed.executionScenarios[0].requires).toBe('mock-server'); + expect(parsed.executionScenarios![0].requires).toBe('mock-server'); }); }); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows.test.ts index 7ec4e3971aa..4a5bd1dd700 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows.test.ts @@ -244,7 +244,7 @@ describe('loadWorkflowTestCasesWithFiles', () => { }); mockedReadFile.mockImplementation((p) => { const filename = String(p); - const parsed = jsonParse(STUB_TEST_CASE); + const parsed = jsonParse>(STUB_TEST_CASE); if (filename.includes('agent-intent')) { return JSON.stringify({ ...parsed, diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/eval-results-dispatcher-contract.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/eval-results-dispatcher-contract.test.ts new file mode 100644 index 00000000000..cf58df6f572 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/__tests__/eval-results-dispatcher-contract.test.ts @@ -0,0 +1,167 @@ +import type { InstanceAiEvalExecutionResult } from '@n8n/api-types'; +import { mkdtempSync, readFileSync } from 'fs'; +import { jsonParse } from 'n8n-workflow'; +import { tmpdir } from 'os'; +import { join } from 'path'; + +import type { CheckOutcome } from '../binaryChecks/types'; +import { aggregateResults } from '../cli/aggregator'; +import { writeEvalResults } from '../cli/index'; +import type { ExecutionScenario, WorkflowTestCase, WorkflowTestCaseResult } from '../types'; + +// Pins the `eval-results.json` fields the lang-tracer dispatcher ingests +// (lang-tracer-dispatcher `src/lib/runner.ts`): it spawns this CLI per case in +// direct (no-LangSmith) mode, reads the file, and projects these fields into +// LangTracer run state. Renaming or dropping any of them breaks LangTracer +// ingestion silently — the dispatcher tolerates absent fields by design. + +const scenario: ExecutionScenario = { + name: 'happy-path', + description: 'baseline', + dataSetup: 'plain', + successCriteria: 'digest arrives', +}; + +const testCase: WorkflowTestCase = { + conversation: [{ role: 'user', text: 'send me a daily digest' }], + complexity: 'simple', + tags: [], + datasets: ['full'], + executionScenarios: [scenario], + outcomeExpectations: ['sends a digest'], +}; + +const passingCheck: CheckOutcome = { + name: 'no-unreachable-nodes', + description: 'all nodes reachable', + kind: 'deterministic', + dimension: 'structure', + status: 'pass', +}; + +function iteration1(): WorkflowTestCaseResult { + return { + testCase, + workflowBuildSuccess: true, + workflowChecks: [passingCheck], + buildExpectationResults: [ + { expectation: 'sends a digest', pass: true, reason: 'digest node present' }, + ], + executionScenarioResults: [{ scenario, success: true, score: 1, reasoning: 'works' }], + }; +} + +function iteration2(): WorkflowTestCaseResult { + return { + testCase, + workflowBuildSuccess: true, + buildExpectationResults: [ + { expectation: 'sends a digest', pass: false, reason: 'digest node missing' }, + ], + executionScenarioResults: [ + { + scenario, + success: false, + score: 0, + reasoning: 'no digest was produced', + failureCategory: 'mock_issue', + rootCause: 'mock returned an empty page', + evalResult: { errors: ['HTTP 500 from the mocked API'] } as InstanceAiEvalExecutionResult, + }, + ], + }; +} + +interface DispatcherView { + experimentName?: string; + testCases: Array<{ + buildSuccessCount: number; + totalRuns: number; + workflowChecksPerRun: Array | null>; + buildExpectations: Array<{ + expectation: string; + passCount: number; + evaluatedCount: number; + }>; + buildExpectationResultsPerRun: Array | null>; + scenarios: Array<{ + name: string; + passCount: number; + totalRuns: number; + runs: Array<{ + passed: boolean; + score: number; + reasoning: string; + failureCategory?: string; + rootCause?: string; + execErrors: string[]; + }>; + }>; + }>; +} + +function writeAndRead(): DispatcherView { + const evaluation = aggregateResults([[iteration1()], [iteration2()]], 2); + const dir = mkdtempSync(join(tmpdir(), 'eval-results-contract-')); + const { jsonPath } = writeEvalResults( + evaluation, + 1234, + dir, + 'exp-dispatcher-contract', + undefined, + undefined, + new Map([[testCase, 'daily-digest']]), + undefined, + undefined, + ); + return jsonParse(readFileSync(jsonPath, 'utf8')); +} + +describe('eval-results.json — dispatcher contract', () => { + it('serializes every field the dispatcher projects into run state', () => { + const report = writeAndRead(); + + expect(report.experimentName).toBe('exp-dispatcher-contract'); + expect(report.testCases).toHaveLength(1); + + const tc = report.testCases[0]; + expect(tc.buildSuccessCount).toBe(2); + expect(tc.totalRuns).toBe(2); + + // Per-iteration build signals. Checks serialize as a name→status map (an + // iteration without checks serializes as null, not as a hole). + expect(tc.workflowChecksPerRun).toEqual([{ 'no-unreachable-nodes': 'pass' }, null]); + expect(tc.buildExpectations).toHaveLength(1); + expect(tc.buildExpectations[0]).toMatchObject({ + expectation: 'sends a digest', + passCount: 1, + evaluatedCount: 2, + }); + expect(tc.buildExpectationResultsPerRun).toEqual([ + [{ expectation: 'sends a digest', pass: true, reason: 'digest node present' }], + [{ expectation: 'sends a digest', pass: false, reason: 'digest node missing' }], + ]); + + // Scenario blocks serialize under the flat `scenarios` key with a flat + // `name` — the shape the dispatcher's fallback reader consumes today. + expect(tc.scenarios).toHaveLength(1); + const sc = tc.scenarios[0]; + expect(sc.name).toBe('happy-path'); + expect(sc.passCount).toBe(1); + expect(sc.totalRuns).toBe(2); + expect(sc.runs).toHaveLength(2); + expect(sc.runs[0]).toMatchObject({ passed: true, score: 1, reasoning: 'works' }); + expect(sc.runs[1]).toMatchObject({ + passed: false, + score: 0, + reasoning: 'no digest was produced', + failureCategory: 'mock_issue', + rootCause: 'mock returned an empty page', + execErrors: ['HTTP 500 from the mocked API'], + }); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/fixtures.ts b/packages/@n8n/instance-ai/evaluations/__tests__/fixtures.ts index bf41ebdb1ad..223922fa0ec 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/fixtures.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/fixtures.ts @@ -56,7 +56,19 @@ export function dataTableConfig(workflowId: string, dataTableId: string): Evalua startNodeName: 'Start', endNodeName: 'End', metrics: [ - { id: 'metric-1', name: 'Correctness', type: 'llm_judge', config: { preset: 'correctness' } }, + { + id: 'metric-1', + name: 'Correctness', + type: 'llm_judge', + config: { + preset: 'correctness', + provider: '@n8n/n8n-nodes-langchain.lmChatOpenAi', + credentialId: 'cred-judge', + model: 'gpt-4.1', + outputType: 'numeric', + inputs: { actualAnswer: '={{ $json.actual }}', expectedAnswer: '={{ $json.expected }}' }, + }, + }, ], datasetSource: 'data_table', datasetRef: { dataTableId }, diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/partial-results-abort.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/partial-results-abort.test.ts index 3c9e1cf8896..8e7cef1f6c9 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/partial-results-abort.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/partial-results-abort.test.ts @@ -45,6 +45,7 @@ function makeTestCase(scenarios: ExecutionScenario[]): WorkflowTestCase { conversation: [{ role: 'user', text: 'build me something' }], complexity: 'complex', tags: ['test'], + datasets: ['full'], executionScenarios: scenarios, }; } diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/runner-prebuilt.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/runner-prebuilt.test.ts index 2147690a2c7..bbed4357656 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/runner-prebuilt.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/runner-prebuilt.test.ts @@ -53,6 +53,7 @@ function makeTestCase(): WorkflowTestCase { conversation: [{ role: 'user', text: 'build me something' }], complexity: 'simple', tags: ['test'], + datasets: ['full'], executionScenarios: [], }; } diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/subagent-runner.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/subagent-runner.test.ts index 028a4c60dad..9cb9987c274 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/subagent-runner.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/subagent-runner.test.ts @@ -34,7 +34,7 @@ function makeClient(): N8nClient { describe('runWorkflowBuildEval', () => { beforeEach(() => { vi.clearAllMocks(); - mockedCleanupBuild.mockResolvedValue(undefined); + mockedCleanupBuild.mockResolvedValue(true); mockedRunBinaryChecks.mockResolvedValue({ feedback: [ { diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/summary.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/summary.test.ts index d2def4e6c10..0d5855f53a5 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/summary.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/summary.test.ts @@ -136,6 +136,7 @@ describe('summary — scenario-less cases (no workflow)', () => { buildSuccessCount: 0, executionScenarios: [], buildExpectations: [], + status: 'verified', }; // Only the first run judged an expectation. expect(getCheckedRunCount(tc)).toBe(1); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/verification-artifact.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/verification-artifact.test.ts index f2d5b70e8f7..fae8fbae73a 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/verification-artifact.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/verification-artifact.test.ts @@ -602,6 +602,7 @@ describe('buildVerificationArtifact', () => { interceptedRequests: Array.from({ length: 30 }, (_, i) => ({ method: 'GET', url: `https://api.example.com/page/${i}`, + nodeType: 'n8n-nodes-base.httpRequest', mockResponse: { page: i }, })), iterationCount: 1, diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/workflow-checks-applicability.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/workflow-checks-applicability.test.ts index 887ad3b7c85..3ba952d2bd7 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/workflow-checks-applicability.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/workflow-checks-applicability.test.ts @@ -6,6 +6,7 @@ function workflowWebhookOnly(): WorkflowResponse { id: 'wf-1', name: 'Webhook only', active: false, + versionId: 'test-version', nodes: [ { name: 'Webhook', type: 'n8n-nodes-base.webhook', parameters: {} }, { diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/workflow-discovery.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/workflow-discovery.test.ts index fe4fd94b944..950883e8755 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/workflow-discovery.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/workflow-discovery.test.ts @@ -9,6 +9,7 @@ const emptyEventOutcome: EventOutcome = { workflowIds: [], executionIds: [], dataTableIds: [], + artifactRefs: [], finalText: '', toolCalls: [], agentActivities: [], diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/write-partial-results.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/write-partial-results.test.ts index 9cafa2b1521..9a454a70359 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/write-partial-results.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/write-partial-results.test.ts @@ -67,6 +67,7 @@ function makeTestCase(scenarios: ExecutionScenario[]): WorkflowTestCase { conversation: [{ role: 'user', text: 'build me something' }], complexity: 'complex', tags: ['test'], + datasets: ['full'], executionScenarios: scenarios, }; } diff --git a/packages/@n8n/instance-ai/evaluations/binaryChecks/checks/code-node-no-http-requests.test.ts b/packages/@n8n/instance-ai/evaluations/binaryChecks/checks/code-node-no-http-requests.test.ts index faaa1dfd189..3ab013b767c 100644 --- a/packages/@n8n/instance-ai/evaluations/binaryChecks/checks/code-node-no-http-requests.test.ts +++ b/packages/@n8n/instance-ai/evaluations/binaryChecks/checks/code-node-no-http-requests.test.ts @@ -6,6 +6,7 @@ function workflowWithCodeNode(parameters: Record): WorkflowResp id: 'wf-1', name: 'Code HTTP test', active: false, + versionId: 'test-version', nodes: [ { name: 'Manual Trigger', @@ -152,6 +153,7 @@ describe('codeNodeNoHttpRequests', () => { id: 'wf-2', name: 'No code', active: false, + versionId: 'test-version', nodes: [ { name: 'Manual Trigger', diff --git a/packages/@n8n/instance-ai/evaluations/binaryChecks/checks/error-routes-consistent.test.ts b/packages/@n8n/instance-ai/evaluations/binaryChecks/checks/error-routes-consistent.test.ts index 0b2f5921a35..6febcb104a6 100644 --- a/packages/@n8n/instance-ai/evaluations/binaryChecks/checks/error-routes-consistent.test.ts +++ b/packages/@n8n/instance-ai/evaluations/binaryChecks/checks/error-routes-consistent.test.ts @@ -10,6 +10,7 @@ function reminderWorkflow(overrides: { id: 'wf-1', name: 'Reminder sender', active: false, + versionId: 'test-version', nodes: [ { name: 'Daily', type: 'n8n-nodes-base.scheduleTrigger', parameters: {} }, { @@ -73,6 +74,7 @@ describe('errorRoutesConsistent', () => { id: 'wf-2', name: 'Branching', active: false, + versionId: 'test-version', nodes: [ { name: 'Start', type: 'n8n-nodes-base.manualTrigger', parameters: {} }, { name: 'IF', type: 'n8n-nodes-base.if', parameters: {} }, @@ -97,6 +99,7 @@ describe('errorRoutesConsistent', () => { id: 'wf-3', name: 'Empty', active: false, + versionId: 'test-version', nodes: [{ name: 'Start', type: 'n8n-nodes-base.manualTrigger', parameters: {} }], connections: {}, }; diff --git a/packages/@n8n/instance-ai/evaluations/binaryChecks/checks/google-sheets-rlc-default-mode.test.ts b/packages/@n8n/instance-ai/evaluations/binaryChecks/checks/google-sheets-rlc-default-mode.test.ts index 99dd16b680b..50543ea1e23 100644 --- a/packages/@n8n/instance-ai/evaluations/binaryChecks/checks/google-sheets-rlc-default-mode.test.ts +++ b/packages/@n8n/instance-ai/evaluations/binaryChecks/checks/google-sheets-rlc-default-mode.test.ts @@ -23,8 +23,8 @@ function sheetWorkflow(documentId: unknown, sheetName?: unknown): WorkflowRespon } describe('googleSheetsRlcDefaultMode', () => { - it('fails when documentId defaults to an empty By ID field (INS-631 shape)', () => { - const result = googleSheetsRlcDefaultMode.run( + it('fails when documentId defaults to an empty By ID field (INS-631 shape)', async () => { + const result = await googleSheetsRlcDefaultMode.run( sheetWorkflow({ __rl: true, mode: 'id', @@ -38,8 +38,8 @@ describe('googleSheetsRlcDefaultMode', () => { expect(result.comment).toContain('Append Row.documentId'); }); - it('fails when documentId is By ID mode with only a placeholder value', () => { - const result = googleSheetsRlcDefaultMode.run( + it('fails when documentId is By ID mode with only a placeholder value', async () => { + const result = await googleSheetsRlcDefaultMode.run( sheetWorkflow({ __rl: true, mode: 'id', @@ -51,8 +51,8 @@ describe('googleSheetsRlcDefaultMode', () => { expect(result.pass).toBe(false); }); - it('fails when documentId is By ID mode with a fabricated ID absent from the prompt (real-build shape)', () => { - const result = googleSheetsRlcDefaultMode.run( + it('fails when documentId is By ID mode with a fabricated ID absent from the prompt (real-build shape)', async () => { + const result = await googleSheetsRlcDefaultMode.run( sheetWorkflow({ __rl: true, mode: 'id', @@ -66,8 +66,8 @@ describe('googleSheetsRlcDefaultMode', () => { expect(result.comment).toContain('Append Row.documentId'); }); - it('passes when documentId uses the From list picker mode', () => { - const result = googleSheetsRlcDefaultMode.run( + it('passes when documentId uses the From list picker mode', async () => { + const result = await googleSheetsRlcDefaultMode.run( sheetWorkflow( { __rl: true, mode: 'list', value: '', cachedResultName: 'SmartAssist Bookings' }, { __rl: true, mode: 'list', value: 0, cachedResultName: 'Sheet1' }, @@ -78,8 +78,8 @@ describe('googleSheetsRlcDefaultMode', () => { expect(result.pass).toBe(true); }); - it('passes when the user supplied a concrete spreadsheet ID', () => { - const result = googleSheetsRlcDefaultMode.run( + it('passes when the user supplied a concrete spreadsheet ID', async () => { + const result = await googleSheetsRlcDefaultMode.run( sheetWorkflow({ __rl: true, mode: 'id', @@ -91,8 +91,8 @@ describe('googleSheetsRlcDefaultMode', () => { expect(result.pass).toBe(true); }); - it('is not applicable when there are no Google Sheets nodes', () => { - const result = googleSheetsRlcDefaultMode.run( + it('is not applicable when there are no Google Sheets nodes', async () => { + const result = await googleSheetsRlcDefaultMode.run( { id: 'wf', name: 'x', diff --git a/packages/@n8n/instance-ai/evaluations/binaryChecks/checks/http-generic-auth-type-matches-prompt.test.ts b/packages/@n8n/instance-ai/evaluations/binaryChecks/checks/http-generic-auth-type-matches-prompt.test.ts index 8bc9f999fbd..b6fc8fafbb8 100644 --- a/packages/@n8n/instance-ai/evaluations/binaryChecks/checks/http-generic-auth-type-matches-prompt.test.ts +++ b/packages/@n8n/instance-ai/evaluations/binaryChecks/checks/http-generic-auth-type-matches-prompt.test.ts @@ -6,6 +6,7 @@ function workflowWithHttpRequest(parameters: Record): WorkflowR id: 'wf-1', name: 'HTTP auth test', active: false, + versionId: 'test-version', nodes: [ { name: 'Manual Trigger', @@ -112,6 +113,7 @@ describe('httpGenericAuthTypeMatchesPrompt', () => { id: 'wf-2', name: 'Slack only', active: false, + versionId: 'test-version', nodes: [ { name: 'Manual Trigger', diff --git a/packages/@n8n/instance-ai/evaluations/binaryChecks/checks/memory-session-key-expression.test.ts b/packages/@n8n/instance-ai/evaluations/binaryChecks/checks/memory-session-key-expression.test.ts index 843f5805782..ab4b8eb4e35 100644 --- a/packages/@n8n/instance-ai/evaluations/binaryChecks/checks/memory-session-key-expression.test.ts +++ b/packages/@n8n/instance-ai/evaluations/binaryChecks/checks/memory-session-key-expression.test.ts @@ -6,6 +6,7 @@ function createWorkflow(memoryParameters: Record): WorkflowResp id: 'workflow-1', name: 'Memory expression test', active: false, + versionId: 'test-version', nodes: [ { name: 'Telegram Trigger', diff --git a/packages/@n8n/instance-ai/evaluations/cli/langtracer-push.ts b/packages/@n8n/instance-ai/evaluations/cli/langtracer-push.ts index f94416343c4..243100afe95 100644 --- a/packages/@n8n/instance-ai/evaluations/cli/langtracer-push.ts +++ b/packages/@n8n/instance-ai/evaluations/cli/langtracer-push.ts @@ -171,9 +171,12 @@ async function main() { throw new Error(`suite "${args.suite}" not found. Available: ${known || '(none)'}.`); } - // Select disk cases: loader applies --filter/--exclude/--tier; then narrow to the - // exact slugs from positional args + --changed (if either was given). - const all = loadWorkflowTestCasesWithFiles(args.filter, args.exclude, args.tier); + // Select disk cases: loader applies --filter/--exclude, --tier narrows by the + // case's datasets (mirrors data/source.ts); then narrow to the exact slugs + // from positional args + --changed (if either was given). + const loaded = loadWorkflowTestCasesWithFiles(args.filter, args.exclude); + const tier = args.tier; + const all = tier ? loaded.filter((c) => c.testCase.datasets.includes(tier)) : loaded; const exactSlugs = new Set([...args.slugs, ...(args.changed ? gitChangedSlugs() : [])]); const selected = exactSlugs.size > 0 ? all.filter((c) => exactSlugs.has(c.fileSlug)) : all; diff --git a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts index 2efdb0642e1..fec10181e3b 100644 --- a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts +++ b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts @@ -71,9 +71,11 @@ export type GatewayStatus = z.infer; /** A node as returned by the n8n REST API — the fields eval code reads. */ export interface WorkflowNodeResponse { + id?: string; name: string; type: string; typeVersion?: number; + position?: [number, number]; parameters?: Record; executeOnce?: boolean; onError?: 'stopWorkflow' | 'continueRegularOutput' | 'continueErrorOutput'; diff --git a/packages/@n8n/instance-ai/evaluations/computer-use/__tests__/graders-llm.test.ts b/packages/@n8n/instance-ai/evaluations/computer-use/__tests__/graders-llm.test.ts index a93260362a1..77093aa3b67 100644 --- a/packages/@n8n/instance-ai/evaluations/computer-use/__tests__/graders-llm.test.ts +++ b/packages/@n8n/instance-ai/evaluations/computer-use/__tests__/graders-llm.test.ts @@ -45,7 +45,13 @@ function trace(overrides: Partial = {}): ScenarioTrace { confirmations: [], finalText: '', durationMs: 0, - tokens: { totalInput: 0, totalOutput: 0, byTool: [] }, + tokens: { + perCall: [], + totalArgsEst: 0, + totalResultsEst: 0, + largestResultEst: 0, + estimated: true, + }, threadId: 'thread-1', ...overrides, }; @@ -62,7 +68,7 @@ function call(toolName: string, args: Record = {}): CapturedToo const grader: LlmTaskCompletedGrader = { type: 'llm.taskCompleted' }; const userPrompt = 'Set up a Slack OAuth credential'; -const category: ScenarioCategory = 'credential-setup'; +const category: ScenarioCategory = 'browser'; // --------------------------------------------------------------------------- // Setup @@ -143,7 +149,7 @@ describe('llm.taskCompleted', () => { ); const sent = captureUserMessage(); - expect(sent).toContain('credential-setup'); + expect(sent).toContain('browser'); expect(sent).toContain('Set up a Slack OAuth credential'); expect(sent).toContain('Final response from the agent.'); }); diff --git a/packages/@n8n/instance-ai/evaluations/computer-use/graders/index.ts b/packages/@n8n/instance-ai/evaluations/computer-use/graders/index.ts index 1fba6010e9e..886c9fde14d 100644 --- a/packages/@n8n/instance-ai/evaluations/computer-use/graders/index.ts +++ b/packages/@n8n/instance-ai/evaluations/computer-use/graders/index.ts @@ -55,7 +55,7 @@ export async function applyGrader(grader: Grader, ctx: GradeContext): Promise ({ toolCallId: `call-${i}`, diff --git a/packages/@n8n/instance-ai/evaluations/discovery/cli.ts b/packages/@n8n/instance-ai/evaluations/discovery/cli.ts index 826c78999ce..2b3928f0833 100644 --- a/packages/@n8n/instance-ai/evaluations/discovery/cli.ts +++ b/packages/@n8n/instance-ai/evaluations/discovery/cli.ts @@ -27,7 +27,8 @@ interface CliArgs { trials: number; passThreshold: number; timeoutMs: number; - maxSteps: number; + /** Optional iteration cap; default uncapped — the wall-clock timeout bounds a run. */ + maxSteps?: number; modelId: string; concurrency: number; nodesJsonPath?: string; @@ -85,7 +86,6 @@ function parseArgs(argv: string[]): CliArgs { trials: 3, passThreshold: 2 / 3, timeoutMs: 60_000, - maxSteps: 5, modelId: DEFAULT_MODEL, concurrency: 3, failOnZeroPass: false, diff --git a/packages/@n8n/instance-ai/evaluations/discovery/runner.ts b/packages/@n8n/instance-ai/evaluations/discovery/runner.ts index a1f1d8d328b..399679db44f 100644 --- a/packages/@n8n/instance-ai/evaluations/discovery/runner.ts +++ b/packages/@n8n/instance-ai/evaluations/discovery/runner.ts @@ -10,11 +10,11 @@ // What's tested: the orchestrator's first dispatch decision. Tools are NOT // stubbed — when the orchestrator loads a runtime skill or reaches for a // Computer Use browser tool, the tool-call event fires before any downstream -// failure, so the discovery check still sees the dispatch intent. maxSteps caps -// the loop so an erroring tool can't drive API spend. +// failure, so the discovery check still sees the dispatch intent. The wall-clock +// timeout bounds the loop so an erroring tool can't drive API spend; scenarios +// or --max-steps can additionally opt into an iteration cap. // --------------------------------------------------------------------------- -import { Memory } from '@n8n/agents'; import type { InstanceAiEvent, TaskList } from '@n8n/api-types'; import { nanoid } from 'nanoid'; @@ -76,7 +76,10 @@ export async function runDiscoveryScenario( options: DiscoveryRunOptions, ): Promise { const started = Date.now(); - const maxSteps = options.scenario?.maxSteps ?? options.maxSteps ?? 5; + // Uncapped by default, matching live behavior: today's orchestrator legitimately + // explores past any small fixed cap (data-table-workflow needs >8 iterations), and + // the wall-clock timeout below bounds runaway runs. Scenarios/CLI opt in to a cap. + const maxSteps = options.scenario?.maxSteps ?? options.maxSteps; const timeoutMs = options.timeoutMs ?? 60_000; const nodesJsonPath = options.nodesJsonPath ?? defaultNodesJsonPath(); @@ -93,7 +96,6 @@ export async function runDiscoveryScenario( const context = applyInstanceState(services.context, options.scenario); const mcpManager = new McpClientManager(); - const memory = new Memory().build(); const threadId = 'discovery-thread-' + nanoid(6); const runId = 'discovery-run-' + nanoid(6); @@ -128,7 +130,7 @@ export async function runDiscoveryScenario( context, orchestrationContext, mcpManager, - memory, + // No memory: discovery measures stateless first-step tool dispatch. memoryConfig: {}, // Eager tool loading — discovery measures dispatch given the full toolset, // not whether the orchestrator can find a tool through search. @@ -138,7 +140,7 @@ export async function runDiscoveryScenario( const streamSource = normalizeStreamSource( await agent.stream(options.scenario.userMessage, { - maxSteps, + maxIterations: maxSteps, abortSignal: abortController.signal, providerOptions: { anthropic: { cacheControl: { type: 'ephemeral' as const } }, diff --git a/packages/@n8n/instance-ai/evaluations/harness/in-memory-event-bus.ts b/packages/@n8n/instance-ai/evaluations/harness/in-memory-event-bus.ts index 5f9cbc09973..dc46295dda8 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/in-memory-event-bus.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/in-memory-event-bus.ts @@ -29,7 +29,9 @@ export function createInMemoryEventBus(): InstanceAiEventBus { }; }, getEventsAfter(threadId, afterId) { - return (storeByThread.get(threadId) ?? []).filter((event) => event.id > afterId); + return (storeByThread.get(threadId) ?? []).filter( + (event) => event.id !== undefined && event.id > afterId, + ); }, getEventsForRun(threadId, runId) { return (storeByThread.get(threadId) ?? []) diff --git a/packages/@n8n/instance-ai/evaluations/harness/langsmith-seed.ts b/packages/@n8n/instance-ai/evaluations/harness/langsmith-seed.ts index 5492560a4da..fcec5823499 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/langsmith-seed.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/langsmith-seed.ts @@ -347,8 +347,10 @@ async function reconstructWithClient( ); } + // `?? NaN` keeps the SDK's optional start_time behavior-identical: an absent + // value still yields NaN comparisons, never a valid epoch-0 date. const byStartTime = (a: Run, b: Run) => - new Date(a.start_time).getTime() - new Date(b.start_time).getTime(); + new Date(a.start_time ?? NaN).getTime() - new Date(b.start_time ?? NaN).getTime(); const rootRuns = runs.filter((r) => r.run_type === 'chain' && !r.parent_run_id).sort(byStartTime); // Real agent tool calls only — the compiled-workflow bookkeeping event is // excluded BY NAME (it must never become a tool-call block in the rebuilt @@ -386,7 +388,7 @@ async function reconstructWithClient( ); } const liveTurnRun = userTurns[liveIndex]; - const boundaryMs = new Date(liveTurnRun.start_time).getTime(); + const boundaryMs = new Date(liveTurnRun.start_time ?? NaN).getTime(); const liveTurn = userMessageOf(liveTurnRun)!; const messages = buildSeedMessages(rootRuns, toolRuns, boundaryMs); @@ -432,7 +434,7 @@ function buildSeedMessages( const messages: Array> = []; for (const root of rootRuns) { - if (new Date(root.start_time).getTime() >= boundaryMs) break; + if (new Date(root.start_time ?? NaN).getTime() >= boundaryMs) break; const userText = userMessageOf(root); if (userText) { @@ -440,7 +442,7 @@ function buildSeedMessages( id: `${root.id}-user`, role: 'user', type: 'llm', - createdAt: new Date(root.start_time).toISOString(), + createdAt: new Date(root.start_time ?? NaN).toISOString(), content: [{ type: 'text', text: userText }], }); } @@ -478,7 +480,7 @@ function buildSeedMessages( role: 'assistant', type: 'llm', // +1ms so the assistant reply orders after its user turn. - createdAt: new Date(new Date(root.start_time).getTime() + 1).toISOString(), + createdAt: new Date(new Date(root.start_time ?? NaN).getTime() + 1).toISOString(), content, }); } diff --git a/packages/@n8n/instance-ai/evaluations/harness/normalize-workflow.ts b/packages/@n8n/instance-ai/evaluations/harness/normalize-workflow.ts index abd50fd9d9a..34cb1ec97e9 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/normalize-workflow.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/normalize-workflow.ts @@ -15,8 +15,11 @@ import type { WorkflowJSON } from '@n8n/workflow-sdk'; import { deepCopy } from 'n8n-workflow'; +import type { IWorkflowBase } from 'n8n-workflow'; -import type { SimpleWorkflow } from '../../../ai-workflow-builder.ee/src/types/workflow'; +/** Same computed type as ai-workflow-builder.ee's `SimpleWorkflow`, declared locally so + * the eval typecheck program doesn't pull the builder package's source tree in. */ +export type SimpleWorkflow = Pick; type NodeRaw = WorkflowJSON['nodes'][number]; diff --git a/packages/@n8n/instance-ai/evaluations/harness/stub-services.ts b/packages/@n8n/instance-ai/evaluations/harness/stub-services.ts index 8aaa3283998..b332f08956b 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/stub-services.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/stub-services.ts @@ -243,6 +243,18 @@ export async function createStubServices( returned: { from: 0, to: 0 }, }; }, + async getResolvedNodeParameters(_executionId: string, nodeName: string) { + return { + nodeName, + runIndex: 0, + itemIndex: 0, + parameters: null, + resolved: null, + failedExpressions: [], + emptyResolutions: [], + suppressed: 'parameter-values-disabled' as const, + }; + }, }; const dataTableService: InstanceAiDataTableService = { @@ -298,6 +310,7 @@ export async function createStubServices( const context: InstanceAiContext = { userId: options.userId ?? 'eval-user', + logger: { info() {}, warn() {}, error() {}, debug() {} }, workflowService, executionService, credentialService, diff --git a/packages/@n8n/instance-ai/evaluations/langtracer/config.ts b/packages/@n8n/instance-ai/evaluations/langtracer/config.ts index e1c00d0b08a..997a1258642 100644 --- a/packages/@n8n/instance-ai/evaluations/langtracer/config.ts +++ b/packages/@n8n/instance-ai/evaluations/langtracer/config.ts @@ -10,10 +10,10 @@ export function resolveLangTracerConfig(env: NodeJS.ProcessEnv = process.env): L const baseUrl = env.LANGTRACER_URL?.trim(); const apiKey = env.LANGTRACER_API_KEY?.trim(); - const missing: string[] = []; - if (!baseUrl) missing.push('LANGTRACER_URL'); - if (!apiKey) missing.push('LANGTRACER_API_KEY'); - if (missing.length > 0) { + if (!baseUrl || !apiKey) { + const missing: string[] = []; + if (!baseUrl) missing.push('LANGTRACER_URL'); + if (!apiKey) missing.push('LANGTRACER_API_KEY'); throw new Error( `--source langtracer needs ${missing.join(' and ')} in the environment (set them in .env.local). ` + 'LANGTRACER_URL is the lang-tracer base URL; LANGTRACER_API_KEY is an MCP bearer key (lt_…), ' + diff --git a/packages/@n8n/instance-ai/evaluations/tsconfig.json b/packages/@n8n/instance-ai/evaluations/tsconfig.json index dad9a2793b1..38cd9fc4785 100644 --- a/packages/@n8n/instance-ai/evaluations/tsconfig.json +++ b/packages/@n8n/instance-ai/evaluations/tsconfig.json @@ -1,14 +1,25 @@ { "extends": [ - "@n8n/typescript-config/tsconfig.common.json", - "@n8n/typescript-config/tsconfig.backend.json" + "@n8n/typescript-config/tsconfig.common.go.json", + "@n8n/typescript-config/tsconfig.backend.go.json" ], "compilerOptions": { "target": "es2023", "lib": ["es2023"], "emitDecoratorMetadata": true, "experimentalDecorators": true, - "types": ["node", "vitest/globals"] + "types": ["node", "vitest/globals"], + "noEmit": true, + // tsgo is fast enough to skip incremental state; avoids a stray tsbuildinfo in dist/. + "incremental": false, + // evaluations/ itself never uses `@/`; this maps it for the ../src files that + // enter this program through direct imports, mirroring the package root config. + "paths": { + "@/*": ["../src/*"] + } }, - "include": ["**/*.ts"] + "include": ["**/*.ts"], + // cli/pairwise.ts deep-imports ai-workflow-builder.ee's evaluations source (its own + // `@/` alias can't resolve here); it's a leaf entrypoint, still linted by eslint. + "exclude": ["dist/**", ".data/**", "computer-use/data/**", "cli/pairwise.ts"] } diff --git a/packages/@n8n/instance-ai/evaluations/utils/user-proxy/agent.ts b/packages/@n8n/instance-ai/evaluations/utils/user-proxy/agent.ts index 363b63fa379..bcf3882ad11 100644 --- a/packages/@n8n/instance-ai/evaluations/utils/user-proxy/agent.ts +++ b/packages/@n8n/instance-ai/evaluations/utils/user-proxy/agent.ts @@ -1,5 +1,5 @@ -import { SYSTEM_PROMPT, TOOL_DESCRIPTIONS } from './prompts'; -import { decisionSchema, type Decision } from './tools'; +import { SYSTEM_PROMPT } from './prompts'; +import { decisionSchema, TOOL_DESCRIPTIONS, type Decision } from './tools'; import { createEvalAgent } from '../../../src/utils/eval-agents'; import type { EvalLogger } from '../../harness/logger'; diff --git a/packages/@n8n/instance-ai/package.json b/packages/@n8n/instance-ai/package.json index 7e86dd24cfc..543185ff03a 100644 --- a/packages/@n8n/instance-ai/package.json +++ b/packages/@n8n/instance-ai/package.json @@ -3,7 +3,7 @@ "version": "1.17.0", "scripts": { "clean": "rimraf dist .turbo", - "typecheck": "tsc --noEmit", + "typecheck": "tsc --noEmit && tsc -p evaluations/tsconfig.json", "build": "tsc -p ./tsconfig.build.json && tsc-alias -p tsconfig.build.json", "build:unchecked": "tsc -p ./tsconfig.build.json --noCheck && tsc-alias -p tsconfig.build.json", "format": "biome format --write src",