fix(core): Fall back to the run's resolved model for AI Assistant verification LLM calls (#34627)

This commit is contained in:
Raúl Gómez Morales
2026-07-23 11:16:03 +00:00
committed by GitHub
parent 84b25bef6b
commit a8c90f84fe
19 changed files with 282 additions and 19 deletions
@@ -18,8 +18,17 @@ export async function reconcileStaleCredentialPlan(args: {
domainContext: NonNullable<OrchestrationContext['domainContext']>;
workflowTaskService: WorkflowTaskService;
logger: OrchestrationContext['logger'];
/** Host-resolved model used when no eval model API key is configured in the environment. */
fallbackModelConfig?: OrchestrationContext['modelId'];
}): Promise<WorkflowBuildOutcome> {
const { buildOutcome, workflowId, domainContext, workflowTaskService, logger } = args;
const {
buildOutcome,
workflowId,
domainContext,
workflowTaskService,
logger,
fallbackModelConfig,
} = args;
// Reconciliation can change something only when the outcome holds mocked
// credentials or an AI root simulated for missing model credentials.
const hasMockedCredentials = Object.keys(buildOutcome.mockedCredentialsByNode ?? {}).length > 0;
@@ -32,7 +41,12 @@ export async function reconcileStaleCredentialPlan(args: {
try {
const workflow = await domainContext.workflowService.getAsWorkflowJSON(workflowId);
const availableCredentials = await buildCredentialMap(domainContext.credentialService);
const patch = await reconcileSimulationPlan({ buildOutcome, workflow, availableCredentials });
const patch = await reconcileSimulationPlan({
buildOutcome,
workflow,
availableCredentials,
fallbackModelConfig,
});
if (!patch) return buildOutcome;
await workflowTaskService.updateBuildOutcome(buildOutcome.workItemId, patch);
@@ -136,6 +136,7 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
domainContext: target.domainContext,
workflowTaskService,
logger: context.logger,
fallbackModelConfig: context.modelId,
});
if (buildOutcome.nodeSimulationPlan === undefined) {
@@ -201,6 +201,34 @@ describe('classifyNodesForSimulation', () => {
expect(mockCreateEvalAgent).toHaveBeenCalledTimes(1);
});
it('forwards the fallback model config to the LLM call', async () => {
setupAgentMock(
JSON.stringify({
Search: { verdict: 'execute', reason: 'POST to a search endpoint', confidence: 'high' },
}),
);
const fallbackModelConfig = {
id: 'anthropic/claude-opus-4-8' as const,
url: 'https://proxy.example.com/anthropic/v1',
apiKey: 'proxy-token',
};
await classifyNodesForSimulation({
workflow: chainWorkflow([
trigger,
{
name: 'Search',
type: 'n8n-nodes-base.httpRequest',
parameters: { method: 'POST', url: 'https://api.example.com/search' },
},
]),
fallbackModelConfig,
});
expect(mockCreateEvalAgent).toHaveBeenCalledWith(
'verification-destructiveness-classifier',
expect.objectContaining({ fallbackModelConfig }),
);
});
it('classifies IO-free Code nodes as execute deterministically', async () => {
const verdicts = await classify([
trigger,
@@ -120,6 +120,42 @@ describe('generateSimulationFixtures', () => {
expect(result).toEqual({ A: [{}] });
});
it('warns on generation failure so the empty-fixture degrade is visible', async () => {
mockCreateEvalAgent.mockImplementation(() => {
throw new Error('Missing API key');
});
const warn = vi.fn();
const logger = { info: vi.fn(), warn, error: vi.fn(), debug: vi.fn() };
const result = await generateSimulationFixtures({
workflow: wf([{ name: 'A', type: 'n8n-nodes-base.slack' }]),
plan: [simulateVerdict('A')],
logger,
});
expect(result).toEqual({ A: [{}] });
expect(warn).toHaveBeenCalledWith(
expect.stringContaining('fixture generation failed'),
expect.objectContaining({ reason: 'generation_failed', nodeCount: 1 }),
);
});
it('forwards the fallback model config to the LLM call', async () => {
setupAgentMock(JSON.stringify({ A: [{ json: { ok: true } }] }));
const fallbackModelConfig = {
id: 'anthropic/claude-opus-4-8' as const,
url: 'https://proxy.example.com/anthropic/v1',
apiKey: 'proxy-token',
};
await generateSimulationFixtures({
workflow: wf([{ name: 'A', type: 'n8n-nodes-base.slack' }]),
plan: [simulateVerdict('A')],
fallbackModelConfig,
});
expect(mockCreateEvalAgent).toHaveBeenCalledWith(
'verification-simulation-fixtures',
expect.objectContaining({ fallbackModelConfig }),
);
});
it('strips markdown fences around the JSON output', async () => {
setupAgentMock('```json\n{"A":[{"json":{"ok":true}}]}\n```');
const result = await generateSimulationFixtures({
@@ -62,6 +62,34 @@ describe('planVerificationSimulation — simulated trigger verdicts', () => {
mockGenerateFixtures.mockResolvedValue({});
});
it('forwards the fallback model config to classification and fixture generation', async () => {
const fallbackModelConfig = {
id: 'anthropic/claude-opus-4-8' as const,
url: 'https://proxy.example.com/anthropic/v1',
apiKey: 'proxy-token',
};
mockClassify.mockResolvedValue([
{
nodeName: 'Send It',
verdict: 'simulate',
reason: 'Sends a message',
confidence: 'high',
source: 'deterministic',
},
]);
await planVerificationSimulation({
workflow: wf([{ name: 'Send It', type: 'n8n-nodes-base.slack' }]),
workflowId: 'wf-1',
fallbackModelConfig,
});
expect(mockClassify).toHaveBeenCalledWith(expect.objectContaining({ fallbackModelConfig }));
expect(mockGenerateFixtures).toHaveBeenCalledWith(
expect.objectContaining({ fallbackModelConfig }),
);
});
it('injects a deterministic simulate verdict for non-deterministic triggers', async () => {
mockClassify.mockResolvedValue([executeVerdict('Fetch Rows')]);
@@ -108,6 +108,7 @@ export function createApplyWorkflowCredentialsTool(context: OrchestrationContext
buildOutcome,
workflow: json,
availableCredentials,
fallbackModelConfig: context.modelId,
});
if (patch) {
await context.workflowTaskService.updateBuildOutcome(input.workItemId, {
@@ -761,6 +761,7 @@ export function createBuildWorkflowTool(context: InstanceAiContext) {
declaredOutputFixtures: compiled.declaredOutputFixtures,
workflowId: saved.id,
outputSchemaLookup: context.outputSchemaLookup,
fallbackModelConfig: context.modelId,
logger: context.logger,
});
const runId = buildContext?.runId ?? context.runId;
@@ -22,6 +22,7 @@ import type { IConnections } from 'n8n-workflow';
import { z } from 'zod';
import { isTriggerNodeType } from './workflow-json-utils';
import type { ModelConfig } from '../../types';
import { HAIKU_MODEL } from '../../utils/eval-agents';
import { generateValidatedJson } from '../../utils/generate-validated-json';
import type { NodeSimulationVerdict } from '../../workflow-loop/workflow-loop-state';
@@ -32,6 +33,8 @@ export interface ClassifyNodesForSimulationInput {
workflow: WorkflowJSON;
/** Node names whose credentials were mocked — always simulated. */
mockedNodeNames?: string[];
/** Host-resolved model used when no eval model API key is configured in the environment. */
fallbackModelConfig?: ModelConfig;
}
const STICKY_NOTE_TYPE = 'n8n-nodes-base.stickyNote';
@@ -354,6 +357,7 @@ function formatNodeBlock(node: WorkflowNode & { name: string }): string {
async function classifyAmbiguousNodes(
nodes: Array<WorkflowNode & { name: string }>,
fallbackModelConfig?: ModelConfig,
): Promise<NodeSimulationVerdict[]> {
const userText = [
'Classify the following n8n workflow nodes.',
@@ -368,6 +372,7 @@ async function classifyAmbiguousNodes(
instructions: SYSTEM_INSTRUCTIONS,
userText,
schema: LlmVerdictSchema,
fallbackModelConfig,
});
const parsed = result.ok ? result.data : undefined;
@@ -426,7 +431,7 @@ export async function classifyNodesForSimulation(
// retired, the plan is the only source of verification pin data, so a
// throw here would leave every node executing for real. Fail destructive.
try {
for (const verdict of await classifyAmbiguousNodes(ambiguous)) {
for (const verdict of await classifyAmbiguousNodes(ambiguous, input.fallbackModelConfig)) {
verdictByName.set(verdict.nodeName, verdict);
}
} catch {
@@ -32,6 +32,8 @@ import { getParentNodes, mapConnectionsByDestination, type IConnections } from '
import { z } from 'zod';
import { isTriggerNodeType } from './workflow-json-utils';
import type { Logger } from '../../logger';
import type { ModelConfig } from '../../types';
import { SONNET_MODEL } from '../../utils/eval-agents';
import { generateValidatedJson } from '../../utils/generate-validated-json';
import type { NodeSimulationVerdict } from '../../workflow-loop/workflow-loop-state';
@@ -53,6 +55,9 @@ export interface GenerateSimulationFixturesInput {
* structure instead of the model's guess at the service's response shape.
*/
outputSchemaLookup?: OutputSchemaLookup;
/** Host-resolved model used when no eval model API key is configured in the environment. */
fallbackModelConfig?: ModelConfig;
logger?: Logger;
}
// Loose on purpose: items may arrive `{json: {...}}`-wrapped, flat, or as an
@@ -196,8 +201,15 @@ export async function generateSimulationFixtures(
instructions: SYSTEM_INSTRUCTIONS,
userText,
schema: FixturesResponseSchema,
fallbackModelConfig: input.fallbackModelConfig,
});
if (!result.ok) return emptyFixtures(nodeNames);
if (!result.ok) {
input.logger?.warn('Simulation fixture generation failed; simulated nodes get empty items', {
reason: result.reason,
nodeCount: nodeNames.length,
});
return emptyFixtures(nodeNames);
}
// Shared normalization + envelope repair, matching the eval pin-data paths:
// wrap-or-passthrough items, then mechanically fix the two known LLM
@@ -22,6 +22,7 @@ import {
} from './generate-simulation-fixtures.service';
import { isMockableTriggerNodeType, isTriggerNodeType } from './workflow-json-utils';
import type { Logger } from '../../logger';
import type { ModelConfig } from '../../types';
import type { NodeSimulationVerdict } from '../../workflow-loop/workflow-loop-state';
export interface PlanVerificationSimulationInput {
@@ -37,6 +38,8 @@ export interface PlanVerificationSimulationInput {
workflowId: string;
/** Node output `__schema__` lookup used to shape generated fixtures. */
outputSchemaLookup?: OutputSchemaLookup;
/** Host-resolved model used when no eval model API key is configured in the environment. */
fallbackModelConfig?: ModelConfig;
logger?: Logger;
}
@@ -255,13 +258,18 @@ export async function planVerificationSimulation({
declaredOutputFixtures,
workflowId,
outputSchemaLookup,
fallbackModelConfig,
logger,
}: PlanVerificationSimulationInput): Promise<VerificationSimulationPlan> {
let nodeSimulationPlan: NodeSimulationVerdict[] | undefined;
let simulationFixtures: SimulationFixtures | undefined;
const declaredFixtures = nonEmptyDeclaredFixtures(declaredOutputFixtures);
try {
nodeSimulationPlan = await classifyNodesForSimulation({ workflow, mockedNodeNames });
nodeSimulationPlan = await classifyNodesForSimulation({
workflow,
mockedNodeNames,
fallbackModelConfig,
});
nodeSimulationPlan = withDeclaredOutputVerdicts(nodeSimulationPlan, declaredFixtures);
nodeSimulationPlan = withSimulatedTriggerVerdicts(nodeSimulationPlan, workflow);
nodeSimulationPlan = withSimulatedCredentiallessAiRootVerdicts(
@@ -280,6 +288,8 @@ export async function planVerificationSimulation({
workflow,
plan: planNeedingGeneratedFixtures,
outputSchemaLookup,
fallbackModelConfig,
logger,
})
: {};
const fixtures = { ...generatedFixtures, ...declaredFixtures };
@@ -20,6 +20,7 @@ import {
findCredentiallessAiRoots,
} from './plan-verification-simulation';
import type { CredentialMap } from './resolve-credentials';
import type { ModelConfig } from '../../types';
import type {
NodeSimulationVerdict,
WorkflowBuildOutcome,
@@ -75,8 +76,10 @@ export async function reconcileSimulationPlan(args: {
buildOutcome: WorkflowBuildOutcome;
workflow: WorkflowJSON;
availableCredentials: CredentialMap;
/** Host-resolved model used when no eval model API key is configured in the environment. */
fallbackModelConfig?: ModelConfig;
}): Promise<SimulationPlanPatch | undefined> {
const { buildOutcome, workflow, availableCredentials } = args;
const { buildOutcome, workflow, availableCredentials, fallbackModelConfig } = args;
const mockedEntries = Object.entries(buildOutcome.mockedCredentialsByNode ?? {});
const satisfiedNames = new Set(
@@ -131,6 +134,7 @@ export async function reconcileSimulationPlan(args: {
const freshVerdicts = await classifyNodesForSimulation({
workflow: scopedWorkflow,
mockedNodeNames: remainingMockedNames,
fallbackModelConfig,
});
freshVerdictByName = new Map(freshVerdicts.map((verdict) => [verdict.nodeName, verdict]));
}
+7
View File
@@ -963,6 +963,13 @@ export interface InstanceAiContext {
*/
tracing?: InstanceAiTraceContext;
projectId?: string;
/**
* Host-resolved model for the current run (proxy-managed on cloud). Domain
* tools pass it as the fallback for utility LLM calls (simulation fixtures,
* destructiveness classification), which otherwise resolve an eval model
* from environment API keys that proxy-managed deployments don't have.
*/
modelId?: ModelConfig;
workflowService: InstanceAiWorkflowService;
executionService: InstanceAiExecutionService;
credentialService: InstanceAiCredentialService;
@@ -81,4 +81,43 @@ describe('eval agent model config', () => {
reasoningEffort: 'high',
});
});
it('throws without env keys or a fallback model config', () => {
expect(() => createEvalAgent('test-agent', { instructions: 'Do the task.' })).toThrow(
/Missing API key/,
);
});
it('uses the fallback model config when no env API key is configured', () => {
const fallbackModelConfig = {
id: 'anthropic/claude-opus-4-8' as const,
url: 'https://proxy.example.com/anthropic/v1',
apiKey: 'proxy-token',
};
createEvalAgent('test-agent', {
instructions: 'Do the task.',
fallbackModelConfig,
});
expect(mockAgentInstances[0]?.model).toHaveBeenCalledWith(fallbackModelConfig);
expect(mockAgentInstances[0]?.thinking).toHaveBeenCalledWith('anthropic', {
mode: 'adaptive',
});
});
it('prefers env-based model resolution over the fallback', () => {
process.env.N8N_AI_ANTHROPIC_KEY = 'env-key';
createEvalAgent('test-agent', {
instructions: 'Do the task.',
fallbackModelConfig: { id: 'anthropic/claude-opus-4-8' as const, url: '', apiKey: 'jwt' },
});
expect(mockAgentInstances[0]?.model).toHaveBeenCalledWith({
id: 'anthropic/claude-sonnet-4-6',
apiKey: 'env-key',
url: undefined,
});
});
});
@@ -79,4 +79,23 @@ describe('generateValidatedJson', () => {
});
expect(await generate()).toEqual({ ok: false, reason: 'generation_failed' });
});
it('forwards the fallback model config to agent creation', async () => {
setupAgentMock('{"ok": true}');
const fallbackModelConfig = {
id: 'anthropic/claude-opus-4-8' as const,
url: 'https://proxy.example.com/anthropic/v1',
apiKey: 'proxy-token',
};
await generateValidatedJson('test-agent', {
instructions: 'instructions',
userText: 'do it',
schema,
fallbackModelConfig,
});
expect(mockCreateEvalAgent).toHaveBeenCalledWith(
'test-agent',
expect.objectContaining({ fallbackModelConfig }),
);
});
});
@@ -1,6 +1,6 @@
/** Shared agent factory + helpers for eval LLM calls (hint generation, mock responses, pin data). */
import { Agent, Tool, type GenerateResult } from '@n8n/agents';
import { Agent, Tool, type GenerateResult, type ModelConfig } from '@n8n/agents';
import { applyAgentThinking } from '../agent/apply-agent-thinking';
@@ -100,20 +100,34 @@ const CACHE_PROVIDER_OPTS = {
providerOptions: EPHEMERAL_CACHE,
};
/**
* Env-based tiered model when configured, otherwise the caller's fallback.
* Deployments where the model is managed outside the environment (e.g. the
* cloud AI service proxy) have no eval API key, so without a fallback every
* in-product eval call would fail before reaching the LLM.
*/
function resolveAgentModel(model?: string, fallbackModelConfig?: ModelConfig): ModelConfig {
try {
const { modelId, apiKey, url } = resolveEvalModelConfig(model);
return { id: modelId, apiKey, url };
} catch (error) {
if (fallbackModelConfig) return fallbackModelConfig;
throw error;
}
}
export function createEvalAgent(
name: string,
options: {
model?: string;
instructions: string;
cache?: boolean;
/** Host-resolved model used when no eval model API key is configured in the environment. */
fallbackModelConfig?: ModelConfig;
},
): Agent {
const { modelId, apiKey, url } = resolveEvalModelConfig(options.model);
const agent = new Agent(name).model({
id: modelId,
apiKey,
url,
});
const model = resolveAgentModel(options.model, options.fallbackModelConfig);
const agent = new Agent(name).model(model);
if (options.cache) {
agent.instructions(options.instructions, CACHE_PROVIDER_OPTS);
@@ -121,7 +135,7 @@ export function createEvalAgent(
agent.instructions(options.instructions);
}
applyAgentThinking(agent, modelId);
applyAgentThinking(agent, model);
return agent;
}
@@ -8,6 +8,7 @@
* helper's own tests mock `eval-agents`.
*/
import type { ModelConfig } from '@n8n/agents';
import type { z } from 'zod';
import { createEvalAgent, extractText } from './eval-agents';
@@ -25,6 +26,8 @@ export interface GenerateValidatedJsonOptions<T> {
instructions: string;
userText: string;
schema: z.ZodType<T>;
/** Host-resolved model used when no eval model API key is configured in the environment. */
fallbackModelConfig?: ModelConfig;
}
function stripMarkdownFences(text: string): string {
@@ -42,6 +45,7 @@ export async function generateValidatedJson<T>(
const llm = createEvalAgent(agentName, {
model: options.model,
instructions: options.instructions,
fallbackModelConfig: options.fallbackModelConfig,
});
const result = await llm.generate([
{ role: 'user' as const, content: [{ type: 'text' as const, text: options.userText }] },
@@ -3767,3 +3767,36 @@ describe('createContext — builder delegate telemetry', () => {
expect(delegate.listAgents).toHaveBeenCalledTimes(1);
});
});
// ---------------------------------------------------------------------------
// createContext — run model wiring
// ---------------------------------------------------------------------------
describe('createContext — run model wiring', () => {
const mockUser = { id: 'user-1', role: { slug: 'global:member' } } as unknown as User;
// Guards the one link of the INS-948 fix that instance-ai's own unit tests
// cannot see: the run's resolved (proxy-aware) model must land on the domain
// context, where simulation fixture/classifier LLM calls read it as their
// fallback on deployments without env model keys (cloud). Dropping this
// wiring regresses silently — every simulated node degrades back to a
// single empty pinned item.
it('copies the host-resolved modelId onto the context', () => {
const service = createAdapterWithGatewayMock(vi.fn());
const modelId = {
id: 'anthropic/claude-opus-4-8' as const,
url: 'https://proxy.example.com/anthropic/v1',
apiKey: 'proxy-token',
};
const context = service.createContext(mockUser, { modelId });
expect(context.modelId).toEqual(modelId);
});
it('leaves modelId undefined when the host does not resolve one', () => {
const service = createAdapterWithGatewayMock(vi.fn());
expect(service.createContext(mockUser).modelId).toBeUndefined();
});
});
@@ -42,6 +42,7 @@ import type {
EvaluationConfigDetail,
UpsertEvaluationConfigInput,
InstanceAiBuilderDelegate,
ModelConfig,
} from '@n8n/instance-ai';
import { braveSearch, searxngSearch, type WebSearchResponse } from '@n8n/ai-utilities';
import {
@@ -296,6 +297,9 @@ export class InstanceAiAdapterService {
/** Per-user config-evals gate (via `isConfigEvalsEnabled`). Falsy →
* eval-config service/tool not wired. */
configEvalsEnabled?: boolean;
/** Host-resolved model for the run — fallback for utility LLM calls
* (simulation fixtures, destructiveness classification). */
modelId?: ModelConfig;
},
): InstanceAiContext {
const {
@@ -306,6 +310,7 @@ export class InstanceAiAdapterService {
credentialIdAllowlist,
agentId,
configEvalsEnabled,
modelId,
} = options ?? {};
// Record gateway availability once per context. Fire-and-forget: the
@@ -317,6 +322,7 @@ export class InstanceAiAdapterService {
return {
userId: user.id,
projectId,
modelId,
workflowService: this.createWorkflowAdapter(user, threadId, projectId),
executionService: this.createExecutionAdapter(user, pushRef, threadId),
credentialService: this.createCredentialAdapter(user, projectId, credentialIdAllowlist),
@@ -2091,6 +2091,11 @@ export class InstanceAiService {
const { searchProxyConfig, tracingProxyConfig, tokenManager, proxyBaseUrl } =
proxyRunConfig ?? (await this.createProxyRunConfig(user));
const modelId =
proxyBaseUrl && tokenManager
? await this.modelService.resolveProxyModel(user, proxyBaseUrl, tokenManager)
: await this.modelService.resolveAgentModelConfig(user);
const configEvalsEnabled = await this.adapterService.isConfigEvalsEnabled(user);
const context = this.adapterService.createContext(user, {
searchProxyConfig,
@@ -2099,6 +2104,7 @@ export class InstanceAiService {
projectId: boundProjectId,
credentialIdAllowlist: this.evalCredentialAllowlists.get(threadId),
configEvalsEnabled,
modelId,
});
// Merge both local gateway and direct browser-use into a single
@@ -2193,11 +2199,6 @@ export class InstanceAiService {
};
}
const modelId =
proxyBaseUrl && tokenManager
? await this.modelService.resolveProxyModel(user, proxyBaseUrl, tokenManager)
: await this.modelService.resolveAgentModelConfig(user);
const taskStorage = new ThreadTaskStorage(memory);
const iterationLog = this.dbIterationLogStorage;
const snapshotStorage = this.dbSnapshotStorage;