fix(core): Agents model list exclude deprecated models and improved error surfacing (no-changelog) (#33631)

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Michael Drury
2026-07-06 16:27:47 +01:00
committed by GitHub
parent eb59191076
commit 1dce924b0d
49 changed files with 2007 additions and 467 deletions
@@ -6438,3 +6438,109 @@ describe('AgentRuntime — toModelOutput error resilience', () => {
expect(toolResultChunk!.isError).toBe(true);
});
});
// ---------------------------------------------------------------------------
// empty model responses (e.g. provider safety blocks)
// ---------------------------------------------------------------------------
describe('AgentRuntime — empty model responses', () => {
beforeEach(() => {
generateText.mockReset();
streamText.mockReset();
});
function makeEmptyStream(finishReason: string, extraChunks: Array<Record<string, unknown>> = []) {
return {
fullStream: makeChunkStream(extraChunks),
finishReason: Promise.resolve(finishReason),
usage: Promise.resolve({ inputTokens: 10, outputTokens: 0, totalTokens: 10 }),
response: Promise.resolve({ messages: [] }),
toolCalls: Promise.resolve([]),
};
}
it.each(['other', 'unknown', 'content-filter'])(
'stream: yields an error chunk when the model returns no output with finish reason "%s"',
async (finishReason) => {
streamText.mockReturnValue(makeEmptyStream(finishReason));
const { runtime } = createRuntime();
const { stream: readableStream } = await runtime.stream('hello');
const chunks = await collectChunks(readableStream);
const errorChunk = chunks.find((c) => c.type === 'error') as
| (StreamChunk & { type: 'error' })
| undefined;
expect(errorChunk).toBeDefined();
expect(String((errorChunk!.error as Error).message)).toContain('no output');
expect(String((errorChunk!.error as Error).message)).toContain(finishReason);
const finishChunk = chunks.find((c) => c.type === 'finish') as
| (StreamChunk & { type: 'finish' })
| undefined;
expect(finishChunk).toBeDefined();
expect(finishChunk!.finishReason).toBe('error');
},
);
it('stream: includes the Google prompt block reason captured from raw chunks', async () => {
streamText.mockReturnValue(
makeEmptyStream('other', [
{
type: 'raw',
rawValue: { promptFeedback: { blockReason: 'PROHIBITED_CONTENT' } },
},
]),
);
const { runtime } = createRuntime(undefined, 'google/gemini-2.5-flash');
const { stream: readableStream } = await runtime.stream('hello');
const chunks = await collectChunks(readableStream);
const errorChunk = chunks.find((c) => c.type === 'error') as
| (StreamChunk & { type: 'error' })
| undefined;
expect(errorChunk).toBeDefined();
expect(String((errorChunk!.error as Error).message)).toContain('PROHIBITED_CONTENT');
});
it('stream: requests raw chunks for google models so block reasons are observable', async () => {
streamText.mockReturnValue(makeStreamSuccess());
const { runtime } = createRuntime(undefined, 'google/gemini-2.5-flash');
const { stream: readableStream } = await runtime.stream('hello');
await collectChunks(readableStream);
const args = streamText.mock.calls[0][0] as Record<string, unknown>;
expect(args.includeRawChunks).toBe(true);
});
it('stream: an empty response finishing with "stop" is not treated as an error', async () => {
streamText.mockReturnValue(makeEmptyStream('stop'));
const { runtime } = createRuntime();
const { stream: readableStream } = await runtime.stream('hello');
const chunks = await collectChunks(readableStream);
expect(chunks.find((c) => c.type === 'error')).toBeUndefined();
const finishChunk = chunks.find((c) => c.type === 'finish') as
| (StreamChunk & { type: 'finish' })
| undefined;
expect(finishChunk!.finishReason).not.toBe('error');
});
it('generate: returns finishReason "error" with a descriptive error when the model returns no output', async () => {
generateText.mockResolvedValue({
finishReason: 'other',
usage: { inputTokens: 10, outputTokens: 0, totalTokens: 10 },
response: { messages: [] },
toolCalls: [],
});
const { runtime } = createRuntime();
const result = await runtime.generate('hello');
expect(result.finishReason).toBe('error');
expect(String((result.error as Error).message)).toContain('no output');
});
});
@@ -719,6 +719,10 @@ export class AgentRuntime {
sink.onTurnFolded?.();
if (turn.aiFinishReason !== 'tool-calls') {
// A rejected/filtered request (e.g. a provider prompt safety block)
// surfaces as an output-less turn instead of an SDK error — throw so
// the failure reaches the caller rather than ending the run silently.
if (turn.errorReason) throw new Error(turn.errorReason.message);
structuredOutput = turn.structuredOutput;
this.emitTurnEnd(turn.newMessages, extractSettledToolCalls(turn.newMessages));
reachedStopCondition = true;
@@ -6,6 +6,7 @@ import type {
RunServices,
SuspendEmission,
} from './run-output-sink';
import { classifyModelTurnError } from './runtime-helpers';
import type { GenerateResult } from '../../types';
import type { ToolResultEntry } from '../../types/sdk/agent';
import { loadAi } from '../model/lazy-ai';
@@ -40,14 +41,17 @@ export class GenerateSink implements RunOutputSink<GenerateResult> {
});
const aiFinishReason = result.finishReason;
const newMessages = fromAiMessages(result.response.messages);
const errorReason = classifyModelTurnError({ aiFinishReason, newMessages });
return {
aiFinishReason,
finishReason: fromAiFinishReason(aiFinishReason),
usage: toTokenUsage(result.usage, result.providerMetadata),
newMessages: fromAiMessages(result.response.messages),
newMessages,
toolCalls: result.toolCalls,
structuredOutput:
ctx.outputSpec && aiFinishReason !== 'tool-calls' ? result.output : undefined,
...(errorReason && { errorReason }),
};
}
@@ -17,6 +17,21 @@ import type { ToolCallBatchResult, ToolCallSuspension } from '../tools/tool-call
type RunCallOptions = (RunOptions & ExecutionOptions) | undefined;
/**
* Recognized ways a model turn can fail while the provider still returns
* "successfully" (no thrown SDK error). Extend this as new failure modes are
* recognized so consumers switch on `type` instead of probing separate fields.
* - `prompt_blocked`: the provider rejected the prompt (e.g. a safety block).
* - `no_output`: the model returned nothing with a non-stop finish reason.
*/
export type ModelTurnErrorType = 'prompt_blocked' | 'no_output';
/** A recognized model-turn failure: a `type` to branch on and a user-facing `message`. */
export interface ModelTurnError {
type: ModelTurnErrorType;
message: string;
}
/** Normalized result of a single LLM turn, independent of generate vs stream. */
export interface ModelTurnResult {
/** Raw AI SDK finish reason (used to detect the `tool-calls` continuation). */
@@ -33,6 +48,13 @@ export interface ModelTurnResult {
}>;
/** Resolved structured output, only when an output spec is set and the turn finished. */
structuredOutput: unknown;
/**
* Set when the turn failed in a recognized way despite the provider returning
* without a thrown error (e.g. a prompt safety block, or empty output). The
* loop surfaces this instead of ending the run silently. Check this single
* field rather than probing per-failure-mode properties.
*/
errorReason?: ModelTurnError;
}
/** Per-iteration inputs for the LLM call, assembled by the shared loop. */
@@ -2,8 +2,10 @@
* Pure utility functions used by AgentRuntime that require no class context.
* These are extracted here to keep agent-runtime.ts focused on orchestration logic.
*/
import type { ModelTurnError } from './run-output-sink';
import type { StreamChunk, TokenUsage } from '../../types';
import type { AgentMessage, ContentToolCall } from '../../types/sdk/message';
import type { RawProviderError } from '../model/raw-error';
/**
* Normalize caller input to `AgentMessage[]` for the runtime. String input becomes a
@@ -21,6 +23,44 @@ export function stringifyError(error: unknown): string {
return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
}
/**
* Finish reasons that indicate the provider rejected or filtered the request
* when they arrive with zero output. `stop`/`length` with empty output are the
* model's own (odd but legal) choice; `tool-calls` always carries calls;
* `error` surfaces through the SDK's thrown error instead.
*/
const EMPTY_RESPONSE_ERROR_FINISH_REASONS = new Set(['other', 'unknown', 'content-filter']);
/**
* Classify a turn that produced no output as a recognized failure, or return
* `undefined` when it doesn't look like a provider rejection. Some providers
* fail this way rather than erroring, reporting the cause only on their raw
* stream events — when a {@link RawProviderError} was captured there, its type
* and reason are carried into the result; otherwise the failure is a generic
* `no_output`.
*/
export function classifyModelTurnError(turn: {
aiFinishReason: string;
newMessages: AgentMessage[];
providerError?: RawProviderError;
}): ModelTurnError | undefined {
if (turn.newMessages.length > 0) return undefined;
if (!EMPTY_RESPONSE_ERROR_FINISH_REASONS.has(turn.aiFinishReason)) return undefined;
const guidance =
'This can be a provider-side false positive — try rephrasing the message, clearing the chat history, or switching models.';
if (turn.providerError) {
return {
type: turn.providerError.type,
message: `The model provider blocked this request (${turn.providerError.reason}) and returned no output (finish reason: ${turn.aiFinishReason}). ${guidance}`,
};
}
return {
type: 'no_output',
message: `The model returned no output (finish reason: ${turn.aiFinishReason}). The provider may have blocked or filtered the request. ${guidance}`,
};
}
/** Extract all settled (resolved or rejected) tool-call blocks from a flat list of agent messages. */
export function extractSettledToolCalls(messages: AgentMessage[]): ContentToolCall[] {
return messages
@@ -6,11 +6,12 @@ import type {
RunServices,
SuspendEmission,
} from './run-output-sink';
import { mergeUsage } from './runtime-helpers';
import { classifyModelTurnError, mergeUsage } from './runtime-helpers';
import type { ExecutionOptions, TokenUsage } from '../../types/sdk/agent';
import type { AgentMessage } from '../../types/sdk/message';
import { loadAi } from '../model/lazy-ai';
import { fromAiFinishReason, fromAiMessages } from '../model/messages';
import { createRawErrorReader, type RawErrorReader } from '../model/raw-error';
import { createRawUsageReader, type RawUsageReader } from '../model/raw-usage';
import { convertChunk, toTokenUsage } from '../streaming/stream';
import type { StreamWriterGuard } from '../streaming/stream-writer-guard';
@@ -32,6 +33,11 @@ export class StreamSink implements RunOutputSink<void> {
// Text streamed for the in-flight turn, retained so a stop landing mid-response
// can still persist what the user already saw. Cleared once the turn is folded.
private partialText = '';
// Reads provider failure signals (e.g. a prompt safety block) from raw
// chunks, so an output-less rejected request can report why. Per-provider
// implementations live behind `RawErrorReader`; undefined when the run's
// provider has no reader.
private rawErrorReader: RawErrorReader | undefined;
constructor(
private readonly guard: StreamWriterGuard,
@@ -100,6 +106,10 @@ export class StreamSink implements RunOutputSink<void> {
this.rawUsageReader = this.options?.recoverUsageOnAbort
? createRawUsageReader(this.services.modelId)
: undefined;
// Some providers report failures (e.g. prompt safety blocks) only on the
// raw stream — no error, no content — so raw chunks are required to
// explain an otherwise silent empty response.
this.rawErrorReader = createRawErrorReader(this.services.modelId);
const { streamText } = loadAi();
const result = streamText({
model: ctx.model,
@@ -108,7 +118,9 @@ export class StreamSink implements RunOutputSink<void> {
abortSignal: ctx.abortSignal,
// Surface the provider's raw message_start/message_delta events so an
// aborted run can recover its usage — the SDK reports none on abort.
...(this.rawUsageReader ? { includeRawChunks: true } : {}),
...(this.rawUsageReader !== undefined || this.rawErrorReader !== undefined
? { includeRawChunks: true }
: {}),
...(ctx.hasTools ? { tools: ctx.aiTools } : {}),
...(ctx.providerOptions ? { providerOptions: ctx.providerOptions } : {}),
...(ctx.outputSpec ? { output: ctx.outputSpec } : {}),
@@ -124,6 +136,7 @@ export class StreamSink implements RunOutputSink<void> {
// reaches the post-loop awaits) can still be billed via getAbortFinish.
if (chunk.type === 'raw') {
this.rawUsageReader?.capture(chunk.rawValue);
this.rawErrorReader?.capture(chunk.rawValue);
continue;
}
// Filter only the SDK's terminal `finish` chunk — the runtime emits its
@@ -167,15 +180,22 @@ export class StreamSink implements RunOutputSink<void> {
const usage = await result.usage;
const providerMetadata = await result.providerMetadata;
const response = await result.response;
const newMessages = fromAiMessages(response.messages);
const errorReason = classifyModelTurnError({
aiFinishReason,
newMessages,
providerError: this.rawErrorReader?.getError(),
});
return {
aiFinishReason,
finishReason: fromAiFinishReason(aiFinishReason),
usage: toTokenUsage(usage, providerMetadata),
newMessages: fromAiMessages(response.messages),
newMessages,
toolCalls: await result.toolCalls,
structuredOutput:
ctx.outputSpec && aiFinishReason !== 'tool-calls' ? await result.output : undefined,
...(errorReason && { errorReason }),
};
}
@@ -0,0 +1,66 @@
import { isRecord } from '@n8n/utils/is-record';
import { getModelProvider } from './prompt-cache';
import { type ProviderId } from './provider-credentials';
import type { ModelTurnErrorType } from '../loop/run-output-sink';
/** A recognized failure signal read from the provider's raw stream events. */
export interface RawProviderError {
type: ModelTurnErrorType;
/** The provider's own reason string (e.g. Google's `PROHIBITED_CONTENT`). */
reason: string;
}
/**
* Reads a provider's native raw stream events (the AI SDK's `chunk.rawValue`
* passthrough, enabled by `includeRawChunks`) for failure signals the SDK does
* not surface — e.g. a prompt safety block that ends the stream with no error
* and no content. Each provider's wire format differs, so the runtime stays
* provider-agnostic and delegates to a per-provider implementation selected
* via {@link createRawErrorReader}.
*/
export interface RawErrorReader {
/** Inspect one raw provider event for a failure signal. */
capture(rawValue: unknown): void;
/** The failure signal seen this turn, or undefined when none was observed. */
getError(): RawProviderError | undefined;
}
/**
* Google reports prompt safety blocks only via `promptFeedback.blockReason` on
* the raw stream — the response then simply contains no candidates.
*/
export class GoogleRawErrorReader implements RawErrorReader {
private blockReason: string | undefined;
capture(rawValue: unknown): void {
if (!isRecord(rawValue) || !isRecord(rawValue.promptFeedback)) return;
const blockReason = rawValue.promptFeedback.blockReason;
if (typeof blockReason === 'string' && blockReason) {
this.blockReason = blockReason;
}
}
getError(): RawProviderError | undefined {
return this.blockReason ? { type: 'prompt_blocked', reason: this.blockReason } : undefined;
}
}
/**
* Per-provider raw-error readers, keyed by provider id. Partial because this is
* best-effort diagnostics: a provider without an entry still gets the generic
* empty-response handling, just without the provider's own reason attached.
*/
const RAW_ERROR_READERS: Partial<Record<ProviderId, () => RawErrorReader>> = {
google: () => new GoogleRawErrorReader(),
};
/**
* Create a fresh raw-error reader for the run's model, or undefined when the
* provider has no reader. The provider is the prefix of the `provider/model` id
* (same convention as `createModel`).
*/
export function createRawErrorReader(modelId: string): RawErrorReader | undefined {
const provider = getModelProvider(modelId) as ProviderId;
return RAW_ERROR_READERS[provider]?.();
}
@@ -71,6 +71,75 @@ describe('fetchProviderCatalog', () => {
expect(catalog['azure-cognitive-services']).toBeUndefined();
});
it('drops models marked deprecated on models.dev', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () =>
await Promise.resolve({
anthropic: {
id: 'anthropic',
name: 'Anthropic',
models: {
'claude-3-haiku-20240307': {
id: 'claude-3-haiku-20240307',
name: 'Claude Haiku 3',
status: 'deprecated',
},
'claude-sonnet-4-6': {
id: 'claude-sonnet-4-6',
name: 'Claude Sonnet 4.6',
},
'claude-beta-model': {
id: 'claude-beta-model',
name: 'Claude Beta Model',
status: 'beta',
},
},
},
}),
});
global.fetch = fetchMock as typeof fetch;
const catalog = await fetchProviderCatalog();
expect(catalog.anthropic.models['claude-3-haiku-20240307']).toBeUndefined();
expect(catalog.anthropic.models['claude-sonnet-4-6'].name).toBe('Claude Sonnet 4.6');
expect(catalog.anthropic.models['claude-beta-model'].name).toBe('Claude Beta Model');
});
it('omits a provider whose models are all deprecated', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () =>
await Promise.resolve({
anthropic: {
id: 'anthropic',
name: 'Anthropic',
models: {
'claude-3-haiku-20240307': {
id: 'claude-3-haiku-20240307',
name: 'Claude Haiku 3',
status: 'deprecated',
},
},
},
openai: {
id: 'openai',
name: 'OpenAI',
models: {
'gpt-5': { id: 'gpt-5', name: 'GPT-5' },
},
},
}),
});
global.fetch = fetchMock as typeof fetch;
const catalog = await fetchProviderCatalog();
expect(catalog.anthropic).toBeUndefined();
expect(catalog.openai.models['gpt-5'].name).toBe('GPT-5');
});
it('strips the "(latest)" suffix from model names and drops duplicate pinned snapshots', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
+6
View File
@@ -68,6 +68,7 @@ interface ModelsDevModel {
release_date?: string;
reasoning?: boolean;
tool_call?: boolean;
status?: string;
cost?: { input?: number; output?: number; cache_read?: number; cache_write?: number };
limit?: { context?: number; output?: number };
}
@@ -138,6 +139,9 @@ export async function fetchProviderCatalog(): Promise<ProviderCatalog> {
const models: Record<string, ModelInfo> = {};
for (const [modelId, model] of Object.entries(provider.models)) {
// Deprecated models still 404 at call time when the provider retires
// them, so never offer them.
if (model.status === 'deprecated') continue;
const info: ModelInfo = {
id: model.id,
name: model.name,
@@ -162,6 +166,8 @@ export async function fetchProviderCatalog(): Promise<ProviderCatalog> {
models[modelId] = info;
}
if (Object.keys(models).length === 0) continue;
const providerId = toAgentProviderId(key);
catalog[providerId] = {
id: providerId,
+8
View File
@@ -24,6 +24,9 @@
],
"web-search": [
"dist/esm/web-search/index.d.ts"
],
"model-discovery": [
"dist/esm/model-discovery/index.d.ts"
]
}
},
@@ -63,6 +66,11 @@
"import": "./dist/esm/web-search/index.js",
"require": "./dist/cjs/web-search/index.js"
},
"./model-discovery": {
"types": "./dist/esm/model-discovery/index.d.ts",
"import": "./dist/esm/model-discovery/index.js",
"require": "./dist/cjs/model-discovery/index.js"
},
"./*": "./*"
},
"scripts": {
@@ -0,0 +1,228 @@
import { listModelsForProvider, MODEL_DISCOVERY_PROVIDERS } from '../index';
function mockFetch(body: unknown, ok = true, status = 200) {
return vi.fn().mockResolvedValue({
ok,
status,
text: async () => JSON.stringify(body),
json: async () => body,
}) as unknown as typeof globalThis.fetch;
}
function calledUrl(fetchFn: unknown): string {
return String((fetchFn as ReturnType<typeof vi.fn>).mock.calls[0][0]);
}
function calledHeaders(fetchFn: unknown): Record<string, string> {
const init = (fetchFn as ReturnType<typeof vi.fn>).mock.calls[0][1] as {
headers: Record<string, string>;
};
return init.headers;
}
describe('model-discovery', () => {
describe('anthropic', () => {
it('lists models from /v1/models with x-api-key auth, newest first', async () => {
const fetch = mockFetch({
data: [
{ id: 'claude-old', display_name: 'Claude Old', created_at: '2024-01-01T00:00:00Z' },
{ id: 'claude-new', display_name: 'Claude New', created_at: '2026-01-01T00:00:00Z' },
],
});
const models = await listModelsForProvider('anthropic', { apiKey: 'key', fetch });
expect(calledUrl(fetch)).toBe('https://api.anthropic.com/v1/models');
expect(calledHeaders(fetch)['x-api-key']).toBe('key');
expect(calledHeaders(fetch)['anthropic-version']).toBe('2023-06-01');
expect(models).toEqual([
{ id: 'claude-new', name: 'Claude New' },
{ id: 'claude-old', name: 'Claude Old' },
]);
});
it('respects a custom baseURL', async () => {
const fetch = mockFetch({ data: [] });
await listModelsForProvider('anthropic', {
apiKey: 'key',
baseURL: 'https://proxy.example.com/',
fetch,
});
expect(calledUrl(fetch)).toBe('https://proxy.example.com/v1/models');
});
});
describe('openai', () => {
it('filters out non-chat models on the official API and sorts by id', async () => {
const fetch = mockFetch({
data: [
{ id: 'gpt-5' },
{ id: 'whisper-1' },
{ id: 'dall-e-3' },
{ id: 'text-embedding-3-small' },
{ id: 'gpt-4o' },
],
});
const models = await listModelsForProvider('openai', { apiKey: 'key', fetch });
expect(calledUrl(fetch)).toBe('https://api.openai.com/v1/models');
expect(calledHeaders(fetch).Authorization).toBe('Bearer key');
expect(models.map((m) => m.id)).toEqual(['gpt-4o', 'gpt-5']);
});
it('includes all models for a custom (non-OpenAI) baseURL', async () => {
const fetch = mockFetch({ data: [{ id: 'whisper-1' }, { id: 'my-model' }] });
const models = await listModelsForProvider('openai', {
apiKey: 'key',
baseURL: 'https://llm.internal.example.com/v1',
fetch,
});
expect(models.map((m) => m.id)).toEqual(['my-model', 'whisper-1']);
});
it('merges caller-supplied headers into the request', async () => {
const fetch = mockFetch({ data: [] });
await listModelsForProvider('openai', {
apiKey: 'key',
headers: { 'x-custom': 'yes' },
fetch,
});
expect(calledHeaders(fetch)['x-custom']).toBe('yes');
});
});
describe('google', () => {
it('lists models from /v1beta/models with header auth, excluding embedding/imagen', async () => {
const fetch = mockFetch({
models: [
{ name: 'models/gemini-2.5-flash', description: 'Fast' },
{ name: 'models/text-embedding-004', description: 'Embeddings' },
{ name: 'models/imagen-3', description: 'Images' },
{ name: 'models/gemini-2.5-pro', description: 'Smart' },
],
});
const models = await listModelsForProvider('google', { apiKey: 'g-key', fetch });
// The API key goes in the x-goog-api-key header, never the query string,
// so it cannot leak through access logs or proxies.
expect(calledUrl(fetch)).toBe('https://generativelanguage.googleapis.com/v1beta/models');
expect(calledHeaders(fetch)['x-goog-api-key']).toBe('g-key');
expect(models).toEqual([
{ id: 'models/gemini-2.5-flash', name: 'models/gemini-2.5-flash' },
{ id: 'models/gemini-2.5-pro', name: 'models/gemini-2.5-pro' },
]);
});
});
describe('groq', () => {
it('keeps only active model objects', async () => {
const fetch = mockFetch({
data: [
{ id: 'llama-3.3-70b', active: true, object: 'model' },
{ id: 'retired-model', active: false, object: 'model' },
{ id: 'not-a-model', active: true, object: 'other' },
],
});
const models = await listModelsForProvider('groq', { apiKey: 'key', fetch });
expect(calledUrl(fetch)).toBe('https://api.groq.com/openai/v1/models');
expect(models.map((m) => m.id)).toEqual(['llama-3.3-70b']);
});
});
describe('mistral', () => {
it('excludes embedding models and sorts by name', async () => {
const fetch = mockFetch({
data: [{ id: 'mistral-large' }, { id: 'mistral-embed' }, { id: 'codestral' }],
});
const models = await listModelsForProvider('mistral', { apiKey: 'key', fetch });
expect(calledUrl(fetch)).toBe('https://api.mistral.ai/v1/models');
expect(models.map((m) => m.id)).toEqual(['codestral', 'mistral-large']);
});
});
describe('cohere', () => {
it('lists chat models by name', async () => {
const fetch = mockFetch({
models: [{ name: 'command-r-plus' }, { name: 'command-a-03-2025' }],
});
const models = await listModelsForProvider('cohere', { apiKey: 'key', fetch });
expect(calledUrl(fetch)).toBe('https://api.cohere.ai/v1/models?page_size=100&endpoint=chat');
expect(models.map((m) => m.id)).toEqual(['command-a-03-2025', 'command-r-plus']);
});
});
describe('nvidia', () => {
it('keeps only supported Nemotron models', async () => {
const fetch = mockFetch({
data: [
{ id: 'nvidia/llama-3.3-nemotron-super-49b-v1' },
{ id: 'meta/llama-3.1-405b-instruct' },
],
});
const models = await listModelsForProvider('nvidia', { apiKey: 'key', fetch });
expect(calledUrl(fetch)).toBe('https://integrate.api.nvidia.com/v1/models');
expect(models.map((m) => m.id)).toEqual(['nvidia/llama-3.3-nemotron-super-49b-v1']);
});
});
describe.each([
['deepseek', 'https://api.deepseek.com/models'],
['openrouter', 'https://openrouter.ai/api/v1/models'],
['xai', 'https://api.x.ai/v1/models'],
['vercel', 'https://ai-gateway.vercel.sh/v1/models'],
] as const)('%s', (provider, expectedUrl) => {
it('lists models with bearer auth, sorted by name', async () => {
const fetch = mockFetch({ data: [{ id: 'model-b' }, { id: 'model-a' }] });
const models = await listModelsForProvider(provider, { apiKey: 'key', fetch });
expect(calledUrl(fetch)).toBe(expectedUrl);
expect(calledHeaders(fetch).Authorization).toBe('Bearer key');
expect(models.map((m) => m.id)).toEqual(['model-a', 'model-b']);
});
});
describe('error handling', () => {
it('throws a descriptive error on a non-2xx response', async () => {
const fetch = mockFetch({ error: { message: 'invalid x-api-key' } }, false, 401);
await expect(listModelsForProvider('anthropic', { apiKey: 'bad', fetch })).rejects.toThrow(
/anthropic.*401/i,
);
});
it('throws for an unknown provider', async () => {
await expect(
listModelsForProvider('not-a-provider', { apiKey: 'key', fetch: mockFetch({}) }),
).rejects.toThrow(/unknown/i);
});
});
it('exposes a registry of all supported providers', () => {
expect(Object.keys(MODEL_DISCOVERY_PROVIDERS).sort()).toEqual([
'anthropic',
'cohere',
'deepseek',
'google',
'groq',
'mistral',
'nvidia',
'openai',
'openrouter',
'vercel',
'xai',
]);
});
});
@@ -0,0 +1,65 @@
import { listAnthropicModels } from './providers/anthropic';
import { listCohereModels } from './providers/cohere';
import { listDeepSeekModels } from './providers/deepseek';
import { listGoogleModels } from './providers/google';
import { listGroqModels } from './providers/groq';
import { listMistralModels } from './providers/mistral';
import { listNvidiaModels } from './providers/nvidia';
import { listOpenAiModels } from './providers/openai';
import { listOpenRouterModels } from './providers/openrouter';
import { listVercelModels } from './providers/vercel';
import { listXaiModels } from './providers/xai';
import type { ListModelsFn, ListModelsOptions, ProviderModel } from './types';
/**
* Live chat-model discovery per provider: each function asks the provider's
* own model-list API which models the given credential can call. Shared by the
* chat sub-nodes' model dropdowns and the agents feature's model catalog, so
* provider knowledge (endpoint, auth, chat-model filtering) lives in one place.
*
* Each provider implements {@link ListModelsFn} in its own file under
* `providers/`, ported 1:1 from the corresponding chat node's `searchModels`
* method or `loadOptions` routing config (see the source reference on each).
* When changing behavior here, keep the node in sync.
*/
export const MODEL_DISCOVERY_PROVIDERS: Record<string, ListModelsFn> = {
anthropic: listAnthropicModels,
cohere: listCohereModels,
deepseek: listDeepSeekModels,
google: listGoogleModels,
groq: listGroqModels,
mistral: listMistralModels,
nvidia: listNvidiaModels,
openai: listOpenAiModels,
openrouter: listOpenRouterModels,
vercel: listVercelModels,
xai: listXaiModels,
};
export function isModelDiscoveryProvider(provider: string): boolean {
return provider in MODEL_DISCOVERY_PROVIDERS;
}
export async function listModelsForProvider(
provider: string,
options: ListModelsOptions,
): Promise<ProviderModel[]> {
const listModels = MODEL_DISCOVERY_PROVIDERS[provider];
if (!listModels) {
throw new Error(`Unknown model discovery provider: "${provider}"`);
}
return await listModels(options);
}
export { listAnthropicModels } from './providers/anthropic';
export { listCohereModels } from './providers/cohere';
export { listDeepSeekModels } from './providers/deepseek';
export { listGoogleModels } from './providers/google';
export { listGroqModels } from './providers/groq';
export { listMistralModels } from './providers/mistral';
export { listNvidiaModels } from './providers/nvidia';
export { listOpenAiModels, shouldIncludeOpenAiModel } from './providers/openai';
export { listOpenRouterModels } from './providers/openrouter';
export { listVercelModels } from './providers/vercel';
export { listXaiModels } from './providers/xai';
export type { ListModelsFn, ListModelsOptions, ProviderModel } from './types';
@@ -0,0 +1,17 @@
import { baseUrl, getJson } from '../request';
import type { ListModelsFn } from '../types';
/** Source: LMChatAnthropic `methods/searchModels.ts` (GET /v1/models, newest first). */
export const listAnthropicModels: ListModelsFn = async (options) => {
const data = (await getJson(
`${baseUrl(options, 'https://api.anthropic.com')}/v1/models`,
{ 'x-api-key': options.apiKey, 'anthropic-version': '2023-06-01' },
options,
'anthropic',
)) as { data?: Array<{ id: string; display_name?: string; created_at?: string }> };
return (data.data ?? [])
.slice()
.sort((a, b) => new Date(b.created_at ?? 0).getTime() - new Date(a.created_at ?? 0).getTime())
.map((model) => ({ id: model.id, name: model.display_name ?? model.id }));
};
@@ -0,0 +1,17 @@
import { baseUrl, bearerHeaders, byName, getJson } from '../request';
import type { ListModelsFn } from '../types';
/** Source: LmChatCohere `loadOptions` routing (chat endpoint models only). */
export const listCohereModels: ListModelsFn = async (options) => {
const data = (await getJson(
`${baseUrl(options, 'https://api.cohere.ai')}/v1/models?page_size=100&endpoint=chat`,
bearerHeaders(options),
options,
'cohere',
)) as { models?: Array<{ name?: unknown }> };
return (data.models ?? [])
.filter((model): model is { name: string } => typeof model.name === 'string')
.map((model) => ({ id: model.name, name: model.name }))
.sort(byName);
};
@@ -0,0 +1,4 @@
import { makeBearerDataListing } from '../request';
/** Source: LmChatDeepSeek `loadOptions` routing. */
export const listDeepSeekModels = makeBearerDataListing('deepseek', 'https://api.deepseek.com');
@@ -0,0 +1,29 @@
import { baseUrl, byName, getJson } from '../request';
import type { ListModelsFn } from '../types';
/**
* Source: LmChatGoogleGemini `loadOptions` routing (GET /v1beta/models,
* embedding/imagen excluded). Ids keep Google's `models/` prefix, matching the
* node dropdown values. Auth uses the `x-goog-api-key` header (Google's
* preferred method) rather than the credential's `?key=` query auth, so the
* key cannot leak through access logs or proxies.
*/
export const listGoogleModels: ListModelsFn = async (options) => {
const base = baseUrl(options, 'https://generativelanguage.googleapis.com');
const data = (await getJson(
`${base}/v1beta/models`,
{ 'x-goog-api-key': options.apiKey },
options,
'google',
)) as { models?: Array<{ name?: unknown }> };
return (data.models ?? [])
.filter(
(model): model is { name: string } =>
typeof model.name === 'string' &&
!model.name.includes('embedding') &&
!model.name.includes('imagen'),
)
.map((model) => ({ id: model.name, name: model.name }))
.sort(byName);
};
@@ -0,0 +1,16 @@
import { baseUrl, bearerHeaders, getJson, idsToModels } from '../request';
import type { ListModelsFn } from '../types';
/** Source: LmChatGroq `loadOptions` routing (active model objects only). */
export const listGroqModels: ListModelsFn = async (options) => {
const data = (await getJson(
`${baseUrl(options, 'https://api.groq.com/openai/v1')}/models`,
bearerHeaders(options),
options,
'groq',
)) as { data?: Array<{ id?: unknown; active?: unknown; object?: unknown }> };
return idsToModels(
(data.data ?? []).filter((model) => model.active === true && model.object === 'model'),
);
};
@@ -0,0 +1,16 @@
import { baseUrl, bearerHeaders, byName, getJson, idsToModels, type IdItem } from '../request';
import type { ListModelsFn } from '../types';
/** Source: LmChatMistralCloud `loadOptions` routing (embedding models excluded). */
export const listMistralModels: ListModelsFn = async (options) => {
const data = (await getJson(
`${baseUrl(options, 'https://api.mistral.ai/v1')}/models`,
bearerHeaders(options),
options,
'mistral',
)) as { data?: IdItem[] };
return idsToModels(data.data ?? [])
.filter((model) => !model.id.includes('embed'))
.sort(byName);
};
@@ -0,0 +1,27 @@
import { baseUrl, bearerHeaders, byName, getJson, idsToModels, type IdItem } from '../request';
import type { ListModelsFn } from '../types';
/** Source: LmChatNvidia `loadOptions` routing — mirrors NEMOTRON_SUPPORTED_MODELS. */
const NVIDIA_SUPPORTED_MODELS = new Set([
'nvidia/llama-3.1-nemotron-nano-8b-v1',
'nvidia/llama-3.3-nemotron-super-49b-v1',
'nvidia/llama-3.3-nemotron-super-49b-v1.5',
'nvidia/nemotron-3-nano-30b-a3b',
'nvidia/nemotron-3-nano-omni-30b-a3b-reasoning',
'nvidia/nemotron-3-super-120b-a12b',
'nvidia/nemotron-nano-12b-v2-vl',
'nvidia/nvidia-nemotron-nano-9b-v2',
]);
export const listNvidiaModels: ListModelsFn = async (options) => {
const data = (await getJson(
`${baseUrl(options, 'https://integrate.api.nvidia.com/v1')}/models`,
bearerHeaders(options),
options,
'nvidia',
)) as { data?: IdItem[] };
return idsToModels(data.data ?? [])
.filter((model) => NVIDIA_SUPPORTED_MODELS.has(model.id))
.sort(byName);
};
@@ -0,0 +1,39 @@
import { baseUrl, bearerHeaders, byName, getJson, idsToModels, type IdItem } from '../request';
import type { ListModelsFn } from '../types';
const OFFICIAL_OPENAI_HOSTNAMES = ['api.openai.com', 'ai-assistant.n8n.io'];
/**
* Source: LMChatOpenAi `methods/loadModels.ts` — on the official API, exclude
* non-chat model families; on custom (proxy/self-hosted) hosts include all.
*/
export function shouldIncludeOpenAiModel(modelId: string, isCustomApi: boolean): boolean {
if (isCustomApi) return true;
return !(
modelId.startsWith('babbage') ||
modelId.startsWith('davinci') ||
modelId.startsWith('computer-use') ||
modelId.startsWith('dall-e') ||
modelId.startsWith('text-embedding') ||
modelId.startsWith('tts') ||
modelId.includes('-tts') ||
modelId.startsWith('whisper') ||
modelId.startsWith('omni-moderation') ||
modelId.startsWith('sora') ||
modelId.includes('-realtime') ||
(modelId.startsWith('gpt-') && modelId.includes('instruct'))
);
}
/** Source: LMChatOpenAi `methods/loadModels.ts` (GET /models, filtered, id asc). */
export const listOpenAiModels: ListModelsFn = async (options) => {
const base = baseUrl(options, 'https://api.openai.com/v1');
const isCustomApi = !OFFICIAL_OPENAI_HOSTNAMES.includes(new URL(base).hostname);
const data = (await getJson(`${base}/models`, bearerHeaders(options), options, 'openai')) as {
data?: IdItem[];
};
return idsToModels(data.data ?? [])
.filter((model) => shouldIncludeOpenAiModel(model.id, isCustomApi))
.sort(byName);
};
@@ -0,0 +1,7 @@
import { makeBearerDataListing } from '../request';
/** Source: LmChatOpenRouter `loadOptions` routing. */
export const listOpenRouterModels = makeBearerDataListing(
'openrouter',
'https://openrouter.ai/api/v1',
);
@@ -0,0 +1,4 @@
import { makeBearerDataListing } from '../request';
/** Source: LmChatVercelAiGateway `loadOptions` routing. */
export const listVercelModels = makeBearerDataListing('vercel', 'https://ai-gateway.vercel.sh/v1');
@@ -0,0 +1,4 @@
import { makeBearerDataListing } from '../request';
/** Source: LmChatXAiGrok `loadOptions` routing. */
export const listXaiModels = makeBearerDataListing('xai', 'https://api.x.ai/v1');
@@ -0,0 +1,62 @@
import type { ListModelsFn, ListModelsOptions, ProviderModel } from './types';
/** GET a provider endpoint and parse JSON, throwing a descriptive error on non-2xx. */
export async function getJson(
url: string,
headers: Record<string, string>,
options: ListModelsOptions,
provider: string,
): Promise<unknown> {
const fetchFn = options.fetch ?? globalThis.fetch;
const response = await fetchFn(url, {
method: 'GET',
headers: { ...headers, ...options.headers },
});
if (!response.ok) {
const body = await response.text().catch(() => '');
throw new Error(
`Failed to list ${provider} models (status ${response.status})${body ? `: ${body.slice(0, 500)}` : ''}`,
);
}
return await response.json();
}
/** Resolve the API base: caller override or the provider default, without a trailing slash. */
export function baseUrl(options: ListModelsOptions, fallback: string): string {
return (options.baseURL ?? fallback).replace(/\/+$/, '');
}
export function bearerHeaders(options: ListModelsOptions): Record<string, string> {
return { Authorization: `Bearer ${options.apiKey}` };
}
export function byName(a: ProviderModel, b: ProviderModel): number {
return a.name.localeCompare(b.name);
}
export interface IdItem {
id?: unknown;
}
/** Map `{ id }` response items to models (name = id), dropping malformed entries. */
export function idsToModels(items: IdItem[]): ProviderModel[] {
return items
.filter((item): item is { id: string } => typeof item.id === 'string' && item.id !== '')
.map((item) => ({ id: item.id, name: item.id }));
}
/**
* The most common provider shape: `GET <base>/models` with bearer auth
* returning `{ data: [{ id }] }`, listed by id ascending.
*/
export function makeBearerDataListing(provider: string, defaultBaseUrl: string): ListModelsFn {
return async (options) => {
const data = (await getJson(
`${baseUrl(options, defaultBaseUrl)}/models`,
bearerHeaders(options),
options,
provider,
)) as { data?: IdItem[] };
return idsToModels(data.data ?? []).sort(byName);
};
}
@@ -0,0 +1,22 @@
/** A chat model as reported by the provider's own model-list API. */
export interface ProviderModel {
/** Provider-native model id, exactly as the provider's API expects it. */
id: string;
/** Human-readable name when the provider supplies one, otherwise the id. */
name: string;
}
export interface ListModelsOptions {
apiKey: string;
/** Override the provider's default API base URL (e.g. a proxy or self-hosted gateway). */
baseURL?: string;
/**
* Transport to use for the request. Callers supply their environment's
* proxy-aware fetch; defaults to the global fetch.
*/
fetch?: typeof globalThis.fetch;
/** Extra headers merged into the request (e.g. custom credential headers). */
headers?: Record<string, string>;
}
export type ListModelsFn = (options: ListModelsOptions) => Promise<ProviderModel[]>;
@@ -54,6 +54,10 @@ export class ListAgentsQueryDto extends Z.class({
sortBy: z.enum(AGENTS_LIST_SORT_OPTIONS).optional(),
}) {}
export class AgentProviderModelsQueryDto extends Z.class({
credentialId: z.string().min(1).max(64).optional(),
}) {}
export class CreateAgentDto extends Z.class({
name: z.string().min(1),
}) {}
@@ -21,3 +21,26 @@ const AGENT_MODEL_PROVIDER_SET = new Set<string>(AGENT_MODEL_PROVIDERS);
export function isAgentModelProvider(provider: string): provider is AgentModelProvider {
return AGENT_MODEL_PROVIDER_SET.has(provider);
}
/** A model offered in the agent model picker. Mirrors the catalog's `ModelInfo` shape. */
export interface AgentCatalogModel {
id: string;
name: string;
releaseDate?: string;
reasoning: boolean;
toolCall: boolean;
cost?: { input: number; output: number; cacheRead?: number; cacheWrite?: number };
limits?: { context?: number; output?: number };
}
/** Response of `GET /agents/v2/catalog/models/:provider`. */
export interface AgentProviderModelsResponse {
provider: string;
/**
* True when the list was confirmed against the provider's own model API
* (only models the credential can actually call). False means the static
* catalog fallback, which may include models the provider has retired.
*/
verified: boolean;
models: AgentCatalogModel[];
}
@@ -1,104 +1,100 @@
import type { ILoadOptionsFunctions } from 'n8n-workflow';
import type { Mocked } from 'vitest';
import { searchModels, type AnthropicModel } from '../searchModels';
import { searchModels } from '../searchModels';
const mockModels = [
{
id: 'claude-3-opus-20240229',
display_name: 'Claude 3 Opus',
type: 'model',
created_at: '2024-02-29T00:00:00Z',
},
{
id: 'claude-3-sonnet-20240229',
display_name: 'Claude 3 Sonnet',
type: 'model',
created_at: '2024-02-29T00:00:00Z',
},
{
id: 'claude-3-haiku-20240307',
display_name: 'Claude 3 Haiku',
type: 'model',
created_at: '2024-03-07T00:00:00Z',
},
{
id: 'claude-2.1',
display_name: 'Claude 2.1',
type: 'model',
created_at: '2023-11-21T00:00:00Z',
},
{
id: 'claude-2.0',
display_name: 'Claude 2.0',
type: 'model',
created_at: '2023-07-11T00:00:00Z',
},
];
describe('searchModels', () => {
let mockContext: Mocked<ILoadOptionsFunctions>;
const mockModels: AnthropicModel[] = [
{
id: 'claude-3-opus-20240229',
display_name: 'Claude 3 Opus',
type: 'model',
created_at: '2024-02-29T00:00:00Z',
},
{
id: 'claude-3-sonnet-20240229',
display_name: 'Claude 3 Sonnet',
type: 'model',
created_at: '2024-02-29T00:00:00Z',
},
{
id: 'claude-3-haiku-20240307',
display_name: 'Claude 3 Haiku',
type: 'model',
created_at: '2024-03-07T00:00:00Z',
},
{
id: 'claude-2.1',
display_name: 'Claude 2.1',
type: 'model',
created_at: '2023-11-21T00:00:00Z',
},
{
id: 'claude-2.0',
display_name: 'Claude 2.0',
type: 'model',
created_at: '2023-07-11T00:00:00Z',
},
];
let fetchSpy: ReturnType<typeof vi.fn>;
beforeEach(() => {
mockContext = {
getCredentials: vi.fn().mockResolvedValue({}),
helpers: {
httpRequestWithAuthentication: vi.fn().mockResolvedValue({
data: mockModels,
}),
},
getCredentials: vi.fn().mockResolvedValue({ apiKey: 'test-api-key' }),
} as unknown as Mocked<ILoadOptionsFunctions>;
fetchSpy = vi.fn().mockResolvedValue({
ok: true,
status: 200,
json: async () => ({ data: mockModels }),
text: async () => '',
});
vi.stubGlobal('fetch', fetchSpy);
});
afterEach(() => {
vi.unstubAllGlobals();
vi.clearAllMocks();
// Reset the getCredentials mock to its default value
mockContext.getCredentials = vi.fn().mockResolvedValue({});
});
it('should fetch models from default Anthropic API URL when no custom URL is provided', async () => {
const result = await searchModels.call(mockContext);
expect(mockContext.getCredentials).toHaveBeenCalledWith('anthropicApi');
expect(mockContext.helpers.httpRequestWithAuthentication).toHaveBeenCalledWith('anthropicApi', {
url: 'https://api.anthropic.com/v1/models',
headers: {
'anthropic-version': '2023-06-01',
},
});
expect(fetchSpy).toHaveBeenCalledWith(
'https://api.anthropic.com/v1/models',
expect.objectContaining({
headers: expect.objectContaining({
'x-api-key': 'test-api-key',
'anthropic-version': '2023-06-01',
}),
}),
);
expect(result.results).toHaveLength(5);
});
it('should fetch models from custom Anthropic API URL when provided in credentials', async () => {
const customUrl = 'https://custom-anthropic-api.example.com';
// Override the default mock to return credentials with a custom URL
mockContext.getCredentials = vi.fn().mockResolvedValue({ url: customUrl });
mockContext.getCredentials = vi
.fn()
.mockResolvedValue({ apiKey: 'test-api-key', url: customUrl });
const result = await searchModels.call(mockContext);
expect(mockContext.getCredentials).toHaveBeenCalledWith('anthropicApi');
expect(mockContext.helpers.httpRequestWithAuthentication).toHaveBeenCalledWith('anthropicApi', {
url: `${customUrl}/v1/models`,
headers: {
'anthropic-version': '2023-06-01',
},
});
expect(fetchSpy).toHaveBeenCalledWith(`${customUrl}/v1/models`, expect.anything());
expect(result.results).toHaveLength(5);
});
it('should use default URL when empty URL is provided in credentials', async () => {
// Override the default mock to return credentials with an empty URL
mockContext.getCredentials = vi.fn().mockResolvedValue({ url: null });
mockContext.getCredentials = vi
.fn()
.mockResolvedValue({ apiKey: 'test-api-key', url: undefined });
const result = await searchModels.call(mockContext);
expect(mockContext.getCredentials).toHaveBeenCalledWith('anthropicApi');
expect(mockContext.helpers.httpRequestWithAuthentication).toHaveBeenCalledWith('anthropicApi', {
url: 'https://api.anthropic.com/v1/models',
headers: {
'anthropic-version': '2023-06-01',
},
});
expect(fetchSpy).toHaveBeenCalledWith('https://api.anthropic.com/v1/models', expect.anything());
expect(result.results).toHaveLength(5);
});
@@ -140,4 +136,25 @@ describe('searchModels', () => {
expect(result.results).toHaveLength(0);
});
it('should include the credential custom header in the request', async () => {
mockContext.getCredentials = vi.fn().mockResolvedValue({
apiKey: 'test-api-key',
header: true,
headerName: 'X-Gateway-Auth',
headerValue: 'gateway-value',
});
await searchModels.call(mockContext);
expect(fetchSpy).toHaveBeenCalledWith(
'https://api.anthropic.com/v1/models',
expect.objectContaining({
headers: expect.objectContaining({
'x-api-key': 'test-api-key',
'X-Gateway-Auth': 'gateway-value',
}),
}),
);
});
});
@@ -1,63 +1,34 @@
import type {
ILoadOptionsFunctions,
INodeListSearchItems,
INodeListSearchResult,
} from 'n8n-workflow';
import { getProxyAgent } from '@n8n/ai-utilities';
import { listAnthropicModels } from '@n8n/ai-utilities/model-discovery';
import type { ILoadOptionsFunctions, INodeListSearchResult } from 'n8n-workflow';
export interface AnthropicModel {
id: string;
display_name: string;
type: string;
created_at: string;
}
import { mergeCustomHeaders } from '../../../../utils/helpers';
export async function searchModels(
this: ILoadOptionsFunctions,
filter?: string,
): Promise<INodeListSearchResult> {
const credentials = await this.getCredentials<{ url?: string }>('anthropicApi');
const credentials = await this.getCredentials('anthropicApi');
const baseURL = (credentials.url as string) ?? 'https://api.anthropic.com';
const baseURL = credentials.url ?? 'https://api.anthropic.com';
const response = (await this.helpers.httpRequestWithAuthentication.call(this, 'anthropicApi', {
url: `${baseURL}/v1/models`,
headers: {
'anthropic-version': '2023-06-01',
},
})) as { data: AnthropicModel[] };
const models = response.data || [];
let results: INodeListSearchItems[] = [];
if (filter) {
for (const model of models) {
if (model.id.toLowerCase().includes(filter.toLowerCase())) {
results.push({
name: model.display_name,
value: model.id,
});
}
}
} else {
results = models.map((model) => ({
name: model.display_name,
value: model.id,
}));
}
// Sort models with more recent ones first (claude-3 before claude-2)
results = results.sort((a, b) => {
const modelA = models.find((m) => m.id === a.value);
const modelB = models.find((m) => m.id === b.value);
if (!modelA || !modelB) return 0;
// Sort by created_at date, most recent first
const dateA = new Date(modelA.created_at);
const dateB = new Date(modelB.created_at);
return dateB.getTime() - dateA.getTime();
// Shared with the agents model catalog: endpoint, auth, and newest-first
// ordering live in @n8n/ai-utilities/model-discovery. The credential's
// optional custom header (for gateway/proxy-backed credentials) is merged in,
// matching what the credential's `authenticate` applies on other requests.
const models = await listAnthropicModels({
apiKey: (credentials.apiKey as string) ?? '',
baseURL,
headers: mergeCustomHeaders(credentials, {}),
fetch: async (url, init) =>
await fetch(url, {
...init,
dispatcher: getProxyAgent(baseURL),
} as RequestInit),
});
return {
results,
results: models
.filter((model) => !filter || model.id.toLowerCase().includes(filter.toLowerCase()))
.map((model) => ({ name: model.name, value: model.id })),
};
}
@@ -1,14 +1,33 @@
import type { ILoadOptionsFunctions } from 'n8n-workflow';
import OpenAI from 'openai';
import type { Mocked, MockedClass } from 'vitest';
import type { Mocked } from 'vitest';
import { searchModels } from '../loadModels';
vi.mock('openai');
const MODEL_IDS = [
'gpt-4',
'gpt-3.5-turbo',
'gpt-3.5-turbo-instruct',
'ft:gpt-3.5-turbo',
'o1-model',
'whisper-1',
'davinci-instruct-beta',
'computer-use-preview',
'whisper-1-preview',
'tts-model',
'other-model',
];
const OFFICIAL_API_RESULTS = [
{ name: 'ft:gpt-3.5-turbo', value: 'ft:gpt-3.5-turbo' },
{ name: 'gpt-3.5-turbo', value: 'gpt-3.5-turbo' },
{ name: 'gpt-4', value: 'gpt-4' },
{ name: 'o1-model', value: 'o1-model' },
{ name: 'other-model', value: 'other-model' },
];
describe('searchModels', () => {
let mockContext: Mocked<ILoadOptionsFunctions>;
let mockOpenAI: Mocked<typeof OpenAI>;
let fetchSpy: ReturnType<typeof vi.fn>;
beforeEach(() => {
mockContext = {
@@ -18,73 +37,41 @@ describe('searchModels', () => {
getNodeParameter: vi.fn().mockReturnValue(''),
} as unknown as Mocked<ILoadOptionsFunctions>;
// Setup OpenAI mock with required properties
const mockOpenAIInstance = {
apiKey: 'test-api-key',
organization: null,
project: null,
_options: {},
models: {
list: vi.fn().mockResolvedValue({
data: [
{ id: 'gpt-4' },
{ id: 'gpt-3.5-turbo' },
{ id: 'gpt-3.5-turbo-instruct' },
{ id: 'ft:gpt-3.5-turbo' },
{ id: 'o1-model' },
{ id: 'whisper-1' },
{ id: 'davinci-instruct-beta' },
{ id: 'computer-use-preview' },
{ id: 'whisper-1-preview' },
{ id: 'tts-model' },
{ id: 'other-model' },
],
}),
},
} as unknown as OpenAI;
(OpenAI as MockedClass<typeof OpenAI>).mockImplementation(function () {
return mockOpenAIInstance;
fetchSpy = vi.fn().mockResolvedValue({
ok: true,
status: 200,
json: async () => ({ data: MODEL_IDS.map((id) => ({ id })) }),
text: async () => '',
});
mockOpenAI = OpenAI as Mocked<typeof OpenAI>;
vi.stubGlobal('fetch', fetchSpy);
});
afterEach(() => {
vi.unstubAllGlobals();
vi.clearAllMocks();
});
it('should return filtered models if custom API endpoint is not provided', async () => {
const result = await searchModels.call(mockContext);
expect(mockOpenAI).toHaveBeenCalledWith(
expect(fetchSpy).toHaveBeenCalledWith(
'https://api.openai.com/v1/models',
expect.objectContaining({
baseURL: 'https://api.openai.com/v1',
apiKey: 'test-api-key',
headers: expect.objectContaining({ Authorization: 'Bearer test-api-key' }),
}),
);
expect(result.results).toEqual([
{ name: 'ft:gpt-3.5-turbo', value: 'ft:gpt-3.5-turbo' },
{ name: 'gpt-3.5-turbo', value: 'gpt-3.5-turbo' },
{ name: 'gpt-4', value: 'gpt-4' },
{ name: 'o1-model', value: 'o1-model' },
{ name: 'other-model', value: 'other-model' },
]);
expect(result.results).toEqual(OFFICIAL_API_RESULTS);
});
it('should initialize OpenAI with correct credentials', async () => {
it('should use the credential url as the API base', async () => {
mockContext.getCredentials.mockResolvedValueOnce({
apiKey: 'test-api-key',
url: 'https://test-url.com',
});
await searchModels.call(mockContext);
expect(mockOpenAI).toHaveBeenCalledWith(
expect.objectContaining({
baseURL: 'https://test-url.com',
apiKey: 'test-api-key',
}),
);
expect(fetchSpy).toHaveBeenCalledWith('https://test-url.com/models', expect.anything());
});
it('should use default OpenAI URL if no custom URL provided', async () => {
@@ -94,18 +81,15 @@ describe('searchModels', () => {
await searchModels.call(mockContext);
expect(mockOpenAI).toHaveBeenCalledWith(
expect.objectContaining({
baseURL: 'https://api.openai.com/v1',
apiKey: 'test-api-key',
}),
);
expect(fetchSpy).toHaveBeenCalledWith('https://api.openai.com/v1/models', expect.anything());
});
it('should include all models for custom API endpoints', async () => {
mockContext.getNodeParameter = vi.fn().mockReturnValue('https://custom-api.com');
const result = await searchModels.call(mockContext);
expect(fetchSpy).toHaveBeenCalledWith('https://custom-api.com/models', expect.anything());
expect(result.results).toEqual([
{ name: 'computer-use-preview', value: 'computer-use-preview' },
{ name: 'davinci-instruct-beta', value: 'davinci-instruct-beta' },
@@ -130,13 +114,7 @@ describe('searchModels', () => {
const result = await searchModels.call(mockContext);
expect(result.results).toEqual([
{ name: 'ft:gpt-3.5-turbo', value: 'ft:gpt-3.5-turbo' },
{ name: 'gpt-3.5-turbo', value: 'gpt-3.5-turbo' },
{ name: 'gpt-4', value: 'gpt-4' },
{ name: 'o1-model', value: 'o1-model' },
{ name: 'other-model', value: 'other-model' },
]);
expect(result.results).toEqual(OFFICIAL_API_RESULTS);
});
it('should filter models based on search term', async () => {
@@ -159,7 +137,7 @@ describe('searchModels', () => {
]);
});
it('should include custom credential headers in the OpenAI client', async () => {
it('should include custom credential headers in the request', async () => {
mockContext.getCredentials.mockResolvedValueOnce({
apiKey: 'test-api-key',
header: true,
@@ -169,48 +147,11 @@ describe('searchModels', () => {
await searchModels.call(mockContext);
expect(mockOpenAI).toHaveBeenCalledWith(
expect(fetchSpy).toHaveBeenCalledWith(
'https://api.openai.com/v1/models',
expect.objectContaining({
defaultHeaders: expect.objectContaining({
'X-Custom-Auth': 'custom-value',
}),
headers: expect.objectContaining({ 'X-Custom-Auth': 'custom-value' }),
}),
);
});
it('should return models sorted alphabetically by id', async () => {
// Setup a mock with scrambled order
const mockUnsortedInstance = {
apiKey: 'test-api-key',
models: {
list: vi.fn().mockResolvedValue({
data: [
{ id: 'gpt-4' },
{ id: 'a-model' },
{ id: 'o1-model' },
{ id: 'gpt-3.5-turbo' },
{ id: 'z-model' },
],
}),
},
} as unknown as OpenAI;
(OpenAI as MockedClass<typeof OpenAI>).mockImplementation(function () {
return mockUnsortedInstance;
});
// Custom API endpoint to include all models
mockContext.getNodeParameter = vi.fn().mockReturnValue('https://custom-api.com');
const result = await searchModels.call(mockContext);
// Verify the results are sorted alphabetically
expect(result.results).toEqual([
{ name: 'a-model', value: 'a-model' },
{ name: 'gpt-3.5-turbo', value: 'gpt-3.5-turbo' },
{ name: 'gpt-4', value: 'gpt-4' },
{ name: 'o1-model', value: 'o1-model' },
{ name: 'z-model', value: 'z-model' },
]);
});
});
@@ -1,11 +1,10 @@
import { getProxyAgent } from '@n8n/ai-utilities';
import { listOpenAiModels } from '@n8n/ai-utilities/model-discovery';
import { AiConfig } from '@n8n/config';
import { Container } from '@n8n/di';
import type { ILoadOptionsFunctions, INodeListSearchResult } from 'n8n-workflow';
import OpenAI from 'openai';
import { mergeCustomHeaders } from '../../../../utils/helpers';
import { shouldIncludeModel } from '../../../vendors/OpenAi/helpers/modelFiltering';
export async function searchModels(
this: ILoadOptionsFunctions,
@@ -17,35 +16,24 @@ export async function searchModels(
(credentials.url as string) ||
'https://api.openai.com/v1';
const { openAiDefaultHeaders } = Container.get(AiConfig);
const defaultHeaders = mergeCustomHeaders(credentials, openAiDefaultHeaders ?? {});
const headers = mergeCustomHeaders(credentials, openAiDefaultHeaders ?? {});
const openai = new OpenAI({
baseURL,
// Shared with the agents model catalog: endpoint, auth, chat-model filtering
// (including include-all on custom hosts) live in @n8n/ai-utilities/model-discovery.
const models = await listOpenAiModels({
apiKey: credentials.apiKey as string,
fetchOptions: {
dispatcher: getProxyAgent(baseURL),
},
defaultHeaders,
baseURL,
headers,
fetch: async (url, init) =>
await fetch(url, {
...init,
dispatcher: getProxyAgent(baseURL),
} as RequestInit),
});
const { data: models = [] } = await openai.models.list();
const url = baseURL && new URL(baseURL);
const isCustomAPI = !!(url && !['api.openai.com', 'ai-assistant.n8n.io'].includes(url.hostname));
const filteredModels = models.filter((model: { id: string }) => {
const includeModel = shouldIncludeModel(model.id, isCustomAPI);
if (!filter) return includeModel;
return includeModel && model.id.toLowerCase().includes(filter.toLowerCase());
});
filteredModels.sort((a, b) => a.id.localeCompare(b.id));
return {
results: filteredModels.map((model: { id: string }) => ({
name: model.id,
value: model.id,
})),
results: models
.filter((model) => !filter || model.id.toLowerCase().includes(filter.toLowerCase()))
.map((model) => ({ name: model.id, value: model.id })),
};
}
@@ -419,20 +419,31 @@ describe('LmChatAnthropic', () => {
describe('searchModels', () => {
let mockLoadContext: ILoadOptionsFunctions;
let mockGetCredentials: Mock;
let mockHttpRequest: Mock;
let fetchSpy: Mock;
beforeEach(() => {
mockGetCredentials = vi.fn();
mockHttpRequest = vi.fn();
fetchSpy = vi.fn();
vi.stubGlobal('fetch', fetchSpy);
mockLoadContext = {
getCredentials: mockGetCredentials,
helpers: {
httpRequestWithAuthentication: mockHttpRequest,
},
} as unknown as ILoadOptionsFunctions;
});
afterEach(() => {
vi.unstubAllGlobals();
});
function mockModelsResponse(models: unknown[]) {
fetchSpy.mockResolvedValue({
ok: true,
status: 200,
json: async () => ({ data: models }),
text: async () => '',
});
}
it('should return all models sorted by creation date', async () => {
const mockModels = [
{
@@ -455,20 +466,21 @@ describe('LmChatAnthropic', () => {
},
];
mockGetCredentials.mockResolvedValue({});
mockHttpRequest.mockResolvedValue({
data: mockModels,
});
mockGetCredentials.mockResolvedValue({ apiKey: 'test-api-key' });
mockModelsResponse(mockModels);
const { searchModels } = lmChatAnthropic.methods.listSearch;
const result = await searchModels.call(mockLoadContext);
expect(mockHttpRequest).toHaveBeenCalledWith('anthropicApi', {
url: 'https://api.anthropic.com/v1/models',
headers: {
'anthropic-version': '2023-06-01',
},
});
expect(fetchSpy).toHaveBeenCalledWith(
'https://api.anthropic.com/v1/models',
expect.objectContaining({
headers: expect.objectContaining({
'x-api-key': 'test-api-key',
'anthropic-version': '2023-06-01',
}),
}),
);
expect(result.results).toHaveLength(3);
// Verify sorted by creation date (newest first)
@@ -499,10 +511,8 @@ describe('LmChatAnthropic', () => {
},
];
mockGetCredentials.mockResolvedValue({});
mockHttpRequest.mockResolvedValue({
data: mockModels,
});
mockGetCredentials.mockResolvedValue({ apiKey: 'test-api-key' });
mockModelsResponse(mockModels);
const { searchModels } = lmChatAnthropic.methods.listSearch;
const result = await searchModels.call(mockLoadContext, 'opus');
@@ -522,10 +532,8 @@ describe('LmChatAnthropic', () => {
},
];
mockGetCredentials.mockResolvedValue({});
mockHttpRequest.mockResolvedValue({
data: mockModels,
});
mockGetCredentials.mockResolvedValue({ apiKey: 'test-api-key' });
mockModelsResponse(mockModels);
const { searchModels } = lmChatAnthropic.methods.listSearch;
const result = await searchModels.call(mockLoadContext, 'SONNET');
@@ -538,28 +546,20 @@ describe('LmChatAnthropic', () => {
const customURL = 'https://custom-anthropic.example.com';
mockGetCredentials.mockResolvedValue({
apiKey: 'test-api-key',
url: customURL,
});
mockHttpRequest.mockResolvedValue({
data: [],
});
mockModelsResponse([]);
const { searchModels } = lmChatAnthropic.methods.listSearch;
await searchModels.call(mockLoadContext);
expect(mockHttpRequest).toHaveBeenCalledWith('anthropicApi', {
url: `${customURL}/v1/models`,
headers: {
'anthropic-version': '2023-06-01',
},
});
expect(fetchSpy).toHaveBeenCalledWith(`${customURL}/v1/models`, expect.anything());
});
it('should handle empty model list', async () => {
mockGetCredentials.mockResolvedValue({});
mockHttpRequest.mockResolvedValue({
data: [],
});
mockGetCredentials.mockResolvedValue({ apiKey: 'test-api-key' });
mockModelsResponse([]);
const { searchModels } = lmChatAnthropic.methods.listSearch;
const result = await searchModels.call(mockLoadContext);
@@ -0,0 +1,219 @@
import type { User } from '@n8n/db';
import { mockLogger } from '@n8n/backend-test-utils';
import { mock } from 'vitest-mock-extended';
import { AgentModelCatalogService } from '../agent-model-catalog.service';
import type { BuilderModelLiveLookupService } from '../builder/builder-model-live-lookup.service';
const fetchProviderCatalog = vi.fn();
vi.mock('@n8n/agents', () => ({
fetchProviderCatalog: (...args: unknown[]) => fetchProviderCatalog(...args) as unknown,
}));
const user = mock<User>({ id: 'user-1' });
const credentialId = 'cred-1';
const catalogFixture = {
anthropic: {
id: 'anthropic',
name: 'Anthropic',
models: {
'claude-sonnet-4-6': {
id: 'claude-sonnet-4-6',
name: 'Claude Sonnet 4.6',
reasoning: true,
toolCall: true,
cost: { input: 3, output: 15 },
},
'claude-opus-4-0': {
id: 'claude-opus-4-0',
name: 'Claude Opus 4',
reasoning: true,
toolCall: true,
},
},
},
google: {
id: 'google',
name: 'Google',
models: {
'gemini-2.5-flash': {
id: 'gemini-2.5-flash',
name: 'Gemini 2.5 Flash',
reasoning: true,
toolCall: true,
},
},
},
};
function makeService() {
const lookupService = mock<BuilderModelLiveLookupService>();
const service = new AgentModelCatalogService(mockLogger(), lookupService);
return { service, lookupService };
}
describe('AgentModelCatalogService', () => {
beforeEach(() => {
fetchProviderCatalog.mockReset();
fetchProviderCatalog.mockResolvedValue(catalogFixture);
});
it('keeps catalog models the provider still reports live, with catalog metadata, and prunes the rest', async () => {
const { service, lookupService } = makeService();
// Provider reports Sonnet but not Opus — Opus (retired) must be pruned.
lookupService.list.mockResolvedValue([
{ name: 'Claude Sonnet 4.6', value: 'claude-sonnet-4-6' },
]);
const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId);
expect(result.verified).toBe(true);
expect(result.models).toHaveLength(1);
expect(result.models[0]).toMatchObject({
id: 'claude-sonnet-4-6',
name: 'Claude Sonnet 4.6',
cost: { input: 3, output: 15 },
});
expect(lookupService.list).toHaveBeenCalledWith(
user,
'project-1',
credentialId,
'anthropicApi',
'anthropic',
);
});
it('verifies a catalog alias when the provider lists only its dated snapshot', async () => {
const { service, lookupService } = makeService();
fetchProviderCatalog.mockResolvedValue({
anthropic: {
id: 'anthropic',
name: 'Anthropic',
models: {
// models.dev keeps the versionless alias and drops the snapshot…
'claude-haiku-4-5': {
id: 'claude-haiku-4-5',
name: 'Claude Haiku 4.5',
reasoning: true,
toolCall: true,
},
'claude-opus-4-0': {
id: 'claude-opus-4-0',
name: 'Claude Opus 4',
reasoning: true,
toolCall: true,
},
},
},
});
// …while Anthropic's API lists the dated snapshot only. The snapshot must
// verify its alias; the retired alias with no live counterpart stays pruned.
lookupService.list.mockResolvedValue([
{ name: 'Claude Haiku 4.5', value: 'claude-haiku-4-5-20251001' },
]);
const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId);
expect(result.verified).toBe(true);
expect(result.models.map((m) => m.id)).toEqual(['claude-haiku-4-5']);
});
it('does not add live models that are missing from the catalog', async () => {
const { service, lookupService } = makeService();
// Live list includes a model models.dev has no entry for, alongside a known one.
lookupService.list.mockResolvedValue([
{ name: 'Claude Sonnet 4.6', value: 'claude-sonnet-4-6' },
{ name: 'Claude Brand New', value: 'claude-brand-new' },
]);
const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId);
expect(result.verified).toBe(true);
// Only the catalog-known model survives; the live-only one is never added.
expect(result.models.map((m) => m.id)).toEqual(['claude-sonnet-4-6']);
});
it('strips the "models/" prefix from google model ids before matching', async () => {
const { service, lookupService } = makeService();
lookupService.list.mockResolvedValue([
{ name: 'models/gemini-2.5-flash', value: 'models/gemini-2.5-flash' },
]);
const result = await service.getProviderModels(user, 'project-1', 'google', credentialId);
expect(result.models).toEqual([
expect.objectContaining({ id: 'gemini-2.5-flash', name: 'Gemini 2.5 Flash' }),
]);
});
it('falls back to the catalog list (verified: false) when the live lookup fails', async () => {
const { service, lookupService } = makeService();
lookupService.list.mockRejectedValue(new Error('provider is down'));
const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId);
expect(result.verified).toBe(false);
expect(result.models.map((m) => m.id).sort()).toEqual(['claude-opus-4-0', 'claude-sonnet-4-6']);
});
it('falls back to the catalog list (verified: false) when no credential is provided', async () => {
const { service, lookupService } = makeService();
const result = await service.getProviderModels(user, 'project-1', 'anthropic', undefined);
expect(result.verified).toBe(false);
expect(result.models).toHaveLength(2);
expect(lookupService.list).not.toHaveBeenCalled();
});
it('falls back to the catalog list for providers without a live lookup', async () => {
const { service, lookupService } = makeService();
fetchProviderCatalog.mockResolvedValue({
'aws-bedrock': {
id: 'aws-bedrock',
name: 'AWS Bedrock',
models: {
'anthropic.claude-sonnet-4-6-v1:0': {
id: 'anthropic.claude-sonnet-4-6-v1:0',
name: 'Claude Sonnet 4.6',
reasoning: true,
toolCall: true,
},
},
},
});
const result = await service.getProviderModels(user, 'project-1', 'aws-bedrock', credentialId);
expect(result.verified).toBe(false);
expect(result.models).toHaveLength(1);
expect(lookupService.list).not.toHaveBeenCalled();
});
it('still returns live models (verified: true) when the catalog fetch fails', async () => {
const { service, lookupService } = makeService();
fetchProviderCatalog.mockRejectedValue(new Error('models.dev unreachable'));
lookupService.list.mockResolvedValue([
{ name: 'Claude Sonnet 4.6', value: 'claude-sonnet-4-6' },
]);
const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId);
expect(result.verified).toBe(true);
expect(result.models).toEqual([
expect.objectContaining({ id: 'claude-sonnet-4-6', name: 'Claude Sonnet 4.6' }),
]);
});
it('returns an empty unverified list when both the lookup and the catalog fail', async () => {
const { service, lookupService } = makeService();
fetchProviderCatalog.mockRejectedValue(new Error('models.dev unreachable'));
lookupService.list.mockRejectedValue(new Error('provider is down'));
const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId);
expect(result.verified).toBe(false);
expect(result.models).toEqual([]);
});
});
@@ -32,7 +32,7 @@ import {
AgentsBuilderToolsService,
getAgentConfigHash,
} from '../builder/agents-builder-tools.service';
import type { BuilderModelLookupService } from '../builder/builder-model-lookup.service';
import type { BuilderModelLiveLookupService } from '../builder/builder-model-live-lookup.service';
import { BUILDER_TOOLS } from '../builder/builder-tool-names';
import type { Agent } from '../entities/agent.entity';
import type { AgentRepository } from '../repositories/agent.repository';
@@ -69,7 +69,7 @@ function makeService() {
const secureRuntime = mock<AgentSecureRuntime>();
const workflowRepository = mock<WorkflowRepository>();
const agentsToolsService = mock<AgentsToolsService>();
const builderModelLookupService = mock<BuilderModelLookupService>();
const builderModelLiveLookupService = mock<BuilderModelLiveLookupService>();
const credentialTypes = mock<CredentialTypes>();
const mcpRegistryService = mock<McpRegistryService>();
const agentTaskService = mock<AgentTaskService>();
@@ -97,7 +97,7 @@ function makeService() {
secureRuntime,
workflowRepository,
agentsToolsService,
builderModelLookupService,
builderModelLiveLookupService,
mcpRegistryService,
mock(),
credentialTypes,
@@ -1,3 +1,12 @@
import type { AgentProviderModelsQueryDto } from '@n8n/api-types';
import type { AuthenticatedRequest } from '@n8n/db';
import type { Response } from 'express';
import { mock } from 'vitest-mock-extended';
import { BadRequestError } from '@/errors/response-errors/bad-request.error';
import type { AgentIntegrationPersistenceService } from '../agent-integration-persistence.service';
import type { AgentModelCatalogService } from '../agent-model-catalog.service';
import { AgentsCatalogController } from '../agents-catalog.controller';
import {
expectProjectScopedAgentRoutes,
@@ -11,8 +20,56 @@ describe('AgentsCatalogController route access scopes', () => {
it.each([
['getModelCatalog', 'agent:read'],
['getProviderModels', 'agent:read'],
['listIntegrations', 'agent:read'],
])('%s uses %s', (handlerName, scope) => {
expect(routes.get(handlerName)?.accessScope?.scope).toBe(scope);
});
});
describe('AgentsCatalogController — getProviderModels', () => {
function makeController() {
const modelCatalogService = mock<AgentModelCatalogService>();
const controller = new AgentsCatalogController(
mock<AgentIntegrationPersistenceService>(),
modelCatalogService,
);
return { controller, modelCatalogService };
}
const req = mock<AuthenticatedRequest<{ projectId: string }>>({
user: { id: 'user-1' },
params: { projectId: 'project-1' },
});
it('delegates to the model catalog service with the user, project, and credential', async () => {
const { controller, modelCatalogService } = makeController();
const response = { provider: 'anthropic', verified: true, models: [] };
modelCatalogService.getProviderModels.mockResolvedValue(response);
const result = await controller.getProviderModels(req, mock<Response>(), 'anthropic', {
credentialId: 'cred-1',
} as AgentProviderModelsQueryDto);
expect(result).toBe(response);
expect(modelCatalogService.getProviderModels).toHaveBeenCalledWith(
req.user,
'project-1',
'anthropic',
'cred-1',
);
});
it('rejects unknown providers', async () => {
const { controller } = makeController();
await expect(
controller.getProviderModels(
req,
mock<Response>(),
'not-a-provider',
{} as AgentProviderModelsQueryDto,
),
).rejects.toThrow(BadRequestError);
});
});
@@ -0,0 +1,144 @@
import type { AgentCatalogModel, AgentProviderModelsResponse } from '@n8n/api-types';
import { Logger } from '@n8n/backend-common';
import type { User } from '@n8n/db';
import { Service } from '@n8n/di';
import { isModelDiscoveryProvider } from '@n8n/ai-utilities/model-discovery';
import { BuilderModelLiveLookupService } from './builder/builder-model-live-lookup.service';
import { LLM_PROVIDER_DEFAULTS } from './builder/interactive/llm-provider-defaults';
/** Google's models API returns ids as `models/<id>`; the AI SDK expects the bare id. */
const GOOGLE_MODEL_ID_PREFIX = 'models/';
function getProviderCredentialType(provider: string): string | undefined {
if (!isModelDiscoveryProvider(provider)) return undefined;
for (const [credentialType, entry] of Object.entries(LLM_PROVIDER_DEFAULTS)) {
if (entry.provider === provider) return credentialType;
}
return undefined;
}
function normalizeLiveModelValue(provider: string, value: string): string {
if (provider === 'google' && value.startsWith(GOOGLE_MODEL_ID_PREFIX)) {
return value.slice(GOOGLE_MODEL_ID_PREFIX.length);
}
return value;
}
/** Dated snapshot suffixes: Anthropic `-20251001`, OpenAI `-2024-08-06`. */
const SNAPSHOT_SUFFIX = /-(?:\d{8}|\d{4}-\d{2}-\d{2})$/;
/**
* The ids a live model verifies. Providers list older models only as dated
* snapshots (e.g. `claude-haiku-4-5-20251001`) while the catalog prefers the
* versionless alias (`claude-haiku-4-5`, which providers resolve to the latest
* snapshot) — so a listed snapshot also verifies its alias. A retired alias
* still prunes: retired models have no live snapshot either.
*/
function liveModelIdVariants(id: string): string[] {
const alias = id.replace(SNAPSHOT_SUFFIX, '');
return alias === id ? [id] : [id, alias];
}
/**
* Builds the model list offered in the agent model picker for one provider.
*
* The curated models.dev catalog is the display list — it is the up-to-date set
* of chat models with names, cost, and limits. But it lags provider
* retirements, so a listed model can 404 at call time. When a credential is
* available we verify the catalog against the provider's own model API (via the
* shared `@n8n/ai-utilities/model-discovery` functions) and prune any catalog
* entry the provider no longer reports. We never add live-only
* models: provider `/models` endpoints return every variant/snapshot and would
* overload the picker. Without a credential, or when the provider has no list
* API wired up, the catalog list is returned unpruned with `verified: false`.
*/
@Service()
export class AgentModelCatalogService {
constructor(
private readonly logger: Logger,
private readonly builderModelLiveLookupService: BuilderModelLiveLookupService,
) {}
async getProviderModels(
user: User,
projectId: string,
provider: string,
credentialId?: string,
): Promise<AgentProviderModelsResponse> {
const catalogModels = await this.getCatalogModels(provider);
const credentialType = getProviderCredentialType(provider);
if (!credentialId || !credentialType) {
return { provider, verified: false, models: Object.values(catalogModels) };
}
let liveModels: Array<{ name: string; value: string }>;
try {
liveModels = await this.builderModelLiveLookupService.list(
user,
projectId,
credentialId,
credentialType,
provider,
);
} catch (error) {
this.logger.warn('Live model list failed — falling back to the static catalog', {
provider,
error: error instanceof Error ? error.message : String(error),
});
return { provider, verified: false, models: Object.values(catalogModels) };
}
const liveModelIds = new Set(
liveModels.flatMap((live) =>
liveModelIdVariants(normalizeLiveModelValue(provider, live.value)),
),
);
const catalogList = Object.values(catalogModels);
// models.dev is the curated display list; the live lookup only verifies it.
// Provider `/models` endpoints return every variant/snapshot, so we never
// add live-only models — we only prune catalog entries the provider no
// longer reports (retired ids that would 404 at call time).
if (catalogList.length > 0) {
return {
provider,
verified: true,
models: catalogList.filter((model) => liveModelIds.has(model.id)),
};
}
// Catalog unavailable (models.dev down or no entry for this provider): there
// is no curated list to prune against, so show the verified live list rather
// than an empty picker.
return {
provider,
verified: true,
models: liveModels.map((live) => {
const id = normalizeLiveModelValue(provider, live.value);
return {
id,
name: normalizeLiveModelValue(provider, live.name) || id,
reasoning: false,
toolCall: true,
};
}),
};
}
private async getCatalogModels(provider: string): Promise<Record<string, AgentCatalogModel>> {
try {
const { fetchProviderCatalog } = await import('@n8n/agents');
const catalog = await fetchProviderCatalog();
return catalog[provider]?.models ?? {};
} catch (error) {
this.logger.warn('Model catalog fetch failed', {
provider,
error: error instanceof Error ? error.message : String(error),
});
return {};
}
}
}
@@ -1,13 +1,24 @@
import type { ChatIntegrationDescriptor } from '@n8n/api-types';
import { Get, ProjectScope, RestController } from '@n8n/decorators';
import {
AgentProviderModelsQueryDto,
isAgentModelProvider,
type AgentProviderModelsResponse,
type ChatIntegrationDescriptor,
} from '@n8n/api-types';
import type { AuthenticatedRequest } from '@n8n/db';
import { Get, Param, ProjectScope, Query, RestController } from '@n8n/decorators';
import type { Response } from 'express';
import { BadRequestError } from '@/errors/response-errors/bad-request.error';
import { AgentIntegrationPersistenceService } from './agent-integration-persistence.service';
import { AgentModelCatalogService } from './agent-model-catalog.service';
import { filterOfferedAgentModelProviders } from './model-catalog';
@RestController('/projects/:projectId/agents/v2')
export class AgentsCatalogController {
constructor(
private readonly agentIntegrationPersistenceService: AgentIntegrationPersistenceService,
private readonly agentModelCatalogService: AgentModelCatalogService,
) {}
@Get('/catalog/models')
@@ -17,6 +28,25 @@ export class AgentsCatalogController {
return filterOfferedAgentModelProviders(await fetchProviderCatalog());
}
@Get('/catalog/models/:provider')
@ProjectScope('agent:read')
async getProviderModels(
req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Param('provider') provider: string,
@Query query: AgentProviderModelsQueryDto,
): Promise<AgentProviderModelsResponse> {
if (!isAgentModelProvider(provider)) {
throw new BadRequestError(`Unknown model provider "${provider}"`);
}
return await this.agentModelCatalogService.getProviderModels(
req.user,
req.params.projectId,
provider,
query.credentialId,
);
}
@Get('/catalog/integrations')
@ProjectScope('agent:read')
listIntegrations(): ChatIntegrationDescriptor[] {
@@ -0,0 +1,108 @@
import type { CustomFetch, HttpTransport, OutboundHttp } from '@n8n/backend-network';
import type { CredentialsEntity, User } from '@n8n/db';
import { mock } from 'vitest-mock-extended';
import type { CredentialsFinderService } from '@/credentials/credentials-finder.service';
import type { CredentialsService } from '@/credentials/credentials.service';
import { BuilderModelLiveLookupService } from '../builder-model-live-lookup.service';
const listModelsForProvider = vi.fn();
vi.mock('@n8n/ai-utilities/model-discovery', () => ({
listModelsForProvider: (...args: unknown[]) => listModelsForProvider(...args) as unknown,
}));
const user = mock<User>({ id: 'user-1' });
const projectId = 'project-1';
function makeService() {
const credentialsService = mock<CredentialsService>();
const credentialsFinderService = mock<CredentialsFinderService>();
const transport = mock<HttpTransport>();
transport.asCustomFetch.mockReturnValue(vi.fn() as unknown as CustomFetch);
const outboundHttp = mock<OutboundHttp>();
outboundHttp.transport.mockReturnValue(transport);
const service = new BuilderModelLiveLookupService(
credentialsService,
credentialsFinderService,
outboundHttp,
);
return { service, credentialsService, credentialsFinderService };
}
function usable(id: string, type: string) {
return [{ id, name: 'My Credential', type }] as Awaited<
ReturnType<CredentialsService['getCredentialsAUserCanUseInAWorkflow']>
>;
}
describe('BuilderModelLiveLookupService', () => {
beforeEach(() => {
listModelsForProvider.mockReset();
});
it('lists models for a credential the user can use in the project', async () => {
const { service, credentialsService, credentialsFinderService } = makeService();
credentialsService.getCredentialsAUserCanUseInAWorkflow.mockResolvedValue(
usable('cred-1', 'anthropicApi'),
);
credentialsFinderService.findCredentialById.mockResolvedValue(mock<CredentialsEntity>());
credentialsService.decrypt.mockResolvedValue({ apiKey: 'sk-key', url: 'https://proxy.local' });
listModelsForProvider.mockResolvedValue([
{ id: 'claude-sonnet-4-6', name: 'Claude Sonnet 4.6' },
]);
const result = await service.list(user, projectId, 'cred-1', 'anthropicApi', 'anthropic');
expect(result).toEqual([{ name: 'Claude Sonnet 4.6', value: 'claude-sonnet-4-6' }]);
expect(credentialsService.getCredentialsAUserCanUseInAWorkflow).toHaveBeenCalledWith(user, {
projectId,
});
// Credential fields are mapped for the provider (anthropic: apiKey + url→baseURL).
expect(listModelsForProvider).toHaveBeenCalledWith(
'anthropic',
expect.objectContaining({ apiKey: 'sk-key', baseURL: 'https://proxy.local' }),
);
});
it('treats an empty provider response as a failed lookup', async () => {
const { service, credentialsService, credentialsFinderService } = makeService();
credentialsService.getCredentialsAUserCanUseInAWorkflow.mockResolvedValue(
usable('cred-1', 'anthropicApi'),
);
credentialsFinderService.findCredentialById.mockResolvedValue(mock<CredentialsEntity>());
credentialsService.decrypt.mockResolvedValue({ apiKey: 'sk-key' });
// An empty list from a chat provider is far more likely a broken request
// or drifted response shape than a real zero-model account — callers must
// fall back rather than prune everything.
listModelsForProvider.mockResolvedValue([]);
await expect(
service.list(user, projectId, 'cred-1', 'anthropicApi', 'anthropic'),
).rejects.toThrow('returned no models');
});
it('rejects a credential that is not available in the project', async () => {
const { service, credentialsService } = makeService();
// The user can read this credential, but it is not in the project's set.
credentialsService.getCredentialsAUserCanUseInAWorkflow.mockResolvedValue([]);
await expect(
service.list(user, projectId, 'cred-other-project', 'anthropicApi', 'anthropic'),
).rejects.toThrow('not found or not accessible');
expect(listModelsForProvider).not.toHaveBeenCalled();
});
it('rejects a credential whose type does not match the provider', async () => {
const { service, credentialsService } = makeService();
credentialsService.getCredentialsAUserCanUseInAWorkflow.mockResolvedValue(
usable('cred-1', 'openAiApi'),
);
await expect(
service.list(user, projectId, 'cred-1', 'anthropicApi', 'anthropic'),
).rejects.toThrow('not found or not accessible');
expect(listModelsForProvider).not.toHaveBeenCalled();
});
});
@@ -43,7 +43,7 @@ import { AgentSkillsService } from '../agent-skills.service';
import { AgentTaskService } from '../agent-task.service';
import { AgentsToolsService } from '../agents-tools.service';
import { AgentsService } from '../agents.service';
import { BuilderModelLookupService } from './builder-model-lookup.service';
import { BuilderModelLiveLookupService } from './builder-model-live-lookup.service';
import { BUILDER_TOOLS } from './builder-tool-names';
import {
collectFromAiParameterReferences,
@@ -311,7 +311,7 @@ export class AgentsBuilderToolsService {
private readonly secureRuntime: AgentSecureRuntime,
private readonly workflowRepository: WorkflowRepository,
private readonly agentsToolsService: AgentsToolsService,
private readonly builderModelLookupService: BuilderModelLookupService,
private readonly builderModelLiveLookupService: BuilderModelLiveLookupService,
private readonly mcpRegistryService: McpRegistryService,
private readonly oauthService: OauthService,
private readonly credentialTypes: CredentialTypes,
@@ -671,8 +671,14 @@ export class AgentsBuilderToolsService {
.build();
const modelLookup: ModelLookup = {
list: async (credentialId, credentialType, lookup) =>
await this.builderModelLookupService.list(user, credentialId, credentialType, lookup),
list: async (credentialId, credentialType, provider) =>
await this.builderModelLiveLookupService.list(
user,
projectId,
credentialId,
credentialType,
provider,
),
};
const tools: BuiltTool[] = [
@@ -0,0 +1,78 @@
import { OutboundHttp } from '@n8n/backend-network';
import type { User } from '@n8n/db';
import { Service } from '@n8n/di';
import { CredentialsFinderService } from '@/credentials/credentials-finder.service';
import { CredentialsService } from '@/credentials/credentials.service';
import { createAiProxyFetch } from '@/utils/ai-proxy-fetch';
import { mapCredentialForProvider } from '../json-config/credential-field-mapping';
/**
* Fetches a provider's live chat-model list for a credential, via the shared
* `@n8n/ai-utilities/model-discovery` functions (the same provider knowledge
* that backs the chat sub-nodes' model dropdowns). Nothing from
* `@n8n/n8n-nodes-langchain` is loaded on this path.
*
* The credential must be usable by the user within the given project the
* same set as the workflow editor's credential picker.
*/
@Service()
export class BuilderModelLiveLookupService {
constructor(
private readonly credentialsService: CredentialsService,
private readonly credentialsFinderService: CredentialsFinderService,
private readonly outboundHttp: OutboundHttp,
) {}
/**
* Returns `{ name, value }` pairs (value = the provider's model id, exactly
* as the provider API expects it). Throws if the credential is not usable by
* the user in the project, its type doesn't match, or the provider has no
* model discovery support.
*/
async list(
user: User,
projectId: string,
credentialId: string,
credentialType: string,
provider: string,
): Promise<Array<{ name: string; value: string }>> {
const usableCredentials = await this.credentialsService.getCredentialsAUserCanUseInAWorkflow(
user,
{ projectId },
);
const usable = usableCredentials.find((c) => c.id === credentialId);
if (!usable || usable.type !== credentialType) {
throw new Error(`Credential ${credentialId} not found or not accessible`);
}
const credential = await this.credentialsFinderService.findCredentialById(credentialId);
if (!credential) {
throw new Error(`Credential ${credentialId} not found or not accessible`);
}
const rawData = await this.credentialsService.decrypt(credential, true);
const { listModelsForProvider } = await import('@n8n/ai-utilities/model-discovery');
const mapped = mapCredentialForProvider(provider, { apiKey: '', ...rawData });
const apiKey = typeof mapped.apiKey === 'string' ? mapped.apiKey : '';
const baseURL =
typeof mapped.baseURL === 'string' && mapped.baseURL ? mapped.baseURL : undefined;
const models = await listModelsForProvider(provider, {
apiKey,
baseURL,
fetch: createAiProxyFetch(this.outboundHttp) as typeof globalThis.fetch,
});
// Every supported chat provider offers models, so an empty list means a
// broken request or a drifted response shape, not a zero-model account.
// Throw so callers fall back (unverified catalog / lookup-failed) instead
// of treating "nothing" as a verified answer and pruning every model.
if (models.length === 0) {
throw new Error(`Provider ${provider} returned no models`);
}
return models.map((model) => ({ name: model.name, value: model.id }));
}
}
@@ -1,82 +0,0 @@
import type { User } from '@n8n/db';
import { ProjectRepository } from '@n8n/db';
import { Service } from '@n8n/di';
import type { INodeCredentials, INodeParameters } from 'n8n-workflow';
import { CredentialsFinderService } from '@/credentials/credentials-finder.service';
import { NodeTypes } from '@/node-types';
import { DynamicNodeParametersService } from '@/services/dynamic-node-parameters.service';
import { getBase } from '@/workflow-execute-additional-data';
import type { ModelLookupConfig } from './interactive/llm-provider-defaults';
@Service()
export class BuilderModelLookupService {
constructor(
private readonly credentialsFinderService: CredentialsFinderService,
private readonly projectRepository: ProjectRepository,
private readonly dynamicNodeParametersService: DynamicNodeParametersService,
private readonly nodeTypes: NodeTypes,
) {}
async list(
user: User,
credentialId: string,
credentialType: string,
lookup: ModelLookupConfig,
): Promise<Array<{ name: string; value: string }>> {
const credential = await this.credentialsFinderService.findCredentialForUser(
credentialId,
user,
['credential:read'],
);
if (!credential || credential.type !== credentialType) {
throw new Error(`Credential ${credentialId} not found or not accessible`);
}
const personalProject = await this.projectRepository.getPersonalProjectForUserOrFail(user.id);
const currentNodeParameters: INodeParameters = {};
const credentials: INodeCredentials = {
[credential.type]: { id: credential.id, name: credential.name },
};
const additionalData = await getBase({
userId: user.id,
projectId: personalProject.id,
currentNodeParameters,
});
const nodeTypeAndVersion = { name: lookup.nodeType, version: lookup.version };
if (lookup.kind === 'listSearch') {
const result = await this.dynamicNodeParametersService.getResourceLocatorResults(
lookup.methodName,
'',
additionalData,
nodeTypeAndVersion,
currentNodeParameters,
credentials,
);
return (result.results ?? []).map((r) => ({
name: String(r.name),
value: String(r.value),
}));
}
const nodeType = this.nodeTypes.getByNameAndVersion(lookup.nodeType, lookup.version);
const property = nodeType.description.properties.find((p) => p.name === lookup.propertyName);
const loadOptions = property?.typeOptions?.loadOptions;
if (!loadOptions) {
throw new Error(
`Property "${lookup.propertyName}" on ${lookup.nodeType} has no loadOptions config`,
);
}
const options = await this.dynamicNodeParametersService.getOptionsViaLoadOptions(
loadOptions,
additionalData,
nodeTypeAndVersion,
currentNodeParameters,
credentials,
);
return options.map((o) => ({ name: String(o.name), value: String(o.value) }));
}
}
@@ -57,7 +57,10 @@ describe('resolve_llm tool', () => {
it('uses the requested model for the requested provider', async () => {
const credentialProvider = makeProvider([{ id: 'c1', name: 'My xAI', type: 'xAiApi' }]);
const modelLookup = makeModelLookup();
const modelLookup = makeModelLookup(async () => [
{ name: 'Grok 4 Fast', value: 'grok-4-fast' },
{ name: 'Grok 4', value: 'grok-4' },
]);
const tool = buildResolveLlmTool({ credentialProvider, modelLookup });
const result = await tool.handler!({ provider: 'xai', model: 'grok-4-fast' }, {});
@@ -146,20 +149,23 @@ describe('resolve_llm tool', () => {
expect(modelLookup.list).not.toHaveBeenCalled();
});
it('skips lookup for providers without modelLookup configured (e.g. Cohere)', async () => {
it('validates the requested model against the lookup for Cohere', async () => {
const credentialProvider = makeProvider([{ id: 'c1', name: 'My Cohere', type: 'cohereApi' }]);
const modelLookup = makeModelLookup();
const modelLookup = makeModelLookup(async () => [
{ name: 'Command R+', value: 'command-r-plus' },
{ name: 'Command R', value: 'command-r' },
]);
const tool = buildResolveLlmTool({ credentialProvider, modelLookup });
const result = await tool.handler!({ provider: 'cohere', model: 'command-x-plus' }, {});
const result = await tool.handler!({ provider: 'cohere', model: 'command-r-plus' }, {});
expect(result).toEqual({
ok: true,
provider: 'cohere',
model: 'command-x-plus',
model: 'command-r-plus',
credentialId: 'c1',
credentialName: 'My Cohere',
});
expect(modelLookup.list).not.toHaveBeenCalled();
expect(modelLookup.list).toHaveBeenCalledWith('c1', 'cohereApi', 'cohere');
});
it('returns the canonical model id when the requested model matches the lookup', async () => {
@@ -183,11 +189,7 @@ describe('resolve_llm tool', () => {
credentialId: 'c1',
credentialName: 'My Anthropic',
});
expect(modelLookup.list).toHaveBeenCalledWith(
'c1',
'anthropicApi',
expect.objectContaining({ kind: 'listSearch', methodName: 'searchModels' }),
);
expect(modelLookup.list).toHaveBeenCalledWith('c1', 'anthropicApi', 'anthropic');
});
it('uniquely-substring-matches a partial requested model id', async () => {
@@ -10,95 +10,58 @@
* Azure variants), omit the entry so the tool falls through to suspending and
* lets the user pick explicitly.
*
* `modelLookup` (optional) points at the chat-model node whose
* `@searchListMethod` or routing-based `loadOptions` returns the live list of
* model ids for the provider. When set, `resolve_llm` validates user-requested
* model strings against that list. When absent, the requested model is passed
* through unchanged.
* Live model validation is available for providers supported by
* `@n8n/ai-utilities/model-discovery` `resolve_llm` checks user-requested
* model strings against the provider's live model list for those.
*/
export type ModelLookupConfig =
| {
kind: 'listSearch';
nodeType: string;
version: number;
methodName: string;
}
| {
kind: 'loadOptionsRouting';
nodeType: string;
version: number;
propertyName: string;
};
export interface LlmProviderDefault {
provider: string;
defaultModel: string;
modelLookup?: ModelLookupConfig;
}
export const LLM_PROVIDER_DEFAULTS: Record<string, LlmProviderDefault> = {
anthropicApi: {
provider: 'anthropic',
defaultModel: 'claude-sonnet-4-6',
modelLookup: {
kind: 'listSearch',
nodeType: '@n8n/n8n-nodes-langchain.lmChatAnthropic',
version: 1.5,
methodName: 'searchModels',
},
},
openAiApi: {
provider: 'openai',
defaultModel: 'gpt-5-mini',
modelLookup: {
kind: 'listSearch',
nodeType: '@n8n/n8n-nodes-langchain.lmChatOpenAi',
version: 1.2,
methodName: 'searchModels',
},
},
googlePalmApi: {
provider: 'google',
defaultModel: 'gemini-2.5-pro',
modelLookup: {
kind: 'loadOptionsRouting',
nodeType: '@n8n/n8n-nodes-langchain.lmChatGoogleGemini',
version: 1.1,
propertyName: 'modelName',
},
},
xAiApi: { provider: 'xai', defaultModel: 'grok-4' },
groqApi: { provider: 'groq', defaultModel: 'llama-3.1-70b-versatile' },
xAiApi: {
provider: 'xai',
defaultModel: 'grok-4',
},
groqApi: {
provider: 'groq',
defaultModel: 'llama-3.3-70b-versatile',
},
mistralCloudApi: {
provider: 'mistral',
defaultModel: 'mistral-large-latest',
modelLookup: {
kind: 'loadOptionsRouting',
nodeType: '@n8n/n8n-nodes-langchain.lmChatMistralCloud',
version: 1,
propertyName: 'model',
},
},
deepSeekApi: { provider: 'deepseek', defaultModel: 'deepseek-chat' },
cohereApi: { provider: 'cohere', defaultModel: 'command-r-plus' },
deepSeekApi: {
provider: 'deepseek',
defaultModel: 'deepseek-chat',
},
cohereApi: {
provider: 'cohere',
defaultModel: 'command-r-plus',
},
openRouterApi: {
provider: 'openrouter',
defaultModel: 'anthropic/claude-sonnet-4.6',
modelLookup: {
kind: 'loadOptionsRouting',
nodeType: '@n8n/n8n-nodes-langchain.lmChatOpenRouter',
version: 1,
propertyName: 'model',
},
},
nvidiaApi: {
provider: 'nvidia',
defaultModel: 'nvidia/llama-3.3-nemotron-super-49b-v1',
modelLookup: {
kind: 'loadOptionsRouting',
nodeType: '@n8n/n8n-nodes-langchain.lmChatNvidia',
version: 1,
propertyName: 'model',
},
},
vercelAiGatewayApi: {
provider: 'vercel',
defaultModel: 'anthropic/claude-sonnet-4.6',
},
};
@@ -1,19 +1,16 @@
import type { BuiltTool, CredentialListItem, CredentialProvider } from '@n8n/agents';
import { Tool } from '@n8n/agents/tool';
import { isModelDiscoveryProvider } from '@n8n/ai-utilities/model-discovery';
import { z } from 'zod';
import { BUILDER_TOOLS } from '../builder-tool-names';
import {
LLM_PROVIDER_DEFAULTS,
type LlmProviderDefault,
type ModelLookupConfig,
} from './llm-provider-defaults';
import { LLM_PROVIDER_DEFAULTS, type LlmProviderDefault } from './llm-provider-defaults';
export interface ModelLookup {
list(
credentialId: string,
credentialType: string,
lookup: ModelLookupConfig,
provider: string,
): Promise<Array<{ name: string; value: string }>>;
}
@@ -52,13 +49,13 @@ async function resolveModelAgainstLookup(
modelLookup: ModelLookup,
) {
const trimmedModel = requestedModel.trim();
if (!defaults.modelLookup || !trimmedModel) {
if (!isModelDiscoveryProvider(defaults.provider) || !trimmedModel) {
return toLlmResolution(credential, defaults, requestedModel);
}
let availableModels: Array<{ name: string; value: string }>;
try {
availableModels = await modelLookup.list(credential.id, credential.type, defaults.modelLookup);
availableModels = await modelLookup.list(credential.id, credential.type, defaults.provider);
} catch (error) {
return {
ok: false as const,
@@ -111,10 +111,11 @@ describe('AgentInfoPanel', () => {
variant: 'ghost',
showToolbar: 'never',
maxHeight: 'none',
placeholder: 'Enter instructions here',
});
expect(editor.props('placeholder')).toBeUndefined();
expect(wrapper.find('[data-testid="agent-instructions-document"]').exists()).toBe(true);
expect(wrapper.text()).not.toContain('characters');
expect(wrapper.text()).not.toContain('Enter instructions here');
});
it('can show the markdown toolbar above instructions', () => {
@@ -127,11 +128,12 @@ describe('AgentInfoPanel', () => {
});
});
it('keeps the instructions placeholder available when instructions are empty', () => {
it('does not pass placeholder text to the instructions editor', () => {
const wrapper = mountPanel('');
const editor = wrapper.findComponent({ name: 'N8nMarkdownEditor' });
expect(editor.props('modelValue')).toBe('');
expect(editor.props('placeholder')).toBe('Enter instructions here');
expect(editor.props('placeholder')).toBeUndefined();
expect(wrapper.text()).not.toContain('Enter instructions here');
});
});
@@ -3,6 +3,7 @@ import type { AgentModelOption } from '../model-providers';
const mocks = vi.hoisted(() => ({
getModelCatalog: vi.fn(),
getProviderModels: vi.fn(),
}));
vi.mock('@n8n/stores/useRootStore', () => ({
@@ -13,6 +14,7 @@ vi.mock('@n8n/stores/useRootStore', () => ({
vi.mock('../composables/useAgentApi', () => ({
getModelCatalog: mocks.getModelCatalog,
getProviderModels: mocks.getProviderModels,
}));
function model(id: string, name = id) {
@@ -36,10 +38,17 @@ function modelIds(models: AgentModelOption[]) {
return models.map((entry) => entry.model);
}
async function flushAsync() {
await new Promise((resolve) => setTimeout(resolve, 0));
}
describe('useModelCatalog', () => {
beforeEach(() => {
vi.resetModules();
mocks.getModelCatalog.mockReset();
mocks.getProviderModels.mockReset();
// Default: no live verification available — fall back to the catalog.
mocks.getProviderModels.mockRejectedValue(new Error('not available'));
});
it('returns models for the agent providers that have selected credentials', async () => {
@@ -91,4 +100,95 @@ describe('useModelCatalog', () => {
]);
expect(result.anthropic?.models).toEqual([]);
});
it('prefers the provider-verified model list over the static catalog', async () => {
mocks.getModelCatalog.mockResolvedValue({
anthropic: provider('anthropic', {
'claude-sonnet-4-6': model('claude-sonnet-4-6', 'Claude Sonnet 4.6'),
'claude-opus-4-0': model('claude-opus-4-0', 'Claude Opus 4'),
}),
});
mocks.getProviderModels.mockResolvedValue({
provider: 'anthropic',
verified: true,
models: [model('claude-sonnet-4-6', 'Claude Sonnet 4.6')],
});
const { useModelCatalog } = await import('../composables/useModelCatalog');
const { ensureLoaded, getModelsForPicker } = useModelCatalog();
await ensureLoaded('project-1');
const credentials = { anthropic: 'anthropic-credential-id' };
// First read triggers the verification fetch; once it lands, only the
// provider-confirmed models remain.
getModelsForPicker(credentials);
await flushAsync();
expect(modelIds(getModelsForPicker(credentials).anthropic?.models ?? [])).toEqual([
'claude-sonnet-4-6',
]);
expect(mocks.getProviderModels).toHaveBeenCalledWith(
{},
'project-1',
'anthropic',
'anthropic-credential-id',
);
});
it('fetches the verified list only once per provider and credential', async () => {
mocks.getModelCatalog.mockResolvedValue({
anthropic: provider('anthropic', {
'claude-sonnet-4-6': model('claude-sonnet-4-6'),
}),
});
mocks.getProviderModels.mockResolvedValue({
provider: 'anthropic',
verified: true,
models: [model('claude-sonnet-4-6')],
});
const { useModelCatalog } = await import('../composables/useModelCatalog');
const { ensureLoaded, getModelsForPicker } = useModelCatalog();
await ensureLoaded('project-1');
const credentials = { anthropic: 'anthropic-credential-id' };
getModelsForPicker(credentials);
getModelsForPicker(credentials);
await flushAsync();
getModelsForPicker(credentials);
await flushAsync();
expect(mocks.getProviderModels).toHaveBeenCalledTimes(1);
});
it('falls back to the catalog list when verification fails or is unverified', async () => {
mocks.getModelCatalog.mockResolvedValue({
anthropic: provider('anthropic', {
'claude-sonnet-4-6': model('claude-sonnet-4-6'),
'claude-opus-4-0': model('claude-opus-4-0'),
}),
openai: provider('openai', {
'gpt-5': model('gpt-5'),
}),
});
mocks.getProviderModels.mockImplementation(async (_ctx, _projectId, providerId) => {
if (providerId === 'anthropic') throw new Error('provider is down');
return { provider: providerId, verified: false, models: [model('gpt-5')] };
});
const { useModelCatalog } = await import('../composables/useModelCatalog');
const { ensureLoaded, getModelsForPicker } = useModelCatalog();
await ensureLoaded('project-1');
const credentials = { anthropic: 'anthropic-cred', openai: 'openai-cred' };
getModelsForPicker(credentials);
await flushAsync();
const result = getModelsForPicker(credentials);
expect(modelIds(result.anthropic?.models ?? []).sort()).toEqual([
'claude-opus-4-0',
'claude-sonnet-4-6',
]);
expect(modelIds(result.openai?.models ?? [])).toEqual(['gpt-5']);
});
});
@@ -201,7 +201,6 @@ function onInstructionsInput(value: string) {
:class="$style.instructionsDocument"
:model-value="instructions"
:readonly="props.disabled"
:placeholder="i18n.baseText('agents.builder.agent.instructions.placeholder')"
:variant="instructionsEditorVariant"
:show-toolbar="instructionsToolbarMode"
max-height="none"
@@ -8,6 +8,7 @@ import type {
AgentTaskConfig,
AgentTaskDto,
AgentIntegrationSettings,
AgentProviderModelsResponse,
AgentVersionListItemDto,
ChatIntegrationDescriptor,
CreateSlackAgentAppResponse,
@@ -346,6 +347,20 @@ export const getModelCatalog = async (
);
};
export const getProviderModels = async (
context: IRestApiContext,
projectId: string,
provider: string,
credentialId?: string,
): Promise<AgentProviderModelsResponse> => {
return await makeRestApiRequest<AgentProviderModelsResponse>(
context,
'GET',
`/projects/${projectId}/agents/v2/catalog/models/${provider}`,
credentialId ? { credentialId } : undefined,
);
};
export const publishAgent = async (
context: IRestApiContext,
projectId: string,
@@ -1,6 +1,11 @@
import { computed, ref } from 'vue';
import { useRootStore } from '@n8n/stores/useRootStore';
import { getModelCatalog, type ProviderCatalog, type ModelInfo } from './useAgentApi';
import {
getModelCatalog,
getProviderModels,
type ProviderCatalog,
type ModelInfo,
} from './useAgentApi';
import {
AGENT_MODEL_PROVIDERS,
type AgentCredentialsByProvider,
@@ -17,6 +22,14 @@ const catalogByProject = ref<Record<string, ProviderCatalog>>({});
const fetchPromises = new Map<string, Promise<void>>();
const loadingProjects = ref(new Set<string>());
// Provider-verified model lists, keyed by project|provider|credential. The
// static catalog can offer models the provider has retired (which then fail at
// call time), so when a credential is selected the provider's own model API is
// asked which models actually work. `null` marks a failed or unverified
// lookup — the static catalog is used for that key.
const verifiedModelsByKey = ref<Record<string, ModelInfo[] | null>>({});
const verifiedFetchesInFlight = new Set<string>();
function createEmptyModelsResponse(): AgentModelsByProvider {
const response: AgentModelsByProvider = {};
for (const provider of AGENT_MODEL_PROVIDERS) {
@@ -82,19 +95,62 @@ export function useModelCatalog() {
return Object.values(p.models).sort((a, b) => a.name.localeCompare(b.name));
}
/**
* Kick off (once per project+provider+credential) the fetch of the
* provider-verified model list. Idempotent and guarded, so it is safe to
* trigger from `getModelsForPicker` when the response lands, the reactive
* map updates and computeds re-evaluate with the verified list.
*/
function ensureVerifiedModels(
projectId: string,
provider: AgentModelProvider,
providerCredentialId: string,
): void {
const key = `${projectId}|${provider}|${providerCredentialId}`;
if (key in verifiedModelsByKey.value || verifiedFetchesInFlight.has(key)) return;
verifiedFetchesInFlight.add(key);
getProviderModels(rootStore.restApiContext, projectId, provider, providerCredentialId)
.then((result) => {
verifiedModelsByKey.value = {
...verifiedModelsByKey.value,
[key]: result.verified ? result.models : null,
};
})
.catch(() => {
verifiedModelsByKey.value = { ...verifiedModelsByKey.value, [key]: null };
})
.finally(() => {
verifiedFetchesInFlight.delete(key);
});
}
function getModelsForPicker(
credentials: AgentCredentialsByProvider | null,
): AgentModelsByProvider {
const response = createEmptyModelsResponse();
for (const provider of AGENT_MODEL_PROVIDERS) {
if (!credentials?.[provider]) continue;
const providerCredentialId = credentials?.[provider];
if (!providerCredentialId) continue;
const providerInfo = catalog.value[provider];
if (!providerInfo) continue;
let models: ModelInfo[] | undefined;
const projectId = activeProjectId.value;
if (projectId) {
ensureVerifiedModels(projectId, provider, providerCredentialId);
models =
verifiedModelsByKey.value[`${projectId}|${provider}|${providerCredentialId}`] ??
undefined;
}
if (!models) {
const providerInfo = catalog.value[provider];
if (!providerInfo) continue;
models = Object.values(providerInfo.models);
}
response[provider] = {
models: Object.values(providerInfo.models)
models: models
.map((model) => toAgentModel(provider, model))
.sort((a, b) => a.name.localeCompare(b.name)),
};