diff --git a/packages/@n8n/agents/src/runtime/__tests__/agent-runtime.test.ts b/packages/@n8n/agents/src/runtime/__tests__/agent-runtime.test.ts index ba9a99ad4dc..eed1571a27f 100644 --- a/packages/@n8n/agents/src/runtime/__tests__/agent-runtime.test.ts +++ b/packages/@n8n/agents/src/runtime/__tests__/agent-runtime.test.ts @@ -6438,3 +6438,109 @@ describe('AgentRuntime — toModelOutput error resilience', () => { expect(toolResultChunk!.isError).toBe(true); }); }); + +// --------------------------------------------------------------------------- +// empty model responses (e.g. provider safety blocks) +// --------------------------------------------------------------------------- + +describe('AgentRuntime — empty model responses', () => { + beforeEach(() => { + generateText.mockReset(); + streamText.mockReset(); + }); + + function makeEmptyStream(finishReason: string, extraChunks: Array> = []) { + return { + fullStream: makeChunkStream(extraChunks), + finishReason: Promise.resolve(finishReason), + usage: Promise.resolve({ inputTokens: 10, outputTokens: 0, totalTokens: 10 }), + response: Promise.resolve({ messages: [] }), + toolCalls: Promise.resolve([]), + }; + } + + it.each(['other', 'unknown', 'content-filter'])( + 'stream: yields an error chunk when the model returns no output with finish reason "%s"', + async (finishReason) => { + streamText.mockReturnValue(makeEmptyStream(finishReason)); + + const { runtime } = createRuntime(); + const { stream: readableStream } = await runtime.stream('hello'); + const chunks = await collectChunks(readableStream); + + const errorChunk = chunks.find((c) => c.type === 'error') as + | (StreamChunk & { type: 'error' }) + | undefined; + expect(errorChunk).toBeDefined(); + expect(String((errorChunk!.error as Error).message)).toContain('no output'); + expect(String((errorChunk!.error as Error).message)).toContain(finishReason); + + const finishChunk = chunks.find((c) => c.type === 'finish') as + | (StreamChunk & { type: 'finish' }) + | undefined; + expect(finishChunk).toBeDefined(); + expect(finishChunk!.finishReason).toBe('error'); + }, + ); + + it('stream: includes the Google prompt block reason captured from raw chunks', async () => { + streamText.mockReturnValue( + makeEmptyStream('other', [ + { + type: 'raw', + rawValue: { promptFeedback: { blockReason: 'PROHIBITED_CONTENT' } }, + }, + ]), + ); + + const { runtime } = createRuntime(undefined, 'google/gemini-2.5-flash'); + const { stream: readableStream } = await runtime.stream('hello'); + const chunks = await collectChunks(readableStream); + + const errorChunk = chunks.find((c) => c.type === 'error') as + | (StreamChunk & { type: 'error' }) + | undefined; + expect(errorChunk).toBeDefined(); + expect(String((errorChunk!.error as Error).message)).toContain('PROHIBITED_CONTENT'); + }); + + it('stream: requests raw chunks for google models so block reasons are observable', async () => { + streamText.mockReturnValue(makeStreamSuccess()); + + const { runtime } = createRuntime(undefined, 'google/gemini-2.5-flash'); + const { stream: readableStream } = await runtime.stream('hello'); + await collectChunks(readableStream); + + const args = streamText.mock.calls[0][0] as Record; + expect(args.includeRawChunks).toBe(true); + }); + + it('stream: an empty response finishing with "stop" is not treated as an error', async () => { + streamText.mockReturnValue(makeEmptyStream('stop')); + + const { runtime } = createRuntime(); + const { stream: readableStream } = await runtime.stream('hello'); + const chunks = await collectChunks(readableStream); + + expect(chunks.find((c) => c.type === 'error')).toBeUndefined(); + const finishChunk = chunks.find((c) => c.type === 'finish') as + | (StreamChunk & { type: 'finish' }) + | undefined; + expect(finishChunk!.finishReason).not.toBe('error'); + }); + + it('generate: returns finishReason "error" with a descriptive error when the model returns no output', async () => { + generateText.mockResolvedValue({ + finishReason: 'other', + usage: { inputTokens: 10, outputTokens: 0, totalTokens: 10 }, + response: { messages: [] }, + toolCalls: [], + }); + + const { runtime } = createRuntime(); + const result = await runtime.generate('hello'); + + expect(result.finishReason).toBe('error'); + expect(String((result.error as Error).message)).toContain('no output'); + }); +}); diff --git a/packages/@n8n/agents/src/runtime/loop/agent-runtime.ts b/packages/@n8n/agents/src/runtime/loop/agent-runtime.ts index cc5d1c50114..24a9fd3911f 100644 --- a/packages/@n8n/agents/src/runtime/loop/agent-runtime.ts +++ b/packages/@n8n/agents/src/runtime/loop/agent-runtime.ts @@ -719,6 +719,10 @@ export class AgentRuntime { sink.onTurnFolded?.(); if (turn.aiFinishReason !== 'tool-calls') { + // A rejected/filtered request (e.g. a provider prompt safety block) + // surfaces as an output-less turn instead of an SDK error — throw so + // the failure reaches the caller rather than ending the run silently. + if (turn.errorReason) throw new Error(turn.errorReason.message); structuredOutput = turn.structuredOutput; this.emitTurnEnd(turn.newMessages, extractSettledToolCalls(turn.newMessages)); reachedStopCondition = true; diff --git a/packages/@n8n/agents/src/runtime/loop/generate-sink.ts b/packages/@n8n/agents/src/runtime/loop/generate-sink.ts index d0a5b66d222..6eafd67db5c 100644 --- a/packages/@n8n/agents/src/runtime/loop/generate-sink.ts +++ b/packages/@n8n/agents/src/runtime/loop/generate-sink.ts @@ -6,6 +6,7 @@ import type { RunServices, SuspendEmission, } from './run-output-sink'; +import { classifyModelTurnError } from './runtime-helpers'; import type { GenerateResult } from '../../types'; import type { ToolResultEntry } from '../../types/sdk/agent'; import { loadAi } from '../model/lazy-ai'; @@ -40,14 +41,17 @@ export class GenerateSink implements RunOutputSink { }); const aiFinishReason = result.finishReason; + const newMessages = fromAiMessages(result.response.messages); + const errorReason = classifyModelTurnError({ aiFinishReason, newMessages }); return { aiFinishReason, finishReason: fromAiFinishReason(aiFinishReason), usage: toTokenUsage(result.usage, result.providerMetadata), - newMessages: fromAiMessages(result.response.messages), + newMessages, toolCalls: result.toolCalls, structuredOutput: ctx.outputSpec && aiFinishReason !== 'tool-calls' ? result.output : undefined, + ...(errorReason && { errorReason }), }; } diff --git a/packages/@n8n/agents/src/runtime/loop/run-output-sink.ts b/packages/@n8n/agents/src/runtime/loop/run-output-sink.ts index 42a46d898f8..07c98403ee2 100644 --- a/packages/@n8n/agents/src/runtime/loop/run-output-sink.ts +++ b/packages/@n8n/agents/src/runtime/loop/run-output-sink.ts @@ -17,6 +17,21 @@ import type { ToolCallBatchResult, ToolCallSuspension } from '../tools/tool-call type RunCallOptions = (RunOptions & ExecutionOptions) | undefined; +/** + * Recognized ways a model turn can fail while the provider still returns + * "successfully" (no thrown SDK error). Extend this as new failure modes are + * recognized so consumers switch on `type` instead of probing separate fields. + * - `prompt_blocked`: the provider rejected the prompt (e.g. a safety block). + * - `no_output`: the model returned nothing with a non-stop finish reason. + */ +export type ModelTurnErrorType = 'prompt_blocked' | 'no_output'; + +/** A recognized model-turn failure: a `type` to branch on and a user-facing `message`. */ +export interface ModelTurnError { + type: ModelTurnErrorType; + message: string; +} + /** Normalized result of a single LLM turn, independent of generate vs stream. */ export interface ModelTurnResult { /** Raw AI SDK finish reason (used to detect the `tool-calls` continuation). */ @@ -33,6 +48,13 @@ export interface ModelTurnResult { }>; /** Resolved structured output, only when an output spec is set and the turn finished. */ structuredOutput: unknown; + /** + * Set when the turn failed in a recognized way despite the provider returning + * without a thrown error (e.g. a prompt safety block, or empty output). The + * loop surfaces this instead of ending the run silently. Check this single + * field rather than probing per-failure-mode properties. + */ + errorReason?: ModelTurnError; } /** Per-iteration inputs for the LLM call, assembled by the shared loop. */ diff --git a/packages/@n8n/agents/src/runtime/loop/runtime-helpers.ts b/packages/@n8n/agents/src/runtime/loop/runtime-helpers.ts index 578f4005b44..4ad388e5b86 100644 --- a/packages/@n8n/agents/src/runtime/loop/runtime-helpers.ts +++ b/packages/@n8n/agents/src/runtime/loop/runtime-helpers.ts @@ -2,8 +2,10 @@ * Pure utility functions used by AgentRuntime that require no class context. * These are extracted here to keep agent-runtime.ts focused on orchestration logic. */ +import type { ModelTurnError } from './run-output-sink'; import type { StreamChunk, TokenUsage } from '../../types'; import type { AgentMessage, ContentToolCall } from '../../types/sdk/message'; +import type { RawProviderError } from '../model/raw-error'; /** * Normalize caller input to `AgentMessage[]` for the runtime. String input becomes a @@ -21,6 +23,44 @@ export function stringifyError(error: unknown): string { return error instanceof Error ? `${error.name}: ${error.message}` : String(error); } +/** + * Finish reasons that indicate the provider rejected or filtered the request + * when they arrive with zero output. `stop`/`length` with empty output are the + * model's own (odd but legal) choice; `tool-calls` always carries calls; + * `error` surfaces through the SDK's thrown error instead. + */ +const EMPTY_RESPONSE_ERROR_FINISH_REASONS = new Set(['other', 'unknown', 'content-filter']); + +/** + * Classify a turn that produced no output as a recognized failure, or return + * `undefined` when it doesn't look like a provider rejection. Some providers + * fail this way rather than erroring, reporting the cause only on their raw + * stream events — when a {@link RawProviderError} was captured there, its type + * and reason are carried into the result; otherwise the failure is a generic + * `no_output`. + */ +export function classifyModelTurnError(turn: { + aiFinishReason: string; + newMessages: AgentMessage[]; + providerError?: RawProviderError; +}): ModelTurnError | undefined { + if (turn.newMessages.length > 0) return undefined; + if (!EMPTY_RESPONSE_ERROR_FINISH_REASONS.has(turn.aiFinishReason)) return undefined; + + const guidance = + 'This can be a provider-side false positive — try rephrasing the message, clearing the chat history, or switching models.'; + if (turn.providerError) { + return { + type: turn.providerError.type, + message: `The model provider blocked this request (${turn.providerError.reason}) and returned no output (finish reason: ${turn.aiFinishReason}). ${guidance}`, + }; + } + return { + type: 'no_output', + message: `The model returned no output (finish reason: ${turn.aiFinishReason}). The provider may have blocked or filtered the request. ${guidance}`, + }; +} + /** Extract all settled (resolved or rejected) tool-call blocks from a flat list of agent messages. */ export function extractSettledToolCalls(messages: AgentMessage[]): ContentToolCall[] { return messages diff --git a/packages/@n8n/agents/src/runtime/loop/stream-sink.ts b/packages/@n8n/agents/src/runtime/loop/stream-sink.ts index c785182af31..7b2d02d00c0 100644 --- a/packages/@n8n/agents/src/runtime/loop/stream-sink.ts +++ b/packages/@n8n/agents/src/runtime/loop/stream-sink.ts @@ -6,11 +6,12 @@ import type { RunServices, SuspendEmission, } from './run-output-sink'; -import { mergeUsage } from './runtime-helpers'; +import { classifyModelTurnError, mergeUsage } from './runtime-helpers'; import type { ExecutionOptions, TokenUsage } from '../../types/sdk/agent'; import type { AgentMessage } from '../../types/sdk/message'; import { loadAi } from '../model/lazy-ai'; import { fromAiFinishReason, fromAiMessages } from '../model/messages'; +import { createRawErrorReader, type RawErrorReader } from '../model/raw-error'; import { createRawUsageReader, type RawUsageReader } from '../model/raw-usage'; import { convertChunk, toTokenUsage } from '../streaming/stream'; import type { StreamWriterGuard } from '../streaming/stream-writer-guard'; @@ -32,6 +33,11 @@ export class StreamSink implements RunOutputSink { // Text streamed for the in-flight turn, retained so a stop landing mid-response // can still persist what the user already saw. Cleared once the turn is folded. private partialText = ''; + // Reads provider failure signals (e.g. a prompt safety block) from raw + // chunks, so an output-less rejected request can report why. Per-provider + // implementations live behind `RawErrorReader`; undefined when the run's + // provider has no reader. + private rawErrorReader: RawErrorReader | undefined; constructor( private readonly guard: StreamWriterGuard, @@ -100,6 +106,10 @@ export class StreamSink implements RunOutputSink { this.rawUsageReader = this.options?.recoverUsageOnAbort ? createRawUsageReader(this.services.modelId) : undefined; + // Some providers report failures (e.g. prompt safety blocks) only on the + // raw stream — no error, no content — so raw chunks are required to + // explain an otherwise silent empty response. + this.rawErrorReader = createRawErrorReader(this.services.modelId); const { streamText } = loadAi(); const result = streamText({ model: ctx.model, @@ -108,7 +118,9 @@ export class StreamSink implements RunOutputSink { abortSignal: ctx.abortSignal, // Surface the provider's raw message_start/message_delta events so an // aborted run can recover its usage — the SDK reports none on abort. - ...(this.rawUsageReader ? { includeRawChunks: true } : {}), + ...(this.rawUsageReader !== undefined || this.rawErrorReader !== undefined + ? { includeRawChunks: true } + : {}), ...(ctx.hasTools ? { tools: ctx.aiTools } : {}), ...(ctx.providerOptions ? { providerOptions: ctx.providerOptions } : {}), ...(ctx.outputSpec ? { output: ctx.outputSpec } : {}), @@ -124,6 +136,7 @@ export class StreamSink implements RunOutputSink { // reaches the post-loop awaits) can still be billed via getAbortFinish. if (chunk.type === 'raw') { this.rawUsageReader?.capture(chunk.rawValue); + this.rawErrorReader?.capture(chunk.rawValue); continue; } // Filter only the SDK's terminal `finish` chunk — the runtime emits its @@ -167,15 +180,22 @@ export class StreamSink implements RunOutputSink { const usage = await result.usage; const providerMetadata = await result.providerMetadata; const response = await result.response; + const newMessages = fromAiMessages(response.messages); + const errorReason = classifyModelTurnError({ + aiFinishReason, + newMessages, + providerError: this.rawErrorReader?.getError(), + }); return { aiFinishReason, finishReason: fromAiFinishReason(aiFinishReason), usage: toTokenUsage(usage, providerMetadata), - newMessages: fromAiMessages(response.messages), + newMessages, toolCalls: await result.toolCalls, structuredOutput: ctx.outputSpec && aiFinishReason !== 'tool-calls' ? await result.output : undefined, + ...(errorReason && { errorReason }), }; } diff --git a/packages/@n8n/agents/src/runtime/model/raw-error.ts b/packages/@n8n/agents/src/runtime/model/raw-error.ts new file mode 100644 index 00000000000..c6da6a9c135 --- /dev/null +++ b/packages/@n8n/agents/src/runtime/model/raw-error.ts @@ -0,0 +1,66 @@ +import { isRecord } from '@n8n/utils/is-record'; + +import { getModelProvider } from './prompt-cache'; +import { type ProviderId } from './provider-credentials'; +import type { ModelTurnErrorType } from '../loop/run-output-sink'; + +/** A recognized failure signal read from the provider's raw stream events. */ +export interface RawProviderError { + type: ModelTurnErrorType; + /** The provider's own reason string (e.g. Google's `PROHIBITED_CONTENT`). */ + reason: string; +} + +/** + * Reads a provider's native raw stream events (the AI SDK's `chunk.rawValue` + * passthrough, enabled by `includeRawChunks`) for failure signals the SDK does + * not surface — e.g. a prompt safety block that ends the stream with no error + * and no content. Each provider's wire format differs, so the runtime stays + * provider-agnostic and delegates to a per-provider implementation selected + * via {@link createRawErrorReader}. + */ +export interface RawErrorReader { + /** Inspect one raw provider event for a failure signal. */ + capture(rawValue: unknown): void; + /** The failure signal seen this turn, or undefined when none was observed. */ + getError(): RawProviderError | undefined; +} + +/** + * Google reports prompt safety blocks only via `promptFeedback.blockReason` on + * the raw stream — the response then simply contains no candidates. + */ +export class GoogleRawErrorReader implements RawErrorReader { + private blockReason: string | undefined; + + capture(rawValue: unknown): void { + if (!isRecord(rawValue) || !isRecord(rawValue.promptFeedback)) return; + const blockReason = rawValue.promptFeedback.blockReason; + if (typeof blockReason === 'string' && blockReason) { + this.blockReason = blockReason; + } + } + + getError(): RawProviderError | undefined { + return this.blockReason ? { type: 'prompt_blocked', reason: this.blockReason } : undefined; + } +} + +/** + * Per-provider raw-error readers, keyed by provider id. Partial because this is + * best-effort diagnostics: a provider without an entry still gets the generic + * empty-response handling, just without the provider's own reason attached. + */ +const RAW_ERROR_READERS: Partial RawErrorReader>> = { + google: () => new GoogleRawErrorReader(), +}; + +/** + * Create a fresh raw-error reader for the run's model, or undefined when the + * provider has no reader. The provider is the prefix of the `provider/model` id + * (same convention as `createModel`). + */ +export function createRawErrorReader(modelId: string): RawErrorReader | undefined { + const provider = getModelProvider(modelId) as ProviderId; + return RAW_ERROR_READERS[provider]?.(); +} diff --git a/packages/@n8n/agents/src/sdk/__tests__/catalog.test.ts b/packages/@n8n/agents/src/sdk/__tests__/catalog.test.ts index ac231b65bc4..d291fd2249c 100644 --- a/packages/@n8n/agents/src/sdk/__tests__/catalog.test.ts +++ b/packages/@n8n/agents/src/sdk/__tests__/catalog.test.ts @@ -71,6 +71,75 @@ describe('fetchProviderCatalog', () => { expect(catalog['azure-cognitive-services']).toBeUndefined(); }); + it('drops models marked deprecated on models.dev', async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: async () => + await Promise.resolve({ + anthropic: { + id: 'anthropic', + name: 'Anthropic', + models: { + 'claude-3-haiku-20240307': { + id: 'claude-3-haiku-20240307', + name: 'Claude Haiku 3', + status: 'deprecated', + }, + 'claude-sonnet-4-6': { + id: 'claude-sonnet-4-6', + name: 'Claude Sonnet 4.6', + }, + 'claude-beta-model': { + id: 'claude-beta-model', + name: 'Claude Beta Model', + status: 'beta', + }, + }, + }, + }), + }); + global.fetch = fetchMock as typeof fetch; + + const catalog = await fetchProviderCatalog(); + + expect(catalog.anthropic.models['claude-3-haiku-20240307']).toBeUndefined(); + expect(catalog.anthropic.models['claude-sonnet-4-6'].name).toBe('Claude Sonnet 4.6'); + expect(catalog.anthropic.models['claude-beta-model'].name).toBe('Claude Beta Model'); + }); + + it('omits a provider whose models are all deprecated', async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: async () => + await Promise.resolve({ + anthropic: { + id: 'anthropic', + name: 'Anthropic', + models: { + 'claude-3-haiku-20240307': { + id: 'claude-3-haiku-20240307', + name: 'Claude Haiku 3', + status: 'deprecated', + }, + }, + }, + openai: { + id: 'openai', + name: 'OpenAI', + models: { + 'gpt-5': { id: 'gpt-5', name: 'GPT-5' }, + }, + }, + }), + }); + global.fetch = fetchMock as typeof fetch; + + const catalog = await fetchProviderCatalog(); + + expect(catalog.anthropic).toBeUndefined(); + expect(catalog.openai.models['gpt-5'].name).toBe('GPT-5'); + }); + it('strips the "(latest)" suffix from model names and drops duplicate pinned snapshots', async () => { const fetchMock = vi.fn().mockResolvedValue({ ok: true, diff --git a/packages/@n8n/agents/src/sdk/catalog.ts b/packages/@n8n/agents/src/sdk/catalog.ts index 6caa795bfed..2d7832e8a4f 100644 --- a/packages/@n8n/agents/src/sdk/catalog.ts +++ b/packages/@n8n/agents/src/sdk/catalog.ts @@ -68,6 +68,7 @@ interface ModelsDevModel { release_date?: string; reasoning?: boolean; tool_call?: boolean; + status?: string; cost?: { input?: number; output?: number; cache_read?: number; cache_write?: number }; limit?: { context?: number; output?: number }; } @@ -138,6 +139,9 @@ export async function fetchProviderCatalog(): Promise { const models: Record = {}; for (const [modelId, model] of Object.entries(provider.models)) { + // Deprecated models still 404 at call time when the provider retires + // them, so never offer them. + if (model.status === 'deprecated') continue; const info: ModelInfo = { id: model.id, name: model.name, @@ -162,6 +166,8 @@ export async function fetchProviderCatalog(): Promise { models[modelId] = info; } + if (Object.keys(models).length === 0) continue; + const providerId = toAgentProviderId(key); catalog[providerId] = { id: providerId, diff --git a/packages/@n8n/ai-utilities/package.json b/packages/@n8n/ai-utilities/package.json index 6d46f1a322f..787dc3a52d0 100644 --- a/packages/@n8n/ai-utilities/package.json +++ b/packages/@n8n/ai-utilities/package.json @@ -24,6 +24,9 @@ ], "web-search": [ "dist/esm/web-search/index.d.ts" + ], + "model-discovery": [ + "dist/esm/model-discovery/index.d.ts" ] } }, @@ -63,6 +66,11 @@ "import": "./dist/esm/web-search/index.js", "require": "./dist/cjs/web-search/index.js" }, + "./model-discovery": { + "types": "./dist/esm/model-discovery/index.d.ts", + "import": "./dist/esm/model-discovery/index.js", + "require": "./dist/cjs/model-discovery/index.js" + }, "./*": "./*" }, "scripts": { diff --git a/packages/@n8n/ai-utilities/src/model-discovery/__tests__/model-discovery.test.ts b/packages/@n8n/ai-utilities/src/model-discovery/__tests__/model-discovery.test.ts new file mode 100644 index 00000000000..9c136bd59a4 --- /dev/null +++ b/packages/@n8n/ai-utilities/src/model-discovery/__tests__/model-discovery.test.ts @@ -0,0 +1,228 @@ +import { listModelsForProvider, MODEL_DISCOVERY_PROVIDERS } from '../index'; + +function mockFetch(body: unknown, ok = true, status = 200) { + return vi.fn().mockResolvedValue({ + ok, + status, + text: async () => JSON.stringify(body), + json: async () => body, + }) as unknown as typeof globalThis.fetch; +} + +function calledUrl(fetchFn: unknown): string { + return String((fetchFn as ReturnType).mock.calls[0][0]); +} + +function calledHeaders(fetchFn: unknown): Record { + const init = (fetchFn as ReturnType).mock.calls[0][1] as { + headers: Record; + }; + return init.headers; +} + +describe('model-discovery', () => { + describe('anthropic', () => { + it('lists models from /v1/models with x-api-key auth, newest first', async () => { + const fetch = mockFetch({ + data: [ + { id: 'claude-old', display_name: 'Claude Old', created_at: '2024-01-01T00:00:00Z' }, + { id: 'claude-new', display_name: 'Claude New', created_at: '2026-01-01T00:00:00Z' }, + ], + }); + + const models = await listModelsForProvider('anthropic', { apiKey: 'key', fetch }); + + expect(calledUrl(fetch)).toBe('https://api.anthropic.com/v1/models'); + expect(calledHeaders(fetch)['x-api-key']).toBe('key'); + expect(calledHeaders(fetch)['anthropic-version']).toBe('2023-06-01'); + expect(models).toEqual([ + { id: 'claude-new', name: 'Claude New' }, + { id: 'claude-old', name: 'Claude Old' }, + ]); + }); + + it('respects a custom baseURL', async () => { + const fetch = mockFetch({ data: [] }); + await listModelsForProvider('anthropic', { + apiKey: 'key', + baseURL: 'https://proxy.example.com/', + fetch, + }); + expect(calledUrl(fetch)).toBe('https://proxy.example.com/v1/models'); + }); + }); + + describe('openai', () => { + it('filters out non-chat models on the official API and sorts by id', async () => { + const fetch = mockFetch({ + data: [ + { id: 'gpt-5' }, + { id: 'whisper-1' }, + { id: 'dall-e-3' }, + { id: 'text-embedding-3-small' }, + { id: 'gpt-4o' }, + ], + }); + + const models = await listModelsForProvider('openai', { apiKey: 'key', fetch }); + + expect(calledUrl(fetch)).toBe('https://api.openai.com/v1/models'); + expect(calledHeaders(fetch).Authorization).toBe('Bearer key'); + expect(models.map((m) => m.id)).toEqual(['gpt-4o', 'gpt-5']); + }); + + it('includes all models for a custom (non-OpenAI) baseURL', async () => { + const fetch = mockFetch({ data: [{ id: 'whisper-1' }, { id: 'my-model' }] }); + + const models = await listModelsForProvider('openai', { + apiKey: 'key', + baseURL: 'https://llm.internal.example.com/v1', + fetch, + }); + + expect(models.map((m) => m.id)).toEqual(['my-model', 'whisper-1']); + }); + + it('merges caller-supplied headers into the request', async () => { + const fetch = mockFetch({ data: [] }); + await listModelsForProvider('openai', { + apiKey: 'key', + headers: { 'x-custom': 'yes' }, + fetch, + }); + expect(calledHeaders(fetch)['x-custom']).toBe('yes'); + }); + }); + + describe('google', () => { + it('lists models from /v1beta/models with header auth, excluding embedding/imagen', async () => { + const fetch = mockFetch({ + models: [ + { name: 'models/gemini-2.5-flash', description: 'Fast' }, + { name: 'models/text-embedding-004', description: 'Embeddings' }, + { name: 'models/imagen-3', description: 'Images' }, + { name: 'models/gemini-2.5-pro', description: 'Smart' }, + ], + }); + + const models = await listModelsForProvider('google', { apiKey: 'g-key', fetch }); + + // The API key goes in the x-goog-api-key header, never the query string, + // so it cannot leak through access logs or proxies. + expect(calledUrl(fetch)).toBe('https://generativelanguage.googleapis.com/v1beta/models'); + expect(calledHeaders(fetch)['x-goog-api-key']).toBe('g-key'); + expect(models).toEqual([ + { id: 'models/gemini-2.5-flash', name: 'models/gemini-2.5-flash' }, + { id: 'models/gemini-2.5-pro', name: 'models/gemini-2.5-pro' }, + ]); + }); + }); + + describe('groq', () => { + it('keeps only active model objects', async () => { + const fetch = mockFetch({ + data: [ + { id: 'llama-3.3-70b', active: true, object: 'model' }, + { id: 'retired-model', active: false, object: 'model' }, + { id: 'not-a-model', active: true, object: 'other' }, + ], + }); + + const models = await listModelsForProvider('groq', { apiKey: 'key', fetch }); + + expect(calledUrl(fetch)).toBe('https://api.groq.com/openai/v1/models'); + expect(models.map((m) => m.id)).toEqual(['llama-3.3-70b']); + }); + }); + + describe('mistral', () => { + it('excludes embedding models and sorts by name', async () => { + const fetch = mockFetch({ + data: [{ id: 'mistral-large' }, { id: 'mistral-embed' }, { id: 'codestral' }], + }); + + const models = await listModelsForProvider('mistral', { apiKey: 'key', fetch }); + + expect(calledUrl(fetch)).toBe('https://api.mistral.ai/v1/models'); + expect(models.map((m) => m.id)).toEqual(['codestral', 'mistral-large']); + }); + }); + + describe('cohere', () => { + it('lists chat models by name', async () => { + const fetch = mockFetch({ + models: [{ name: 'command-r-plus' }, { name: 'command-a-03-2025' }], + }); + + const models = await listModelsForProvider('cohere', { apiKey: 'key', fetch }); + + expect(calledUrl(fetch)).toBe('https://api.cohere.ai/v1/models?page_size=100&endpoint=chat'); + expect(models.map((m) => m.id)).toEqual(['command-a-03-2025', 'command-r-plus']); + }); + }); + + describe('nvidia', () => { + it('keeps only supported Nemotron models', async () => { + const fetch = mockFetch({ + data: [ + { id: 'nvidia/llama-3.3-nemotron-super-49b-v1' }, + { id: 'meta/llama-3.1-405b-instruct' }, + ], + }); + + const models = await listModelsForProvider('nvidia', { apiKey: 'key', fetch }); + + expect(calledUrl(fetch)).toBe('https://integrate.api.nvidia.com/v1/models'); + expect(models.map((m) => m.id)).toEqual(['nvidia/llama-3.3-nemotron-super-49b-v1']); + }); + }); + + describe.each([ + ['deepseek', 'https://api.deepseek.com/models'], + ['openrouter', 'https://openrouter.ai/api/v1/models'], + ['xai', 'https://api.x.ai/v1/models'], + ['vercel', 'https://ai-gateway.vercel.sh/v1/models'], + ] as const)('%s', (provider, expectedUrl) => { + it('lists models with bearer auth, sorted by name', async () => { + const fetch = mockFetch({ data: [{ id: 'model-b' }, { id: 'model-a' }] }); + + const models = await listModelsForProvider(provider, { apiKey: 'key', fetch }); + + expect(calledUrl(fetch)).toBe(expectedUrl); + expect(calledHeaders(fetch).Authorization).toBe('Bearer key'); + expect(models.map((m) => m.id)).toEqual(['model-a', 'model-b']); + }); + }); + + describe('error handling', () => { + it('throws a descriptive error on a non-2xx response', async () => { + const fetch = mockFetch({ error: { message: 'invalid x-api-key' } }, false, 401); + + await expect(listModelsForProvider('anthropic', { apiKey: 'bad', fetch })).rejects.toThrow( + /anthropic.*401/i, + ); + }); + + it('throws for an unknown provider', async () => { + await expect( + listModelsForProvider('not-a-provider', { apiKey: 'key', fetch: mockFetch({}) }), + ).rejects.toThrow(/unknown/i); + }); + }); + + it('exposes a registry of all supported providers', () => { + expect(Object.keys(MODEL_DISCOVERY_PROVIDERS).sort()).toEqual([ + 'anthropic', + 'cohere', + 'deepseek', + 'google', + 'groq', + 'mistral', + 'nvidia', + 'openai', + 'openrouter', + 'vercel', + 'xai', + ]); + }); +}); diff --git a/packages/@n8n/ai-utilities/src/model-discovery/index.ts b/packages/@n8n/ai-utilities/src/model-discovery/index.ts new file mode 100644 index 00000000000..44c81300c10 --- /dev/null +++ b/packages/@n8n/ai-utilities/src/model-discovery/index.ts @@ -0,0 +1,65 @@ +import { listAnthropicModels } from './providers/anthropic'; +import { listCohereModels } from './providers/cohere'; +import { listDeepSeekModels } from './providers/deepseek'; +import { listGoogleModels } from './providers/google'; +import { listGroqModels } from './providers/groq'; +import { listMistralModels } from './providers/mistral'; +import { listNvidiaModels } from './providers/nvidia'; +import { listOpenAiModels } from './providers/openai'; +import { listOpenRouterModels } from './providers/openrouter'; +import { listVercelModels } from './providers/vercel'; +import { listXaiModels } from './providers/xai'; +import type { ListModelsFn, ListModelsOptions, ProviderModel } from './types'; + +/** + * Live chat-model discovery per provider: each function asks the provider's + * own model-list API which models the given credential can call. Shared by the + * chat sub-nodes' model dropdowns and the agents feature's model catalog, so + * provider knowledge (endpoint, auth, chat-model filtering) lives in one place. + * + * Each provider implements {@link ListModelsFn} in its own file under + * `providers/`, ported 1:1 from the corresponding chat node's `searchModels` + * method or `loadOptions` routing config (see the source reference on each). + * When changing behavior here, keep the node in sync. + */ +export const MODEL_DISCOVERY_PROVIDERS: Record = { + anthropic: listAnthropicModels, + cohere: listCohereModels, + deepseek: listDeepSeekModels, + google: listGoogleModels, + groq: listGroqModels, + mistral: listMistralModels, + nvidia: listNvidiaModels, + openai: listOpenAiModels, + openrouter: listOpenRouterModels, + vercel: listVercelModels, + xai: listXaiModels, +}; + +export function isModelDiscoveryProvider(provider: string): boolean { + return provider in MODEL_DISCOVERY_PROVIDERS; +} + +export async function listModelsForProvider( + provider: string, + options: ListModelsOptions, +): Promise { + const listModels = MODEL_DISCOVERY_PROVIDERS[provider]; + if (!listModels) { + throw new Error(`Unknown model discovery provider: "${provider}"`); + } + return await listModels(options); +} + +export { listAnthropicModels } from './providers/anthropic'; +export { listCohereModels } from './providers/cohere'; +export { listDeepSeekModels } from './providers/deepseek'; +export { listGoogleModels } from './providers/google'; +export { listGroqModels } from './providers/groq'; +export { listMistralModels } from './providers/mistral'; +export { listNvidiaModels } from './providers/nvidia'; +export { listOpenAiModels, shouldIncludeOpenAiModel } from './providers/openai'; +export { listOpenRouterModels } from './providers/openrouter'; +export { listVercelModels } from './providers/vercel'; +export { listXaiModels } from './providers/xai'; +export type { ListModelsFn, ListModelsOptions, ProviderModel } from './types'; diff --git a/packages/@n8n/ai-utilities/src/model-discovery/providers/anthropic.ts b/packages/@n8n/ai-utilities/src/model-discovery/providers/anthropic.ts new file mode 100644 index 00000000000..714a04bb1b8 --- /dev/null +++ b/packages/@n8n/ai-utilities/src/model-discovery/providers/anthropic.ts @@ -0,0 +1,17 @@ +import { baseUrl, getJson } from '../request'; +import type { ListModelsFn } from '../types'; + +/** Source: LMChatAnthropic `methods/searchModels.ts` (GET /v1/models, newest first). */ +export const listAnthropicModels: ListModelsFn = async (options) => { + const data = (await getJson( + `${baseUrl(options, 'https://api.anthropic.com')}/v1/models`, + { 'x-api-key': options.apiKey, 'anthropic-version': '2023-06-01' }, + options, + 'anthropic', + )) as { data?: Array<{ id: string; display_name?: string; created_at?: string }> }; + + return (data.data ?? []) + .slice() + .sort((a, b) => new Date(b.created_at ?? 0).getTime() - new Date(a.created_at ?? 0).getTime()) + .map((model) => ({ id: model.id, name: model.display_name ?? model.id })); +}; diff --git a/packages/@n8n/ai-utilities/src/model-discovery/providers/cohere.ts b/packages/@n8n/ai-utilities/src/model-discovery/providers/cohere.ts new file mode 100644 index 00000000000..c10562953a6 --- /dev/null +++ b/packages/@n8n/ai-utilities/src/model-discovery/providers/cohere.ts @@ -0,0 +1,17 @@ +import { baseUrl, bearerHeaders, byName, getJson } from '../request'; +import type { ListModelsFn } from '../types'; + +/** Source: LmChatCohere `loadOptions` routing (chat endpoint models only). */ +export const listCohereModels: ListModelsFn = async (options) => { + const data = (await getJson( + `${baseUrl(options, 'https://api.cohere.ai')}/v1/models?page_size=100&endpoint=chat`, + bearerHeaders(options), + options, + 'cohere', + )) as { models?: Array<{ name?: unknown }> }; + + return (data.models ?? []) + .filter((model): model is { name: string } => typeof model.name === 'string') + .map((model) => ({ id: model.name, name: model.name })) + .sort(byName); +}; diff --git a/packages/@n8n/ai-utilities/src/model-discovery/providers/deepseek.ts b/packages/@n8n/ai-utilities/src/model-discovery/providers/deepseek.ts new file mode 100644 index 00000000000..4f57b494800 --- /dev/null +++ b/packages/@n8n/ai-utilities/src/model-discovery/providers/deepseek.ts @@ -0,0 +1,4 @@ +import { makeBearerDataListing } from '../request'; + +/** Source: LmChatDeepSeek `loadOptions` routing. */ +export const listDeepSeekModels = makeBearerDataListing('deepseek', 'https://api.deepseek.com'); diff --git a/packages/@n8n/ai-utilities/src/model-discovery/providers/google.ts b/packages/@n8n/ai-utilities/src/model-discovery/providers/google.ts new file mode 100644 index 00000000000..f9b548490ec --- /dev/null +++ b/packages/@n8n/ai-utilities/src/model-discovery/providers/google.ts @@ -0,0 +1,29 @@ +import { baseUrl, byName, getJson } from '../request'; +import type { ListModelsFn } from '../types'; + +/** + * Source: LmChatGoogleGemini `loadOptions` routing (GET /v1beta/models, + * embedding/imagen excluded). Ids keep Google's `models/` prefix, matching the + * node dropdown values. Auth uses the `x-goog-api-key` header (Google's + * preferred method) rather than the credential's `?key=` query auth, so the + * key cannot leak through access logs or proxies. + */ +export const listGoogleModels: ListModelsFn = async (options) => { + const base = baseUrl(options, 'https://generativelanguage.googleapis.com'); + const data = (await getJson( + `${base}/v1beta/models`, + { 'x-goog-api-key': options.apiKey }, + options, + 'google', + )) as { models?: Array<{ name?: unknown }> }; + + return (data.models ?? []) + .filter( + (model): model is { name: string } => + typeof model.name === 'string' && + !model.name.includes('embedding') && + !model.name.includes('imagen'), + ) + .map((model) => ({ id: model.name, name: model.name })) + .sort(byName); +}; diff --git a/packages/@n8n/ai-utilities/src/model-discovery/providers/groq.ts b/packages/@n8n/ai-utilities/src/model-discovery/providers/groq.ts new file mode 100644 index 00000000000..2d3c747b36d --- /dev/null +++ b/packages/@n8n/ai-utilities/src/model-discovery/providers/groq.ts @@ -0,0 +1,16 @@ +import { baseUrl, bearerHeaders, getJson, idsToModels } from '../request'; +import type { ListModelsFn } from '../types'; + +/** Source: LmChatGroq `loadOptions` routing (active model objects only). */ +export const listGroqModels: ListModelsFn = async (options) => { + const data = (await getJson( + `${baseUrl(options, 'https://api.groq.com/openai/v1')}/models`, + bearerHeaders(options), + options, + 'groq', + )) as { data?: Array<{ id?: unknown; active?: unknown; object?: unknown }> }; + + return idsToModels( + (data.data ?? []).filter((model) => model.active === true && model.object === 'model'), + ); +}; diff --git a/packages/@n8n/ai-utilities/src/model-discovery/providers/mistral.ts b/packages/@n8n/ai-utilities/src/model-discovery/providers/mistral.ts new file mode 100644 index 00000000000..dbfb8a8078c --- /dev/null +++ b/packages/@n8n/ai-utilities/src/model-discovery/providers/mistral.ts @@ -0,0 +1,16 @@ +import { baseUrl, bearerHeaders, byName, getJson, idsToModels, type IdItem } from '../request'; +import type { ListModelsFn } from '../types'; + +/** Source: LmChatMistralCloud `loadOptions` routing (embedding models excluded). */ +export const listMistralModels: ListModelsFn = async (options) => { + const data = (await getJson( + `${baseUrl(options, 'https://api.mistral.ai/v1')}/models`, + bearerHeaders(options), + options, + 'mistral', + )) as { data?: IdItem[] }; + + return idsToModels(data.data ?? []) + .filter((model) => !model.id.includes('embed')) + .sort(byName); +}; diff --git a/packages/@n8n/ai-utilities/src/model-discovery/providers/nvidia.ts b/packages/@n8n/ai-utilities/src/model-discovery/providers/nvidia.ts new file mode 100644 index 00000000000..3cd12b2a0d3 --- /dev/null +++ b/packages/@n8n/ai-utilities/src/model-discovery/providers/nvidia.ts @@ -0,0 +1,27 @@ +import { baseUrl, bearerHeaders, byName, getJson, idsToModels, type IdItem } from '../request'; +import type { ListModelsFn } from '../types'; + +/** Source: LmChatNvidia `loadOptions` routing — mirrors NEMOTRON_SUPPORTED_MODELS. */ +const NVIDIA_SUPPORTED_MODELS = new Set([ + 'nvidia/llama-3.1-nemotron-nano-8b-v1', + 'nvidia/llama-3.3-nemotron-super-49b-v1', + 'nvidia/llama-3.3-nemotron-super-49b-v1.5', + 'nvidia/nemotron-3-nano-30b-a3b', + 'nvidia/nemotron-3-nano-omni-30b-a3b-reasoning', + 'nvidia/nemotron-3-super-120b-a12b', + 'nvidia/nemotron-nano-12b-v2-vl', + 'nvidia/nvidia-nemotron-nano-9b-v2', +]); + +export const listNvidiaModels: ListModelsFn = async (options) => { + const data = (await getJson( + `${baseUrl(options, 'https://integrate.api.nvidia.com/v1')}/models`, + bearerHeaders(options), + options, + 'nvidia', + )) as { data?: IdItem[] }; + + return idsToModels(data.data ?? []) + .filter((model) => NVIDIA_SUPPORTED_MODELS.has(model.id)) + .sort(byName); +}; diff --git a/packages/@n8n/ai-utilities/src/model-discovery/providers/openai.ts b/packages/@n8n/ai-utilities/src/model-discovery/providers/openai.ts new file mode 100644 index 00000000000..623df1fb910 --- /dev/null +++ b/packages/@n8n/ai-utilities/src/model-discovery/providers/openai.ts @@ -0,0 +1,39 @@ +import { baseUrl, bearerHeaders, byName, getJson, idsToModels, type IdItem } from '../request'; +import type { ListModelsFn } from '../types'; + +const OFFICIAL_OPENAI_HOSTNAMES = ['api.openai.com', 'ai-assistant.n8n.io']; + +/** + * Source: LMChatOpenAi `methods/loadModels.ts` — on the official API, exclude + * non-chat model families; on custom (proxy/self-hosted) hosts include all. + */ +export function shouldIncludeOpenAiModel(modelId: string, isCustomApi: boolean): boolean { + if (isCustomApi) return true; + return !( + modelId.startsWith('babbage') || + modelId.startsWith('davinci') || + modelId.startsWith('computer-use') || + modelId.startsWith('dall-e') || + modelId.startsWith('text-embedding') || + modelId.startsWith('tts') || + modelId.includes('-tts') || + modelId.startsWith('whisper') || + modelId.startsWith('omni-moderation') || + modelId.startsWith('sora') || + modelId.includes('-realtime') || + (modelId.startsWith('gpt-') && modelId.includes('instruct')) + ); +} + +/** Source: LMChatOpenAi `methods/loadModels.ts` (GET /models, filtered, id asc). */ +export const listOpenAiModels: ListModelsFn = async (options) => { + const base = baseUrl(options, 'https://api.openai.com/v1'); + const isCustomApi = !OFFICIAL_OPENAI_HOSTNAMES.includes(new URL(base).hostname); + const data = (await getJson(`${base}/models`, bearerHeaders(options), options, 'openai')) as { + data?: IdItem[]; + }; + + return idsToModels(data.data ?? []) + .filter((model) => shouldIncludeOpenAiModel(model.id, isCustomApi)) + .sort(byName); +}; diff --git a/packages/@n8n/ai-utilities/src/model-discovery/providers/openrouter.ts b/packages/@n8n/ai-utilities/src/model-discovery/providers/openrouter.ts new file mode 100644 index 00000000000..388bbfad164 --- /dev/null +++ b/packages/@n8n/ai-utilities/src/model-discovery/providers/openrouter.ts @@ -0,0 +1,7 @@ +import { makeBearerDataListing } from '../request'; + +/** Source: LmChatOpenRouter `loadOptions` routing. */ +export const listOpenRouterModels = makeBearerDataListing( + 'openrouter', + 'https://openrouter.ai/api/v1', +); diff --git a/packages/@n8n/ai-utilities/src/model-discovery/providers/vercel.ts b/packages/@n8n/ai-utilities/src/model-discovery/providers/vercel.ts new file mode 100644 index 00000000000..c70433d29cf --- /dev/null +++ b/packages/@n8n/ai-utilities/src/model-discovery/providers/vercel.ts @@ -0,0 +1,4 @@ +import { makeBearerDataListing } from '../request'; + +/** Source: LmChatVercelAiGateway `loadOptions` routing. */ +export const listVercelModels = makeBearerDataListing('vercel', 'https://ai-gateway.vercel.sh/v1'); diff --git a/packages/@n8n/ai-utilities/src/model-discovery/providers/xai.ts b/packages/@n8n/ai-utilities/src/model-discovery/providers/xai.ts new file mode 100644 index 00000000000..fe9af290d45 --- /dev/null +++ b/packages/@n8n/ai-utilities/src/model-discovery/providers/xai.ts @@ -0,0 +1,4 @@ +import { makeBearerDataListing } from '../request'; + +/** Source: LmChatXAiGrok `loadOptions` routing. */ +export const listXaiModels = makeBearerDataListing('xai', 'https://api.x.ai/v1'); diff --git a/packages/@n8n/ai-utilities/src/model-discovery/request.ts b/packages/@n8n/ai-utilities/src/model-discovery/request.ts new file mode 100644 index 00000000000..bf43ebb764a --- /dev/null +++ b/packages/@n8n/ai-utilities/src/model-discovery/request.ts @@ -0,0 +1,62 @@ +import type { ListModelsFn, ListModelsOptions, ProviderModel } from './types'; + +/** GET a provider endpoint and parse JSON, throwing a descriptive error on non-2xx. */ +export async function getJson( + url: string, + headers: Record, + options: ListModelsOptions, + provider: string, +): Promise { + const fetchFn = options.fetch ?? globalThis.fetch; + const response = await fetchFn(url, { + method: 'GET', + headers: { ...headers, ...options.headers }, + }); + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error( + `Failed to list ${provider} models (status ${response.status})${body ? `: ${body.slice(0, 500)}` : ''}`, + ); + } + return await response.json(); +} + +/** Resolve the API base: caller override or the provider default, without a trailing slash. */ +export function baseUrl(options: ListModelsOptions, fallback: string): string { + return (options.baseURL ?? fallback).replace(/\/+$/, ''); +} + +export function bearerHeaders(options: ListModelsOptions): Record { + return { Authorization: `Bearer ${options.apiKey}` }; +} + +export function byName(a: ProviderModel, b: ProviderModel): number { + return a.name.localeCompare(b.name); +} + +export interface IdItem { + id?: unknown; +} + +/** Map `{ id }` response items to models (name = id), dropping malformed entries. */ +export function idsToModels(items: IdItem[]): ProviderModel[] { + return items + .filter((item): item is { id: string } => typeof item.id === 'string' && item.id !== '') + .map((item) => ({ id: item.id, name: item.id })); +} + +/** + * The most common provider shape: `GET /models` with bearer auth + * returning `{ data: [{ id }] }`, listed by id ascending. + */ +export function makeBearerDataListing(provider: string, defaultBaseUrl: string): ListModelsFn { + return async (options) => { + const data = (await getJson( + `${baseUrl(options, defaultBaseUrl)}/models`, + bearerHeaders(options), + options, + provider, + )) as { data?: IdItem[] }; + return idsToModels(data.data ?? []).sort(byName); + }; +} diff --git a/packages/@n8n/ai-utilities/src/model-discovery/types.ts b/packages/@n8n/ai-utilities/src/model-discovery/types.ts new file mode 100644 index 00000000000..9b0d87e5893 --- /dev/null +++ b/packages/@n8n/ai-utilities/src/model-discovery/types.ts @@ -0,0 +1,22 @@ +/** A chat model as reported by the provider's own model-list API. */ +export interface ProviderModel { + /** Provider-native model id, exactly as the provider's API expects it. */ + id: string; + /** Human-readable name when the provider supplies one, otherwise the id. */ + name: string; +} + +export interface ListModelsOptions { + apiKey: string; + /** Override the provider's default API base URL (e.g. a proxy or self-hosted gateway). */ + baseURL?: string; + /** + * Transport to use for the request. Callers supply their environment's + * proxy-aware fetch; defaults to the global fetch. + */ + fetch?: typeof globalThis.fetch; + /** Extra headers merged into the request (e.g. custom credential headers). */ + headers?: Record; +} + +export type ListModelsFn = (options: ListModelsOptions) => Promise; diff --git a/packages/@n8n/api-types/src/agents/dto.ts b/packages/@n8n/api-types/src/agents/dto.ts index ddd074c32f0..0f6284accf2 100644 --- a/packages/@n8n/api-types/src/agents/dto.ts +++ b/packages/@n8n/api-types/src/agents/dto.ts @@ -54,6 +54,10 @@ export class ListAgentsQueryDto extends Z.class({ sortBy: z.enum(AGENTS_LIST_SORT_OPTIONS).optional(), }) {} +export class AgentProviderModelsQueryDto extends Z.class({ + credentialId: z.string().min(1).max(64).optional(), +}) {} + export class CreateAgentDto extends Z.class({ name: z.string().min(1), }) {} diff --git a/packages/@n8n/api-types/src/agents/model-providers.ts b/packages/@n8n/api-types/src/agents/model-providers.ts index 9fd834a193e..ba4805fcdd3 100644 --- a/packages/@n8n/api-types/src/agents/model-providers.ts +++ b/packages/@n8n/api-types/src/agents/model-providers.ts @@ -21,3 +21,26 @@ const AGENT_MODEL_PROVIDER_SET = new Set(AGENT_MODEL_PROVIDERS); export function isAgentModelProvider(provider: string): provider is AgentModelProvider { return AGENT_MODEL_PROVIDER_SET.has(provider); } + +/** A model offered in the agent model picker. Mirrors the catalog's `ModelInfo` shape. */ +export interface AgentCatalogModel { + id: string; + name: string; + releaseDate?: string; + reasoning: boolean; + toolCall: boolean; + cost?: { input: number; output: number; cacheRead?: number; cacheWrite?: number }; + limits?: { context?: number; output?: number }; +} + +/** Response of `GET /agents/v2/catalog/models/:provider`. */ +export interface AgentProviderModelsResponse { + provider: string; + /** + * True when the list was confirmed against the provider's own model API + * (only models the credential can actually call). False means the static + * catalog fallback, which may include models the provider has retired. + */ + verified: boolean; + models: AgentCatalogModel[]; +} diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMChatAnthropic/methods/__tests__/searchModels.test.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMChatAnthropic/methods/__tests__/searchModels.test.ts index 232b1431a42..ee65ff0828d 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LMChatAnthropic/methods/__tests__/searchModels.test.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMChatAnthropic/methods/__tests__/searchModels.test.ts @@ -1,104 +1,100 @@ import type { ILoadOptionsFunctions } from 'n8n-workflow'; import type { Mocked } from 'vitest'; -import { searchModels, type AnthropicModel } from '../searchModels'; +import { searchModels } from '../searchModels'; + +const mockModels = [ + { + id: 'claude-3-opus-20240229', + display_name: 'Claude 3 Opus', + type: 'model', + created_at: '2024-02-29T00:00:00Z', + }, + { + id: 'claude-3-sonnet-20240229', + display_name: 'Claude 3 Sonnet', + type: 'model', + created_at: '2024-02-29T00:00:00Z', + }, + { + id: 'claude-3-haiku-20240307', + display_name: 'Claude 3 Haiku', + type: 'model', + created_at: '2024-03-07T00:00:00Z', + }, + { + id: 'claude-2.1', + display_name: 'Claude 2.1', + type: 'model', + created_at: '2023-11-21T00:00:00Z', + }, + { + id: 'claude-2.0', + display_name: 'Claude 2.0', + type: 'model', + created_at: '2023-07-11T00:00:00Z', + }, +]; describe('searchModels', () => { let mockContext: Mocked; - - const mockModels: AnthropicModel[] = [ - { - id: 'claude-3-opus-20240229', - display_name: 'Claude 3 Opus', - type: 'model', - created_at: '2024-02-29T00:00:00Z', - }, - { - id: 'claude-3-sonnet-20240229', - display_name: 'Claude 3 Sonnet', - type: 'model', - created_at: '2024-02-29T00:00:00Z', - }, - { - id: 'claude-3-haiku-20240307', - display_name: 'Claude 3 Haiku', - type: 'model', - created_at: '2024-03-07T00:00:00Z', - }, - { - id: 'claude-2.1', - display_name: 'Claude 2.1', - type: 'model', - created_at: '2023-11-21T00:00:00Z', - }, - { - id: 'claude-2.0', - display_name: 'Claude 2.0', - type: 'model', - created_at: '2023-07-11T00:00:00Z', - }, - ]; + let fetchSpy: ReturnType; beforeEach(() => { mockContext = { - getCredentials: vi.fn().mockResolvedValue({}), - helpers: { - httpRequestWithAuthentication: vi.fn().mockResolvedValue({ - data: mockModels, - }), - }, + getCredentials: vi.fn().mockResolvedValue({ apiKey: 'test-api-key' }), } as unknown as Mocked; + + fetchSpy = vi.fn().mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ data: mockModels }), + text: async () => '', + }); + vi.stubGlobal('fetch', fetchSpy); }); afterEach(() => { + vi.unstubAllGlobals(); vi.clearAllMocks(); - // Reset the getCredentials mock to its default value - mockContext.getCredentials = vi.fn().mockResolvedValue({}); }); it('should fetch models from default Anthropic API URL when no custom URL is provided', async () => { const result = await searchModels.call(mockContext); expect(mockContext.getCredentials).toHaveBeenCalledWith('anthropicApi'); - expect(mockContext.helpers.httpRequestWithAuthentication).toHaveBeenCalledWith('anthropicApi', { - url: 'https://api.anthropic.com/v1/models', - headers: { - 'anthropic-version': '2023-06-01', - }, - }); + expect(fetchSpy).toHaveBeenCalledWith( + 'https://api.anthropic.com/v1/models', + expect.objectContaining({ + headers: expect.objectContaining({ + 'x-api-key': 'test-api-key', + 'anthropic-version': '2023-06-01', + }), + }), + ); expect(result.results).toHaveLength(5); }); it('should fetch models from custom Anthropic API URL when provided in credentials', async () => { const customUrl = 'https://custom-anthropic-api.example.com'; - // Override the default mock to return credentials with a custom URL - mockContext.getCredentials = vi.fn().mockResolvedValue({ url: customUrl }); + mockContext.getCredentials = vi + .fn() + .mockResolvedValue({ apiKey: 'test-api-key', url: customUrl }); const result = await searchModels.call(mockContext); - expect(mockContext.getCredentials).toHaveBeenCalledWith('anthropicApi'); - expect(mockContext.helpers.httpRequestWithAuthentication).toHaveBeenCalledWith('anthropicApi', { - url: `${customUrl}/v1/models`, - headers: { - 'anthropic-version': '2023-06-01', - }, - }); + expect(fetchSpy).toHaveBeenCalledWith(`${customUrl}/v1/models`, expect.anything()); expect(result.results).toHaveLength(5); }); it('should use default URL when empty URL is provided in credentials', async () => { - // Override the default mock to return credentials with an empty URL - mockContext.getCredentials = vi.fn().mockResolvedValue({ url: null }); + mockContext.getCredentials = vi + .fn() + .mockResolvedValue({ apiKey: 'test-api-key', url: undefined }); const result = await searchModels.call(mockContext); - expect(mockContext.getCredentials).toHaveBeenCalledWith('anthropicApi'); - expect(mockContext.helpers.httpRequestWithAuthentication).toHaveBeenCalledWith('anthropicApi', { - url: 'https://api.anthropic.com/v1/models', - headers: { - 'anthropic-version': '2023-06-01', - }, - }); + expect(fetchSpy).toHaveBeenCalledWith('https://api.anthropic.com/v1/models', expect.anything()); expect(result.results).toHaveLength(5); }); @@ -140,4 +136,25 @@ describe('searchModels', () => { expect(result.results).toHaveLength(0); }); + + it('should include the credential custom header in the request', async () => { + mockContext.getCredentials = vi.fn().mockResolvedValue({ + apiKey: 'test-api-key', + header: true, + headerName: 'X-Gateway-Auth', + headerValue: 'gateway-value', + }); + + await searchModels.call(mockContext); + + expect(fetchSpy).toHaveBeenCalledWith( + 'https://api.anthropic.com/v1/models', + expect.objectContaining({ + headers: expect.objectContaining({ + 'x-api-key': 'test-api-key', + 'X-Gateway-Auth': 'gateway-value', + }), + }), + ); + }); }); diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMChatAnthropic/methods/searchModels.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMChatAnthropic/methods/searchModels.ts index 94a926abf52..595410725b9 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LMChatAnthropic/methods/searchModels.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMChatAnthropic/methods/searchModels.ts @@ -1,63 +1,34 @@ -import type { - ILoadOptionsFunctions, - INodeListSearchItems, - INodeListSearchResult, -} from 'n8n-workflow'; +import { getProxyAgent } from '@n8n/ai-utilities'; +import { listAnthropicModels } from '@n8n/ai-utilities/model-discovery'; +import type { ILoadOptionsFunctions, INodeListSearchResult } from 'n8n-workflow'; -export interface AnthropicModel { - id: string; - display_name: string; - type: string; - created_at: string; -} +import { mergeCustomHeaders } from '../../../../utils/helpers'; export async function searchModels( this: ILoadOptionsFunctions, filter?: string, ): Promise { - const credentials = await this.getCredentials<{ url?: string }>('anthropicApi'); + const credentials = await this.getCredentials('anthropicApi'); + const baseURL = (credentials.url as string) ?? 'https://api.anthropic.com'; - const baseURL = credentials.url ?? 'https://api.anthropic.com'; - const response = (await this.helpers.httpRequestWithAuthentication.call(this, 'anthropicApi', { - url: `${baseURL}/v1/models`, - headers: { - 'anthropic-version': '2023-06-01', - }, - })) as { data: AnthropicModel[] }; - - const models = response.data || []; - let results: INodeListSearchItems[] = []; - - if (filter) { - for (const model of models) { - if (model.id.toLowerCase().includes(filter.toLowerCase())) { - results.push({ - name: model.display_name, - value: model.id, - }); - } - } - } else { - results = models.map((model) => ({ - name: model.display_name, - value: model.id, - })); - } - - // Sort models with more recent ones first (claude-3 before claude-2) - results = results.sort((a, b) => { - const modelA = models.find((m) => m.id === a.value); - const modelB = models.find((m) => m.id === b.value); - - if (!modelA || !modelB) return 0; - - // Sort by created_at date, most recent first - const dateA = new Date(modelA.created_at); - const dateB = new Date(modelB.created_at); - return dateB.getTime() - dateA.getTime(); + // Shared with the agents model catalog: endpoint, auth, and newest-first + // ordering live in @n8n/ai-utilities/model-discovery. The credential's + // optional custom header (for gateway/proxy-backed credentials) is merged in, + // matching what the credential's `authenticate` applies on other requests. + const models = await listAnthropicModels({ + apiKey: (credentials.apiKey as string) ?? '', + baseURL, + headers: mergeCustomHeaders(credentials, {}), + fetch: async (url, init) => + await fetch(url, { + ...init, + dispatcher: getProxyAgent(baseURL), + } as RequestInit), }); return { - results, + results: models + .filter((model) => !filter || model.id.toLowerCase().includes(filter.toLowerCase())) + .map((model) => ({ name: model.name, value: model.id })), }; } diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/methods/__tests__/loadModels.test.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/methods/__tests__/loadModels.test.ts index 5d205fc4bd1..b507fbfc7d4 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/methods/__tests__/loadModels.test.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/methods/__tests__/loadModels.test.ts @@ -1,14 +1,33 @@ import type { ILoadOptionsFunctions } from 'n8n-workflow'; -import OpenAI from 'openai'; -import type { Mocked, MockedClass } from 'vitest'; +import type { Mocked } from 'vitest'; import { searchModels } from '../loadModels'; -vi.mock('openai'); +const MODEL_IDS = [ + 'gpt-4', + 'gpt-3.5-turbo', + 'gpt-3.5-turbo-instruct', + 'ft:gpt-3.5-turbo', + 'o1-model', + 'whisper-1', + 'davinci-instruct-beta', + 'computer-use-preview', + 'whisper-1-preview', + 'tts-model', + 'other-model', +]; + +const OFFICIAL_API_RESULTS = [ + { name: 'ft:gpt-3.5-turbo', value: 'ft:gpt-3.5-turbo' }, + { name: 'gpt-3.5-turbo', value: 'gpt-3.5-turbo' }, + { name: 'gpt-4', value: 'gpt-4' }, + { name: 'o1-model', value: 'o1-model' }, + { name: 'other-model', value: 'other-model' }, +]; describe('searchModels', () => { let mockContext: Mocked; - let mockOpenAI: Mocked; + let fetchSpy: ReturnType; beforeEach(() => { mockContext = { @@ -18,73 +37,41 @@ describe('searchModels', () => { getNodeParameter: vi.fn().mockReturnValue(''), } as unknown as Mocked; - // Setup OpenAI mock with required properties - const mockOpenAIInstance = { - apiKey: 'test-api-key', - organization: null, - project: null, - _options: {}, - models: { - list: vi.fn().mockResolvedValue({ - data: [ - { id: 'gpt-4' }, - { id: 'gpt-3.5-turbo' }, - { id: 'gpt-3.5-turbo-instruct' }, - { id: 'ft:gpt-3.5-turbo' }, - { id: 'o1-model' }, - { id: 'whisper-1' }, - { id: 'davinci-instruct-beta' }, - { id: 'computer-use-preview' }, - { id: 'whisper-1-preview' }, - { id: 'tts-model' }, - { id: 'other-model' }, - ], - }), - }, - } as unknown as OpenAI; - - (OpenAI as MockedClass).mockImplementation(function () { - return mockOpenAIInstance; + fetchSpy = vi.fn().mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ data: MODEL_IDS.map((id) => ({ id })) }), + text: async () => '', }); - - mockOpenAI = OpenAI as Mocked; + vi.stubGlobal('fetch', fetchSpy); }); afterEach(() => { + vi.unstubAllGlobals(); vi.clearAllMocks(); }); it('should return filtered models if custom API endpoint is not provided', async () => { const result = await searchModels.call(mockContext); - expect(mockOpenAI).toHaveBeenCalledWith( + expect(fetchSpy).toHaveBeenCalledWith( + 'https://api.openai.com/v1/models', expect.objectContaining({ - baseURL: 'https://api.openai.com/v1', - apiKey: 'test-api-key', + headers: expect.objectContaining({ Authorization: 'Bearer test-api-key' }), }), ); - expect(result.results).toEqual([ - { name: 'ft:gpt-3.5-turbo', value: 'ft:gpt-3.5-turbo' }, - { name: 'gpt-3.5-turbo', value: 'gpt-3.5-turbo' }, - { name: 'gpt-4', value: 'gpt-4' }, - { name: 'o1-model', value: 'o1-model' }, - { name: 'other-model', value: 'other-model' }, - ]); + expect(result.results).toEqual(OFFICIAL_API_RESULTS); }); - it('should initialize OpenAI with correct credentials', async () => { + it('should use the credential url as the API base', async () => { mockContext.getCredentials.mockResolvedValueOnce({ apiKey: 'test-api-key', url: 'https://test-url.com', }); + await searchModels.call(mockContext); - expect(mockOpenAI).toHaveBeenCalledWith( - expect.objectContaining({ - baseURL: 'https://test-url.com', - apiKey: 'test-api-key', - }), - ); + expect(fetchSpy).toHaveBeenCalledWith('https://test-url.com/models', expect.anything()); }); it('should use default OpenAI URL if no custom URL provided', async () => { @@ -94,18 +81,15 @@ describe('searchModels', () => { await searchModels.call(mockContext); - expect(mockOpenAI).toHaveBeenCalledWith( - expect.objectContaining({ - baseURL: 'https://api.openai.com/v1', - apiKey: 'test-api-key', - }), - ); + expect(fetchSpy).toHaveBeenCalledWith('https://api.openai.com/v1/models', expect.anything()); }); it('should include all models for custom API endpoints', async () => { mockContext.getNodeParameter = vi.fn().mockReturnValue('https://custom-api.com'); const result = await searchModels.call(mockContext); + + expect(fetchSpy).toHaveBeenCalledWith('https://custom-api.com/models', expect.anything()); expect(result.results).toEqual([ { name: 'computer-use-preview', value: 'computer-use-preview' }, { name: 'davinci-instruct-beta', value: 'davinci-instruct-beta' }, @@ -130,13 +114,7 @@ describe('searchModels', () => { const result = await searchModels.call(mockContext); - expect(result.results).toEqual([ - { name: 'ft:gpt-3.5-turbo', value: 'ft:gpt-3.5-turbo' }, - { name: 'gpt-3.5-turbo', value: 'gpt-3.5-turbo' }, - { name: 'gpt-4', value: 'gpt-4' }, - { name: 'o1-model', value: 'o1-model' }, - { name: 'other-model', value: 'other-model' }, - ]); + expect(result.results).toEqual(OFFICIAL_API_RESULTS); }); it('should filter models based on search term', async () => { @@ -159,7 +137,7 @@ describe('searchModels', () => { ]); }); - it('should include custom credential headers in the OpenAI client', async () => { + it('should include custom credential headers in the request', async () => { mockContext.getCredentials.mockResolvedValueOnce({ apiKey: 'test-api-key', header: true, @@ -169,48 +147,11 @@ describe('searchModels', () => { await searchModels.call(mockContext); - expect(mockOpenAI).toHaveBeenCalledWith( + expect(fetchSpy).toHaveBeenCalledWith( + 'https://api.openai.com/v1/models', expect.objectContaining({ - defaultHeaders: expect.objectContaining({ - 'X-Custom-Auth': 'custom-value', - }), + headers: expect.objectContaining({ 'X-Custom-Auth': 'custom-value' }), }), ); }); - - it('should return models sorted alphabetically by id', async () => { - // Setup a mock with scrambled order - const mockUnsortedInstance = { - apiKey: 'test-api-key', - models: { - list: vi.fn().mockResolvedValue({ - data: [ - { id: 'gpt-4' }, - { id: 'a-model' }, - { id: 'o1-model' }, - { id: 'gpt-3.5-turbo' }, - { id: 'z-model' }, - ], - }), - }, - } as unknown as OpenAI; - - (OpenAI as MockedClass).mockImplementation(function () { - return mockUnsortedInstance; - }); - - // Custom API endpoint to include all models - mockContext.getNodeParameter = vi.fn().mockReturnValue('https://custom-api.com'); - - const result = await searchModels.call(mockContext); - - // Verify the results are sorted alphabetically - expect(result.results).toEqual([ - { name: 'a-model', value: 'a-model' }, - { name: 'gpt-3.5-turbo', value: 'gpt-3.5-turbo' }, - { name: 'gpt-4', value: 'gpt-4' }, - { name: 'o1-model', value: 'o1-model' }, - { name: 'z-model', value: 'z-model' }, - ]); - }); }); diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/methods/loadModels.ts b/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/methods/loadModels.ts index 57803188d35..dba8c158693 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/methods/loadModels.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LMChatOpenAi/methods/loadModels.ts @@ -1,11 +1,10 @@ import { getProxyAgent } from '@n8n/ai-utilities'; +import { listOpenAiModels } from '@n8n/ai-utilities/model-discovery'; import { AiConfig } from '@n8n/config'; import { Container } from '@n8n/di'; import type { ILoadOptionsFunctions, INodeListSearchResult } from 'n8n-workflow'; -import OpenAI from 'openai'; import { mergeCustomHeaders } from '../../../../utils/helpers'; -import { shouldIncludeModel } from '../../../vendors/OpenAi/helpers/modelFiltering'; export async function searchModels( this: ILoadOptionsFunctions, @@ -17,35 +16,24 @@ export async function searchModels( (credentials.url as string) || 'https://api.openai.com/v1'; const { openAiDefaultHeaders } = Container.get(AiConfig); - const defaultHeaders = mergeCustomHeaders(credentials, openAiDefaultHeaders ?? {}); + const headers = mergeCustomHeaders(credentials, openAiDefaultHeaders ?? {}); - const openai = new OpenAI({ - baseURL, + // Shared with the agents model catalog: endpoint, auth, chat-model filtering + // (including include-all on custom hosts) live in @n8n/ai-utilities/model-discovery. + const models = await listOpenAiModels({ apiKey: credentials.apiKey as string, - fetchOptions: { - dispatcher: getProxyAgent(baseURL), - }, - defaultHeaders, + baseURL, + headers, + fetch: async (url, init) => + await fetch(url, { + ...init, + dispatcher: getProxyAgent(baseURL), + } as RequestInit), }); - const { data: models = [] } = await openai.models.list(); - - const url = baseURL && new URL(baseURL); - const isCustomAPI = !!(url && !['api.openai.com', 'ai-assistant.n8n.io'].includes(url.hostname)); - - const filteredModels = models.filter((model: { id: string }) => { - const includeModel = shouldIncludeModel(model.id, isCustomAPI); - - if (!filter) return includeModel; - - return includeModel && model.id.toLowerCase().includes(filter.toLowerCase()); - }); - - filteredModels.sort((a, b) => a.id.localeCompare(b.id)); return { - results: filteredModels.map((model: { id: string }) => ({ - name: model.id, - value: model.id, - })), + results: models + .filter((model) => !filter || model.id.toLowerCase().includes(filter.toLowerCase())) + .map((model) => ({ name: model.id, value: model.id })), }; } diff --git a/packages/@n8n/nodes-langchain/nodes/llms/test/LmChatAnthropic.test.ts b/packages/@n8n/nodes-langchain/nodes/llms/test/LmChatAnthropic.test.ts index df2b79d3405..07a9eedf8af 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/test/LmChatAnthropic.test.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/test/LmChatAnthropic.test.ts @@ -419,20 +419,31 @@ describe('LmChatAnthropic', () => { describe('searchModels', () => { let mockLoadContext: ILoadOptionsFunctions; let mockGetCredentials: Mock; - let mockHttpRequest: Mock; + let fetchSpy: Mock; beforeEach(() => { mockGetCredentials = vi.fn(); - mockHttpRequest = vi.fn(); + fetchSpy = vi.fn(); + vi.stubGlobal('fetch', fetchSpy); mockLoadContext = { getCredentials: mockGetCredentials, - helpers: { - httpRequestWithAuthentication: mockHttpRequest, - }, } as unknown as ILoadOptionsFunctions; }); + afterEach(() => { + vi.unstubAllGlobals(); + }); + + function mockModelsResponse(models: unknown[]) { + fetchSpy.mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ data: models }), + text: async () => '', + }); + } + it('should return all models sorted by creation date', async () => { const mockModels = [ { @@ -455,20 +466,21 @@ describe('LmChatAnthropic', () => { }, ]; - mockGetCredentials.mockResolvedValue({}); - mockHttpRequest.mockResolvedValue({ - data: mockModels, - }); + mockGetCredentials.mockResolvedValue({ apiKey: 'test-api-key' }); + mockModelsResponse(mockModels); const { searchModels } = lmChatAnthropic.methods.listSearch; const result = await searchModels.call(mockLoadContext); - expect(mockHttpRequest).toHaveBeenCalledWith('anthropicApi', { - url: 'https://api.anthropic.com/v1/models', - headers: { - 'anthropic-version': '2023-06-01', - }, - }); + expect(fetchSpy).toHaveBeenCalledWith( + 'https://api.anthropic.com/v1/models', + expect.objectContaining({ + headers: expect.objectContaining({ + 'x-api-key': 'test-api-key', + 'anthropic-version': '2023-06-01', + }), + }), + ); expect(result.results).toHaveLength(3); // Verify sorted by creation date (newest first) @@ -499,10 +511,8 @@ describe('LmChatAnthropic', () => { }, ]; - mockGetCredentials.mockResolvedValue({}); - mockHttpRequest.mockResolvedValue({ - data: mockModels, - }); + mockGetCredentials.mockResolvedValue({ apiKey: 'test-api-key' }); + mockModelsResponse(mockModels); const { searchModels } = lmChatAnthropic.methods.listSearch; const result = await searchModels.call(mockLoadContext, 'opus'); @@ -522,10 +532,8 @@ describe('LmChatAnthropic', () => { }, ]; - mockGetCredentials.mockResolvedValue({}); - mockHttpRequest.mockResolvedValue({ - data: mockModels, - }); + mockGetCredentials.mockResolvedValue({ apiKey: 'test-api-key' }); + mockModelsResponse(mockModels); const { searchModels } = lmChatAnthropic.methods.listSearch; const result = await searchModels.call(mockLoadContext, 'SONNET'); @@ -538,28 +546,20 @@ describe('LmChatAnthropic', () => { const customURL = 'https://custom-anthropic.example.com'; mockGetCredentials.mockResolvedValue({ + apiKey: 'test-api-key', url: customURL, }); - mockHttpRequest.mockResolvedValue({ - data: [], - }); + mockModelsResponse([]); const { searchModels } = lmChatAnthropic.methods.listSearch; await searchModels.call(mockLoadContext); - expect(mockHttpRequest).toHaveBeenCalledWith('anthropicApi', { - url: `${customURL}/v1/models`, - headers: { - 'anthropic-version': '2023-06-01', - }, - }); + expect(fetchSpy).toHaveBeenCalledWith(`${customURL}/v1/models`, expect.anything()); }); it('should handle empty model list', async () => { - mockGetCredentials.mockResolvedValue({}); - mockHttpRequest.mockResolvedValue({ - data: [], - }); + mockGetCredentials.mockResolvedValue({ apiKey: 'test-api-key' }); + mockModelsResponse([]); const { searchModels } = lmChatAnthropic.methods.listSearch; const result = await searchModels.call(mockLoadContext); diff --git a/packages/cli/src/modules/agents/__tests__/agent-model-catalog.service.test.ts b/packages/cli/src/modules/agents/__tests__/agent-model-catalog.service.test.ts new file mode 100644 index 00000000000..fc3de616b06 --- /dev/null +++ b/packages/cli/src/modules/agents/__tests__/agent-model-catalog.service.test.ts @@ -0,0 +1,219 @@ +import type { User } from '@n8n/db'; +import { mockLogger } from '@n8n/backend-test-utils'; +import { mock } from 'vitest-mock-extended'; + +import { AgentModelCatalogService } from '../agent-model-catalog.service'; +import type { BuilderModelLiveLookupService } from '../builder/builder-model-live-lookup.service'; + +const fetchProviderCatalog = vi.fn(); +vi.mock('@n8n/agents', () => ({ + fetchProviderCatalog: (...args: unknown[]) => fetchProviderCatalog(...args) as unknown, +})); + +const user = mock({ id: 'user-1' }); +const credentialId = 'cred-1'; + +const catalogFixture = { + anthropic: { + id: 'anthropic', + name: 'Anthropic', + models: { + 'claude-sonnet-4-6': { + id: 'claude-sonnet-4-6', + name: 'Claude Sonnet 4.6', + reasoning: true, + toolCall: true, + cost: { input: 3, output: 15 }, + }, + 'claude-opus-4-0': { + id: 'claude-opus-4-0', + name: 'Claude Opus 4', + reasoning: true, + toolCall: true, + }, + }, + }, + google: { + id: 'google', + name: 'Google', + models: { + 'gemini-2.5-flash': { + id: 'gemini-2.5-flash', + name: 'Gemini 2.5 Flash', + reasoning: true, + toolCall: true, + }, + }, + }, +}; + +function makeService() { + const lookupService = mock(); + const service = new AgentModelCatalogService(mockLogger(), lookupService); + return { service, lookupService }; +} + +describe('AgentModelCatalogService', () => { + beforeEach(() => { + fetchProviderCatalog.mockReset(); + fetchProviderCatalog.mockResolvedValue(catalogFixture); + }); + + it('keeps catalog models the provider still reports live, with catalog metadata, and prunes the rest', async () => { + const { service, lookupService } = makeService(); + // Provider reports Sonnet but not Opus — Opus (retired) must be pruned. + lookupService.list.mockResolvedValue([ + { name: 'Claude Sonnet 4.6', value: 'claude-sonnet-4-6' }, + ]); + + const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId); + + expect(result.verified).toBe(true); + expect(result.models).toHaveLength(1); + expect(result.models[0]).toMatchObject({ + id: 'claude-sonnet-4-6', + name: 'Claude Sonnet 4.6', + cost: { input: 3, output: 15 }, + }); + expect(lookupService.list).toHaveBeenCalledWith( + user, + 'project-1', + credentialId, + 'anthropicApi', + 'anthropic', + ); + }); + + it('verifies a catalog alias when the provider lists only its dated snapshot', async () => { + const { service, lookupService } = makeService(); + fetchProviderCatalog.mockResolvedValue({ + anthropic: { + id: 'anthropic', + name: 'Anthropic', + models: { + // models.dev keeps the versionless alias and drops the snapshot… + 'claude-haiku-4-5': { + id: 'claude-haiku-4-5', + name: 'Claude Haiku 4.5', + reasoning: true, + toolCall: true, + }, + 'claude-opus-4-0': { + id: 'claude-opus-4-0', + name: 'Claude Opus 4', + reasoning: true, + toolCall: true, + }, + }, + }, + }); + // …while Anthropic's API lists the dated snapshot only. The snapshot must + // verify its alias; the retired alias with no live counterpart stays pruned. + lookupService.list.mockResolvedValue([ + { name: 'Claude Haiku 4.5', value: 'claude-haiku-4-5-20251001' }, + ]); + + const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId); + + expect(result.verified).toBe(true); + expect(result.models.map((m) => m.id)).toEqual(['claude-haiku-4-5']); + }); + + it('does not add live models that are missing from the catalog', async () => { + const { service, lookupService } = makeService(); + // Live list includes a model models.dev has no entry for, alongside a known one. + lookupService.list.mockResolvedValue([ + { name: 'Claude Sonnet 4.6', value: 'claude-sonnet-4-6' }, + { name: 'Claude Brand New', value: 'claude-brand-new' }, + ]); + + const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId); + + expect(result.verified).toBe(true); + // Only the catalog-known model survives; the live-only one is never added. + expect(result.models.map((m) => m.id)).toEqual(['claude-sonnet-4-6']); + }); + + it('strips the "models/" prefix from google model ids before matching', async () => { + const { service, lookupService } = makeService(); + lookupService.list.mockResolvedValue([ + { name: 'models/gemini-2.5-flash', value: 'models/gemini-2.5-flash' }, + ]); + + const result = await service.getProviderModels(user, 'project-1', 'google', credentialId); + + expect(result.models).toEqual([ + expect.objectContaining({ id: 'gemini-2.5-flash', name: 'Gemini 2.5 Flash' }), + ]); + }); + + it('falls back to the catalog list (verified: false) when the live lookup fails', async () => { + const { service, lookupService } = makeService(); + lookupService.list.mockRejectedValue(new Error('provider is down')); + + const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId); + + expect(result.verified).toBe(false); + expect(result.models.map((m) => m.id).sort()).toEqual(['claude-opus-4-0', 'claude-sonnet-4-6']); + }); + + it('falls back to the catalog list (verified: false) when no credential is provided', async () => { + const { service, lookupService } = makeService(); + + const result = await service.getProviderModels(user, 'project-1', 'anthropic', undefined); + + expect(result.verified).toBe(false); + expect(result.models).toHaveLength(2); + expect(lookupService.list).not.toHaveBeenCalled(); + }); + + it('falls back to the catalog list for providers without a live lookup', async () => { + const { service, lookupService } = makeService(); + fetchProviderCatalog.mockResolvedValue({ + 'aws-bedrock': { + id: 'aws-bedrock', + name: 'AWS Bedrock', + models: { + 'anthropic.claude-sonnet-4-6-v1:0': { + id: 'anthropic.claude-sonnet-4-6-v1:0', + name: 'Claude Sonnet 4.6', + reasoning: true, + toolCall: true, + }, + }, + }, + }); + + const result = await service.getProviderModels(user, 'project-1', 'aws-bedrock', credentialId); + + expect(result.verified).toBe(false); + expect(result.models).toHaveLength(1); + expect(lookupService.list).not.toHaveBeenCalled(); + }); + + it('still returns live models (verified: true) when the catalog fetch fails', async () => { + const { service, lookupService } = makeService(); + fetchProviderCatalog.mockRejectedValue(new Error('models.dev unreachable')); + lookupService.list.mockResolvedValue([ + { name: 'Claude Sonnet 4.6', value: 'claude-sonnet-4-6' }, + ]); + + const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId); + + expect(result.verified).toBe(true); + expect(result.models).toEqual([ + expect.objectContaining({ id: 'claude-sonnet-4-6', name: 'Claude Sonnet 4.6' }), + ]); + }); + + it('returns an empty unverified list when both the lookup and the catalog fail', async () => { + const { service, lookupService } = makeService(); + fetchProviderCatalog.mockRejectedValue(new Error('models.dev unreachable')); + lookupService.list.mockRejectedValue(new Error('provider is down')); + + const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId); + + expect(result.verified).toBe(false); + expect(result.models).toEqual([]); + }); +}); diff --git a/packages/cli/src/modules/agents/__tests__/agents-builder-tools.service.test.ts b/packages/cli/src/modules/agents/__tests__/agents-builder-tools.service.test.ts index 9481ef13cf1..7663ed37433 100644 --- a/packages/cli/src/modules/agents/__tests__/agents-builder-tools.service.test.ts +++ b/packages/cli/src/modules/agents/__tests__/agents-builder-tools.service.test.ts @@ -32,7 +32,7 @@ import { AgentsBuilderToolsService, getAgentConfigHash, } from '../builder/agents-builder-tools.service'; -import type { BuilderModelLookupService } from '../builder/builder-model-lookup.service'; +import type { BuilderModelLiveLookupService } from '../builder/builder-model-live-lookup.service'; import { BUILDER_TOOLS } from '../builder/builder-tool-names'; import type { Agent } from '../entities/agent.entity'; import type { AgentRepository } from '../repositories/agent.repository'; @@ -69,7 +69,7 @@ function makeService() { const secureRuntime = mock(); const workflowRepository = mock(); const agentsToolsService = mock(); - const builderModelLookupService = mock(); + const builderModelLiveLookupService = mock(); const credentialTypes = mock(); const mcpRegistryService = mock(); const agentTaskService = mock(); @@ -97,7 +97,7 @@ function makeService() { secureRuntime, workflowRepository, agentsToolsService, - builderModelLookupService, + builderModelLiveLookupService, mcpRegistryService, mock(), credentialTypes, diff --git a/packages/cli/src/modules/agents/__tests__/agents-catalog.controller.test.ts b/packages/cli/src/modules/agents/__tests__/agents-catalog.controller.test.ts index 4509e886bfa..6db7179ecfe 100644 --- a/packages/cli/src/modules/agents/__tests__/agents-catalog.controller.test.ts +++ b/packages/cli/src/modules/agents/__tests__/agents-catalog.controller.test.ts @@ -1,3 +1,12 @@ +import type { AgentProviderModelsQueryDto } from '@n8n/api-types'; +import type { AuthenticatedRequest } from '@n8n/db'; +import type { Response } from 'express'; +import { mock } from 'vitest-mock-extended'; + +import { BadRequestError } from '@/errors/response-errors/bad-request.error'; + +import type { AgentIntegrationPersistenceService } from '../agent-integration-persistence.service'; +import type { AgentModelCatalogService } from '../agent-model-catalog.service'; import { AgentsCatalogController } from '../agents-catalog.controller'; import { expectProjectScopedAgentRoutes, @@ -11,8 +20,56 @@ describe('AgentsCatalogController route access scopes', () => { it.each([ ['getModelCatalog', 'agent:read'], + ['getProviderModels', 'agent:read'], ['listIntegrations', 'agent:read'], ])('%s uses %s', (handlerName, scope) => { expect(routes.get(handlerName)?.accessScope?.scope).toBe(scope); }); }); + +describe('AgentsCatalogController — getProviderModels', () => { + function makeController() { + const modelCatalogService = mock(); + const controller = new AgentsCatalogController( + mock(), + modelCatalogService, + ); + return { controller, modelCatalogService }; + } + + const req = mock>({ + user: { id: 'user-1' }, + params: { projectId: 'project-1' }, + }); + + it('delegates to the model catalog service with the user, project, and credential', async () => { + const { controller, modelCatalogService } = makeController(); + const response = { provider: 'anthropic', verified: true, models: [] }; + modelCatalogService.getProviderModels.mockResolvedValue(response); + + const result = await controller.getProviderModels(req, mock(), 'anthropic', { + credentialId: 'cred-1', + } as AgentProviderModelsQueryDto); + + expect(result).toBe(response); + expect(modelCatalogService.getProviderModels).toHaveBeenCalledWith( + req.user, + 'project-1', + 'anthropic', + 'cred-1', + ); + }); + + it('rejects unknown providers', async () => { + const { controller } = makeController(); + + await expect( + controller.getProviderModels( + req, + mock(), + 'not-a-provider', + {} as AgentProviderModelsQueryDto, + ), + ).rejects.toThrow(BadRequestError); + }); +}); diff --git a/packages/cli/src/modules/agents/agent-model-catalog.service.ts b/packages/cli/src/modules/agents/agent-model-catalog.service.ts new file mode 100644 index 00000000000..c0955e37617 --- /dev/null +++ b/packages/cli/src/modules/agents/agent-model-catalog.service.ts @@ -0,0 +1,144 @@ +import type { AgentCatalogModel, AgentProviderModelsResponse } from '@n8n/api-types'; +import { Logger } from '@n8n/backend-common'; +import type { User } from '@n8n/db'; +import { Service } from '@n8n/di'; + +import { isModelDiscoveryProvider } from '@n8n/ai-utilities/model-discovery'; + +import { BuilderModelLiveLookupService } from './builder/builder-model-live-lookup.service'; +import { LLM_PROVIDER_DEFAULTS } from './builder/interactive/llm-provider-defaults'; + +/** Google's models API returns ids as `models/`; the AI SDK expects the bare id. */ +const GOOGLE_MODEL_ID_PREFIX = 'models/'; + +function getProviderCredentialType(provider: string): string | undefined { + if (!isModelDiscoveryProvider(provider)) return undefined; + for (const [credentialType, entry] of Object.entries(LLM_PROVIDER_DEFAULTS)) { + if (entry.provider === provider) return credentialType; + } + return undefined; +} + +function normalizeLiveModelValue(provider: string, value: string): string { + if (provider === 'google' && value.startsWith(GOOGLE_MODEL_ID_PREFIX)) { + return value.slice(GOOGLE_MODEL_ID_PREFIX.length); + } + return value; +} + +/** Dated snapshot suffixes: Anthropic `-20251001`, OpenAI `-2024-08-06`. */ +const SNAPSHOT_SUFFIX = /-(?:\d{8}|\d{4}-\d{2}-\d{2})$/; + +/** + * The ids a live model verifies. Providers list older models only as dated + * snapshots (e.g. `claude-haiku-4-5-20251001`) while the catalog prefers the + * versionless alias (`claude-haiku-4-5`, which providers resolve to the latest + * snapshot) — so a listed snapshot also verifies its alias. A retired alias + * still prunes: retired models have no live snapshot either. + */ +function liveModelIdVariants(id: string): string[] { + const alias = id.replace(SNAPSHOT_SUFFIX, ''); + return alias === id ? [id] : [id, alias]; +} + +/** + * Builds the model list offered in the agent model picker for one provider. + * + * The curated models.dev catalog is the display list — it is the up-to-date set + * of chat models with names, cost, and limits. But it lags provider + * retirements, so a listed model can 404 at call time. When a credential is + * available we verify the catalog against the provider's own model API (via the + * shared `@n8n/ai-utilities/model-discovery` functions) and prune any catalog + * entry the provider no longer reports. We never add live-only + * models: provider `/models` endpoints return every variant/snapshot and would + * overload the picker. Without a credential, or when the provider has no list + * API wired up, the catalog list is returned unpruned with `verified: false`. + */ +@Service() +export class AgentModelCatalogService { + constructor( + private readonly logger: Logger, + private readonly builderModelLiveLookupService: BuilderModelLiveLookupService, + ) {} + + async getProviderModels( + user: User, + projectId: string, + provider: string, + credentialId?: string, + ): Promise { + const catalogModels = await this.getCatalogModels(provider); + const credentialType = getProviderCredentialType(provider); + + if (!credentialId || !credentialType) { + return { provider, verified: false, models: Object.values(catalogModels) }; + } + + let liveModels: Array<{ name: string; value: string }>; + try { + liveModels = await this.builderModelLiveLookupService.list( + user, + projectId, + credentialId, + credentialType, + provider, + ); + } catch (error) { + this.logger.warn('Live model list failed — falling back to the static catalog', { + provider, + error: error instanceof Error ? error.message : String(error), + }); + return { provider, verified: false, models: Object.values(catalogModels) }; + } + + const liveModelIds = new Set( + liveModels.flatMap((live) => + liveModelIdVariants(normalizeLiveModelValue(provider, live.value)), + ), + ); + const catalogList = Object.values(catalogModels); + + // models.dev is the curated display list; the live lookup only verifies it. + // Provider `/models` endpoints return every variant/snapshot, so we never + // add live-only models — we only prune catalog entries the provider no + // longer reports (retired ids that would 404 at call time). + if (catalogList.length > 0) { + return { + provider, + verified: true, + models: catalogList.filter((model) => liveModelIds.has(model.id)), + }; + } + + // Catalog unavailable (models.dev down or no entry for this provider): there + // is no curated list to prune against, so show the verified live list rather + // than an empty picker. + return { + provider, + verified: true, + models: liveModels.map((live) => { + const id = normalizeLiveModelValue(provider, live.value); + return { + id, + name: normalizeLiveModelValue(provider, live.name) || id, + reasoning: false, + toolCall: true, + }; + }), + }; + } + + private async getCatalogModels(provider: string): Promise> { + try { + const { fetchProviderCatalog } = await import('@n8n/agents'); + const catalog = await fetchProviderCatalog(); + return catalog[provider]?.models ?? {}; + } catch (error) { + this.logger.warn('Model catalog fetch failed', { + provider, + error: error instanceof Error ? error.message : String(error), + }); + return {}; + } + } +} diff --git a/packages/cli/src/modules/agents/agents-catalog.controller.ts b/packages/cli/src/modules/agents/agents-catalog.controller.ts index e07f28efd53..09180041682 100644 --- a/packages/cli/src/modules/agents/agents-catalog.controller.ts +++ b/packages/cli/src/modules/agents/agents-catalog.controller.ts @@ -1,13 +1,24 @@ -import type { ChatIntegrationDescriptor } from '@n8n/api-types'; -import { Get, ProjectScope, RestController } from '@n8n/decorators'; +import { + AgentProviderModelsQueryDto, + isAgentModelProvider, + type AgentProviderModelsResponse, + type ChatIntegrationDescriptor, +} from '@n8n/api-types'; +import type { AuthenticatedRequest } from '@n8n/db'; +import { Get, Param, ProjectScope, Query, RestController } from '@n8n/decorators'; +import type { Response } from 'express'; + +import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { AgentIntegrationPersistenceService } from './agent-integration-persistence.service'; +import { AgentModelCatalogService } from './agent-model-catalog.service'; import { filterOfferedAgentModelProviders } from './model-catalog'; @RestController('/projects/:projectId/agents/v2') export class AgentsCatalogController { constructor( private readonly agentIntegrationPersistenceService: AgentIntegrationPersistenceService, + private readonly agentModelCatalogService: AgentModelCatalogService, ) {} @Get('/catalog/models') @@ -17,6 +28,25 @@ export class AgentsCatalogController { return filterOfferedAgentModelProviders(await fetchProviderCatalog()); } + @Get('/catalog/models/:provider') + @ProjectScope('agent:read') + async getProviderModels( + req: AuthenticatedRequest<{ projectId: string }>, + _res: Response, + @Param('provider') provider: string, + @Query query: AgentProviderModelsQueryDto, + ): Promise { + if (!isAgentModelProvider(provider)) { + throw new BadRequestError(`Unknown model provider "${provider}"`); + } + return await this.agentModelCatalogService.getProviderModels( + req.user, + req.params.projectId, + provider, + query.credentialId, + ); + } + @Get('/catalog/integrations') @ProjectScope('agent:read') listIntegrations(): ChatIntegrationDescriptor[] { diff --git a/packages/cli/src/modules/agents/builder/__tests__/builder-model-live-lookup.service.test.ts b/packages/cli/src/modules/agents/builder/__tests__/builder-model-live-lookup.service.test.ts new file mode 100644 index 00000000000..89a33910818 --- /dev/null +++ b/packages/cli/src/modules/agents/builder/__tests__/builder-model-live-lookup.service.test.ts @@ -0,0 +1,108 @@ +import type { CustomFetch, HttpTransport, OutboundHttp } from '@n8n/backend-network'; +import type { CredentialsEntity, User } from '@n8n/db'; +import { mock } from 'vitest-mock-extended'; + +import type { CredentialsFinderService } from '@/credentials/credentials-finder.service'; +import type { CredentialsService } from '@/credentials/credentials.service'; + +import { BuilderModelLiveLookupService } from '../builder-model-live-lookup.service'; + +const listModelsForProvider = vi.fn(); +vi.mock('@n8n/ai-utilities/model-discovery', () => ({ + listModelsForProvider: (...args: unknown[]) => listModelsForProvider(...args) as unknown, +})); + +const user = mock({ id: 'user-1' }); +const projectId = 'project-1'; + +function makeService() { + const credentialsService = mock(); + const credentialsFinderService = mock(); + const transport = mock(); + transport.asCustomFetch.mockReturnValue(vi.fn() as unknown as CustomFetch); + const outboundHttp = mock(); + outboundHttp.transport.mockReturnValue(transport); + + const service = new BuilderModelLiveLookupService( + credentialsService, + credentialsFinderService, + outboundHttp, + ); + return { service, credentialsService, credentialsFinderService }; +} + +function usable(id: string, type: string) { + return [{ id, name: 'My Credential', type }] as Awaited< + ReturnType + >; +} + +describe('BuilderModelLiveLookupService', () => { + beforeEach(() => { + listModelsForProvider.mockReset(); + }); + + it('lists models for a credential the user can use in the project', async () => { + const { service, credentialsService, credentialsFinderService } = makeService(); + credentialsService.getCredentialsAUserCanUseInAWorkflow.mockResolvedValue( + usable('cred-1', 'anthropicApi'), + ); + credentialsFinderService.findCredentialById.mockResolvedValue(mock()); + credentialsService.decrypt.mockResolvedValue({ apiKey: 'sk-key', url: 'https://proxy.local' }); + listModelsForProvider.mockResolvedValue([ + { id: 'claude-sonnet-4-6', name: 'Claude Sonnet 4.6' }, + ]); + + const result = await service.list(user, projectId, 'cred-1', 'anthropicApi', 'anthropic'); + + expect(result).toEqual([{ name: 'Claude Sonnet 4.6', value: 'claude-sonnet-4-6' }]); + expect(credentialsService.getCredentialsAUserCanUseInAWorkflow).toHaveBeenCalledWith(user, { + projectId, + }); + // Credential fields are mapped for the provider (anthropic: apiKey + url→baseURL). + expect(listModelsForProvider).toHaveBeenCalledWith( + 'anthropic', + expect.objectContaining({ apiKey: 'sk-key', baseURL: 'https://proxy.local' }), + ); + }); + + it('treats an empty provider response as a failed lookup', async () => { + const { service, credentialsService, credentialsFinderService } = makeService(); + credentialsService.getCredentialsAUserCanUseInAWorkflow.mockResolvedValue( + usable('cred-1', 'anthropicApi'), + ); + credentialsFinderService.findCredentialById.mockResolvedValue(mock()); + credentialsService.decrypt.mockResolvedValue({ apiKey: 'sk-key' }); + // An empty list from a chat provider is far more likely a broken request + // or drifted response shape than a real zero-model account — callers must + // fall back rather than prune everything. + listModelsForProvider.mockResolvedValue([]); + + await expect( + service.list(user, projectId, 'cred-1', 'anthropicApi', 'anthropic'), + ).rejects.toThrow('returned no models'); + }); + + it('rejects a credential that is not available in the project', async () => { + const { service, credentialsService } = makeService(); + // The user can read this credential, but it is not in the project's set. + credentialsService.getCredentialsAUserCanUseInAWorkflow.mockResolvedValue([]); + + await expect( + service.list(user, projectId, 'cred-other-project', 'anthropicApi', 'anthropic'), + ).rejects.toThrow('not found or not accessible'); + expect(listModelsForProvider).not.toHaveBeenCalled(); + }); + + it('rejects a credential whose type does not match the provider', async () => { + const { service, credentialsService } = makeService(); + credentialsService.getCredentialsAUserCanUseInAWorkflow.mockResolvedValue( + usable('cred-1', 'openAiApi'), + ); + + await expect( + service.list(user, projectId, 'cred-1', 'anthropicApi', 'anthropic'), + ).rejects.toThrow('not found or not accessible'); + expect(listModelsForProvider).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cli/src/modules/agents/builder/agents-builder-tools.service.ts b/packages/cli/src/modules/agents/builder/agents-builder-tools.service.ts index 62abbe6c751..a72d95782b8 100644 --- a/packages/cli/src/modules/agents/builder/agents-builder-tools.service.ts +++ b/packages/cli/src/modules/agents/builder/agents-builder-tools.service.ts @@ -43,7 +43,7 @@ import { AgentSkillsService } from '../agent-skills.service'; import { AgentTaskService } from '../agent-task.service'; import { AgentsToolsService } from '../agents-tools.service'; import { AgentsService } from '../agents.service'; -import { BuilderModelLookupService } from './builder-model-lookup.service'; +import { BuilderModelLiveLookupService } from './builder-model-live-lookup.service'; import { BUILDER_TOOLS } from './builder-tool-names'; import { collectFromAiParameterReferences, @@ -311,7 +311,7 @@ export class AgentsBuilderToolsService { private readonly secureRuntime: AgentSecureRuntime, private readonly workflowRepository: WorkflowRepository, private readonly agentsToolsService: AgentsToolsService, - private readonly builderModelLookupService: BuilderModelLookupService, + private readonly builderModelLiveLookupService: BuilderModelLiveLookupService, private readonly mcpRegistryService: McpRegistryService, private readonly oauthService: OauthService, private readonly credentialTypes: CredentialTypes, @@ -671,8 +671,14 @@ export class AgentsBuilderToolsService { .build(); const modelLookup: ModelLookup = { - list: async (credentialId, credentialType, lookup) => - await this.builderModelLookupService.list(user, credentialId, credentialType, lookup), + list: async (credentialId, credentialType, provider) => + await this.builderModelLiveLookupService.list( + user, + projectId, + credentialId, + credentialType, + provider, + ), }; const tools: BuiltTool[] = [ diff --git a/packages/cli/src/modules/agents/builder/builder-model-live-lookup.service.ts b/packages/cli/src/modules/agents/builder/builder-model-live-lookup.service.ts new file mode 100644 index 00000000000..1721e12c1a8 --- /dev/null +++ b/packages/cli/src/modules/agents/builder/builder-model-live-lookup.service.ts @@ -0,0 +1,78 @@ +import { OutboundHttp } from '@n8n/backend-network'; +import type { User } from '@n8n/db'; +import { Service } from '@n8n/di'; + +import { CredentialsFinderService } from '@/credentials/credentials-finder.service'; +import { CredentialsService } from '@/credentials/credentials.service'; +import { createAiProxyFetch } from '@/utils/ai-proxy-fetch'; + +import { mapCredentialForProvider } from '../json-config/credential-field-mapping'; + +/** + * Fetches a provider's live chat-model list for a credential, via the shared + * `@n8n/ai-utilities/model-discovery` functions (the same provider knowledge + * that backs the chat sub-nodes' model dropdowns). Nothing from + * `@n8n/n8n-nodes-langchain` is loaded on this path. + * + * The credential must be usable by the user within the given project — the + * same set as the workflow editor's credential picker. + */ +@Service() +export class BuilderModelLiveLookupService { + constructor( + private readonly credentialsService: CredentialsService, + private readonly credentialsFinderService: CredentialsFinderService, + private readonly outboundHttp: OutboundHttp, + ) {} + + /** + * Returns `{ name, value }` pairs (value = the provider's model id, exactly + * as the provider API expects it). Throws if the credential is not usable by + * the user in the project, its type doesn't match, or the provider has no + * model discovery support. + */ + async list( + user: User, + projectId: string, + credentialId: string, + credentialType: string, + provider: string, + ): Promise> { + const usableCredentials = await this.credentialsService.getCredentialsAUserCanUseInAWorkflow( + user, + { projectId }, + ); + const usable = usableCredentials.find((c) => c.id === credentialId); + if (!usable || usable.type !== credentialType) { + throw new Error(`Credential ${credentialId} not found or not accessible`); + } + + const credential = await this.credentialsFinderService.findCredentialById(credentialId); + if (!credential) { + throw new Error(`Credential ${credentialId} not found or not accessible`); + } + const rawData = await this.credentialsService.decrypt(credential, true); + + const { listModelsForProvider } = await import('@n8n/ai-utilities/model-discovery'); + const mapped = mapCredentialForProvider(provider, { apiKey: '', ...rawData }); + const apiKey = typeof mapped.apiKey === 'string' ? mapped.apiKey : ''; + const baseURL = + typeof mapped.baseURL === 'string' && mapped.baseURL ? mapped.baseURL : undefined; + + const models = await listModelsForProvider(provider, { + apiKey, + baseURL, + fetch: createAiProxyFetch(this.outboundHttp) as typeof globalThis.fetch, + }); + + // Every supported chat provider offers models, so an empty list means a + // broken request or a drifted response shape, not a zero-model account. + // Throw so callers fall back (unverified catalog / lookup-failed) instead + // of treating "nothing" as a verified answer and pruning every model. + if (models.length === 0) { + throw new Error(`Provider ${provider} returned no models`); + } + + return models.map((model) => ({ name: model.name, value: model.id })); + } +} diff --git a/packages/cli/src/modules/agents/builder/builder-model-lookup.service.ts b/packages/cli/src/modules/agents/builder/builder-model-lookup.service.ts deleted file mode 100644 index e29a4fc6889..00000000000 --- a/packages/cli/src/modules/agents/builder/builder-model-lookup.service.ts +++ /dev/null @@ -1,82 +0,0 @@ -import type { User } from '@n8n/db'; -import { ProjectRepository } from '@n8n/db'; -import { Service } from '@n8n/di'; -import type { INodeCredentials, INodeParameters } from 'n8n-workflow'; - -import { CredentialsFinderService } from '@/credentials/credentials-finder.service'; -import { NodeTypes } from '@/node-types'; -import { DynamicNodeParametersService } from '@/services/dynamic-node-parameters.service'; -import { getBase } from '@/workflow-execute-additional-data'; - -import type { ModelLookupConfig } from './interactive/llm-provider-defaults'; - -@Service() -export class BuilderModelLookupService { - constructor( - private readonly credentialsFinderService: CredentialsFinderService, - private readonly projectRepository: ProjectRepository, - private readonly dynamicNodeParametersService: DynamicNodeParametersService, - private readonly nodeTypes: NodeTypes, - ) {} - - async list( - user: User, - credentialId: string, - credentialType: string, - lookup: ModelLookupConfig, - ): Promise> { - const credential = await this.credentialsFinderService.findCredentialForUser( - credentialId, - user, - ['credential:read'], - ); - if (!credential || credential.type !== credentialType) { - throw new Error(`Credential ${credentialId} not found or not accessible`); - } - - const personalProject = await this.projectRepository.getPersonalProjectForUserOrFail(user.id); - const currentNodeParameters: INodeParameters = {}; - const credentials: INodeCredentials = { - [credential.type]: { id: credential.id, name: credential.name }, - }; - const additionalData = await getBase({ - userId: user.id, - projectId: personalProject.id, - currentNodeParameters, - }); - const nodeTypeAndVersion = { name: lookup.nodeType, version: lookup.version }; - - if (lookup.kind === 'listSearch') { - const result = await this.dynamicNodeParametersService.getResourceLocatorResults( - lookup.methodName, - '', - additionalData, - nodeTypeAndVersion, - currentNodeParameters, - credentials, - ); - return (result.results ?? []).map((r) => ({ - name: String(r.name), - value: String(r.value), - })); - } - - const nodeType = this.nodeTypes.getByNameAndVersion(lookup.nodeType, lookup.version); - const property = nodeType.description.properties.find((p) => p.name === lookup.propertyName); - const loadOptions = property?.typeOptions?.loadOptions; - if (!loadOptions) { - throw new Error( - `Property "${lookup.propertyName}" on ${lookup.nodeType} has no loadOptions config`, - ); - } - - const options = await this.dynamicNodeParametersService.getOptionsViaLoadOptions( - loadOptions, - additionalData, - nodeTypeAndVersion, - currentNodeParameters, - credentials, - ); - return options.map((o) => ({ name: String(o.name), value: String(o.value) })); - } -} diff --git a/packages/cli/src/modules/agents/builder/interactive/__tests__/resolve-llm.tool.test.ts b/packages/cli/src/modules/agents/builder/interactive/__tests__/resolve-llm.tool.test.ts index e0d8198e0c5..8956e5d7b2c 100644 --- a/packages/cli/src/modules/agents/builder/interactive/__tests__/resolve-llm.tool.test.ts +++ b/packages/cli/src/modules/agents/builder/interactive/__tests__/resolve-llm.tool.test.ts @@ -57,7 +57,10 @@ describe('resolve_llm tool', () => { it('uses the requested model for the requested provider', async () => { const credentialProvider = makeProvider([{ id: 'c1', name: 'My xAI', type: 'xAiApi' }]); - const modelLookup = makeModelLookup(); + const modelLookup = makeModelLookup(async () => [ + { name: 'Grok 4 Fast', value: 'grok-4-fast' }, + { name: 'Grok 4', value: 'grok-4' }, + ]); const tool = buildResolveLlmTool({ credentialProvider, modelLookup }); const result = await tool.handler!({ provider: 'xai', model: 'grok-4-fast' }, {}); @@ -146,20 +149,23 @@ describe('resolve_llm tool', () => { expect(modelLookup.list).not.toHaveBeenCalled(); }); - it('skips lookup for providers without modelLookup configured (e.g. Cohere)', async () => { + it('validates the requested model against the lookup for Cohere', async () => { const credentialProvider = makeProvider([{ id: 'c1', name: 'My Cohere', type: 'cohereApi' }]); - const modelLookup = makeModelLookup(); + const modelLookup = makeModelLookup(async () => [ + { name: 'Command R+', value: 'command-r-plus' }, + { name: 'Command R', value: 'command-r' }, + ]); const tool = buildResolveLlmTool({ credentialProvider, modelLookup }); - const result = await tool.handler!({ provider: 'cohere', model: 'command-x-plus' }, {}); + const result = await tool.handler!({ provider: 'cohere', model: 'command-r-plus' }, {}); expect(result).toEqual({ ok: true, provider: 'cohere', - model: 'command-x-plus', + model: 'command-r-plus', credentialId: 'c1', credentialName: 'My Cohere', }); - expect(modelLookup.list).not.toHaveBeenCalled(); + expect(modelLookup.list).toHaveBeenCalledWith('c1', 'cohereApi', 'cohere'); }); it('returns the canonical model id when the requested model matches the lookup', async () => { @@ -183,11 +189,7 @@ describe('resolve_llm tool', () => { credentialId: 'c1', credentialName: 'My Anthropic', }); - expect(modelLookup.list).toHaveBeenCalledWith( - 'c1', - 'anthropicApi', - expect.objectContaining({ kind: 'listSearch', methodName: 'searchModels' }), - ); + expect(modelLookup.list).toHaveBeenCalledWith('c1', 'anthropicApi', 'anthropic'); }); it('uniquely-substring-matches a partial requested model id', async () => { diff --git a/packages/cli/src/modules/agents/builder/interactive/llm-provider-defaults.ts b/packages/cli/src/modules/agents/builder/interactive/llm-provider-defaults.ts index 082cf4a4ec0..acfc45e5365 100644 --- a/packages/cli/src/modules/agents/builder/interactive/llm-provider-defaults.ts +++ b/packages/cli/src/modules/agents/builder/interactive/llm-provider-defaults.ts @@ -10,95 +10,58 @@ * Azure variants), omit the entry so the tool falls through to suspending and * lets the user pick explicitly. * - * `modelLookup` (optional) points at the chat-model node whose - * `@searchListMethod` or routing-based `loadOptions` returns the live list of - * model ids for the provider. When set, `resolve_llm` validates user-requested - * model strings against that list. When absent, the requested model is passed - * through unchanged. + * Live model validation is available for providers supported by + * `@n8n/ai-utilities/model-discovery` — `resolve_llm` checks user-requested + * model strings against the provider's live model list for those. */ -export type ModelLookupConfig = - | { - kind: 'listSearch'; - nodeType: string; - version: number; - methodName: string; - } - | { - kind: 'loadOptionsRouting'; - nodeType: string; - version: number; - propertyName: string; - }; - export interface LlmProviderDefault { provider: string; defaultModel: string; - modelLookup?: ModelLookupConfig; } export const LLM_PROVIDER_DEFAULTS: Record = { anthropicApi: { provider: 'anthropic', defaultModel: 'claude-sonnet-4-6', - modelLookup: { - kind: 'listSearch', - nodeType: '@n8n/n8n-nodes-langchain.lmChatAnthropic', - version: 1.5, - methodName: 'searchModels', - }, }, openAiApi: { provider: 'openai', defaultModel: 'gpt-5-mini', - modelLookup: { - kind: 'listSearch', - nodeType: '@n8n/n8n-nodes-langchain.lmChatOpenAi', - version: 1.2, - methodName: 'searchModels', - }, }, googlePalmApi: { provider: 'google', defaultModel: 'gemini-2.5-pro', - modelLookup: { - kind: 'loadOptionsRouting', - nodeType: '@n8n/n8n-nodes-langchain.lmChatGoogleGemini', - version: 1.1, - propertyName: 'modelName', - }, }, - xAiApi: { provider: 'xai', defaultModel: 'grok-4' }, - groqApi: { provider: 'groq', defaultModel: 'llama-3.1-70b-versatile' }, + xAiApi: { + provider: 'xai', + defaultModel: 'grok-4', + }, + groqApi: { + provider: 'groq', + defaultModel: 'llama-3.3-70b-versatile', + }, mistralCloudApi: { provider: 'mistral', defaultModel: 'mistral-large-latest', - modelLookup: { - kind: 'loadOptionsRouting', - nodeType: '@n8n/n8n-nodes-langchain.lmChatMistralCloud', - version: 1, - propertyName: 'model', - }, }, - deepSeekApi: { provider: 'deepseek', defaultModel: 'deepseek-chat' }, - cohereApi: { provider: 'cohere', defaultModel: 'command-r-plus' }, + deepSeekApi: { + provider: 'deepseek', + defaultModel: 'deepseek-chat', + }, + cohereApi: { + provider: 'cohere', + defaultModel: 'command-r-plus', + }, openRouterApi: { provider: 'openrouter', defaultModel: 'anthropic/claude-sonnet-4.6', - modelLookup: { - kind: 'loadOptionsRouting', - nodeType: '@n8n/n8n-nodes-langchain.lmChatOpenRouter', - version: 1, - propertyName: 'model', - }, }, nvidiaApi: { provider: 'nvidia', defaultModel: 'nvidia/llama-3.3-nemotron-super-49b-v1', - modelLookup: { - kind: 'loadOptionsRouting', - nodeType: '@n8n/n8n-nodes-langchain.lmChatNvidia', - version: 1, - propertyName: 'model', - }, + }, + vercelAiGatewayApi: { + provider: 'vercel', + defaultModel: 'anthropic/claude-sonnet-4.6', }, }; diff --git a/packages/cli/src/modules/agents/builder/interactive/resolve-llm.tool.ts b/packages/cli/src/modules/agents/builder/interactive/resolve-llm.tool.ts index 17fdcc4b4d3..e9d2cf464ab 100644 --- a/packages/cli/src/modules/agents/builder/interactive/resolve-llm.tool.ts +++ b/packages/cli/src/modules/agents/builder/interactive/resolve-llm.tool.ts @@ -1,19 +1,16 @@ import type { BuiltTool, CredentialListItem, CredentialProvider } from '@n8n/agents'; import { Tool } from '@n8n/agents/tool'; +import { isModelDiscoveryProvider } from '@n8n/ai-utilities/model-discovery'; import { z } from 'zod'; import { BUILDER_TOOLS } from '../builder-tool-names'; -import { - LLM_PROVIDER_DEFAULTS, - type LlmProviderDefault, - type ModelLookupConfig, -} from './llm-provider-defaults'; +import { LLM_PROVIDER_DEFAULTS, type LlmProviderDefault } from './llm-provider-defaults'; export interface ModelLookup { list( credentialId: string, credentialType: string, - lookup: ModelLookupConfig, + provider: string, ): Promise>; } @@ -52,13 +49,13 @@ async function resolveModelAgainstLookup( modelLookup: ModelLookup, ) { const trimmedModel = requestedModel.trim(); - if (!defaults.modelLookup || !trimmedModel) { + if (!isModelDiscoveryProvider(defaults.provider) || !trimmedModel) { return toLlmResolution(credential, defaults, requestedModel); } let availableModels: Array<{ name: string; value: string }>; try { - availableModels = await modelLookup.list(credential.id, credential.type, defaults.modelLookup); + availableModels = await modelLookup.list(credential.id, credential.type, defaults.provider); } catch (error) { return { ok: false as const, diff --git a/packages/frontend/editor-ui/src/features/agents/__tests__/AgentInfoPanel.spec.ts b/packages/frontend/editor-ui/src/features/agents/__tests__/AgentInfoPanel.spec.ts index 36ff2f46e25..0bf9843c743 100644 --- a/packages/frontend/editor-ui/src/features/agents/__tests__/AgentInfoPanel.spec.ts +++ b/packages/frontend/editor-ui/src/features/agents/__tests__/AgentInfoPanel.spec.ts @@ -111,10 +111,11 @@ describe('AgentInfoPanel', () => { variant: 'ghost', showToolbar: 'never', maxHeight: 'none', - placeholder: 'Enter instructions here', }); + expect(editor.props('placeholder')).toBeUndefined(); expect(wrapper.find('[data-testid="agent-instructions-document"]').exists()).toBe(true); expect(wrapper.text()).not.toContain('characters'); + expect(wrapper.text()).not.toContain('Enter instructions here'); }); it('can show the markdown toolbar above instructions', () => { @@ -127,11 +128,12 @@ describe('AgentInfoPanel', () => { }); }); - it('keeps the instructions placeholder available when instructions are empty', () => { + it('does not pass placeholder text to the instructions editor', () => { const wrapper = mountPanel(''); const editor = wrapper.findComponent({ name: 'N8nMarkdownEditor' }); expect(editor.props('modelValue')).toBe(''); - expect(editor.props('placeholder')).toBe('Enter instructions here'); + expect(editor.props('placeholder')).toBeUndefined(); + expect(wrapper.text()).not.toContain('Enter instructions here'); }); }); diff --git a/packages/frontend/editor-ui/src/features/agents/__tests__/useModelCatalog.test.ts b/packages/frontend/editor-ui/src/features/agents/__tests__/useModelCatalog.test.ts index a93a33bb236..cac09f559cb 100644 --- a/packages/frontend/editor-ui/src/features/agents/__tests__/useModelCatalog.test.ts +++ b/packages/frontend/editor-ui/src/features/agents/__tests__/useModelCatalog.test.ts @@ -3,6 +3,7 @@ import type { AgentModelOption } from '../model-providers'; const mocks = vi.hoisted(() => ({ getModelCatalog: vi.fn(), + getProviderModels: vi.fn(), })); vi.mock('@n8n/stores/useRootStore', () => ({ @@ -13,6 +14,7 @@ vi.mock('@n8n/stores/useRootStore', () => ({ vi.mock('../composables/useAgentApi', () => ({ getModelCatalog: mocks.getModelCatalog, + getProviderModels: mocks.getProviderModels, })); function model(id: string, name = id) { @@ -36,10 +38,17 @@ function modelIds(models: AgentModelOption[]) { return models.map((entry) => entry.model); } +async function flushAsync() { + await new Promise((resolve) => setTimeout(resolve, 0)); +} + describe('useModelCatalog', () => { beforeEach(() => { vi.resetModules(); mocks.getModelCatalog.mockReset(); + mocks.getProviderModels.mockReset(); + // Default: no live verification available — fall back to the catalog. + mocks.getProviderModels.mockRejectedValue(new Error('not available')); }); it('returns models for the agent providers that have selected credentials', async () => { @@ -91,4 +100,95 @@ describe('useModelCatalog', () => { ]); expect(result.anthropic?.models).toEqual([]); }); + + it('prefers the provider-verified model list over the static catalog', async () => { + mocks.getModelCatalog.mockResolvedValue({ + anthropic: provider('anthropic', { + 'claude-sonnet-4-6': model('claude-sonnet-4-6', 'Claude Sonnet 4.6'), + 'claude-opus-4-0': model('claude-opus-4-0', 'Claude Opus 4'), + }), + }); + mocks.getProviderModels.mockResolvedValue({ + provider: 'anthropic', + verified: true, + models: [model('claude-sonnet-4-6', 'Claude Sonnet 4.6')], + }); + + const { useModelCatalog } = await import('../composables/useModelCatalog'); + const { ensureLoaded, getModelsForPicker } = useModelCatalog(); + await ensureLoaded('project-1'); + const credentials = { anthropic: 'anthropic-credential-id' }; + + // First read triggers the verification fetch; once it lands, only the + // provider-confirmed models remain. + getModelsForPicker(credentials); + await flushAsync(); + + expect(modelIds(getModelsForPicker(credentials).anthropic?.models ?? [])).toEqual([ + 'claude-sonnet-4-6', + ]); + expect(mocks.getProviderModels).toHaveBeenCalledWith( + {}, + 'project-1', + 'anthropic', + 'anthropic-credential-id', + ); + }); + + it('fetches the verified list only once per provider and credential', async () => { + mocks.getModelCatalog.mockResolvedValue({ + anthropic: provider('anthropic', { + 'claude-sonnet-4-6': model('claude-sonnet-4-6'), + }), + }); + mocks.getProviderModels.mockResolvedValue({ + provider: 'anthropic', + verified: true, + models: [model('claude-sonnet-4-6')], + }); + + const { useModelCatalog } = await import('../composables/useModelCatalog'); + const { ensureLoaded, getModelsForPicker } = useModelCatalog(); + await ensureLoaded('project-1'); + const credentials = { anthropic: 'anthropic-credential-id' }; + + getModelsForPicker(credentials); + getModelsForPicker(credentials); + await flushAsync(); + getModelsForPicker(credentials); + await flushAsync(); + + expect(mocks.getProviderModels).toHaveBeenCalledTimes(1); + }); + + it('falls back to the catalog list when verification fails or is unverified', async () => { + mocks.getModelCatalog.mockResolvedValue({ + anthropic: provider('anthropic', { + 'claude-sonnet-4-6': model('claude-sonnet-4-6'), + 'claude-opus-4-0': model('claude-opus-4-0'), + }), + openai: provider('openai', { + 'gpt-5': model('gpt-5'), + }), + }); + mocks.getProviderModels.mockImplementation(async (_ctx, _projectId, providerId) => { + if (providerId === 'anthropic') throw new Error('provider is down'); + return { provider: providerId, verified: false, models: [model('gpt-5')] }; + }); + + const { useModelCatalog } = await import('../composables/useModelCatalog'); + const { ensureLoaded, getModelsForPicker } = useModelCatalog(); + await ensureLoaded('project-1'); + const credentials = { anthropic: 'anthropic-cred', openai: 'openai-cred' }; + + getModelsForPicker(credentials); + await flushAsync(); + + const result = getModelsForPicker(credentials); + expect(modelIds(result.anthropic?.models ?? []).sort()).toEqual([ + 'claude-opus-4-0', + 'claude-sonnet-4-6', + ]); + expect(modelIds(result.openai?.models ?? [])).toEqual(['gpt-5']); + }); }); diff --git a/packages/frontend/editor-ui/src/features/agents/components/AgentInfoPanel.vue b/packages/frontend/editor-ui/src/features/agents/components/AgentInfoPanel.vue index a8220eeba83..93bf6017dcb 100644 --- a/packages/frontend/editor-ui/src/features/agents/components/AgentInfoPanel.vue +++ b/packages/frontend/editor-ui/src/features/agents/components/AgentInfoPanel.vue @@ -201,7 +201,6 @@ function onInstructionsInput(value: string) { :class="$style.instructionsDocument" :model-value="instructions" :readonly="props.disabled" - :placeholder="i18n.baseText('agents.builder.agent.instructions.placeholder')" :variant="instructionsEditorVariant" :show-toolbar="instructionsToolbarMode" max-height="none" diff --git a/packages/frontend/editor-ui/src/features/agents/composables/useAgentApi.ts b/packages/frontend/editor-ui/src/features/agents/composables/useAgentApi.ts index 9186203a33b..498043056ee 100644 --- a/packages/frontend/editor-ui/src/features/agents/composables/useAgentApi.ts +++ b/packages/frontend/editor-ui/src/features/agents/composables/useAgentApi.ts @@ -8,6 +8,7 @@ import type { AgentTaskConfig, AgentTaskDto, AgentIntegrationSettings, + AgentProviderModelsResponse, AgentVersionListItemDto, ChatIntegrationDescriptor, CreateSlackAgentAppResponse, @@ -346,6 +347,20 @@ export const getModelCatalog = async ( ); }; +export const getProviderModels = async ( + context: IRestApiContext, + projectId: string, + provider: string, + credentialId?: string, +): Promise => { + return await makeRestApiRequest( + context, + 'GET', + `/projects/${projectId}/agents/v2/catalog/models/${provider}`, + credentialId ? { credentialId } : undefined, + ); +}; + export const publishAgent = async ( context: IRestApiContext, projectId: string, diff --git a/packages/frontend/editor-ui/src/features/agents/composables/useModelCatalog.ts b/packages/frontend/editor-ui/src/features/agents/composables/useModelCatalog.ts index 07f689b986a..1552fc9ad21 100644 --- a/packages/frontend/editor-ui/src/features/agents/composables/useModelCatalog.ts +++ b/packages/frontend/editor-ui/src/features/agents/composables/useModelCatalog.ts @@ -1,6 +1,11 @@ import { computed, ref } from 'vue'; import { useRootStore } from '@n8n/stores/useRootStore'; -import { getModelCatalog, type ProviderCatalog, type ModelInfo } from './useAgentApi'; +import { + getModelCatalog, + getProviderModels, + type ProviderCatalog, + type ModelInfo, +} from './useAgentApi'; import { AGENT_MODEL_PROVIDERS, type AgentCredentialsByProvider, @@ -17,6 +22,14 @@ const catalogByProject = ref>({}); const fetchPromises = new Map>(); const loadingProjects = ref(new Set()); +// Provider-verified model lists, keyed by project|provider|credential. The +// static catalog can offer models the provider has retired (which then fail at +// call time), so when a credential is selected the provider's own model API is +// asked which models actually work. `null` marks a failed or unverified +// lookup — the static catalog is used for that key. +const verifiedModelsByKey = ref>({}); +const verifiedFetchesInFlight = new Set(); + function createEmptyModelsResponse(): AgentModelsByProvider { const response: AgentModelsByProvider = {}; for (const provider of AGENT_MODEL_PROVIDERS) { @@ -82,19 +95,62 @@ export function useModelCatalog() { return Object.values(p.models).sort((a, b) => a.name.localeCompare(b.name)); } + /** + * Kick off (once per project+provider+credential) the fetch of the + * provider-verified model list. Idempotent and guarded, so it is safe to + * trigger from `getModelsForPicker` — when the response lands, the reactive + * map updates and computeds re-evaluate with the verified list. + */ + function ensureVerifiedModels( + projectId: string, + provider: AgentModelProvider, + providerCredentialId: string, + ): void { + const key = `${projectId}|${provider}|${providerCredentialId}`; + if (key in verifiedModelsByKey.value || verifiedFetchesInFlight.has(key)) return; + + verifiedFetchesInFlight.add(key); + getProviderModels(rootStore.restApiContext, projectId, provider, providerCredentialId) + .then((result) => { + verifiedModelsByKey.value = { + ...verifiedModelsByKey.value, + [key]: result.verified ? result.models : null, + }; + }) + .catch(() => { + verifiedModelsByKey.value = { ...verifiedModelsByKey.value, [key]: null }; + }) + .finally(() => { + verifiedFetchesInFlight.delete(key); + }); + } + function getModelsForPicker( credentials: AgentCredentialsByProvider | null, ): AgentModelsByProvider { const response = createEmptyModelsResponse(); for (const provider of AGENT_MODEL_PROVIDERS) { - if (!credentials?.[provider]) continue; + const providerCredentialId = credentials?.[provider]; + if (!providerCredentialId) continue; - const providerInfo = catalog.value[provider]; - if (!providerInfo) continue; + let models: ModelInfo[] | undefined; + const projectId = activeProjectId.value; + if (projectId) { + ensureVerifiedModels(projectId, provider, providerCredentialId); + models = + verifiedModelsByKey.value[`${projectId}|${provider}|${providerCredentialId}`] ?? + undefined; + } + + if (!models) { + const providerInfo = catalog.value[provider]; + if (!providerInfo) continue; + models = Object.values(providerInfo.models); + } response[provider] = { - models: Object.values(providerInfo.models) + models: models .map((model) => toAgentModel(provider, model)) .sort((a, b) => a.name.localeCompare(b.name)), };