mirror of
https://github.com/n8n-io/n8n.git
synced 2026-08-30 18:01:23 +08:00
fix(core): Agents model list exclude deprecated models and improved error surfacing (no-changelog) (#33631)
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -6438,3 +6438,109 @@ describe('AgentRuntime — toModelOutput error resilience', () => {
|
||||
expect(toolResultChunk!.isError).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// empty model responses (e.g. provider safety blocks)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('AgentRuntime — empty model responses', () => {
|
||||
beforeEach(() => {
|
||||
generateText.mockReset();
|
||||
streamText.mockReset();
|
||||
});
|
||||
|
||||
function makeEmptyStream(finishReason: string, extraChunks: Array<Record<string, unknown>> = []) {
|
||||
return {
|
||||
fullStream: makeChunkStream(extraChunks),
|
||||
finishReason: Promise.resolve(finishReason),
|
||||
usage: Promise.resolve({ inputTokens: 10, outputTokens: 0, totalTokens: 10 }),
|
||||
response: Promise.resolve({ messages: [] }),
|
||||
toolCalls: Promise.resolve([]),
|
||||
};
|
||||
}
|
||||
|
||||
it.each(['other', 'unknown', 'content-filter'])(
|
||||
'stream: yields an error chunk when the model returns no output with finish reason "%s"',
|
||||
async (finishReason) => {
|
||||
streamText.mockReturnValue(makeEmptyStream(finishReason));
|
||||
|
||||
const { runtime } = createRuntime();
|
||||
const { stream: readableStream } = await runtime.stream('hello');
|
||||
const chunks = await collectChunks(readableStream);
|
||||
|
||||
const errorChunk = chunks.find((c) => c.type === 'error') as
|
||||
| (StreamChunk & { type: 'error' })
|
||||
| undefined;
|
||||
expect(errorChunk).toBeDefined();
|
||||
expect(String((errorChunk!.error as Error).message)).toContain('no output');
|
||||
expect(String((errorChunk!.error as Error).message)).toContain(finishReason);
|
||||
|
||||
const finishChunk = chunks.find((c) => c.type === 'finish') as
|
||||
| (StreamChunk & { type: 'finish' })
|
||||
| undefined;
|
||||
expect(finishChunk).toBeDefined();
|
||||
expect(finishChunk!.finishReason).toBe('error');
|
||||
},
|
||||
);
|
||||
|
||||
it('stream: includes the Google prompt block reason captured from raw chunks', async () => {
|
||||
streamText.mockReturnValue(
|
||||
makeEmptyStream('other', [
|
||||
{
|
||||
type: 'raw',
|
||||
rawValue: { promptFeedback: { blockReason: 'PROHIBITED_CONTENT' } },
|
||||
},
|
||||
]),
|
||||
);
|
||||
|
||||
const { runtime } = createRuntime(undefined, 'google/gemini-2.5-flash');
|
||||
const { stream: readableStream } = await runtime.stream('hello');
|
||||
const chunks = await collectChunks(readableStream);
|
||||
|
||||
const errorChunk = chunks.find((c) => c.type === 'error') as
|
||||
| (StreamChunk & { type: 'error' })
|
||||
| undefined;
|
||||
expect(errorChunk).toBeDefined();
|
||||
expect(String((errorChunk!.error as Error).message)).toContain('PROHIBITED_CONTENT');
|
||||
});
|
||||
|
||||
it('stream: requests raw chunks for google models so block reasons are observable', async () => {
|
||||
streamText.mockReturnValue(makeStreamSuccess());
|
||||
|
||||
const { runtime } = createRuntime(undefined, 'google/gemini-2.5-flash');
|
||||
const { stream: readableStream } = await runtime.stream('hello');
|
||||
await collectChunks(readableStream);
|
||||
|
||||
const args = streamText.mock.calls[0][0] as Record<string, unknown>;
|
||||
expect(args.includeRawChunks).toBe(true);
|
||||
});
|
||||
|
||||
it('stream: an empty response finishing with "stop" is not treated as an error', async () => {
|
||||
streamText.mockReturnValue(makeEmptyStream('stop'));
|
||||
|
||||
const { runtime } = createRuntime();
|
||||
const { stream: readableStream } = await runtime.stream('hello');
|
||||
const chunks = await collectChunks(readableStream);
|
||||
|
||||
expect(chunks.find((c) => c.type === 'error')).toBeUndefined();
|
||||
const finishChunk = chunks.find((c) => c.type === 'finish') as
|
||||
| (StreamChunk & { type: 'finish' })
|
||||
| undefined;
|
||||
expect(finishChunk!.finishReason).not.toBe('error');
|
||||
});
|
||||
|
||||
it('generate: returns finishReason "error" with a descriptive error when the model returns no output', async () => {
|
||||
generateText.mockResolvedValue({
|
||||
finishReason: 'other',
|
||||
usage: { inputTokens: 10, outputTokens: 0, totalTokens: 10 },
|
||||
response: { messages: [] },
|
||||
toolCalls: [],
|
||||
});
|
||||
|
||||
const { runtime } = createRuntime();
|
||||
const result = await runtime.generate('hello');
|
||||
|
||||
expect(result.finishReason).toBe('error');
|
||||
expect(String((result.error as Error).message)).toContain('no output');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -719,6 +719,10 @@ export class AgentRuntime {
|
||||
sink.onTurnFolded?.();
|
||||
|
||||
if (turn.aiFinishReason !== 'tool-calls') {
|
||||
// A rejected/filtered request (e.g. a provider prompt safety block)
|
||||
// surfaces as an output-less turn instead of an SDK error — throw so
|
||||
// the failure reaches the caller rather than ending the run silently.
|
||||
if (turn.errorReason) throw new Error(turn.errorReason.message);
|
||||
structuredOutput = turn.structuredOutput;
|
||||
this.emitTurnEnd(turn.newMessages, extractSettledToolCalls(turn.newMessages));
|
||||
reachedStopCondition = true;
|
||||
|
||||
@@ -6,6 +6,7 @@ import type {
|
||||
RunServices,
|
||||
SuspendEmission,
|
||||
} from './run-output-sink';
|
||||
import { classifyModelTurnError } from './runtime-helpers';
|
||||
import type { GenerateResult } from '../../types';
|
||||
import type { ToolResultEntry } from '../../types/sdk/agent';
|
||||
import { loadAi } from '../model/lazy-ai';
|
||||
@@ -40,14 +41,17 @@ export class GenerateSink implements RunOutputSink<GenerateResult> {
|
||||
});
|
||||
|
||||
const aiFinishReason = result.finishReason;
|
||||
const newMessages = fromAiMessages(result.response.messages);
|
||||
const errorReason = classifyModelTurnError({ aiFinishReason, newMessages });
|
||||
return {
|
||||
aiFinishReason,
|
||||
finishReason: fromAiFinishReason(aiFinishReason),
|
||||
usage: toTokenUsage(result.usage, result.providerMetadata),
|
||||
newMessages: fromAiMessages(result.response.messages),
|
||||
newMessages,
|
||||
toolCalls: result.toolCalls,
|
||||
structuredOutput:
|
||||
ctx.outputSpec && aiFinishReason !== 'tool-calls' ? result.output : undefined,
|
||||
...(errorReason && { errorReason }),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -17,6 +17,21 @@ import type { ToolCallBatchResult, ToolCallSuspension } from '../tools/tool-call
|
||||
|
||||
type RunCallOptions = (RunOptions & ExecutionOptions) | undefined;
|
||||
|
||||
/**
|
||||
* Recognized ways a model turn can fail while the provider still returns
|
||||
* "successfully" (no thrown SDK error). Extend this as new failure modes are
|
||||
* recognized so consumers switch on `type` instead of probing separate fields.
|
||||
* - `prompt_blocked`: the provider rejected the prompt (e.g. a safety block).
|
||||
* - `no_output`: the model returned nothing with a non-stop finish reason.
|
||||
*/
|
||||
export type ModelTurnErrorType = 'prompt_blocked' | 'no_output';
|
||||
|
||||
/** A recognized model-turn failure: a `type` to branch on and a user-facing `message`. */
|
||||
export interface ModelTurnError {
|
||||
type: ModelTurnErrorType;
|
||||
message: string;
|
||||
}
|
||||
|
||||
/** Normalized result of a single LLM turn, independent of generate vs stream. */
|
||||
export interface ModelTurnResult {
|
||||
/** Raw AI SDK finish reason (used to detect the `tool-calls` continuation). */
|
||||
@@ -33,6 +48,13 @@ export interface ModelTurnResult {
|
||||
}>;
|
||||
/** Resolved structured output, only when an output spec is set and the turn finished. */
|
||||
structuredOutput: unknown;
|
||||
/**
|
||||
* Set when the turn failed in a recognized way despite the provider returning
|
||||
* without a thrown error (e.g. a prompt safety block, or empty output). The
|
||||
* loop surfaces this instead of ending the run silently. Check this single
|
||||
* field rather than probing per-failure-mode properties.
|
||||
*/
|
||||
errorReason?: ModelTurnError;
|
||||
}
|
||||
|
||||
/** Per-iteration inputs for the LLM call, assembled by the shared loop. */
|
||||
|
||||
@@ -2,8 +2,10 @@
|
||||
* Pure utility functions used by AgentRuntime that require no class context.
|
||||
* These are extracted here to keep agent-runtime.ts focused on orchestration logic.
|
||||
*/
|
||||
import type { ModelTurnError } from './run-output-sink';
|
||||
import type { StreamChunk, TokenUsage } from '../../types';
|
||||
import type { AgentMessage, ContentToolCall } from '../../types/sdk/message';
|
||||
import type { RawProviderError } from '../model/raw-error';
|
||||
|
||||
/**
|
||||
* Normalize caller input to `AgentMessage[]` for the runtime. String input becomes a
|
||||
@@ -21,6 +23,44 @@ export function stringifyError(error: unknown): string {
|
||||
return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
||||
}
|
||||
|
||||
/**
|
||||
* Finish reasons that indicate the provider rejected or filtered the request
|
||||
* when they arrive with zero output. `stop`/`length` with empty output are the
|
||||
* model's own (odd but legal) choice; `tool-calls` always carries calls;
|
||||
* `error` surfaces through the SDK's thrown error instead.
|
||||
*/
|
||||
const EMPTY_RESPONSE_ERROR_FINISH_REASONS = new Set(['other', 'unknown', 'content-filter']);
|
||||
|
||||
/**
|
||||
* Classify a turn that produced no output as a recognized failure, or return
|
||||
* `undefined` when it doesn't look like a provider rejection. Some providers
|
||||
* fail this way rather than erroring, reporting the cause only on their raw
|
||||
* stream events — when a {@link RawProviderError} was captured there, its type
|
||||
* and reason are carried into the result; otherwise the failure is a generic
|
||||
* `no_output`.
|
||||
*/
|
||||
export function classifyModelTurnError(turn: {
|
||||
aiFinishReason: string;
|
||||
newMessages: AgentMessage[];
|
||||
providerError?: RawProviderError;
|
||||
}): ModelTurnError | undefined {
|
||||
if (turn.newMessages.length > 0) return undefined;
|
||||
if (!EMPTY_RESPONSE_ERROR_FINISH_REASONS.has(turn.aiFinishReason)) return undefined;
|
||||
|
||||
const guidance =
|
||||
'This can be a provider-side false positive — try rephrasing the message, clearing the chat history, or switching models.';
|
||||
if (turn.providerError) {
|
||||
return {
|
||||
type: turn.providerError.type,
|
||||
message: `The model provider blocked this request (${turn.providerError.reason}) and returned no output (finish reason: ${turn.aiFinishReason}). ${guidance}`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
type: 'no_output',
|
||||
message: `The model returned no output (finish reason: ${turn.aiFinishReason}). The provider may have blocked or filtered the request. ${guidance}`,
|
||||
};
|
||||
}
|
||||
|
||||
/** Extract all settled (resolved or rejected) tool-call blocks from a flat list of agent messages. */
|
||||
export function extractSettledToolCalls(messages: AgentMessage[]): ContentToolCall[] {
|
||||
return messages
|
||||
|
||||
@@ -6,11 +6,12 @@ import type {
|
||||
RunServices,
|
||||
SuspendEmission,
|
||||
} from './run-output-sink';
|
||||
import { mergeUsage } from './runtime-helpers';
|
||||
import { classifyModelTurnError, mergeUsage } from './runtime-helpers';
|
||||
import type { ExecutionOptions, TokenUsage } from '../../types/sdk/agent';
|
||||
import type { AgentMessage } from '../../types/sdk/message';
|
||||
import { loadAi } from '../model/lazy-ai';
|
||||
import { fromAiFinishReason, fromAiMessages } from '../model/messages';
|
||||
import { createRawErrorReader, type RawErrorReader } from '../model/raw-error';
|
||||
import { createRawUsageReader, type RawUsageReader } from '../model/raw-usage';
|
||||
import { convertChunk, toTokenUsage } from '../streaming/stream';
|
||||
import type { StreamWriterGuard } from '../streaming/stream-writer-guard';
|
||||
@@ -32,6 +33,11 @@ export class StreamSink implements RunOutputSink<void> {
|
||||
// Text streamed for the in-flight turn, retained so a stop landing mid-response
|
||||
// can still persist what the user already saw. Cleared once the turn is folded.
|
||||
private partialText = '';
|
||||
// Reads provider failure signals (e.g. a prompt safety block) from raw
|
||||
// chunks, so an output-less rejected request can report why. Per-provider
|
||||
// implementations live behind `RawErrorReader`; undefined when the run's
|
||||
// provider has no reader.
|
||||
private rawErrorReader: RawErrorReader | undefined;
|
||||
|
||||
constructor(
|
||||
private readonly guard: StreamWriterGuard,
|
||||
@@ -100,6 +106,10 @@ export class StreamSink implements RunOutputSink<void> {
|
||||
this.rawUsageReader = this.options?.recoverUsageOnAbort
|
||||
? createRawUsageReader(this.services.modelId)
|
||||
: undefined;
|
||||
// Some providers report failures (e.g. prompt safety blocks) only on the
|
||||
// raw stream — no error, no content — so raw chunks are required to
|
||||
// explain an otherwise silent empty response.
|
||||
this.rawErrorReader = createRawErrorReader(this.services.modelId);
|
||||
const { streamText } = loadAi();
|
||||
const result = streamText({
|
||||
model: ctx.model,
|
||||
@@ -108,7 +118,9 @@ export class StreamSink implements RunOutputSink<void> {
|
||||
abortSignal: ctx.abortSignal,
|
||||
// Surface the provider's raw message_start/message_delta events so an
|
||||
// aborted run can recover its usage — the SDK reports none on abort.
|
||||
...(this.rawUsageReader ? { includeRawChunks: true } : {}),
|
||||
...(this.rawUsageReader !== undefined || this.rawErrorReader !== undefined
|
||||
? { includeRawChunks: true }
|
||||
: {}),
|
||||
...(ctx.hasTools ? { tools: ctx.aiTools } : {}),
|
||||
...(ctx.providerOptions ? { providerOptions: ctx.providerOptions } : {}),
|
||||
...(ctx.outputSpec ? { output: ctx.outputSpec } : {}),
|
||||
@@ -124,6 +136,7 @@ export class StreamSink implements RunOutputSink<void> {
|
||||
// reaches the post-loop awaits) can still be billed via getAbortFinish.
|
||||
if (chunk.type === 'raw') {
|
||||
this.rawUsageReader?.capture(chunk.rawValue);
|
||||
this.rawErrorReader?.capture(chunk.rawValue);
|
||||
continue;
|
||||
}
|
||||
// Filter only the SDK's terminal `finish` chunk — the runtime emits its
|
||||
@@ -167,15 +180,22 @@ export class StreamSink implements RunOutputSink<void> {
|
||||
const usage = await result.usage;
|
||||
const providerMetadata = await result.providerMetadata;
|
||||
const response = await result.response;
|
||||
const newMessages = fromAiMessages(response.messages);
|
||||
const errorReason = classifyModelTurnError({
|
||||
aiFinishReason,
|
||||
newMessages,
|
||||
providerError: this.rawErrorReader?.getError(),
|
||||
});
|
||||
|
||||
return {
|
||||
aiFinishReason,
|
||||
finishReason: fromAiFinishReason(aiFinishReason),
|
||||
usage: toTokenUsage(usage, providerMetadata),
|
||||
newMessages: fromAiMessages(response.messages),
|
||||
newMessages,
|
||||
toolCalls: await result.toolCalls,
|
||||
structuredOutput:
|
||||
ctx.outputSpec && aiFinishReason !== 'tool-calls' ? await result.output : undefined,
|
||||
...(errorReason && { errorReason }),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import { isRecord } from '@n8n/utils/is-record';
|
||||
|
||||
import { getModelProvider } from './prompt-cache';
|
||||
import { type ProviderId } from './provider-credentials';
|
||||
import type { ModelTurnErrorType } from '../loop/run-output-sink';
|
||||
|
||||
/** A recognized failure signal read from the provider's raw stream events. */
|
||||
export interface RawProviderError {
|
||||
type: ModelTurnErrorType;
|
||||
/** The provider's own reason string (e.g. Google's `PROHIBITED_CONTENT`). */
|
||||
reason: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a provider's native raw stream events (the AI SDK's `chunk.rawValue`
|
||||
* passthrough, enabled by `includeRawChunks`) for failure signals the SDK does
|
||||
* not surface — e.g. a prompt safety block that ends the stream with no error
|
||||
* and no content. Each provider's wire format differs, so the runtime stays
|
||||
* provider-agnostic and delegates to a per-provider implementation selected
|
||||
* via {@link createRawErrorReader}.
|
||||
*/
|
||||
export interface RawErrorReader {
|
||||
/** Inspect one raw provider event for a failure signal. */
|
||||
capture(rawValue: unknown): void;
|
||||
/** The failure signal seen this turn, or undefined when none was observed. */
|
||||
getError(): RawProviderError | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Google reports prompt safety blocks only via `promptFeedback.blockReason` on
|
||||
* the raw stream — the response then simply contains no candidates.
|
||||
*/
|
||||
export class GoogleRawErrorReader implements RawErrorReader {
|
||||
private blockReason: string | undefined;
|
||||
|
||||
capture(rawValue: unknown): void {
|
||||
if (!isRecord(rawValue) || !isRecord(rawValue.promptFeedback)) return;
|
||||
const blockReason = rawValue.promptFeedback.blockReason;
|
||||
if (typeof blockReason === 'string' && blockReason) {
|
||||
this.blockReason = blockReason;
|
||||
}
|
||||
}
|
||||
|
||||
getError(): RawProviderError | undefined {
|
||||
return this.blockReason ? { type: 'prompt_blocked', reason: this.blockReason } : undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-provider raw-error readers, keyed by provider id. Partial because this is
|
||||
* best-effort diagnostics: a provider without an entry still gets the generic
|
||||
* empty-response handling, just without the provider's own reason attached.
|
||||
*/
|
||||
const RAW_ERROR_READERS: Partial<Record<ProviderId, () => RawErrorReader>> = {
|
||||
google: () => new GoogleRawErrorReader(),
|
||||
};
|
||||
|
||||
/**
|
||||
* Create a fresh raw-error reader for the run's model, or undefined when the
|
||||
* provider has no reader. The provider is the prefix of the `provider/model` id
|
||||
* (same convention as `createModel`).
|
||||
*/
|
||||
export function createRawErrorReader(modelId: string): RawErrorReader | undefined {
|
||||
const provider = getModelProvider(modelId) as ProviderId;
|
||||
return RAW_ERROR_READERS[provider]?.();
|
||||
}
|
||||
@@ -71,6 +71,75 @@ describe('fetchProviderCatalog', () => {
|
||||
expect(catalog['azure-cognitive-services']).toBeUndefined();
|
||||
});
|
||||
|
||||
it('drops models marked deprecated on models.dev', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () =>
|
||||
await Promise.resolve({
|
||||
anthropic: {
|
||||
id: 'anthropic',
|
||||
name: 'Anthropic',
|
||||
models: {
|
||||
'claude-3-haiku-20240307': {
|
||||
id: 'claude-3-haiku-20240307',
|
||||
name: 'Claude Haiku 3',
|
||||
status: 'deprecated',
|
||||
},
|
||||
'claude-sonnet-4-6': {
|
||||
id: 'claude-sonnet-4-6',
|
||||
name: 'Claude Sonnet 4.6',
|
||||
},
|
||||
'claude-beta-model': {
|
||||
id: 'claude-beta-model',
|
||||
name: 'Claude Beta Model',
|
||||
status: 'beta',
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
global.fetch = fetchMock as typeof fetch;
|
||||
|
||||
const catalog = await fetchProviderCatalog();
|
||||
|
||||
expect(catalog.anthropic.models['claude-3-haiku-20240307']).toBeUndefined();
|
||||
expect(catalog.anthropic.models['claude-sonnet-4-6'].name).toBe('Claude Sonnet 4.6');
|
||||
expect(catalog.anthropic.models['claude-beta-model'].name).toBe('Claude Beta Model');
|
||||
});
|
||||
|
||||
it('omits a provider whose models are all deprecated', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () =>
|
||||
await Promise.resolve({
|
||||
anthropic: {
|
||||
id: 'anthropic',
|
||||
name: 'Anthropic',
|
||||
models: {
|
||||
'claude-3-haiku-20240307': {
|
||||
id: 'claude-3-haiku-20240307',
|
||||
name: 'Claude Haiku 3',
|
||||
status: 'deprecated',
|
||||
},
|
||||
},
|
||||
},
|
||||
openai: {
|
||||
id: 'openai',
|
||||
name: 'OpenAI',
|
||||
models: {
|
||||
'gpt-5': { id: 'gpt-5', name: 'GPT-5' },
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
global.fetch = fetchMock as typeof fetch;
|
||||
|
||||
const catalog = await fetchProviderCatalog();
|
||||
|
||||
expect(catalog.anthropic).toBeUndefined();
|
||||
expect(catalog.openai.models['gpt-5'].name).toBe('GPT-5');
|
||||
});
|
||||
|
||||
it('strips the "(latest)" suffix from model names and drops duplicate pinned snapshots', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
|
||||
@@ -68,6 +68,7 @@ interface ModelsDevModel {
|
||||
release_date?: string;
|
||||
reasoning?: boolean;
|
||||
tool_call?: boolean;
|
||||
status?: string;
|
||||
cost?: { input?: number; output?: number; cache_read?: number; cache_write?: number };
|
||||
limit?: { context?: number; output?: number };
|
||||
}
|
||||
@@ -138,6 +139,9 @@ export async function fetchProviderCatalog(): Promise<ProviderCatalog> {
|
||||
|
||||
const models: Record<string, ModelInfo> = {};
|
||||
for (const [modelId, model] of Object.entries(provider.models)) {
|
||||
// Deprecated models still 404 at call time when the provider retires
|
||||
// them, so never offer them.
|
||||
if (model.status === 'deprecated') continue;
|
||||
const info: ModelInfo = {
|
||||
id: model.id,
|
||||
name: model.name,
|
||||
@@ -162,6 +166,8 @@ export async function fetchProviderCatalog(): Promise<ProviderCatalog> {
|
||||
models[modelId] = info;
|
||||
}
|
||||
|
||||
if (Object.keys(models).length === 0) continue;
|
||||
|
||||
const providerId = toAgentProviderId(key);
|
||||
catalog[providerId] = {
|
||||
id: providerId,
|
||||
|
||||
@@ -24,6 +24,9 @@
|
||||
],
|
||||
"web-search": [
|
||||
"dist/esm/web-search/index.d.ts"
|
||||
],
|
||||
"model-discovery": [
|
||||
"dist/esm/model-discovery/index.d.ts"
|
||||
]
|
||||
}
|
||||
},
|
||||
@@ -63,6 +66,11 @@
|
||||
"import": "./dist/esm/web-search/index.js",
|
||||
"require": "./dist/cjs/web-search/index.js"
|
||||
},
|
||||
"./model-discovery": {
|
||||
"types": "./dist/esm/model-discovery/index.d.ts",
|
||||
"import": "./dist/esm/model-discovery/index.js",
|
||||
"require": "./dist/cjs/model-discovery/index.js"
|
||||
},
|
||||
"./*": "./*"
|
||||
},
|
||||
"scripts": {
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
import { listModelsForProvider, MODEL_DISCOVERY_PROVIDERS } from '../index';
|
||||
|
||||
function mockFetch(body: unknown, ok = true, status = 200) {
|
||||
return vi.fn().mockResolvedValue({
|
||||
ok,
|
||||
status,
|
||||
text: async () => JSON.stringify(body),
|
||||
json: async () => body,
|
||||
}) as unknown as typeof globalThis.fetch;
|
||||
}
|
||||
|
||||
function calledUrl(fetchFn: unknown): string {
|
||||
return String((fetchFn as ReturnType<typeof vi.fn>).mock.calls[0][0]);
|
||||
}
|
||||
|
||||
function calledHeaders(fetchFn: unknown): Record<string, string> {
|
||||
const init = (fetchFn as ReturnType<typeof vi.fn>).mock.calls[0][1] as {
|
||||
headers: Record<string, string>;
|
||||
};
|
||||
return init.headers;
|
||||
}
|
||||
|
||||
describe('model-discovery', () => {
|
||||
describe('anthropic', () => {
|
||||
it('lists models from /v1/models with x-api-key auth, newest first', async () => {
|
||||
const fetch = mockFetch({
|
||||
data: [
|
||||
{ id: 'claude-old', display_name: 'Claude Old', created_at: '2024-01-01T00:00:00Z' },
|
||||
{ id: 'claude-new', display_name: 'Claude New', created_at: '2026-01-01T00:00:00Z' },
|
||||
],
|
||||
});
|
||||
|
||||
const models = await listModelsForProvider('anthropic', { apiKey: 'key', fetch });
|
||||
|
||||
expect(calledUrl(fetch)).toBe('https://api.anthropic.com/v1/models');
|
||||
expect(calledHeaders(fetch)['x-api-key']).toBe('key');
|
||||
expect(calledHeaders(fetch)['anthropic-version']).toBe('2023-06-01');
|
||||
expect(models).toEqual([
|
||||
{ id: 'claude-new', name: 'Claude New' },
|
||||
{ id: 'claude-old', name: 'Claude Old' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('respects a custom baseURL', async () => {
|
||||
const fetch = mockFetch({ data: [] });
|
||||
await listModelsForProvider('anthropic', {
|
||||
apiKey: 'key',
|
||||
baseURL: 'https://proxy.example.com/',
|
||||
fetch,
|
||||
});
|
||||
expect(calledUrl(fetch)).toBe('https://proxy.example.com/v1/models');
|
||||
});
|
||||
});
|
||||
|
||||
describe('openai', () => {
|
||||
it('filters out non-chat models on the official API and sorts by id', async () => {
|
||||
const fetch = mockFetch({
|
||||
data: [
|
||||
{ id: 'gpt-5' },
|
||||
{ id: 'whisper-1' },
|
||||
{ id: 'dall-e-3' },
|
||||
{ id: 'text-embedding-3-small' },
|
||||
{ id: 'gpt-4o' },
|
||||
],
|
||||
});
|
||||
|
||||
const models = await listModelsForProvider('openai', { apiKey: 'key', fetch });
|
||||
|
||||
expect(calledUrl(fetch)).toBe('https://api.openai.com/v1/models');
|
||||
expect(calledHeaders(fetch).Authorization).toBe('Bearer key');
|
||||
expect(models.map((m) => m.id)).toEqual(['gpt-4o', 'gpt-5']);
|
||||
});
|
||||
|
||||
it('includes all models for a custom (non-OpenAI) baseURL', async () => {
|
||||
const fetch = mockFetch({ data: [{ id: 'whisper-1' }, { id: 'my-model' }] });
|
||||
|
||||
const models = await listModelsForProvider('openai', {
|
||||
apiKey: 'key',
|
||||
baseURL: 'https://llm.internal.example.com/v1',
|
||||
fetch,
|
||||
});
|
||||
|
||||
expect(models.map((m) => m.id)).toEqual(['my-model', 'whisper-1']);
|
||||
});
|
||||
|
||||
it('merges caller-supplied headers into the request', async () => {
|
||||
const fetch = mockFetch({ data: [] });
|
||||
await listModelsForProvider('openai', {
|
||||
apiKey: 'key',
|
||||
headers: { 'x-custom': 'yes' },
|
||||
fetch,
|
||||
});
|
||||
expect(calledHeaders(fetch)['x-custom']).toBe('yes');
|
||||
});
|
||||
});
|
||||
|
||||
describe('google', () => {
|
||||
it('lists models from /v1beta/models with header auth, excluding embedding/imagen', async () => {
|
||||
const fetch = mockFetch({
|
||||
models: [
|
||||
{ name: 'models/gemini-2.5-flash', description: 'Fast' },
|
||||
{ name: 'models/text-embedding-004', description: 'Embeddings' },
|
||||
{ name: 'models/imagen-3', description: 'Images' },
|
||||
{ name: 'models/gemini-2.5-pro', description: 'Smart' },
|
||||
],
|
||||
});
|
||||
|
||||
const models = await listModelsForProvider('google', { apiKey: 'g-key', fetch });
|
||||
|
||||
// The API key goes in the x-goog-api-key header, never the query string,
|
||||
// so it cannot leak through access logs or proxies.
|
||||
expect(calledUrl(fetch)).toBe('https://generativelanguage.googleapis.com/v1beta/models');
|
||||
expect(calledHeaders(fetch)['x-goog-api-key']).toBe('g-key');
|
||||
expect(models).toEqual([
|
||||
{ id: 'models/gemini-2.5-flash', name: 'models/gemini-2.5-flash' },
|
||||
{ id: 'models/gemini-2.5-pro', name: 'models/gemini-2.5-pro' },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('groq', () => {
|
||||
it('keeps only active model objects', async () => {
|
||||
const fetch = mockFetch({
|
||||
data: [
|
||||
{ id: 'llama-3.3-70b', active: true, object: 'model' },
|
||||
{ id: 'retired-model', active: false, object: 'model' },
|
||||
{ id: 'not-a-model', active: true, object: 'other' },
|
||||
],
|
||||
});
|
||||
|
||||
const models = await listModelsForProvider('groq', { apiKey: 'key', fetch });
|
||||
|
||||
expect(calledUrl(fetch)).toBe('https://api.groq.com/openai/v1/models');
|
||||
expect(models.map((m) => m.id)).toEqual(['llama-3.3-70b']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('mistral', () => {
|
||||
it('excludes embedding models and sorts by name', async () => {
|
||||
const fetch = mockFetch({
|
||||
data: [{ id: 'mistral-large' }, { id: 'mistral-embed' }, { id: 'codestral' }],
|
||||
});
|
||||
|
||||
const models = await listModelsForProvider('mistral', { apiKey: 'key', fetch });
|
||||
|
||||
expect(calledUrl(fetch)).toBe('https://api.mistral.ai/v1/models');
|
||||
expect(models.map((m) => m.id)).toEqual(['codestral', 'mistral-large']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('cohere', () => {
|
||||
it('lists chat models by name', async () => {
|
||||
const fetch = mockFetch({
|
||||
models: [{ name: 'command-r-plus' }, { name: 'command-a-03-2025' }],
|
||||
});
|
||||
|
||||
const models = await listModelsForProvider('cohere', { apiKey: 'key', fetch });
|
||||
|
||||
expect(calledUrl(fetch)).toBe('https://api.cohere.ai/v1/models?page_size=100&endpoint=chat');
|
||||
expect(models.map((m) => m.id)).toEqual(['command-a-03-2025', 'command-r-plus']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('nvidia', () => {
|
||||
it('keeps only supported Nemotron models', async () => {
|
||||
const fetch = mockFetch({
|
||||
data: [
|
||||
{ id: 'nvidia/llama-3.3-nemotron-super-49b-v1' },
|
||||
{ id: 'meta/llama-3.1-405b-instruct' },
|
||||
],
|
||||
});
|
||||
|
||||
const models = await listModelsForProvider('nvidia', { apiKey: 'key', fetch });
|
||||
|
||||
expect(calledUrl(fetch)).toBe('https://integrate.api.nvidia.com/v1/models');
|
||||
expect(models.map((m) => m.id)).toEqual(['nvidia/llama-3.3-nemotron-super-49b-v1']);
|
||||
});
|
||||
});
|
||||
|
||||
describe.each([
|
||||
['deepseek', 'https://api.deepseek.com/models'],
|
||||
['openrouter', 'https://openrouter.ai/api/v1/models'],
|
||||
['xai', 'https://api.x.ai/v1/models'],
|
||||
['vercel', 'https://ai-gateway.vercel.sh/v1/models'],
|
||||
] as const)('%s', (provider, expectedUrl) => {
|
||||
it('lists models with bearer auth, sorted by name', async () => {
|
||||
const fetch = mockFetch({ data: [{ id: 'model-b' }, { id: 'model-a' }] });
|
||||
|
||||
const models = await listModelsForProvider(provider, { apiKey: 'key', fetch });
|
||||
|
||||
expect(calledUrl(fetch)).toBe(expectedUrl);
|
||||
expect(calledHeaders(fetch).Authorization).toBe('Bearer key');
|
||||
expect(models.map((m) => m.id)).toEqual(['model-a', 'model-b']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('error handling', () => {
|
||||
it('throws a descriptive error on a non-2xx response', async () => {
|
||||
const fetch = mockFetch({ error: { message: 'invalid x-api-key' } }, false, 401);
|
||||
|
||||
await expect(listModelsForProvider('anthropic', { apiKey: 'bad', fetch })).rejects.toThrow(
|
||||
/anthropic.*401/i,
|
||||
);
|
||||
});
|
||||
|
||||
it('throws for an unknown provider', async () => {
|
||||
await expect(
|
||||
listModelsForProvider('not-a-provider', { apiKey: 'key', fetch: mockFetch({}) }),
|
||||
).rejects.toThrow(/unknown/i);
|
||||
});
|
||||
});
|
||||
|
||||
it('exposes a registry of all supported providers', () => {
|
||||
expect(Object.keys(MODEL_DISCOVERY_PROVIDERS).sort()).toEqual([
|
||||
'anthropic',
|
||||
'cohere',
|
||||
'deepseek',
|
||||
'google',
|
||||
'groq',
|
||||
'mistral',
|
||||
'nvidia',
|
||||
'openai',
|
||||
'openrouter',
|
||||
'vercel',
|
||||
'xai',
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,65 @@
|
||||
import { listAnthropicModels } from './providers/anthropic';
|
||||
import { listCohereModels } from './providers/cohere';
|
||||
import { listDeepSeekModels } from './providers/deepseek';
|
||||
import { listGoogleModels } from './providers/google';
|
||||
import { listGroqModels } from './providers/groq';
|
||||
import { listMistralModels } from './providers/mistral';
|
||||
import { listNvidiaModels } from './providers/nvidia';
|
||||
import { listOpenAiModels } from './providers/openai';
|
||||
import { listOpenRouterModels } from './providers/openrouter';
|
||||
import { listVercelModels } from './providers/vercel';
|
||||
import { listXaiModels } from './providers/xai';
|
||||
import type { ListModelsFn, ListModelsOptions, ProviderModel } from './types';
|
||||
|
||||
/**
|
||||
* Live chat-model discovery per provider: each function asks the provider's
|
||||
* own model-list API which models the given credential can call. Shared by the
|
||||
* chat sub-nodes' model dropdowns and the agents feature's model catalog, so
|
||||
* provider knowledge (endpoint, auth, chat-model filtering) lives in one place.
|
||||
*
|
||||
* Each provider implements {@link ListModelsFn} in its own file under
|
||||
* `providers/`, ported 1:1 from the corresponding chat node's `searchModels`
|
||||
* method or `loadOptions` routing config (see the source reference on each).
|
||||
* When changing behavior here, keep the node in sync.
|
||||
*/
|
||||
export const MODEL_DISCOVERY_PROVIDERS: Record<string, ListModelsFn> = {
|
||||
anthropic: listAnthropicModels,
|
||||
cohere: listCohereModels,
|
||||
deepseek: listDeepSeekModels,
|
||||
google: listGoogleModels,
|
||||
groq: listGroqModels,
|
||||
mistral: listMistralModels,
|
||||
nvidia: listNvidiaModels,
|
||||
openai: listOpenAiModels,
|
||||
openrouter: listOpenRouterModels,
|
||||
vercel: listVercelModels,
|
||||
xai: listXaiModels,
|
||||
};
|
||||
|
||||
export function isModelDiscoveryProvider(provider: string): boolean {
|
||||
return provider in MODEL_DISCOVERY_PROVIDERS;
|
||||
}
|
||||
|
||||
export async function listModelsForProvider(
|
||||
provider: string,
|
||||
options: ListModelsOptions,
|
||||
): Promise<ProviderModel[]> {
|
||||
const listModels = MODEL_DISCOVERY_PROVIDERS[provider];
|
||||
if (!listModels) {
|
||||
throw new Error(`Unknown model discovery provider: "${provider}"`);
|
||||
}
|
||||
return await listModels(options);
|
||||
}
|
||||
|
||||
export { listAnthropicModels } from './providers/anthropic';
|
||||
export { listCohereModels } from './providers/cohere';
|
||||
export { listDeepSeekModels } from './providers/deepseek';
|
||||
export { listGoogleModels } from './providers/google';
|
||||
export { listGroqModels } from './providers/groq';
|
||||
export { listMistralModels } from './providers/mistral';
|
||||
export { listNvidiaModels } from './providers/nvidia';
|
||||
export { listOpenAiModels, shouldIncludeOpenAiModel } from './providers/openai';
|
||||
export { listOpenRouterModels } from './providers/openrouter';
|
||||
export { listVercelModels } from './providers/vercel';
|
||||
export { listXaiModels } from './providers/xai';
|
||||
export type { ListModelsFn, ListModelsOptions, ProviderModel } from './types';
|
||||
@@ -0,0 +1,17 @@
|
||||
import { baseUrl, getJson } from '../request';
|
||||
import type { ListModelsFn } from '../types';
|
||||
|
||||
/** Source: LMChatAnthropic `methods/searchModels.ts` (GET /v1/models, newest first). */
|
||||
export const listAnthropicModels: ListModelsFn = async (options) => {
|
||||
const data = (await getJson(
|
||||
`${baseUrl(options, 'https://api.anthropic.com')}/v1/models`,
|
||||
{ 'x-api-key': options.apiKey, 'anthropic-version': '2023-06-01' },
|
||||
options,
|
||||
'anthropic',
|
||||
)) as { data?: Array<{ id: string; display_name?: string; created_at?: string }> };
|
||||
|
||||
return (data.data ?? [])
|
||||
.slice()
|
||||
.sort((a, b) => new Date(b.created_at ?? 0).getTime() - new Date(a.created_at ?? 0).getTime())
|
||||
.map((model) => ({ id: model.id, name: model.display_name ?? model.id }));
|
||||
};
|
||||
@@ -0,0 +1,17 @@
|
||||
import { baseUrl, bearerHeaders, byName, getJson } from '../request';
|
||||
import type { ListModelsFn } from '../types';
|
||||
|
||||
/** Source: LmChatCohere `loadOptions` routing (chat endpoint models only). */
|
||||
export const listCohereModels: ListModelsFn = async (options) => {
|
||||
const data = (await getJson(
|
||||
`${baseUrl(options, 'https://api.cohere.ai')}/v1/models?page_size=100&endpoint=chat`,
|
||||
bearerHeaders(options),
|
||||
options,
|
||||
'cohere',
|
||||
)) as { models?: Array<{ name?: unknown }> };
|
||||
|
||||
return (data.models ?? [])
|
||||
.filter((model): model is { name: string } => typeof model.name === 'string')
|
||||
.map((model) => ({ id: model.name, name: model.name }))
|
||||
.sort(byName);
|
||||
};
|
||||
@@ -0,0 +1,4 @@
|
||||
import { makeBearerDataListing } from '../request';
|
||||
|
||||
/** Source: LmChatDeepSeek `loadOptions` routing. */
|
||||
export const listDeepSeekModels = makeBearerDataListing('deepseek', 'https://api.deepseek.com');
|
||||
@@ -0,0 +1,29 @@
|
||||
import { baseUrl, byName, getJson } from '../request';
|
||||
import type { ListModelsFn } from '../types';
|
||||
|
||||
/**
|
||||
* Source: LmChatGoogleGemini `loadOptions` routing (GET /v1beta/models,
|
||||
* embedding/imagen excluded). Ids keep Google's `models/` prefix, matching the
|
||||
* node dropdown values. Auth uses the `x-goog-api-key` header (Google's
|
||||
* preferred method) rather than the credential's `?key=` query auth, so the
|
||||
* key cannot leak through access logs or proxies.
|
||||
*/
|
||||
export const listGoogleModels: ListModelsFn = async (options) => {
|
||||
const base = baseUrl(options, 'https://generativelanguage.googleapis.com');
|
||||
const data = (await getJson(
|
||||
`${base}/v1beta/models`,
|
||||
{ 'x-goog-api-key': options.apiKey },
|
||||
options,
|
||||
'google',
|
||||
)) as { models?: Array<{ name?: unknown }> };
|
||||
|
||||
return (data.models ?? [])
|
||||
.filter(
|
||||
(model): model is { name: string } =>
|
||||
typeof model.name === 'string' &&
|
||||
!model.name.includes('embedding') &&
|
||||
!model.name.includes('imagen'),
|
||||
)
|
||||
.map((model) => ({ id: model.name, name: model.name }))
|
||||
.sort(byName);
|
||||
};
|
||||
@@ -0,0 +1,16 @@
|
||||
import { baseUrl, bearerHeaders, getJson, idsToModels } from '../request';
|
||||
import type { ListModelsFn } from '../types';
|
||||
|
||||
/** Source: LmChatGroq `loadOptions` routing (active model objects only). */
|
||||
export const listGroqModels: ListModelsFn = async (options) => {
|
||||
const data = (await getJson(
|
||||
`${baseUrl(options, 'https://api.groq.com/openai/v1')}/models`,
|
||||
bearerHeaders(options),
|
||||
options,
|
||||
'groq',
|
||||
)) as { data?: Array<{ id?: unknown; active?: unknown; object?: unknown }> };
|
||||
|
||||
return idsToModels(
|
||||
(data.data ?? []).filter((model) => model.active === true && model.object === 'model'),
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,16 @@
|
||||
import { baseUrl, bearerHeaders, byName, getJson, idsToModels, type IdItem } from '../request';
|
||||
import type { ListModelsFn } from '../types';
|
||||
|
||||
/** Source: LmChatMistralCloud `loadOptions` routing (embedding models excluded). */
|
||||
export const listMistralModels: ListModelsFn = async (options) => {
|
||||
const data = (await getJson(
|
||||
`${baseUrl(options, 'https://api.mistral.ai/v1')}/models`,
|
||||
bearerHeaders(options),
|
||||
options,
|
||||
'mistral',
|
||||
)) as { data?: IdItem[] };
|
||||
|
||||
return idsToModels(data.data ?? [])
|
||||
.filter((model) => !model.id.includes('embed'))
|
||||
.sort(byName);
|
||||
};
|
||||
@@ -0,0 +1,27 @@
|
||||
import { baseUrl, bearerHeaders, byName, getJson, idsToModels, type IdItem } from '../request';
|
||||
import type { ListModelsFn } from '../types';
|
||||
|
||||
/** Source: LmChatNvidia `loadOptions` routing — mirrors NEMOTRON_SUPPORTED_MODELS. */
|
||||
const NVIDIA_SUPPORTED_MODELS = new Set([
|
||||
'nvidia/llama-3.1-nemotron-nano-8b-v1',
|
||||
'nvidia/llama-3.3-nemotron-super-49b-v1',
|
||||
'nvidia/llama-3.3-nemotron-super-49b-v1.5',
|
||||
'nvidia/nemotron-3-nano-30b-a3b',
|
||||
'nvidia/nemotron-3-nano-omni-30b-a3b-reasoning',
|
||||
'nvidia/nemotron-3-super-120b-a12b',
|
||||
'nvidia/nemotron-nano-12b-v2-vl',
|
||||
'nvidia/nvidia-nemotron-nano-9b-v2',
|
||||
]);
|
||||
|
||||
export const listNvidiaModels: ListModelsFn = async (options) => {
|
||||
const data = (await getJson(
|
||||
`${baseUrl(options, 'https://integrate.api.nvidia.com/v1')}/models`,
|
||||
bearerHeaders(options),
|
||||
options,
|
||||
'nvidia',
|
||||
)) as { data?: IdItem[] };
|
||||
|
||||
return idsToModels(data.data ?? [])
|
||||
.filter((model) => NVIDIA_SUPPORTED_MODELS.has(model.id))
|
||||
.sort(byName);
|
||||
};
|
||||
@@ -0,0 +1,39 @@
|
||||
import { baseUrl, bearerHeaders, byName, getJson, idsToModels, type IdItem } from '../request';
|
||||
import type { ListModelsFn } from '../types';
|
||||
|
||||
const OFFICIAL_OPENAI_HOSTNAMES = ['api.openai.com', 'ai-assistant.n8n.io'];
|
||||
|
||||
/**
|
||||
* Source: LMChatOpenAi `methods/loadModels.ts` — on the official API, exclude
|
||||
* non-chat model families; on custom (proxy/self-hosted) hosts include all.
|
||||
*/
|
||||
export function shouldIncludeOpenAiModel(modelId: string, isCustomApi: boolean): boolean {
|
||||
if (isCustomApi) return true;
|
||||
return !(
|
||||
modelId.startsWith('babbage') ||
|
||||
modelId.startsWith('davinci') ||
|
||||
modelId.startsWith('computer-use') ||
|
||||
modelId.startsWith('dall-e') ||
|
||||
modelId.startsWith('text-embedding') ||
|
||||
modelId.startsWith('tts') ||
|
||||
modelId.includes('-tts') ||
|
||||
modelId.startsWith('whisper') ||
|
||||
modelId.startsWith('omni-moderation') ||
|
||||
modelId.startsWith('sora') ||
|
||||
modelId.includes('-realtime') ||
|
||||
(modelId.startsWith('gpt-') && modelId.includes('instruct'))
|
||||
);
|
||||
}
|
||||
|
||||
/** Source: LMChatOpenAi `methods/loadModels.ts` (GET /models, filtered, id asc). */
|
||||
export const listOpenAiModels: ListModelsFn = async (options) => {
|
||||
const base = baseUrl(options, 'https://api.openai.com/v1');
|
||||
const isCustomApi = !OFFICIAL_OPENAI_HOSTNAMES.includes(new URL(base).hostname);
|
||||
const data = (await getJson(`${base}/models`, bearerHeaders(options), options, 'openai')) as {
|
||||
data?: IdItem[];
|
||||
};
|
||||
|
||||
return idsToModels(data.data ?? [])
|
||||
.filter((model) => shouldIncludeOpenAiModel(model.id, isCustomApi))
|
||||
.sort(byName);
|
||||
};
|
||||
@@ -0,0 +1,7 @@
|
||||
import { makeBearerDataListing } from '../request';
|
||||
|
||||
/** Source: LmChatOpenRouter `loadOptions` routing. */
|
||||
export const listOpenRouterModels = makeBearerDataListing(
|
||||
'openrouter',
|
||||
'https://openrouter.ai/api/v1',
|
||||
);
|
||||
@@ -0,0 +1,4 @@
|
||||
import { makeBearerDataListing } from '../request';
|
||||
|
||||
/** Source: LmChatVercelAiGateway `loadOptions` routing. */
|
||||
export const listVercelModels = makeBearerDataListing('vercel', 'https://ai-gateway.vercel.sh/v1');
|
||||
@@ -0,0 +1,4 @@
|
||||
import { makeBearerDataListing } from '../request';
|
||||
|
||||
/** Source: LmChatXAiGrok `loadOptions` routing. */
|
||||
export const listXaiModels = makeBearerDataListing('xai', 'https://api.x.ai/v1');
|
||||
@@ -0,0 +1,62 @@
|
||||
import type { ListModelsFn, ListModelsOptions, ProviderModel } from './types';
|
||||
|
||||
/** GET a provider endpoint and parse JSON, throwing a descriptive error on non-2xx. */
|
||||
export async function getJson(
|
||||
url: string,
|
||||
headers: Record<string, string>,
|
||||
options: ListModelsOptions,
|
||||
provider: string,
|
||||
): Promise<unknown> {
|
||||
const fetchFn = options.fetch ?? globalThis.fetch;
|
||||
const response = await fetchFn(url, {
|
||||
method: 'GET',
|
||||
headers: { ...headers, ...options.headers },
|
||||
});
|
||||
if (!response.ok) {
|
||||
const body = await response.text().catch(() => '');
|
||||
throw new Error(
|
||||
`Failed to list ${provider} models (status ${response.status})${body ? `: ${body.slice(0, 500)}` : ''}`,
|
||||
);
|
||||
}
|
||||
return await response.json();
|
||||
}
|
||||
|
||||
/** Resolve the API base: caller override or the provider default, without a trailing slash. */
|
||||
export function baseUrl(options: ListModelsOptions, fallback: string): string {
|
||||
return (options.baseURL ?? fallback).replace(/\/+$/, '');
|
||||
}
|
||||
|
||||
export function bearerHeaders(options: ListModelsOptions): Record<string, string> {
|
||||
return { Authorization: `Bearer ${options.apiKey}` };
|
||||
}
|
||||
|
||||
export function byName(a: ProviderModel, b: ProviderModel): number {
|
||||
return a.name.localeCompare(b.name);
|
||||
}
|
||||
|
||||
export interface IdItem {
|
||||
id?: unknown;
|
||||
}
|
||||
|
||||
/** Map `{ id }` response items to models (name = id), dropping malformed entries. */
|
||||
export function idsToModels(items: IdItem[]): ProviderModel[] {
|
||||
return items
|
||||
.filter((item): item is { id: string } => typeof item.id === 'string' && item.id !== '')
|
||||
.map((item) => ({ id: item.id, name: item.id }));
|
||||
}
|
||||
|
||||
/**
|
||||
* The most common provider shape: `GET <base>/models` with bearer auth
|
||||
* returning `{ data: [{ id }] }`, listed by id ascending.
|
||||
*/
|
||||
export function makeBearerDataListing(provider: string, defaultBaseUrl: string): ListModelsFn {
|
||||
return async (options) => {
|
||||
const data = (await getJson(
|
||||
`${baseUrl(options, defaultBaseUrl)}/models`,
|
||||
bearerHeaders(options),
|
||||
options,
|
||||
provider,
|
||||
)) as { data?: IdItem[] };
|
||||
return idsToModels(data.data ?? []).sort(byName);
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
/** A chat model as reported by the provider's own model-list API. */
|
||||
export interface ProviderModel {
|
||||
/** Provider-native model id, exactly as the provider's API expects it. */
|
||||
id: string;
|
||||
/** Human-readable name when the provider supplies one, otherwise the id. */
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface ListModelsOptions {
|
||||
apiKey: string;
|
||||
/** Override the provider's default API base URL (e.g. a proxy or self-hosted gateway). */
|
||||
baseURL?: string;
|
||||
/**
|
||||
* Transport to use for the request. Callers supply their environment's
|
||||
* proxy-aware fetch; defaults to the global fetch.
|
||||
*/
|
||||
fetch?: typeof globalThis.fetch;
|
||||
/** Extra headers merged into the request (e.g. custom credential headers). */
|
||||
headers?: Record<string, string>;
|
||||
}
|
||||
|
||||
export type ListModelsFn = (options: ListModelsOptions) => Promise<ProviderModel[]>;
|
||||
@@ -54,6 +54,10 @@ export class ListAgentsQueryDto extends Z.class({
|
||||
sortBy: z.enum(AGENTS_LIST_SORT_OPTIONS).optional(),
|
||||
}) {}
|
||||
|
||||
export class AgentProviderModelsQueryDto extends Z.class({
|
||||
credentialId: z.string().min(1).max(64).optional(),
|
||||
}) {}
|
||||
|
||||
export class CreateAgentDto extends Z.class({
|
||||
name: z.string().min(1),
|
||||
}) {}
|
||||
|
||||
@@ -21,3 +21,26 @@ const AGENT_MODEL_PROVIDER_SET = new Set<string>(AGENT_MODEL_PROVIDERS);
|
||||
export function isAgentModelProvider(provider: string): provider is AgentModelProvider {
|
||||
return AGENT_MODEL_PROVIDER_SET.has(provider);
|
||||
}
|
||||
|
||||
/** A model offered in the agent model picker. Mirrors the catalog's `ModelInfo` shape. */
|
||||
export interface AgentCatalogModel {
|
||||
id: string;
|
||||
name: string;
|
||||
releaseDate?: string;
|
||||
reasoning: boolean;
|
||||
toolCall: boolean;
|
||||
cost?: { input: number; output: number; cacheRead?: number; cacheWrite?: number };
|
||||
limits?: { context?: number; output?: number };
|
||||
}
|
||||
|
||||
/** Response of `GET /agents/v2/catalog/models/:provider`. */
|
||||
export interface AgentProviderModelsResponse {
|
||||
provider: string;
|
||||
/**
|
||||
* True when the list was confirmed against the provider's own model API
|
||||
* (only models the credential can actually call). False means the static
|
||||
* catalog fallback, which may include models the provider has retired.
|
||||
*/
|
||||
verified: boolean;
|
||||
models: AgentCatalogModel[];
|
||||
}
|
||||
|
||||
+83
-66
@@ -1,104 +1,100 @@
|
||||
import type { ILoadOptionsFunctions } from 'n8n-workflow';
|
||||
import type { Mocked } from 'vitest';
|
||||
|
||||
import { searchModels, type AnthropicModel } from '../searchModels';
|
||||
import { searchModels } from '../searchModels';
|
||||
|
||||
const mockModels = [
|
||||
{
|
||||
id: 'claude-3-opus-20240229',
|
||||
display_name: 'Claude 3 Opus',
|
||||
type: 'model',
|
||||
created_at: '2024-02-29T00:00:00Z',
|
||||
},
|
||||
{
|
||||
id: 'claude-3-sonnet-20240229',
|
||||
display_name: 'Claude 3 Sonnet',
|
||||
type: 'model',
|
||||
created_at: '2024-02-29T00:00:00Z',
|
||||
},
|
||||
{
|
||||
id: 'claude-3-haiku-20240307',
|
||||
display_name: 'Claude 3 Haiku',
|
||||
type: 'model',
|
||||
created_at: '2024-03-07T00:00:00Z',
|
||||
},
|
||||
{
|
||||
id: 'claude-2.1',
|
||||
display_name: 'Claude 2.1',
|
||||
type: 'model',
|
||||
created_at: '2023-11-21T00:00:00Z',
|
||||
},
|
||||
{
|
||||
id: 'claude-2.0',
|
||||
display_name: 'Claude 2.0',
|
||||
type: 'model',
|
||||
created_at: '2023-07-11T00:00:00Z',
|
||||
},
|
||||
];
|
||||
|
||||
describe('searchModels', () => {
|
||||
let mockContext: Mocked<ILoadOptionsFunctions>;
|
||||
|
||||
const mockModels: AnthropicModel[] = [
|
||||
{
|
||||
id: 'claude-3-opus-20240229',
|
||||
display_name: 'Claude 3 Opus',
|
||||
type: 'model',
|
||||
created_at: '2024-02-29T00:00:00Z',
|
||||
},
|
||||
{
|
||||
id: 'claude-3-sonnet-20240229',
|
||||
display_name: 'Claude 3 Sonnet',
|
||||
type: 'model',
|
||||
created_at: '2024-02-29T00:00:00Z',
|
||||
},
|
||||
{
|
||||
id: 'claude-3-haiku-20240307',
|
||||
display_name: 'Claude 3 Haiku',
|
||||
type: 'model',
|
||||
created_at: '2024-03-07T00:00:00Z',
|
||||
},
|
||||
{
|
||||
id: 'claude-2.1',
|
||||
display_name: 'Claude 2.1',
|
||||
type: 'model',
|
||||
created_at: '2023-11-21T00:00:00Z',
|
||||
},
|
||||
{
|
||||
id: 'claude-2.0',
|
||||
display_name: 'Claude 2.0',
|
||||
type: 'model',
|
||||
created_at: '2023-07-11T00:00:00Z',
|
||||
},
|
||||
];
|
||||
let fetchSpy: ReturnType<typeof vi.fn>;
|
||||
|
||||
beforeEach(() => {
|
||||
mockContext = {
|
||||
getCredentials: vi.fn().mockResolvedValue({}),
|
||||
helpers: {
|
||||
httpRequestWithAuthentication: vi.fn().mockResolvedValue({
|
||||
data: mockModels,
|
||||
}),
|
||||
},
|
||||
getCredentials: vi.fn().mockResolvedValue({ apiKey: 'test-api-key' }),
|
||||
} as unknown as Mocked<ILoadOptionsFunctions>;
|
||||
|
||||
fetchSpy = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({ data: mockModels }),
|
||||
text: async () => '',
|
||||
});
|
||||
vi.stubGlobal('fetch', fetchSpy);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
vi.clearAllMocks();
|
||||
// Reset the getCredentials mock to its default value
|
||||
mockContext.getCredentials = vi.fn().mockResolvedValue({});
|
||||
});
|
||||
|
||||
it('should fetch models from default Anthropic API URL when no custom URL is provided', async () => {
|
||||
const result = await searchModels.call(mockContext);
|
||||
|
||||
expect(mockContext.getCredentials).toHaveBeenCalledWith('anthropicApi');
|
||||
expect(mockContext.helpers.httpRequestWithAuthentication).toHaveBeenCalledWith('anthropicApi', {
|
||||
url: 'https://api.anthropic.com/v1/models',
|
||||
headers: {
|
||||
'anthropic-version': '2023-06-01',
|
||||
},
|
||||
});
|
||||
expect(fetchSpy).toHaveBeenCalledWith(
|
||||
'https://api.anthropic.com/v1/models',
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({
|
||||
'x-api-key': 'test-api-key',
|
||||
'anthropic-version': '2023-06-01',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(result.results).toHaveLength(5);
|
||||
});
|
||||
|
||||
it('should fetch models from custom Anthropic API URL when provided in credentials', async () => {
|
||||
const customUrl = 'https://custom-anthropic-api.example.com';
|
||||
// Override the default mock to return credentials with a custom URL
|
||||
mockContext.getCredentials = vi.fn().mockResolvedValue({ url: customUrl });
|
||||
mockContext.getCredentials = vi
|
||||
.fn()
|
||||
.mockResolvedValue({ apiKey: 'test-api-key', url: customUrl });
|
||||
|
||||
const result = await searchModels.call(mockContext);
|
||||
|
||||
expect(mockContext.getCredentials).toHaveBeenCalledWith('anthropicApi');
|
||||
expect(mockContext.helpers.httpRequestWithAuthentication).toHaveBeenCalledWith('anthropicApi', {
|
||||
url: `${customUrl}/v1/models`,
|
||||
headers: {
|
||||
'anthropic-version': '2023-06-01',
|
||||
},
|
||||
});
|
||||
expect(fetchSpy).toHaveBeenCalledWith(`${customUrl}/v1/models`, expect.anything());
|
||||
expect(result.results).toHaveLength(5);
|
||||
});
|
||||
|
||||
it('should use default URL when empty URL is provided in credentials', async () => {
|
||||
// Override the default mock to return credentials with an empty URL
|
||||
mockContext.getCredentials = vi.fn().mockResolvedValue({ url: null });
|
||||
mockContext.getCredentials = vi
|
||||
.fn()
|
||||
.mockResolvedValue({ apiKey: 'test-api-key', url: undefined });
|
||||
|
||||
const result = await searchModels.call(mockContext);
|
||||
|
||||
expect(mockContext.getCredentials).toHaveBeenCalledWith('anthropicApi');
|
||||
expect(mockContext.helpers.httpRequestWithAuthentication).toHaveBeenCalledWith('anthropicApi', {
|
||||
url: 'https://api.anthropic.com/v1/models',
|
||||
headers: {
|
||||
'anthropic-version': '2023-06-01',
|
||||
},
|
||||
});
|
||||
expect(fetchSpy).toHaveBeenCalledWith('https://api.anthropic.com/v1/models', expect.anything());
|
||||
expect(result.results).toHaveLength(5);
|
||||
});
|
||||
|
||||
@@ -140,4 +136,25 @@ describe('searchModels', () => {
|
||||
|
||||
expect(result.results).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should include the credential custom header in the request', async () => {
|
||||
mockContext.getCredentials = vi.fn().mockResolvedValue({
|
||||
apiKey: 'test-api-key',
|
||||
header: true,
|
||||
headerName: 'X-Gateway-Auth',
|
||||
headerValue: 'gateway-value',
|
||||
});
|
||||
|
||||
await searchModels.call(mockContext);
|
||||
|
||||
expect(fetchSpy).toHaveBeenCalledWith(
|
||||
'https://api.anthropic.com/v1/models',
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({
|
||||
'x-api-key': 'test-api-key',
|
||||
'X-Gateway-Auth': 'gateway-value',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,63 +1,34 @@
|
||||
import type {
|
||||
ILoadOptionsFunctions,
|
||||
INodeListSearchItems,
|
||||
INodeListSearchResult,
|
||||
} from 'n8n-workflow';
|
||||
import { getProxyAgent } from '@n8n/ai-utilities';
|
||||
import { listAnthropicModels } from '@n8n/ai-utilities/model-discovery';
|
||||
import type { ILoadOptionsFunctions, INodeListSearchResult } from 'n8n-workflow';
|
||||
|
||||
export interface AnthropicModel {
|
||||
id: string;
|
||||
display_name: string;
|
||||
type: string;
|
||||
created_at: string;
|
||||
}
|
||||
import { mergeCustomHeaders } from '../../../../utils/helpers';
|
||||
|
||||
export async function searchModels(
|
||||
this: ILoadOptionsFunctions,
|
||||
filter?: string,
|
||||
): Promise<INodeListSearchResult> {
|
||||
const credentials = await this.getCredentials<{ url?: string }>('anthropicApi');
|
||||
const credentials = await this.getCredentials('anthropicApi');
|
||||
const baseURL = (credentials.url as string) ?? 'https://api.anthropic.com';
|
||||
|
||||
const baseURL = credentials.url ?? 'https://api.anthropic.com';
|
||||
const response = (await this.helpers.httpRequestWithAuthentication.call(this, 'anthropicApi', {
|
||||
url: `${baseURL}/v1/models`,
|
||||
headers: {
|
||||
'anthropic-version': '2023-06-01',
|
||||
},
|
||||
})) as { data: AnthropicModel[] };
|
||||
|
||||
const models = response.data || [];
|
||||
let results: INodeListSearchItems[] = [];
|
||||
|
||||
if (filter) {
|
||||
for (const model of models) {
|
||||
if (model.id.toLowerCase().includes(filter.toLowerCase())) {
|
||||
results.push({
|
||||
name: model.display_name,
|
||||
value: model.id,
|
||||
});
|
||||
}
|
||||
}
|
||||
} else {
|
||||
results = models.map((model) => ({
|
||||
name: model.display_name,
|
||||
value: model.id,
|
||||
}));
|
||||
}
|
||||
|
||||
// Sort models with more recent ones first (claude-3 before claude-2)
|
||||
results = results.sort((a, b) => {
|
||||
const modelA = models.find((m) => m.id === a.value);
|
||||
const modelB = models.find((m) => m.id === b.value);
|
||||
|
||||
if (!modelA || !modelB) return 0;
|
||||
|
||||
// Sort by created_at date, most recent first
|
||||
const dateA = new Date(modelA.created_at);
|
||||
const dateB = new Date(modelB.created_at);
|
||||
return dateB.getTime() - dateA.getTime();
|
||||
// Shared with the agents model catalog: endpoint, auth, and newest-first
|
||||
// ordering live in @n8n/ai-utilities/model-discovery. The credential's
|
||||
// optional custom header (for gateway/proxy-backed credentials) is merged in,
|
||||
// matching what the credential's `authenticate` applies on other requests.
|
||||
const models = await listAnthropicModels({
|
||||
apiKey: (credentials.apiKey as string) ?? '',
|
||||
baseURL,
|
||||
headers: mergeCustomHeaders(credentials, {}),
|
||||
fetch: async (url, init) =>
|
||||
await fetch(url, {
|
||||
...init,
|
||||
dispatcher: getProxyAgent(baseURL),
|
||||
} as RequestInit),
|
||||
});
|
||||
|
||||
return {
|
||||
results,
|
||||
results: models
|
||||
.filter((model) => !filter || model.id.toLowerCase().includes(filter.toLowerCase()))
|
||||
.map((model) => ({ name: model.name, value: model.id })),
|
||||
};
|
||||
}
|
||||
|
||||
+45
-104
@@ -1,14 +1,33 @@
|
||||
import type { ILoadOptionsFunctions } from 'n8n-workflow';
|
||||
import OpenAI from 'openai';
|
||||
import type { Mocked, MockedClass } from 'vitest';
|
||||
import type { Mocked } from 'vitest';
|
||||
|
||||
import { searchModels } from '../loadModels';
|
||||
|
||||
vi.mock('openai');
|
||||
const MODEL_IDS = [
|
||||
'gpt-4',
|
||||
'gpt-3.5-turbo',
|
||||
'gpt-3.5-turbo-instruct',
|
||||
'ft:gpt-3.5-turbo',
|
||||
'o1-model',
|
||||
'whisper-1',
|
||||
'davinci-instruct-beta',
|
||||
'computer-use-preview',
|
||||
'whisper-1-preview',
|
||||
'tts-model',
|
||||
'other-model',
|
||||
];
|
||||
|
||||
const OFFICIAL_API_RESULTS = [
|
||||
{ name: 'ft:gpt-3.5-turbo', value: 'ft:gpt-3.5-turbo' },
|
||||
{ name: 'gpt-3.5-turbo', value: 'gpt-3.5-turbo' },
|
||||
{ name: 'gpt-4', value: 'gpt-4' },
|
||||
{ name: 'o1-model', value: 'o1-model' },
|
||||
{ name: 'other-model', value: 'other-model' },
|
||||
];
|
||||
|
||||
describe('searchModels', () => {
|
||||
let mockContext: Mocked<ILoadOptionsFunctions>;
|
||||
let mockOpenAI: Mocked<typeof OpenAI>;
|
||||
let fetchSpy: ReturnType<typeof vi.fn>;
|
||||
|
||||
beforeEach(() => {
|
||||
mockContext = {
|
||||
@@ -18,73 +37,41 @@ describe('searchModels', () => {
|
||||
getNodeParameter: vi.fn().mockReturnValue(''),
|
||||
} as unknown as Mocked<ILoadOptionsFunctions>;
|
||||
|
||||
// Setup OpenAI mock with required properties
|
||||
const mockOpenAIInstance = {
|
||||
apiKey: 'test-api-key',
|
||||
organization: null,
|
||||
project: null,
|
||||
_options: {},
|
||||
models: {
|
||||
list: vi.fn().mockResolvedValue({
|
||||
data: [
|
||||
{ id: 'gpt-4' },
|
||||
{ id: 'gpt-3.5-turbo' },
|
||||
{ id: 'gpt-3.5-turbo-instruct' },
|
||||
{ id: 'ft:gpt-3.5-turbo' },
|
||||
{ id: 'o1-model' },
|
||||
{ id: 'whisper-1' },
|
||||
{ id: 'davinci-instruct-beta' },
|
||||
{ id: 'computer-use-preview' },
|
||||
{ id: 'whisper-1-preview' },
|
||||
{ id: 'tts-model' },
|
||||
{ id: 'other-model' },
|
||||
],
|
||||
}),
|
||||
},
|
||||
} as unknown as OpenAI;
|
||||
|
||||
(OpenAI as MockedClass<typeof OpenAI>).mockImplementation(function () {
|
||||
return mockOpenAIInstance;
|
||||
fetchSpy = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({ data: MODEL_IDS.map((id) => ({ id })) }),
|
||||
text: async () => '',
|
||||
});
|
||||
|
||||
mockOpenAI = OpenAI as Mocked<typeof OpenAI>;
|
||||
vi.stubGlobal('fetch', fetchSpy);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it('should return filtered models if custom API endpoint is not provided', async () => {
|
||||
const result = await searchModels.call(mockContext);
|
||||
|
||||
expect(mockOpenAI).toHaveBeenCalledWith(
|
||||
expect(fetchSpy).toHaveBeenCalledWith(
|
||||
'https://api.openai.com/v1/models',
|
||||
expect.objectContaining({
|
||||
baseURL: 'https://api.openai.com/v1',
|
||||
apiKey: 'test-api-key',
|
||||
headers: expect.objectContaining({ Authorization: 'Bearer test-api-key' }),
|
||||
}),
|
||||
);
|
||||
expect(result.results).toEqual([
|
||||
{ name: 'ft:gpt-3.5-turbo', value: 'ft:gpt-3.5-turbo' },
|
||||
{ name: 'gpt-3.5-turbo', value: 'gpt-3.5-turbo' },
|
||||
{ name: 'gpt-4', value: 'gpt-4' },
|
||||
{ name: 'o1-model', value: 'o1-model' },
|
||||
{ name: 'other-model', value: 'other-model' },
|
||||
]);
|
||||
expect(result.results).toEqual(OFFICIAL_API_RESULTS);
|
||||
});
|
||||
|
||||
it('should initialize OpenAI with correct credentials', async () => {
|
||||
it('should use the credential url as the API base', async () => {
|
||||
mockContext.getCredentials.mockResolvedValueOnce({
|
||||
apiKey: 'test-api-key',
|
||||
url: 'https://test-url.com',
|
||||
});
|
||||
|
||||
await searchModels.call(mockContext);
|
||||
|
||||
expect(mockOpenAI).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
baseURL: 'https://test-url.com',
|
||||
apiKey: 'test-api-key',
|
||||
}),
|
||||
);
|
||||
expect(fetchSpy).toHaveBeenCalledWith('https://test-url.com/models', expect.anything());
|
||||
});
|
||||
|
||||
it('should use default OpenAI URL if no custom URL provided', async () => {
|
||||
@@ -94,18 +81,15 @@ describe('searchModels', () => {
|
||||
|
||||
await searchModels.call(mockContext);
|
||||
|
||||
expect(mockOpenAI).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
baseURL: 'https://api.openai.com/v1',
|
||||
apiKey: 'test-api-key',
|
||||
}),
|
||||
);
|
||||
expect(fetchSpy).toHaveBeenCalledWith('https://api.openai.com/v1/models', expect.anything());
|
||||
});
|
||||
|
||||
it('should include all models for custom API endpoints', async () => {
|
||||
mockContext.getNodeParameter = vi.fn().mockReturnValue('https://custom-api.com');
|
||||
|
||||
const result = await searchModels.call(mockContext);
|
||||
|
||||
expect(fetchSpy).toHaveBeenCalledWith('https://custom-api.com/models', expect.anything());
|
||||
expect(result.results).toEqual([
|
||||
{ name: 'computer-use-preview', value: 'computer-use-preview' },
|
||||
{ name: 'davinci-instruct-beta', value: 'davinci-instruct-beta' },
|
||||
@@ -130,13 +114,7 @@ describe('searchModels', () => {
|
||||
|
||||
const result = await searchModels.call(mockContext);
|
||||
|
||||
expect(result.results).toEqual([
|
||||
{ name: 'ft:gpt-3.5-turbo', value: 'ft:gpt-3.5-turbo' },
|
||||
{ name: 'gpt-3.5-turbo', value: 'gpt-3.5-turbo' },
|
||||
{ name: 'gpt-4', value: 'gpt-4' },
|
||||
{ name: 'o1-model', value: 'o1-model' },
|
||||
{ name: 'other-model', value: 'other-model' },
|
||||
]);
|
||||
expect(result.results).toEqual(OFFICIAL_API_RESULTS);
|
||||
});
|
||||
|
||||
it('should filter models based on search term', async () => {
|
||||
@@ -159,7 +137,7 @@ describe('searchModels', () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it('should include custom credential headers in the OpenAI client', async () => {
|
||||
it('should include custom credential headers in the request', async () => {
|
||||
mockContext.getCredentials.mockResolvedValueOnce({
|
||||
apiKey: 'test-api-key',
|
||||
header: true,
|
||||
@@ -169,48 +147,11 @@ describe('searchModels', () => {
|
||||
|
||||
await searchModels.call(mockContext);
|
||||
|
||||
expect(mockOpenAI).toHaveBeenCalledWith(
|
||||
expect(fetchSpy).toHaveBeenCalledWith(
|
||||
'https://api.openai.com/v1/models',
|
||||
expect.objectContaining({
|
||||
defaultHeaders: expect.objectContaining({
|
||||
'X-Custom-Auth': 'custom-value',
|
||||
}),
|
||||
headers: expect.objectContaining({ 'X-Custom-Auth': 'custom-value' }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should return models sorted alphabetically by id', async () => {
|
||||
// Setup a mock with scrambled order
|
||||
const mockUnsortedInstance = {
|
||||
apiKey: 'test-api-key',
|
||||
models: {
|
||||
list: vi.fn().mockResolvedValue({
|
||||
data: [
|
||||
{ id: 'gpt-4' },
|
||||
{ id: 'a-model' },
|
||||
{ id: 'o1-model' },
|
||||
{ id: 'gpt-3.5-turbo' },
|
||||
{ id: 'z-model' },
|
||||
],
|
||||
}),
|
||||
},
|
||||
} as unknown as OpenAI;
|
||||
|
||||
(OpenAI as MockedClass<typeof OpenAI>).mockImplementation(function () {
|
||||
return mockUnsortedInstance;
|
||||
});
|
||||
|
||||
// Custom API endpoint to include all models
|
||||
mockContext.getNodeParameter = vi.fn().mockReturnValue('https://custom-api.com');
|
||||
|
||||
const result = await searchModels.call(mockContext);
|
||||
|
||||
// Verify the results are sorted alphabetically
|
||||
expect(result.results).toEqual([
|
||||
{ name: 'a-model', value: 'a-model' },
|
||||
{ name: 'gpt-3.5-turbo', value: 'gpt-3.5-turbo' },
|
||||
{ name: 'gpt-4', value: 'gpt-4' },
|
||||
{ name: 'o1-model', value: 'o1-model' },
|
||||
{ name: 'z-model', value: 'z-model' },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
import { getProxyAgent } from '@n8n/ai-utilities';
|
||||
import { listOpenAiModels } from '@n8n/ai-utilities/model-discovery';
|
||||
import { AiConfig } from '@n8n/config';
|
||||
import { Container } from '@n8n/di';
|
||||
import type { ILoadOptionsFunctions, INodeListSearchResult } from 'n8n-workflow';
|
||||
import OpenAI from 'openai';
|
||||
|
||||
import { mergeCustomHeaders } from '../../../../utils/helpers';
|
||||
import { shouldIncludeModel } from '../../../vendors/OpenAi/helpers/modelFiltering';
|
||||
|
||||
export async function searchModels(
|
||||
this: ILoadOptionsFunctions,
|
||||
@@ -17,35 +16,24 @@ export async function searchModels(
|
||||
(credentials.url as string) ||
|
||||
'https://api.openai.com/v1';
|
||||
const { openAiDefaultHeaders } = Container.get(AiConfig);
|
||||
const defaultHeaders = mergeCustomHeaders(credentials, openAiDefaultHeaders ?? {});
|
||||
const headers = mergeCustomHeaders(credentials, openAiDefaultHeaders ?? {});
|
||||
|
||||
const openai = new OpenAI({
|
||||
baseURL,
|
||||
// Shared with the agents model catalog: endpoint, auth, chat-model filtering
|
||||
// (including include-all on custom hosts) live in @n8n/ai-utilities/model-discovery.
|
||||
const models = await listOpenAiModels({
|
||||
apiKey: credentials.apiKey as string,
|
||||
fetchOptions: {
|
||||
dispatcher: getProxyAgent(baseURL),
|
||||
},
|
||||
defaultHeaders,
|
||||
baseURL,
|
||||
headers,
|
||||
fetch: async (url, init) =>
|
||||
await fetch(url, {
|
||||
...init,
|
||||
dispatcher: getProxyAgent(baseURL),
|
||||
} as RequestInit),
|
||||
});
|
||||
const { data: models = [] } = await openai.models.list();
|
||||
|
||||
const url = baseURL && new URL(baseURL);
|
||||
const isCustomAPI = !!(url && !['api.openai.com', 'ai-assistant.n8n.io'].includes(url.hostname));
|
||||
|
||||
const filteredModels = models.filter((model: { id: string }) => {
|
||||
const includeModel = shouldIncludeModel(model.id, isCustomAPI);
|
||||
|
||||
if (!filter) return includeModel;
|
||||
|
||||
return includeModel && model.id.toLowerCase().includes(filter.toLowerCase());
|
||||
});
|
||||
|
||||
filteredModels.sort((a, b) => a.id.localeCompare(b.id));
|
||||
|
||||
return {
|
||||
results: filteredModels.map((model: { id: string }) => ({
|
||||
name: model.id,
|
||||
value: model.id,
|
||||
})),
|
||||
results: models
|
||||
.filter((model) => !filter || model.id.toLowerCase().includes(filter.toLowerCase()))
|
||||
.map((model) => ({ name: model.id, value: model.id })),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -419,20 +419,31 @@ describe('LmChatAnthropic', () => {
|
||||
describe('searchModels', () => {
|
||||
let mockLoadContext: ILoadOptionsFunctions;
|
||||
let mockGetCredentials: Mock;
|
||||
let mockHttpRequest: Mock;
|
||||
let fetchSpy: Mock;
|
||||
|
||||
beforeEach(() => {
|
||||
mockGetCredentials = vi.fn();
|
||||
mockHttpRequest = vi.fn();
|
||||
fetchSpy = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchSpy);
|
||||
|
||||
mockLoadContext = {
|
||||
getCredentials: mockGetCredentials,
|
||||
helpers: {
|
||||
httpRequestWithAuthentication: mockHttpRequest,
|
||||
},
|
||||
} as unknown as ILoadOptionsFunctions;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
function mockModelsResponse(models: unknown[]) {
|
||||
fetchSpy.mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({ data: models }),
|
||||
text: async () => '',
|
||||
});
|
||||
}
|
||||
|
||||
it('should return all models sorted by creation date', async () => {
|
||||
const mockModels = [
|
||||
{
|
||||
@@ -455,20 +466,21 @@ describe('LmChatAnthropic', () => {
|
||||
},
|
||||
];
|
||||
|
||||
mockGetCredentials.mockResolvedValue({});
|
||||
mockHttpRequest.mockResolvedValue({
|
||||
data: mockModels,
|
||||
});
|
||||
mockGetCredentials.mockResolvedValue({ apiKey: 'test-api-key' });
|
||||
mockModelsResponse(mockModels);
|
||||
|
||||
const { searchModels } = lmChatAnthropic.methods.listSearch;
|
||||
const result = await searchModels.call(mockLoadContext);
|
||||
|
||||
expect(mockHttpRequest).toHaveBeenCalledWith('anthropicApi', {
|
||||
url: 'https://api.anthropic.com/v1/models',
|
||||
headers: {
|
||||
'anthropic-version': '2023-06-01',
|
||||
},
|
||||
});
|
||||
expect(fetchSpy).toHaveBeenCalledWith(
|
||||
'https://api.anthropic.com/v1/models',
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({
|
||||
'x-api-key': 'test-api-key',
|
||||
'anthropic-version': '2023-06-01',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(result.results).toHaveLength(3);
|
||||
// Verify sorted by creation date (newest first)
|
||||
@@ -499,10 +511,8 @@ describe('LmChatAnthropic', () => {
|
||||
},
|
||||
];
|
||||
|
||||
mockGetCredentials.mockResolvedValue({});
|
||||
mockHttpRequest.mockResolvedValue({
|
||||
data: mockModels,
|
||||
});
|
||||
mockGetCredentials.mockResolvedValue({ apiKey: 'test-api-key' });
|
||||
mockModelsResponse(mockModels);
|
||||
|
||||
const { searchModels } = lmChatAnthropic.methods.listSearch;
|
||||
const result = await searchModels.call(mockLoadContext, 'opus');
|
||||
@@ -522,10 +532,8 @@ describe('LmChatAnthropic', () => {
|
||||
},
|
||||
];
|
||||
|
||||
mockGetCredentials.mockResolvedValue({});
|
||||
mockHttpRequest.mockResolvedValue({
|
||||
data: mockModels,
|
||||
});
|
||||
mockGetCredentials.mockResolvedValue({ apiKey: 'test-api-key' });
|
||||
mockModelsResponse(mockModels);
|
||||
|
||||
const { searchModels } = lmChatAnthropic.methods.listSearch;
|
||||
const result = await searchModels.call(mockLoadContext, 'SONNET');
|
||||
@@ -538,28 +546,20 @@ describe('LmChatAnthropic', () => {
|
||||
const customURL = 'https://custom-anthropic.example.com';
|
||||
|
||||
mockGetCredentials.mockResolvedValue({
|
||||
apiKey: 'test-api-key',
|
||||
url: customURL,
|
||||
});
|
||||
mockHttpRequest.mockResolvedValue({
|
||||
data: [],
|
||||
});
|
||||
mockModelsResponse([]);
|
||||
|
||||
const { searchModels } = lmChatAnthropic.methods.listSearch;
|
||||
await searchModels.call(mockLoadContext);
|
||||
|
||||
expect(mockHttpRequest).toHaveBeenCalledWith('anthropicApi', {
|
||||
url: `${customURL}/v1/models`,
|
||||
headers: {
|
||||
'anthropic-version': '2023-06-01',
|
||||
},
|
||||
});
|
||||
expect(fetchSpy).toHaveBeenCalledWith(`${customURL}/v1/models`, expect.anything());
|
||||
});
|
||||
|
||||
it('should handle empty model list', async () => {
|
||||
mockGetCredentials.mockResolvedValue({});
|
||||
mockHttpRequest.mockResolvedValue({
|
||||
data: [],
|
||||
});
|
||||
mockGetCredentials.mockResolvedValue({ apiKey: 'test-api-key' });
|
||||
mockModelsResponse([]);
|
||||
|
||||
const { searchModels } = lmChatAnthropic.methods.listSearch;
|
||||
const result = await searchModels.call(mockLoadContext);
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
import type { User } from '@n8n/db';
|
||||
import { mockLogger } from '@n8n/backend-test-utils';
|
||||
import { mock } from 'vitest-mock-extended';
|
||||
|
||||
import { AgentModelCatalogService } from '../agent-model-catalog.service';
|
||||
import type { BuilderModelLiveLookupService } from '../builder/builder-model-live-lookup.service';
|
||||
|
||||
const fetchProviderCatalog = vi.fn();
|
||||
vi.mock('@n8n/agents', () => ({
|
||||
fetchProviderCatalog: (...args: unknown[]) => fetchProviderCatalog(...args) as unknown,
|
||||
}));
|
||||
|
||||
const user = mock<User>({ id: 'user-1' });
|
||||
const credentialId = 'cred-1';
|
||||
|
||||
const catalogFixture = {
|
||||
anthropic: {
|
||||
id: 'anthropic',
|
||||
name: 'Anthropic',
|
||||
models: {
|
||||
'claude-sonnet-4-6': {
|
||||
id: 'claude-sonnet-4-6',
|
||||
name: 'Claude Sonnet 4.6',
|
||||
reasoning: true,
|
||||
toolCall: true,
|
||||
cost: { input: 3, output: 15 },
|
||||
},
|
||||
'claude-opus-4-0': {
|
||||
id: 'claude-opus-4-0',
|
||||
name: 'Claude Opus 4',
|
||||
reasoning: true,
|
||||
toolCall: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
google: {
|
||||
id: 'google',
|
||||
name: 'Google',
|
||||
models: {
|
||||
'gemini-2.5-flash': {
|
||||
id: 'gemini-2.5-flash',
|
||||
name: 'Gemini 2.5 Flash',
|
||||
reasoning: true,
|
||||
toolCall: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
function makeService() {
|
||||
const lookupService = mock<BuilderModelLiveLookupService>();
|
||||
const service = new AgentModelCatalogService(mockLogger(), lookupService);
|
||||
return { service, lookupService };
|
||||
}
|
||||
|
||||
describe('AgentModelCatalogService', () => {
|
||||
beforeEach(() => {
|
||||
fetchProviderCatalog.mockReset();
|
||||
fetchProviderCatalog.mockResolvedValue(catalogFixture);
|
||||
});
|
||||
|
||||
it('keeps catalog models the provider still reports live, with catalog metadata, and prunes the rest', async () => {
|
||||
const { service, lookupService } = makeService();
|
||||
// Provider reports Sonnet but not Opus — Opus (retired) must be pruned.
|
||||
lookupService.list.mockResolvedValue([
|
||||
{ name: 'Claude Sonnet 4.6', value: 'claude-sonnet-4-6' },
|
||||
]);
|
||||
|
||||
const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId);
|
||||
|
||||
expect(result.verified).toBe(true);
|
||||
expect(result.models).toHaveLength(1);
|
||||
expect(result.models[0]).toMatchObject({
|
||||
id: 'claude-sonnet-4-6',
|
||||
name: 'Claude Sonnet 4.6',
|
||||
cost: { input: 3, output: 15 },
|
||||
});
|
||||
expect(lookupService.list).toHaveBeenCalledWith(
|
||||
user,
|
||||
'project-1',
|
||||
credentialId,
|
||||
'anthropicApi',
|
||||
'anthropic',
|
||||
);
|
||||
});
|
||||
|
||||
it('verifies a catalog alias when the provider lists only its dated snapshot', async () => {
|
||||
const { service, lookupService } = makeService();
|
||||
fetchProviderCatalog.mockResolvedValue({
|
||||
anthropic: {
|
||||
id: 'anthropic',
|
||||
name: 'Anthropic',
|
||||
models: {
|
||||
// models.dev keeps the versionless alias and drops the snapshot…
|
||||
'claude-haiku-4-5': {
|
||||
id: 'claude-haiku-4-5',
|
||||
name: 'Claude Haiku 4.5',
|
||||
reasoning: true,
|
||||
toolCall: true,
|
||||
},
|
||||
'claude-opus-4-0': {
|
||||
id: 'claude-opus-4-0',
|
||||
name: 'Claude Opus 4',
|
||||
reasoning: true,
|
||||
toolCall: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
// …while Anthropic's API lists the dated snapshot only. The snapshot must
|
||||
// verify its alias; the retired alias with no live counterpart stays pruned.
|
||||
lookupService.list.mockResolvedValue([
|
||||
{ name: 'Claude Haiku 4.5', value: 'claude-haiku-4-5-20251001' },
|
||||
]);
|
||||
|
||||
const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId);
|
||||
|
||||
expect(result.verified).toBe(true);
|
||||
expect(result.models.map((m) => m.id)).toEqual(['claude-haiku-4-5']);
|
||||
});
|
||||
|
||||
it('does not add live models that are missing from the catalog', async () => {
|
||||
const { service, lookupService } = makeService();
|
||||
// Live list includes a model models.dev has no entry for, alongside a known one.
|
||||
lookupService.list.mockResolvedValue([
|
||||
{ name: 'Claude Sonnet 4.6', value: 'claude-sonnet-4-6' },
|
||||
{ name: 'Claude Brand New', value: 'claude-brand-new' },
|
||||
]);
|
||||
|
||||
const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId);
|
||||
|
||||
expect(result.verified).toBe(true);
|
||||
// Only the catalog-known model survives; the live-only one is never added.
|
||||
expect(result.models.map((m) => m.id)).toEqual(['claude-sonnet-4-6']);
|
||||
});
|
||||
|
||||
it('strips the "models/" prefix from google model ids before matching', async () => {
|
||||
const { service, lookupService } = makeService();
|
||||
lookupService.list.mockResolvedValue([
|
||||
{ name: 'models/gemini-2.5-flash', value: 'models/gemini-2.5-flash' },
|
||||
]);
|
||||
|
||||
const result = await service.getProviderModels(user, 'project-1', 'google', credentialId);
|
||||
|
||||
expect(result.models).toEqual([
|
||||
expect.objectContaining({ id: 'gemini-2.5-flash', name: 'Gemini 2.5 Flash' }),
|
||||
]);
|
||||
});
|
||||
|
||||
it('falls back to the catalog list (verified: false) when the live lookup fails', async () => {
|
||||
const { service, lookupService } = makeService();
|
||||
lookupService.list.mockRejectedValue(new Error('provider is down'));
|
||||
|
||||
const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId);
|
||||
|
||||
expect(result.verified).toBe(false);
|
||||
expect(result.models.map((m) => m.id).sort()).toEqual(['claude-opus-4-0', 'claude-sonnet-4-6']);
|
||||
});
|
||||
|
||||
it('falls back to the catalog list (verified: false) when no credential is provided', async () => {
|
||||
const { service, lookupService } = makeService();
|
||||
|
||||
const result = await service.getProviderModels(user, 'project-1', 'anthropic', undefined);
|
||||
|
||||
expect(result.verified).toBe(false);
|
||||
expect(result.models).toHaveLength(2);
|
||||
expect(lookupService.list).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('falls back to the catalog list for providers without a live lookup', async () => {
|
||||
const { service, lookupService } = makeService();
|
||||
fetchProviderCatalog.mockResolvedValue({
|
||||
'aws-bedrock': {
|
||||
id: 'aws-bedrock',
|
||||
name: 'AWS Bedrock',
|
||||
models: {
|
||||
'anthropic.claude-sonnet-4-6-v1:0': {
|
||||
id: 'anthropic.claude-sonnet-4-6-v1:0',
|
||||
name: 'Claude Sonnet 4.6',
|
||||
reasoning: true,
|
||||
toolCall: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = await service.getProviderModels(user, 'project-1', 'aws-bedrock', credentialId);
|
||||
|
||||
expect(result.verified).toBe(false);
|
||||
expect(result.models).toHaveLength(1);
|
||||
expect(lookupService.list).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('still returns live models (verified: true) when the catalog fetch fails', async () => {
|
||||
const { service, lookupService } = makeService();
|
||||
fetchProviderCatalog.mockRejectedValue(new Error('models.dev unreachable'));
|
||||
lookupService.list.mockResolvedValue([
|
||||
{ name: 'Claude Sonnet 4.6', value: 'claude-sonnet-4-6' },
|
||||
]);
|
||||
|
||||
const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId);
|
||||
|
||||
expect(result.verified).toBe(true);
|
||||
expect(result.models).toEqual([
|
||||
expect.objectContaining({ id: 'claude-sonnet-4-6', name: 'Claude Sonnet 4.6' }),
|
||||
]);
|
||||
});
|
||||
|
||||
it('returns an empty unverified list when both the lookup and the catalog fail', async () => {
|
||||
const { service, lookupService } = makeService();
|
||||
fetchProviderCatalog.mockRejectedValue(new Error('models.dev unreachable'));
|
||||
lookupService.list.mockRejectedValue(new Error('provider is down'));
|
||||
|
||||
const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId);
|
||||
|
||||
expect(result.verified).toBe(false);
|
||||
expect(result.models).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -32,7 +32,7 @@ import {
|
||||
AgentsBuilderToolsService,
|
||||
getAgentConfigHash,
|
||||
} from '../builder/agents-builder-tools.service';
|
||||
import type { BuilderModelLookupService } from '../builder/builder-model-lookup.service';
|
||||
import type { BuilderModelLiveLookupService } from '../builder/builder-model-live-lookup.service';
|
||||
import { BUILDER_TOOLS } from '../builder/builder-tool-names';
|
||||
import type { Agent } from '../entities/agent.entity';
|
||||
import type { AgentRepository } from '../repositories/agent.repository';
|
||||
@@ -69,7 +69,7 @@ function makeService() {
|
||||
const secureRuntime = mock<AgentSecureRuntime>();
|
||||
const workflowRepository = mock<WorkflowRepository>();
|
||||
const agentsToolsService = mock<AgentsToolsService>();
|
||||
const builderModelLookupService = mock<BuilderModelLookupService>();
|
||||
const builderModelLiveLookupService = mock<BuilderModelLiveLookupService>();
|
||||
const credentialTypes = mock<CredentialTypes>();
|
||||
const mcpRegistryService = mock<McpRegistryService>();
|
||||
const agentTaskService = mock<AgentTaskService>();
|
||||
@@ -97,7 +97,7 @@ function makeService() {
|
||||
secureRuntime,
|
||||
workflowRepository,
|
||||
agentsToolsService,
|
||||
builderModelLookupService,
|
||||
builderModelLiveLookupService,
|
||||
mcpRegistryService,
|
||||
mock(),
|
||||
credentialTypes,
|
||||
|
||||
@@ -1,3 +1,12 @@
|
||||
import type { AgentProviderModelsQueryDto } from '@n8n/api-types';
|
||||
import type { AuthenticatedRequest } from '@n8n/db';
|
||||
import type { Response } from 'express';
|
||||
import { mock } from 'vitest-mock-extended';
|
||||
|
||||
import { BadRequestError } from '@/errors/response-errors/bad-request.error';
|
||||
|
||||
import type { AgentIntegrationPersistenceService } from '../agent-integration-persistence.service';
|
||||
import type { AgentModelCatalogService } from '../agent-model-catalog.service';
|
||||
import { AgentsCatalogController } from '../agents-catalog.controller';
|
||||
import {
|
||||
expectProjectScopedAgentRoutes,
|
||||
@@ -11,8 +20,56 @@ describe('AgentsCatalogController route access scopes', () => {
|
||||
|
||||
it.each([
|
||||
['getModelCatalog', 'agent:read'],
|
||||
['getProviderModels', 'agent:read'],
|
||||
['listIntegrations', 'agent:read'],
|
||||
])('%s uses %s', (handlerName, scope) => {
|
||||
expect(routes.get(handlerName)?.accessScope?.scope).toBe(scope);
|
||||
});
|
||||
});
|
||||
|
||||
describe('AgentsCatalogController — getProviderModels', () => {
|
||||
function makeController() {
|
||||
const modelCatalogService = mock<AgentModelCatalogService>();
|
||||
const controller = new AgentsCatalogController(
|
||||
mock<AgentIntegrationPersistenceService>(),
|
||||
modelCatalogService,
|
||||
);
|
||||
return { controller, modelCatalogService };
|
||||
}
|
||||
|
||||
const req = mock<AuthenticatedRequest<{ projectId: string }>>({
|
||||
user: { id: 'user-1' },
|
||||
params: { projectId: 'project-1' },
|
||||
});
|
||||
|
||||
it('delegates to the model catalog service with the user, project, and credential', async () => {
|
||||
const { controller, modelCatalogService } = makeController();
|
||||
const response = { provider: 'anthropic', verified: true, models: [] };
|
||||
modelCatalogService.getProviderModels.mockResolvedValue(response);
|
||||
|
||||
const result = await controller.getProviderModels(req, mock<Response>(), 'anthropic', {
|
||||
credentialId: 'cred-1',
|
||||
} as AgentProviderModelsQueryDto);
|
||||
|
||||
expect(result).toBe(response);
|
||||
expect(modelCatalogService.getProviderModels).toHaveBeenCalledWith(
|
||||
req.user,
|
||||
'project-1',
|
||||
'anthropic',
|
||||
'cred-1',
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects unknown providers', async () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
await expect(
|
||||
controller.getProviderModels(
|
||||
req,
|
||||
mock<Response>(),
|
||||
'not-a-provider',
|
||||
{} as AgentProviderModelsQueryDto,
|
||||
),
|
||||
).rejects.toThrow(BadRequestError);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
import type { AgentCatalogModel, AgentProviderModelsResponse } from '@n8n/api-types';
|
||||
import { Logger } from '@n8n/backend-common';
|
||||
import type { User } from '@n8n/db';
|
||||
import { Service } from '@n8n/di';
|
||||
|
||||
import { isModelDiscoveryProvider } from '@n8n/ai-utilities/model-discovery';
|
||||
|
||||
import { BuilderModelLiveLookupService } from './builder/builder-model-live-lookup.service';
|
||||
import { LLM_PROVIDER_DEFAULTS } from './builder/interactive/llm-provider-defaults';
|
||||
|
||||
/** Google's models API returns ids as `models/<id>`; the AI SDK expects the bare id. */
|
||||
const GOOGLE_MODEL_ID_PREFIX = 'models/';
|
||||
|
||||
function getProviderCredentialType(provider: string): string | undefined {
|
||||
if (!isModelDiscoveryProvider(provider)) return undefined;
|
||||
for (const [credentialType, entry] of Object.entries(LLM_PROVIDER_DEFAULTS)) {
|
||||
if (entry.provider === provider) return credentialType;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function normalizeLiveModelValue(provider: string, value: string): string {
|
||||
if (provider === 'google' && value.startsWith(GOOGLE_MODEL_ID_PREFIX)) {
|
||||
return value.slice(GOOGLE_MODEL_ID_PREFIX.length);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/** Dated snapshot suffixes: Anthropic `-20251001`, OpenAI `-2024-08-06`. */
|
||||
const SNAPSHOT_SUFFIX = /-(?:\d{8}|\d{4}-\d{2}-\d{2})$/;
|
||||
|
||||
/**
|
||||
* The ids a live model verifies. Providers list older models only as dated
|
||||
* snapshots (e.g. `claude-haiku-4-5-20251001`) while the catalog prefers the
|
||||
* versionless alias (`claude-haiku-4-5`, which providers resolve to the latest
|
||||
* snapshot) — so a listed snapshot also verifies its alias. A retired alias
|
||||
* still prunes: retired models have no live snapshot either.
|
||||
*/
|
||||
function liveModelIdVariants(id: string): string[] {
|
||||
const alias = id.replace(SNAPSHOT_SUFFIX, '');
|
||||
return alias === id ? [id] : [id, alias];
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the model list offered in the agent model picker for one provider.
|
||||
*
|
||||
* The curated models.dev catalog is the display list — it is the up-to-date set
|
||||
* of chat models with names, cost, and limits. But it lags provider
|
||||
* retirements, so a listed model can 404 at call time. When a credential is
|
||||
* available we verify the catalog against the provider's own model API (via the
|
||||
* shared `@n8n/ai-utilities/model-discovery` functions) and prune any catalog
|
||||
* entry the provider no longer reports. We never add live-only
|
||||
* models: provider `/models` endpoints return every variant/snapshot and would
|
||||
* overload the picker. Without a credential, or when the provider has no list
|
||||
* API wired up, the catalog list is returned unpruned with `verified: false`.
|
||||
*/
|
||||
@Service()
|
||||
export class AgentModelCatalogService {
|
||||
constructor(
|
||||
private readonly logger: Logger,
|
||||
private readonly builderModelLiveLookupService: BuilderModelLiveLookupService,
|
||||
) {}
|
||||
|
||||
async getProviderModels(
|
||||
user: User,
|
||||
projectId: string,
|
||||
provider: string,
|
||||
credentialId?: string,
|
||||
): Promise<AgentProviderModelsResponse> {
|
||||
const catalogModels = await this.getCatalogModels(provider);
|
||||
const credentialType = getProviderCredentialType(provider);
|
||||
|
||||
if (!credentialId || !credentialType) {
|
||||
return { provider, verified: false, models: Object.values(catalogModels) };
|
||||
}
|
||||
|
||||
let liveModels: Array<{ name: string; value: string }>;
|
||||
try {
|
||||
liveModels = await this.builderModelLiveLookupService.list(
|
||||
user,
|
||||
projectId,
|
||||
credentialId,
|
||||
credentialType,
|
||||
provider,
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.warn('Live model list failed — falling back to the static catalog', {
|
||||
provider,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return { provider, verified: false, models: Object.values(catalogModels) };
|
||||
}
|
||||
|
||||
const liveModelIds = new Set(
|
||||
liveModels.flatMap((live) =>
|
||||
liveModelIdVariants(normalizeLiveModelValue(provider, live.value)),
|
||||
),
|
||||
);
|
||||
const catalogList = Object.values(catalogModels);
|
||||
|
||||
// models.dev is the curated display list; the live lookup only verifies it.
|
||||
// Provider `/models` endpoints return every variant/snapshot, so we never
|
||||
// add live-only models — we only prune catalog entries the provider no
|
||||
// longer reports (retired ids that would 404 at call time).
|
||||
if (catalogList.length > 0) {
|
||||
return {
|
||||
provider,
|
||||
verified: true,
|
||||
models: catalogList.filter((model) => liveModelIds.has(model.id)),
|
||||
};
|
||||
}
|
||||
|
||||
// Catalog unavailable (models.dev down or no entry for this provider): there
|
||||
// is no curated list to prune against, so show the verified live list rather
|
||||
// than an empty picker.
|
||||
return {
|
||||
provider,
|
||||
verified: true,
|
||||
models: liveModels.map((live) => {
|
||||
const id = normalizeLiveModelValue(provider, live.value);
|
||||
return {
|
||||
id,
|
||||
name: normalizeLiveModelValue(provider, live.name) || id,
|
||||
reasoning: false,
|
||||
toolCall: true,
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
private async getCatalogModels(provider: string): Promise<Record<string, AgentCatalogModel>> {
|
||||
try {
|
||||
const { fetchProviderCatalog } = await import('@n8n/agents');
|
||||
const catalog = await fetchProviderCatalog();
|
||||
return catalog[provider]?.models ?? {};
|
||||
} catch (error) {
|
||||
this.logger.warn('Model catalog fetch failed', {
|
||||
provider,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return {};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,24 @@
|
||||
import type { ChatIntegrationDescriptor } from '@n8n/api-types';
|
||||
import { Get, ProjectScope, RestController } from '@n8n/decorators';
|
||||
import {
|
||||
AgentProviderModelsQueryDto,
|
||||
isAgentModelProvider,
|
||||
type AgentProviderModelsResponse,
|
||||
type ChatIntegrationDescriptor,
|
||||
} from '@n8n/api-types';
|
||||
import type { AuthenticatedRequest } from '@n8n/db';
|
||||
import { Get, Param, ProjectScope, Query, RestController } from '@n8n/decorators';
|
||||
import type { Response } from 'express';
|
||||
|
||||
import { BadRequestError } from '@/errors/response-errors/bad-request.error';
|
||||
|
||||
import { AgentIntegrationPersistenceService } from './agent-integration-persistence.service';
|
||||
import { AgentModelCatalogService } from './agent-model-catalog.service';
|
||||
import { filterOfferedAgentModelProviders } from './model-catalog';
|
||||
|
||||
@RestController('/projects/:projectId/agents/v2')
|
||||
export class AgentsCatalogController {
|
||||
constructor(
|
||||
private readonly agentIntegrationPersistenceService: AgentIntegrationPersistenceService,
|
||||
private readonly agentModelCatalogService: AgentModelCatalogService,
|
||||
) {}
|
||||
|
||||
@Get('/catalog/models')
|
||||
@@ -17,6 +28,25 @@ export class AgentsCatalogController {
|
||||
return filterOfferedAgentModelProviders(await fetchProviderCatalog());
|
||||
}
|
||||
|
||||
@Get('/catalog/models/:provider')
|
||||
@ProjectScope('agent:read')
|
||||
async getProviderModels(
|
||||
req: AuthenticatedRequest<{ projectId: string }>,
|
||||
_res: Response,
|
||||
@Param('provider') provider: string,
|
||||
@Query query: AgentProviderModelsQueryDto,
|
||||
): Promise<AgentProviderModelsResponse> {
|
||||
if (!isAgentModelProvider(provider)) {
|
||||
throw new BadRequestError(`Unknown model provider "${provider}"`);
|
||||
}
|
||||
return await this.agentModelCatalogService.getProviderModels(
|
||||
req.user,
|
||||
req.params.projectId,
|
||||
provider,
|
||||
query.credentialId,
|
||||
);
|
||||
}
|
||||
|
||||
@Get('/catalog/integrations')
|
||||
@ProjectScope('agent:read')
|
||||
listIntegrations(): ChatIntegrationDescriptor[] {
|
||||
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
import type { CustomFetch, HttpTransport, OutboundHttp } from '@n8n/backend-network';
|
||||
import type { CredentialsEntity, User } from '@n8n/db';
|
||||
import { mock } from 'vitest-mock-extended';
|
||||
|
||||
import type { CredentialsFinderService } from '@/credentials/credentials-finder.service';
|
||||
import type { CredentialsService } from '@/credentials/credentials.service';
|
||||
|
||||
import { BuilderModelLiveLookupService } from '../builder-model-live-lookup.service';
|
||||
|
||||
const listModelsForProvider = vi.fn();
|
||||
vi.mock('@n8n/ai-utilities/model-discovery', () => ({
|
||||
listModelsForProvider: (...args: unknown[]) => listModelsForProvider(...args) as unknown,
|
||||
}));
|
||||
|
||||
const user = mock<User>({ id: 'user-1' });
|
||||
const projectId = 'project-1';
|
||||
|
||||
function makeService() {
|
||||
const credentialsService = mock<CredentialsService>();
|
||||
const credentialsFinderService = mock<CredentialsFinderService>();
|
||||
const transport = mock<HttpTransport>();
|
||||
transport.asCustomFetch.mockReturnValue(vi.fn() as unknown as CustomFetch);
|
||||
const outboundHttp = mock<OutboundHttp>();
|
||||
outboundHttp.transport.mockReturnValue(transport);
|
||||
|
||||
const service = new BuilderModelLiveLookupService(
|
||||
credentialsService,
|
||||
credentialsFinderService,
|
||||
outboundHttp,
|
||||
);
|
||||
return { service, credentialsService, credentialsFinderService };
|
||||
}
|
||||
|
||||
function usable(id: string, type: string) {
|
||||
return [{ id, name: 'My Credential', type }] as Awaited<
|
||||
ReturnType<CredentialsService['getCredentialsAUserCanUseInAWorkflow']>
|
||||
>;
|
||||
}
|
||||
|
||||
describe('BuilderModelLiveLookupService', () => {
|
||||
beforeEach(() => {
|
||||
listModelsForProvider.mockReset();
|
||||
});
|
||||
|
||||
it('lists models for a credential the user can use in the project', async () => {
|
||||
const { service, credentialsService, credentialsFinderService } = makeService();
|
||||
credentialsService.getCredentialsAUserCanUseInAWorkflow.mockResolvedValue(
|
||||
usable('cred-1', 'anthropicApi'),
|
||||
);
|
||||
credentialsFinderService.findCredentialById.mockResolvedValue(mock<CredentialsEntity>());
|
||||
credentialsService.decrypt.mockResolvedValue({ apiKey: 'sk-key', url: 'https://proxy.local' });
|
||||
listModelsForProvider.mockResolvedValue([
|
||||
{ id: 'claude-sonnet-4-6', name: 'Claude Sonnet 4.6' },
|
||||
]);
|
||||
|
||||
const result = await service.list(user, projectId, 'cred-1', 'anthropicApi', 'anthropic');
|
||||
|
||||
expect(result).toEqual([{ name: 'Claude Sonnet 4.6', value: 'claude-sonnet-4-6' }]);
|
||||
expect(credentialsService.getCredentialsAUserCanUseInAWorkflow).toHaveBeenCalledWith(user, {
|
||||
projectId,
|
||||
});
|
||||
// Credential fields are mapped for the provider (anthropic: apiKey + url→baseURL).
|
||||
expect(listModelsForProvider).toHaveBeenCalledWith(
|
||||
'anthropic',
|
||||
expect.objectContaining({ apiKey: 'sk-key', baseURL: 'https://proxy.local' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('treats an empty provider response as a failed lookup', async () => {
|
||||
const { service, credentialsService, credentialsFinderService } = makeService();
|
||||
credentialsService.getCredentialsAUserCanUseInAWorkflow.mockResolvedValue(
|
||||
usable('cred-1', 'anthropicApi'),
|
||||
);
|
||||
credentialsFinderService.findCredentialById.mockResolvedValue(mock<CredentialsEntity>());
|
||||
credentialsService.decrypt.mockResolvedValue({ apiKey: 'sk-key' });
|
||||
// An empty list from a chat provider is far more likely a broken request
|
||||
// or drifted response shape than a real zero-model account — callers must
|
||||
// fall back rather than prune everything.
|
||||
listModelsForProvider.mockResolvedValue([]);
|
||||
|
||||
await expect(
|
||||
service.list(user, projectId, 'cred-1', 'anthropicApi', 'anthropic'),
|
||||
).rejects.toThrow('returned no models');
|
||||
});
|
||||
|
||||
it('rejects a credential that is not available in the project', async () => {
|
||||
const { service, credentialsService } = makeService();
|
||||
// The user can read this credential, but it is not in the project's set.
|
||||
credentialsService.getCredentialsAUserCanUseInAWorkflow.mockResolvedValue([]);
|
||||
|
||||
await expect(
|
||||
service.list(user, projectId, 'cred-other-project', 'anthropicApi', 'anthropic'),
|
||||
).rejects.toThrow('not found or not accessible');
|
||||
expect(listModelsForProvider).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects a credential whose type does not match the provider', async () => {
|
||||
const { service, credentialsService } = makeService();
|
||||
credentialsService.getCredentialsAUserCanUseInAWorkflow.mockResolvedValue(
|
||||
usable('cred-1', 'openAiApi'),
|
||||
);
|
||||
|
||||
await expect(
|
||||
service.list(user, projectId, 'cred-1', 'anthropicApi', 'anthropic'),
|
||||
).rejects.toThrow('not found or not accessible');
|
||||
expect(listModelsForProvider).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -43,7 +43,7 @@ import { AgentSkillsService } from '../agent-skills.service';
|
||||
import { AgentTaskService } from '../agent-task.service';
|
||||
import { AgentsToolsService } from '../agents-tools.service';
|
||||
import { AgentsService } from '../agents.service';
|
||||
import { BuilderModelLookupService } from './builder-model-lookup.service';
|
||||
import { BuilderModelLiveLookupService } from './builder-model-live-lookup.service';
|
||||
import { BUILDER_TOOLS } from './builder-tool-names';
|
||||
import {
|
||||
collectFromAiParameterReferences,
|
||||
@@ -311,7 +311,7 @@ export class AgentsBuilderToolsService {
|
||||
private readonly secureRuntime: AgentSecureRuntime,
|
||||
private readonly workflowRepository: WorkflowRepository,
|
||||
private readonly agentsToolsService: AgentsToolsService,
|
||||
private readonly builderModelLookupService: BuilderModelLookupService,
|
||||
private readonly builderModelLiveLookupService: BuilderModelLiveLookupService,
|
||||
private readonly mcpRegistryService: McpRegistryService,
|
||||
private readonly oauthService: OauthService,
|
||||
private readonly credentialTypes: CredentialTypes,
|
||||
@@ -671,8 +671,14 @@ export class AgentsBuilderToolsService {
|
||||
.build();
|
||||
|
||||
const modelLookup: ModelLookup = {
|
||||
list: async (credentialId, credentialType, lookup) =>
|
||||
await this.builderModelLookupService.list(user, credentialId, credentialType, lookup),
|
||||
list: async (credentialId, credentialType, provider) =>
|
||||
await this.builderModelLiveLookupService.list(
|
||||
user,
|
||||
projectId,
|
||||
credentialId,
|
||||
credentialType,
|
||||
provider,
|
||||
),
|
||||
};
|
||||
|
||||
const tools: BuiltTool[] = [
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import { OutboundHttp } from '@n8n/backend-network';
|
||||
import type { User } from '@n8n/db';
|
||||
import { Service } from '@n8n/di';
|
||||
|
||||
import { CredentialsFinderService } from '@/credentials/credentials-finder.service';
|
||||
import { CredentialsService } from '@/credentials/credentials.service';
|
||||
import { createAiProxyFetch } from '@/utils/ai-proxy-fetch';
|
||||
|
||||
import { mapCredentialForProvider } from '../json-config/credential-field-mapping';
|
||||
|
||||
/**
|
||||
* Fetches a provider's live chat-model list for a credential, via the shared
|
||||
* `@n8n/ai-utilities/model-discovery` functions (the same provider knowledge
|
||||
* that backs the chat sub-nodes' model dropdowns). Nothing from
|
||||
* `@n8n/n8n-nodes-langchain` is loaded on this path.
|
||||
*
|
||||
* The credential must be usable by the user within the given project — the
|
||||
* same set as the workflow editor's credential picker.
|
||||
*/
|
||||
@Service()
|
||||
export class BuilderModelLiveLookupService {
|
||||
constructor(
|
||||
private readonly credentialsService: CredentialsService,
|
||||
private readonly credentialsFinderService: CredentialsFinderService,
|
||||
private readonly outboundHttp: OutboundHttp,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Returns `{ name, value }` pairs (value = the provider's model id, exactly
|
||||
* as the provider API expects it). Throws if the credential is not usable by
|
||||
* the user in the project, its type doesn't match, or the provider has no
|
||||
* model discovery support.
|
||||
*/
|
||||
async list(
|
||||
user: User,
|
||||
projectId: string,
|
||||
credentialId: string,
|
||||
credentialType: string,
|
||||
provider: string,
|
||||
): Promise<Array<{ name: string; value: string }>> {
|
||||
const usableCredentials = await this.credentialsService.getCredentialsAUserCanUseInAWorkflow(
|
||||
user,
|
||||
{ projectId },
|
||||
);
|
||||
const usable = usableCredentials.find((c) => c.id === credentialId);
|
||||
if (!usable || usable.type !== credentialType) {
|
||||
throw new Error(`Credential ${credentialId} not found or not accessible`);
|
||||
}
|
||||
|
||||
const credential = await this.credentialsFinderService.findCredentialById(credentialId);
|
||||
if (!credential) {
|
||||
throw new Error(`Credential ${credentialId} not found or not accessible`);
|
||||
}
|
||||
const rawData = await this.credentialsService.decrypt(credential, true);
|
||||
|
||||
const { listModelsForProvider } = await import('@n8n/ai-utilities/model-discovery');
|
||||
const mapped = mapCredentialForProvider(provider, { apiKey: '', ...rawData });
|
||||
const apiKey = typeof mapped.apiKey === 'string' ? mapped.apiKey : '';
|
||||
const baseURL =
|
||||
typeof mapped.baseURL === 'string' && mapped.baseURL ? mapped.baseURL : undefined;
|
||||
|
||||
const models = await listModelsForProvider(provider, {
|
||||
apiKey,
|
||||
baseURL,
|
||||
fetch: createAiProxyFetch(this.outboundHttp) as typeof globalThis.fetch,
|
||||
});
|
||||
|
||||
// Every supported chat provider offers models, so an empty list means a
|
||||
// broken request or a drifted response shape, not a zero-model account.
|
||||
// Throw so callers fall back (unverified catalog / lookup-failed) instead
|
||||
// of treating "nothing" as a verified answer and pruning every model.
|
||||
if (models.length === 0) {
|
||||
throw new Error(`Provider ${provider} returned no models`);
|
||||
}
|
||||
|
||||
return models.map((model) => ({ name: model.name, value: model.id }));
|
||||
}
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
import type { User } from '@n8n/db';
|
||||
import { ProjectRepository } from '@n8n/db';
|
||||
import { Service } from '@n8n/di';
|
||||
import type { INodeCredentials, INodeParameters } from 'n8n-workflow';
|
||||
|
||||
import { CredentialsFinderService } from '@/credentials/credentials-finder.service';
|
||||
import { NodeTypes } from '@/node-types';
|
||||
import { DynamicNodeParametersService } from '@/services/dynamic-node-parameters.service';
|
||||
import { getBase } from '@/workflow-execute-additional-data';
|
||||
|
||||
import type { ModelLookupConfig } from './interactive/llm-provider-defaults';
|
||||
|
||||
@Service()
|
||||
export class BuilderModelLookupService {
|
||||
constructor(
|
||||
private readonly credentialsFinderService: CredentialsFinderService,
|
||||
private readonly projectRepository: ProjectRepository,
|
||||
private readonly dynamicNodeParametersService: DynamicNodeParametersService,
|
||||
private readonly nodeTypes: NodeTypes,
|
||||
) {}
|
||||
|
||||
async list(
|
||||
user: User,
|
||||
credentialId: string,
|
||||
credentialType: string,
|
||||
lookup: ModelLookupConfig,
|
||||
): Promise<Array<{ name: string; value: string }>> {
|
||||
const credential = await this.credentialsFinderService.findCredentialForUser(
|
||||
credentialId,
|
||||
user,
|
||||
['credential:read'],
|
||||
);
|
||||
if (!credential || credential.type !== credentialType) {
|
||||
throw new Error(`Credential ${credentialId} not found or not accessible`);
|
||||
}
|
||||
|
||||
const personalProject = await this.projectRepository.getPersonalProjectForUserOrFail(user.id);
|
||||
const currentNodeParameters: INodeParameters = {};
|
||||
const credentials: INodeCredentials = {
|
||||
[credential.type]: { id: credential.id, name: credential.name },
|
||||
};
|
||||
const additionalData = await getBase({
|
||||
userId: user.id,
|
||||
projectId: personalProject.id,
|
||||
currentNodeParameters,
|
||||
});
|
||||
const nodeTypeAndVersion = { name: lookup.nodeType, version: lookup.version };
|
||||
|
||||
if (lookup.kind === 'listSearch') {
|
||||
const result = await this.dynamicNodeParametersService.getResourceLocatorResults(
|
||||
lookup.methodName,
|
||||
'',
|
||||
additionalData,
|
||||
nodeTypeAndVersion,
|
||||
currentNodeParameters,
|
||||
credentials,
|
||||
);
|
||||
return (result.results ?? []).map((r) => ({
|
||||
name: String(r.name),
|
||||
value: String(r.value),
|
||||
}));
|
||||
}
|
||||
|
||||
const nodeType = this.nodeTypes.getByNameAndVersion(lookup.nodeType, lookup.version);
|
||||
const property = nodeType.description.properties.find((p) => p.name === lookup.propertyName);
|
||||
const loadOptions = property?.typeOptions?.loadOptions;
|
||||
if (!loadOptions) {
|
||||
throw new Error(
|
||||
`Property "${lookup.propertyName}" on ${lookup.nodeType} has no loadOptions config`,
|
||||
);
|
||||
}
|
||||
|
||||
const options = await this.dynamicNodeParametersService.getOptionsViaLoadOptions(
|
||||
loadOptions,
|
||||
additionalData,
|
||||
nodeTypeAndVersion,
|
||||
currentNodeParameters,
|
||||
credentials,
|
||||
);
|
||||
return options.map((o) => ({ name: String(o.name), value: String(o.value) }));
|
||||
}
|
||||
}
|
||||
+13
-11
@@ -57,7 +57,10 @@ describe('resolve_llm tool', () => {
|
||||
|
||||
it('uses the requested model for the requested provider', async () => {
|
||||
const credentialProvider = makeProvider([{ id: 'c1', name: 'My xAI', type: 'xAiApi' }]);
|
||||
const modelLookup = makeModelLookup();
|
||||
const modelLookup = makeModelLookup(async () => [
|
||||
{ name: 'Grok 4 Fast', value: 'grok-4-fast' },
|
||||
{ name: 'Grok 4', value: 'grok-4' },
|
||||
]);
|
||||
const tool = buildResolveLlmTool({ credentialProvider, modelLookup });
|
||||
const result = await tool.handler!({ provider: 'xai', model: 'grok-4-fast' }, {});
|
||||
|
||||
@@ -146,20 +149,23 @@ describe('resolve_llm tool', () => {
|
||||
expect(modelLookup.list).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('skips lookup for providers without modelLookup configured (e.g. Cohere)', async () => {
|
||||
it('validates the requested model against the lookup for Cohere', async () => {
|
||||
const credentialProvider = makeProvider([{ id: 'c1', name: 'My Cohere', type: 'cohereApi' }]);
|
||||
const modelLookup = makeModelLookup();
|
||||
const modelLookup = makeModelLookup(async () => [
|
||||
{ name: 'Command R+', value: 'command-r-plus' },
|
||||
{ name: 'Command R', value: 'command-r' },
|
||||
]);
|
||||
const tool = buildResolveLlmTool({ credentialProvider, modelLookup });
|
||||
const result = await tool.handler!({ provider: 'cohere', model: 'command-x-plus' }, {});
|
||||
const result = await tool.handler!({ provider: 'cohere', model: 'command-r-plus' }, {});
|
||||
|
||||
expect(result).toEqual({
|
||||
ok: true,
|
||||
provider: 'cohere',
|
||||
model: 'command-x-plus',
|
||||
model: 'command-r-plus',
|
||||
credentialId: 'c1',
|
||||
credentialName: 'My Cohere',
|
||||
});
|
||||
expect(modelLookup.list).not.toHaveBeenCalled();
|
||||
expect(modelLookup.list).toHaveBeenCalledWith('c1', 'cohereApi', 'cohere');
|
||||
});
|
||||
|
||||
it('returns the canonical model id when the requested model matches the lookup', async () => {
|
||||
@@ -183,11 +189,7 @@ describe('resolve_llm tool', () => {
|
||||
credentialId: 'c1',
|
||||
credentialName: 'My Anthropic',
|
||||
});
|
||||
expect(modelLookup.list).toHaveBeenCalledWith(
|
||||
'c1',
|
||||
'anthropicApi',
|
||||
expect.objectContaining({ kind: 'listSearch', methodName: 'searchModels' }),
|
||||
);
|
||||
expect(modelLookup.list).toHaveBeenCalledWith('c1', 'anthropicApi', 'anthropic');
|
||||
});
|
||||
|
||||
it('uniquely-substring-matches a partial requested model id', async () => {
|
||||
|
||||
@@ -10,95 +10,58 @@
|
||||
* Azure variants), omit the entry so the tool falls through to suspending and
|
||||
* lets the user pick explicitly.
|
||||
*
|
||||
* `modelLookup` (optional) points at the chat-model node whose
|
||||
* `@searchListMethod` or routing-based `loadOptions` returns the live list of
|
||||
* model ids for the provider. When set, `resolve_llm` validates user-requested
|
||||
* model strings against that list. When absent, the requested model is passed
|
||||
* through unchanged.
|
||||
* Live model validation is available for providers supported by
|
||||
* `@n8n/ai-utilities/model-discovery` — `resolve_llm` checks user-requested
|
||||
* model strings against the provider's live model list for those.
|
||||
*/
|
||||
export type ModelLookupConfig =
|
||||
| {
|
||||
kind: 'listSearch';
|
||||
nodeType: string;
|
||||
version: number;
|
||||
methodName: string;
|
||||
}
|
||||
| {
|
||||
kind: 'loadOptionsRouting';
|
||||
nodeType: string;
|
||||
version: number;
|
||||
propertyName: string;
|
||||
};
|
||||
|
||||
export interface LlmProviderDefault {
|
||||
provider: string;
|
||||
defaultModel: string;
|
||||
modelLookup?: ModelLookupConfig;
|
||||
}
|
||||
|
||||
export const LLM_PROVIDER_DEFAULTS: Record<string, LlmProviderDefault> = {
|
||||
anthropicApi: {
|
||||
provider: 'anthropic',
|
||||
defaultModel: 'claude-sonnet-4-6',
|
||||
modelLookup: {
|
||||
kind: 'listSearch',
|
||||
nodeType: '@n8n/n8n-nodes-langchain.lmChatAnthropic',
|
||||
version: 1.5,
|
||||
methodName: 'searchModels',
|
||||
},
|
||||
},
|
||||
openAiApi: {
|
||||
provider: 'openai',
|
||||
defaultModel: 'gpt-5-mini',
|
||||
modelLookup: {
|
||||
kind: 'listSearch',
|
||||
nodeType: '@n8n/n8n-nodes-langchain.lmChatOpenAi',
|
||||
version: 1.2,
|
||||
methodName: 'searchModels',
|
||||
},
|
||||
},
|
||||
googlePalmApi: {
|
||||
provider: 'google',
|
||||
defaultModel: 'gemini-2.5-pro',
|
||||
modelLookup: {
|
||||
kind: 'loadOptionsRouting',
|
||||
nodeType: '@n8n/n8n-nodes-langchain.lmChatGoogleGemini',
|
||||
version: 1.1,
|
||||
propertyName: 'modelName',
|
||||
},
|
||||
},
|
||||
xAiApi: { provider: 'xai', defaultModel: 'grok-4' },
|
||||
groqApi: { provider: 'groq', defaultModel: 'llama-3.1-70b-versatile' },
|
||||
xAiApi: {
|
||||
provider: 'xai',
|
||||
defaultModel: 'grok-4',
|
||||
},
|
||||
groqApi: {
|
||||
provider: 'groq',
|
||||
defaultModel: 'llama-3.3-70b-versatile',
|
||||
},
|
||||
mistralCloudApi: {
|
||||
provider: 'mistral',
|
||||
defaultModel: 'mistral-large-latest',
|
||||
modelLookup: {
|
||||
kind: 'loadOptionsRouting',
|
||||
nodeType: '@n8n/n8n-nodes-langchain.lmChatMistralCloud',
|
||||
version: 1,
|
||||
propertyName: 'model',
|
||||
},
|
||||
},
|
||||
deepSeekApi: { provider: 'deepseek', defaultModel: 'deepseek-chat' },
|
||||
cohereApi: { provider: 'cohere', defaultModel: 'command-r-plus' },
|
||||
deepSeekApi: {
|
||||
provider: 'deepseek',
|
||||
defaultModel: 'deepseek-chat',
|
||||
},
|
||||
cohereApi: {
|
||||
provider: 'cohere',
|
||||
defaultModel: 'command-r-plus',
|
||||
},
|
||||
openRouterApi: {
|
||||
provider: 'openrouter',
|
||||
defaultModel: 'anthropic/claude-sonnet-4.6',
|
||||
modelLookup: {
|
||||
kind: 'loadOptionsRouting',
|
||||
nodeType: '@n8n/n8n-nodes-langchain.lmChatOpenRouter',
|
||||
version: 1,
|
||||
propertyName: 'model',
|
||||
},
|
||||
},
|
||||
nvidiaApi: {
|
||||
provider: 'nvidia',
|
||||
defaultModel: 'nvidia/llama-3.3-nemotron-super-49b-v1',
|
||||
modelLookup: {
|
||||
kind: 'loadOptionsRouting',
|
||||
nodeType: '@n8n/n8n-nodes-langchain.lmChatNvidia',
|
||||
version: 1,
|
||||
propertyName: 'model',
|
||||
},
|
||||
},
|
||||
vercelAiGatewayApi: {
|
||||
provider: 'vercel',
|
||||
defaultModel: 'anthropic/claude-sonnet-4.6',
|
||||
},
|
||||
};
|
||||
|
||||
@@ -1,19 +1,16 @@
|
||||
import type { BuiltTool, CredentialListItem, CredentialProvider } from '@n8n/agents';
|
||||
import { Tool } from '@n8n/agents/tool';
|
||||
import { isModelDiscoveryProvider } from '@n8n/ai-utilities/model-discovery';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { BUILDER_TOOLS } from '../builder-tool-names';
|
||||
import {
|
||||
LLM_PROVIDER_DEFAULTS,
|
||||
type LlmProviderDefault,
|
||||
type ModelLookupConfig,
|
||||
} from './llm-provider-defaults';
|
||||
import { LLM_PROVIDER_DEFAULTS, type LlmProviderDefault } from './llm-provider-defaults';
|
||||
|
||||
export interface ModelLookup {
|
||||
list(
|
||||
credentialId: string,
|
||||
credentialType: string,
|
||||
lookup: ModelLookupConfig,
|
||||
provider: string,
|
||||
): Promise<Array<{ name: string; value: string }>>;
|
||||
}
|
||||
|
||||
@@ -52,13 +49,13 @@ async function resolveModelAgainstLookup(
|
||||
modelLookup: ModelLookup,
|
||||
) {
|
||||
const trimmedModel = requestedModel.trim();
|
||||
if (!defaults.modelLookup || !trimmedModel) {
|
||||
if (!isModelDiscoveryProvider(defaults.provider) || !trimmedModel) {
|
||||
return toLlmResolution(credential, defaults, requestedModel);
|
||||
}
|
||||
|
||||
let availableModels: Array<{ name: string; value: string }>;
|
||||
try {
|
||||
availableModels = await modelLookup.list(credential.id, credential.type, defaults.modelLookup);
|
||||
availableModels = await modelLookup.list(credential.id, credential.type, defaults.provider);
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false as const,
|
||||
|
||||
@@ -111,10 +111,11 @@ describe('AgentInfoPanel', () => {
|
||||
variant: 'ghost',
|
||||
showToolbar: 'never',
|
||||
maxHeight: 'none',
|
||||
placeholder: 'Enter instructions here',
|
||||
});
|
||||
expect(editor.props('placeholder')).toBeUndefined();
|
||||
expect(wrapper.find('[data-testid="agent-instructions-document"]').exists()).toBe(true);
|
||||
expect(wrapper.text()).not.toContain('characters');
|
||||
expect(wrapper.text()).not.toContain('Enter instructions here');
|
||||
});
|
||||
|
||||
it('can show the markdown toolbar above instructions', () => {
|
||||
@@ -127,11 +128,12 @@ describe('AgentInfoPanel', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps the instructions placeholder available when instructions are empty', () => {
|
||||
it('does not pass placeholder text to the instructions editor', () => {
|
||||
const wrapper = mountPanel('');
|
||||
|
||||
const editor = wrapper.findComponent({ name: 'N8nMarkdownEditor' });
|
||||
expect(editor.props('modelValue')).toBe('');
|
||||
expect(editor.props('placeholder')).toBe('Enter instructions here');
|
||||
expect(editor.props('placeholder')).toBeUndefined();
|
||||
expect(wrapper.text()).not.toContain('Enter instructions here');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { AgentModelOption } from '../model-providers';
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
getModelCatalog: vi.fn(),
|
||||
getProviderModels: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('@n8n/stores/useRootStore', () => ({
|
||||
@@ -13,6 +14,7 @@ vi.mock('@n8n/stores/useRootStore', () => ({
|
||||
|
||||
vi.mock('../composables/useAgentApi', () => ({
|
||||
getModelCatalog: mocks.getModelCatalog,
|
||||
getProviderModels: mocks.getProviderModels,
|
||||
}));
|
||||
|
||||
function model(id: string, name = id) {
|
||||
@@ -36,10 +38,17 @@ function modelIds(models: AgentModelOption[]) {
|
||||
return models.map((entry) => entry.model);
|
||||
}
|
||||
|
||||
async function flushAsync() {
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
describe('useModelCatalog', () => {
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
mocks.getModelCatalog.mockReset();
|
||||
mocks.getProviderModels.mockReset();
|
||||
// Default: no live verification available — fall back to the catalog.
|
||||
mocks.getProviderModels.mockRejectedValue(new Error('not available'));
|
||||
});
|
||||
|
||||
it('returns models for the agent providers that have selected credentials', async () => {
|
||||
@@ -91,4 +100,95 @@ describe('useModelCatalog', () => {
|
||||
]);
|
||||
expect(result.anthropic?.models).toEqual([]);
|
||||
});
|
||||
|
||||
it('prefers the provider-verified model list over the static catalog', async () => {
|
||||
mocks.getModelCatalog.mockResolvedValue({
|
||||
anthropic: provider('anthropic', {
|
||||
'claude-sonnet-4-6': model('claude-sonnet-4-6', 'Claude Sonnet 4.6'),
|
||||
'claude-opus-4-0': model('claude-opus-4-0', 'Claude Opus 4'),
|
||||
}),
|
||||
});
|
||||
mocks.getProviderModels.mockResolvedValue({
|
||||
provider: 'anthropic',
|
||||
verified: true,
|
||||
models: [model('claude-sonnet-4-6', 'Claude Sonnet 4.6')],
|
||||
});
|
||||
|
||||
const { useModelCatalog } = await import('../composables/useModelCatalog');
|
||||
const { ensureLoaded, getModelsForPicker } = useModelCatalog();
|
||||
await ensureLoaded('project-1');
|
||||
const credentials = { anthropic: 'anthropic-credential-id' };
|
||||
|
||||
// First read triggers the verification fetch; once it lands, only the
|
||||
// provider-confirmed models remain.
|
||||
getModelsForPicker(credentials);
|
||||
await flushAsync();
|
||||
|
||||
expect(modelIds(getModelsForPicker(credentials).anthropic?.models ?? [])).toEqual([
|
||||
'claude-sonnet-4-6',
|
||||
]);
|
||||
expect(mocks.getProviderModels).toHaveBeenCalledWith(
|
||||
{},
|
||||
'project-1',
|
||||
'anthropic',
|
||||
'anthropic-credential-id',
|
||||
);
|
||||
});
|
||||
|
||||
it('fetches the verified list only once per provider and credential', async () => {
|
||||
mocks.getModelCatalog.mockResolvedValue({
|
||||
anthropic: provider('anthropic', {
|
||||
'claude-sonnet-4-6': model('claude-sonnet-4-6'),
|
||||
}),
|
||||
});
|
||||
mocks.getProviderModels.mockResolvedValue({
|
||||
provider: 'anthropic',
|
||||
verified: true,
|
||||
models: [model('claude-sonnet-4-6')],
|
||||
});
|
||||
|
||||
const { useModelCatalog } = await import('../composables/useModelCatalog');
|
||||
const { ensureLoaded, getModelsForPicker } = useModelCatalog();
|
||||
await ensureLoaded('project-1');
|
||||
const credentials = { anthropic: 'anthropic-credential-id' };
|
||||
|
||||
getModelsForPicker(credentials);
|
||||
getModelsForPicker(credentials);
|
||||
await flushAsync();
|
||||
getModelsForPicker(credentials);
|
||||
await flushAsync();
|
||||
|
||||
expect(mocks.getProviderModels).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('falls back to the catalog list when verification fails or is unverified', async () => {
|
||||
mocks.getModelCatalog.mockResolvedValue({
|
||||
anthropic: provider('anthropic', {
|
||||
'claude-sonnet-4-6': model('claude-sonnet-4-6'),
|
||||
'claude-opus-4-0': model('claude-opus-4-0'),
|
||||
}),
|
||||
openai: provider('openai', {
|
||||
'gpt-5': model('gpt-5'),
|
||||
}),
|
||||
});
|
||||
mocks.getProviderModels.mockImplementation(async (_ctx, _projectId, providerId) => {
|
||||
if (providerId === 'anthropic') throw new Error('provider is down');
|
||||
return { provider: providerId, verified: false, models: [model('gpt-5')] };
|
||||
});
|
||||
|
||||
const { useModelCatalog } = await import('../composables/useModelCatalog');
|
||||
const { ensureLoaded, getModelsForPicker } = useModelCatalog();
|
||||
await ensureLoaded('project-1');
|
||||
const credentials = { anthropic: 'anthropic-cred', openai: 'openai-cred' };
|
||||
|
||||
getModelsForPicker(credentials);
|
||||
await flushAsync();
|
||||
|
||||
const result = getModelsForPicker(credentials);
|
||||
expect(modelIds(result.anthropic?.models ?? []).sort()).toEqual([
|
||||
'claude-opus-4-0',
|
||||
'claude-sonnet-4-6',
|
||||
]);
|
||||
expect(modelIds(result.openai?.models ?? [])).toEqual(['gpt-5']);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -201,7 +201,6 @@ function onInstructionsInput(value: string) {
|
||||
:class="$style.instructionsDocument"
|
||||
:model-value="instructions"
|
||||
:readonly="props.disabled"
|
||||
:placeholder="i18n.baseText('agents.builder.agent.instructions.placeholder')"
|
||||
:variant="instructionsEditorVariant"
|
||||
:show-toolbar="instructionsToolbarMode"
|
||||
max-height="none"
|
||||
|
||||
@@ -8,6 +8,7 @@ import type {
|
||||
AgentTaskConfig,
|
||||
AgentTaskDto,
|
||||
AgentIntegrationSettings,
|
||||
AgentProviderModelsResponse,
|
||||
AgentVersionListItemDto,
|
||||
ChatIntegrationDescriptor,
|
||||
CreateSlackAgentAppResponse,
|
||||
@@ -346,6 +347,20 @@ export const getModelCatalog = async (
|
||||
);
|
||||
};
|
||||
|
||||
export const getProviderModels = async (
|
||||
context: IRestApiContext,
|
||||
projectId: string,
|
||||
provider: string,
|
||||
credentialId?: string,
|
||||
): Promise<AgentProviderModelsResponse> => {
|
||||
return await makeRestApiRequest<AgentProviderModelsResponse>(
|
||||
context,
|
||||
'GET',
|
||||
`/projects/${projectId}/agents/v2/catalog/models/${provider}`,
|
||||
credentialId ? { credentialId } : undefined,
|
||||
);
|
||||
};
|
||||
|
||||
export const publishAgent = async (
|
||||
context: IRestApiContext,
|
||||
projectId: string,
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
import { computed, ref } from 'vue';
|
||||
import { useRootStore } from '@n8n/stores/useRootStore';
|
||||
import { getModelCatalog, type ProviderCatalog, type ModelInfo } from './useAgentApi';
|
||||
import {
|
||||
getModelCatalog,
|
||||
getProviderModels,
|
||||
type ProviderCatalog,
|
||||
type ModelInfo,
|
||||
} from './useAgentApi';
|
||||
import {
|
||||
AGENT_MODEL_PROVIDERS,
|
||||
type AgentCredentialsByProvider,
|
||||
@@ -17,6 +22,14 @@ const catalogByProject = ref<Record<string, ProviderCatalog>>({});
|
||||
const fetchPromises = new Map<string, Promise<void>>();
|
||||
const loadingProjects = ref(new Set<string>());
|
||||
|
||||
// Provider-verified model lists, keyed by project|provider|credential. The
|
||||
// static catalog can offer models the provider has retired (which then fail at
|
||||
// call time), so when a credential is selected the provider's own model API is
|
||||
// asked which models actually work. `null` marks a failed or unverified
|
||||
// lookup — the static catalog is used for that key.
|
||||
const verifiedModelsByKey = ref<Record<string, ModelInfo[] | null>>({});
|
||||
const verifiedFetchesInFlight = new Set<string>();
|
||||
|
||||
function createEmptyModelsResponse(): AgentModelsByProvider {
|
||||
const response: AgentModelsByProvider = {};
|
||||
for (const provider of AGENT_MODEL_PROVIDERS) {
|
||||
@@ -82,19 +95,62 @@ export function useModelCatalog() {
|
||||
return Object.values(p.models).sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
/**
|
||||
* Kick off (once per project+provider+credential) the fetch of the
|
||||
* provider-verified model list. Idempotent and guarded, so it is safe to
|
||||
* trigger from `getModelsForPicker` — when the response lands, the reactive
|
||||
* map updates and computeds re-evaluate with the verified list.
|
||||
*/
|
||||
function ensureVerifiedModels(
|
||||
projectId: string,
|
||||
provider: AgentModelProvider,
|
||||
providerCredentialId: string,
|
||||
): void {
|
||||
const key = `${projectId}|${provider}|${providerCredentialId}`;
|
||||
if (key in verifiedModelsByKey.value || verifiedFetchesInFlight.has(key)) return;
|
||||
|
||||
verifiedFetchesInFlight.add(key);
|
||||
getProviderModels(rootStore.restApiContext, projectId, provider, providerCredentialId)
|
||||
.then((result) => {
|
||||
verifiedModelsByKey.value = {
|
||||
...verifiedModelsByKey.value,
|
||||
[key]: result.verified ? result.models : null,
|
||||
};
|
||||
})
|
||||
.catch(() => {
|
||||
verifiedModelsByKey.value = { ...verifiedModelsByKey.value, [key]: null };
|
||||
})
|
||||
.finally(() => {
|
||||
verifiedFetchesInFlight.delete(key);
|
||||
});
|
||||
}
|
||||
|
||||
function getModelsForPicker(
|
||||
credentials: AgentCredentialsByProvider | null,
|
||||
): AgentModelsByProvider {
|
||||
const response = createEmptyModelsResponse();
|
||||
|
||||
for (const provider of AGENT_MODEL_PROVIDERS) {
|
||||
if (!credentials?.[provider]) continue;
|
||||
const providerCredentialId = credentials?.[provider];
|
||||
if (!providerCredentialId) continue;
|
||||
|
||||
const providerInfo = catalog.value[provider];
|
||||
if (!providerInfo) continue;
|
||||
let models: ModelInfo[] | undefined;
|
||||
const projectId = activeProjectId.value;
|
||||
if (projectId) {
|
||||
ensureVerifiedModels(projectId, provider, providerCredentialId);
|
||||
models =
|
||||
verifiedModelsByKey.value[`${projectId}|${provider}|${providerCredentialId}`] ??
|
||||
undefined;
|
||||
}
|
||||
|
||||
if (!models) {
|
||||
const providerInfo = catalog.value[provider];
|
||||
if (!providerInfo) continue;
|
||||
models = Object.values(providerInfo.models);
|
||||
}
|
||||
|
||||
response[provider] = {
|
||||
models: Object.values(providerInfo.models)
|
||||
models: models
|
||||
.map((model) => toAgentModel(provider, model))
|
||||
.sort((a, b) => a.name.localeCompare(b.name)),
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user