From d8aafc634cecbf55ece3121c4b135605fd9513aa Mon Sep 17 00:00:00 2001 From: Jaakko Husso Date: Wed, 2 Sep 2026 11:52:50 +0000 Subject: [PATCH] fix(core): Report a model id in Instance AI traces and run metrics (no-changelog) (#37415) Co-authored-by: Claude Opus 5 (1M context) --- packages/@n8n/instance-ai/src/index.ts | 2 + .../__tests__/langsmith-tracing.test.ts | 50 ++++++++++++++ .../instance-ai/src/tracing/trace-payloads.ts | 13 ++-- .../utils/__tests__/model-config-id.test.ts | 66 +++++++++++++++++++ .../instance-ai/src/utils/model-config-id.ts | 41 ++++++++++++ .../src/events/maps/instance-ai.event-map.ts | 4 +- .../__tests__/observability.test.ts | 31 ++++++++- .../instance-ai/instance-ai.service.ts | 3 +- .../src/modules/instance-ai/observability.ts | 14 +++- 9 files changed, 213 insertions(+), 11 deletions(-) create mode 100644 packages/@n8n/instance-ai/src/utils/__tests__/model-config-id.test.ts create mode 100644 packages/@n8n/instance-ai/src/utils/model-config-id.ts diff --git a/packages/@n8n/instance-ai/src/index.ts b/packages/@n8n/instance-ai/src/index.ts index 8b8496d35ab..47080bd74d4 100644 --- a/packages/@n8n/instance-ai/src/index.ts +++ b/packages/@n8n/instance-ai/src/index.ts @@ -187,6 +187,8 @@ const loadValidateAttachments = lazyModule( export { MAX_STEPS } from './constants/max-steps'; export { parseModelHeadersJson } from './utils/parse-model-headers'; +export { modelConfigId } from './utils/model-config-id'; +export { isEndpointModelConfig } from './utils/modal-session'; export { resolveCustomModelExperimentDefaultsFromEnv } from './utils/custom-model-defaults'; export { WorkflowSaveConflictError } from './errors/workflow-save-conflict.error'; export { WorkflowNotFoundError } from './errors/workflow-not-found.error'; diff --git a/packages/@n8n/instance-ai/src/tracing/__tests__/langsmith-tracing.test.ts b/packages/@n8n/instance-ai/src/tracing/__tests__/langsmith-tracing.test.ts index b6b816f8bc0..6cf88761ae4 100644 --- a/packages/@n8n/instance-ai/src/tracing/__tests__/langsmith-tracing.test.ts +++ b/packages/@n8n/instance-ai/src/tracing/__tests__/langsmith-tracing.test.ts @@ -1734,6 +1734,56 @@ describe('createInstanceAiTraceContext', () => { expect(JSON.stringify(tracing?.orchestratorRun.metadata)).not.toContain('sk-ant-secret'); }); + it('records a model id for a pre-built AI SDK model instead of dumping the instance', async () => { + // What the proxy routes hand over: `modelId` + `config.provider`, no `id`. + // `provider` is a prototype getter on the real SDK model, hence the class. + class FakeChatLanguageModel { + readonly specificationVersion = 'v4'; + + readonly modelId = 'kimi-k3'; + + readonly config = { + provider: 'moonshotai.chat', + url: () => 'https://proxy.example.com/kimi/v1', + headers: function getHeaders() { + return {}; + }, + includeUsage: true, + }; + + readonly chunkSchema = { '~standard': { vendor: 'zod', version: 1 } }; + + get provider() { + return this.config.provider; + } + } + + const tracing = await createInstanceAiTraceContext({ + threadId: 'thread-1', + messageId: 'message-1', + runId: 'run-1', + userId: 'user-1', + modelId: new FakeChatLanguageModel(), + input: { message: 'What workflows do I have?' }, + }); + + expect(tracing?.messageRun.metadata).toEqual( + expect.objectContaining({ model_id: 'moonshotai/kimi-k3' }), + ); + + await tracing?.finishRun(tracing.orchestratorRun, { + outputs: { result: 'done' }, + metadata: { model_id: new FakeChatLanguageModel() }, + }); + + expect(tracing?.orchestratorRun.metadata).toEqual( + expect.objectContaining({ model_id: 'moonshotai/kimi-k3' }), + ); + const serialized = JSON.stringify(tracing?.orchestratorRun.metadata); + expect(serialized).not.toContain('chunkSchema'); + expect(serialized).not.toContain('[function'); + }); + it('traces suspendable tools and HITL suspension spans', async () => { const tracing = await createInstanceAiTraceContext({ threadId: 'thread-1', diff --git a/packages/@n8n/instance-ai/src/tracing/trace-payloads.ts b/packages/@n8n/instance-ai/src/tracing/trace-payloads.ts index c86ac66bb6c..c9e38f003d9 100644 --- a/packages/@n8n/instance-ai/src/tracing/trace-payloads.ts +++ b/packages/@n8n/instance-ai/src/tracing/trace-payloads.ts @@ -16,6 +16,7 @@ import { } from '../tools/tool-ids'; import type { InstanceAiToolRegistry } from '../types'; import { formatAgentRoleLabel, formatTraceLabel } from './trace-labels'; +import { modelConfigId } from '../utils/model-config-id'; const MAX_TRACE_DEPTH = 4; const MAX_PROMPT_SCHEMA_TRACE_DEPTH = 12; @@ -1342,12 +1343,12 @@ export function rawTracePayload(value: unknown): Record { } export function serializeModelIdForTrace(modelId: unknown): unknown { - if (typeof modelId === 'string' && modelId.length > 0) { - return truncateString(modelId); - } - - if (isRecord(modelId) && typeof modelId.id === 'string') { - return truncateString(modelId.id); + // Falling through to `sanitizeTraceValue` dumps the whole model instance — + // config, zod chunk schema, bound functions — into the span attribute, so + // every recognizable variant has to be handled by `modelConfigId`. + const id = modelConfigId(modelId); + if (id !== undefined) { + return truncateString(id); } return sanitizeTraceValue(modelId); diff --git a/packages/@n8n/instance-ai/src/utils/__tests__/model-config-id.test.ts b/packages/@n8n/instance-ai/src/utils/__tests__/model-config-id.test.ts new file mode 100644 index 00000000000..def1a9d7abf --- /dev/null +++ b/packages/@n8n/instance-ai/src/utils/__tests__/model-config-id.test.ts @@ -0,0 +1,66 @@ +import { modelConfigId } from '../model-config-id'; + +describe('modelConfigId', () => { + it('returns plain-string model ids as-is', () => { + expect(modelConfigId('anthropic/claude-sonnet-4-6')).toBe('anthropic/claude-sonnet-4-6'); + }); + + it('reads the id off endpoint and Vertex configs', () => { + expect( + modelConfigId({ + id: 'anthropic/claude-sonnet-4-6', + url: 'https://api.anthropic.com/v1/messages', + apiKey: 'sk-ant-secret', + }), + ).toBe('anthropic/claude-sonnet-4-6'); + expect( + modelConfigId({ + id: 'google-vertex-anthropic/claude-opus-4-8', + project: 'p', + location: 'europe-west1', + }), + ).toBe('google-vertex-anthropic/claude-opus-4-8'); + }); + + it('reads `modelId` off a pre-built AI SDK model and drops the transport suffix', () => { + // Shape of what `createProxyLanguageModel` returns: `modelId` + `config.provider`, + // no `id`. `provider` is a prototype getter, so only `config` is an own property. + expect( + modelConfigId({ + specificationVersion: 'v4', + modelId: 'kimi-k3', + config: { provider: 'moonshotai.chat', includeUsage: true }, + }), + ).toBe('moonshotai/kimi-k3'); + expect( + modelConfigId({ + specificationVersion: 'v4', + modelId: 'claude-opus-4-8', + config: { provider: 'anthropic.messages' }, + }), + ).toBe('anthropic/claude-opus-4-8'); + }); + + it('prefers a top-level provider over the config one', () => { + expect( + modelConfigId({ + modelId: 'claude-opus-4-8', + provider: 'anthropic.messages', + config: { provider: 'ignored.chat' }, + }), + ).toBe('anthropic/claude-opus-4-8'); + }); + + it('falls back to the bare model name when no provider is readable', () => { + expect(modelConfigId({ modelId: 'kimi-k3' })).toBe('kimi-k3'); + expect(modelConfigId({ modelId: 'kimi-k3', config: { provider: 42 } })).toBe('kimi-k3'); + }); + + it('returns undefined when no id can be read', () => { + expect(modelConfigId(undefined)).toBeUndefined(); + expect(modelConfigId('')).toBeUndefined(); + expect(modelConfigId({})).toBeUndefined(); + expect(modelConfigId({ modelId: '' })).toBeUndefined(); + expect(modelConfigId({ modelId: 7 })).toBeUndefined(); + }); +}); diff --git a/packages/@n8n/instance-ai/src/utils/model-config-id.ts b/packages/@n8n/instance-ai/src/utils/model-config-id.ts new file mode 100644 index 00000000000..e65031e6821 --- /dev/null +++ b/packages/@n8n/instance-ai/src/utils/model-config-id.ts @@ -0,0 +1,41 @@ +import { isRecord } from '@n8n/utils/is-record'; + +/** Normalize the provider half of a `provider/model` id. The AI SDK reports the + * transport too (`anthropic.messages`, `moonshotai.chat`), which we drop so + * proxy-built models group with the plain-string model ids. */ +function normalizeProvider(provider: unknown): string | undefined { + if (typeof provider !== 'string') return undefined; + return provider.split('.', 1)[0] || undefined; +} + +/** + * Best-effort `provider/model` id for any `ModelConfig` variant — for telemetry, + * tracing, and logs. + * + * Takes `unknown` so callers holding an unnarrowed value can use it too. Returns + * `undefined` when no id can be read, so callers pick their own fallback. + * + * The pre-built AI SDK `LanguageModel` variant (proxy routes) is the reason this + * exists: it carries `modelId` plus a `provider` and no `id`, so an `id`-only + * lookup silently misses every proxy-backed run. + */ +export function modelConfigId(config: unknown): string | undefined { + if (typeof config === 'string') { + return config.length > 0 ? config : undefined; + } + + if (!isRecord(config)) return undefined; + + if (typeof config.id === 'string') return config.id; + + if (typeof config.modelId === 'string' && config.modelId.length > 0) { + // `provider` is a prototype getter on AI SDK model instances; `config.provider` + // is the own property the openai-compatible and Anthropic providers set. + const provider = normalizeProvider( + config.provider ?? (isRecord(config.config) ? config.config.provider : undefined), + ); + return provider ? `${provider}/${config.modelId}` : config.modelId; + } + + return undefined; +} diff --git a/packages/cli/src/events/maps/instance-ai.event-map.ts b/packages/cli/src/events/maps/instance-ai.event-map.ts index ad67ccb3e39..539cb932bbf 100644 --- a/packages/cli/src/events/maps/instance-ai.event-map.ts +++ b/packages/cli/src/events/maps/instance-ai.event-map.ts @@ -36,7 +36,9 @@ export type InstanceAiEventMap = { status: 'completed' | 'cancelled' | 'error' | 'suspended'; /** Wall-clock duration of the run, or undefined when the start time is unknown. */ durationMs?: number; - /** Model identifier for built-in providers; 'custom' for OpenAI-compatible/native instances. */ + /** Model identifier for managed models (built-in providers and proxy-built + * instances); 'custom' for user-configured OpenAI-compatible endpoints, + * whose free-form ids would inflate the metric label's cardinality. */ model: string; toolCalls: number; toolErrors: number; diff --git a/packages/cli/src/modules/instance-ai/__tests__/observability.test.ts b/packages/cli/src/modules/instance-ai/__tests__/observability.test.ts index 6bcf1d69334..36d128c957c 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/observability.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/observability.test.ts @@ -1,6 +1,6 @@ -import type { InstanceAiTraceContext } from '@n8n/instance-ai'; +import type { InstanceAiTraceContext, ModelConfig } from '@n8n/instance-ai'; -import { buildInstanceAiObservabilityContext } from '../observability'; +import { buildInstanceAiObservabilityContext, runMetricsModelLabel } from '../observability'; describe('Instance AI observability', () => { it('builds a flat correlation context from run and trace details', () => { @@ -49,3 +49,30 @@ describe('Instance AI observability', () => { }); }); }); + +describe('runMetricsModelLabel', () => { + it('reports managed model ids as-is', () => { + expect(runMetricsModelLabel('anthropic/claude-sonnet-4-6')).toBe('anthropic/claude-sonnet-4-6'); + }); + + it('reports the model id of a proxy-built AI SDK instance', () => { + const proxyModel = { + specificationVersion: 'v4', + modelId: 'kimi-k3', + config: { provider: 'moonshotai.chat' }, + } as unknown as ModelConfig; + + expect(runMetricsModelLabel(proxyModel)).toBe('moonshotai/kimi-k3'); + }); + + it("collapses user-configured endpoints and unknown configs to 'custom'", () => { + expect( + runMetricsModelLabel({ + id: 'custom/some-self-hosted-model', + url: 'https://llm.example.com/v1', + }), + ).toBe('custom'); + expect(runMetricsModelLabel(undefined)).toBe('custom'); + expect(runMetricsModelLabel({} as unknown as ModelConfig)).toBe('custom'); + }); +}); diff --git a/packages/cli/src/modules/instance-ai/instance-ai.service.ts b/packages/cli/src/modules/instance-ai/instance-ai.service.ts index 3f49e1bd137..6de575bb042 100644 --- a/packages/cli/src/modules/instance-ai/instance-ai.service.ts +++ b/packages/cli/src/modules/instance-ai/instance-ai.service.ts @@ -173,6 +173,7 @@ import { INSTANCE_AI_RUN_TIMEOUT_REASON, InstanceAiLivenessService } from './liv import { InstanceAiMcpRegistryService } from './mcp'; import { buildInstanceAiObservabilityContext, + runMetricsModelLabel, type InstanceAiObservabilityContext, } from './observability'; import { @@ -6626,7 +6627,7 @@ export class InstanceAiService { // Duration is reported once, by the run's terminal event. durationMs: status !== 'suspended' && startedAt !== undefined ? Date.now() - startedAt : undefined, - model: typeof options?.modelId === 'string' ? options.modelId : 'custom', + model: runMetricsModelLabel(options?.modelId), toolCalls: options?.workSummary?.totalToolCalls ?? 0, toolErrors: options?.workSummary?.totalToolErrors ?? 0, ...(options?.usage ? { usage: options.usage } : {}), diff --git a/packages/cli/src/modules/instance-ai/observability.ts b/packages/cli/src/modules/instance-ai/observability.ts index dda95a931c8..95b9de1fef0 100644 --- a/packages/cli/src/modules/instance-ai/observability.ts +++ b/packages/cli/src/modules/instance-ai/observability.ts @@ -1,4 +1,5 @@ -import type { InstanceAiTraceContext } from '@n8n/instance-ai'; +import { isEndpointModelConfig, modelConfigId } from '@n8n/instance-ai'; +import type { InstanceAiTraceContext, ModelConfig } from '@n8n/instance-ai'; export type InstanceAiObservabilityContext = { threadId: string; @@ -35,3 +36,14 @@ export function buildInstanceAiObservabilityContext( ...(context.role ? { role: context.role } : {}), }; } + +/** + * `model` is a Prometheus label on the Instance AI run metrics, so it has to stay + * low-cardinality. Managed models (built-in ids and the pre-built AI SDK + * instances the proxy hands over) come from a fixed set and are reported as-is; + * a user-configured endpoint's model id is free-form, so it collapses to 'custom'. + */ +export function runMetricsModelLabel(modelId: ModelConfig | undefined): string { + if (modelId === undefined || isEndpointModelConfig(modelId)) return 'custom'; + return modelConfigId(modelId) ?? 'custom'; +}