From 91ebeecb4e148372041b3f6e21bdb123c2eacb17 Mon Sep 17 00:00:00 2001 From: Riqwan Thamir Date: Thu, 13 Aug 2026 10:15:03 +0000 Subject: [PATCH] feat(core): Improve Instance AI model experiments (no-changelog) (#35034) Co-authored-by: Cursor --- .github/WORKFLOWS.md | 4 +- .github/workflows/test-evals-instance-ai.yml | 245 ++++++++++++++++-- packages/@n8n/agents/package.json | 1 + packages/@n8n/agents/src/index.ts | 2 + .../runtime/__tests__/model-factory.test.ts | 103 +++++++- .../runtime/__tests__/provider-quirks.test.ts | 109 +++++++- .../agents/src/runtime/loop/agent-runtime.ts | 1 + .../agents/src/runtime/loop/generate-sink.ts | 1 + .../src/runtime/loop/run-output-sink.ts | 1 + .../src/runtime/loop/runtime-context.ts | 3 + .../agents/src/runtime/loop/stream-sink.ts | 1 + .../agents/src/runtime/model/model-factory.ts | 122 +++++++-- .../agents/src/runtime/model/prompt-cache.ts | 15 +- .../src/runtime/model/provider-credentials.ts | 13 + .../src/runtime/model/provider-quirks.ts | 97 +++++-- .../agents/src/runtime/model/raw-usage.ts | 1 + packages/@n8n/agents/src/sdk/catalog.ts | 1 + packages/@n8n/agents/src/types/index.ts | 2 + packages/@n8n/agents/src/types/sdk/agent.ts | 6 + .../@n8n/agents/src/types/sdk/provider.ts | 44 +++- .../src/agents/provider-capabilities.ts | 20 +- .../config/src/configs/instance-ai.config.ts | 21 ++ packages/@n8n/config/test/config.test.ts | 3 + .../@n8n/instance-ai/docs/configuration.md | 15 +- packages/@n8n/instance-ai/eslint.config.mjs | 2 + .../@n8n/instance-ai/evaluations/README.md | 28 ++ .../__tests__/checklist-verifier.test.ts | 12 +- .../evaluations/checklist/verifier.ts | 8 + .../evaluations/harness/cleanup.ts | 10 +- .../skills/workflow-builder/SKILL.md | 3 +- .../__tests__/apply-agent-thinking.test.ts | 89 ++++++- .../agent/__tests__/instance-agent.test.ts | 40 +++ .../src/agent/apply-agent-thinking.ts | 52 +++- .../instance-ai/src/agent/instance-agent.ts | 26 +- packages/@n8n/instance-ai/src/index.ts | 3 + packages/@n8n/instance-ai/src/types.ts | 16 +- .../__tests__/custom-model-defaults.test.ts | 102 ++++++++ .../src/utils/__tests__/eval-agents.test.ts | 93 ++++++- .../src/utils/__tests__/modal-session.test.ts | 65 +++++ .../src/utils/custom-model-defaults.ts | 119 +++++++++ .../@n8n/instance-ai/src/utils/eval-agents.ts | 56 +++- .../instance-ai/src/utils/modal-session.ts | 50 ++++ .../src/utils/parse-model-headers.ts | 28 ++ .../workspace/__tests__/sandbox-setup.test.ts | 13 +- .../src/workspace/pack-workspace-sdk.ts | 28 +- .../src/workspace/sandbox-setup.ts | 2 +- packages/@n8n/instance-ai/tsconfig.json | 2 +- .../agents-builder-settings.service.test.ts | 26 +- .../agents-builder.service.session.test.ts | 2 +- .../agents-builder-settings.service.ts | 28 +- .../instance-ai-settings.service.test.ts | 82 ++++++ .../instance-ai-settings.service.ts | 100 ++++++- .../__tests__/provider-capabilities.test.ts | 17 +- pnpm-lock.yaml | 239 +++++++++++------ pnpm-workspace.yaml | 5 + 55 files changed, 1942 insertions(+), 235 deletions(-) create mode 100644 packages/@n8n/instance-ai/src/utils/__tests__/custom-model-defaults.test.ts create mode 100644 packages/@n8n/instance-ai/src/utils/__tests__/modal-session.test.ts create mode 100644 packages/@n8n/instance-ai/src/utils/custom-model-defaults.ts create mode 100644 packages/@n8n/instance-ai/src/utils/modal-session.ts create mode 100644 packages/@n8n/instance-ai/src/utils/parse-model-headers.ts diff --git a/.github/WORKFLOWS.md b/.github/WORKFLOWS.md index 59aca73b1a8..71f81ed4923 100644 --- a/.github/WORKFLOWS.md +++ b/.github/WORKFLOWS.md @@ -209,7 +209,7 @@ the lab bench.** The gate deliberately exposes only PR re-runs. Anything that isn't PR gating — baselines, model experiments, arbitrary branch runs — goes through `test-evals-instance-ai.yml`'s own dispatch form ("Instance AI Evals: Experiments"): full knob set (branch, filter, tier, suite, -iterations, experiment-name, model), no per-PR cancellation (dispatches run in parallel, e.g. concurrent +iterations, experiment-name, model, model-url, model-key, reasoning-effort, supports-structured-outputs), no per-PR cancellation (dispatches run in parallel, e.g. concurrent model-comparison arms), and SHA-keyed docker cache hits on master. Evals never run on fork PRs: the event trigger gates on `head.repo.fork`, and the `pr` re-run path refuses fork PRs in `resolve` (dispatched runs carry secrets). @@ -763,7 +763,7 @@ Adding a new channel requires inviting the bot first; the first run otherwise fa | Cloud/CDN | `CLOUDFLARE_API_TOKEN`, `CLOUDFLARE_ACCOUNT_ID` | | GitHub Automation | `N8N_ASSISTANT_APP_ID`, `N8N_ASSISTANT_PRIVATE_KEY` | | Benchmarking | `BENCHMARK_ARM_*`, `N8N_BENCHMARK_LICENSE_CERT` | -| AI/Evals | `ANTHROPIC_API_KEY`, `EVALS_LANGSMITH_*` | +| AI/Evals | `EVALS_ANTHROPIC_KEY`, `EVALS_OPENAI_KEY`, `EVALS_OPENROUTER_KEY`, `EVALS_XAI_KEY`, `EVALS_BASETEN_KEY`, `EVALS_FIREWORKS_KEY`, `EVALS_TOGETHER_KEY`, `EVALS_DATABRICKS_KEY`, `EVALS_MODAL_KEY`, `EVALS_LYCEUM_KEY`, `EVALS_AZURE_FOUNDRY_KEY`, `EVALS_VERTEX_KEY`, `EVALS_VERTEX_PROJECT_ID`, `EVALS_VERTEX_LOCATION`, `EVALS_LANGSMITH_*` | ### Scoping diff --git a/.github/workflows/test-evals-instance-ai.yml b/.github/workflows/test-evals-instance-ai.yml index b969c1f3fcb..95d4997327c 100644 --- a/.github/workflows/test-evals-instance-ai.yml +++ b/.github/workflows/test-evals-instance-ai.yml @@ -60,7 +60,37 @@ on: type: string default: '' model: - description: 'Model override for all Instance AI agents in the lane backends, provider/model format. Empty = backend default.' + description: 'Model override (provider/model). Empty = anthropic/claude-opus-4-8. Vertex Claude: google-vertex-anthropic/. OpenAI-compat experiments use custom/ + model-url + model-key. See evaluations/README.md.' + required: false + type: string + default: '' + model-url: + description: 'Optional N8N_INSTANCE_AI_MODEL_URL (required for custom/* OpenAI-compatible /v1; also anthropic/* Azure Foundry). See evaluations/README.md.' + required: false + type: string + default: '' + model-key: + description: 'Which EVALS_* secret to use for custom/* (or anthropic/* + model-url). One of: baseten, fireworks, together, modal, databricks, azure, lyceum. Empty = keyless custom, or native openai/openrouter/xai/anthropic prefix routing.' + required: false + type: string + default: '' + reasoning-effort: + description: 'Optional custom/* reasoning effort override (none|minimal|low|medium|high|xhigh|max). Empty = runtime known-model map; still empty = omit.' + required: false + type: string + default: '' + supports-structured-outputs: + description: 'Optional custom/* supportsStructuredOutputs override (true|false). Empty = runtime known-model map; still empty = omit.' + required: false + type: string + default: '' + lanes: + description: 'Parallel n8n lane containers (1-11). Empty = 10, or 1 when model-key=baseten (fits Baseten Basic verified ~500k TPM).' + required: false + type: string + default: '' + eval-concurrency: + description: 'Concurrent scenario executions. Empty = 32, or 2 when model-key=baseten (~0.5M TPM / ~12 RPM — fits Baseten Basic verified).' required: false type: string default: '' @@ -95,7 +125,31 @@ on: required: false default: '' model: - description: 'Model for all Instance AI agents (provider/model). Empty = backend default (anthropic/claude-opus-4-8). For experiments: anthropic/claude-sonnet-4-6, anthropic/claude-sonnet-5.' + description: 'Model for all Instance AI agents (provider/model). Empty = backend default (anthropic/claude-opus-4-8). Vertex Claude: google-vertex-anthropic/. OpenAI-compat experiments: custom/ + model-url + model-key.' + required: false + default: '' + model-url: + description: 'Base URL (N8N_INSTANCE_AI_MODEL_URL). Required for custom/*; optional for anthropic/* Azure Foundry.' + required: false + default: '' + model-key: + description: 'EVALS_* secret alias for custom/*: baseten | fireworks | together | modal | databricks | azure | lyceum. Empty = keyless custom or native provider prefix.' + required: false + default: '' + reasoning-effort: + description: 'Optional custom/* reasoning effort override (none|minimal|low|medium|high|xhigh|max). Empty = runtime known-model map; still empty = omit.' + required: false + default: '' + supports-structured-outputs: + description: 'Optional custom/* supportsStructuredOutputs override (true|false). Empty = runtime known-model map; still empty = omit.' + required: false + default: '' + lanes: + description: 'Parallel n8n lane containers (1-11). Empty = 10, or 1 when model-key=baseten (fits Baseten Basic verified ~500k TPM).' + required: false + default: '' + eval-concurrency: + description: 'Concurrent scenarios. Empty = 32, or 2 when model-key=baseten (~0.5M TPM / ~12 RPM — fits Baseten Basic verified).' required: false default: '' @@ -105,16 +159,19 @@ jobs: # 8vcpu/32GB: full-suite N=10 baselines need ~150 min of eval time and # did not fit the 4vcpu tier at any timeout. runs-on: blacksmith-8vcpu-ubuntu-2204 - # Long ceiling only for high-N baseline captures; regular runs keep a - # tight guard so a wedged run fails fast. Plain comparison (not fromJSON): - # it coerces the string input to a number, and a malformed dispatch value - # (NaN) falls back to the 90-minute guard instead of erroring the job. - timeout-minutes: ${{ (inputs.iterations || '3') >= 5 && 240 || 90 }} + # Long ceiling for high-N baselines (240) and for Baseten (600: auto + # 1 lane / c=2 is ~16× slower than the Anthropic default, so a ~18 min + # full suite becomes ~5 h; +2 h buffer for provider backoff). Plain + # comparison (not fromJSON): coerces the string input to a number; + # malformed NaN falls back to the 90-minute guard instead of erroring. + timeout-minutes: ${{ (inputs['model-key'] || '') == 'baseten' && 600 || ((inputs.iterations || '3') >= 5 && 240 || 90) }} env: - # Each port hosts an independent n8n container. The eval CLI's - # work-stealing allocator dispatches builds across them, capped per-lane. - # 10 lanes x 2.5 GB caps = 25 GB, leaving ~7 GB for sandbox + CLI + OS. - LANE_PORTS: '5678,5679,5680,5681,5682,5683,5684,5685,5686,5687' + # Contiguous ports from 5678; 5678..5688 avoids Node fetch()'s blocked- + # port list (max 11 lanes). Defaults: 10 lanes / c=32 for Anthropic; + # model-key=baseten auto-throttles to 1 / 2 to stay under Baseten Basic + # verified (~500k TPM). + LANES: ${{ inputs.lanes != '' && inputs.lanes || ((inputs['model-key'] || '') == 'baseten' && '1' || '10') }} + EVAL_CONCURRENCY: ${{ inputs.eval-concurrency != '' && inputs.eval-concurrency || ((inputs['model-key'] || '') == 'baseten' && '2' || '32') }} permissions: contents: read pull-requests: write @@ -174,7 +231,20 @@ jobs: - name: Start n8n containers env: EVALS_ANTHROPIC_KEY: ${{ secrets.EVALS_ANTHROPIC_KEY }} + EVALS_OPENAI_KEY: ${{ secrets.EVALS_OPENAI_KEY }} INSTANCE_AI_BRAVE_SEARCH_API_KEY: ${{ secrets.INSTANCE_AI_BRAVE_SEARCH_API_KEY }} + EVALS_OPENROUTER_KEY: ${{ secrets.EVALS_OPENROUTER_KEY }} + EVALS_XAI_KEY: ${{ secrets.EVALS_XAI_KEY }} + EVALS_BASETEN_KEY: ${{ secrets.EVALS_BASETEN_KEY }} + EVALS_FIREWORKS_KEY: ${{ secrets.EVALS_FIREWORKS_KEY }} + EVALS_TOGETHER_KEY: ${{ secrets.EVALS_TOGETHER_KEY }} + EVALS_DATABRICKS_KEY: ${{ secrets.EVALS_DATABRICKS_KEY }} + EVALS_MODAL_KEY: ${{ secrets.EVALS_MODAL_KEY }} + EVALS_LYCEUM_KEY: ${{ secrets.EVALS_LYCEUM_KEY }} + EVALS_AZURE_FOUNDRY_KEY: ${{ secrets.EVALS_AZURE_FOUNDRY_KEY }} + EVALS_VERTEX_KEY: ${{ secrets.EVALS_VERTEX_KEY }} + EVALS_VERTEX_PROJECT_ID: ${{ secrets.EVALS_VERTEX_PROJECT_ID }} + EVALS_VERTEX_LOCATION: ${{ secrets.EVALS_VERTEX_LOCATION }} N8N_LICENSE_ACTIVATION_KEY: ${{ secrets.N8N_LICENSE_ACTIVATION_KEY }} N8N_LICENSE_CERT: ${{ secrets.N8N_LICENSE_CERT }} N8N_ENCRYPTION_KEY: ${{ secrets.N8N_ENCRYPTION_KEY }} @@ -187,6 +257,10 @@ jobs: SANDBOX_NAME_PREFIX: evals-ci-${{ github.head_ref || inputs.branch || github.ref_name }} SANDBOX_PROVIDER: ${{ inputs.sandbox-provider || 'n8n-sandbox' }} INSTANCE_AI_MODEL: ${{ inputs.model }} + INSTANCE_AI_MODEL_URL: ${{ inputs.model-url }} + INSTANCE_AI_MODEL_KEY: ${{ inputs.model-key }} + INSTANCE_AI_REASONING_EFFORT: ${{ inputs.reasoning-effort }} + INSTANCE_AI_SUPPORTS_STRUCTURED_OUTPUTS: ${{ inputs.supports-structured-outputs }} run: | # Build provider-specific env args SANDBOX_ARGS=() @@ -216,8 +290,112 @@ jobs: if [ -n "$INSTANCE_AI_MODEL" ]; then MODEL_ARGS+=(-e N8N_INSTANCE_AI_MODEL="$INSTANCE_AI_MODEL") fi + # Anthropic-compatible custom base (Azure Foundry Claude). Drop /v1/messages + # if pasted from the Foundry Details "Endpoint" field — the Anthropic SDK + # appends /v1/messages itself. + if [ -n "$INSTANCE_AI_MODEL_URL" ]; then + MODEL_ARGS+=(-e N8N_INSTANCE_AI_MODEL_URL="$INSTANCE_AI_MODEL_URL") + fi - IFS=',' read -ra PORTS <<< "$LANE_PORTS" + # custom/* experiment knobs: pass through workflow inputs only. + # When unset, the n8n runtime looks up custom-model-defaults.ts and + # omits the field if still unresolved — do not re-implement that map here. + if [ -n "$INSTANCE_AI_REASONING_EFFORT" ]; then + MODEL_ARGS+=(-e N8N_INSTANCE_AI_REASONING_EFFORT="$INSTANCE_AI_REASONING_EFFORT") + echo "custom reasoning effort override: $INSTANCE_AI_REASONING_EFFORT" + fi + if [ -n "$INSTANCE_AI_SUPPORTS_STRUCTURED_OUTPUTS" ]; then + MODEL_ARGS+=(-e N8N_INSTANCE_AI_SUPPORTS_STRUCTURED_OUTPUTS="$INSTANCE_AI_SUPPORTS_STRUCTURED_OUTPUTS") + echo "custom supportsStructuredOutputs override: $INSTANCE_AI_SUPPORTS_STRUCTURED_OUTPUTS" + fi + + # EvalMock / pin-data / in-product eval LLMs always use Anthropic Sonnet, + # independent of the builder experiment model (custom/Kimi, openai/*, …). + # ANTHROPIC_API_KEY is the provider-native key resolveEvalModelConfig prefers + # when N8N_INSTANCE_AI_EVAL_MODEL differs from N8N_INSTANCE_AI_MODEL. + MODEL_ARGS+=( + -e N8N_INSTANCE_AI_EVAL_MODEL=anthropic/claude-sonnet-4-6 + -e ANTHROPIC_API_KEY="$EVALS_ANTHROPIC_KEY" + ) + + # Lane builds use the provider matching N8N_INSTANCE_AI_MODEL. + # The eval CLI step keeps EVALS_ANTHROPIC_KEY for Sonnet verifier/mocks. + MODEL_API_KEY="$EVALS_ANTHROPIC_KEY" + MODEL_PROVIDER="${INSTANCE_AI_MODEL%%/*}" + require_secret() { + local name="$1" value="$2" + if [ -z "$value" ]; then + echo "::error::${name} secret is empty (model=${INSTANCE_AI_MODEL:-default} model-key=${INSTANCE_AI_MODEL_KEY:-})" + exit 1 + fi + } + resolve_model_key() { + case "$INSTANCE_AI_MODEL_KEY" in + baseten) require_secret EVALS_BASETEN_KEY "$EVALS_BASETEN_KEY"; MODEL_API_KEY="$EVALS_BASETEN_KEY" ;; + fireworks) require_secret EVALS_FIREWORKS_KEY "$EVALS_FIREWORKS_KEY"; MODEL_API_KEY="$EVALS_FIREWORKS_KEY" ;; + together) require_secret EVALS_TOGETHER_KEY "$EVALS_TOGETHER_KEY"; MODEL_API_KEY="$EVALS_TOGETHER_KEY" ;; + modal) require_secret EVALS_MODAL_KEY "$EVALS_MODAL_KEY"; MODEL_API_KEY="$EVALS_MODAL_KEY" ;; + databricks) require_secret EVALS_DATABRICKS_KEY "$EVALS_DATABRICKS_KEY"; MODEL_API_KEY="$EVALS_DATABRICKS_KEY" ;; + azure) require_secret EVALS_AZURE_FOUNDRY_KEY "$EVALS_AZURE_FOUNDRY_KEY"; MODEL_API_KEY="$EVALS_AZURE_FOUNDRY_KEY" ;; + lyceum) require_secret EVALS_LYCEUM_KEY "$EVALS_LYCEUM_KEY"; MODEL_API_KEY="$EVALS_LYCEUM_KEY" ;; + '') ;; + *) + echo "::error::unknown model-key '${INSTANCE_AI_MODEL_KEY}' (expected baseten|fireworks|together|modal|databricks|azure|lyceum)" + exit 1 + ;; + esac + } + if [[ "$INSTANCE_AI_MODEL" == custom/* ]]; then + if [ -z "$INSTANCE_AI_MODEL_URL" ]; then + echo "::error::model custom/* requires model-url (OpenAI-compatible base URL ending in /v1)" + exit 1 + fi + # model-key selects EVALS_*; empty model-key = keyless custom router. + if [ -n "$INSTANCE_AI_MODEL_KEY" ]; then + resolve_model_key + else + MODEL_API_KEY="" + fi + elif [ "$MODEL_PROVIDER" = "google-vertex-anthropic" ]; then + if [ -n "$INSTANCE_AI_MODEL_URL" ]; then + echo "::error::google-vertex-anthropic/* does not use model-url (leave it empty)" + exit 1 + fi + require_secret EVALS_VERTEX_KEY "$EVALS_VERTEX_KEY" + require_secret EVALS_VERTEX_PROJECT_ID "$EVALS_VERTEX_PROJECT_ID" + MODEL_API_KEY="" + MODEL_ARGS+=( + -e N8N_INSTANCE_AI_VERTEX_PROJECT_ID="$EVALS_VERTEX_PROJECT_ID" + -e N8N_INSTANCE_AI_VERTEX_LOCATION="${EVALS_VERTEX_LOCATION:-global}" + -e N8N_INSTANCE_AI_VERTEX_SERVICE_ACCOUNT_JSON="$EVALS_VERTEX_KEY" + ) + elif [ -n "$INSTANCE_AI_MODEL_URL" ]; then + # anthropic/* + model-url = Azure Foundry Claude (or model-key=azure). + if [ -n "$INSTANCE_AI_MODEL_KEY" ]; then + resolve_model_key + else + require_secret EVALS_AZURE_FOUNDRY_KEY "$EVALS_AZURE_FOUNDRY_KEY" + MODEL_API_KEY="$EVALS_AZURE_FOUNDRY_KEY" + fi + else + case "$MODEL_PROVIDER" in + openai) require_secret EVALS_OPENAI_KEY "$EVALS_OPENAI_KEY"; MODEL_API_KEY="$EVALS_OPENAI_KEY" ;; + openrouter) require_secret EVALS_OPENROUTER_KEY "$EVALS_OPENROUTER_KEY"; MODEL_API_KEY="$EVALS_OPENROUTER_KEY" ;; + xai) require_secret EVALS_XAI_KEY "$EVALS_XAI_KEY"; MODEL_API_KEY="$EVALS_XAI_KEY" ;; + esac + fi + + if [ "$LANES" -lt 1 ] || [ "$LANES" -gt 11 ]; then + echo "::error::lanes must be 1-11 (got $LANES)" + exit 1 + fi + PORTS=() + for i in $(seq 0 $((LANES - 1))); do + PORTS+=($((5678 + i))) + done + # Reuse across later steps (Create test users / Assert / Run Evals). + echo "LANE_PORTS=${PORTS[*]}" >> "$GITHUB_ENV" + echo "Starting $LANES lane(s) on ports ${PORTS[*]} (eval concurrency $EVAL_CONCURRENCY)" for i in "${!PORTS[@]}"; do port="${PORTS[$i]}" # Bounded and self-healing: a lane that exhausts its capped heap is @@ -242,8 +420,8 @@ jobs: -e E2E_TESTS=true \ -e N8N_ENABLED_MODULES=instance-ai \ -e N8N_AI_ENABLED=true \ - -e N8N_INSTANCE_AI_MODEL_API_KEY="$EVALS_ANTHROPIC_KEY" \ -e INSTANCE_AI_BRAVE_SEARCH_API_KEY="$INSTANCE_AI_BRAVE_SEARCH_API_KEY" \ + -e N8N_INSTANCE_AI_MODEL_API_KEY="$MODEL_API_KEY" \ -e N8N_AI_ASSISTANT_BASE_URL="" \ -e N8N_INSTANCE_AI_SANDBOX_ENABLED=true \ "${SANDBOX_ARGS[@]}" \ @@ -283,7 +461,7 @@ jobs: - name: Create test users run: | - IFS=',' read -ra PORTS <<< "$LANE_PORTS" + read -ra PORTS <<< "$LANE_PORTS" for port in "${PORTS[@]}"; do curl -sf -X POST "http://localhost:$port/rest/e2e/reset" \ -H "Content-Type: application/json" \ @@ -304,7 +482,7 @@ jobs: SANDBOX_PROVIDER: ${{ inputs.sandbox-provider || 'n8n-sandbox' }} INSTANCE_AI_MODEL: ${{ inputs.model }} run: | - IFS=',' read -ra PORTS <<< "$LANE_PORTS" + read -ra PORTS <<< "$LANE_PORTS" bad=0 for i in "${!PORTS[@]}"; do port="${PORTS[$i]}" @@ -323,13 +501,15 @@ jobs: echo " lane $lane: sandboxEnabled=true sandboxProvider=$SANDBOX_PROVIDER ok" fi # /preferences returns the effective model name (user pref || config), - # i.e. the name part of provider/model. + # i.e. everything after the first provider/ segment (openrouter/moonshotai/kimi-k3 + # → moonshotai/kimi-k3). Use #*/ not ##*/ so nested OpenRouter ids match. if [ -n "$INSTANCE_AI_MODEL" ]; then + expected_model="${INSTANCE_AI_MODEL#*/}" effective=$(curl -sf -b "/tmp/cookies-$port.txt" \ "http://localhost:$port/rest/instance-ai/preferences" \ | jq -r '.data.modelName') - if [ "$effective" != "${INSTANCE_AI_MODEL##*/}" ]; then - echo "::error::lane $lane (port $port): expected model '${INSTANCE_AI_MODEL##*/}', got '$effective'" + if [ "$effective" != "$expected_model" ]; then + echo "::error::lane $lane (port $port): expected model '$expected_model', got '$effective'" bad=$((bad+1)) else echo " lane $lane: model=$effective ok" @@ -364,7 +544,10 @@ jobs: continue-on-error: true working-directory: packages/@n8n/instance-ai env: + # Host-side verifier/judges always use Anthropic — never the builder experiment model. + N8N_INSTANCE_AI_EVAL_MODEL: anthropic/claude-sonnet-4-6 N8N_INSTANCE_AI_MODEL_API_KEY: ${{ secrets.EVALS_ANTHROPIC_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.EVALS_ANTHROPIC_KEY }} LANGSMITH_TRACING: 'true' LANGSMITH_ENDPOINT: ${{ secrets.EVALS_LANGSMITH_ENDPOINT }} LANGSMITH_API_KEY: ${{ secrets.EVALS_LANGSMITH_API_KEY }} @@ -383,13 +566,14 @@ jobs: EXPERIMENT_NAME: ${{ inputs.experiment-name }} EVAL_PR_NUMBER: ${{ inputs.pr-number || github.event.pull_request.number }} run: | - IFS=',' read -ra PORTS <<< "$LANE_PORTS" + read -ra PORTS <<< "$LANE_PORTS" URLS=() for port in "${PORTS[@]}"; do URLS+=("http://localhost:$port") done BASE_URLS=$(IFS=,; printf '%s' "${URLS[*]}") - ARGS=(--base-url "$BASE_URLS" --concurrency 32 --verbose --iterations "${ITERATIONS:-3}") + echo "Lanes: $LANES | eval concurrency: $EVAL_CONCURRENCY" + ARGS=(--base-url "$BASE_URLS" --concurrency "$EVAL_CONCURRENCY" --verbose --iterations "${ITERATIONS:-3}") # LangTracer is the only CI case source (no disk fallback by design). ARGS+=(--source langtracer --suite "${SUITE:-baseline}") # Pin the LangSmith cohort: langtracer mode otherwise derives a @@ -420,6 +604,17 @@ jobs: if: ${{ always() }} env: EVALS_ANTHROPIC_KEY: ${{ secrets.EVALS_ANTHROPIC_KEY }} + EVALS_OPENAI_KEY: ${{ secrets.EVALS_OPENAI_KEY }} + EVALS_OPENROUTER_KEY: ${{ secrets.EVALS_OPENROUTER_KEY }} + EVALS_XAI_KEY: ${{ secrets.EVALS_XAI_KEY }} + EVALS_BASETEN_KEY: ${{ secrets.EVALS_BASETEN_KEY }} + EVALS_FIREWORKS_KEY: ${{ secrets.EVALS_FIREWORKS_KEY }} + EVALS_TOGETHER_KEY: ${{ secrets.EVALS_TOGETHER_KEY }} + EVALS_DATABRICKS_KEY: ${{ secrets.EVALS_DATABRICKS_KEY }} + EVALS_MODAL_KEY: ${{ secrets.EVALS_MODAL_KEY }} + EVALS_LYCEUM_KEY: ${{ secrets.EVALS_LYCEUM_KEY }} + EVALS_AZURE_FOUNDRY_KEY: ${{ secrets.EVALS_AZURE_FOUNDRY_KEY }} + EVALS_VERTEX_KEY: ${{ secrets.EVALS_VERTEX_KEY }} DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }} N8N_LICENSE_ACTIVATION_KEY: ${{ secrets.N8N_LICENSE_ACTIVATION_KEY }} N8N_LICENSE_CERT: ${{ secrets.N8N_LICENSE_CERT }} @@ -430,9 +625,13 @@ jobs: # ::add-mask:: is a single-line workflow command. Multi-line secrets # (e.g. N8N_LICENSE_CERT is PEM-encoded) must be masked one line at # a time, otherwise only the first line is registered. - for v in "$EVALS_ANTHROPIC_KEY" "$DAYTONA_API_KEY" \ - "$N8N_LICENSE_ACTIVATION_KEY" "$N8N_LICENSE_CERT" \ - "$N8N_ENCRYPTION_KEY" "$EVALS_LANGSMITH_API_KEY"; do + for v in "$EVALS_ANTHROPIC_KEY" "$EVALS_OPENAI_KEY" "$EVALS_OPENROUTER_KEY" \ + "$EVALS_XAI_KEY" "$EVALS_BASETEN_KEY" "$EVALS_FIREWORKS_KEY" \ + "$EVALS_TOGETHER_KEY" "$EVALS_DATABRICKS_KEY" "$EVALS_MODAL_KEY" \ + "$EVALS_LYCEUM_KEY" "$EVALS_AZURE_FOUNDRY_KEY" "$EVALS_VERTEX_KEY" \ + "$DAYTONA_API_KEY" \ + "$N8N_LICENSE_ACTIVATION_KEY" "$N8N_LICENSE_CERT" "$N8N_ENCRYPTION_KEY" \ + "$EVALS_LANGSMITH_API_KEY"; do [ -z "$v" ] && continue while IFS= read -r line; do [ -n "$line" ] && echo "::add-mask::$line" diff --git a/packages/@n8n/agents/package.json b/packages/@n8n/agents/package.json index 38e22560f20..bf6fa98fee3 100644 --- a/packages/@n8n/agents/package.json +++ b/packages/@n8n/agents/package.json @@ -98,6 +98,7 @@ "@ai-sdk/deepseek": "catalog:", "@ai-sdk/gateway": "catalog:", "@ai-sdk/google": "catalog:", + "@ai-sdk/google-vertex": "catalog:", "@ai-sdk/groq": "catalog:", "@ai-sdk/mistral": "catalog:", "@ai-sdk/openai": "catalog:", diff --git a/packages/@n8n/agents/src/index.ts b/packages/@n8n/agents/src/index.ts index adf503cb509..f668ae49030 100644 --- a/packages/@n8n/agents/src/index.ts +++ b/packages/@n8n/agents/src/index.ts @@ -25,7 +25,9 @@ export type { Provider, ThinkingConfig, ThinkingConfigFor, + AnthropicThinkingEffort, AnthropicThinkingConfig, + OpenAIReasoningEffort, OpenAIThinkingConfig, GoogleThinkingConfig, XaiThinkingConfig, diff --git a/packages/@n8n/agents/src/runtime/__tests__/model-factory.test.ts b/packages/@n8n/agents/src/runtime/__tests__/model-factory.test.ts index 0b573224a92..3774de7ec22 100644 --- a/packages/@n8n/agents/src/runtime/__tests__/model-factory.test.ts +++ b/packages/@n8n/agents/src/runtime/__tests__/model-factory.test.ts @@ -7,6 +7,8 @@ type ProviderOpts = { baseURL?: string; fetch?: typeof globalThis.fetch; headers?: Record; + includeUsage?: boolean; + supportsStructuredOutputs?: boolean; }; // All providers are mocked via vi.mock so require() inside the registry entries @@ -162,6 +164,8 @@ vi.mock('@ai-sdk/openai-compatible', () => ({ baseURL: opts.baseURL, headers: opts.headers, fetch: opts.fetch, + includeUsage: opts.includeUsage, + supportsStructuredOutputs: opts.supportsStructuredOutputs, specificationVersion: 'v3', }), })); @@ -184,6 +188,25 @@ vi.mock('@ai-sdk/amazon-bedrock', () => ({ }), })); +vi.mock('@ai-sdk/google-vertex/anthropic', () => ({ + createVertexAnthropic: + (opts?: { + project?: string; + location?: string; + googleAuthOptions?: { credentials?: Record }; + fetch?: typeof globalThis.fetch; + }) => + (model: string) => ({ + provider: 'google-vertex-anthropic', + modelId: model, + project: opts?.project, + location: opts?.location, + googleAuthOptions: opts?.googleAuthOptions, + fetch: opts?.fetch, + specificationVersion: 'v3', + }), +})); + const { mockProxyAgent } = vi.hoisted(() => ({ mockProxyAgent: vi.fn() })); vi.mock('undici', () => ({ ProxyAgent: mockProxyAgent, @@ -204,9 +227,9 @@ describe('createModel', () => { }); it('should accept a string config', () => { - const model = createModel('anthropic/claude-sonnet-4-5') as unknown as Record; + const model = createModel('anthropic/claude-opus-5') as unknown as Record; expect(model.provider).toBe('anthropic'); - expect(model.modelId).toBe('claude-sonnet-4-5'); + expect(model.modelId).toBe('claude-opus-5'); }); it('should accept an object config with baseURL', () => { @@ -360,7 +383,7 @@ describe('createModel', () => { expect(model.apiKey).toBe('or-test'); }); - it('should create model for nvidia', () => { + it('should create model for nvidia with SDK defaults for usage and structured outputs', () => { const model = createModel({ id: 'nvidia/nvidia/llama-3.3-nemotron-super-49b-v1', apiKey: 'nv-test', @@ -370,6 +393,28 @@ describe('createModel', () => { expect(model.modelId).toBe('nvidia/llama-3.3-nemotron-super-49b-v1'); expect(model.apiKey).toBe('nv-test'); expect(model.baseURL).toBe('https://integrate.api.nvidia.com/v1'); + expect(model.includeUsage).toBeUndefined(); + expect(model.supportsStructuredOutputs).toBeUndefined(); + }); + + it('should have undefined supportsStructuredOutputs for custom when unset', () => { + const model = createModel({ + id: 'custom/Kimi-K3', + apiKey: 'key', + baseURL: 'https://example.com/v1', + }) as unknown as Record; + expect(model.provider).toBe('custom'); + expect(model.supportsStructuredOutputs).toBe(undefined); + }); + + it('should forward supportsStructuredOutputs for custom when set', () => { + const model = createModel({ + id: 'custom/Kimi-K3', + apiKey: 'key', + baseURL: 'https://example.com/v1', + supportsStructuredOutputs: true, + }) as unknown as Record; + expect(model.supportsStructuredOutputs).toBe(true); }); }); @@ -467,6 +512,58 @@ describe('createModel', () => { }); }); + describe('google-vertex-anthropic', () => { + it('should create model with project, location, and service-account JSON', () => { + const model = createModel({ + id: 'google-vertex-anthropic/claude-opus-4-8', + project: 'my-project', + location: 'global', + googleCredentials: JSON.stringify({ + client_email: 'svc@my-project.iam.gserviceaccount.com', + private_key: '-----BEGIN PRIVATE KEY-----\\nABC\\n-----END PRIVATE KEY-----\\n', + }), + }) as unknown as Record; + expect(model.provider).toBe('google-vertex-anthropic'); + expect(model.modelId).toBe('claude-opus-4-8'); + expect(model.project).toBe('my-project'); + expect(model.location).toBe('global'); + expect(model.googleAuthOptions).toEqual({ + credentials: { + client_email: 'svc@my-project.iam.gserviceaccount.com', + private_key: '-----BEGIN PRIVATE KEY-----\nABC\n-----END PRIVATE KEY-----\n', + }, + }); + }); + + it('should default location to global when omitted', () => { + const model = createModel({ + id: 'google-vertex-anthropic/claude-opus-4-8', + project: 'my-project', + }) as unknown as Record; + expect(model.location).toBe('global'); + expect(model.googleAuthOptions).toBeUndefined(); + }); + + it('should throw if project is missing', () => { + expect(() => + createModel({ + id: 'google-vertex-anthropic/claude-opus-4-8', + location: 'global', + }), + ).toThrow(/Invalid credentials for provider "google-vertex-anthropic"/); + }); + + it('should throw if googleCredentials is not valid JSON', () => { + expect(() => + createModel({ + id: 'google-vertex-anthropic/claude-opus-4-8', + project: 'my-project', + googleCredentials: 'not-json', + }), + ).toThrow(/googleCredentials must be valid JSON/); + }); + }); + describe('unsupported provider', () => { it('should throw for ollama', () => { expect(() => createModel('ollama/llama3')).toThrow(/Unsupported provider: "ollama"/); diff --git a/packages/@n8n/agents/src/runtime/__tests__/provider-quirks.test.ts b/packages/@n8n/agents/src/runtime/__tests__/provider-quirks.test.ts index ea78eb4f598..a0f33b9fc88 100644 --- a/packages/@n8n/agents/src/runtime/__tests__/provider-quirks.test.ts +++ b/packages/@n8n/agents/src/runtime/__tests__/provider-quirks.test.ts @@ -1,6 +1,11 @@ import { describe, it, expect } from 'vitest'; -import { applyToolProviderOptionDefaults, getProviderQuirks } from '../model/provider-quirks'; +import { + HIGH_REASONING_DEFAULT_MAX_OUTPUT_TOKENS, + applyToolProviderOptionDefaults, + getProviderQuirks, + resolveDefaultMaxOutputTokens, +} from '../model/provider-quirks'; describe('getProviderQuirks', () => { it('returns an empty object for an unknown provider', () => { @@ -93,12 +98,80 @@ describe('thinkingToProviderOptions', () => { }); }); + it('anthropic: enabled mode with effort keeps default budgetTokens', () => { + expect( + getProviderQuirks('anthropic').thinkingToProviderOptions?.( + { mode: 'enabled', effort: 'medium' }, + 'anthropic/claude-sonnet-4-5', + ), + ).toEqual({ + anthropic: { + thinking: { type: 'enabled', budgetTokens: 10000 }, + effort: 'medium', + }, + }); + }); + + it('anthropic: enabled mode with effort forwards explicit budgetTokens', () => { + expect( + getProviderQuirks('anthropic').thinkingToProviderOptions?.( + { mode: 'enabled', effort: 'medium', budgetTokens: 5000 }, + 'anthropic/claude-sonnet-4-5', + ), + ).toEqual({ + anthropic: { + thinking: { type: 'enabled', budgetTokens: 5000 }, + effort: 'medium', + }, + }); + }); + + it('google-vertex-anthropic: emits Anthropic adaptive thinking under the anthropic namespace', () => { + expect( + getProviderQuirks('google-vertex-anthropic').thinkingToProviderOptions?.( + { mode: 'adaptive', effort: 'medium' }, + 'google-vertex-anthropic/claude-opus-4-8', + ), + ).toEqual({ + anthropic: { + thinking: { type: 'adaptive', display: 'summarized' }, + effort: 'medium', + }, + }); + }); + + it('google-vertex-anthropic: tool defaults land under the anthropic namespace', () => { + expect(getProviderQuirks('google-vertex-anthropic').providerOptionsNamespace).toBe('anthropic'); + expect(applyToolProviderOptionDefaults(undefined)).toEqual({ + anthropic: { eagerInputStreaming: false }, + }); + }); + it('openai: defaults reasoningEffort to medium', () => { expect(getProviderQuirks('openai').thinkingToProviderOptions?.({}, 'openai/gpt-5')).toEqual({ openai: { reasoningEffort: 'medium', reasoningSummary: null }, }); }); + it('openai: forwards GPT-5.6 Sol compatible reasoningEffort values', () => { + expect( + getProviderQuirks('openai').thinkingToProviderOptions?.( + { reasoningEffort: 'medium' }, + 'openai/gpt-5.6-sol', + ), + ).toEqual({ + openai: { reasoningEffort: 'medium', reasoningSummary: null }, + }); + expect( + getProviderQuirks('openai').thinkingToProviderOptions?.( + { reasoningEffort: 'xhigh' }, + 'openai/gpt-5.6-sol', + ), + ).toEqual({ + openai: { reasoningEffort: 'xhigh', reasoningSummary: null }, + }); + }); + it('google: forwards thinkingBudget and thinkingLevel when set', () => { expect( getProviderQuirks('google').thinkingToProviderOptions?.( @@ -115,4 +188,38 @@ describe('thinkingToProviderOptions', () => { xai: { reasoningEffort: 'high' }, }); }); + + it('custom: maps reasoningEffort to providerOptions.custom', () => { + expect( + getProviderQuirks('custom').thinkingToProviderOptions?.( + { + reasoningEffort: 'high', + }, + 'custom/Kimi-K3', + ), + ).toEqual({ + custom: { reasoningEffort: 'high' }, + }); + }); + + it('custom: omits reasoning effort when unset', () => { + expect(getProviderQuirks('custom').thinkingToProviderOptions?.({}, 'custom/Kimi-K3')).toEqual( + {}, + ); + }); +}); + +describe('resolveDefaultMaxOutputTokens', () => { + it.each([ + 'custom/accounts/fireworks/models/kimi-k3', + 'openrouter/moonshotai/kimi-k3', + 'custom/Kimi-K3', + ] as const)('raises the output cap to the Kimi K3 default for %s', (modelId) => { + expect(resolveDefaultMaxOutputTokens(modelId)).toBe(HIGH_REASONING_DEFAULT_MAX_OUTPUT_TOKENS); + }); + + it('leaves unrelated models unset', () => { + expect(resolveDefaultMaxOutputTokens('custom/deepseek-ai/DeepSeek-V4-Pro')).toBeUndefined(); + expect(resolveDefaultMaxOutputTokens('anthropic/claude-sonnet-4-5')).toBeUndefined(); + }); }); diff --git a/packages/@n8n/agents/src/runtime/loop/agent-runtime.ts b/packages/@n8n/agents/src/runtime/loop/agent-runtime.ts index a7f1185917d..3f04578ebda 100644 --- a/packages/@n8n/agents/src/runtime/loop/agent-runtime.ts +++ b/packages/@n8n/agents/src/runtime/loop/agent-runtime.ts @@ -883,6 +883,7 @@ export class AgentRuntime { reasoning: staticLoopContext.reasoning, providerOptions: staticLoopContext.providerOptions, outputSpec: staticLoopContext.outputSpec, + maxOutputTokens: staticLoopContext.maxOutputTokens, aiSdkOptions: this.buildAiSdkOptions(toolMap, options), }; let turn = await sink.callModel(modelCallContext); diff --git a/packages/@n8n/agents/src/runtime/loop/generate-sink.ts b/packages/@n8n/agents/src/runtime/loop/generate-sink.ts index 55e355536ab..0f7cee308b1 100644 --- a/packages/@n8n/agents/src/runtime/loop/generate-sink.ts +++ b/packages/@n8n/agents/src/runtime/loop/generate-sink.ts @@ -39,6 +39,7 @@ export class GenerateSink implements RunOutputSink { ...(ctx.hasTools ? { tools: ctx.aiTools } : {}), ...(ctx.providerOptions ? { providerOptions: ctx.providerOptions } : {}), ...(ctx.outputSpec ? { output: ctx.outputSpec } : {}), + ...(ctx.maxOutputTokens !== undefined ? { maxOutputTokens: ctx.maxOutputTokens } : {}), ...ctx.aiSdkOptions, }); diff --git a/packages/@n8n/agents/src/runtime/loop/run-output-sink.ts b/packages/@n8n/agents/src/runtime/loop/run-output-sink.ts index 0ede1adef91..e9ffffadbfa 100644 --- a/packages/@n8n/agents/src/runtime/loop/run-output-sink.ts +++ b/packages/@n8n/agents/src/runtime/loop/run-output-sink.ts @@ -69,6 +69,7 @@ export interface ModelCallContext { reasoning?: LanguageModelCallOptions['reasoning']; providerOptions?: Record; outputSpec?: ReturnType; + maxOutputTokens?: number; aiSdkOptions: { telemetry?: TelemetryOptions; repairToolCall?: ToolCallRepairFunction>; diff --git a/packages/@n8n/agents/src/runtime/loop/runtime-context.ts b/packages/@n8n/agents/src/runtime/loop/runtime-context.ts index dc43f850dbd..96bd6c27ce4 100644 --- a/packages/@n8n/agents/src/runtime/loop/runtime-context.ts +++ b/packages/@n8n/agents/src/runtime/loop/runtime-context.ts @@ -22,6 +22,7 @@ import { getProviderQuirks, PROVIDER_QUIRKS, providerIdFromModelId, + resolveDefaultMaxOutputTokens, } from '../model/provider-quirks'; import type { DeferredToolManager } from '../tools/deferred-tool-manager'; import { buildToolMap, toAiSdkProviderTools, toAiSdkTools } from '../tools/tool-adapter'; @@ -38,6 +39,7 @@ export interface StaticLoopContext { reasoning: AgentRuntimeConfig['reasoning']; providerOptions?: Record; outputSpec?: ReturnType; + maxOutputTokens?: number; } /** @@ -87,6 +89,7 @@ export class RuntimeContextBuilder { reasoning: this.config.reasoning, providerOptions: providerOptions as Record | undefined, outputSpec, + maxOutputTokens: execOptions?.maxOutputTokens ?? resolveDefaultMaxOutputTokens(this.modelId), }; } diff --git a/packages/@n8n/agents/src/runtime/loop/stream-sink.ts b/packages/@n8n/agents/src/runtime/loop/stream-sink.ts index 98f40781399..7de731f679a 100644 --- a/packages/@n8n/agents/src/runtime/loop/stream-sink.ts +++ b/packages/@n8n/agents/src/runtime/loop/stream-sink.ts @@ -194,6 +194,7 @@ export class StreamSink implements RunOutputSink { ...(ctx.hasTools ? { tools: ctx.aiTools } : {}), ...(ctx.providerOptions ? { providerOptions: ctx.providerOptions } : {}), ...(ctx.outputSpec ? { output: ctx.outputSpec } : {}), + ...(ctx.maxOutputTokens !== undefined ? { maxOutputTokens: ctx.maxOutputTokens } : {}), ...ctx.aiSdkOptions, ...this.buildSmoothStreamTransformOptions(), }); diff --git a/packages/@n8n/agents/src/runtime/model/model-factory.ts b/packages/@n8n/agents/src/runtime/model/model-factory.ts index 617e13de80d..1a5dcaa28ff 100644 --- a/packages/@n8n/agents/src/runtime/model/model-factory.ts +++ b/packages/@n8n/agents/src/runtime/model/model-factory.ts @@ -65,6 +65,84 @@ type ProviderRegistry = { [P in ProviderId]: RegistryEntry

; }; +type OpenAiCompatibleCreds = { + apiKey?: string; + baseURL?: string; + headers?: Record; +}; + +/** + * Parse a Vertex service-account JSON string into `googleAuthOptions`. + * Accepts either a full SA JSON blob or the subset `{ client_email, private_key }`. + * Returns undefined when unset so ADC can take over. + */ +function parseGoogleVertexAuthOptions( + googleCredentials: string | undefined, +): { credentials: Record } | undefined { + if (!googleCredentials?.trim()) return undefined; + let parsed: unknown; + try { + parsed = JSON.parse(googleCredentials); + } catch { + throw new Error( + 'Invalid credentials for provider "google-vertex-anthropic": googleCredentials must be valid JSON', + ); + } + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw new Error( + 'Invalid credentials for provider "google-vertex-anthropic": googleCredentials must be a JSON object', + ); + } + const credentials = { ...(parsed as Record) }; + // SA keys often arrive with literal `\n` escapes when pasted into env files. + if (typeof credentials.private_key === 'string') { + credentials.private_key = credentials.private_key.replace(/\\n/g, '\n'); + } + return { credentials }; +} + +/** + * Shared builder for OpenAI-compatible HTTP providers. Prefer this over + * `@ai-sdk/` packages that pull optional NAPI binaries or v4-only types. + */ +function buildOpenAiCompatible( + name: string, + defaultBaseURL: string | undefined, + creds: OpenAiCompatibleCreds, + model: string, + fetch: FetchFn | undefined, + options?: { includeUsage?: boolean; supportsStructuredOutputs?: boolean }, +): LanguageModel { + const { createOpenAICompatible } = + require('@ai-sdk/openai-compatible') as typeof import('@ai-sdk/openai-compatible'); + const baseURL = creds.baseURL ?? defaultBaseURL; + if (!baseURL) { + throw new Error(`baseURL is required for OpenAI-compatible provider "${name}"`); + } + return createOpenAICompatible({ + name, + baseURL, + apiKey: creds.apiKey, + headers: creds.headers, + fetch, + includeUsage: options?.includeUsage, + supportsStructuredOutputs: options?.supportsStructuredOutputs, + })(model); +} + +type OpenAiCompatibleProviderId = 'nvidia'; + +function openAiCompatibleEntry

( + name: P, + defaultBaseURL: string, + options?: { includeUsage?: boolean; supportsStructuredOutputs?: boolean }, +): RegistryEntry

{ + return { + build: (creds, model, fetch) => + buildOpenAiCompatible(name, defaultBaseURL, creds, model, fetch, options), + }; +} + /** * Registry of language model providers. * Each entry maps a provider id to a builder that loads its @ai-sdk/* package @@ -86,17 +164,10 @@ const LANGUAGE_PROVIDERS: ProviderRegistry = { }, }, custom: { - build: (creds, model, fetch) => { - const { createOpenAICompatible } = - require('@ai-sdk/openai-compatible') as typeof import('@ai-sdk/openai-compatible'); - return createOpenAICompatible({ - name: 'custom', - baseURL: creds.baseURL, - apiKey: creds.apiKey, - headers: creds.headers, - fetch, - })(model); - }, + build: (creds, model, fetch) => + buildOpenAiCompatible('custom', undefined, creds, model, fetch, { + supportsStructuredOutputs: creds.supportsStructuredOutputs, + }), }, anthropic: { build: (creds, model, fetch) => { @@ -116,6 +187,21 @@ const LANGUAGE_PROVIDERS: ProviderRegistry = { return createAnthropic({ ...creds, baseURL: normalizedBaseURL, fetch })(model); }, }, + 'google-vertex-anthropic': { + build: (creds, model, fetch) => { + const { createVertexAnthropic } = + require('@ai-sdk/google-vertex/anthropic') as typeof import('@ai-sdk/google-vertex/anthropic'); + const googleAuthOptions = parseGoogleVertexAuthOptions(creds.googleCredentials); + return createVertexAnthropic({ + project: creds.project, + location: creds.location, + baseURL: creds.baseURL, + headers: creds.headers, + ...(googleAuthOptions ? { googleAuthOptions } : {}), + fetch, + })(model); + }, + }, google: { build: (creds, model, fetch) => { const { createGoogle } = require('@ai-sdk/google') as typeof import('@ai-sdk/google'); @@ -165,19 +251,7 @@ const LANGUAGE_PROVIDERS: ProviderRegistry = { return createOpenRouter({ apiKey: creds.apiKey, baseURL: creds.baseURL, fetch })(model); }, }, - nvidia: { - build: (creds, model, fetch) => { - const { createOpenAICompatible } = - require('@ai-sdk/openai-compatible') as typeof import('@ai-sdk/openai-compatible'); - return createOpenAICompatible({ - name: 'nvidia', - baseURL: creds.baseURL ?? 'https://integrate.api.nvidia.com/v1', - apiKey: creds.apiKey, - headers: creds.headers, - fetch, - })(model); - }, - }, + nvidia: openAiCompatibleEntry('nvidia', 'https://integrate.api.nvidia.com/v1', {}), 'azure-openai': { build: (creds, model, fetch) => { const { createAzure } = require('@ai-sdk/azure') as typeof import('@ai-sdk/azure'); diff --git a/packages/@n8n/agents/src/runtime/model/prompt-cache.ts b/packages/@n8n/agents/src/runtime/model/prompt-cache.ts index 446528e0776..f64bee9c3b6 100644 --- a/packages/@n8n/agents/src/runtime/model/prompt-cache.ts +++ b/packages/@n8n/agents/src/runtime/model/prompt-cache.ts @@ -40,7 +40,7 @@ export function getEffectiveAnthropicCacheTtl( config: PromptCachingConfig | undefined, modelId: string, ): '5m' | '1h' | undefined { - if (getModelProvider(modelId) !== 'anthropic' || !isEnabledForProvider(config, 'anthropic')) { + if (!isAnthropicMessagesProvider(modelId) || !isEnabledForProvider(config, 'anthropic')) { return undefined; } return getAnthropicCacheTtl(config); @@ -59,6 +59,12 @@ export function getModelProvider(modelId: string): string { return modelId.split('/')[0]; } +/** Providers that speak the Anthropic Messages API (including Vertex Claude). */ +export function isAnthropicMessagesProvider(modelId: string): boolean { + const provider = getModelProvider(modelId); + return provider === 'anthropic' || provider === 'google-vertex-anthropic'; +} + /** * Deterministic, non-reversible per-agent-version OpenAI `promptCacheKey`. * Hashes the agent name, model id, and base instructions together so the key @@ -144,7 +150,7 @@ export function buildInstructionPromptCacheOptions( config: PromptCachingConfig | undefined, modelId: string, ): ProviderOptions | undefined { - if (getModelProvider(modelId) !== 'anthropic' || !isEnabledForProvider(config, 'anthropic')) { + if (!isAnthropicMessagesProvider(modelId) || !isEnabledForProvider(config, 'anthropic')) { return undefined; } @@ -201,10 +207,7 @@ export function applyRuntimeCacheBreakpoints(params: { staticToolCacheName: string | undefined; }): { messages: ModelMessage[]; aiTools: ToolSet } { const { system, messages, aiTools, promptCaching, modelId, staticToolCacheName } = params; - if ( - getModelProvider(modelId) !== 'anthropic' || - !isEnabledForProvider(promptCaching, 'anthropic') - ) { + if (!isAnthropicMessagesProvider(modelId) || !isEnabledForProvider(promptCaching, 'anthropic')) { return { messages, aiTools }; } diff --git a/packages/@n8n/agents/src/runtime/model/provider-credentials.ts b/packages/@n8n/agents/src/runtime/model/provider-credentials.ts index cfe5b279b5c..d3cb0ba92f5 100644 --- a/packages/@n8n/agents/src/runtime/model/provider-credentials.ts +++ b/packages/@n8n/agents/src/runtime/model/provider-credentials.ts @@ -19,8 +19,21 @@ export const PROVIDER_CREDENTIAL_SCHEMAS = { }), custom: apiKeyCreds.extend({ baseURL: z.string().min(1, 'baseURL is required'), + supportsStructuredOutputs: z.boolean().optional(), }), anthropic: apiKeyCreds, + /** + * Claude on Google Vertex (Anthropic Messages via `:rawPredict`). + * `googleCredentials` is a service-account JSON string; omit it to use ADC + * (`gcloud auth application-default login`). + */ + 'google-vertex-anthropic': z.object({ + project: z.string().min(1, 'project is required'), + location: z.string().min(1, 'location is required').default('global'), + googleCredentials: z.string().optional(), + baseURL: z.string().optional(), + headers: z.record(z.string(), z.string()).optional(), + }), google: apiKeyCreds, xai: apiKeyCreds, groq: apiKeyCreds, diff --git a/packages/@n8n/agents/src/runtime/model/provider-quirks.ts b/packages/@n8n/agents/src/runtime/model/provider-quirks.ts index 217ed77f18c..5135b564dea 100644 --- a/packages/@n8n/agents/src/runtime/model/provider-quirks.ts +++ b/packages/@n8n/agents/src/runtime/model/provider-quirks.ts @@ -3,12 +3,19 @@ import type { AnthropicThinkingConfig, GoogleThinkingConfig, JSONObject, + OpenAIReasoningEffort, OpenAIThinkingConfig, ThinkingConfig, XaiThinkingConfig, } from '../../types'; export interface ProviderQuirks { + /** + * Namespace used in AI SDK `providerOptions` / `providerMetadata`. + * Defaults to the registry key. Vertex Anthropic still speaks the Anthropic + * Messages wire format, so its quirks emit under `anthropic`. + */ + providerOptionsNamespace?: string; /** providerMetadata keys on reasoning parts that must be copied to providerOptions and survive replay. */ reasoningReplayKeys?: string[]; /** Defaults merged under this provider's namespace into every tool's providerOptions (explicit tool values win). */ @@ -22,6 +29,31 @@ export interface ProviderQuirks { ) => Record>; } +/** Shared Anthropic Messages thinking mapping (also used by Vertex Anthropic). */ +function anthropicThinkingToProviderOptions( + thinking: ThinkingConfig, + modelId: string, +): Record> { + const cfg = thinking as AnthropicThinkingConfig; + const adaptive = cfg.mode === 'adaptive' ? cfg : undefined; + if (anthropicUsesAdaptiveThinking(modelId)) { + return { + anthropic: { + thinking: { type: 'adaptive', display: adaptive?.display ?? 'summarized' }, + effort: adaptive?.effort ?? 'medium', + }, + }; + } + const budgetTokens = cfg.mode === 'adaptive' ? undefined : cfg.budgetTokens; + const effort = cfg.mode === 'adaptive' ? undefined : cfg.effort; + return { + anthropic: { + thinking: { type: 'enabled', budgetTokens: budgetTokens ?? 10000 }, + ...(effort !== undefined ? { effort } : {}), + }, + }; +} + /** Anthropic model families that take the adaptive thinking API. */ const ANTHROPIC_ADAPTIVE_THINKING = /claude-(?:opus-(?:5|4-6|4-7|4-8)|sonnet-(?:5|4-6)|fable-5)/; /** Anthropic model families that predate it and still take a fixed token budget. */ @@ -39,6 +71,21 @@ function anthropicUsesAdaptiveThinking(modelId: string): boolean { return modelId.includes('claude-'); } +/** Map effort to top-level `reasoningEffort` (OpenAI-compatible chat). */ +function reasoningEffortQuirk( + provider: ProviderId, + defaultEffort?: OpenAIReasoningEffort, +): ProviderQuirks { + return { + thinkingToProviderOptions: (thinking) => { + const cfg = thinking as OpenAIThinkingConfig; + const reasoningEffort = cfg.reasoningEffort ?? defaultEffort; + if (reasoningEffort === undefined) return {}; + return { [provider]: { reasoningEffort } }; + }, + }; +} + /** * Declarative registry of provider-specific behavior the AI SDK doesn't * normalize away. Each entry documents why the quirk exists and its upstream @@ -61,24 +108,14 @@ export const PROVIDER_QUIRKS: Partial> = { // QUIRK(anthropic): the two thinking APIs are mutually exclusive — an // adaptive model rejects `type: 'enabled'` and vice versa — so the model // decides the shape and the config only fills in its details. - thinkingToProviderOptions: (thinking, modelId) => { - const cfg = thinking as AnthropicThinkingConfig; - const adaptive = cfg.mode === 'adaptive' ? cfg : undefined; - if (anthropicUsesAdaptiveThinking(modelId)) { - return { - anthropic: { - thinking: { type: 'adaptive', display: adaptive?.display ?? 'summarized' }, - effort: adaptive?.effort ?? 'medium', - }, - }; - } - const budgetTokens = cfg.mode === 'adaptive' ? undefined : cfg.budgetTokens; - return { - anthropic: { - thinking: { type: 'enabled', budgetTokens: budgetTokens ?? 10000 }, - }, - }; - }, + thinkingToProviderOptions: anthropicThinkingToProviderOptions, + }, + // Vertex Claude uses AnthropicLanguageModel under the hood — providerOptions + // stay under the `anthropic` namespace even though the model id prefix differs. + 'google-vertex-anthropic': { + providerOptionsNamespace: 'anthropic', + toolProviderOptionDefaults: { eagerInputStreaming: false }, + thinkingToProviderOptions: anthropicThinkingToProviderOptions, }, openai: { // QUIRK(openai): the Responses API pairs each function_call item with a @@ -129,6 +166,8 @@ export const PROVIDER_QUIRKS: Partial> = { return { xai: { reasoningEffort: cfg.reasoningEffort ?? 'high' } }; }, }, + // custom/*: only forward an explicit effort — no provider-level default. + custom: reasoningEffortQuirk('custom'), }; export function getProviderQuirks(providerId: string): ProviderQuirks { @@ -139,6 +178,25 @@ export function providerIdFromModelId(modelId: string): string { return modelId.split('/')[0]; } +/** + * Default completion-token cap for reasoning-heavy Kimi K3 models. + * Context windows are often 131072 shared input+output; requesting the full + * window as max_tokens overflows once any prompt tokens are present. + */ +export const HIGH_REASONING_DEFAULT_MAX_OUTPUT_TOKENS = 65_536; + +/** + * Provider/model-specific default for AI SDK `maxOutputTokens`. Unknown models + * otherwise fall back to a 4096 cap that reasoning-heavy agent turns can exhaust + * before emitting text or tool calls. + */ +export function resolveDefaultMaxOutputTokens(modelId: string): number | undefined { + if (modelId.toLowerCase().includes('kimi-k3')) { + return HIGH_REASONING_DEFAULT_MAX_OUTPUT_TOKENS; + } + return undefined; +} + /** Merge every registered tool providerOptions default under its provider namespace; explicit tool values win. */ export function applyToolProviderOptionDefaults( toolProviderOptions: Record | undefined, @@ -146,7 +204,8 @@ export function applyToolProviderOptionDefaults( const result = { ...toolProviderOptions }; for (const [provider, quirks] of Object.entries(PROVIDER_QUIRKS)) { if (!quirks.toolProviderOptionDefaults) continue; - result[provider] = { ...quirks.toolProviderOptionDefaults, ...result[provider] }; + const namespace = quirks.providerOptionsNamespace ?? provider; + result[namespace] = { ...quirks.toolProviderOptionDefaults, ...result[namespace] }; } return result; } diff --git a/packages/@n8n/agents/src/runtime/model/raw-usage.ts b/packages/@n8n/agents/src/runtime/model/raw-usage.ts index 3a81ebe8c75..1580f816ea2 100644 --- a/packages/@n8n/agents/src/runtime/model/raw-usage.ts +++ b/packages/@n8n/agents/src/runtime/model/raw-usage.ts @@ -74,6 +74,7 @@ export class AnthropicRawUsageReader implements RawUsageReader { */ const RAW_USAGE_READERS: Partial RawUsageReader>> = { anthropic: () => new AnthropicRawUsageReader(), + 'google-vertex-anthropic': () => new AnthropicRawUsageReader(), }; /** diff --git a/packages/@n8n/agents/src/sdk/catalog.ts b/packages/@n8n/agents/src/sdk/catalog.ts index 3ec77e0e42a..b2518636c60 100644 --- a/packages/@n8n/agents/src/sdk/catalog.ts +++ b/packages/@n8n/agents/src/sdk/catalog.ts @@ -11,6 +11,7 @@ const MODELS_DEV_PROVIDER_ALIASES: Record = { const AGENT_PROVIDER_NAMES: Record = { 'aws-bedrock': 'AWS Bedrock', 'azure-openai': 'Azure OpenAI', + 'google-vertex-anthropic': 'Google Vertex Anthropic', }; /** Cost per million tokens. */ diff --git a/packages/@n8n/agents/src/types/index.ts b/packages/@n8n/agents/src/types/index.ts index 46deea5ee5d..7dd05e9f634 100644 --- a/packages/@n8n/agents/src/types/index.ts +++ b/packages/@n8n/agents/src/types/index.ts @@ -27,7 +27,9 @@ export type { BuiltFileStore } from './sdk/file-store'; export type { Provider, + AnthropicThinkingEffort, AnthropicThinkingConfig, + OpenAIReasoningEffort, OpenAIThinkingConfig, GoogleThinkingConfig, XaiThinkingConfig, diff --git a/packages/@n8n/agents/src/types/sdk/agent.ts b/packages/@n8n/agents/src/types/sdk/agent.ts index 3f63c1c2dfe..a98db4a7654 100644 --- a/packages/@n8n/agents/src/types/sdk/agent.ts +++ b/packages/@n8n/agents/src/types/sdk/agent.ts @@ -179,6 +179,12 @@ export interface ExecutionOptions { maxIterations?: number; abortSignal?: AbortSignal; providerOptions?: ProviderOptions; + /** + * Cap on completion tokens for each model call (`max_tokens` / + * `maxOutputTokens`). When unset, provider/model defaults from + * `resolveDefaultMaxOutputTokens` apply. + */ + maxOutputTokens?: number; /** AI SDK `smoothStream` transform. Enabled by default; pass `false` to disable. */ smoothStream?: SmoothStreamOptions | false; /** diff --git a/packages/@n8n/agents/src/types/sdk/provider.ts b/packages/@n8n/agents/src/types/sdk/provider.ts index 35489a555d5..cf47db39d25 100644 --- a/packages/@n8n/agents/src/types/sdk/provider.ts +++ b/packages/@n8n/agents/src/types/sdk/provider.ts @@ -6,15 +6,16 @@ export type Provider = | 'anthropic' | 'cerebras' + | 'custom' | 'deepinfra' | 'deepseek' | 'google' + | 'google-vertex-anthropic' | 'groq' | 'mistral' | 'openai' | 'openrouter' | 'perplexity' - | 'togetherai' | 'vercel' | 'xai' | (string & {}); @@ -26,6 +27,8 @@ export type Provider = * - `'adaptive'`: the model decides how much to think per request. * - `'enabled'` (default): a fixed token budget controlled by `budgetTokens`. */ +export type AnthropicThinkingEffort = 'low' | 'medium' | 'high' | 'xhigh' | 'max'; + export type AnthropicThinkingConfig = | { mode: 'adaptive'; @@ -35,18 +38,43 @@ export type AnthropicThinkingConfig = * reasoning streams and replay metadata are available. */ display?: 'omitted' | 'summarized'; - /** Reasoning effort for adaptive thinking. Defaults to 'medium'. */ - effort?: 'low' | 'medium' | 'high' | 'xhigh' | 'max'; + /** + * Adaptive-thinking effort (`output_config.effort` on the Anthropic API). + * When omitted, Anthropic/SDK defaults apply. + */ + effort?: AnthropicThinkingEffort; } | { mode?: 'enabled'; - /** Token budget for extended thinking. Defaults to 10000. */ + /** Token budget for extended thinking. Defaults to 10000 when `effort` is omitted. */ budgetTokens?: number; + /** + * Maps to Anthropic `output_config.effort`. Used with `type: 'enabled'` for + * Anthropic-compatible gateways that reject `adaptive`. + */ + effort?: AnthropicThinkingEffort; }; +/** + * OpenAI / AI SDK reasoning effort. + * GPT-5.6 Sol supports none|low|medium|high|xhigh|max (no `minimal`). + * Older reasoning models may also accept `minimal`. + */ +export type OpenAIReasoningEffort = + | 'none' + | 'minimal' + | 'low' + | 'medium' + | 'high' + | 'xhigh' + | 'max'; + export interface OpenAIThinkingConfig { - /** Reasoning effort level. Defaults to 'medium'. */ - reasoningEffort?: 'low' | 'medium' | 'high'; + /** + * Reasoning effort level. Mapped to AI SDK `providerOptions..reasoningEffort`. + * OpenAI quirks may default when unset; `custom` only forwards an explicit value. + */ + reasoningEffort?: OpenAIReasoningEffort; } export interface GoogleThinkingConfig { @@ -58,7 +86,7 @@ export interface GoogleThinkingConfig { export interface XaiThinkingConfig { /** Reasoning effort level. */ - reasoningEffort?: 'low' | 'high'; + reasoningEffort?: 'low' | 'medium' | 'high'; } /** @@ -66,7 +94,7 @@ export interface XaiThinkingConfig { * Known providers get their specific config; unknown providers default * to OpenAI-style (reasoningEffort) since most providers follow that API. */ -export type ThinkingConfigFor

= P extends 'anthropic' +export type ThinkingConfigFor

= P extends 'anthropic' | 'google-vertex-anthropic' ? AnthropicThinkingConfig : P extends 'google' ? GoogleThinkingConfig diff --git a/packages/@n8n/api-types/src/agents/provider-capabilities.ts b/packages/@n8n/api-types/src/agents/provider-capabilities.ts index e4e638e79c3..9163dda988c 100644 --- a/packages/@n8n/api-types/src/agents/provider-capabilities.ts +++ b/packages/@n8n/api-types/src/agents/provider-capabilities.ts @@ -78,6 +78,16 @@ export const PROVIDER_CAPABILITIES: Record = { providerTools: ['anthropic.web_search'], attachments: { image: true, pdf: true, audio: false }, }, + // Claude on Vertex — same Messages capabilities as Anthropic; providerTools + // stay empty because Vertex only exposes a subset and Instance AI does not + // wire Vertex-specific Anthropic provider tools yet. + 'google-vertex-anthropic': { + thinking: 'budgetTokens', + promptCaching: 'ttl', + webSearch: false, + providerTools: [], + attachments: { image: true, pdf: true, audio: false }, + }, openai: { thinking: 'reasoningEffort', promptCaching: true, @@ -93,7 +103,7 @@ export const PROVIDER_CAPABILITIES: Record = { attachments: { image: true, pdf: true, audio: true }, }, xai: { - thinking: false, + thinking: 'reasoningEffort', promptCaching: false, webSearch: false, providerTools: [], @@ -127,6 +137,14 @@ export const PROVIDER_CAPABILITIES: Record = { providerTools: [], attachments: NO_ATTACHMENTS, }, + // OpenAI-compatible custom endpoints (N8N_INSTANCE_AI_MODEL_URL / custom/*). + custom: { + thinking: 'reasoningEffort', + promptCaching: false, + webSearch: false, + providerTools: [], + attachments: NO_ATTACHMENTS, + }, cohere: { thinking: false, promptCaching: false, diff --git a/packages/@n8n/config/src/configs/instance-ai.config.ts b/packages/@n8n/config/src/configs/instance-ai.config.ts index 31449704fd9..283e7c7eb20 100644 --- a/packages/@n8n/config/src/configs/instance-ai.config.ts +++ b/packages/@n8n/config/src/configs/instance-ai.config.ts @@ -16,6 +16,27 @@ export class InstanceAiConfig { @Env('N8N_INSTANCE_AI_MODEL_API_KEY') modelApiKey: string = ''; + /** + * Google Cloud project for `google-vertex-anthropic/*` models. + * Falls back to `GOOGLE_VERTEX_PROJECT`, then `project_id` in the service-account JSON. + */ + @Env('N8N_INSTANCE_AI_VERTEX_PROJECT_ID') + vertexProjectId: string = ''; + + /** + * Vertex location for `google-vertex-anthropic/*` models (e.g. `global`, `us-east5`). + * Empty falls back to `GOOGLE_VERTEX_LOCATION`, then `global`. + */ + @Env('N8N_INSTANCE_AI_VERTEX_LOCATION') + vertexLocation: string = ''; + + /** + * Service-account JSON for `google-vertex-anthropic/*` models. + * Omit to use ADC (`gcloud auth application-default login`). + */ + @Env('N8N_INSTANCE_AI_VERTEX_SERVICE_ACCOUNT_JSON') + vertexServiceAccountJson: string = ''; + /** Comma-separated name=url pairs for MCP servers (e.g. "github=https://mcp.github.com/sse"). */ @Env('N8N_INSTANCE_AI_MCP_SERVERS') mcpServers: string = ''; diff --git a/packages/@n8n/config/test/config.test.ts b/packages/@n8n/config/test/config.test.ts index 3651aa8a4d0..91ca2570ca0 100644 --- a/packages/@n8n/config/test/config.test.ts +++ b/packages/@n8n/config/test/config.test.ts @@ -322,6 +322,9 @@ describe('GlobalConfig', () => { model: 'anthropic/claude-opus-4-8', modelUrl: '', modelApiKey: '', + vertexProjectId: '', + vertexLocation: '', + vertexServiceAccountJson: '', mcpServers: '', localGatewayDisabled: false, browserUseEnabled: true, diff --git a/packages/@n8n/instance-ai/docs/configuration.md b/packages/@n8n/instance-ai/docs/configuration.md index e4595a2f954..73961d3d395 100644 --- a/packages/@n8n/instance-ai/docs/configuration.md +++ b/packages/@n8n/instance-ai/docs/configuration.md @@ -11,9 +11,14 @@ persisted in settings takes precedence over `N8N_INSTANCE_AI_SANDBOX_PROVIDER`. | Variable | Type | Default | Description | |----------|------|---------|-------------| -| `N8N_INSTANCE_AI_MODEL` | string | `anthropic/claude-opus-4-8` | LLM model in `provider/model` format for built-in providers, or a bare model name when `N8N_INSTANCE_AI_MODEL_URL` is set. Must be set for the module to enable. | +| `N8N_INSTANCE_AI_MODEL` | string | `anthropic/claude-opus-4-8` | LLM model in `provider/model` format for built-in providers, or a bare model name when `N8N_INSTANCE_AI_MODEL_URL` is set. Must be set for the module to enable. Opus 5: `anthropic/claude-opus-5`. Vertex Claude: `google-vertex-anthropic/claude-opus-4-8`. | | `N8N_INSTANCE_AI_MODEL_URL` | string | `''` | Base URL for an OpenAI-compatible endpoint (e.g. `http://localhost:1234/v1` for LM Studio). When set, model requests go to this URL instead of the built-in provider. | | `N8N_INSTANCE_AI_MODEL_API_KEY` | string | `''` | API key for the custom model endpoint. Optional — some local servers don't require one. | +| `N8N_INSTANCE_AI_VERTEX_PROJECT_ID` | string | `''` | Google Cloud project for `google-vertex-anthropic/*`. Falls back to `GOOGLE_VERTEX_PROJECT`, then `project_id` in the service-account JSON. | +| `N8N_INSTANCE_AI_VERTEX_LOCATION` | string | `''` | Vertex location for `google-vertex-anthropic/*` (e.g. `global`, `us-east5`). Empty falls back to `GOOGLE_VERTEX_LOCATION`, then `global`. | +| `N8N_INSTANCE_AI_VERTEX_SERVICE_ACCOUNT_JSON` | string | `''` | Service-account JSON for Vertex Claude. Omit to use ADC (`gcloud auth application-default login`). | +| `N8N_INSTANCE_AI_REASONING_EFFORT` | string | unset | Optional reasoning effort for `custom/*` (`none`/`minimal`/`low`/`medium`/`high`/`xhigh`/`max`). Unset = known-model map in `src/utils/custom-model-defaults.ts`; still unresolved = omit. | +| `N8N_INSTANCE_AI_SUPPORTS_STRUCTURED_OUTPUTS` | string | unset | Optional `true`/`false` for `custom/*` structured-output support. Unset = known-model map; still unresolved = omit. | | `N8N_INSTANCE_AI_MCP_SERVERS` | string | `''` | Comma-separated MCP server configs. Format: `name=url,name=url` | | `N8N_INSTANCE_AI_LOCAL_GATEWAY_DISABLED` | boolean | `false` | Disable the local gateway (filesystem, shell, browser) for all users | @@ -32,6 +37,7 @@ persisted in settings takes precedence over `N8N_INSTANCE_AI_SANDBOX_PROVIDER`. |----------|------|---------|-------------| | `N8N_INSTANCE_AI_RUN_DEBUG_ENABLED` | boolean | `false` | Capture orchestrator LLM steps and workflow code snapshots for the dev debug panel and eval LLM debug reports. | | `N8N_INSTANCE_AI_EVAL_TIMING` | boolean | `false` | When `true`, logs a per-execution `[EvalMock][timing]` phase breakdown (hints / bypass-pin / http-mock / ai-turn) for the eval mock-execution path, to attribute mocked-execution latency. A no-op otherwise. | +| `EVAL_MODAL_LLM_HEADERS` | string | `''` | Eval-only JSON object of extra HTTP headers for Modal (or other custom) LLM endpoints. Used by eval helpers (`createEvalAgent`, mock generation, verification) with `N8N_INSTANCE_AI_MODEL_URL`. | ### Memory @@ -235,6 +241,13 @@ N8N_INSTANCE_AI_GATEWAY_API_KEY=my-secret-key N8N_INSTANCE_AI_MODEL=your-tool-capable-model N8N_INSTANCE_AI_MODEL_URL=http://localhost:1234/v1 +# Direct Google Vertex Claude (no AI assistant proxy) +# Keep N8N_AI_ASSISTANT_BASE_URL unset/empty. +N8N_INSTANCE_AI_MODEL=google-vertex-anthropic/claude-opus-4-8 +N8N_INSTANCE_AI_VERTEX_PROJECT_ID=my-gcp-project +N8N_INSTANCE_AI_VERTEX_LOCATION=global +N8N_INSTANCE_AI_VERTEX_SERVICE_ACCOUNT_JSON='{"type":"service_account",...}' + # Output filtering — secrets + email only, with a custom placeholder N8N_INSTANCE_AI_OUTPUT_REDACTION_PII=email N8N_INSTANCE_AI_OUTPUT_REDACTION_PLACEHOLDER=‹redacted› diff --git a/packages/@n8n/instance-ai/eslint.config.mjs b/packages/@n8n/instance-ai/eslint.config.mjs index 1d0558b9c5a..804783b3861 100644 --- a/packages/@n8n/instance-ai/eslint.config.mjs +++ b/packages/@n8n/instance-ai/eslint.config.mjs @@ -28,6 +28,8 @@ export default defineConfig( // Local eval scratch output — never linted, never committed. '.data/**', 'evaluations/.data/**', + 'evaluations/.output/**', + '.output/**', // Deep-imports ai-workflow-builder.ee's evaluations source, so it sits outside // evaluations/tsconfig.json (see its exclude) and the eslint project service. 'evaluations/cli/pairwise.ts', diff --git a/packages/@n8n/instance-ai/evaluations/README.md b/packages/@n8n/instance-ai/evaluations/README.md index 12ad2fde739..92a068d60ab 100644 --- a/packages/@n8n/instance-ai/evaluations/README.md +++ b/packages/@n8n/instance-ai/evaluations/README.md @@ -296,6 +296,8 @@ Not yet covered: an automatic "unexpected artifact" fail (a build producing an a |----------|----------|-------------| | `N8N_INSTANCE_AI_MODEL` | Yes | Model used by Instance AI and, by default, the eval helper calls for mock generation and verification | | `N8N_INSTANCE_AI_MODEL_API_KEY` | No | Generic eval-model API key override | +| `N8N_INSTANCE_AI_MODEL_URL` | No | OpenAI-compatible base URL for custom eval models (used with `custom/...` model ids) | +| `EVAL_MODAL_LLM_HEADERS` | No | Eval-only JSON object of extra HTTP headers for Modal (or other custom) LLM endpoints | | `OPENAI_API_KEY` | No | Provider-specific key used automatically when `N8N_INSTANCE_AI_MODEL` starts with `openai/` | | `ANTHROPIC_API_KEY` | No | Provider-specific key used automatically when `N8N_INSTANCE_AI_MODEL` starts with `anthropic/` | | `N8N_EVAL_EMAIL` | No | n8n login email (defaults to E2E test owner) | @@ -1000,6 +1002,32 @@ Evals run automatically on PRs that change Instance AI code (path-filtered). The The job is **non-blocking**. Results are posted as a PR comment and uploaded as artifacts. When `LANGSMITH_API_KEY` is set via the `EVALS_LANGSMITH_API_KEY` secret, runs also land as LangSmith experiments tagged with commit SHA + branch, so you can compare against master side-by-side. +For model A/B experiments, dispatch **Instance AI Evals: Experiments** (`test-evals-instance-ai.yml`). Native providers use `model` alone (`anthropic/*`, `openai/*`, `openrouter/*`, `xai/*`, `google-vertex-anthropic/*`). OpenAI-compatible vendors use **`custom/` + `model-url` + `model-key`** — no first-class provider prefixes. + +| Experiment | `model` | `model-url` | `model-key` → secret | +|------------|---------|-------------|----------------------| +| Anthropic / OpenAI / OpenRouter / xAI | `anthropic/…`, `openai/…`, etc. | empty | (prefix → `EVALS_*`) | +| Google Vertex Claude | `google-vertex-anthropic/claude-opus-4-8` | empty | (prefix → `EVALS_VERTEX_KEY` + `EVALS_VERTEX_PROJECT_ID`; optional `EVALS_VERTEX_LOCATION`, default `global`) | +| Baseten | `custom/` | `https://inference.baseten.co/v1` | `baseten` → `EVALS_BASETEN_KEY` | +| Fireworks | `custom/accounts/fireworks/models/…` | `https://api.fireworks.ai/inference/v1` | `fireworks` → `EVALS_FIREWORKS_KEY` | +| Together | `custom/moonshotai/Kimi-K3` | `https://api.together.ai/v1` | `together` → `EVALS_TOGETHER_KEY` | +| Modal | `custom/…` | `https://….modal.direct…/v1` | `modal` → `EVALS_MODAL_KEY` | +| Databricks | `custom/workspace.default.kimi-k3` | `https://….databricks.com/ai-gateway/mlflow/v1` | `databricks` → `EVALS_DATABRICKS_KEY` | +| Lyceum | `custom/moonshotai/Kimi-K3` | OpenAI-compatible `/v1` base URL | `lyceum` → `EVALS_LYCEUM_KEY` | +| Azure OpenAI | `custom/` | `https://….openai.azure.com/openai/v1` | `azure` → `EVALS_AZURE_FOUNDRY_KEY` | +| Keyless custom router | `custom/` | `https://host/v1` | empty (no API key) | +| Azure Foundry Claude | `anthropic/` | Foundry Anthropic base | `azure` (or omit — defaults to Foundry key) | + +`lanes` / `eval-concurrency` default to **10 / 32**. For `model-key=baseten` they auto-throttle to **1 / 2** (~0.5M TPM — fits [Baseten Basic verified](https://docs.baseten.co/inference/model-apis/rate-limits-and-budgets)); override the inputs if you have more headroom. + +Verifier/mocks always use `EVALS_ANTHROPIC_KEY`. + +For `custom/*`, optional dispatch inputs `reasoning-effort` and `supports-structured-outputs` +override `N8N_INSTANCE_AI_REASONING_EFFORT` / `N8N_INSTANCE_AI_SUPPORTS_STRUCTURED_OUTPUTS`. +When unset, the runtime looks up +`packages/@n8n/instance-ai/src/utils/custom-model-defaults.ts` (substring match on the model id). +If still unresolved, the field is omitted from the request (no blanket custom default). + ## Architecture ``` diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/checklist-verifier.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/checklist-verifier.test.ts index a6bd02d3f9c..5466fd7ef06 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/checklist-verifier.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/checklist-verifier.test.ts @@ -14,6 +14,7 @@ vi.mock('../../src/utils/eval-agents', () => ({ import { createEvalAgent, resolveEvalModelConfig } from '../../src/utils/eval-agents'; import { + VERIFIER_MAX_OUTPUT_TOKENS, VERIFY_ATTEMPT_TIMEOUTS_MS, VERIFY_INACTIVITY_TIMEOUT_MS, verifyChecklist, @@ -39,7 +40,12 @@ const GOOD_OUTPUT = { ], }; -type StreamFn = Mock<(messages: unknown, opts: { abortSignal: AbortSignal }) => Promise>; +type StreamFn = Mock< + ( + messages: unknown, + opts: { abortSignal: AbortSignal; maxOutputTokens?: number }, + ) => Promise +>; /** Wire createEvalAgent().structuredOutput().stream() to the given stream mock. */ function mockVerifierAgent(stream: StreamFn): void { @@ -155,6 +161,10 @@ describe('verifyChecklist (agents stream path)', () => { expect(attempts).toHaveLength(1); expect(attempts[0].status).toBe('success'); expect(stream).toHaveBeenCalledTimes(1); + expect(stream).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ maxOutputTokens: VERIFIER_MAX_OUTPUT_TOKENS }), + ); }); it('falls back to parsing the streamed text when no structured output arrives', async () => { diff --git a/packages/@n8n/instance-ai/evaluations/checklist/verifier.ts b/packages/@n8n/instance-ai/evaluations/checklist/verifier.ts index acfffc97d58..b2a9b136bce 100644 --- a/packages/@n8n/instance-ai/evaluations/checklist/verifier.ts +++ b/packages/@n8n/instance-ai/evaluations/checklist/verifier.ts @@ -36,6 +36,12 @@ export const VERIFY_ATTEMPT_TIMEOUTS_MS = [60_000, 120_000, 240_000]; /** Abort when the stream goes silent for this long AFTER its first chunk (agents path only). * Pre-first-chunk time is bounded by the attempt cap, not this window. */ export const VERIFY_INACTIVITY_TIMEOUT_MS = 45_000; +/** + * Completion budget for checklist JSON. Some OpenAI-compatible defaults (~250) + * truncate mid-result (~298 tokens observed); 4096 leaves headroom for thinking + * tokens plus multi-item checklists. + */ +export const VERIFIER_MAX_OUTPUT_TOKENS = 4_096; const VERIFIER_DEBUG = process.env.N8N_EVAL_VERIFIER_DEBUG === '1'; function jitteredPauseMs(attempt: number): number { @@ -168,6 +174,7 @@ async function runNativeOpenAiVerifier( const model = resolveEvalModelConfig(); const requestBody = { model: model.providerModelId, + max_output_tokens: VERIFIER_MAX_OUTPUT_TOKENS, ...(supportsOpenAiReasoning(model.providerModelId) ? { reasoning: { effort: 'high' } } : {}), input: [ { @@ -455,6 +462,7 @@ export async function verifyChecklist( const streamResult = await agent.stream(messages, { abortSignal: abortController.signal, smoothStream: false, + maxOutputTokens: VERIFIER_MAX_OUTPUT_TOKENS, }); const streamed = await consumeVerifierStream( streamResult.stream, diff --git a/packages/@n8n/instance-ai/evaluations/harness/cleanup.ts b/packages/@n8n/instance-ai/evaluations/harness/cleanup.ts index 9ea0be027ce..102d187c056 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/cleanup.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/cleanup.ts @@ -246,9 +246,9 @@ export async function runWorkflowChecks(args: { } function hasAnthropicKey(): boolean { - return Boolean( - process.env.N8N_INSTANCE_AI_MODEL_API_KEY ?? - process.env.N8N_AI_ANTHROPIC_KEY ?? - process.env.ANTHROPIC_API_KEY, - ); + return [ + process.env.N8N_AI_ANTHROPIC_KEY, + process.env.ANTHROPIC_API_KEY, + process.env.N8N_INSTANCE_AI_MODEL_API_KEY, + ].some((value) => Boolean(value?.trim())); } diff --git a/packages/@n8n/instance-ai/skills/workflow-builder/SKILL.md b/packages/@n8n/instance-ai/skills/workflow-builder/SKILL.md index 5bbba855399..c519f7d5886 100644 --- a/packages/@n8n/instance-ai/skills/workflow-builder/SKILL.md +++ b/packages/@n8n/instance-ai/skills/workflow-builder/SKILL.md @@ -476,7 +476,8 @@ unsolicited `sticky()`, forbidden builder constructs (e.g. `.map()`), and repeated `.onTrue()` / `.onFalse()` overwrites on the same IF variable. Fix every reported error and warning before calling `build-workflow`. -- Code nodes need not always be necessary. You can use other n8n nodes to do the same thing. +- Avoid code node where possible, use n8n nodes that help do the same thing. + If it makes it simpler, go ahead and use code node. - SDK builder code is a restricted subset of TypeScript that builds a static graph; it is not a Code node and does not run. Build strings with template literals; do runtime joining, aggregation, or transforms in a Code node or diff --git a/packages/@n8n/instance-ai/src/agent/__tests__/apply-agent-thinking.test.ts b/packages/@n8n/instance-ai/src/agent/__tests__/apply-agent-thinking.test.ts index d2f1108cfca..eaede18763e 100644 --- a/packages/@n8n/instance-ai/src/agent/__tests__/apply-agent-thinking.test.ts +++ b/packages/@n8n/instance-ai/src/agent/__tests__/apply-agent-thinking.test.ts @@ -22,16 +22,25 @@ describe('applyAgentThinking', () => { vi.clearAllMocks(); }); - it('enables adaptive thinking for Anthropic', () => { + it('enables adaptive thinking with medium effort for Anthropic', () => { const agent = new Agent('test'); - applyAgentThinking(agent, 'anthropic/claude-opus-4-8'); + applyAgentThinking(agent, 'anthropic/claude-opus-5'); expect(mockAgentInstances[0]?.thinking).toHaveBeenCalledWith('anthropic', { mode: 'adaptive', effort: 'medium', }); }); - it('enables adaptive thinking for dotted Anthropic provider IDs', () => { + it('enables adaptive thinking with medium effort for Google Vertex Anthropic', () => { + const agent = new Agent('test'); + applyAgentThinking(agent, 'google-vertex-anthropic/claude-opus-4-8'); + expect(mockAgentInstances[0]?.thinking).toHaveBeenCalledWith('google-vertex-anthropic', { + mode: 'adaptive', + effort: 'medium', + }); + }); + + it('enables adaptive thinking with medium effort for dotted Anthropic provider IDs', () => { const agent = new Agent('test'); applyAgentThinking(agent, 'anthropic.messages/claude-opus-4-8'); expect(mockAgentInstances[0]?.thinking).toHaveBeenCalledWith('anthropic', { @@ -40,7 +49,7 @@ describe('applyAgentThinking', () => { }); }); - it('enables adaptive thinking for AI SDK Anthropic model objects', () => { + it('enables adaptive thinking with medium effort for AI SDK Anthropic model objects', () => { const agent = new Agent('test'); applyAgentThinking(agent, { modelId: 'claude-opus-4-8', @@ -52,17 +61,85 @@ describe('applyAgentThinking', () => { }); }); - it('enables OpenAI reasoning for supported models', () => { + it.each(['openai/gpt-5.6-sol', 'openai/gpt-5.6-terra', 'openai/gpt-5.6-luna'] as const)( + 'enables medium OpenAI reasoning effort for %s', + (modelId) => { + const agent = new Agent('test'); + applyAgentThinking(agent, modelId); + expect(mockAgentInstances[0]?.thinking).toHaveBeenCalledWith('openai', { + reasoningEffort: 'medium', + }); + }, + ); + + it('enables high OpenAI reasoning effort for non-GPT-5.6 models', () => { const agent = new Agent('test'); - applyAgentThinking(agent, 'openai/gpt-5.5'); + applyAgentThinking(agent, 'openai/gpt-4.1'); expect(mockAgentInstances[0]?.thinking).toHaveBeenCalledWith('openai', { reasoningEffort: 'high', }); }); + it('enables mapped low reasoning effort for known custom Kimi K3 models', () => { + const agent = new Agent('test'); + applyAgentThinking(agent, 'custom/Kimi-K3'); + expect(mockAgentInstances[0]?.thinking).toHaveBeenCalledWith('custom', { + reasoningEffort: 'low', + }); + }); + + it('enables mapped low reasoning effort for Databricks AI Gateway Kimi-K3', () => { + const agent = new Agent('test'); + applyAgentThinking(agent, 'custom/workspace.default.kimi-k3'); + expect(mockAgentInstances[0]?.thinking).toHaveBeenCalledWith('custom', { + reasoningEffort: 'low', + }); + }); + + it('skips thinking for unknown custom models without an env override', () => { + const agent = new Agent('test'); + applyAgentThinking(agent, 'custom/unknown-model'); + expect(mockAgentInstances[0]?.thinking).not.toHaveBeenCalled(); + }); + + it('prefers N8N_INSTANCE_AI_REASONING_EFFORT over the known-model map', () => { + const previous = process.env.N8N_INSTANCE_AI_REASONING_EFFORT; + process.env.N8N_INSTANCE_AI_REASONING_EFFORT = 'high'; + try { + const agent = new Agent('test'); + applyAgentThinking(agent, 'custom/Kimi-K3'); + expect(mockAgentInstances[0]?.thinking).toHaveBeenCalledWith('custom', { + reasoningEffort: 'high', + }); + } finally { + if (previous === undefined) delete process.env.N8N_INSTANCE_AI_REASONING_EFFORT; + else process.env.N8N_INSTANCE_AI_REASONING_EFFORT = previous; + } + }); + it('skips providers without thinking support', () => { const agent = new Agent('test'); applyAgentThinking(agent, 'google/gemini-2.5-pro'); expect(mockAgentInstances[0]?.thinking).not.toHaveBeenCalled(); }); + + it('skips OpenRouter models (thinking unsupported)', () => { + const agent = new Agent('test'); + applyAgentThinking(agent, 'openrouter/moonshotai/kimi-k3'); + expect(mockAgentInstances[0]?.thinking).not.toHaveBeenCalled(); + }); + + it('enables medium reasoning effort for Grok 4.5 via xAI', () => { + const agent = new Agent('test'); + applyAgentThinking(agent, 'xai/grok-4.5'); + expect(mockAgentInstances[0]?.thinking).toHaveBeenCalledWith('xai', { + reasoningEffort: 'medium', + }); + }); + + it('skips xAI models that are not Grok 4.5', () => { + const agent = new Agent('test'); + applyAgentThinking(agent, 'xai/grok-3'); + expect(mockAgentInstances[0]?.thinking).not.toHaveBeenCalled(); + }); }); diff --git a/packages/@n8n/instance-ai/src/agent/__tests__/instance-agent.test.ts b/packages/@n8n/instance-ai/src/agent/__tests__/instance-agent.test.ts index deb87ea98e2..2fdda2b719b 100644 --- a/packages/@n8n/instance-ai/src/agent/__tests__/instance-agent.test.ts +++ b/packages/@n8n/instance-ai/src/agent/__tests__/instance-agent.test.ts @@ -807,4 +807,44 @@ describe('createInstanceAgent', () => { expect(mockAgentInstances[0]?.mcpConnectionFailures).not.toHaveBeenCalled(); }); + + it('sticks Modal endpoint requests to the conversation thread via Modal-Session-ID', async () => { + // Sub-agents (build-agent, eval-setup, etc.) reuse context.modelId / orchestration + // context.modelId, so mutating the sticky header here covers those paths too. + const modalModel = { + id: 'custom/moonshotai/Kimi-K3' as const, + url: 'https://n8ngmbh--ep-kimi-k3-server.us-west.modal.direct/v1', + apiKey: 'wk-test.ws-test', + }; + const context = { + threadId: 'thread-sticky-1', + runLabel: 'modal-sticky-run', + localGatewayStatus: undefined, + licenseHints: undefined, + localMcpServer: undefined, + modelId: modalModel, + }; + const orchestrationContext = { + runId: 'modal-sticky-run', + threadId: 'thread-sticky-1', + modelId: modalModel, + }; + + await createInstanceAgent({ + modelId: modalModel, + context, + orchestrationContext, + memoryConfig: {}, + mcpManager: createMcpManagerStub(), + thinkingEnabled: false, + } as never); + + const stickyModel = { + ...modalModel, + headers: { 'Modal-Session-ID': 'thread-sticky-1' }, + }; + expect(mockAgentInstances[0]?.model).toHaveBeenCalledWith(stickyModel); + expect(context.modelId).toEqual(stickyModel); + expect(orchestrationContext.modelId).toEqual(stickyModel); + }); }); diff --git a/packages/@n8n/instance-ai/src/agent/apply-agent-thinking.ts b/packages/@n8n/instance-ai/src/agent/apply-agent-thinking.ts index acb03e823a3..879179669ea 100644 --- a/packages/@n8n/instance-ai/src/agent/apply-agent-thinking.ts +++ b/packages/@n8n/instance-ai/src/agent/apply-agent-thinking.ts @@ -2,6 +2,8 @@ import type { Agent, ModelConfig } from '@n8n/agents'; import { PROVIDER_CAPABILITIES } from '@n8n/api-types'; import { isRecord } from '@n8n/utils/is-record'; +import { resolveCustomModelExperimentDefaultsFromEnv } from '../utils/custom-model-defaults'; + function normalizeProvider(provider: string): string { return provider.split('.')[0] ?? provider; } @@ -33,18 +35,60 @@ function resolveModelProvider(modelId: ModelConfig): string | undefined { return provider && model ? normalizeProvider(provider) : undefined; } +function resolveModelIdString(modelId: ModelConfig): string | undefined { + if (typeof modelId === 'string') return modelId; + if (!isRecord(modelId)) return undefined; + + const id = getStringProperty(modelId, 'id'); + if (id) return id; + + const provider = getStringProperty(modelId, 'provider') ?? getProviderFromConfig(modelId); + const model = getStringProperty(modelId, 'modelId'); + return provider && model ? `${provider}/${model}` : undefined; +} + +/** Grok 4.5 via xAI (`xai/grok-4.5`). */ +function isGrok45Model(modelId: ModelConfig): boolean { + const id = resolveModelIdString(modelId)?.toLowerCase() ?? ''; + return id.includes('grok-4.5'); +} + +/** GPT-5.6 family via OpenAI (`openai/gpt-5.6-sol`, `openai/gpt-5.6-terra`, `openai/gpt-5.6-luna`). */ +function isGpt56Model(modelId: ModelConfig): boolean { + const id = resolveModelIdString(modelId)?.toLowerCase() ?? ''; + return id.includes('gpt-5.6'); +} + export function applyAgentThinking(agent: Agent, modelId: ModelConfig): void { const provider = resolveModelProvider(modelId); if (!provider || !PROVIDER_CAPABILITIES[provider]?.thinking) return; - if (provider === 'openai') { - agent.thinking('openai', { reasoningEffort: 'high' }); + if (provider === 'custom') { + // No blanket custom default: env override → known-model map → omit. + const resolvedModelId = resolveModelIdString(modelId) ?? ''; + const { reasoningEffort } = resolveCustomModelExperimentDefaultsFromEnv(resolvedModelId); + if (reasoningEffort !== undefined) { + agent.thinking('custom', { reasoningEffort }); + } return; } - if (provider === 'anthropic') { - agent.thinking('anthropic', { mode: 'adaptive', effort: 'medium' }); + if (provider === 'openai') { + agent.thinking('openai', { + reasoningEffort: isGpt56Model(modelId) ? 'medium' : 'high', + }); return; } + + if (provider === 'anthropic' || provider === 'google-vertex-anthropic') { + agent.thinking(provider, { mode: 'adaptive', effort: 'medium' }); + return; + } + + if (provider === 'xai') { + if (isGrok45Model(modelId)) { + agent.thinking('xai', { reasoningEffort: 'medium' }); + } + } } diff --git a/packages/@n8n/instance-ai/src/agent/instance-agent.ts b/packages/@n8n/instance-ai/src/agent/instance-agent.ts index 78d3170cbc0..bbcfd840e94 100644 --- a/packages/@n8n/instance-ai/src/agent/instance-agent.ts +++ b/packages/@n8n/instance-ai/src/agent/instance-agent.ts @@ -24,7 +24,26 @@ import type { CreateInstanceAgentOptions, InstanceAiContext, InstanceAiToolRegistry, + ModelConfig, } from '../types'; +import { withModalSession } from '../utils/modal-session'; + +function resolveModalSessionModelId( + modelId: ModelConfig, + context: InstanceAiContext, + orchestrationThreadId: string | undefined, +): ModelConfig { + const threadId = [context.threadId, orchestrationThreadId] + .map((value) => value?.trim()) + .find((value): value is string => value !== undefined && value.length > 0); + if (!threadId) return modelId; + + const stickyModelId = withModalSession(modelId, threadId); + if (stickyModelId === modelId) return modelId; + + context.modelId = stickyModelId; + return stickyModelId; +} // ── Agent factory ─────────────────────────────────────────────────────────── @@ -53,7 +72,7 @@ export async function createInstanceAgent( options: CreateInstanceAgentOptions, ): Promise<{ agent: Agent; mcpConnectionFailures: Array<{ server: string; error: string }> }> { const { - modelId, + modelId: rawModelId, context, orchestrationContext, mcpServers = [], @@ -61,6 +80,11 @@ export async function createInstanceAgent( memoryConfig, } = options; + const modelId = resolveModalSessionModelId(rawModelId, context, orchestrationContext?.threadId); + if (orchestrationContext && orchestrationContext.modelId !== modelId) { + orchestrationContext.modelId = modelId; + } + // Build native n8n domain tools (context captured via closures — per-run). // Thread the trace handle in so domain tools (e.g. build-workflow) can emit // explicit child runs that land on the active trace — orchestration tools diff --git a/packages/@n8n/instance-ai/src/index.ts b/packages/@n8n/instance-ai/src/index.ts index 6625c931ff6..dacf3205a01 100644 --- a/packages/@n8n/instance-ai/src/index.ts +++ b/packages/@n8n/instance-ai/src/index.ts @@ -182,6 +182,8 @@ const loadValidateAttachments = lazyModule( ); export { MAX_STEPS } from './constants/max-steps'; +export { parseModelHeadersJson } from './utils/parse-model-headers'; +export { resolveCustomModelExperimentDefaultsFromEnv } from './utils/custom-model-defaults'; export { WorkflowSaveConflictError } from './errors/workflow-save-conflict.error'; export { WorkflowNotFoundError } from './errors/workflow-not-found.error'; export { @@ -612,6 +614,7 @@ export type { LocalMcpServer, McpServerConfig, ModelConfig, + VertexAnthropicModelConfig, InstanceAiMemoryConfig, CreateInstanceAgentOptions, TaskStorage, diff --git a/packages/@n8n/instance-ai/src/types.ts b/packages/@n8n/instance-ai/src/types.ts index 6d00fc2f65c..3d44d2c4ef5 100644 --- a/packages/@n8n/instance-ai/src/types.ts +++ b/packages/@n8n/instance-ai/src/types.ts @@ -1400,16 +1400,22 @@ export interface InstanceAiMemoryConfig { type NativeLanguageModelConfig = Extract; +/** Direct Google Vertex Claude (Anthropic Messages via `:rawPredict`). */ +export type VertexAnthropicModelConfig = { + id: `google-vertex-anthropic/${string}`; + project: string; + location: string; + googleCredentials?: string; +}; + /** Model identifier: plain string for built-in providers, object for OpenAI-compatible endpoints, - * or a pre-built LanguageModel instance (e.g. from @ai-sdk/anthropic with a custom baseURL). + * Vertex Claude, or a pre-built LanguageModel instance (e.g. from @ai-sdk/anthropic with a custom baseURL). * - * The LanguageModel variant exists for proxy routes that need a provider-native transport. - * For example, Vertex AI Anthropic routes use the native Messages API at `/v1/messages`, so - * we must use `@ai-sdk/anthropic` directly instead of routing through an OpenAI-compatible - * `/chat/completions` adapter. */ + * The LanguageModel variant exists for proxy routes that need a provider-native transport. */ export type ModelConfig = | string | { id: `${string}/${string}`; url: string; apiKey?: string; headers?: Record } + | VertexAnthropicModelConfig | NativeLanguageModelConfig; /** Configuration for routing requests through an AI service proxy (LangSmith tracing, Brave Search, etc.). */ diff --git a/packages/@n8n/instance-ai/src/utils/__tests__/custom-model-defaults.test.ts b/packages/@n8n/instance-ai/src/utils/__tests__/custom-model-defaults.test.ts new file mode 100644 index 00000000000..77f75e0d0a4 --- /dev/null +++ b/packages/@n8n/instance-ai/src/utils/__tests__/custom-model-defaults.test.ts @@ -0,0 +1,102 @@ +import { + parseReasoningEffort, + parseSupportsStructuredOutputs, + resolveCustomModelExperimentDefaults, + resolveCustomModelExperimentDefaultsFromEnv, +} from '../custom-model-defaults'; + +describe('resolveCustomModelExperimentDefaults', () => { + it('maps known Kimi K3 models to low effort and structured outputs', () => { + expect(resolveCustomModelExperimentDefaults('custom/moonshotai/Kimi-K3')).toEqual({ + reasoningEffort: 'low', + supportsStructuredOutputs: true, + }); + }); + + it('maps GLM 5.2 models to medium effort', () => { + expect(resolveCustomModelExperimentDefaults('custom/zai-org/GLM-5.2-Fast')).toEqual({ + reasoningEffort: 'medium', + supportsStructuredOutputs: true, + }); + }); + + it('maps DeepSeek models to structured outputs only', () => { + expect(resolveCustomModelExperimentDefaults('custom/deepseek-ai/DeepSeek-V4-Pro')).toEqual({ + supportsStructuredOutputs: true, + }); + }); + + it('returns nothing for unknown custom models', () => { + expect(resolveCustomModelExperimentDefaults('custom/unknown-model')).toEqual({}); + }); + + it('lets explicit overrides win over the model map', () => { + expect( + resolveCustomModelExperimentDefaults('custom/moonshotai/Kimi-K3', { + reasoningEffort: 'high', + supportsStructuredOutputs: false, + }), + ).toEqual({ + reasoningEffort: 'high', + supportsStructuredOutputs: false, + }); + }); +}); + +describe('resolveCustomModelExperimentDefaultsFromEnv', () => { + const previousEffort = process.env.N8N_INSTANCE_AI_REASONING_EFFORT; + const previousStructured = process.env.N8N_INSTANCE_AI_SUPPORTS_STRUCTURED_OUTPUTS; + + afterEach(() => { + if (previousEffort === undefined) delete process.env.N8N_INSTANCE_AI_REASONING_EFFORT; + else process.env.N8N_INSTANCE_AI_REASONING_EFFORT = previousEffort; + if (previousStructured === undefined) { + delete process.env.N8N_INSTANCE_AI_SUPPORTS_STRUCTURED_OUTPUTS; + } else { + process.env.N8N_INSTANCE_AI_SUPPORTS_STRUCTURED_OUTPUTS = previousStructured; + } + }); + + it('reads env overrides ahead of the model map', () => { + process.env.N8N_INSTANCE_AI_REASONING_EFFORT = 'high'; + process.env.N8N_INSTANCE_AI_SUPPORTS_STRUCTURED_OUTPUTS = 'false'; + expect(resolveCustomModelExperimentDefaultsFromEnv('custom/moonshotai/Kimi-K3')).toEqual({ + reasoningEffort: 'high', + supportsStructuredOutputs: false, + }); + }); + + it('falls back to the model map when env is unset', () => { + delete process.env.N8N_INSTANCE_AI_REASONING_EFFORT; + delete process.env.N8N_INSTANCE_AI_SUPPORTS_STRUCTURED_OUTPUTS; + expect(resolveCustomModelExperimentDefaultsFromEnv('custom/moonshotai/Kimi-K3')).toEqual({ + reasoningEffort: 'low', + supportsStructuredOutputs: true, + }); + }); + + it('falls back to the model map when env overrides are non-empty but invalid', () => { + process.env.N8N_INSTANCE_AI_REASONING_EFFORT = 'nope'; + process.env.N8N_INSTANCE_AI_SUPPORTS_STRUCTURED_OUTPUTS = 'maybe'; + + expect(resolveCustomModelExperimentDefaultsFromEnv('custom/moonshotai/Kimi-K3')).toEqual({ + reasoningEffort: 'low', + supportsStructuredOutputs: true, + }); + }); +}); + +describe('parse helpers', () => { + it('parses reasoning effort values', () => { + expect(parseReasoningEffort(' medium ')).toBe('medium'); + expect(parseReasoningEffort('')).toBeUndefined(); + expect(parseReasoningEffort('nope')).toBeUndefined(); + }); + + it('parses structured-output flags', () => { + expect(parseSupportsStructuredOutputs('true')).toBe(true); + expect(parseSupportsStructuredOutputs('0')).toBe(false); + expect(parseSupportsStructuredOutputs('')).toBeUndefined(); + expect(parseSupportsStructuredOutputs('maybe')).toBeUndefined(); + }); +}); diff --git a/packages/@n8n/instance-ai/src/utils/__tests__/eval-agents.test.ts b/packages/@n8n/instance-ai/src/utils/__tests__/eval-agents.test.ts index 986e2c9d1b5..cbab43e6388 100644 --- a/packages/@n8n/instance-ai/src/utils/__tests__/eval-agents.test.ts +++ b/packages/@n8n/instance-ai/src/utils/__tests__/eval-agents.test.ts @@ -25,6 +25,8 @@ const MODEL_ENV_KEYS = [ 'N8N_INSTANCE_AI_MODEL', 'N8N_INSTANCE_AI_EVAL_MODEL', 'N8N_INSTANCE_AI_MODEL_API_KEY', + 'N8N_INSTANCE_AI_MODEL_URL', + 'EVAL_MODAL_LLM_HEADERS', 'N8N_AI_ANTHROPIC_KEY', 'ANTHROPIC_API_KEY', 'OPENAI_API_KEY', @@ -73,12 +75,12 @@ describe('eval agent model config', () => { process.env.OPENAI_API_KEY = 'openai-key'; createEvalAgent('test-agent', { - model: 'openai/gpt-5.5', + model: 'openai/gpt-5.6-sol', instructions: 'Do the task.', }); expect(mockAgentInstances[0]?.thinking).toHaveBeenCalledWith('openai', { - reasoningEffort: 'high', + reasoningEffort: 'medium', }); }); @@ -121,4 +123,91 @@ describe('eval agent model config', () => { url: undefined, }); }); + + it('supports custom endpoints that authenticate with EVAL_MODAL_LLM_HEADERS only', () => { + process.env.N8N_INSTANCE_AI_MODEL = 'custom/moonshotai/Kimi-K3'; + process.env.N8N_INSTANCE_AI_MODEL_URL = 'https://example.modal.direct/v1'; + process.env.EVAL_MODAL_LLM_HEADERS = '{"Modal-Key":"wk-test","Modal-Secret":"ws-test"}'; + + const config = resolveEvalModelConfig(); + + expect(config).toMatchObject({ + modelId: 'custom/moonshotai/Kimi-K3', + apiKey: '', + url: 'https://example.modal.direct/v1', + headers: { 'Modal-Key': 'wk-test', 'Modal-Secret': 'ws-test' }, + }); + + createEvalAgent('test-agent', { instructions: 'Do the task.' }); + + expect(mockAgentInstances[0]?.model).toHaveBeenCalledWith({ + id: 'custom/moonshotai/Kimi-K3', + apiKey: '', + url: 'https://example.modal.direct/v1', + headers: { 'Modal-Key': 'wk-test', 'Modal-Secret': 'ws-test' }, + }); + }); + + it('supports keyless custom/* OpenAI-compatible routers (URL only, no API key)', () => { + process.env.N8N_INSTANCE_AI_MODEL = 'custom/Kimi-K3'; + process.env.N8N_INSTANCE_AI_MODEL_URL = 'https://router.example.com/v1'; + + const config = resolveEvalModelConfig(); + + expect(config).toMatchObject({ + modelId: 'custom/Kimi-K3', + provider: 'custom', + providerModelId: 'Kimi-K3', + apiKey: '', + url: 'https://router.example.com/v1', + }); + + createEvalAgent('test-agent', { instructions: 'Do the task.' }); + + expect(mockAgentInstances[0]?.model).toHaveBeenCalledWith({ + id: 'custom/Kimi-K3', + apiKey: '', + url: 'https://router.example.com/v1', + }); + }); + + it('still requires an API key for custom/* without a model URL', () => { + process.env.N8N_INSTANCE_AI_MODEL = 'custom/Kimi-K3'; + + expect(() => resolveEvalModelConfig()).toThrow( + /Missing API key for eval model "custom\/Kimi-K3"/, + ); + }); + + it('keeps Anthropic eval model separate from a custom/* builder (no builder URL/key leak)', () => { + process.env.N8N_INSTANCE_AI_MODEL = 'custom/Kimi-K3'; + process.env.N8N_INSTANCE_AI_MODEL_URL = 'https://router.example.com/v1'; + process.env.N8N_INSTANCE_AI_MODEL_API_KEY = ''; + process.env.N8N_INSTANCE_AI_EVAL_MODEL = 'anthropic/claude-sonnet-4-6'; + process.env.ANTHROPIC_API_KEY = 'anthropic-eval-key'; + + const config = resolveEvalModelConfig(); + + expect(config).toEqual({ + modelId: 'anthropic/claude-sonnet-4-6', + provider: 'anthropic', + providerModelId: 'claude-sonnet-4-6', + apiKey: 'anthropic-eval-key', + url: undefined, + headers: undefined, + }); + }); + + it('prefers Anthropic keys over a non-Anthropic builder MODEL_API_KEY for the eval model', () => { + process.env.N8N_INSTANCE_AI_MODEL = 'openai/gpt-5.6-sol'; + process.env.N8N_INSTANCE_AI_MODEL_API_KEY = 'openai-builder-key'; + process.env.N8N_INSTANCE_AI_EVAL_MODEL = 'anthropic/claude-sonnet-4-6'; + process.env.ANTHROPIC_API_KEY = 'anthropic-eval-key'; + + expect(resolveEvalModelConfig()).toMatchObject({ + modelId: 'anthropic/claude-sonnet-4-6', + apiKey: 'anthropic-eval-key', + url: undefined, + }); + }); }); diff --git a/packages/@n8n/instance-ai/src/utils/__tests__/modal-session.test.ts b/packages/@n8n/instance-ai/src/utils/__tests__/modal-session.test.ts new file mode 100644 index 00000000000..9d03424e64c --- /dev/null +++ b/packages/@n8n/instance-ai/src/utils/__tests__/modal-session.test.ts @@ -0,0 +1,65 @@ +import { MODAL_SESSION_HEADER, isModalModelEndpoint, withModalSession } from '../modal-session'; + +describe('isModalModelEndpoint', () => { + it('accepts Modal public hostnames', () => { + expect(isModalModelEndpoint('https://n8ngmbh--ep-kimi-k3-server.us-west.modal.direct/v1')).toBe( + true, + ); + expect(isModalModelEndpoint('https://my-app--server.modal.run/v1')).toBe(true); + }); + + it('rejects non-Modal and invalid URLs', () => { + expect(isModalModelEndpoint('https://api.together.ai/v1')).toBe(false); + expect(isModalModelEndpoint('https://modal.example.com/v1')).toBe(false); + expect(isModalModelEndpoint('')).toBe(false); + expect(isModalModelEndpoint(undefined)).toBe(false); + expect(isModalModelEndpoint('not-a-url')).toBe(false); + }); +}); + +describe('withModalSession', () => { + const modalConfig = { + id: 'custom/moonshotai/Kimi-K3' as const, + url: 'https://n8ngmbh--ep-kimi-k3-server.us-west.modal.direct/v1', + apiKey: 'wk-test.ws-test', + }; + + it('adds Modal-Session-ID for Modal endpoint configs', () => { + expect(withModalSession(modalConfig, 'thread-abc')).toEqual({ + ...modalConfig, + headers: { [MODAL_SESSION_HEADER]: 'thread-abc' }, + }); + }); + + it('preserves existing headers and overwrites a prior Modal-Session-ID', () => { + const withAuth = { + ...modalConfig, + headers: { 'Modal-Key': 'wk-test', [MODAL_SESSION_HEADER]: 'old-thread' }, + }; + expect(withModalSession(withAuth, 'thread-new')).toEqual({ + ...modalConfig, + headers: { + 'Modal-Key': 'wk-test', + [MODAL_SESSION_HEADER]: 'thread-new', + }, + }); + }); + + it('returns the same reference when the thread id is already set', () => { + const sticky = withModalSession(modalConfig, 'thread-abc'); + expect(withModalSession(sticky, 'thread-abc')).toBe(sticky); + }); + + it('is a no-op for non-Modal endpoints and non-endpoint configs', () => { + const databricks = { + id: 'custom/workspace.default.kimi-k3' as const, + url: 'https://dbc-a5846527-f2d2.cloud.databricks.com/ai-gateway/mlflow/v1', + apiKey: 'dapi-test', + }; + expect(withModalSession(databricks, 'thread-abc')).toBe(databricks); + expect(withModalSession('anthropic/claude-opus-4-8', 'thread-abc')).toBe( + 'anthropic/claude-opus-4-8', + ); + expect(withModalSession(modalConfig, ' ')).toBe(modalConfig); + }); +}); diff --git a/packages/@n8n/instance-ai/src/utils/custom-model-defaults.ts b/packages/@n8n/instance-ai/src/utils/custom-model-defaults.ts new file mode 100644 index 00000000000..bc583c97a47 --- /dev/null +++ b/packages/@n8n/instance-ai/src/utils/custom-model-defaults.ts @@ -0,0 +1,119 @@ +import { z } from 'zod'; + +const customModelReasoningEffortSchema = z.enum([ + 'none', + 'minimal', + 'low', + 'medium', + 'high', + 'xhigh', + 'max', +]); + +export type CustomModelReasoningEffort = z.infer; + +export type CustomModelExperimentDefaults = { + reasoningEffort?: CustomModelReasoningEffort; + supportsStructuredOutputs?: boolean; +}; + +const customModelDefaultEntrySchema = z + .object({ + match: z.string().min(1), + reasoningEffort: customModelReasoningEffortSchema.optional(), + supportsStructuredOutputs: z.boolean().optional(), + }) + .strict(); + +type CustomModelDefaultEntry = z.infer; + +/** + * Known custom/* model experiment defaults. First case-insensitive substring + * match wins. Add entries here when a new custom model needs stable knobs. + */ +const CUSTOM_MODEL_DEFAULTS = [ + { + match: 'kimi-k3', + reasoningEffort: 'low', + supportsStructuredOutputs: true, + }, + { + match: 'glm-5.2', + reasoningEffort: 'medium', + supportsStructuredOutputs: true, + }, + { + match: 'deepseek', + supportsStructuredOutputs: true, + }, +] satisfies CustomModelDefaultEntry[]; + +export function parseReasoningEffort( + value: string | undefined, +): CustomModelReasoningEffort | undefined { + const trimmed = value?.trim().toLowerCase(); + if (!trimmed) return undefined; + const parsed = customModelReasoningEffortSchema.safeParse(trimmed); + return parsed.success ? parsed.data : undefined; +} + +export function parseSupportsStructuredOutputs(value: string | undefined): boolean | undefined { + const trimmed = value?.trim().toLowerCase(); + if (!trimmed) return undefined; + if (trimmed === 'true' || trimmed === '1') return true; + if (trimmed === 'false' || trimmed === '0') return false; + return undefined; +} + +/** + * Resolve optional custom/* experiment knobs. + * Explicit overrides win; otherwise the first case-insensitive substring match + * in the shared model map is used. Unresolved fields stay undefined so callers + * can omit them from the request. + */ +export function resolveCustomModelExperimentDefaults( + modelId: string, + overrides: CustomModelExperimentDefaults = {}, +): CustomModelExperimentDefaults { + const matched = CUSTOM_MODEL_DEFAULTS.find((entry) => + modelId.toLowerCase().includes(entry.match.toLowerCase()), + ); + + const reasoningEffort = overrides.reasoningEffort ?? matched?.reasoningEffort; + const supportsStructuredOutputs = + overrides.supportsStructuredOutputs ?? matched?.supportsStructuredOutputs; + + return { + ...(reasoningEffort !== undefined ? { reasoningEffort } : {}), + ...(supportsStructuredOutputs !== undefined ? { supportsStructuredOutputs } : {}), + }; +} + +export function resolveCustomModelExperimentDefaultsFromEnv( + modelId: string, + envOverrides: { + reasoningEffort?: string; + supportsStructuredOutputs?: string; + } = {}, +): CustomModelExperimentDefaults { + const reasoningEffortRaw = + envOverrides.reasoningEffort ?? process.env.N8N_INSTANCE_AI_REASONING_EFFORT; + const supportsStructuredOutputsRaw = + envOverrides.supportsStructuredOutputs ?? + process.env.N8N_INSTANCE_AI_SUPPORTS_STRUCTURED_OUTPUTS; + + const reasoningEffort = parseReasoningEffort(reasoningEffortRaw); + const supportsStructuredOutputs = parseSupportsStructuredOutputs(supportsStructuredOutputsRaw); + + const customConfig: CustomModelExperimentDefaults = {}; + + if (reasoningEffort !== undefined) { + customConfig.reasoningEffort = reasoningEffort; + } + + if (supportsStructuredOutputs !== undefined) { + customConfig.supportsStructuredOutputs = supportsStructuredOutputs; + } + + return resolveCustomModelExperimentDefaults(modelId, customConfig); +} diff --git a/packages/@n8n/instance-ai/src/utils/eval-agents.ts b/packages/@n8n/instance-ai/src/utils/eval-agents.ts index b80b78767d4..4bb5601131a 100644 --- a/packages/@n8n/instance-ai/src/utils/eval-agents.ts +++ b/packages/@n8n/instance-ai/src/utils/eval-agents.ts @@ -2,6 +2,7 @@ import { Agent, Tool, type GenerateResult, type ModelConfig } from '@n8n/agents'; +import { parseModelHeadersJson } from './parse-model-headers'; import { applyAgentThinking } from '../agent/apply-agent-thinking'; export { Tool }; @@ -30,6 +31,7 @@ export interface EvalModelConfig { providerModelId: string; apiKey: string; url?: string; + headers?: Record; } function getModelId(model?: string): string { @@ -41,16 +43,36 @@ function getModelId(model?: string): string { return modelId; } +/** + * True when the resolved model is the Instance AI builder model (or there is + * no separate builder model). False when resolving a dedicated eval model + * (N8N_INSTANCE_AI_EVAL_MODEL / explicit arg) that differs from the builder — + * in that case we must not reuse the builder's API key or custom base URL. + */ +function isResolvingBuilderModel(modelId: string): boolean { + const builderModel = process.env.N8N_INSTANCE_AI_MODEL?.trim(); + if (!builderModel) return true; + return modelId === builderModel; +} + function getApiKey(modelId: string): string { const [provider] = modelId.split('/'); const providerKeyEnv = PROVIDER_API_KEY_ENV[provider]; const providerKey = providerKeyEnv ? process.env[providerKeyEnv] : undefined; - const key = - process.env.N8N_INSTANCE_AI_MODEL_API_KEY ?? - (provider === 'anthropic' ? process.env.N8N_AI_ANTHROPIC_KEY : undefined) ?? - providerKey; + const anthropicLegacy = provider === 'anthropic' ? process.env.N8N_AI_ANTHROPIC_KEY : undefined; + const genericKey = process.env.N8N_INSTANCE_AI_MODEL_API_KEY; + + // Builder model: prefer the lane's N8N_INSTANCE_AI_MODEL_API_KEY. + // Separate eval model (e.g. Anthropic mocks while builder is custom/openai): + // prefer provider-native keys so an OpenAI/empty builder key is not sent to Anthropic. + const key = isResolvingBuilderModel(modelId) + ? (genericKey ?? anthropicLegacy ?? providerKey) + : (anthropicLegacy ?? providerKey ?? genericKey); if (!key) { + // custom/* OpenAI-compatible routers may be keyless (URL only) or + // header-auth (URL + headers). Both are valid without an API key. + if (isResolvingBuilderModel(modelId) && allowsKeylessCustomEndpoint(provider)) return ''; throw new Error( `Missing API key for eval model "${modelId}". Set N8N_INSTANCE_AI_MODEL_API_KEY${ provider === 'anthropic' @@ -70,6 +92,17 @@ function getModelUrl(): string | undefined { return url; } +function getModelHeaders(): Record | undefined { + return parseModelHeadersJson(process.env.EVAL_MODAL_LLM_HEADERS); +} + +function allowsKeylessCustomEndpoint(provider: string): boolean { + if (!getModelUrl()) return false; + if (getModelHeaders()) return true; + + return provider === 'custom'; +} + export function resolveEvalModelConfig(model?: string): EvalModelConfig { const modelId = getModelId(model); const [provider, ...rest] = modelId.split('/'); @@ -78,12 +111,16 @@ export function resolveEvalModelConfig(model?: string): EvalModelConfig { if (joinedProviderModelId.length > 0) { providerModelId = joinedProviderModelId; } + // Builder endpoint (URL/headers) only applies when resolving that builder model. + // A dedicated Anthropic eval model must hit Anthropic, not the custom/Foundry base. + const attachBuilderEndpoint = isResolvingBuilderModel(modelId); return { modelId, provider, providerModelId, apiKey: getApiKey(modelId), - url: getModelUrl(), + url: attachBuilderEndpoint ? getModelUrl() : undefined, + headers: attachBuilderEndpoint ? getModelHeaders() : undefined, }; } @@ -108,8 +145,13 @@ const CACHE_PROVIDER_OPTS = { */ function resolveAgentModel(model?: string, fallbackModelConfig?: ModelConfig): ModelConfig { try { - const { modelId, apiKey, url } = resolveEvalModelConfig(model); - return { id: modelId, apiKey, url }; + const { modelId, apiKey, url, headers } = resolveEvalModelConfig(model); + return { + id: modelId, + apiKey, + url, + ...(headers ? { headers } : {}), + }; } catch (error) { if (fallbackModelConfig) return fallbackModelConfig; throw error; diff --git a/packages/@n8n/instance-ai/src/utils/modal-session.ts b/packages/@n8n/instance-ai/src/utils/modal-session.ts new file mode 100644 index 00000000000..bac50293f32 --- /dev/null +++ b/packages/@n8n/instance-ai/src/utils/modal-session.ts @@ -0,0 +1,50 @@ +import { isRecord } from '@n8n/utils/is-record'; + +import type { ModelConfig } from '../types'; + +export const MODAL_SESSION_HEADER = 'Modal-Session-ID'; + +export function isEndpointModelConfig(modelId: ModelConfig): modelId is { + id: `${string}/${string}`; + url: string; + apiKey?: string; + headers?: Record; + supportsStructuredOutputs?: boolean; +} { + if (!isRecord(modelId)) return false; + if ('doGenerate' in modelId || 'specificationVersion' in modelId) return false; + if (!('url' in modelId) || !('id' in modelId)) return false; + + const id = modelId.id; + const url = modelId.url; + if (typeof id !== 'string' || !id.includes('/')) return false; + if (typeof url !== 'string') return false; + return true; +} + +export function isModalModelEndpoint(url: string | undefined): boolean { + if (!url?.trim()) return false; + try { + const host = new URL(url).hostname.toLowerCase(); + return host.endsWith('.modal.direct') || host.endsWith('.modal.run'); + } catch { + return false; + } +} + +export function withModalSession(modelId: ModelConfig, threadId: string): ModelConfig { + const trimmedThreadId = threadId.trim(); + if (!trimmedThreadId || !isEndpointModelConfig(modelId)) return modelId; + if (!isModalModelEndpoint(modelId.url)) return modelId; + + const existing = modelId.headers?.[MODAL_SESSION_HEADER]; + if (existing === trimmedThreadId) return modelId; + + return { + ...modelId, + headers: { + ...modelId.headers, + [MODAL_SESSION_HEADER]: trimmedThreadId, + }, + }; +} diff --git a/packages/@n8n/instance-ai/src/utils/parse-model-headers.ts b/packages/@n8n/instance-ai/src/utils/parse-model-headers.ts new file mode 100644 index 00000000000..c45c66723a8 --- /dev/null +++ b/packages/@n8n/instance-ai/src/utils/parse-model-headers.ts @@ -0,0 +1,28 @@ +/** + * Parse eval-only LLM header JSON (`EVAL_MODAL_LLM_HEADERS`) — a JSON object of + * HTTP header names to string values (e.g. Modal proxy auth). + */ +export function parseModelHeadersJson(raw: string | undefined): Record | undefined { + const trimmed = raw?.trim(); + if (!trimmed) return undefined; + + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch { + return undefined; + } + + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + return undefined; + } + + const headers: Record = {}; + for (const [key, value] of Object.entries(parsed)) { + if (typeof key !== 'string' || !key.trim()) continue; + if (typeof value !== 'string') continue; + headers[key.trim()] = value; + } + + return Object.keys(headers).length > 0 ? headers : undefined; +} diff --git a/packages/@n8n/instance-ai/src/workspace/__tests__/sandbox-setup.test.ts b/packages/@n8n/instance-ai/src/workspace/__tests__/sandbox-setup.test.ts index b3b132fc9ee..ad5b1532485 100644 --- a/packages/@n8n/instance-ai/src/workspace/__tests__/sandbox-setup.test.ts +++ b/packages/@n8n/instance-ai/src/workspace/__tests__/sandbox-setup.test.ts @@ -566,12 +566,20 @@ describe('setupSandboxWorkspace', () => { }); it('retries packing linked workspace packages after a null pack result', async () => { + const utilsTarball = Buffer.from('utils'); const workflowTarball = Buffer.from('workflow'); const sdkTarball = Buffer.from('sdk'); const packSandboxLinkedWorkspacePackages = vi .fn() .mockResolvedValueOnce(null) .mockResolvedValueOnce([ + { + filename: 'n8n-utils.tgz', + tarball: utilsTarball, + version: '1.41.0', + packageName: '@n8n/utils', + packagePath: '/host/utils', + }, { filename: 'n8n-workflow.tgz', tarball: workflowTarball, @@ -622,6 +630,9 @@ describe('setupSandboxWorkspace', () => { await linkWorkspaceSdkIfEnabled(workspace, '/workspace', logger); expect(packSandboxLinkedWorkspacePackages).toHaveBeenCalledTimes(2); + expect(writeFile).toHaveBeenCalledWith('/workspace/n8n-utils.tgz', utilsTarball, { + recursive: true, + }); expect(writeFile).toHaveBeenCalledWith('/workspace/n8n-workflow.tgz', workflowTarball, { recursive: true, }); @@ -631,7 +642,7 @@ describe('setupSandboxWorkspace', () => { expect(runInSandbox).toHaveBeenCalledWith( workspace, expect.stringContaining( - "npm install '/workspace/n8n-workflow.tgz' '/workspace/workflow-sdk.tgz'", + "npm install '/workspace/n8n-utils.tgz' '/workspace/n8n-workflow.tgz' '/workspace/workflow-sdk.tgz'", ), '/workspace', ); diff --git a/packages/@n8n/instance-ai/src/workspace/pack-workspace-sdk.ts b/packages/@n8n/instance-ai/src/workspace/pack-workspace-sdk.ts index d7cc2fb2dc6..453838726c0 100644 --- a/packages/@n8n/instance-ai/src/workspace/pack-workspace-sdk.ts +++ b/packages/@n8n/instance-ai/src/workspace/pack-workspace-sdk.ts @@ -11,9 +11,10 @@ * packages into tarballs on the host so the sandbox can `npm install` them * post-creation and override the registry copies. * - * Both `@n8n/workflow-sdk` and its `n8n-workflow` dependency are linked: - * packing only the SDK still leaves npm's `n8n-workflow` in place, which - * breaks when master is ahead of the registry (e.g. unreleased exports). + * `@n8n/workflow-sdk`, `n8n-workflow`, and `@n8n/utils` are linked together: + * packing only the SDK still leaves npm's copies of those deps in place, which + * breaks when master is ahead of the registry (e.g. unreleased exports like + * `@n8n/utils/sleep`). * * We use `pnpm pack` (not `npm pack`) because the workspace `package.json` * uses pnpm protocols (`workspace:*`, `catalog:`) that npm can't resolve; @@ -35,7 +36,11 @@ const execFileAsync = promisify(execFile); const ENV_FLAG = 'N8N_INSTANCE_AI_SANDBOX_LINK_SDK'; /** Packages installed into the sandbox when workspace linking is enabled. */ -export const SANDBOX_LINKED_WORKSPACE_PACKAGES = ['n8n-workflow', '@n8n/workflow-sdk'] as const; +export const SANDBOX_LINKED_WORKSPACE_PACKAGES = [ + '@n8n/utils', + 'n8n-workflow', + '@n8n/workflow-sdk', +] as const; export interface WorkspacePackageTarball { /** Raw tarball bytes, ready to upload to the sandbox. */ @@ -130,7 +135,7 @@ export async function packSandboxLinkedWorkspacePackages( const tarball = await packWorkspacePackage(logger, packageName); if (!tarball) { throw new Error( - `${ENV_FLAG} is enabled, but ${packageName} could not be packed. Run \`pnpm build\` for packages/workflow and packages/@n8n/workflow-sdk, or unset ${ENV_FLAG}.`, + `${ENV_FLAG} is enabled, but ${packageName} could not be packed. Run \`pnpm build\` for packages/@n8n/utils, packages/workflow, and packages/@n8n/workflow-sdk, or unset ${ENV_FLAG}.`, ); } packed.push(tarball); @@ -159,9 +164,18 @@ export async function packWorkspaceSdk( function resolvePackagePath(name: string): string | null { try { - const pkgJsonPath = hostRequire.resolve(`${name}/package.json`); - return path.dirname(pkgJsonPath); + return path.dirname(hostRequire.resolve(`${name}/package.json`)); } catch { + // Packages that omit a `package.json` export still live under node_modules. + for (const base of hostRequire.resolve.paths(name) ?? []) { + const candidate = path.join(base, name, 'package.json'); + try { + hostRequire(candidate); + return path.dirname(candidate); + } catch { + // keep looking + } + } return null; } } diff --git a/packages/@n8n/instance-ai/src/workspace/sandbox-setup.ts b/packages/@n8n/instance-ai/src/workspace/sandbox-setup.ts index f84011d8f05..1b4f330caea 100644 --- a/packages/@n8n/instance-ai/src/workspace/sandbox-setup.ts +++ b/packages/@n8n/instance-ai/src/workspace/sandbox-setup.ts @@ -190,7 +190,7 @@ export async function linkWorkspaceSdkIfEnabled( if (!packedPackages?.length) { linkedPackagesPromise = null; throw new Error( - 'N8N_INSTANCE_AI_SANDBOX_LINK_SDK is enabled, but workspace packages could not be packed. Run `pnpm build` in packages/workflow and packages/@n8n/workflow-sdk, or unset N8N_INSTANCE_AI_SANDBOX_LINK_SDK.', + 'N8N_INSTANCE_AI_SANDBOX_LINK_SDK is enabled, but workspace packages could not be packed. Run `pnpm build` in packages/@n8n/utils, packages/workflow, and packages/@n8n/workflow-sdk, or unset N8N_INSTANCE_AI_SANDBOX_LINK_SDK.', ); } diff --git a/packages/@n8n/instance-ai/tsconfig.json b/packages/@n8n/instance-ai/tsconfig.json index 1672efb9f1c..ca0d4fafbb8 100644 --- a/packages/@n8n/instance-ai/tsconfig.json +++ b/packages/@n8n/instance-ai/tsconfig.json @@ -13,5 +13,5 @@ }, "types": ["node", "vitest/globals", "vite/client"] }, - "include": ["src/**/*.ts"] + "include": ["src/**/*.ts", "src/**/*.json"] } diff --git a/packages/cli/src/modules/agents/builder/__tests__/agents-builder-settings.service.test.ts b/packages/cli/src/modules/agents/builder/__tests__/agents-builder-settings.service.test.ts index 7bcf30c4e84..de32f3a6755 100644 --- a/packages/cli/src/modules/agents/builder/__tests__/agents-builder-settings.service.test.ts +++ b/packages/cli/src/modules/agents/builder/__tests__/agents-builder-settings.service.test.ts @@ -13,7 +13,13 @@ import { BUILDER_NOT_CONFIGURED_CODE } from '@n8n/api-types'; import { AgentsBuilderSettingsService } from '../agents-builder-settings.service'; import { BuilderNotConfiguredError } from '../errors'; -const ENV_KEYS = ['N8N_AI_ANTHROPIC_KEY', 'ANTHROPIC_API_KEY'] as const; +const ENV_KEYS = [ + 'N8N_AI_ANTHROPIC_KEY', + 'ANTHROPIC_API_KEY', + 'N8N_INSTANCE_AI_MODEL', + 'N8N_INSTANCE_AI_MODEL_API_KEY', + 'N8N_INSTANCE_AI_MODEL_URL', +] as const; function makeJwt(exp: number): string { const header = Buffer.from(JSON.stringify({ alg: 'HS256' })).toString('base64url'); @@ -110,6 +116,24 @@ describe('AgentsBuilderSettingsService', () => { ); }); + it('mode=default + proxy disabled + Instance AI env set → returns Instance AI model config', async () => { + mockPersistedSettings({ mode: 'default' }); + aiService.isProxyEnabled.mockReturnValue(false); + process.env.N8N_INSTANCE_AI_MODEL = 'custom/Kimi-K3'; + process.env.N8N_INSTANCE_AI_MODEL_API_KEY = 'custom-key'; + process.env.N8N_AI_ANTHROPIC_KEY = 'sk-env'; + + const result = await service.resolveModelConfig(user); + + expect(result).toEqual({ + config: { + id: 'custom/Kimi-K3', + apiKey: 'custom-key', + }, + isProxied: false, + }); + }); + it('mode=default + proxy disabled + env set → returns env-var anthropic config', async () => { mockPersistedSettings({ mode: 'default' }); aiService.isProxyEnabled.mockReturnValue(false); diff --git a/packages/cli/src/modules/agents/builder/__tests__/agents-builder.service.session.test.ts b/packages/cli/src/modules/agents/builder/__tests__/agents-builder.service.session.test.ts index fe7d5847047..2fb523262d1 100644 --- a/packages/cli/src/modules/agents/builder/__tests__/agents-builder.service.session.test.ts +++ b/packages/cli/src/modules/agents/builder/__tests__/agents-builder.service.session.test.ts @@ -368,7 +368,7 @@ describe('AgentsBuilderService session isolation', () => { it.each([ ['Anthropic', 'anthropic/claude-sonnet-host-resolved'], - ['OpenAI', 'openai/gpt-5.5'], + ['OpenAI', 'openai/gpt-5.6-sol'], ['Google', 'google/gemini-2.5-pro'], ])('enables generic reasoning for a %s builder model', async (_provider, modelConfig) => { const { service, user, credentialProvider } = setup(); diff --git a/packages/cli/src/modules/agents/builder/agents-builder-settings.service.ts b/packages/cli/src/modules/agents/builder/agents-builder-settings.service.ts index a3f248c0652..94a056a4ccc 100644 --- a/packages/cli/src/modules/agents/builder/agents-builder-settings.service.ts +++ b/packages/cli/src/modules/agents/builder/agents-builder-settings.service.ts @@ -40,6 +40,24 @@ function readEnvAnthropicKey(): string | null { return key && key.length > 0 ? key : null; } +/** + * Dev/CI backstop: reuse the Instance AI orchestrator model env so local setups + * that already point Instance AI at a custom OpenAI-compatible endpoint also + * configure the agent-builder settings path without a separate credential. + */ +function readEnvInstanceAiModelConfig(): ModelConfig | null { + const model = process.env.N8N_INSTANCE_AI_MODEL?.trim(); + const apiKey = process.env.N8N_INSTANCE_AI_MODEL_API_KEY?.trim(); + if (!model || !model.includes('/') || !apiKey) return null; + + const modelUrl = process.env.N8N_INSTANCE_AI_MODEL_URL?.trim(); + const id = model as `${string}/${string}`; + if (modelUrl) { + return { id, url: modelUrl, apiKey }; + } + return { id, apiKey }; +} + interface BuilderTelemetryProxyConfig { apiUrl: string; getAuthHeaders: () => Promise>; @@ -56,7 +74,10 @@ interface ResolvedBuilderModelConfig { * the builder LLM runs on: * 1. `mode: 'custom'` — admin-picked credential. Wins regardless of platform. * 2. `mode: 'default'` + AI proxy enabled — runs through the n8n AI assistant proxy. - * 3. `mode: 'default'` + env-var backstop set (`N8N_AI_ANTHROPIC_KEY` / + * 3. `mode: 'default'` + Instance AI model env (`N8N_INSTANCE_AI_MODEL` + + * `N8N_INSTANCE_AI_MODEL_API_KEY`) — same provider/model as Instance AI + * (e.g. `custom/Kimi-K3` or `openai/gpt-5-mini`). + * 4. `mode: 'default'` + Anthropic env backstop (`N8N_AI_ANTHROPIC_KEY` / * `ANTHROPIC_API_KEY`) — direct Anthropic API calls. */ @Service() @@ -151,6 +172,11 @@ export class AgentsBuilderSettingsService { return await this.resolveProxyModel(user); } + const instanceAiEnv = readEnvInstanceAiModelConfig(); + if (instanceAiEnv) { + return { config: instanceAiEnv, isProxied: false }; + } + const envKey = readEnvAnthropicKey(); if (envKey) { return { diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-settings.service.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai-settings.service.test.ts index dd5d59739b1..f87d3eaabe8 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-settings.service.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai-settings.service.test.ts @@ -90,6 +90,8 @@ describe('InstanceAiSettingsService', () => { vi.stubEnv('N8N_INSTANCE_AI_MODEL', ''); vi.stubEnv('OPENAI_API_KEY', ''); vi.stubEnv('ANTHROPIC_API_KEY', ''); + vi.stubEnv('GOOGLE_VERTEX_PROJECT', ''); + vi.stubEnv('GOOGLE_VERTEX_LOCATION', ''); persistedSettingsValue = undefined; logger.scoped.mockReturnValue(logger); Container.set(Logger, logger); @@ -107,6 +109,9 @@ describe('InstanceAiSettingsService', () => { searxngUrl: '', daytonaApiUrl: '', daytonaApiKey: '', + vertexProjectId: '', + vertexLocation: '', + vertexServiceAccountJson: '', }); globalConfig.deployment.type = 'default'; instanceCredentialBroker.listForUse.mockResolvedValue([]); @@ -1797,6 +1802,23 @@ describe('InstanceAiSettingsService', () => { expect((await service.getAdminSettings()).modelEnvConfigured).toBe(true); }); + it('reports Vertex as env-configured only when a project id is resolvable', async () => { + globalConfig.instanceAi.model = 'google-vertex-anthropic/claude-opus-4-8'; + expect((await service.getAdminSettings()).modelEnvConfigured).toBe(false); + + globalConfig.instanceAi.vertexServiceAccountJson = + '{"client_email":"svc@example.com","private_key":"k"}'; + expect((await service.getAdminSettings()).modelEnvConfigured).toBe(false); + + globalConfig.instanceAi.vertexServiceAccountJson = + '{"project_id":"from-json","client_email":"svc@example.com","private_key":"k"}'; + expect((await service.getAdminSettings()).modelEnvConfigured).toBe(true); + + globalConfig.instanceAi.vertexServiceAccountJson = ''; + globalConfig.instanceAi.vertexProjectId = 'from-env'; + expect((await service.getAdminSettings()).modelEnvConfigured).toBe(true); + }); + it('reports environment configuration for the selected sandbox provider', async () => { globalConfig.instanceAi.sandboxProvider = 'daytona'; globalConfig.instanceAi.daytonaApiKey = 'dtn-key'; @@ -1985,6 +2007,66 @@ describe('InstanceAiSettingsService', () => { globalConfig.instanceAi.modelApiKey = ''; await expect(service.resolveModelConfig(mock())).resolves.toBe('openai/gpt-4'); }); + + it('builds google-vertex-anthropic configs from Vertex environment variables', async () => { + aiService.isProxyEnabled.mockReturnValue(false); + vi.stubEnv('N8N_INSTANCE_AI_MODEL', 'google-vertex-anthropic/claude-opus-4-8'); + Object.assign(globalConfig.instanceAi, { + model: 'google-vertex-anthropic/claude-opus-4-8', + modelUrl: '', + modelApiKey: '', + vertexProjectId: 'instance-ai-494613', + vertexLocation: 'global', + vertexServiceAccountJson: '{"client_email":"svc@example.com","private_key":"k"}', + }); + + await expect(service.resolveModelConfig(mock())).resolves.toEqual({ + id: 'google-vertex-anthropic/claude-opus-4-8', + project: 'instance-ai-494613', + location: 'global', + googleCredentials: '{"client_email":"svc@example.com","private_key":"k"}', + }); + }); + + it('derives the Vertex project from service-account JSON when env project is unset', async () => { + aiService.isProxyEnabled.mockReturnValue(false); + vi.stubEnv('N8N_INSTANCE_AI_MODEL', 'google-vertex-anthropic/claude-opus-4-8'); + Object.assign(globalConfig.instanceAi, { + model: 'google-vertex-anthropic/claude-opus-4-8', + modelUrl: '', + modelApiKey: '', + vertexProjectId: '', + vertexLocation: '', + vertexServiceAccountJson: + '{"project_id":"from-json","client_email":"svc@example.com","private_key":"k"}', + }); + + await expect(service.resolveModelConfig(mock())).resolves.toEqual({ + id: 'google-vertex-anthropic/claude-opus-4-8', + project: 'from-json', + location: 'global', + googleCredentials: + '{"project_id":"from-json","client_email":"svc@example.com","private_key":"k"}', + }); + }); + + it('falls back to GOOGLE_VERTEX_LOCATION when the n8n Vertex location env is empty', async () => { + aiService.isProxyEnabled.mockReturnValue(false); + vi.stubEnv('N8N_INSTANCE_AI_MODEL', 'google-vertex-anthropic/claude-opus-4-8'); + vi.stubEnv('GOOGLE_VERTEX_LOCATION', 'us-east5'); + Object.assign(globalConfig.instanceAi, { + model: 'google-vertex-anthropic/claude-opus-4-8', + modelUrl: '', + modelApiKey: '', + vertexProjectId: 'instance-ai-494613', + vertexLocation: '', + vertexServiceAccountJson: '{"client_email":"svc@example.com","private_key":"k"}', + }); + + await expect(service.resolveModelConfig(mock())).resolves.toMatchObject({ + location: 'us-east5', + }); + }); }); describe('isSetupCompleted', () => { diff --git a/packages/cli/src/modules/instance-ai/instance-ai-settings.service.ts b/packages/cli/src/modules/instance-ai/instance-ai-settings.service.ts index 6b0a6fc1d54..242db203d32 100644 --- a/packages/cli/src/modules/instance-ai/instance-ai-settings.service.ts +++ b/packages/cli/src/modules/instance-ai/instance-ai-settings.service.ts @@ -22,7 +22,11 @@ import type { InstanceAiConfig, DeploymentConfig } from '@n8n/config'; import { DbLock, DbLockService, SettingsRepository, UserRepository } from '@n8n/db'; import type { CredentialsEntity, ICredentialsDb, OperationContext, User } from '@n8n/db'; import { Container, Service } from '@n8n/di'; -import type { ModelConfig } from '@n8n/instance-ai'; +import { + resolveCustomModelExperimentDefaultsFromEnv, + type ModelConfig, + type VertexAnthropicModelConfig, +} from '@n8n/instance-ai'; import { hasGlobalScope } from '@n8n/permissions'; import { ensureError } from '@n8n/utils/errors/ensure-error'; import type { ICredentialDataDecryptedObject, IUserSettings } from 'n8n-workflow'; @@ -162,6 +166,25 @@ function modelCredentialHeaders( return Object.keys(headers).length ? headers : undefined; } +function isVertexAnthropicModelId( + id: `${string}/${string}`, +): id is `google-vertex-anthropic/${string}` { + return id.startsWith('google-vertex-anthropic/'); +} + +/** `project_id` from a GCP service-account JSON blob, if present and parseable. */ +function projectIdFromServiceAccountJson(json: string | undefined): string { + if (!json?.trim()) return ''; + try { + const parsed: unknown = JSON.parse(json); + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) return ''; + const projectId = (parsed as Record).project_id; + return typeof projectId === 'string' ? projectId.trim() : ''; + } catch { + return ''; + } +} + function validateSearchCredential({ type, data, @@ -380,14 +403,20 @@ export class InstanceAiSettingsService { credentialSelection: AdminCredentialSelection, ): InstanceAiAdminSettingsResponse { const c = this.config; - const modelProviderApiKeyEnv = MODEL_PROVIDER_API_KEY_ENV.get(c.model.split('/', 1)[0] ?? ''); + const modelProvider = c.model.split('/', 1)[0] ?? ''; + const modelProviderApiKeyEnv = MODEL_PROVIDER_API_KEY_ENV.get(modelProvider); const isProxyEnabled = this.aiService.isProxyEnabled(); const isManaged = this.isCloud || isProxyEnabled; const providerModelApiKeyConfigured = Boolean( modelProviderApiKeyEnv && process.env[modelProviderApiKeyEnv]?.trim(), ); + const vertexModelEnvConfigured = + modelProvider === 'google-vertex-anthropic' && Boolean(this.resolveVertexProjectId()); const modelConnectionEnvConfigured = Boolean( - c.modelApiKey.trim() || c.modelUrl.trim() || providerModelApiKeyConfigured, + c.modelApiKey.trim() || + c.modelUrl.trim() || + providerModelApiKeyConfigured || + vertexModelEnvConfigured, ); const sandboxEnvConfigured = this.hasEnvironmentSandboxConnection(); const searchEnvConfigured = this.hasEnvironmentSearchConnection(); @@ -1351,11 +1380,12 @@ export class InstanceAiSettingsService { const provider = config.includes('/') ? config.slice(0, config.indexOf('/')) : 'custom'; return `${provider}/${modelName}`; } - if ('id' in config && typeof config.id === 'string') { + if ('id' in config && typeof config.id === 'string' && 'url' in config) { const provider = config.id.includes('/') ? config.id.slice(0, config.id.indexOf('/')) : 'custom'; - return { ...config, id: `${provider}/${modelName}` }; + const id: `${string}/${string}` = `${provider}/${modelName}`; + return { ...config, id }; } return config; } @@ -1571,6 +1601,9 @@ export class InstanceAiSettingsService { private hasEnvironmentModelConnection(): boolean { const provider = this.config.model.split('/', 1)[0] ?? ''; + if (provider === 'google-vertex-anthropic') { + return Boolean(this.resolveVertexProjectId()); + } const providerApiKeyEnv = MODEL_PROVIDER_API_KEY_ENV.get(provider); return Boolean( this.config.modelApiKey.trim() || @@ -1604,18 +1637,71 @@ export class InstanceAiSettingsService { const id: `${string}/${string}` = model.includes('/') ? (model as `${string}/${string}`) : `custom/${model}`; + const customOptions = this.customModelOptionsFor(id); + + const vertexConfig = this.vertexAnthropicModelConfig(id); + if (vertexConfig) return vertexConfig; if (modelUrl) { - return { id, url: modelUrl, ...(modelApiKey ? { apiKey: modelApiKey } : {}) }; + return { + id, + url: modelUrl, + ...(modelApiKey ? { apiKey: modelApiKey } : {}), + ...customOptions, + }; } if (modelApiKey) { - return { id, url: '', apiKey: modelApiKey }; + return { + id, + url: '', + apiKey: modelApiKey, + ...customOptions, + }; } return model; } + private resolveVertexProjectId(): string { + return ( + this.config.vertexProjectId?.trim() || + process.env.GOOGLE_VERTEX_PROJECT?.trim() || + projectIdFromServiceAccountJson(this.config.vertexServiceAccountJson) || + '' + ); + } + + private resolveVertexLocation(): string { + return ( + this.config.vertexLocation?.trim() || process.env.GOOGLE_VERTEX_LOCATION?.trim() || 'global' + ); + } + + /** + * Build a typed Vertex Anthropic model config from Instance AI env vars. + * Returns null when the model id is not `google-vertex-anthropic/*`. + */ + private vertexAnthropicModelConfig(id: `${string}/${string}`): VertexAnthropicModelConfig | null { + if (!isVertexAnthropicModelId(id)) return null; + + const googleCredentials = this.config.vertexServiceAccountJson?.trim() || undefined; + + return { + id, + project: this.resolveVertexProjectId(), + location: this.resolveVertexLocation(), + ...(googleCredentials ? { googleCredentials } : {}), + }; + } + + /** Optional custom/* knobs from env override → known-model map → omit. */ + private customModelOptionsFor(modelId: string): { supportsStructuredOutputs?: boolean } { + if (!modelId.startsWith('custom/')) return {}; + const { supportsStructuredOutputs } = resolveCustomModelExperimentDefaultsFromEnv(modelId); + return supportsStructuredOutputs !== undefined ? { supportsStructuredOutputs } : {}; + } + private extractModelName(model: string): string { const slash = model.indexOf('/'); return slash >= 0 ? model.slice(slash + 1) : model; diff --git a/packages/frontend/editor-ui/src/features/agents/__tests__/provider-capabilities.test.ts b/packages/frontend/editor-ui/src/features/agents/__tests__/provider-capabilities.test.ts index 8e426da0bcc..f73234bbdd3 100644 --- a/packages/frontend/editor-ui/src/features/agents/__tests__/provider-capabilities.test.ts +++ b/packages/frontend/editor-ui/src/features/agents/__tests__/provider-capabilities.test.ts @@ -13,9 +13,12 @@ describe('provider-capabilities', () => { expect([...REASONING_EFFORT_OPTIONS]).toEqual(['low', 'medium', 'high']); }); - it('uses budget-tokens for Anthropic and reasoning-effort for OpenAI', () => { + it('uses budget-tokens for Anthropic and reasoning-effort for OpenAI/xAI/custom', () => { expect(PROVIDER_CAPABILITIES.anthropic.thinking).toBe('budgetTokens'); expect(PROVIDER_CAPABILITIES.openai.thinking).toBe('reasoningEffort'); + expect(PROVIDER_CAPABILITIES.openrouter.thinking).toBe(false); + expect(PROVIDER_CAPABILITIES.xai.thinking).toBe('reasoningEffort'); + expect(PROVIDER_CAPABILITIES.custom.thinking).toBe('reasoningEffort'); }); it('keeps the canonical Anthropic cache-ttl order', () => { @@ -36,6 +39,7 @@ describe('provider-capabilities', () => { 'deepseek', 'mistral', 'openrouter', + 'custom', 'cohere', 'ollama', ]; @@ -45,16 +49,7 @@ describe('provider-capabilities', () => { }); it('marks providers without thinking support as `false`', () => { - const noThinking = [ - 'google', - 'xai', - 'groq', - 'deepseek', - 'mistral', - 'openrouter', - 'cohere', - 'ollama', - ]; + const noThinking = ['google', 'groq', 'deepseek', 'mistral', 'cohere', 'ollama']; for (const provider of noThinking) { expect(PROVIDER_CAPABILITIES[provider]?.thinking).toBe(false); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c295de3e8a1..51fe12bd612 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -27,6 +27,9 @@ catalogs: '@ai-sdk/google': specifier: ^4.0.24 version: 4.0.24 + '@ai-sdk/google-vertex': + specifier: ^5.0.47 + version: 5.0.48 '@ai-sdk/groq': specifier: ^4.0.13 version: 4.0.13 @@ -966,6 +969,9 @@ importers: '@ai-sdk/google': specifier: 'catalog:' version: 4.0.24(zod@3.25.76) + '@ai-sdk/google-vertex': + specifier: 'catalog:' + version: 5.0.48(zod@3.25.76) '@ai-sdk/groq': specifier: 'catalog:' version: 4.0.13(zod@3.25.76) @@ -1120,7 +1126,7 @@ importers: version: 1.0.27(@langchain/core@1.2.0(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.1))(openai@6.46.0(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76))(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10)))(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.1))(cheerio@1.1.0)(openai@6.46.0(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76))(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10)) '@langchain/community': specifier: 'catalog:' - version: 1.1.27(b405db3381a109939c4610cb9de005a5) + version: 1.1.27(d4d69bb701b08e92536f507cd2608666) '@langchain/core': specifier: 'catalog:' version: 1.2.0(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.1))(openai@6.46.0(@smithy/signature-v4@5.3.5)(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76))(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10)) @@ -3198,7 +3204,7 @@ importers: version: 1.0.1(@langchain/core@1.2.0(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.1))(openai@6.46.0(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76))(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10)))(encoding@0.1.13) '@langchain/community': specifier: 'catalog:' - version: 1.1.27(6421211f9ce0f081394dd8f3f58c2bce) + version: 1.1.27(ff1df20be70f541448eb6f5cafd5a6b6) '@langchain/core': specifier: 'catalog:' version: 1.2.0(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.1))(openai@6.46.0(@smithy/signature-v4@5.3.5)(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76))(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10)) @@ -7223,6 +7229,12 @@ packages: peerDependencies: zod: 3.25.76 + '@ai-sdk/anthropic@4.0.36': + resolution: {integrity: sha512-Wg5jfray0X4+qkrr73GZ7U7K2JNGx+S+rNY9LorVlXhkhTqnvtNLYWRA1WxSxlCDCw3yjRCJdL9kyc+b7naAog==} + engines: {node: '>=22'} + peerDependencies: + zod: 3.25.76 + '@ai-sdk/azure@4.0.21': resolution: {integrity: sha512-2DbEPXIdj6dvc30tPE7Rz+t3ELbjOvIDawAHUvLIipc15sY7+uvS203jh1cjiPUyZ+NAIa3B+HWEpHrjlAoyKg==} engines: {node: '>=22'} @@ -7247,12 +7259,24 @@ packages: peerDependencies: zod: 3.25.76 + '@ai-sdk/google-vertex@5.0.48': + resolution: {integrity: sha512-04QG2F+q5ZZ3WqOaaZqj0J+pTsKVSUTJs43oIwk7Nx1MyZsHutbWiHAj48BXYi8InnnIDj65WL1pSZJuciAcTw==} + engines: {node: '>=22'} + peerDependencies: + zod: 3.25.76 + '@ai-sdk/google@4.0.24': resolution: {integrity: sha512-W+kTGb/RRe2SEYwbKcKYM3N2EQryS2Iqjgou6wPsyP/2BwBU7Q23CF6/YWE8k7xtseDLcSY59jsR2MDpY4azRg==} engines: {node: '>=22'} peerDependencies: zod: 3.25.76 + '@ai-sdk/google@4.0.39': + resolution: {integrity: sha512-+mRx7UBZn9PkJ4J6YXowaRZKMZYa290cknVqqOw/roZaDg186IUOLn9JHNQkvgaj91/MLW1AZNESy1ZD2yXDCg==} + engines: {node: '>=22'} + peerDependencies: + zod: 3.25.76 + '@ai-sdk/groq@4.0.13': resolution: {integrity: sha512-bEUGDm3v3X/LvrpMioPg8NhaFBmwt+MjFa0cFdQVOJOT6LBLlGEYwr70g8B2/ttjZj4fW2qIhM/m8DDoYpXdrg==} engines: {node: '>=22'} @@ -7271,6 +7295,12 @@ packages: peerDependencies: zod: 3.25.76 + '@ai-sdk/openai-compatible@3.0.28': + resolution: {integrity: sha512-vF/852mCFiASq2fAPE1mA0E0w78NH68/X2Gukuh4TeJrMIp+CJfoR5QLuY2+pTx7UiLmuYWDpCQhWaupuM7+Xg==} + engines: {node: '>=22'} + peerDependencies: + zod: 3.25.76 + '@ai-sdk/openai@4.0.20': resolution: {integrity: sha512-/Hu+btLIO2zuYqCp//vBBAGdM/BeN4gds+NWrlv7Y9WrcTXwnEEwh6P3QZIWh2Tt1I1lswuDIpAJ4t/c/rws3Q==} engines: {node: '>=22'} @@ -7293,6 +7323,12 @@ packages: peerDependencies: zod: 3.25.76 + '@ai-sdk/provider-utils@5.0.25': + resolution: {integrity: sha512-xscPPHCSjCHWrdhai25sbHCJeKNLW/3D1uSpUZa4cEtTKXA8OnPQ3+Rfu1SmM5Ea/Mf8Dfn3cllw9zeMzo/zFA==} + engines: {node: '>=22'} + peerDependencies: + zod: 3.25.76 + '@ai-sdk/provider@4.0.3': resolution: {integrity: sha512-e0CpNWJUY7OxAFAnCZkw+ri9QOHWwTs1tXP42782KFGCU07qt8NiXCrCVowyCB5dP2r5/Uls+g2oPd8kOJn9dw==} engines: {node: '>=22'} @@ -7301,6 +7337,10 @@ packages: resolution: {integrity: sha512-tbHKNLirllUNF3ZlkCsXnwab2ZV1Sl4b1H/Cp9ruCce15IBmskE8Gwkk0yo9xDWY+jho2of7lVXtwSsyrq7cwQ==} engines: {node: '>=22'} + '@ai-sdk/provider@4.0.7': + resolution: {integrity: sha512-6or44XprPzKbr8zkmzosowSE0pxkvJcoojBL+mCZvPUt3kvXp3XSNqeVun9golb1acEfSo6yaEBRT18h2VU+1Q==} + engines: {node: '>=22'} + '@ai-sdk/xai@4.0.18': resolution: {integrity: sha512-2hrfujGCzNj89+BkREXU3siHlutdWcDMMkKc/+RITRk8T8NPIuAZ4yF8g1/2W+Xessbqz+4PQP7al9HPTfbimg==} engines: {node: '>=22'} @@ -16744,8 +16784,8 @@ packages: resolution: {integrity: sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==} engines: {node: '>=14'} - gcp-metadata@7.0.0: - resolution: {integrity: sha512-3PfRTzvT3Msu0Hy8Gf9ypxJvaClG2IB9pyH0r8QOmRBW5mUcrHgYpF4GYP+XulDbfhxEhBYtJtJJQb5S2wM+LA==} + gcp-metadata@8.1.2: + resolution: {integrity: sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==} engines: {node: '>=18'} generate-function@2.3.1: @@ -16922,8 +16962,8 @@ packages: engines: {node: '>=0.6.0'} hasBin: true - google-auth-library@10.1.0: - resolution: {integrity: sha512-GspVjZj1RbyRWpQ9FbAXMKjFGzZwDKnUHi66JJ+tcjcu5/xYAP1pdlWotCuIkMwjfVsxxDvsGZXGLzRt72D0sQ==} + google-auth-library@10.9.1: + resolution: {integrity: sha512-i1ydyHrqcIxXkWh/uBmVkzCvIuq5yiK2ATndIe5XxKholrG/MTYP9xGYka4sQhrbIAgGjL2B6NOE7rFaiF3fXw==} engines: {node: '>=18'} google-auth-library@9.15.1: @@ -16934,8 +16974,8 @@ packages: resolution: {integrity: sha512-V6eky/xz2mcKfAd1Ioxyd6nmA61gao3n01C+YeuIwu3vzM9EDR6wcVzMSIbLMDXWeoi9SHYctXuKYC5uJUT3eQ==} engines: {node: '>=14'} - google-logging-utils@1.1.1: - resolution: {integrity: sha512-rcX58I7nqpu4mbKztFeOAObbomBbHU2oIb/d3tJfF3dizGSApqtSwYJigGCooHdnMyQBIw8BrWyK96w3YXgr6A==} + google-logging-utils@1.1.3: + resolution: {integrity: sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==} engines: {node: '>=14'} gopd@1.2.0: @@ -16968,10 +17008,6 @@ packages: resolution: {integrity: sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==} engines: {node: '>=14.0.0'} - gtoken@8.0.0: - resolution: {integrity: sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw==} - engines: {node: '>=18'} - handlebars@4.7.9: resolution: {integrity: sha512-4E71E0rpOaQuJR2A3xDZ+GM1HyWYv1clR58tC8emQNeQe3RH7MAzSbat+V0wG78LQBo6m6bzSG/L4pBuCsgnUQ==} engines: {node: '>=0.4.7'} @@ -23270,6 +23306,12 @@ snapshots: '@ai-sdk/provider-utils': 5.0.18(zod@3.25.76) zod: 3.25.76 + '@ai-sdk/anthropic@4.0.36(zod@3.25.76)': + dependencies: + '@ai-sdk/provider': 4.0.7 + '@ai-sdk/provider-utils': 5.0.25(zod@3.25.76) + zod: 3.25.76 + '@ai-sdk/azure@4.0.21(zod@3.25.76)': dependencies: '@ai-sdk/deepseek': 3.0.13(zod@3.25.76) @@ -23297,12 +23339,30 @@ snapshots: '@vercel/oidc': 3.2.0 zod: 3.25.76 + '@ai-sdk/google-vertex@5.0.48(zod@3.25.76)': + dependencies: + '@ai-sdk/anthropic': 4.0.36(zod@3.25.76) + '@ai-sdk/google': 4.0.39(zod@3.25.76) + '@ai-sdk/openai-compatible': 3.0.28(zod@3.25.76) + '@ai-sdk/provider': 4.0.7 + '@ai-sdk/provider-utils': 5.0.25(zod@3.25.76) + google-auth-library: 10.9.1 + zod: 3.25.76 + transitivePeerDependencies: + - supports-color + '@ai-sdk/google@4.0.24(zod@3.25.76)': dependencies: '@ai-sdk/provider': 4.0.3 '@ai-sdk/provider-utils': 5.0.12(zod@3.25.76) zod: 3.25.76 + '@ai-sdk/google@4.0.39(zod@3.25.76)': + dependencies: + '@ai-sdk/provider': 4.0.7 + '@ai-sdk/provider-utils': 5.0.25(zod@3.25.76) + zod: 3.25.76 + '@ai-sdk/groq@4.0.13(zod@3.25.76)': dependencies: '@ai-sdk/provider': 4.0.3 @@ -23321,6 +23381,12 @@ snapshots: '@ai-sdk/provider-utils': 5.0.12(zod@3.25.76) zod: 3.25.76 + '@ai-sdk/openai-compatible@3.0.28(zod@3.25.76)': + dependencies: + '@ai-sdk/provider': 4.0.7 + '@ai-sdk/provider-utils': 5.0.25(zod@3.25.76) + zod: 3.25.76 + '@ai-sdk/openai@4.0.20(zod@3.25.76)': dependencies: '@ai-sdk/provider': 4.0.3 @@ -23352,6 +23418,15 @@ snapshots: undici: 7.29.0 zod: 3.25.76 + '@ai-sdk/provider-utils@5.0.25(zod@3.25.76)': + dependencies: + '@ai-sdk/provider': 4.0.7 + '@standard-schema/spec': 1.1.0 + '@workflow/serde': 4.1.0 + eventsource-parser: 3.0.8 + undici: 7.29.0 + zod: 3.25.76 + '@ai-sdk/provider@4.0.3': dependencies: json-schema: 0.4.0 @@ -23360,6 +23435,10 @@ snapshots: dependencies: json-schema: 0.4.0 + '@ai-sdk/provider@4.0.7': + dependencies: + json-schema: 0.4.0 + '@ai-sdk/xai@4.0.18(zod@3.25.76)': dependencies: '@ai-sdk/openai-compatible': 3.0.14(zod@3.25.76) @@ -26849,7 +26928,62 @@ snapshots: - aws-crt - encoding - '@langchain/community@1.1.27(6421211f9ce0f081394dd8f3f58c2bce)': + '@langchain/community@1.1.27(d4d69bb701b08e92536f507cd2608666)': + dependencies: + '@browserbasehq/stagehand': 1.14.0(@playwright/test@1.60.0)(bufferutil@4.0.9)(deepmerge@4.3.1)(dotenv@17.4.2)(encoding@0.1.13)(openai@6.46.0(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76))(utf-8-validate@5.0.10)(zod@3.25.76) + '@ibm-cloud/watsonx-ai': 1.1.2 + '@langchain/classic': 1.0.27(@langchain/core@1.2.0(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.1))(openai@6.46.0(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76))(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10)))(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.1))(cheerio@1.1.0)(openai@6.46.0(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76))(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10)) + '@langchain/core': 1.2.0(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.1))(openai@6.46.0(@smithy/signature-v4@5.3.5)(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76))(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10)) + '@langchain/openai': 1.4.1(@langchain/core@1.2.0(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.1))(openai@6.46.0(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76))(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10)))(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10)) + binary-extensions: 2.2.0 + flat: 5.0.2 + ibm-cloud-sdk-core: 5.5.0 + js-yaml: 4.3.1 + langsmith: 0.6.0(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.1))(openai@6.46.0(@smithy/signature-v4@5.3.5)(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76))(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10)) + math-expression-evaluator: 2.0.7 + openai: 6.46.0(@smithy/signature-v4@5.3.5)(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76) + uuid: 13.0.1 + zod: 3.25.76 + optionalDependencies: + '@aws-crypto/sha256-js': 5.2.0 + '@azure/storage-blob': 12.32.0 + '@browserbasehq/sdk': 2.12.0(encoding@0.1.13) + '@getzep/zep-cloud': 1.0.6(d33d83a49ccae0e9b43f6f5606890e08) + '@google-cloud/storage': 7.12.1(encoding@0.1.13) + '@libsql/client': 0.17.2(bufferutil@4.0.9)(encoding@0.1.13)(utf-8-validate@5.0.10) + '@mozilla/readability': 0.6.0 + '@smithy/eventstream-codec': 4.4.14 + '@smithy/protocol-http': 5.3.12 + '@smithy/util-utf8': 4.4.14 + '@supabase/supabase-js': 2.50.0(bufferutil@4.0.9)(utf-8-validate@5.0.10) + '@zilliz/milvus2-sdk-node': 2.5.7 + chromadb: 3.2.0 + crypto-js: 4.2.0 + discord.js: 14.27.0(bufferutil@4.0.9)(utf-8-validate@5.0.10) + epub2: 3.0.2(ts-toolbelt@9.6.0) + fast-xml-parser: 5.7.2 + google-auth-library: 10.9.1 + html-to-text: 9.0.5 + ignore: 7.0.5 + ioredis: 5.3.2 + jsdom: 23.0.1(bufferutil@4.0.9)(utf-8-validate@5.0.10) + jsonwebtoken: 9.0.3 + lodash: 4.18.1(patch_hash=81ead1b07fc8efda87bd2724dbd876ab49f4a0aa03f96c3770060b2ea47d9e0b) + mammoth: 1.12.0 + mysql2: 3.23.1(@types/node@20.19.41) + pdf-parse: 2.4.5 + pg: 8.21.0(pg-native@3.8.0) + playwright: 1.60.0 + puppeteer: 24.41.0(bufferutil@4.0.9)(typescript@7.0.2)(utf-8-validate@5.0.10) + ws: 8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10) + transitivePeerDependencies: + - '@opentelemetry/api' + - '@opentelemetry/exporter-trace-otlp-proto' + - '@opentelemetry/sdk-trace-base' + - '@smithy/hash-node' + - peggy + + '@langchain/community@1.1.27(ff1df20be70f541448eb6f5cafd5a6b6)': dependencies: '@browserbasehq/stagehand': 1.14.0(@playwright/test@1.60.0)(bufferutil@4.0.9)(deepmerge@4.3.1)(dotenv@17.4.2)(encoding@0.1.13)(openai@6.46.0(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76))(utf-8-validate@5.0.10)(zod@3.25.76) '@ibm-cloud/watsonx-ai': 1.1.2 @@ -26892,7 +27026,7 @@ snapshots: discord.js: 14.27.0(bufferutil@4.0.9)(utf-8-validate@5.0.10) epub2: 3.0.2(ts-toolbelt@9.6.0) fast-xml-parser: 5.7.2 - google-auth-library: 10.1.0 + google-auth-library: 10.9.1 html-to-text: 9.0.5 ignore: 5.3.2 ioredis: 5.3.2 @@ -26916,61 +27050,6 @@ snapshots: - '@smithy/hash-node' - peggy - '@langchain/community@1.1.27(b405db3381a109939c4610cb9de005a5)': - dependencies: - '@browserbasehq/stagehand': 1.14.0(@playwright/test@1.60.0)(bufferutil@4.0.9)(deepmerge@4.3.1)(dotenv@17.4.2)(encoding@0.1.13)(openai@6.46.0(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76))(utf-8-validate@5.0.10)(zod@3.25.76) - '@ibm-cloud/watsonx-ai': 1.1.2 - '@langchain/classic': 1.0.27(@langchain/core@1.2.0(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.1))(openai@6.46.0(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76))(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10)))(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.1))(cheerio@1.1.0)(openai@6.46.0(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76))(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10)) - '@langchain/core': 1.2.0(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.1))(openai@6.46.0(@smithy/signature-v4@5.3.5)(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76))(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10)) - '@langchain/openai': 1.4.1(@langchain/core@1.2.0(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.1))(openai@6.46.0(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76))(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10)))(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10)) - binary-extensions: 2.2.0 - flat: 5.0.2 - ibm-cloud-sdk-core: 5.5.0 - js-yaml: 4.3.1 - langsmith: 0.6.0(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.1))(openai@6.46.0(@smithy/signature-v4@5.3.5)(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76))(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10)) - math-expression-evaluator: 2.0.7 - openai: 6.46.0(@smithy/signature-v4@5.3.5)(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76) - uuid: 13.0.1 - zod: 3.25.76 - optionalDependencies: - '@aws-crypto/sha256-js': 5.2.0 - '@azure/storage-blob': 12.32.0 - '@browserbasehq/sdk': 2.12.0(encoding@0.1.13) - '@getzep/zep-cloud': 1.0.6(d33d83a49ccae0e9b43f6f5606890e08) - '@google-cloud/storage': 7.12.1(encoding@0.1.13) - '@libsql/client': 0.17.2(bufferutil@4.0.9)(encoding@0.1.13)(utf-8-validate@5.0.10) - '@mozilla/readability': 0.6.0 - '@smithy/eventstream-codec': 4.4.14 - '@smithy/protocol-http': 5.3.12 - '@smithy/util-utf8': 4.4.14 - '@supabase/supabase-js': 2.50.0(bufferutil@4.0.9)(utf-8-validate@5.0.10) - '@zilliz/milvus2-sdk-node': 2.5.7 - chromadb: 3.2.0 - crypto-js: 4.2.0 - discord.js: 14.27.0(bufferutil@4.0.9)(utf-8-validate@5.0.10) - epub2: 3.0.2(ts-toolbelt@9.6.0) - fast-xml-parser: 5.7.2 - google-auth-library: 10.1.0 - html-to-text: 9.0.5 - ignore: 7.0.5 - ioredis: 5.3.2 - jsdom: 23.0.1(bufferutil@4.0.9)(utf-8-validate@5.0.10) - jsonwebtoken: 9.0.3 - lodash: 4.18.1(patch_hash=81ead1b07fc8efda87bd2724dbd876ab49f4a0aa03f96c3770060b2ea47d9e0b) - mammoth: 1.12.0 - mysql2: 3.23.1(@types/node@20.19.41) - pdf-parse: 2.4.5 - pg: 8.21.0(pg-native@3.8.0) - playwright: 1.60.0 - puppeteer: 24.41.0(bufferutil@4.0.9)(typescript@7.0.2)(utf-8-validate@5.0.10) - ws: 8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10) - transitivePeerDependencies: - - '@opentelemetry/api' - - '@opentelemetry/exporter-trace-otlp-proto' - - '@opentelemetry/sdk-trace-base' - - '@smithy/hash-node' - - peggy - '@langchain/core@1.2.0(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.1))(openai@6.46.0(@smithy/signature-v4@5.3.5)(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76))(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))': dependencies: '@cfworker/json-schema': 4.1.0 @@ -26995,7 +27074,7 @@ snapshots: '@langchain/google-gauth@2.1.24(@langchain/core@1.2.0(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.1))(openai@6.46.0(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76))(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10)))': dependencies: '@langchain/google-common': 2.1.24(patch_hash=b50436a6c7b0a82d54472dff331ba1b42ab8b816995f83295eef2ee4b482a87d)(@langchain/core@1.2.0(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.1))(openai@6.46.0(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.76))(ws@8.21.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))) - google-auth-library: 10.1.0 + google-auth-library: 10.9.1 transitivePeerDependencies: - '@langchain/core' - supports-color @@ -35325,10 +35404,10 @@ snapshots: transitivePeerDependencies: - supports-color - gcp-metadata@7.0.0: + gcp-metadata@8.1.2: dependencies: gaxios: 7.1.4 - google-logging-utils: 1.1.1 + google-logging-utils: 1.1.3 json-bigint: 1.0.0 transitivePeerDependencies: - supports-color @@ -35539,14 +35618,13 @@ snapshots: dependencies: minimist: 1.2.8 - google-auth-library@10.1.0: + google-auth-library@10.9.1: dependencies: base64-js: 1.5.1 ecdsa-sig-formatter: 1.0.11 gaxios: 7.1.4 - gcp-metadata: 7.0.0 - google-logging-utils: 1.1.1 - gtoken: 8.0.0 + gcp-metadata: 8.1.2 + google-logging-utils: 1.1.3 jws: 4.0.1 transitivePeerDependencies: - supports-color @@ -35580,7 +35658,7 @@ snapshots: - encoding - supports-color - google-logging-utils@1.1.1: {} + google-logging-utils@1.1.3: {} gopd@1.2.0: {} @@ -35631,13 +35709,6 @@ snapshots: transitivePeerDependencies: - supports-color - gtoken@8.0.0: - dependencies: - gaxios: 7.1.4 - jws: 4.0.1 - transitivePeerDependencies: - - supports-color - handlebars@4.7.9: dependencies: minimist: 1.2.8 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 5c0119dd91f..1df26d3aa77 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -16,6 +16,7 @@ catalog: '@ai-sdk/deepseek': ^3.0.13 '@ai-sdk/gateway': ^4.0.37 '@ai-sdk/google': ^4.0.24 + '@ai-sdk/google-vertex': ^5.0.47 '@ai-sdk/groq': ^4.0.13 '@ai-sdk/mistral': ^4.0.14 '@ai-sdk/openai': ^4.0.20 @@ -291,3 +292,7 @@ minimumReleaseAge: 4320 minimumReleaseAgeExclude: - '@n8n/*' - '@n8n_io/*' + # Fresh Claude model IDs (e.g. Opus 5) ship in @ai-sdk/anthropic patches; allow + # installing before the 3-day minimumReleaseAge so agents/evals aren't stuck on + # the unknown-model 4096 max_tokens default. + - '@ai-sdk/anthropic'