diff --git a/.changeset/nice-mirrors-report.md b/.changeset/nice-mirrors-report.md new file mode 100644 index 0000000000..09553cd2a8 --- /dev/null +++ b/.changeset/nice-mirrors-report.md @@ -0,0 +1,5 @@ +--- +"claude-dev": minor +--- + +Updated Baseten models to use dynamic fetching, added docs diff --git a/docs/provider-config/baseten.mdx b/docs/provider-config/baseten.mdx new file mode 100644 index 0000000000..5645a26381 --- /dev/null +++ b/docs/provider-config/baseten.mdx @@ -0,0 +1,114 @@ +--- +title: "Baseten" +description: "Learn how to configure and use Baseten's Model APIs with Cline. Access frontier open-source models with enterprise-grade performance, reliability, and competitive pricing." +--- + +Baseten provides on-demand frontier model APIs designed for production applications, not just experimentation. Built on the Baseten Inference Stack, these APIs deliver enterprise-grade performance and reliability with optimized inference for leading open-source models from OpenAI, DeepSeek, Meta, Moonshot AI, and Alibaba Cloud. + +**Website:** [https://www.baseten.co/products/model-apis/](https://www.baseten.co/products/model-apis/) + +### Getting an API Key + +1. **Sign Up/Sign In:** Go to [Baseten](https://www.baseten.co/) and create an account or sign in. +2. **Navigate to API Keys:** Access your dashboard and go to the API Keys section. +3. **Create a Key:** Generate a new API key. Give it a descriptive name (e.g., "Cline"). +4. **Copy the Key:** Copy the API key immediately and store it securely. + +### Supported Models + +Cline supports all current models under Baseten Model APIs, including: + +**Reasoning Models:** +- `deepseek-ai/DeepSeek-R1` - DeepSeek's first-generation reasoning model (163K context) - $2.55/$5.95 per 1M tokens +- `deepseek-ai/DeepSeek-R1-0528` - Latest revision of DeepSeek's reasoning model (163K context) - $2.55/$5.95 per 1M tokens +- `deepseek-ai/DeepSeek-V3.1` - Hybrid reasoning with advanced tool calling (163K context) - $0.50/$1.50 per 1M tokens +- `deepseek-ai/DeepSeek-V3-0324` - Fast general-purpose with enhanced reasoning (163K context) - $0.77/$0.77 per 1M tokens + +**Flagship Models:** +- `openai/gpt-oss-120b` (OpenAI) - 120B MoE with strong reasoning capabilities (128K context) - $0.10/$0.50 per 1M tokens +- `moonshotai/Kimi-K2-Instruct` (Moonshot AI) - 1 trillion parameter model for agentic tasks (131K context) - $0.60/$2.50 per 1M tokens +- `moonshotai/Kimi-K2-Instruct-0905` (Moonshot AI) - September update with enhanced capabilities (262K context) - $0.60/$2.50 per 1M tokens + +**Meta Llama 4 Series:** +- `meta-llama/Llama-4-Maverick-17B-128E-Instruct` - High-efficiency processing (1M context!) - $0.19/$0.72 per 1M tokens +- `meta-llama/Llama-4-Scout-17B-16E-Instruct` - Precise context understanding (1M context!) - $0.13/$0.50 per 1M tokens + +**Coding Specialists:** +- `Qwen/Qwen3-Coder-480B-A35B-Instruct` (Alibaba Cloud) - Advanced coding and reasoning (262K context) - $0.38/$1.53 per 1M tokens +- `Qwen/Qwen3-235B-A22B-Instruct-2507` (Alibaba Cloud) - Math and reasoning expert (262K context) - $0.22/$0.80 per 1M tokens + +### Configuration in Cline + +1. **Open Cline Settings:** Click the settings icon (⚙️) in the Cline panel. +2. **Select Provider:** Choose "Baseten" from the "API Provider" dropdown. +3. **Enter API Key:** Paste your Baseten API key into the "Baseten API Key" field. +4. **Select Model:** Choose your desired model from the "Model" dropdown. + +### Production-First Architecture + +Baseten's Model APIs are built for production environments with several key advantages: + +#### Enterprise-Grade Reliability +- **Four nines of uptime** (99.99%) through active-active redundancy +- **Cloud-agnostic, multi-cluster autoscaling** for consistent availability +- **SOC 2 Type II certified** and **HIPAA compliant** for security requirements + +#### Optimized Performance +- **Pre-optimized models** shipped with the Baseten Inference Stack +- **Latest-generation GPUs** with multi-cloud infrastructure +- **Ultra-fast inference** optimized from the bottom up for production workloads + +#### Cost Efficiency +- **5-10x less expensive** than closed alternatives +- **Optimized multi-cloud infrastructure** for efficient resource utilization +- **Transparent pricing** with no hidden costs or rate limit surprises + +#### Developer Experience +- **OpenAI compatible API** - migrate by swapping a single URL +- **Drop-in replacement** for closed models with comprehensive observability +- **Seamless scaling** from Model APIs to dedicated deployments + +### Special Features + +#### Function Calling & Tool Use +All Baseten models support structured outputs, function calling, and tool use as part of the Baseten Inference Stack, making them ideal for agentic applications. + +#### Reasoning Capabilities +DeepSeek models offer enhanced reasoning with step-by-step thought processes, while maintaining production-ready performance. + +#### Long Context Support +- **Up to 1 million tokens** for Llama 4 models (Maverick and Scout) +- **262K tokens** for Qwen3 models +- **163K tokens** for DeepSeek models +- **Perfect for code repositories** and complex multi-turn conversations + +#### Quantization Optimizations +Models are deployed with advanced quantization techniques (fp4, fp8, fp16) for optimal performance while maintaining quality. + +### Migration from Other Providers + +Baseten's OpenAI compatibility makes migration straightforward: + +**From OpenAI:** +- Swap `api.openai.com` with `inference.baseten.co/v1` +- Keep existing request/response formats +- Benefit from significant cost savings + +**From Other Providers:** +- Use standard OpenAI SDK format +- Maintain existing prompting strategies +- Access to newer open-source models + +### Tips and Notes + +- **Model Selection:** Choose models based on your specific use case - reasoning models for complex tasks, coding models for development work, and flagship models for general applications. +- **Cost Optimization:** Baseten offers some of the most competitive pricing in the market, especially for open-source models. +- **Context Windows:** Take advantage of large context windows (up to 1M tokens) for including substantial codebases and documentation. +- **Enterprise Ready:** Baseten is designed for production use with enterprise-grade security, compliance, and reliability. +- **Dynamic Model Updates:** Cline automatically fetches the latest model list from Baseten, ensuring access to new models as they're released. +- **Multi-Cloud Capacity Management (MCM):** Baseten's multi-cloud infrastructure ensures high availability and low latency globally. +- **Support:** Baseten provides dedicated support for production deployments and can work with you on dedicated resources as you scale. + +### Pricing Information + +Current pricing is highly competitive and transparent. For the most up-to-date pricing, visit the [Baseten Model APIs page](https://www.baseten.co/products/model-apis/). Prices typically range from $0.10-$6.00 per million tokens, making Baseten significantly more cost-effective than many closed-model alternatives while providing access to state-of-the-art open-source models. diff --git a/src/core/api/providers/baseten.ts b/src/core/api/providers/baseten.ts index 6160fdc026..03a723812f 100644 --- a/src/core/api/providers/baseten.ts +++ b/src/core/api/providers/baseten.ts @@ -146,20 +146,15 @@ export class BasetenHandler implements ApiHandler { } } - /** - * Checks if the current model supports vision/images - */ - supportsImages(): boolean { - const model = this.getModel() - return model.info.supportsImages === true - } - /** * Checks if the current model supports tools */ supportsTools(): boolean { - const _model = this.getModel() - // Baseten models support tools via OpenAI-compatible API - return true + const model = this.getModel() + const modelInfo = model.info as any + + // Use dynamic API data when available, fallback to true since all current Baseten models support tools + // (as of 2025-09-16 - could change if Baseten add non-tool models in future, currently no plans to do so) + return modelInfo.supportedFeatures ? modelInfo.supportedFeatures.includes("tools") : true } } diff --git a/src/core/controller/models/refreshBasetenModels.ts b/src/core/controller/models/refreshBasetenModels.ts index 8daff9e5e5..a8a2fde441 100644 --- a/src/core/controller/models/refreshBasetenModels.ts +++ b/src/core/controller/models/refreshBasetenModels.ts @@ -2,6 +2,7 @@ import { GlobalFileNames } from "@core/storage/disk" import { EmptyRequest } from "@shared/proto/cline/common" import { OpenRouterCompatibleModelInfo, OpenRouterModelInfo } from "@shared/proto/cline/models" import { fileExistsAtPath } from "@utils/fs" +import { parsePrice } from "@utils/model-utils" import axios from "axios" import fs from "fs/promises" import path from "path" @@ -24,11 +25,11 @@ export async function refreshBasetenModels( // Get the Baseten API key from the controller's state const basetenApiKey = controller.stateManager.getSecretKey("basetenApiKey") - const models: Record> = {} + const models: Record & { supportedFeatures?: string[] }> = {} try { if (!basetenApiKey) { console.log("No Baseten API key found, using static models as fallback") - // Don't throw an error, just use static models + // Don't throw an error, just use static models, althought this might be slightly out of date for (const [modelId, modelInfo] of Object.entries(basetenModels)) { models[modelId] = { maxTokens: modelInfo.maxTokens, @@ -69,25 +70,20 @@ export async function refreshBasetenModels( continue } - // Only include models that are listed in the static basetenModels - if (!(rawModel.id in basetenModels)) { - console.log(`Skipping model ${rawModel.id} - not in static basetenModels list`) - continue - } - // Check if we have static pricing information for this model const staticModelInfo = basetenModels[rawModel.id as keyof typeof basetenModels] - const modelInfo: Partial = { - maxTokens: staticModelInfo?.maxTokens || 8192, - contextWindow: staticModelInfo?.contextWindow || 8192, - supportsImages: staticModelInfo?.supportsImages || false, + const modelInfo: Partial & { supportedFeatures?: string[] } = { + maxTokens: rawModel.max_completion_tokens || staticModelInfo?.maxTokens, + contextWindow: rawModel.context_length || staticModelInfo?.contextWindow, + supportsImages: false, // Baseten model APIs does not support image input supportsPromptCache: staticModelInfo?.supportsPromptCache || false, - inputPrice: staticModelInfo?.inputPrice || 0, - outputPrice: staticModelInfo?.outputPrice || 0, + inputPrice: parsePrice(rawModel.pricing?.prompt) || staticModelInfo?.inputPrice || 0, + outputPrice: parsePrice(rawModel.pricing?.completion) || staticModelInfo?.outputPrice || 0, cacheWritesPrice: staticModelInfo?.cacheWritesPrice || 0, cacheReadsPrice: staticModelInfo?.cacheReadsPrice || 0, description: generateModelDescription(rawModel, staticModelInfo), + supportedFeatures: rawModel.supported_features || [], } models[rawModel.id] = modelInfo @@ -125,11 +121,9 @@ export async function refreshBasetenModels( const cachedModels = await readBasetenModels(controller) if (cachedModels && Object.keys(cachedModels).length > 0) { console.log("Using cached Baseten models") - // Filter cached models to only include those in static basetenModels + // Use all cached models (no filtering) for (const [modelId, modelInfo] of Object.entries(cachedModels)) { - if (modelId in basetenModels) { - models[modelId] = modelInfo - } + models[modelId] = modelInfo } } else { // Fall back to static models from shared/api.ts @@ -165,6 +159,7 @@ export async function refreshBasetenModels( cacheReadsPrice: model.cacheReadsPrice ?? 0, description: model.description ?? "", tiers: model.tiers ?? [], + // Note: supportedFeatures is preserved as custom property but not part of OpenRouterModelInfo proto } } @@ -219,14 +214,47 @@ function isValidChatModel(rawModel: any): boolean { * Generates a descriptive name for the model */ function generateModelDescription(rawModel: any, staticModelInfo?: any): string { - // Use static description if available + // Use static description if available and preferred if (staticModelInfo?.description) { return staticModelInfo.description } - // Generate description based on model characteristics - const modelId = rawModel.id - const ownedBy = rawModel.owned_by || "Unknown" + // Use API description if available + if (rawModel.description) { + const contextWindow = rawModel.context_length + const quantization = rawModel.quantization + const features = rawModel.supported_features || [] - return `${ownedBy} model: ${modelId}` + let description = rawModel.description + + // Add technical details if available + const technicalDetails = [] + if (contextWindow) { + technicalDetails.push(`${contextWindow.toLocaleString()} token context`) + } + if (quantization) { + technicalDetails.push(`${quantization} precision`) + } + if (features.length > 0) { + const featureList = features.join(", ") + technicalDetails.push(`supports ${featureList}`) + } + + if (technicalDetails.length > 0) { + description += ` (${technicalDetails.join(", ")})` + } + + return description + } + + // Fallback: use name or model ID + const modelName = rawModel.name || rawModel.id + const contextWindow = rawModel.context_length + const ownedBy = rawModel.owned_by || "Baseten" + + if (contextWindow) { + return `${ownedBy} ${modelName} with ${contextWindow.toLocaleString()} token context window` + } + + return `${ownedBy} model: ${modelName}` } diff --git a/src/shared/api.ts b/src/shared/api.ts index ab1259c548..b10617c650 100644 --- a/src/shared/api.ts +++ b/src/shared/api.ts @@ -3271,8 +3271,35 @@ export const huaweiCloudMaasModels = { // Baseten // https://baseten.co/products/model-apis/ +// Extended ModelInfo to include supportedFeatures, like tools +export interface BasetenModelInfo extends ModelInfo { + supportedFeatures?: string[] +} + export const basetenModels = { - "deepseek-ai/DeepSeek-R1-0528": { + "Qwen/Qwen3-235B-A22B-Instruct-2507": { + maxTokens: 262144, + contextWindow: 262144, + supportsImages: false, + supportsPromptCache: false, + inputPrice: 0.22, + outputPrice: 0.8, + cacheWritesPrice: 0, + cacheReadsPrice: 0, + description: "Mixture-of-experts LLM with math and reasoning capabilities", + }, + "meta-llama/Llama-4-Maverick-17B-128E-Instruct": { + maxTokens: 131072, + contextWindow: 1000000, + supportsImages: false, + supportsPromptCache: false, + inputPrice: 0.19, + outputPrice: 0.72, + cacheWritesPrice: 0, + cacheReadsPrice: 0, + description: "High-efficiency language processing", + }, + "deepseek-ai/DeepSeek-R1": { maxTokens: 131072, contextWindow: 163840, supportsImages: false, @@ -3281,8 +3308,7 @@ export const basetenModels = { outputPrice: 5.95, cacheWritesPrice: 0, cacheReadsPrice: 0, - description: - "DeepSeek R1 0528 - A state-of-the-art 671B-parameter MoE LLM with o1-style reasoning licensed for commercial use.", + description: "DeepSeek's first-generation reasoning model", }, "deepseek-ai/DeepSeek-V3-0324": { maxTokens: 131072, @@ -3293,64 +3319,84 @@ export const basetenModels = { outputPrice: 0.77, cacheWritesPrice: 0, cacheReadsPrice: 0, - description: "DeepSeek V3 0324 - A state-of-the-art 671B-parameter MoE LLM licensed for commercial use.", - }, - "meta-llama/Llama-4-Maverick-17B-128E-Instruct": { - maxTokens: 8192, - contextWindow: 131072, - supportsImages: false, - supportsPromptCache: false, - inputPrice: 0.19, - outputPrice: 0.72, - cacheWritesPrice: 0, - cacheReadsPrice: 0, - description: "Meta's Llama 4 Maverick - A SOTA mixture-of-experts multi-modal LLM with 400 billion total parameters.", + description: "Fast general-purpose LLM with enhanced reasoning capabilities", }, "meta-llama/Llama-4-Scout-17B-16E-Instruct": { - maxTokens: 8192, - contextWindow: 131072, + maxTokens: 131072, + contextWindow: 1000000, supportsImages: false, supportsPromptCache: false, inputPrice: 0.13, outputPrice: 0.5, cacheWritesPrice: 0, cacheReadsPrice: 0, - description: "Meta's Llama 4 Scout - A SOTA mixture-of-experts multi-modal LLM with 109 billion total parameters.", + description: "Precise context understanding with efficient reasoning capabilities", }, - "moonshotai/Kimi-K2-Instruct": { + "deepseek-ai/DeepSeek-V3.1": { maxTokens: 131072, - contextWindow: 131072, + contextWindow: 163840, + supportsImages: false, + supportsPromptCache: false, + inputPrice: 0.5, + outputPrice: 1.5, + cacheWritesPrice: 0, + cacheReadsPrice: 0, + description: "Extremely capable general-purpose LLM with hybrid reasoning capabilities and advanced tool calling", + }, + "Qwen/Qwen3-Coder-480B-A35B-Instruct": { + maxTokens: 262144, + contextWindow: 262144, + supportsImages: false, + supportsPromptCache: false, + inputPrice: 0.38, + outputPrice: 1.53, + cacheWritesPrice: 0, + cacheReadsPrice: 0, + description: "Mixture-of-experts LLM with advanced coding and reasoning capabilities", + }, + "openai/gpt-oss-120b": { + maxTokens: 128072, + contextWindow: 128072, + supportsImages: false, + supportsPromptCache: false, + inputPrice: 0.1, + outputPrice: 0.5, + cacheWritesPrice: 0, + cacheReadsPrice: 0, + description: "Extremely capable general-purpose LLM with strong, controllable reasoning capabilities", + }, + "moonshotai/Kimi-K2-Instruct-0905": { + maxTokens: 168000, + contextWindow: 262000, supportsImages: false, supportsPromptCache: false, inputPrice: 0.6, outputPrice: 2.5, cacheWritesPrice: 0, cacheReadsPrice: 0, - description: "Moonshot AI's Kimi K2 - The world's first 1 trillion parameter open source model.", + description: "State of the art language model for agentic and coding tasks. Septemeber Update.", }, - "Qwen/Qwen3-235B-A22B-Instruct-2507": { - maxTokens: 163800, - contextWindow: 163800, + "moonshotai/Kimi-K2-Instruct": { + maxTokens: 131000, + contextWindow: 131000, supportsImages: false, supportsPromptCache: false, - inputPrice: 0.22, - outputPrice: 0.8, + inputPrice: 0.6, + outputPrice: 2.5, cacheWritesPrice: 0, cacheReadsPrice: 0, - description: - "Qwen3-235B-A22B-Instruct-2507 is a multilingual, instruction-tuned mixture-of-experts language model based on the Qwen3-235B architecture, with 22B active parameters per forward pass.", + description: "State of the art language model for agentic and coding tasks", }, - "Qwen/Qwen3-Coder-480B-A35B-Instruct": { - maxTokens: 163800, - contextWindow: 163800, + "deepseek-ai/DeepSeek-R1-0528": { + maxTokens: 131072, + contextWindow: 163840, supportsImages: false, supportsPromptCache: false, - inputPrice: 1.7, - outputPrice: 1.7, + inputPrice: 2.55, + outputPrice: 5.95, cacheWritesPrice: 0, cacheReadsPrice: 0, - description: - "Qwen3-Coder-480B-A35B-Instruct is a 480B parameter, instruction-tuned, agentic coding model that excels at function calling, tool use, and long-context reasoning over repositories.", + description: "The latest revision of DeepSeek's first-generation reasoning model", }, } as const satisfies Record export type BasetenModelId = keyof typeof basetenModels diff --git a/src/utils/model-utils.ts b/src/utils/model-utils.ts index a87865ed78..d4fe5d1696 100644 --- a/src/utils/model-utils.ts +++ b/src/utils/model-utils.ts @@ -58,6 +58,23 @@ export function isLocalModel(providerInfo: ApiProviderInfo): boolean { return localProviders.includes(normalize(providerInfo.providerId)) } +/** + * Parses a price string and converts it from per-token to per-million-tokens + * @param priceString The price string to parse (e.g. from API responses) + * @returns The price multiplied by 1,000,000 for per-million-token pricing, or 0 if invalid + */ +export function parsePrice(priceString: string | undefined): number { + if (!priceString || priceString === "" || priceString === "0") { + return 0 + } + const parsed = parseFloat(priceString) + if (Number.isNaN(parsed)) { + return 0 + } + // Convert from per-token to per-million-tokens (multiply by 1,000,000) + return parsed * 1_000_000 +} + function normalize(text: string): string { return text.trim().toLowerCase() } diff --git a/webview-ui/src/components/settings/BasetenModelPicker.tsx b/webview-ui/src/components/settings/BasetenModelPicker.tsx index a767579078..9b9b42d727 100644 --- a/webview-ui/src/components/settings/BasetenModelPicker.tsx +++ b/webview-ui/src/components/settings/BasetenModelPicker.tsx @@ -30,12 +30,6 @@ const BasetenModelPicker: React.FC = ({ isPopup, curren const dropdownListRef = useRef(null) const handleModelChange = (newModelId: string) => { - // Only allow selection of models that are in the static basetenModels - if (!(newModelId in basetenModels)) { - console.warn(`Model ${newModelId} is not in the static basetenModels list`) - return - } - // Use dynamic models if available, otherwise fall back to static models const modelInfo = dynamicBasetenModels?.[newModelId] || basetenModels[newModelId as keyof typeof basetenModels] @@ -60,27 +54,13 @@ const BasetenModelPicker: React.FC = ({ isPopup, curren useMount(() => { ModelsServiceClient.refreshBasetenModels(EmptyRequest.create({})) .then((response) => { - // Filter to only include models that are listed in the static basetenModels - const filteredModels: Record = {} - - // Always include the default model - filteredModels[basetenDefaultModelId] = basetenModels[basetenDefaultModelId] - - // Only include models from the API response that exist in static basetenModels - for (const [modelId, modelInfo] of Object.entries(response.models)) { - if (modelId in basetenModels) { - filteredModels[modelId] = modelInfo - } - } - - setBasetenModels(filteredModels) + setBasetenModels({ + [basetenDefaultModelId]: basetenModels[basetenDefaultModelId], + ...response.models, + }) }) .catch((err) => { console.error("Failed to refresh Baseten models:", err) - // On error, fall back to only static models - setBasetenModels({ - [basetenDefaultModelId]: basetenModels[basetenDefaultModelId], - }) }) }) @@ -113,24 +93,8 @@ const BasetenModelPicker: React.FC = ({ isPopup, curren }, []) const allBasetenModels = useMemo(() => { - // Only include models that are listed in the static basetenModels - const filteredModels: Record = {} - - // Start with static models - for (const [modelId, modelInfo] of Object.entries(basetenModels)) { - filteredModels[modelId] = modelInfo - } - - // Override with dynamic models, but only if they exist in static basetenModels - if (dynamicBasetenModels) { - for (const [modelId, modelInfo] of Object.entries(dynamicBasetenModels)) { - if (modelId in basetenModels) { - filteredModels[modelId] = modelInfo - } - } - } - - return filteredModels + // Merge static models with dynamic models, with dynamic taking precedence + return { ...basetenModels, ...(dynamicBasetenModels || {}) } }, [dynamicBasetenModels]) const modelIds = useMemo(() => { @@ -163,6 +127,28 @@ const BasetenModelPicker: React.FC = ({ isPopup, curren return results }, [searchableItems, debouncedSearchTerm, fuse]) + // Safe HTML parser for highlighted search results + const parseHighlightedText = React.useCallback((htmlString: string) => { + // Split by highlight spans and reconstruct as React elements + const parts = htmlString.split(/(.*?<\/span>)/g) + + return parts + .map((part) => { + if (part.startsWith('')) { + // Extract text content from span + const text = part.replace(/(.*?)<\/span>/, "$1") + return ( + + {text} + + ) + } + // Return plain text without wrapping in span + return part || null + }) + .filter((part) => part !== null && part !== "") + }, []) + const handleKeyDown = (event: KeyboardEvent) => { if (!isDropdownVisible) { return @@ -266,17 +252,17 @@ const BasetenModelPicker: React.FC = ({ isPopup, curren className={`px-2.5 py-1.5 cursor-pointer break-all whitespace-normal hover:bg-[var(--vscode-list-activeSelectionBackground)] ${ index === selectedIndex ? "bg-[var(--vscode-list-activeSelectionBackground)]" : "" }`} - dangerouslySetInnerHTML={{ - __html: item.html, - }} key={item.id} onClick={() => { handleModelChange(item.id) setIsDropdownVisible(false) }} onMouseEnter={() => setSelectedIndex(index)} - ref={(el: HTMLDivElement | null) => (itemRefs.current[index] = el)} - /> + ref={(el: HTMLDivElement | null) => { + itemRefs.current[index] = el + }}> + {parseHighlightedText(item.html)} + ))} )}