mirror of
https://github.com/cline/cline.git
synced 2026-10-24 02:02:27 +08:00
feat(vscode): add Local Models settings page for managing Ollama models
This commit is contained in:
Saoud Rizwan
@@ -524,6 +524,7 @@
|
||||
"nice-grpc": "^2.1.12",
|
||||
"nice-grpc-common": "^2.0.3",
|
||||
"node-machine-id": "^1.1.12",
|
||||
"ollama": "^0.6.4",
|
||||
"open": "^10.1.2",
|
||||
"open-graph-scraper": "^6.9.0",
|
||||
"openai": "^6.21.0",
|
||||
|
||||
@@ -65,6 +65,71 @@ service ModelsService {
|
||||
rpc writeProviderConfig(WriteProviderConfigRequest) returns (ProviderConfigResponse);
|
||||
// Commits a mode-specific model ID with optional user-authored metadata overrides
|
||||
rpc commitModelSelection(CommitModelSelectionRequest) returns (Empty);
|
||||
// Reports the local machine's hardware and the Ollama server's installed, loaded, and downloading models
|
||||
rpc getLocalModelsStatus(StringRequest) returns (LocalModelsStatus);
|
||||
// Downloads an Ollama model, streaming progress. Re-subscribes if the model is already downloading.
|
||||
rpc pullOllamaModel(OllamaModelRequest) returns (stream OllamaPullProgress);
|
||||
// Cancels an in-progress Ollama model download
|
||||
rpc cancelOllamaPull(OllamaModelRequest) returns (Empty);
|
||||
// Loads an Ollama model into memory
|
||||
rpc loadOllamaModel(OllamaModelRequest) returns (Empty);
|
||||
// Unloads an Ollama model from memory
|
||||
rpc unloadOllamaModel(OllamaModelRequest) returns (Empty);
|
||||
// Deletes a downloaded Ollama model
|
||||
rpc deleteOllamaModel(OllamaModelRequest) returns (Empty);
|
||||
}
|
||||
|
||||
message OllamaModelRequest {
|
||||
string base_url = 1;
|
||||
string model = 2;
|
||||
optional int64 context_window = 3;
|
||||
}
|
||||
|
||||
message LocalGpuInfo {
|
||||
string name = 1;
|
||||
double memory_gb = 2;
|
||||
}
|
||||
|
||||
message LocalHardwareInfo {
|
||||
string platform = 1;
|
||||
string arch = 2;
|
||||
string cpu_model = 3;
|
||||
int32 cpu_cores = 4;
|
||||
double total_memory_gb = 5;
|
||||
// True on Apple Silicon, where the GPU shares system memory
|
||||
bool unified_memory = 6;
|
||||
repeated LocalGpuInfo gpus = 7;
|
||||
}
|
||||
|
||||
message OllamaInstalledModel {
|
||||
string name = 1;
|
||||
int64 size_bytes = 2;
|
||||
string parameter_size = 3;
|
||||
string quantization = 4;
|
||||
}
|
||||
|
||||
message OllamaRunningModel {
|
||||
string name = 1;
|
||||
int64 size_bytes = 2;
|
||||
int64 size_vram_bytes = 3;
|
||||
int64 expires_at_ms = 4;
|
||||
int64 context_length = 5;
|
||||
}
|
||||
|
||||
message OllamaPullProgress {
|
||||
string model = 1;
|
||||
string status = 2;
|
||||
int64 completed = 3;
|
||||
int64 total = 4;
|
||||
}
|
||||
|
||||
message LocalModelsStatus {
|
||||
LocalHardwareInfo hardware = 1;
|
||||
bool ollama_reachable = 2;
|
||||
string ollama_version = 3;
|
||||
repeated OllamaInstalledModel installed = 4;
|
||||
repeated OllamaRunningModel running = 5;
|
||||
repeated OllamaPullProgress active_pulls = 6;
|
||||
}
|
||||
|
||||
// List of VS Code LM models
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Empty } from "@shared/proto/cline/common"
|
||||
import { OllamaModelRequest } from "@shared/proto/cline/models"
|
||||
import { cancelOllamaPull as cancelPull } from "@/services/local-models/ollama"
|
||||
import { Controller } from ".."
|
||||
|
||||
export async function cancelOllamaPull(_controller: Controller, request: OllamaModelRequest): Promise<Empty> {
|
||||
cancelPull(request.model)
|
||||
return Empty.create()
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Empty } from "@shared/proto/cline/common"
|
||||
import { OllamaModelRequest } from "@shared/proto/cline/models"
|
||||
import { createOllamaClient } from "@/services/local-models/ollama"
|
||||
import { Controller } from ".."
|
||||
|
||||
export async function deleteOllamaModel(_controller: Controller, request: OllamaModelRequest): Promise<Empty> {
|
||||
await createOllamaClient(request.baseUrl).delete({ model: request.model })
|
||||
return Empty.create()
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import { StringRequest } from "@shared/proto/cline/common"
|
||||
import { LocalModelsStatus, OllamaInstalledModel, OllamaRunningModel } from "@shared/proto/cline/models"
|
||||
import { getLocalHardwareInfo } from "@/services/local-models/hardware"
|
||||
import { createOllamaClient, getActiveOllamaPulls } from "@/services/local-models/ollama"
|
||||
import { Controller } from ".."
|
||||
|
||||
export async function getLocalModelsStatus(_controller: Controller, request: StringRequest): Promise<LocalModelsStatus> {
|
||||
const hardware = await getLocalHardwareInfo()
|
||||
const client = createOllamaClient(request.value)
|
||||
try {
|
||||
const [{ version }, list, ps] = await Promise.all([client.version(), client.list(), client.ps()])
|
||||
return LocalModelsStatus.create({
|
||||
hardware,
|
||||
ollamaReachable: true,
|
||||
ollamaVersion: version,
|
||||
installed: list.models.map((model) =>
|
||||
OllamaInstalledModel.create({
|
||||
name: model.name,
|
||||
sizeBytes: model.size,
|
||||
parameterSize: model.details?.parameter_size ?? "",
|
||||
quantization: model.details?.quantization_level ?? "",
|
||||
}),
|
||||
),
|
||||
running: ps.models.map((model) =>
|
||||
OllamaRunningModel.create({
|
||||
name: model.name,
|
||||
sizeBytes: model.size,
|
||||
sizeVramBytes: model.size_vram,
|
||||
expiresAtMs: model.expires_at ? new Date(model.expires_at).getTime() : 0,
|
||||
// Reported by Ollama 0.7+, but not yet in the SDK's types.
|
||||
contextLength: (model as { context_length?: number }).context_length ?? 0,
|
||||
}),
|
||||
),
|
||||
activePulls: getActiveOllamaPulls(),
|
||||
})
|
||||
} catch {
|
||||
return LocalModelsStatus.create({ hardware, ollamaReachable: false, activePulls: getActiveOllamaPulls() })
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { Empty } from "@shared/proto/cline/common"
|
||||
import { OllamaModelRequest } from "@shared/proto/cline/models"
|
||||
import { createOllamaClient } from "@/services/local-models/ollama"
|
||||
import { Controller } from ".."
|
||||
|
||||
/**
|
||||
* Loads a model into memory by sending it an empty prompt. `num_ctx` must
|
||||
* match what Cline sends with each request, otherwise Ollama reloads the model
|
||||
* on the first message.
|
||||
*/
|
||||
export async function loadOllamaModel(_controller: Controller, request: OllamaModelRequest): Promise<Empty> {
|
||||
await createOllamaClient(request.baseUrl).generate({
|
||||
model: request.model,
|
||||
prompt: "",
|
||||
// Stay loaded until explicitly unloaded. Cline's own requests reset this to the server's keep-alive.
|
||||
keep_alive: -1,
|
||||
...(request.contextWindow ? { options: { num_ctx: request.contextWindow } } : {}),
|
||||
})
|
||||
return Empty.create()
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
import { OllamaModelRequest, OllamaPullProgress } from "@shared/proto/cline/models"
|
||||
import { getOrStartOllamaPull } from "@/services/local-models/ollama"
|
||||
import { getRequestRegistry, StreamingResponseHandler } from "../grpc-handler"
|
||||
import { Controller } from "../index"
|
||||
|
||||
/**
|
||||
* Streams download progress for an Ollama model. Cancelling the stream only
|
||||
* unsubscribes; the download keeps going until it finishes or
|
||||
* `cancelOllamaPull` is called.
|
||||
*/
|
||||
export async function pullOllamaModel(
|
||||
_controller: Controller,
|
||||
request: OllamaModelRequest,
|
||||
responseStream: StreamingResponseHandler<OllamaPullProgress>,
|
||||
requestId?: string,
|
||||
): Promise<void> {
|
||||
const model = request.model.trim()
|
||||
if (!model) {
|
||||
throw new Error("Model name is required")
|
||||
}
|
||||
|
||||
const pull = getOrStartOllamaPull(request.baseUrl, model)
|
||||
let subscribed = true
|
||||
const listener = (progress: OllamaPullProgress) => {
|
||||
void responseStream(progress, false).catch(() => {})
|
||||
}
|
||||
const unsubscribe = () => {
|
||||
subscribed = false
|
||||
pull.listeners.delete(listener)
|
||||
}
|
||||
pull.listeners.add(listener)
|
||||
if (requestId) {
|
||||
getRequestRegistry().registerRequest(requestId, unsubscribe, { type: "ollama_pull" }, responseStream)
|
||||
}
|
||||
listener(pull.progress)
|
||||
|
||||
try {
|
||||
await pull.done
|
||||
if (subscribed) {
|
||||
await responseStream(OllamaPullProgress.create({ model, status: "success" }), true)
|
||||
}
|
||||
} catch (error) {
|
||||
if (subscribed) {
|
||||
throw error
|
||||
}
|
||||
} finally {
|
||||
if (requestId) {
|
||||
getRequestRegistry().cancelRequest(requestId)
|
||||
}
|
||||
unsubscribe()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Empty } from "@shared/proto/cline/common"
|
||||
import { OllamaModelRequest } from "@shared/proto/cline/models"
|
||||
import { createOllamaClient } from "@/services/local-models/ollama"
|
||||
import { Controller } from ".."
|
||||
|
||||
export async function unloadOllamaModel(_controller: Controller, request: OllamaModelRequest): Promise<Empty> {
|
||||
await createOllamaClient(request.baseUrl).generate({ model: request.model, prompt: "", keep_alive: 0 })
|
||||
return Empty.create()
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { execFile } from "child_process"
|
||||
import os from "os"
|
||||
import { promisify } from "util"
|
||||
import { LocalGpuInfo, LocalHardwareInfo } from "@/shared/proto/cline/models"
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
const BYTES_PER_GB = 1024 ** 3
|
||||
|
||||
let hardwarePromise: Promise<LocalHardwareInfo> | undefined
|
||||
|
||||
async function detectNvidiaGpus(): Promise<LocalGpuInfo[]> {
|
||||
try {
|
||||
const { stdout } = await execFileAsync("nvidia-smi", ["--query-gpu=name,memory.total", "--format=csv,noheader,nounits"], {
|
||||
timeout: 3000,
|
||||
windowsHide: true,
|
||||
})
|
||||
return stdout
|
||||
.split("\n")
|
||||
.map((line) => line.split(",").map((part) => part.trim()))
|
||||
.filter(([name, memoryMib]) => name && Number(memoryMib) > 0)
|
||||
.map(([name, memoryMib]) => LocalGpuInfo.create({ name, memoryGb: Number(memoryMib) / 1024 }))
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
async function detectHardware(): Promise<LocalHardwareInfo> {
|
||||
const platform = os.platform()
|
||||
const arch = os.arch()
|
||||
const cpus = os.cpus()
|
||||
// Apple Silicon GPUs share system memory, so there is no separate VRAM to query.
|
||||
const unifiedMemory = platform === "darwin" && arch === "arm64"
|
||||
return LocalHardwareInfo.create({
|
||||
platform,
|
||||
arch,
|
||||
cpuModel: cpus[0]?.model.trim() ?? "",
|
||||
cpuCores: cpus.length,
|
||||
totalMemoryGb: os.totalmem() / BYTES_PER_GB,
|
||||
unifiedMemory,
|
||||
gpus: unifiedMemory ? [] : await detectNvidiaGpus(),
|
||||
})
|
||||
}
|
||||
|
||||
/** Hardware doesn't change while the extension runs, so detection happens once. */
|
||||
export function getLocalHardwareInfo(): Promise<LocalHardwareInfo> {
|
||||
hardwarePromise ??= detectHardware()
|
||||
return hardwarePromise
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import { Ollama } from "ollama"
|
||||
import { fetch } from "@/shared/net"
|
||||
import { OllamaPullProgress } from "@/shared/proto/cline/models"
|
||||
|
||||
export const DEFAULT_OLLAMA_BASE_URL = "http://localhost:11434"
|
||||
const PROGRESS_THROTTLE_MS = 250
|
||||
|
||||
type PullListener = (progress: OllamaPullProgress) => void
|
||||
|
||||
interface ActivePull {
|
||||
progress: OllamaPullProgress
|
||||
listeners: Set<PullListener>
|
||||
abort: () => void
|
||||
done: Promise<void>
|
||||
}
|
||||
|
||||
// Downloads outlive the settings page that started them (an 18 GB pull takes a
|
||||
// while), so they are tracked here and any reopened page can re-subscribe.
|
||||
const activePulls = new Map<string, ActivePull>()
|
||||
|
||||
export function createOllamaClient(baseUrl: string | undefined): Ollama {
|
||||
const host = baseUrl?.trim().replace(/\/+$/, "") || DEFAULT_OLLAMA_BASE_URL
|
||||
return new Ollama({ host, fetch })
|
||||
}
|
||||
|
||||
export function getActiveOllamaPulls(): OllamaPullProgress[] {
|
||||
return [...activePulls.values()].map((pull) => pull.progress)
|
||||
}
|
||||
|
||||
export function getOrStartOllamaPull(baseUrl: string | undefined, model: string): ActivePull {
|
||||
const existing = activePulls.get(model)
|
||||
if (existing) {
|
||||
return existing
|
||||
}
|
||||
|
||||
const client = createOllamaClient(baseUrl)
|
||||
let aborted = false
|
||||
let abortStream: (() => void) | undefined
|
||||
const pull: ActivePull = {
|
||||
progress: OllamaPullProgress.create({ model, status: "starting" }),
|
||||
listeners: new Set(),
|
||||
abort: () => {
|
||||
aborted = true
|
||||
abortStream?.()
|
||||
},
|
||||
done: Promise.resolve(),
|
||||
}
|
||||
|
||||
pull.done = (async () => {
|
||||
try {
|
||||
const stream = await client.pull({ model, stream: true })
|
||||
abortStream = () => stream.abort()
|
||||
if (aborted) {
|
||||
stream.abort()
|
||||
}
|
||||
let lastNotifiedAt = 0
|
||||
for await (const part of stream) {
|
||||
const statusChanged = part.status !== pull.progress.status
|
||||
pull.progress = OllamaPullProgress.create({
|
||||
model,
|
||||
status: part.status,
|
||||
// Layer-less status lines ("pulling manifest") carry no byte counts; keep the last ones.
|
||||
completed: part.completed ?? (statusChanged ? 0 : pull.progress.completed),
|
||||
total: part.total ?? (statusChanged ? 0 : pull.progress.total),
|
||||
})
|
||||
const now = Date.now()
|
||||
if (statusChanged || now - lastNotifiedAt >= PROGRESS_THROTTLE_MS) {
|
||||
lastNotifiedAt = now
|
||||
for (const listener of pull.listeners) {
|
||||
listener(pull.progress)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (aborted) {
|
||||
throw new Error("Download cancelled")
|
||||
}
|
||||
throw error
|
||||
} finally {
|
||||
activePulls.delete(model)
|
||||
}
|
||||
})()
|
||||
// Rejections are surfaced to subscribers; avoid an unhandled rejection when nobody is listening.
|
||||
pull.done.catch(() => {})
|
||||
|
||||
activePulls.set(model, pull)
|
||||
return pull
|
||||
}
|
||||
|
||||
export function cancelOllamaPull(model: string): void {
|
||||
activePulls.get(model)?.abort()
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { describe, expect, it } from "bun:test"
|
||||
import { evaluateLocalModelFit, getRecommendedLocalModel, type LocalHardware, normalizeOllamaModelName } from "./local-models"
|
||||
|
||||
const mac = (totalMemoryGb: number): LocalHardware => ({ totalMemoryGb, unifiedMemory: true, gpus: [] })
|
||||
const pc = (totalMemoryGb: number, vramGb?: number): LocalHardware => ({
|
||||
totalMemoryGb,
|
||||
unifiedMemory: false,
|
||||
gpus: vramGb ? [{ name: "GPU", memoryGb: vramGb }] : [],
|
||||
})
|
||||
|
||||
describe("local models", () => {
|
||||
it("rates fit against GPU memory first, then system memory", () => {
|
||||
expect(evaluateLocalModelFit(23, pc(64, 24))).toBe("great")
|
||||
expect(evaluateLocalModelFit(23, pc(32, 12))).toBe("slow")
|
||||
expect(evaluateLocalModelFit(56, pc(16, 8))).toBe("too-large")
|
||||
expect(evaluateLocalModelFit(10, pc(32))).toBe("slow")
|
||||
})
|
||||
|
||||
it("recommends Qwen3.8 27B when it fits on the GPU", () => {
|
||||
expect(getRecommendedLocalModel(mac(32))?.id).toBe("qwen3.8:27b")
|
||||
expect(getRecommendedLocalModel(pc(32, 24))?.id).toBe("qwen3.8:27b")
|
||||
})
|
||||
|
||||
it("falls back to smaller models on smaller machines", () => {
|
||||
expect(getRecommendedLocalModel(mac(24))?.id).toBe("gpt-oss:20b")
|
||||
expect(getRecommendedLocalModel(mac(16))?.id).toBe("qwen3:8b")
|
||||
})
|
||||
|
||||
it("prefers mixture-of-experts models when only the CPU can run them", () => {
|
||||
expect(getRecommendedLocalModel(pc(32))?.id).toBe("gpt-oss:20b")
|
||||
expect(getRecommendedLocalModel(pc(4))).toBeUndefined()
|
||||
})
|
||||
|
||||
it("treats untagged Ollama names as :latest", () => {
|
||||
expect(normalizeOllamaModelName("laguna-xs-2.1")).toBe("laguna-xs-2.1:latest")
|
||||
expect(normalizeOllamaModelName("qwen3.8:27b")).toBe("qwen3.8:27b")
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,132 @@
|
||||
/**
|
||||
* Curated Ollama models that work well with Cline, each with the context
|
||||
* window Cline should run it at. Ordered by how strongly we recommend them, so
|
||||
* the first one that fits a machine is its recommendation.
|
||||
*
|
||||
* `memoryGb` estimates weights plus KV cache at `contextWindow`; it is what
|
||||
* decides whether a model fits on the user's hardware.
|
||||
*/
|
||||
export interface LocalModelCatalogEntry {
|
||||
id: string
|
||||
name: string
|
||||
description: string
|
||||
sizeGb: number
|
||||
memoryGb: number
|
||||
contextWindow: number
|
||||
tags: string[]
|
||||
}
|
||||
|
||||
export const LOCAL_MODEL_CATALOG: LocalModelCatalogEntry[] = [
|
||||
{
|
||||
id: "qwen3.8:27b",
|
||||
name: "Qwen3.8 27B",
|
||||
description: "The strongest all-round local model for agentic coding. Reliable tool use across long tasks.",
|
||||
sizeGb: 18,
|
||||
memoryGb: 23,
|
||||
contextWindow: 65536,
|
||||
tags: ["Best overall", "Vision"],
|
||||
},
|
||||
{
|
||||
id: "laguna-xs-2.1",
|
||||
name: "Laguna XS 2.1",
|
||||
description:
|
||||
"Mixture-of-experts model from Poolside built for long-horizon coding. Only 3B active parameters, so it's fast.",
|
||||
sizeGb: 20,
|
||||
memoryGb: 24,
|
||||
contextWindow: 65536,
|
||||
tags: ["Fast"],
|
||||
},
|
||||
{
|
||||
id: "qwen3.6:35b-a3b",
|
||||
name: "Qwen3.6 35B A3B",
|
||||
description: "Fast mixture-of-experts Qwen with solid agentic coding. A good pick when speed matters more than depth.",
|
||||
sizeGb: 24,
|
||||
memoryGb: 26,
|
||||
contextWindow: 65536,
|
||||
tags: ["Fast", "Vision"],
|
||||
},
|
||||
{
|
||||
id: "devstral-small-2:24b",
|
||||
name: "Devstral Small 2",
|
||||
description: "Mistral's agentic coding model, tuned for exploring codebases and editing many files.",
|
||||
sizeGb: 15,
|
||||
memoryGb: 21,
|
||||
contextWindow: 32768,
|
||||
tags: ["Vision"],
|
||||
},
|
||||
{
|
||||
id: "gpt-oss:20b",
|
||||
name: "gpt-oss 20B",
|
||||
description: "OpenAI's open-weight reasoning model. Light on memory and quick, a great fit for 16-24 GB machines.",
|
||||
sizeGb: 14,
|
||||
memoryGb: 16,
|
||||
contextWindow: 65536,
|
||||
tags: ["Fast"],
|
||||
},
|
||||
{
|
||||
id: "qwen3-coder-next",
|
||||
name: "Qwen3-Coder-Next",
|
||||
description: "80B mixture-of-experts coder with 3B active parameters, for machines with 64 GB or more.",
|
||||
sizeGb: 52,
|
||||
memoryGb: 56,
|
||||
contextWindow: 65536,
|
||||
tags: ["Fast"],
|
||||
},
|
||||
{
|
||||
id: "qwen3:8b",
|
||||
name: "Qwen3 8B",
|
||||
description: "Small enough for 8-16 GB machines. Handles simple edits, but struggles with larger multi-step tasks.",
|
||||
sizeGb: 5.2,
|
||||
memoryGb: 10,
|
||||
contextWindow: 32768,
|
||||
tags: ["Lightweight"],
|
||||
},
|
||||
]
|
||||
|
||||
/** Context window for installed models that aren't in the catalog; matches the SDK's Ollama default. */
|
||||
export const DEFAULT_LOCAL_CONTEXT_WINDOW = 32768
|
||||
|
||||
export interface LocalHardware {
|
||||
totalMemoryGb: number
|
||||
unifiedMemory: boolean
|
||||
gpus: { name: string; memoryGb: number }[]
|
||||
}
|
||||
|
||||
/**
|
||||
* - `great`: fits entirely in GPU (or Apple unified) memory
|
||||
* - `slow`: only fits by spilling onto the CPU
|
||||
* - `too-large`: doesn't fit in memory at all
|
||||
*/
|
||||
export type LocalModelFit = "great" | "slow" | "too-large"
|
||||
|
||||
export function getLocalMemoryBudget(hardware: LocalHardware): { gpuGb: number; totalGb: number } {
|
||||
if (hardware.unifiedMemory) {
|
||||
// macOS lets the GPU use roughly three quarters of unified memory.
|
||||
return { gpuGb: hardware.totalMemoryGb * 0.75, totalGb: hardware.totalMemoryGb * 0.85 }
|
||||
}
|
||||
const gpuGb = hardware.gpus.reduce((sum, gpu) => sum + gpu.memoryGb, 0)
|
||||
return { gpuGb, totalGb: gpuGb + hardware.totalMemoryGb * 0.7 }
|
||||
}
|
||||
|
||||
export function evaluateLocalModelFit(memoryGb: number, hardware: LocalHardware): LocalModelFit {
|
||||
const { gpuGb, totalGb } = getLocalMemoryBudget(hardware)
|
||||
if (memoryGb <= gpuGb) {
|
||||
return "great"
|
||||
}
|
||||
return memoryGb <= totalGb ? "slow" : "too-large"
|
||||
}
|
||||
|
||||
export function getRecommendedLocalModel(hardware: LocalHardware): LocalModelCatalogEntry | undefined {
|
||||
const great = LOCAL_MODEL_CATALOG.find((entry) => evaluateLocalModelFit(entry.memoryGb, hardware) === "great")
|
||||
if (great) {
|
||||
return great
|
||||
}
|
||||
// Mixture-of-experts models only compute a few billion parameters per token, so they hold up far better on CPU.
|
||||
const slow = LOCAL_MODEL_CATALOG.filter((entry) => evaluateLocalModelFit(entry.memoryGb, hardware) === "slow")
|
||||
return slow.find((entry) => entry.tags.includes("Fast")) ?? slow[0]
|
||||
}
|
||||
|
||||
/** Ollama treats an untagged name as `:latest`. */
|
||||
export function normalizeOllamaModelName(name: string): string {
|
||||
return name.includes(":") ? name : `${name}:latest`
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import { ResetStateRequest } from "@shared/proto/cline/state"
|
||||
import type { UserOrganization } from "@shared/proto/index.cline"
|
||||
import {
|
||||
CheckCheck,
|
||||
Cpu,
|
||||
FlaskConical,
|
||||
HardDriveDownload,
|
||||
Info,
|
||||
@@ -29,6 +30,7 @@ import ApiConfigurationSection from "./sections/ApiConfigurationSection"
|
||||
import DebugSection from "./sections/DebugSection"
|
||||
import FeatureSettingsSection from "./sections/FeatureSettingsSection"
|
||||
import GeneralSettingsSection from "./sections/GeneralSettingsSection"
|
||||
import LocalModelsSection from "./sections/LocalModelsSection"
|
||||
import { RemoteConfigSection } from "./sections/RemoteConfigSection"
|
||||
import TerminalSettingsSection from "./sections/TerminalSettingsSection"
|
||||
import { getNextSettingsNavigationRequestId, resolveSettingsTarget, type SettingsNavigationRequest } from "./settingsTargets"
|
||||
@@ -36,7 +38,7 @@ import { getNextSettingsNavigationRequestId, resolveSettingsTarget, type Setting
|
||||
const IS_DEV = process.env.IS_DEV
|
||||
|
||||
// Tab definitions
|
||||
type SettingsTabID = "api-config" | "features" | "terminal" | "general" | "about" | "debug" | "remote-config"
|
||||
type SettingsTabID = "api-config" | "local-models" | "features" | "terminal" | "general" | "about" | "debug" | "remote-config"
|
||||
interface SettingsTab {
|
||||
id: SettingsTabID
|
||||
name: string
|
||||
@@ -54,6 +56,13 @@ const SETTINGS_TABS: SettingsTab[] = [
|
||||
headerText: "API Configuration",
|
||||
icon: SlidersHorizontal,
|
||||
},
|
||||
{
|
||||
id: "local-models",
|
||||
name: "Local Models",
|
||||
tooltipText: "Local Models",
|
||||
headerText: "Local Models",
|
||||
icon: Cpu,
|
||||
},
|
||||
{
|
||||
id: "features",
|
||||
name: "Features",
|
||||
@@ -130,6 +139,7 @@ const SettingsView = ({ navigationRequest, onDone }: SettingsViewProps) => {
|
||||
const TAB_CONTENT_MAP: Record<SettingsTabID, React.FC<any>> = useMemo(
|
||||
() => ({
|
||||
"api-config": ApiConfigurationSection,
|
||||
"local-models": LocalModelsSection,
|
||||
general: GeneralSettingsSection,
|
||||
features: FeatureSettingsSection,
|
||||
terminal: TerminalSettingsSection,
|
||||
|
||||
@@ -0,0 +1,455 @@
|
||||
import {
|
||||
DEFAULT_LOCAL_CONTEXT_WINDOW,
|
||||
evaluateLocalModelFit,
|
||||
getLocalMemoryBudget,
|
||||
getRecommendedLocalModel,
|
||||
LOCAL_MODEL_CATALOG,
|
||||
type LocalModelCatalogEntry,
|
||||
type LocalModelFit,
|
||||
normalizeOllamaModelName,
|
||||
} from "@shared/local-models"
|
||||
import { StringRequest } from "@shared/proto/cline/common"
|
||||
import {
|
||||
type LocalHardwareInfo,
|
||||
type LocalModelsStatus,
|
||||
OllamaModelRequest,
|
||||
type OllamaRunningModel,
|
||||
} from "@shared/proto/cline/models"
|
||||
import { VSCodeLink } from "@vscode/webview-ui-toolkit/react"
|
||||
import { Download, Loader2, Play, RefreshCw, Square, Trash2, X } from "lucide-react"
|
||||
import { type ReactNode, useCallback, useEffect, useMemo, useRef, useState } from "react"
|
||||
import { useEvent } from "react-use"
|
||||
import { Badge } from "@/components/ui/badge"
|
||||
import { Button } from "@/components/ui/button"
|
||||
import { Progress } from "@/components/ui/progress"
|
||||
import { useExtensionState } from "@/context/ExtensionStateContext"
|
||||
import { useProviderConfig } from "@/hooks/useProviderConfig"
|
||||
import { ModelsServiceClient } from "@/services/grpc-client"
|
||||
import Section from "../Section"
|
||||
|
||||
interface LocalModelsSectionProps {
|
||||
renderSectionHeader: (tabId: string) => JSX.Element | null
|
||||
}
|
||||
|
||||
interface PullState {
|
||||
status: string
|
||||
completed: number
|
||||
total: number
|
||||
}
|
||||
|
||||
type ModelAction = "using" | "loading" | "unloading" | "deleting"
|
||||
|
||||
const FIT_LABELS: Record<LocalModelFit, { text: string; className: string }> = {
|
||||
great: { text: "Runs well on this machine", className: "text-success" },
|
||||
slow: { text: "Will run slowly on this machine", className: "text-warning" },
|
||||
"too-large": { text: "Too large for this machine", className: "text-error" },
|
||||
}
|
||||
|
||||
const formatGb = (bytes: number) => `${(bytes / 1e9).toFixed(1)} GB`
|
||||
const formatContext = (tokens: number) => `${Math.round(tokens / 1024)}K context`
|
||||
|
||||
function describeHardware(hardware: LocalHardwareInfo): string {
|
||||
const memory = `${Math.round(hardware.totalMemoryGb)} GB`
|
||||
if (hardware.unifiedMemory) {
|
||||
return `${hardware.cpuModel} · ${memory} unified memory`
|
||||
}
|
||||
if (hardware.gpus.length > 0) {
|
||||
const gpus = hardware.gpus.map((gpu) => `${gpu.name} (${Math.round(gpu.memoryGb)} GB)`).join(", ")
|
||||
return `${gpus} · ${memory} RAM`
|
||||
}
|
||||
return `${hardware.cpuModel} · ${memory} RAM · No supported GPU detected`
|
||||
}
|
||||
|
||||
function describeRunning(running: OllamaRunningModel): string {
|
||||
const placement =
|
||||
running.sizeVramBytes <= 0
|
||||
? "on CPU"
|
||||
: running.sizeVramBytes >= running.sizeBytes
|
||||
? "on GPU"
|
||||
: `${Math.round((running.sizeVramBytes / running.sizeBytes) * 100)}% on GPU`
|
||||
const parts = [`Loaded ${placement}`, formatGb(running.sizeBytes)]
|
||||
if (running.contextLength > 0) {
|
||||
parts.push(formatContext(running.contextLength))
|
||||
}
|
||||
// Ollama reports models pinned with keep_alive -1 as expiring centuries from now.
|
||||
const oneYearMs = 365 * 24 * 60 * 60 * 1000
|
||||
if (running.expiresAtMs > 0 && running.expiresAtMs - Date.now() < oneYearMs) {
|
||||
const time = new Date(running.expiresAtMs).toLocaleTimeString([], { hour: "numeric", minute: "2-digit" })
|
||||
parts.push(`unloads at ${time} if idle`)
|
||||
}
|
||||
return parts.join(" · ")
|
||||
}
|
||||
|
||||
const LocalModelsSection = ({ renderSectionHeader }: LocalModelsSectionProps) => {
|
||||
const { apiConfiguration, mode, planActSeparateModelsSetting } = useExtensionState()
|
||||
const { config, write, commitSelection } = useProviderConfig("ollama")
|
||||
const baseUrl = config?.baseUrl ?? apiConfiguration?.ollamaBaseUrl ?? ""
|
||||
|
||||
const [status, setStatus] = useState<LocalModelsStatus | undefined>(undefined)
|
||||
const [refreshing, setRefreshing] = useState(false)
|
||||
const [pulls, setPulls] = useState<Record<string, PullState>>({})
|
||||
const [actions, setActions] = useState<Record<string, ModelAction>>({})
|
||||
const [errors, setErrors] = useState<Record<string, string>>({})
|
||||
const pullSubscriptions = useRef(new Map<string, () => void>())
|
||||
|
||||
const setModelError = useCallback((model: string, message: string | undefined) => {
|
||||
setErrors(({ [model]: _, ...rest }) => (message ? { ...rest, [model]: message } : rest))
|
||||
}, [])
|
||||
|
||||
const refresh = useCallback(async () => {
|
||||
setRefreshing(true)
|
||||
try {
|
||||
setStatus(await ModelsServiceClient.getLocalModelsStatus(StringRequest.create({ value: baseUrl })))
|
||||
} catch (error) {
|
||||
console.error("Failed to read local models status:", error)
|
||||
} finally {
|
||||
setRefreshing(false)
|
||||
}
|
||||
}, [baseUrl])
|
||||
|
||||
const subscribeToPull = useCallback(
|
||||
(model: string) => {
|
||||
if (pullSubscriptions.current.has(model)) {
|
||||
return
|
||||
}
|
||||
setModelError(model, undefined)
|
||||
setPulls((prev) => ({ ...prev, [model]: prev[model] ?? { status: "starting", completed: 0, total: 0 } }))
|
||||
const finish = () => {
|
||||
pullSubscriptions.current.delete(model)
|
||||
setPulls(({ [model]: _, ...rest }) => rest)
|
||||
void refresh()
|
||||
}
|
||||
const cancel = ModelsServiceClient.pullOllamaModel(OllamaModelRequest.create({ baseUrl, model }), {
|
||||
onResponse: (progress) => {
|
||||
if (progress.status !== "success") {
|
||||
setPulls((prev) => ({ ...prev, [model]: progress }))
|
||||
}
|
||||
},
|
||||
onError: (error) => {
|
||||
if (error.message !== "Download cancelled") {
|
||||
setModelError(model, error.message)
|
||||
}
|
||||
},
|
||||
onComplete: finish,
|
||||
})
|
||||
pullSubscriptions.current.set(model, cancel)
|
||||
},
|
||||
[baseUrl, refresh, setModelError],
|
||||
)
|
||||
|
||||
useEffect(() => {
|
||||
void refresh()
|
||||
}, [refresh])
|
||||
|
||||
// Unsubscribing leaves downloads running on the host; reopening this page picks them back up.
|
||||
useEffect(() => {
|
||||
for (const pull of status?.activePulls ?? []) {
|
||||
subscribeToPull(pull.model)
|
||||
}
|
||||
}, [status, subscribeToPull])
|
||||
useEffect(() => {
|
||||
const subscriptions = pullSubscriptions.current
|
||||
return () => {
|
||||
for (const cancel of subscriptions.values()) {
|
||||
cancel()
|
||||
}
|
||||
subscriptions.clear()
|
||||
}
|
||||
}, [])
|
||||
|
||||
// Loaded models can be unloaded by Ollama's idle timeout or from outside Cline.
|
||||
useEvent("focus", () => void refresh())
|
||||
const nextExpiryMs = useMemo(() => {
|
||||
const expiries = (status?.running ?? []).map((model) => model.expiresAtMs).filter((ms) => ms > Date.now())
|
||||
return expiries.length > 0 ? Math.min(...expiries) : undefined
|
||||
}, [status])
|
||||
useEffect(() => {
|
||||
if (!nextExpiryMs || nextExpiryMs - Date.now() > 60 * 60 * 1000) {
|
||||
return
|
||||
}
|
||||
const timer = setTimeout(() => void refresh(), nextExpiryMs - Date.now() + 1000)
|
||||
return () => clearTimeout(timer)
|
||||
}, [nextExpiryMs, refresh])
|
||||
|
||||
const runAction = useCallback(
|
||||
async (model: string, action: ModelAction, run: () => Promise<unknown>) => {
|
||||
setActions((prev) => ({ ...prev, [model]: action }))
|
||||
setModelError(model, undefined)
|
||||
try {
|
||||
await run()
|
||||
} catch (error) {
|
||||
setModelError(model, error instanceof Error ? error.message : String(error))
|
||||
} finally {
|
||||
setActions(({ [model]: _, ...rest }) => rest)
|
||||
void refresh()
|
||||
}
|
||||
},
|
||||
[refresh, setModelError],
|
||||
)
|
||||
|
||||
const currentProvider = mode === "plan" ? apiConfiguration?.planModeApiProvider : apiConfiguration?.actModeApiProvider
|
||||
const selectedModelId = (mode === "plan" ? config?.planSelection : config?.actSelection)?.modelId
|
||||
const activeModel = currentProvider === "ollama" && selectedModelId ? normalizeOllamaModelName(selectedModelId) : undefined
|
||||
|
||||
const contextWindowFor = useCallback(
|
||||
(model: string) => {
|
||||
if (model === activeModel && config?.contextWindow) {
|
||||
return config.contextWindow
|
||||
}
|
||||
return (
|
||||
LOCAL_MODEL_CATALOG.find((entry) => normalizeOllamaModelName(entry.id) === model)?.contextWindow ??
|
||||
DEFAULT_LOCAL_CONTEXT_WINDOW
|
||||
)
|
||||
},
|
||||
[activeModel, config?.contextWindow],
|
||||
)
|
||||
|
||||
const loadModel = useCallback(
|
||||
(model: string) =>
|
||||
ModelsServiceClient.loadOllamaModel(
|
||||
OllamaModelRequest.create({ baseUrl, model, contextWindow: contextWindowFor(model) }),
|
||||
),
|
||||
[baseUrl, contextWindowFor],
|
||||
)
|
||||
|
||||
const useWithCline = useCallback(
|
||||
(model: string, contextWindow: number) =>
|
||||
runAction(model, "using", async () => {
|
||||
await write({ contextWindow })
|
||||
const modes = planActSeparateModelsSetting ? [mode] : (["plan", "act"] as const)
|
||||
for (const targetMode of modes) {
|
||||
await commitSelection(targetMode, { providerId: "ollama", modelId: model })
|
||||
}
|
||||
// Warm the model up so the first message doesn't wait on a cold load.
|
||||
await ModelsServiceClient.loadOllamaModel(OllamaModelRequest.create({ baseUrl, model, contextWindow }))
|
||||
}),
|
||||
[baseUrl, commitSelection, mode, planActSeparateModelsSetting, runAction, write],
|
||||
)
|
||||
|
||||
const installed = useMemo(
|
||||
() => new Map((status?.installed ?? []).map((model) => [normalizeOllamaModelName(model.name), model])),
|
||||
[status],
|
||||
)
|
||||
const running = useMemo(
|
||||
() => new Map((status?.running ?? []).map((model) => [normalizeOllamaModelName(model.name), model])),
|
||||
[status],
|
||||
)
|
||||
const hardware = status?.hardware
|
||||
const recommended = hardware ? getRecommendedLocalModel(hardware) : undefined
|
||||
const catalogIds = new Set(LOCAL_MODEL_CATALOG.map((entry) => normalizeOllamaModelName(entry.id)))
|
||||
const otherInstalled = [...installed.entries()].filter(([name]) => !catalogIds.has(name))
|
||||
const reachable = status?.ollamaReachable ?? false
|
||||
|
||||
const renderModelRow = (
|
||||
model: string,
|
||||
{ title, entry, details }: { title: string; entry?: LocalModelCatalogEntry; details: ReactNode },
|
||||
) => {
|
||||
const name = normalizeOllamaModelName(model)
|
||||
const isInstalled = installed.has(name)
|
||||
const runningModel = running.get(name)
|
||||
const pull = pulls[model]
|
||||
const action = actions[name]
|
||||
const error = errors[model] ?? errors[name]
|
||||
const isActive = name === activeModel
|
||||
const fit = entry && hardware ? evaluateLocalModelFit(entry.memoryGb, hardware) : undefined
|
||||
const pullPercent = pull && pull.total > 0 ? (pull.completed / pull.total) * 100 : 0
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-1.5 p-3 rounded-sm border border-editor-group-border" key={model}>
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
<span className="font-semibold">{title}</span>
|
||||
{entry && entry === recommended && <Badge variant="cline">Recommended for you</Badge>}
|
||||
{isActive && <Badge variant="success">In use</Badge>}
|
||||
{entry?.tags.map((tag) => (
|
||||
<Badge key={tag} variant="outline">
|
||||
{tag}
|
||||
</Badge>
|
||||
))}
|
||||
</div>
|
||||
{entry && <p className="text-xs text-description m-0">{entry.description}</p>}
|
||||
<div className="text-xs text-description">
|
||||
{details}
|
||||
{fit && <span className={FIT_LABELS[fit].className}> · {FIT_LABELS[fit].text}</span>}
|
||||
</div>
|
||||
{runningModel && (
|
||||
<div className="text-xs flex items-center gap-1.5">
|
||||
<span className="inline-block size-2 rounded-full bg-success" />
|
||||
{describeRunning(runningModel)}
|
||||
</div>
|
||||
)}
|
||||
{pull && (
|
||||
<div className="flex flex-col gap-1">
|
||||
<Progress className="h-1.5" value={pullPercent} />
|
||||
<div className="text-xs text-description flex justify-between gap-2">
|
||||
<span className="truncate">
|
||||
{pull.status.startsWith("pulling") && pull.total > 0
|
||||
? `Downloading ${formatGb(pull.completed)} of ${formatGb(pull.total)}`
|
||||
: pull.status}
|
||||
</span>
|
||||
{pull.total > 0 && <span>{Math.floor(pullPercent)}%</span>}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{error && <p className="text-xs text-error m-0 break-words">{error}</p>}
|
||||
<div className="flex gap-2 flex-wrap mt-1">
|
||||
{!isInstalled && !pull && (
|
||||
<Button
|
||||
disabled={!reachable}
|
||||
onClick={() => subscribeToPull(model)}
|
||||
size="sm"
|
||||
variant={fit === "too-large" ? "secondary" : "default"}>
|
||||
<Download />
|
||||
Download{entry ? ` (${entry.sizeGb} GB)` : ""}
|
||||
</Button>
|
||||
)}
|
||||
{pull && (
|
||||
<Button
|
||||
onClick={() =>
|
||||
void ModelsServiceClient.cancelOllamaPull(OllamaModelRequest.create({ baseUrl, model }))
|
||||
}
|
||||
size="sm"
|
||||
variant="secondary">
|
||||
<X />
|
||||
Cancel
|
||||
</Button>
|
||||
)}
|
||||
{isInstalled && !isActive && (
|
||||
<Button
|
||||
disabled={!!action}
|
||||
onClick={() => void useWithCline(name, entry?.contextWindow ?? DEFAULT_LOCAL_CONTEXT_WINDOW)}
|
||||
size="sm">
|
||||
{action === "using" ? <Loader2 className="animate-spin" /> : null}
|
||||
Use with Cline
|
||||
</Button>
|
||||
)}
|
||||
{isInstalled && !runningModel && (
|
||||
<Button
|
||||
disabled={!!action}
|
||||
onClick={() => void runAction(name, "loading", () => loadModel(name))}
|
||||
size="sm"
|
||||
variant="secondary">
|
||||
{action === "loading" ? <Loader2 className="animate-spin" /> : <Play />}
|
||||
{action === "loading" ? "Loading…" : "Load"}
|
||||
</Button>
|
||||
)}
|
||||
{runningModel && (
|
||||
<Button
|
||||
disabled={!!action}
|
||||
onClick={() =>
|
||||
void runAction(name, "unloading", () =>
|
||||
ModelsServiceClient.unloadOllamaModel(OllamaModelRequest.create({ baseUrl, model: name })),
|
||||
)
|
||||
}
|
||||
size="sm"
|
||||
variant="secondary">
|
||||
{action === "unloading" ? <Loader2 className="animate-spin" /> : <Square />}
|
||||
Unload
|
||||
</Button>
|
||||
)}
|
||||
{isInstalled && (
|
||||
<Button
|
||||
aria-label={`Delete ${title}`}
|
||||
disabled={!!action}
|
||||
onClick={() =>
|
||||
void runAction(name, "deleting", () =>
|
||||
ModelsServiceClient.deleteOllamaModel(OllamaModelRequest.create({ baseUrl, model: name })),
|
||||
)
|
||||
}
|
||||
size="sm"
|
||||
title="Delete from disk"
|
||||
variant="ghost">
|
||||
{action === "deleting" ? <Loader2 className="animate-spin" /> : <Trash2 />}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
const budget = hardware ? getLocalMemoryBudget(hardware) : undefined
|
||||
|
||||
return (
|
||||
<div>
|
||||
{renderSectionHeader("local-models")}
|
||||
<Section>
|
||||
<p className="text-sm text-description m-0">
|
||||
Run models on your own machine with{" "}
|
||||
<VSCodeLink href="https://ollama.com" style={{ display: "inline", fontSize: "inherit" }}>
|
||||
Ollama
|
||||
</VSCodeLink>
|
||||
. Download one, then click <span className="font-semibold">Use with Cline</span> to switch Cline to it with
|
||||
the right settings.
|
||||
</p>
|
||||
|
||||
{hardware && (
|
||||
<div className="flex flex-col gap-1 p-3 rounded-sm bg-code-block-background">
|
||||
<span className="text-xs uppercase tracking-wide text-description">Your machine</span>
|
||||
<span className="text-sm">{describeHardware(hardware)}</span>
|
||||
{budget && (
|
||||
<span className="text-xs text-description">
|
||||
{budget.gpuGb > 0
|
||||
? `Models needing up to ~${Math.floor(budget.gpuGb)} GB run fully on the GPU.`
|
||||
: "Without a supported GPU, models run on the CPU and respond slowly."}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="flex items-center gap-2 text-sm">
|
||||
<span className={`inline-block size-2 rounded-full ${reachable ? "bg-success" : "bg-error"}`} />
|
||||
<span className="flex-1">
|
||||
{!status
|
||||
? "Checking for Ollama…"
|
||||
: reachable
|
||||
? `Ollama ${status.ollamaVersion} is running`
|
||||
: "Ollama isn't running"}
|
||||
</span>
|
||||
<Button aria-label="Refresh" disabled={refreshing} onClick={() => void refresh()} size="icon" variant="icon">
|
||||
<RefreshCw className={refreshing ? "animate-spin" : ""} />
|
||||
</Button>
|
||||
</div>
|
||||
{status && !reachable && (
|
||||
<p className="text-xs text-description m-0">
|
||||
<VSCodeLink href="https://ollama.com/download" style={{ display: "inline", fontSize: "inherit" }}>
|
||||
Install Ollama
|
||||
</VSCodeLink>{" "}
|
||||
and start it (or run <code>ollama serve</code>). Cline looks for it at{" "}
|
||||
{baseUrl || "http://localhost:11434"}, which you can change in API Configuration.
|
||||
</p>
|
||||
)}
|
||||
|
||||
<h4 className="m-0 mt-2">Recommended models</h4>
|
||||
{LOCAL_MODEL_CATALOG.map((entry) =>
|
||||
renderModelRow(entry.id, {
|
||||
title: entry.name,
|
||||
entry,
|
||||
details: `${entry.id} · ${entry.sizeGb} GB download · needs ~${entry.memoryGb} GB memory · ${formatContext(entry.contextWindow)}`,
|
||||
}),
|
||||
)}
|
||||
|
||||
{otherInstalled.length > 0 && (
|
||||
<>
|
||||
<h4 className="m-0 mt-2">Other downloaded models</h4>
|
||||
{otherInstalled.map(([name, model]) =>
|
||||
renderModelRow(name, {
|
||||
title: model.name,
|
||||
details: [formatGb(model.sizeBytes), model.parameterSize, model.quantization]
|
||||
.filter(Boolean)
|
||||
.join(" · "),
|
||||
}),
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
|
||||
<p className="text-xs text-description m-0 mt-2">
|
||||
Ollama unloads a model after a few minutes without requests to free up memory, so the next message may take a
|
||||
moment while it loads again. Set the <code>OLLAMA_KEEP_ALIVE</code> environment variable for the Ollama server
|
||||
to keep models loaded longer. The context window and server URL can be adjusted under API Configuration →
|
||||
Ollama.
|
||||
</p>
|
||||
</Section>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default LocalModelsSection
|
||||
@@ -3,6 +3,7 @@ export const featureSettingControlId = (featureId: string) => `${featureSettingE
|
||||
|
||||
export const SETTINGS_NAVIGATION_TARGETS = {
|
||||
"api-config": { tabId: "api-config" },
|
||||
"local-models": { tabId: "local-models" },
|
||||
features: { tabId: "features" },
|
||||
checkpoints: {
|
||||
tabId: "features",
|
||||
|
||||
@@ -428,6 +428,7 @@
|
||||
"nice-grpc": "^2.1.12",
|
||||
"nice-grpc-common": "^2.0.3",
|
||||
"node-machine-id": "^1.1.12",
|
||||
"ollama": "^0.6.4",
|
||||
"open": "^10.1.2",
|
||||
"open-graph-scraper": "^6.9.0",
|
||||
"openai": "^6.21.0",
|
||||
@@ -4251,6 +4252,8 @@
|
||||
|
||||
"obug": ["obug@2.1.4", "", {}, "sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA=="],
|
||||
|
||||
"ollama": ["ollama@0.6.4", "", { "dependencies": { "whatwg-fetch": "^3.6.20" } }, "sha512-KzM6b5Sq0TgipeMTdImuELr/V4eNqrTf37PRzUTHCPPieFmboYw6qPn+pX7+lg0nSP1jsKwy4AFkiQFXhqHeZA=="],
|
||||
|
||||
"ollama-ai-provider-v2": ["ollama-ai-provider-v2@4.0.1", "", { "dependencies": { "@ai-sdk/provider": "^4.0.2", "@ai-sdk/provider-utils": "^5.0.5" }, "peerDependencies": { "ai": "^7.0.0", "zod": "^4.0.16" } }, "sha512-JHw/Knt4kOTSwQ0oqorbdgzukK6iE73U3MPUyOuCEBpAQGAzm80CQDwyaXMwF80yFjY21RY9qfzkoLyPFFts/w=="],
|
||||
|
||||
"on-exit-leak-free": ["on-exit-leak-free@2.1.2", "", {}, "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA=="],
|
||||
@@ -5083,6 +5086,8 @@
|
||||
|
||||
"whatwg-encoding": ["whatwg-encoding@3.1.1", "", { "dependencies": { "iconv-lite": "0.6.3" } }, "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ=="],
|
||||
|
||||
"whatwg-fetch": ["whatwg-fetch@3.6.20", "", {}, "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg=="],
|
||||
|
||||
"whatwg-mimetype": ["whatwg-mimetype@4.0.0", "", {}, "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="],
|
||||
|
||||
"whatwg-url": ["whatwg-url@14.2.0", "", { "dependencies": { "tr46": "^5.1.0", "webidl-conversions": "^7.0.0" } }, "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw=="],
|
||||
|
||||
Reference in New Issue
Block a user