opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-24 16:03:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(llm): support model import for vllm (#24499)

Browse Source
This commit is contained in:
cwz_eikoh
2026-03-19 15:03:35 +08:00
committed by GitHub
parent dfaf5bca39
commit bdc211a779
3 changed files with 99 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/llm/drivers/llm_container/vllm.go
+91 -23
View File
@@ -2,9 +2,13 @@ package llm_container
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
@@ -474,38 +478,102 @@ func (v *vllm) UninstallModel(ctx context.Context, userCred mcclient.TokenCreden
return nil
}
func (v *vllm) DownloadModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, tmpDir string, modelName string, modelTag string) (string, []string, error) {
lc, err := llm.GetLLMContainer()
func resolveHfdRevision(modelTag string) string {
if strings.TrimSpace(modelTag) == "" {
return "main"
}
return strings.TrimSpace(modelTag)
}
type hfModelAPIResponse struct {
Siblings []struct {
RFilename string `json:"rfilename"`
} `json:"siblings"`
}
func escapeURLPathPreserveSlash(p string) string {
if p == "" {
return ""
}
parts := strings.Split(p, "/")
for i := range parts {
parts[i] = url.PathEscape(parts[i])
}
return strings.Join(parts, "/")
}
func isNonEmptyFile(p string) bool {
st, err := os.Stat(p)
if err != nil {
return "", nil, errors.Wrap(err, "get llm container")
return false
}
return !st.IsDir() && st.Size() > 0
}
func (v *vllm) DownloadModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, tmpDir string, modelName string, modelTag string) (string, []string, error) {
// Download HF model on host into tmpDir for instant-model import.
// We place files under tmpDir/huggingface/<org>/<repo> so that the archive contains relative paths.
if strings.TrimSpace(tmpDir) == "" {
return "", nil, errors.Error("tmpDir is empty")
}
if strings.TrimSpace(modelName) == "" {
return "", nil, errors.Error("modelName is empty")
}
// Logic to download model inside the container
// modelName is expected to be like "facebook/opt-125m"
targetDir := path.Join(api.LLM_VLLM_MODELS_PATH, modelName)
// Check if already exists
checkCmd := fmt.Sprintf("[ -d '%s' ] && echo 'EXIST'", targetDir)
out, _ := exec(ctx, lc.CmpId, checkCmd, 10)
if strings.Contains(out, "EXIST") {
log.Infof("Model %s already exists at %s", modelName, targetDir)
localDir := filepath.Join(tmpDir, "huggingface", filepath.FromSlash(modelName))
if err := os.MkdirAll(localDir, 0755); err != nil {
return "", nil, errors.Wrap(err, "mkdir local model dir")
}
// If already downloaded, short-circuit (directory exists and non-empty).
if entries, err := os.ReadDir(localDir); err == nil && len(entries) > 0 {
targetDir := path.Join(api.LLM_VLLM_MODELS_PATH, modelName)
log.Infof("Model %s already exists in import dir %s", modelName, localDir)
return modelName, []string{targetDir}, nil
}
// Try to use huggingface-cli
// Assuming container has internet access and tools
downloadCmd := fmt.Sprintf("mkdir -p %s && huggingface-cli download %s --local-dir %s --local-dir-use-symlinks False", targetDir, modelName, targetDir)
// If huggingface-cli is missing, try installing it (if pip available)
// fallback to pip install
fullCmd := fmt.Sprintf("if ! command -v huggingface-cli &> /dev/null; then pip install -U huggingface_hub; fi; %s", downloadCmd)
log.Infof("Downloading model %s with cmd: %s", modelName, fullCmd)
_, err = exec(ctx, lc.CmpId, fullCmd, 3600) // 1 hour timeout for large models
rev := resolveHfdRevision(modelTag)
apiURL := fmt.Sprintf("%s/api/models/%s?revision=%s", api.LLM_VLLM_HF_ENDPOINT, escapeURLPathPreserveSlash(modelName), url.QueryEscape(rev))
log.Infof("Downloading HF model via HF Mirror API: %s", func() string {
b, _ := json.Marshal(map[string]string{
"model": modelName,
"revision": rev,
"dir": localDir,
"endpoint": api.LLM_VLLM_HF_ENDPOINT,
"api": apiURL,
})
return string(b)
}())
metaBody, err := llm.HttpGet(ctx, apiURL)
if err != nil {
return "", nil, errors.Wrapf(err, "failed to download model %s", modelName)
return "", nil, errors.Wrapf(err, "fetch hf model metadata failed: %s", apiURL)
}
meta := hfModelAPIResponse{}
if err := json.Unmarshal(metaBody, &meta); err != nil {
return "", nil, errors.Wrap(err, "unmarshal hf model metadata")
}
if len(meta.Siblings) == 0 {
return "", nil, errors.Errorf("hf model metadata has no siblings: %s", apiURL)
}
for _, s := range meta.Siblings {
rf := strings.TrimSpace(s.RFilename)
if rf == "" {
continue
}
dst := filepath.Join(localDir, filepath.FromSlash(rf))
if isNonEmptyFile(dst) {
continue
}
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
return "", nil, errors.Wrapf(err, "mkdir for %s", dst)
}
fileURL := fmt.Sprintf("%s/%s/resolve/%s/%s", api.LLM_VLLM_HF_ENDPOINT, escapeURLPathPreserveSlash(modelName), url.PathEscape(rev), escapeURLPathPreserveSlash(rf))
if err := llm.HttpDownloadFile(ctx, fileURL, dst); err != nil {
return "", nil, errors.Wrapf(err, "download file failed: %s -> %s", fileURL, dst)
}
}
targetDir := path.Join(api.LLM_VLLM_MODELS_PATH, modelName)
return modelName, []string{targetDir}, nil
}
pkg/llm/models/instantmodel.go
+5 -1
View File
@@ -846,7 +846,11 @@ func (model *SInstantModel) DoImport(ctx context.Context, userCred mcclient.Toke
imgFileSize := imgFileStat.Size()
imgParams := imageapi.ImageCreateInput{}
imgParams.GenerateName = fmt.Sprintf("%s-%s", input.ModelName, input.ModelTag)
safeModelName := strings.ReplaceAll(strings.TrimSpace(input.ModelName), "/", "_")
if safeModelName == "" {
safeModelName = "instant-model"
}
imgParams.GenerateName = fmt.Sprintf("%s-%s", safeModelName, strings.TrimSpace(input.ModelTag))
imgParams.DiskFormat = "tgz"
imgParams.Size = &imgFileSize
imgParams.Properties = map[string]string{
pkg/mcclient/options/llm/instantmodel.go
+3 -3
View File
@@ -29,7 +29,7 @@ func (o *LLMInstantModelShowOptions) Params() (jsonutils.JSONObject, error) {
type LLMInstantModelCreateOptions struct {
options.BaseCreateOptions
LLM_TYPE string `help:"llm container type" choices:"ollama" json:"llm_type"`
LLM_TYPE string `help:"llm container type" choices:"ollama|vllm" json:"llm_type"`
MODEL_NAME string `json:"model_name"`
MODEL_TAG string `json:"model_tag"`
@@ -63,8 +63,8 @@ func (o *LLMInstantModelDeleteOptions) Params() (jsonutils.JSONObject, error) {
}
type LLMInstantModelImportOptions struct {
LLM_TYPE string `help:"llm container type" choices:"ollama" json:"llm_type"`
MODEL_NAME string `help:"model name to import, e.g. qwen3" json:"model_name"`
LLM_TYPE string `help:"llm container type" choices:"ollama|vllm" json:"llm_type"`
MODEL_NAME string `help:"model name to import, e.g. qwen3 or Qwen/Qwen3-VL-8B-Instruct" json:"model_name"`
MODEL_TAG string `help:"model tag to import, e.g. 8b" json:"model_tag"`
}