diff --git a/pkg/apis/llm/instantmodel.go b/pkg/apis/llm/instantmodel.go index 9bed5bb75d..6a706b06ce 100644 --- a/pkg/apis/llm/instantmodel.go +++ b/pkg/apis/llm/instantmodel.go @@ -6,6 +6,7 @@ import ( const ( InstantModelSourceHuggingFace = "huggingface" + InstantModelSourceModelScope = "model_scope" ) type InstantModelListInput struct { @@ -30,6 +31,7 @@ type InstantModelImportInput struct { Source string `json:"source,omitempty"` RepoId string `json:"repo_id,omitempty"` Revision string `json:"revision,omitempty"` + FilePath string `json:"file_path,omitempty"` // model_scope file glob; empty means full snapshot } type InstantModelCreateInput struct { diff --git a/pkg/apis/llm/instantmodel_modelscope.go b/pkg/apis/llm/instantmodel_modelscope.go new file mode 100644 index 0000000000..a288c4b65a --- /dev/null +++ b/pkg/apis/llm/instantmodel_modelscope.go @@ -0,0 +1,47 @@ +package llm + +type InstantModelModelScopeSearchInput struct { + Q string `json:"q"` + Page int `json:"page,omitempty"` + PageSize int `json:"page_size,omitempty"` +} + +type InstantModelModelScopeSearchOutput struct { + Data []InstantModelModelScopeSearchResult `json:"data"` + Page int `json:"page"` + HasMore bool `json:"has_more"` + Total int `json:"total,omitempty"` +} + +type InstantModelModelScopeSearchResult struct { + ModelId string `json:"model_id"` + Name string `json:"name,omitempty"` + Author string `json:"author,omitempty"` + PipelineTag string `json:"pipeline_tag,omitempty"` + Tags []string `json:"tags,omitempty"` + Downloads int64 `json:"downloads,omitempty"` + Likes int64 `json:"likes,omitempty"` + LastModified string `json:"last_modified,omitempty"` + Supported bool `json:"supported"` + UnsupportedReason string `json:"unsupported_reason,omitempty"` +} + +type InstantModelModelScopeRepoInfoInput struct { + ModelId string `json:"model_id"` + Revision string `json:"revision,omitempty"` +} + +type InstantModelModelScopeRepoInfo struct { + ModelId string `json:"model_id"` + RequestedRevision string `json:"requested_revision,omitempty"` + ResolvedRevision string `json:"resolved_revision,omitempty"` + Siblings []string `json:"siblings,omitempty"` + ConfigPresent bool `json:"config_present"` + SafetensorsPresent bool `json:"safetensors_present"` + GgufPresent bool `json:"gguf_present"` + ReadmePresent bool `json:"readme_present"` + SizeBytes int64 `json:"size_bytes,omitempty"` + Supported bool `json:"supported"` + UnsupportedReason string `json:"unsupported_reason,omitempty"` + ImportMode string `json:"import_mode,omitempty"` +} diff --git a/pkg/llm/drivers/llm_container/comfyui.go b/pkg/llm/drivers/llm_container/comfyui.go index 5d13eba163..17e6ecd0db 100644 --- a/pkg/llm/drivers/llm_container/comfyui.go +++ b/pkg/llm/drivers/llm_container/comfyui.go @@ -388,15 +388,16 @@ func (c *comfyui) UninstallModel(ctx context.Context, userCred mcclient.TokenCre return nil } -func (c *comfyui) DownloadModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, tmpDir string, modelName string, modelTag string, progress func(progress float32)) (string, []string, error) { +func (c *comfyui) DownloadModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, tmpDir string, input api.InstantModelImportInput, progress func(progress float32)) (string, []string, error) { if strings.TrimSpace(tmpDir) == "" { return "", nil, errors.Error("tmpDir is empty") } - if strings.TrimSpace(modelName) == "" { + modelName := resolveImportModelName(input) + if modelName == "" { return "", nil, errors.Error("modelName is empty") } - rev := resolveHfdRevision(modelTag) + rev := resolveImportRevision(input, resolveHfdRevision("")) apiURL := fmt.Sprintf("%s/api/models/%s?revision=%s", api.LLM_COMFYUI_HF_ENDPOINT, escapeURLPathPreserveSlash(modelName), url.QueryEscape(rev)) log.Infof("Downloading HF model for ComfyUI via HF Mirror API: %s", func() string { b, _ := json.Marshal(map[string]string{ diff --git a/pkg/llm/drivers/llm_container/huggingface_download.go b/pkg/llm/drivers/llm_container/huggingface_download.go new file mode 100644 index 0000000000..6d0e0118c7 --- /dev/null +++ b/pkg/llm/drivers/llm_container/huggingface_download.go @@ -0,0 +1,224 @@ +package llm_container + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "os" + "path" + "path/filepath" + "strings" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/llm/models" +) + +type hfModelAPIResponse struct { + Siblings []hfModelSibling `json:"siblings"` +} + +type hfModelSibling struct { + RFilename string `json:"rfilename"` + Size int64 `json:"size"` +} + +func resolveHfdRevision(modelTag string) string { + if strings.TrimSpace(modelTag) == "" { + return "main" + } + return strings.TrimSpace(modelTag) +} + +func escapeURLPathPreserveSlash(p string) string { + if p == "" { + return "" + } + parts := strings.Split(p, "/") + for i := range parts { + parts[i] = url.PathEscape(parts[i]) + } + return strings.Join(parts, "/") +} + +func buildHuggingFaceModelAPIURL(endpoint, modelName, revision string) string { + return fmt.Sprintf("%s/api/models/%s?revision=%s&blobs=true", + strings.TrimRight(endpoint, "/"), + escapeURLPathPreserveSlash(modelName), + url.QueryEscape(revision), + ) +} + +func isNonEmptyFile(p string) bool { + st, err := os.Stat(p) + if err != nil { + return false + } + return !st.IsDir() && st.Size() > 0 +} + +func isCompleteFile(p string, expectedSize int64) bool { + st, err := os.Stat(p) + if err != nil || st.IsDir() { + return false + } + if expectedSize > 0 { + return st.Size() == expectedSize + } + return true +} + +func isHuggingFaceImportComplete(localDir string, siblings []hfModelSibling) bool { + if len(siblings) == 0 { + return false + } + for _, sibling := range siblings { + rf := strings.TrimSpace(sibling.RFilename) + if rf == "" { + continue + } + dst := filepath.Join(localDir, filepath.FromSlash(rf)) + if !isCompleteFile(dst, sibling.Size) { + return false + } + } + return true +} + +func hfSiblingDownloadProgress(localDir string, siblings []hfModelSibling) (int64, int64, int, int) { + totalSize := int64(0) + completedSize := int64(0) + totalFiles := 0 + completedFiles := 0 + for _, sibling := range siblings { + rf := strings.TrimSpace(sibling.RFilename) + if rf == "" { + continue + } + totalFiles++ + if sibling.Size > 0 { + totalSize += sibling.Size + } + dst := filepath.Join(localDir, filepath.FromSlash(rf)) + if isCompleteFile(dst, sibling.Size) { + completedFiles++ + if sibling.Size > 0 { + completedSize += sibling.Size + } + } + } + return totalSize, completedSize, totalFiles, completedFiles +} + +func resolveImportModelName(input api.InstantModelImportInput) string { + if repo := strings.TrimSpace(input.RepoId); repo != "" { + return repo + } + return strings.TrimSpace(input.ModelName) +} + +func resolveImportRevision(input api.InstantModelImportInput, defaultRev string) string { + if rev := strings.TrimSpace(input.Revision); rev != "" { + return rev + } + if rev := strings.TrimSpace(input.ModelTag); rev != "" { + return rev + } + return defaultRev +} + +func downloadHuggingFaceSnapshot( + ctx context.Context, + llm *models.SLLM, + tmpDir string, + input api.InstantModelImportInput, + endpoint string, + modelsPath string, + progress func(progress float32), +) (string, []string, error) { + if strings.TrimSpace(tmpDir) == "" { + return "", nil, errors.Error("tmpDir is empty") + } + modelName := resolveImportModelName(input) + if modelName == "" { + return "", nil, errors.Error("modelName is empty") + } + + modelBase := filepath.Base(modelName) + localDir := filepath.Join(tmpDir, "huggingface", modelBase) + if err := os.MkdirAll(localDir, 0755); err != nil { + return "", nil, errors.Wrap(err, "mkdir local model dir") + } + + rev := resolveImportRevision(input, resolveHfdRevision("")) + apiURL := buildHuggingFaceModelAPIURL(endpoint, modelName, rev) + log.Infof("Downloading HF model via HF Mirror API: %s", func() string { + b, _ := json.Marshal(map[string]string{ + "model": modelName, + "revision": rev, + "dir": localDir, + "endpoint": endpoint, + "api": apiURL, + }) + return string(b) + }()) + metaBody, err := llm.HttpGet(ctx, apiURL) + if err != nil { + return "", nil, errors.Wrapf(err, "fetch hf model metadata failed: %s", apiURL) + } + meta := hfModelAPIResponse{} + if err := json.Unmarshal(metaBody, &meta); err != nil { + return "", nil, errors.Wrap(err, "unmarshal hf model metadata") + } + if len(meta.Siblings) == 0 { + return "", nil, errors.Errorf("hf model metadata has no siblings: %s", apiURL) + } + totalSize, completedSize, totalFiles, completedFiles := hfSiblingDownloadProgress(localDir, meta.Siblings) + if totalSize > 0 { + reportInstantModelDownloadProgress(progress, completedSize, totalSize) + } else { + reportInstantModelStepProgress(progress, completedFiles, totalFiles) + } + if isHuggingFaceImportComplete(localDir, meta.Siblings) { + targetDir := path.Join(modelsPath, modelBase) + log.Infof("Model %s already exists in import dir %s", modelName, localDir) + return modelName, []string{targetDir}, nil + } + + for _, s := range meta.Siblings { + rf := strings.TrimSpace(s.RFilename) + if rf == "" { + continue + } + dst := filepath.Join(localDir, filepath.FromSlash(rf)) + if isCompleteFile(dst, s.Size) { + continue + } + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return "", nil, errors.Wrapf(err, "mkdir for %s", dst) + } + fileURL := fmt.Sprintf("%s/%s/resolve/%s/%s", endpoint, escapeURLPathPreserveSlash(modelName), url.PathEscape(rev), escapeURLPathPreserveSlash(rf)) + fileCompleted := completedSize + fileCompletedSteps := completedFiles + downloadProgress := instantModelFileDownloadProgress(progress, fileCompleted, totalSize, s.Size) + if totalSize <= 0 { + downloadProgress = instantModelStepDownloadProgress(progress, fileCompletedSteps, totalFiles) + } + if err := llm.HttpDownloadFileWithProgress(ctx, fileURL, dst, downloadProgress); err != nil { + return "", nil, errors.Wrapf(err, "download file failed: %s -> %s", fileURL, dst) + } + if totalSize > 0 && s.Size > 0 { + completedSize += s.Size + reportInstantModelDownloadProgress(progress, completedSize, totalSize) + } else if totalSize <= 0 { + completedFiles++ + reportInstantModelStepProgress(progress, completedFiles, totalFiles) + } + } + + targetDir := path.Join(modelsPath, modelBase) + return modelName, []string{targetDir}, nil +} diff --git a/pkg/llm/drivers/llm_container/modelscope_download.go b/pkg/llm/drivers/llm_container/modelscope_download.go new file mode 100644 index 0000000000..b9052e0022 --- /dev/null +++ b/pkg/llm/drivers/llm_container/modelscope_download.go @@ -0,0 +1,147 @@ +package llm_container + +import ( + "context" + "encoding/json" + "os" + "path" + "path/filepath" + "strings" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/llm/hub" + "yunion.io/x/onecloud/pkg/llm/models" +) + +func isModelScopeImportComplete(localDir string, files []hub.ModelScopeFileEntry) bool { + if len(files) == 0 { + return false + } + for _, f := range files { + dst := filepath.Join(localDir, filepath.FromSlash(f.Path)) + if !isCompleteFile(dst, f.Size) { + return false + } + } + return true +} + +func modelScopeDownloadProgress(localDir string, files []hub.ModelScopeFileEntry) (int64, int64, int, int) { + totalSize := int64(0) + completedSize := int64(0) + totalFiles := 0 + completedFiles := 0 + for _, f := range files { + totalFiles++ + if f.Size > 0 { + totalSize += f.Size + } + dst := filepath.Join(localDir, filepath.FromSlash(f.Path)) + if isCompleteFile(dst, f.Size) { + completedFiles++ + if f.Size > 0 { + completedSize += f.Size + } + } + } + return totalSize, completedSize, totalFiles, completedFiles +} + +func downloadModelScopeSnapshot( + ctx context.Context, + llm *models.SLLM, + tmpDir string, + input api.InstantModelImportInput, + modelsPath string, + progress func(progress float32), +) (string, []string, error) { + if strings.TrimSpace(tmpDir) == "" { + return "", nil, errors.Error("tmpDir is empty") + } + modelID := resolveImportModelName(input) + if modelID == "" { + return "", nil, errors.Error("model_id is empty") + } + + modelBase := filepath.Base(modelID) + localDir := filepath.Join(tmpDir, "modelscope", modelBase) + if err := os.MkdirAll(localDir, 0755); err != nil { + return "", nil, errors.Wrap(err, "mkdir local model dir") + } + + revision := resolveImportRevision(input, hub.DefaultModelScopeRevision) + endpoint := hub.ResolveModelScopeEndpoint() + + allFiles, err := hub.FetchModelScopeFiles(ctx, modelID, revision) + if err != nil { + return "", nil, err + } + if len(allFiles) == 0 { + return "", nil, errors.Errorf("modelscope model %s has no downloadable files", modelID) + } + + files, err := hub.MatchModelScopeFilePaths(allFiles, input.FilePath) + if err != nil { + return "", nil, err + } + + log.Infof("Downloading ModelScope model: %s", func() string { + b, _ := json.Marshal(map[string]string{ + "model_id": modelID, + "revision": revision, + "dir": localDir, + "endpoint": endpoint, + "pattern": input.FilePath, + }) + return string(b) + }()) + + totalSize, completedSize, totalFiles, completedFiles := modelScopeDownloadProgress(localDir, files) + if totalSize > 0 { + reportInstantModelDownloadProgress(progress, completedSize, totalSize) + } else { + reportInstantModelStepProgress(progress, completedFiles, totalFiles) + } + if isModelScopeImportComplete(localDir, files) { + targetDir := path.Join(modelsPath, modelBase) + log.Infof("ModelScope model %s already exists in import dir %s", modelID, localDir) + return modelID, []string{targetDir}, nil + } + + for _, f := range files { + rf := strings.TrimSpace(f.Path) + if rf == "" { + continue + } + dst := filepath.Join(localDir, filepath.FromSlash(rf)) + if isCompleteFile(dst, f.Size) { + continue + } + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return "", nil, errors.Wrapf(err, "mkdir for %s", dst) + } + fileURL := hub.BuildModelScopeFileDownloadURL(endpoint, modelID, rf) + fileCompleted := completedSize + fileCompletedSteps := completedFiles + downloadProgress := instantModelFileDownloadProgress(progress, fileCompleted, totalSize, f.Size) + if totalSize <= 0 { + downloadProgress = instantModelStepDownloadProgress(progress, fileCompletedSteps, totalFiles) + } + if err := llm.HttpDownloadFileWithProgress(ctx, fileURL, dst, downloadProgress); err != nil { + return "", nil, errors.Wrapf(err, "download file failed: %s -> %s", fileURL, dst) + } + if totalSize > 0 && f.Size > 0 { + completedSize += f.Size + reportInstantModelDownloadProgress(progress, completedSize, totalSize) + } else if totalSize <= 0 { + completedFiles++ + reportInstantModelStepProgress(progress, completedFiles, totalFiles) + } + } + + targetDir := path.Join(modelsPath, modelBase) + return modelID, []string{targetDir}, nil +} diff --git a/pkg/llm/drivers/llm_container/modelscope_download_test.go b/pkg/llm/drivers/llm_container/modelscope_download_test.go new file mode 100644 index 0000000000..39bebdb325 --- /dev/null +++ b/pkg/llm/drivers/llm_container/modelscope_download_test.go @@ -0,0 +1,46 @@ +package llm_container + +import ( + "testing" + + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/llm/hub" +) + +func TestBuildModelScopeFilesURL(t *testing.T) { + got := hub.BuildModelScopeFilesURL("https://www.modelscope.cn", "Qwen/Qwen3-0.6B", "master") + want := "https://www.modelscope.cn/api/v1/models/Qwen/Qwen3-0.6B/repo/files?Recursive=true&Revision=master" + if got != want { + t.Fatalf("BuildModelScopeFilesURL=%q want %q", got, want) + } +} + +func TestBuildModelScopeFileDownloadURL(t *testing.T) { + got := hub.BuildModelScopeFileDownloadURL("https://www.modelscope.cn", "Qwen/Qwen3-0.6B", "config.json") + if got == "" { + t.Fatal("empty download url") + } +} + +func TestMatchModelScopeFilePaths(t *testing.T) { + files := []hub.ModelScopeFileEntry{ + {Path: "config.json", Size: 100}, + {Path: "model.safetensors", Size: 200}, + } + matched, err := hub.MatchModelScopeFilePaths(files, "*.safetensors") + if err != nil { + t.Fatalf("MatchModelScopeFilePaths: %v", err) + } + if len(matched) != 1 || matched[0].Path != "model.safetensors" { + t.Fatalf("unexpected matched: %+v", matched) + } +} + +func TestResolveImportRevisionModelScope(t *testing.T) { + rev := resolveImportRevision(api.InstantModelImportInput{ + Source: api.InstantModelSourceModelScope, + }, hub.DefaultModelScopeRevision) + if rev != hub.DefaultModelScopeRevision { + t.Fatalf("rev=%q", rev) + } +} diff --git a/pkg/llm/drivers/llm_container/ollama.go b/pkg/llm/drivers/llm_container/ollama.go index ff17aa2ef6..49a0ca247e 100644 --- a/pkg/llm/drivers/llm_container/ollama.go +++ b/pkg/llm/drivers/llm_container/ollama.go @@ -198,7 +198,9 @@ func (o *ollama) GetContainerSpecs(ctx context.Context, llm *models.SLLM, image // return err // } -func (o *ollama) DownloadModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, tmpDir string, modelName string, modelTag string, progress func(progress float32)) (string, []string, error) { +func (o *ollama) DownloadModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, tmpDir string, input api.InstantModelImportInput, progress func(progress float32)) (string, []string, error) { + modelName := resolveImportModelName(input) + modelTag := resolveImportRevision(input, "latest") // 1. download manifest from registry namespace, repo := getNamespaceAndRepo(modelName) manifestsUrl := fmt.Sprintf(api.LLM_OLLAMA_LIBRARY_BASE_URL, fmt.Sprintf("%s/%s/manifests/%s", namespace, repo, modelTag)) diff --git a/pkg/llm/drivers/llm_container/openclaw.go b/pkg/llm/drivers/llm_container/openclaw.go index fcaf55f4e2..93ff58a913 100644 --- a/pkg/llm/drivers/llm_container/openclaw.go +++ b/pkg/llm/drivers/llm_container/openclaw.go @@ -436,6 +436,6 @@ func (c *openclaw) UninstallModel(ctx context.Context, userCred mcclient.TokenCr return nil } -func (c *openclaw) DownloadModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, tmpDir string, modelName string, modelTag string, progress func(progress float32)) (string, []string, error) { +func (c *openclaw) DownloadModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, tmpDir string, input api.InstantModelImportInput, progress func(progress float32)) (string, []string, error) { return "", nil, nil } diff --git a/pkg/llm/drivers/llm_container/sglang.go b/pkg/llm/drivers/llm_container/sglang.go index 4d12c43453..b5e8a8c2ac 100644 --- a/pkg/llm/drivers/llm_container/sglang.go +++ b/pkg/llm/drivers/llm_container/sglang.go @@ -2,12 +2,8 @@ package llm_container import ( "context" - "encoding/json" "fmt" - "net/url" - "os" "path" - "path/filepath" "strconv" "strings" "unicode" @@ -431,88 +427,12 @@ func (s *sglang) UninstallModel(ctx context.Context, userCred mcclient.TokenCred return nil } -func (s *sglang) DownloadModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, tmpDir string, modelName string, modelTag string, progress func(progress float32)) (string, []string, error) { - if strings.TrimSpace(tmpDir) == "" { - return "", nil, errors.Error("tmpDir is empty") +func (s *sglang) DownloadModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, tmpDir string, input api.InstantModelImportInput, progress func(progress float32)) (string, []string, error) { + source := strings.ToLower(strings.TrimSpace(input.Source)) + if source == api.InstantModelSourceModelScope { + return downloadModelScopeSnapshot(ctx, llm, tmpDir, input, api.LLM_SGLANG_MODELS_PATH, progress) } - if strings.TrimSpace(modelName) == "" { - return "", nil, errors.Error("modelName is empty") - } - - modelBase := filepath.Base(modelName) - localDir := filepath.Join(tmpDir, "huggingface", modelBase) - if err := os.MkdirAll(localDir, 0755); err != nil { - return "", nil, errors.Wrap(err, "mkdir local model dir") - } - - rev := resolveHfdRevision(modelTag) - apiURL := buildHuggingFaceModelAPIURL(api.LLM_SGLANG_HF_ENDPOINT, modelName, rev) - log.Infof("Downloading HF model via HF Mirror API for SGLang: %s", func() string { - b, _ := json.Marshal(map[string]string{ - "model": modelName, - "revision": rev, - "dir": localDir, - "endpoint": api.LLM_SGLANG_HF_ENDPOINT, - "api": apiURL, - }) - return string(b) - }()) - metaBody, err := llm.HttpGet(ctx, apiURL) - if err != nil { - return "", nil, errors.Wrapf(err, "fetch hf model metadata failed: %s", apiURL) - } - meta := hfModelAPIResponse{} - if err := json.Unmarshal(metaBody, &meta); err != nil { - return "", nil, errors.Wrap(err, "unmarshal hf model metadata") - } - if len(meta.Siblings) == 0 { - return "", nil, errors.Errorf("hf model metadata has no siblings: %s", apiURL) - } - totalSize, completedSize, totalFiles, completedFiles := hfSiblingDownloadProgress(localDir, meta.Siblings) - if totalSize > 0 { - reportInstantModelDownloadProgress(progress, completedSize, totalSize) - } else { - reportInstantModelStepProgress(progress, completedFiles, totalFiles) - } - if isHuggingFaceImportComplete(localDir, meta.Siblings) { - targetDir := path.Join(api.LLM_SGLANG_MODELS_PATH, modelBase) - log.Infof("Model %s already exists in import dir %s", modelName, localDir) - return modelName, []string{targetDir}, nil - } - - for _, sibling := range meta.Siblings { - rf := strings.TrimSpace(sibling.RFilename) - if rf == "" { - continue - } - dst := filepath.Join(localDir, filepath.FromSlash(rf)) - if isCompleteFile(dst, sibling.Size) { - continue - } - if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { - return "", nil, errors.Wrapf(err, "mkdir for %s", dst) - } - fileURL := fmt.Sprintf("%s/%s/resolve/%s/%s", api.LLM_SGLANG_HF_ENDPOINT, escapeURLPathPreserveSlash(modelName), url.PathEscape(rev), escapeURLPathPreserveSlash(rf)) - fileCompleted := completedSize - fileCompletedSteps := completedFiles - downloadProgress := instantModelFileDownloadProgress(progress, fileCompleted, totalSize, sibling.Size) - if totalSize <= 0 { - downloadProgress = instantModelStepDownloadProgress(progress, fileCompletedSteps, totalFiles) - } - if err := llm.HttpDownloadFileWithProgress(ctx, fileURL, dst, downloadProgress); err != nil { - return "", nil, errors.Wrapf(err, "download file failed: %s -> %s", fileURL, dst) - } - if totalSize > 0 && sibling.Size > 0 { - completedSize += sibling.Size - reportInstantModelDownloadProgress(progress, completedSize, totalSize) - } else if totalSize <= 0 { - completedFiles++ - reportInstantModelStepProgress(progress, completedFiles, totalFiles) - } - } - - targetDir := path.Join(api.LLM_SGLANG_MODELS_PATH, modelBase) - return modelName, []string{targetDir}, nil + return downloadHuggingFaceSnapshot(ctx, llm, tmpDir, input, api.LLM_SGLANG_HF_ENDPOINT, api.LLM_SGLANG_MODELS_PATH, progress) } var protectedSGLangArgKeys = map[string]struct{}{ diff --git a/pkg/llm/drivers/llm_container/vllm.go b/pkg/llm/drivers/llm_container/vllm.go index 24b49af9af..963f416871 100644 --- a/pkg/llm/drivers/llm_container/vllm.go +++ b/pkg/llm/drivers/llm_container/vllm.go @@ -2,12 +2,8 @@ package llm_container import ( "context" - "encoding/json" "fmt" - "net/url" - "os" "path" - "path/filepath" "strconv" "strings" "unicode" @@ -667,184 +663,10 @@ func (v *vllm) UninstallModel(ctx context.Context, userCred mcclient.TokenCreden return nil } -func resolveHfdRevision(modelTag string) string { - if strings.TrimSpace(modelTag) == "" { - return "main" +func (v *vllm) DownloadModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, tmpDir string, input api.InstantModelImportInput, progress func(progress float32)) (string, []string, error) { + source := strings.ToLower(strings.TrimSpace(input.Source)) + if source == api.InstantModelSourceModelScope { + return downloadModelScopeSnapshot(ctx, llm, tmpDir, input, api.LLM_VLLM_MODELS_PATH, progress) } - return strings.TrimSpace(modelTag) -} - -type hfModelAPIResponse struct { - Siblings []hfModelSibling `json:"siblings"` -} - -type hfModelSibling struct { - RFilename string `json:"rfilename"` - Size int64 `json:"size"` -} - -func escapeURLPathPreserveSlash(p string) string { - if p == "" { - return "" - } - parts := strings.Split(p, "/") - for i := range parts { - parts[i] = url.PathEscape(parts[i]) - } - return strings.Join(parts, "/") -} - -func buildHuggingFaceModelAPIURL(endpoint, modelName, revision string) string { - return fmt.Sprintf("%s/api/models/%s?revision=%s&blobs=true", - strings.TrimRight(endpoint, "/"), - escapeURLPathPreserveSlash(modelName), - url.QueryEscape(revision), - ) -} - -func isNonEmptyFile(p string) bool { - st, err := os.Stat(p) - if err != nil { - return false - } - return !st.IsDir() && st.Size() > 0 -} - -func isCompleteFile(p string, expectedSize int64) bool { - st, err := os.Stat(p) - if err != nil || st.IsDir() { - return false - } - if expectedSize > 0 { - return st.Size() == expectedSize - } - return true -} - -func isHuggingFaceImportComplete(localDir string, siblings []hfModelSibling) bool { - if len(siblings) == 0 { - return false - } - for _, sibling := range siblings { - rf := strings.TrimSpace(sibling.RFilename) - if rf == "" { - continue - } - dst := filepath.Join(localDir, filepath.FromSlash(rf)) - if !isCompleteFile(dst, sibling.Size) { - return false - } - } - return true -} - -func hfSiblingDownloadProgress(localDir string, siblings []hfModelSibling) (int64, int64, int, int) { - totalSize := int64(0) - completedSize := int64(0) - totalFiles := 0 - completedFiles := 0 - for _, sibling := range siblings { - rf := strings.TrimSpace(sibling.RFilename) - if rf == "" { - continue - } - totalFiles++ - if sibling.Size > 0 { - totalSize += sibling.Size - } - dst := filepath.Join(localDir, filepath.FromSlash(rf)) - if isCompleteFile(dst, sibling.Size) { - completedFiles++ - if sibling.Size > 0 { - completedSize += sibling.Size - } - } - } - return totalSize, completedSize, totalFiles, completedFiles -} - -func (v *vllm) DownloadModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, tmpDir string, modelName string, modelTag string, progress func(progress float32)) (string, []string, error) { - // Download HF model on host into tmpDir for instant-model import. - // We place files under tmpDir/huggingface/ so that the archive contains relative paths. - if strings.TrimSpace(tmpDir) == "" { - return "", nil, errors.Error("tmpDir is empty") - } - if strings.TrimSpace(modelName) == "" { - return "", nil, errors.Error("modelName is empty") - } - - modelBase := filepath.Base(modelName) - localDir := filepath.Join(tmpDir, "huggingface", modelBase) - if err := os.MkdirAll(localDir, 0755); err != nil { - return "", nil, errors.Wrap(err, "mkdir local model dir") - } - - rev := resolveHfdRevision(modelTag) - apiURL := buildHuggingFaceModelAPIURL(api.LLM_VLLM_HF_ENDPOINT, modelName, rev) - log.Infof("Downloading HF model via HF Mirror API: %s", func() string { - b, _ := json.Marshal(map[string]string{ - "model": modelName, - "revision": rev, - "dir": localDir, - "endpoint": api.LLM_VLLM_HF_ENDPOINT, - "api": apiURL, - }) - return string(b) - }()) - metaBody, err := llm.HttpGet(ctx, apiURL) - if err != nil { - return "", nil, errors.Wrapf(err, "fetch hf model metadata failed: %s", apiURL) - } - meta := hfModelAPIResponse{} - if err := json.Unmarshal(metaBody, &meta); err != nil { - return "", nil, errors.Wrap(err, "unmarshal hf model metadata") - } - if len(meta.Siblings) == 0 { - return "", nil, errors.Errorf("hf model metadata has no siblings: %s", apiURL) - } - totalSize, completedSize, totalFiles, completedFiles := hfSiblingDownloadProgress(localDir, meta.Siblings) - if totalSize > 0 { - reportInstantModelDownloadProgress(progress, completedSize, totalSize) - } else { - reportInstantModelStepProgress(progress, completedFiles, totalFiles) - } - if isHuggingFaceImportComplete(localDir, meta.Siblings) { - targetDir := path.Join(api.LLM_VLLM_MODELS_PATH, modelBase) - log.Infof("Model %s already exists in import dir %s", modelName, localDir) - return modelName, []string{targetDir}, nil - } - - for _, s := range meta.Siblings { - rf := strings.TrimSpace(s.RFilename) - if rf == "" { - continue - } - dst := filepath.Join(localDir, filepath.FromSlash(rf)) - if isCompleteFile(dst, s.Size) { - continue - } - if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { - return "", nil, errors.Wrapf(err, "mkdir for %s", dst) - } - fileURL := fmt.Sprintf("%s/%s/resolve/%s/%s", api.LLM_VLLM_HF_ENDPOINT, escapeURLPathPreserveSlash(modelName), url.PathEscape(rev), escapeURLPathPreserveSlash(rf)) - fileCompleted := completedSize - fileCompletedSteps := completedFiles - downloadProgress := instantModelFileDownloadProgress(progress, fileCompleted, totalSize, s.Size) - if totalSize <= 0 { - downloadProgress = instantModelStepDownloadProgress(progress, fileCompletedSteps, totalFiles) - } - if err := llm.HttpDownloadFileWithProgress(ctx, fileURL, dst, downloadProgress); err != nil { - return "", nil, errors.Wrapf(err, "download file failed: %s -> %s", fileURL, dst) - } - if totalSize > 0 && s.Size > 0 { - completedSize += s.Size - reportInstantModelDownloadProgress(progress, completedSize, totalSize) - } else if totalSize <= 0 { - completedFiles++ - reportInstantModelStepProgress(progress, completedFiles, totalFiles) - } - } - - targetDir := path.Join(api.LLM_VLLM_MODELS_PATH, modelBase) - return modelName, []string{targetDir}, nil + return downloadHuggingFaceSnapshot(ctx, llm, tmpDir, input, api.LLM_VLLM_HF_ENDPOINT, api.LLM_VLLM_MODELS_PATH, progress) } diff --git a/pkg/llm/hub/doc.go b/pkg/llm/hub/doc.go new file mode 100644 index 0000000000..4d639d94ad --- /dev/null +++ b/pkg/llm/hub/doc.go @@ -0,0 +1 @@ +package hub // import "yunion.io/x/onecloud/pkg/llm/hub" diff --git a/pkg/llm/hub/modelscope.go b/pkg/llm/hub/modelscope.go new file mode 100644 index 0000000000..eaf4222dab --- /dev/null +++ b/pkg/llm/hub/modelscope.go @@ -0,0 +1,157 @@ +package hub + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "path/filepath" + "strings" + + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/httputils" + + "yunion.io/x/onecloud/pkg/llm/options" +) + +const DefaultModelScopeRevision = "master" + +type ModelScopeFileEntry struct { + Path string + Size int64 +} + +type modelScopeFilesResponse struct { + Data modelScopeFilesData `json:"Data"` +} + +type modelScopeFilesData struct { + Files []modelScopeFileEntryRaw `json:"Files"` +} + +type modelScopeFileEntryRaw struct { + Path string `json:"Path"` + Name string `json:"Name"` + Size int64 `json:"Size"` + Type string `json:"Type"` +} + +func ResolveModelScopeEndpoint() string { + endpoint := strings.TrimSpace(options.Options.ModelScopeEndpoint) + if endpoint == "" { + endpoint = "https://www.modelscope.cn" + } + return strings.TrimRight(endpoint, "/") +} + +func EscapeURLPathPreserveSlash(p string) string { + if p == "" { + return "" + } + parts := strings.Split(p, "/") + for i := range parts { + parts[i] = url.PathEscape(parts[i]) + } + return strings.Join(parts, "/") +} + +func BuildModelScopeFilesURL(endpoint, modelID, revision string) string { + base := fmt.Sprintf("%s/api/v1/models/%s/repo/files?Recursive=true", + endpoint, + EscapeURLPathPreserveSlash(modelID), + ) + if revision != "" { + return base + "&Revision=" + url.QueryEscape(revision) + } + return base +} + +func BuildModelScopeFileDownloadURL(endpoint, modelID, filePath string) string { + return fmt.Sprintf("%s/api/v1/models/%s/repo?FilePath=%s", + endpoint, + EscapeURLPathPreserveSlash(modelID), + url.QueryEscape(filePath), + ) +} + +func ModelScopeHTTPGet(ctx context.Context, reqURL string) ([]byte, error) { + client := httputils.GetTimeoutClient(0) + transport := httputils.GetTransport(true) + client.Transport = transport + + header := http.Header{} + if token := strings.TrimSpace(options.Options.ModelScopeToken); token != "" { + header.Set("Authorization", "Bearer "+token) + } + + resp, err := httputils.Request(client, ctx, httputils.GET, reqURL, header, nil, false) + if err != nil { + return nil, errors.Wrap(err, "http request failed") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, errors.Errorf("unexpected status code: %d for %s", resp.StatusCode, reqURL) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, "read response body") + } + return body, nil +} + +func FetchModelScopeFiles(ctx context.Context, modelID, revision string) ([]ModelScopeFileEntry, error) { + if revision == "" { + revision = DefaultModelScopeRevision + } + reqURL := BuildModelScopeFilesURL(ResolveModelScopeEndpoint(), modelID, revision) + body, err := ModelScopeHTTPGet(ctx, reqURL) + if err != nil { + return nil, errors.Wrapf(err, "fetch modelscope files: %s", reqURL) + } + resp := modelScopeFilesResponse{} + if err := json.Unmarshal(body, &resp); err != nil { + return nil, errors.Wrap(err, "unmarshal modelscope files response") + } + out := make([]ModelScopeFileEntry, 0, len(resp.Data.Files)) + for _, f := range resp.Data.Files { + if strings.EqualFold(f.Type, "tree") { + continue + } + name := strings.TrimSpace(f.Path) + if name == "" { + name = strings.TrimSpace(f.Name) + } + if name == "" || name == ".gitignore" || name == ".gitattributes" { + continue + } + out = append(out, ModelScopeFileEntry{ + Path: name, + Size: f.Size, + }) + } + return out, nil +} + +func MatchModelScopeFilePaths(files []ModelScopeFileEntry, pattern string) ([]ModelScopeFileEntry, error) { + pattern = strings.TrimSpace(pattern) + if pattern == "" { + return files, nil + } + matched := make([]ModelScopeFileEntry, 0) + for _, f := range files { + ok, err := filepath.Match(pattern, f.Path) + if err != nil { + return nil, errors.Wrapf(err, "invalid model_scope file_path pattern %q", pattern) + } + if ok { + matched = append(matched, f) + } + } + if len(matched) == 0 { + return nil, errors.Errorf("no file found in model that matches %q", pattern) + } + return matched, nil +} diff --git a/pkg/llm/models/instantmodel.go b/pkg/llm/models/instantmodel.go index bcadf7e53c..67d3900f49 100644 --- a/pkg/llm/models/instantmodel.go +++ b/pkg/llm/models/instantmodel.go @@ -1217,7 +1217,7 @@ func (model *SInstantModel) DoImport(ctx context.Context, userCred mcclient.Toke } // download model from registry - modelId, mounts, err := drv.DownloadModel(ctx, userCred, nil, tmpDir, input.ModelName, input.ModelTag, progress.setDownloadProgress) + modelId, mounts, err := drv.DownloadModel(ctx, userCred, nil, tmpDir, input, progress.setDownloadProgress) if err != nil { err = errors.Wrap(err, "DownloadModel") return diff --git a/pkg/llm/models/instantmodel_huggingface.go b/pkg/llm/models/instantmodel_huggingface.go index a2336dd8df..bd55c7b593 100644 --- a/pkg/llm/models/instantmodel_huggingface.go +++ b/pkg/llm/models/instantmodel_huggingface.go @@ -442,6 +442,9 @@ func (man *SInstantModelManager) GetPropertyProxy( if isHuggingFaceURL(finalURL) && options.Options.HuggingFaceToken != "" { req.Header.Set("Authorization", "Bearer "+options.Options.HuggingFaceToken) } + if isModelScopeURL(finalURL) && options.Options.ModelScopeToken != "" { + req.Header.Set("Authorization", "Bearer "+options.Options.ModelScopeToken) + } resp, err := proxyHTTPClient.Do(req) if err != nil { @@ -512,6 +515,14 @@ func isHuggingFaceURL(u string) bool { return ep != "" && strings.HasPrefix(u, ep) } +func isModelScopeURL(u string) bool { + if strings.Contains(u, "modelscope.cn") { + return true + } + ep := strings.TrimRight(options.Options.ModelScopeEndpoint, "/") + return ep != "" && strings.HasPrefix(u, ep) +} + func truncateProxyBody(b []byte, n int) string { if len(b) <= n { return string(b) diff --git a/pkg/llm/models/instantmodel_huggingface_import.go b/pkg/llm/models/instantmodel_huggingface_import.go index ebe6ebb178..a93ea7cd4a 100644 --- a/pkg/llm/models/instantmodel_huggingface_import.go +++ b/pkg/llm/models/instantmodel_huggingface_import.go @@ -10,9 +10,13 @@ import ( ) const defaultHuggingFaceRevision = "main" +const defaultModelScopeRevision = "master" func normalizeInstantModelSource(source string, repoID string) string { source = strings.TrimSpace(source) + if strings.EqualFold(source, apis.InstantModelSourceModelScope) { + return apis.InstantModelSourceModelScope + } if strings.EqualFold(source, apis.InstantModelSourceHuggingFace) || (source == "" && strings.TrimSpace(repoID) != "") { return apis.InstantModelSourceHuggingFace } @@ -29,8 +33,15 @@ func resolveImportRepoAndRevision(input apis.InstantModelImportInput) (string, s if revision == "" { revision = strings.TrimSpace(input.ModelTag) } - if source == apis.InstantModelSourceHuggingFace && repoID != "" && revision == "" { - revision = defaultHuggingFaceRevision + switch source { + case apis.InstantModelSourceHuggingFace: + if repoID != "" && revision == "" { + revision = defaultHuggingFaceRevision + } + case apis.InstantModelSourceModelScope: + if repoID != "" && revision == "" { + revision = defaultModelScopeRevision + } } return source, repoID, revision } diff --git a/pkg/llm/models/instantmodel_modelscope.go b/pkg/llm/models/instantmodel_modelscope.go new file mode 100644 index 0000000000..4e45b6880b --- /dev/null +++ b/pkg/llm/models/instantmodel_modelscope.go @@ -0,0 +1,290 @@ +package models + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "path/filepath" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + apis "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/llm/hub" + "yunion.io/x/onecloud/pkg/llm/options" + "yunion.io/x/onecloud/pkg/mcclient" +) + +const ( + modelScopeImportMode = "snapshot" + modelScopeSearchPageSize = 20 +) + +type modelScopeOpenAPISearchResponse struct { + Success bool `json:"success"` + Data modelScopeOpenAPISearchData `json:"data"` +} + +type modelScopeOpenAPISearchData struct { + Models []modelScopeOpenAPIModel `json:"models"` + TotalCount int `json:"total_count"` + PageNumber int `json:"page_number"` + PageSize int `json:"page_size"` +} + +type modelScopeOpenAPIModel struct { + Id string `json:"id"` + DisplayName string `json:"display_name"` + Description string `json:"description"` + Downloads int64 `json:"downloads"` + Likes int64 `json:"likes"` + Tasks []string `json:"tasks"` + Tags []string `json:"tags"` + LastModified string `json:"last_modified"` + Gated bool `json:"gated"` + Private bool `json:"private"` +} + +func (man *SInstantModelManager) GetPropertyModelscopeSearch(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + return man.getPropertyModelScopeSearch(ctx, userCred, query) +} + +func (man *SInstantModelManager) GetPropertyModelScopeSearch(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + return man.getPropertyModelScopeSearch(ctx, userCred, query) +} + +func (man *SInstantModelManager) getPropertyModelScopeSearch(ctx context.Context, _ mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + input := apis.InstantModelModelScopeSearchInput{} + if query != nil { + if err := query.Unmarshal(&input); err != nil { + return nil, errors.Wrap(err, "query.Unmarshal") + } + } + input.Q = strings.TrimSpace(input.Q) + if input.Page <= 0 { + input.Page = 1 + } + if input.PageSize <= 0 { + input.PageSize = modelScopeSearchPageSize + } + if input.PageSize > 100 { + input.PageSize = 100 + } + + endpoint := hub.ResolveModelScopeEndpoint() + reqURL := fmt.Sprintf("%s/openapi/v1/models?search=%s&page_number=%d&page_size=%d", + endpoint, + url.QueryEscape(input.Q), + input.Page, + input.PageSize, + ) + body, err := modelScopeHTTPRequest(ctx, http.MethodGet, reqURL, nil) + if err != nil { + return nil, errors.Wrap(err, "modelScopeHTTPRequest") + } + + resp := modelScopeOpenAPISearchResponse{} + if err := json.Unmarshal(body, &resp); err != nil { + return nil, errors.Wrap(err, "json.Unmarshal") + } + if !resp.Success { + return nil, errors.Errorf("modelscope search failed for %s", reqURL) + } + results := normalizeModelScopeOpenAPISearchResults(resp.Data.Models) + hasMore := input.Page*input.PageSize < resp.Data.TotalCount + return jsonutils.Marshal(apis.InstantModelModelScopeSearchOutput{ + Data: results, + Page: input.Page, + HasMore: hasMore, + Total: resp.Data.TotalCount, + }), nil +} + +func (man *SInstantModelManager) GetPropertyModelscopeRepoInfo(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + return man.getPropertyModelScopeRepoInfo(ctx, userCred, query) +} + +func (man *SInstantModelManager) GetPropertyModelScopeRepoInfo(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + return man.getPropertyModelScopeRepoInfo(ctx, userCred, query) +} + +func (man *SInstantModelManager) getPropertyModelScopeRepoInfo(ctx context.Context, _ mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + input := apis.InstantModelModelScopeRepoInfoInput{} + if query != nil { + if err := query.Unmarshal(&input); err != nil { + return nil, errors.Wrap(err, "query.Unmarshal") + } + } + input.ModelId = strings.TrimSpace(input.ModelId) + if input.ModelId == "" { + return nil, httperrors.NewMissingParameterError("model_id") + } + revision := strings.TrimSpace(input.Revision) + if revision == "" { + revision = defaultModelScopeRevision + } + + info, err := getModelScopeRepoInfo(ctx, input.ModelId, revision) + if err != nil { + return nil, errors.Wrap(err, "getModelScopeRepoInfo") + } + return jsonutils.Marshal(info), nil +} + +func normalizeModelScopeOpenAPISearchResults(items []modelScopeOpenAPIModel) []apis.InstantModelModelScopeSearchResult { + results := make([]apis.InstantModelModelScopeSearchResult, 0, len(items)) + for _, item := range items { + modelID := strings.TrimSpace(item.Id) + if modelID == "" { + continue + } + pipelineTag := "" + if len(item.Tasks) > 0 { + pipelineTag = item.Tasks[0] + } + result := apis.InstantModelModelScopeSearchResult{ + ModelId: modelID, + Name: firstNonEmpty(item.DisplayName, modelID), + PipelineTag: pipelineTag, + Tags: item.Tags, + Downloads: item.Downloads, + Likes: item.Likes, + LastModified: item.LastModified, + Supported: true, + } + switch { + case item.Private: + result.Supported = false + result.UnsupportedReason = "private repositories are not supported in this phase" + case item.Gated: + result.Supported = false + result.UnsupportedReason = "gated repositories are not supported in this phase" + case isModelScopeGgufOnly(item.Tags): + result.Supported = false + result.UnsupportedReason = "gguf repositories are not supported for modelscope snapshot import" + } + results = append(results, result) + } + return results +} + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if strings.TrimSpace(v) != "" { + return strings.TrimSpace(v) + } + } + return "" +} + +func isModelScopeGgufOnly(tags []string) bool { + for _, tag := range tags { + lower := strings.ToLower(strings.TrimSpace(tag)) + if lower == "gguf" || strings.Contains(lower, "library:gguf") { + return true + } + } + return false +} + +func getModelScopeRepoInfo(ctx context.Context, modelID, revision string) (*apis.InstantModelModelScopeRepoInfo, error) { + files, err := hub.FetchModelScopeFiles(ctx, modelID, revision) + if err != nil { + return nil, err + } + info := &apis.InstantModelModelScopeRepoInfo{ + ModelId: modelID, + RequestedRevision: revision, + ResolvedRevision: revision, + Supported: true, + ImportMode: modelScopeImportMode, + } + for _, f := range files { + name := f.Path + info.Siblings = append(info.Siblings, name) + info.SizeBytes += f.Size + base := strings.ToLower(filepath.Base(name)) + switch { + case base == "config.json": + info.ConfigPresent = true + case base == "readme.md": + info.ReadmePresent = true + case strings.HasSuffix(strings.ToLower(name), ".safetensors"): + info.SafetensorsPresent = true + case strings.HasSuffix(strings.ToLower(name), ".gguf"): + info.GgufPresent = true + } + } + switch { + case info.GgufPresent: + info.Supported = false + info.UnsupportedReason = "gguf repositories are not supported for modelscope snapshot import" + case !info.ConfigPresent: + info.Supported = false + info.UnsupportedReason = "config.json is required for modelscope snapshot import" + case !info.SafetensorsPresent: + info.Supported = false + info.UnsupportedReason = "no safetensors weights detected for modelscope snapshot import" + } + if !info.Supported { + info.ImportMode = "" + } + return info, nil +} + +func modelScopeHTTPRequest(ctx context.Context, method, reqURL string, body []byte) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, method, reqURL, http.NoBody) + if err != nil { + return nil, errors.Wrap(err, "http.NewRequestWithContext") + } + if token := strings.TrimSpace(options.Options.ModelScopeToken); token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + client := &http.Client{Timeout: 60 * time.Second} + resp, err := client.Do(req) + if err != nil { + return nil, errors.Wrap(err, "client.Do") + } + defer resp.Body.Close() + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, "io.ReadAll") + } + if resp.StatusCode != http.StatusOK { + return nil, errors.Errorf("unexpected status code: %d for %s", resp.StatusCode, reqURL) + } + return respBody, nil +} + +// FetchModelScopeWeightSize sums root-level weight file sizes from ModelScope. +func FetchModelScopeWeightSize(ctx context.Context, modelID, revision string) (int64, error) { + return fetchModelScopeWeightSize(ctx, modelID, revision) +} + +func fetchModelScopeWeightSize(ctx context.Context, modelID, revision string) (int64, error) { + files, err := hub.FetchModelScopeFiles(ctx, modelID, revision) + if err != nil { + return 0, errors.Wrap(err, "FetchModelScopeFiles") + } + var total int64 + for _, f := range files { + if strings.Contains(f.Path, "/") { + continue + } + if _, skip := huggingFaceWeightExcludeNames[f.Path]; skip { + continue + } + ext := strings.ToLower(filepath.Ext(f.Path)) + if _, ok := huggingFaceWeightExtensions[ext]; !ok { + continue + } + total += f.Size + } + return total, nil +} diff --git a/pkg/llm/models/instantmodel_modelscope_test.go b/pkg/llm/models/instantmodel_modelscope_test.go new file mode 100644 index 0000000000..6d8c5f54c4 --- /dev/null +++ b/pkg/llm/models/instantmodel_modelscope_test.go @@ -0,0 +1,29 @@ +package models + +import ( + "testing" +) + +func TestNormalizeModelScopeOpenAPISearchResults(t *testing.T) { + items := []modelScopeOpenAPIModel{ + {Id: "Qwen/Qwen3-0.6B", Downloads: 10, Likes: 2, Tasks: []string{"text-generation"}}, + {Id: "org/gguf-model", Tags: []string{"library:gguf"}}, + } + out := normalizeModelScopeOpenAPISearchResults(items) + if len(out) != 2 { + t.Fatalf("len=%d", len(out)) + } + if out[0].ModelId != "Qwen/Qwen3-0.6B" || !out[0].Supported { + t.Fatalf("first=%+v", out[0]) + } + if out[1].Supported { + t.Fatalf("gguf should be unsupported: %+v", out[1]) + } +} + +func TestResolveModelCatalogSourceCustomURL(t *testing.T) { + custom := "https://example.com/catalog.yaml" + if got := resolveModelCatalogSource(custom); got != custom { + t.Fatalf("got=%q want=%q", got, custom) + } +} diff --git a/pkg/llm/models/llm_container_driver.go b/pkg/llm/models/llm_container_driver.go index 3d5566a2b6..4cfb136b72 100644 --- a/pkg/llm/models/llm_container_driver.go +++ b/pkg/llm/models/llm_container_driver.go @@ -75,7 +75,7 @@ type ILLMContainerInstantModel interface { PreInstallModel(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, instMdl *SLLMInstantModel) error InstallModel(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, dirs []string, mdlIds []string) error UninstallModel(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, instMdl *SLLMInstantModel) error - DownloadModel(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, tmpDir string, modelName string, modelTag string, progress func(progress float32)) (string, []string, error) + DownloadModel(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, tmpDir string, input llm.InstantModelImportInput, progress func(progress float32)) (string, []string, error) } type ILLMContainerDriver interface { diff --git a/pkg/llm/models/llm_deployment_orphan_backfill.go b/pkg/llm/models/llm_deployment_orphan_backfill.go index 3cde975572..c2fb76882d 100644 --- a/pkg/llm/models/llm_deployment_orphan_backfill.go +++ b/pkg/llm/models/llm_deployment_orphan_backfill.go @@ -6,6 +6,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/tristate" + "yunion.io/x/sqlchemy" "yunion.io/x/onecloud/pkg/apis" computeapi "yunion.io/x/onecloud/pkg/apis/compute" @@ -22,6 +23,21 @@ var orphanLLMBackfillExcludedStatuses = []string{ api.LLM_STATUS_START_DELETE, } +var orphanLLMBackfillSupportedTypes = []string{ + string(api.LLM_CONTAINER_OLLAMA), + string(api.LLM_CONTAINER_VLLM), + string(api.LLM_CONTAINER_SGLANG), +} + +func isOrphanLLMBackfillSupportedType(llmType string) bool { + for _, t := range orphanLLMBackfillSupportedTypes { + if llmType == t { + return true + } + } + return false +} + // BackfillOrphanLLMDeployments creates one SLLMDeployment per orphan SLLM instance // (no llm_deployment_id) and links the instance to it. Idempotent across restarts. func BackfillOrphanLLMDeployments(ctx context.Context) error { @@ -58,6 +74,9 @@ func fetchOrphanLLMsForBackfill() ([]SLLM, error) { q = q.Equals("deleted", false) q = q.Equals("pending_deleted", false) q = q.NotIn("status", orphanLLMBackfillExcludedStatuses) + skuQ := GetLLMSkuManager().Query().SubQuery() + q = q.Join(skuQ, sqlchemy.Equals(q.Field("llm_sku_id"), skuQ.Field("id"))) + q = q.Filter(sqlchemy.In(skuQ.Field("llm_type"), orphanLLMBackfillSupportedTypes)) var llms []SLLM if err := db.FetchModelObjects(GetLLMManager(), q, &llms); err != nil { @@ -155,6 +174,9 @@ func (man *SLLMDeploymentManager) adoptOrphanLLM(ctx context.Context, userCred m return errors.Wrapf(err, "fetch LLMSku %s", llm.LLMSkuId) } sku := skuObj.(*SLLMSku) + if !isOrphanLLMBackfillSupportedType(sku.LLMType) { + return errors.Errorf("llm type %q is not supported for orphan backfill", sku.LLMType) + } nets, err := netsFromOrphanLLM(llm) if err != nil { diff --git a/pkg/llm/models/llm_model_set.go b/pkg/llm/models/llm_model_set.go index 3c30e89911..26fab1e29c 100644 --- a/pkg/llm/models/llm_model_set.go +++ b/pkg/llm/models/llm_model_set.go @@ -24,6 +24,8 @@ const ( // configured. Overridable via options.ModelCatalogURL — values without an // http:// or https:// prefix are treated as local file paths. DefaultModelCatalogURL = "https://www.cloudpods.org/llm-catalog.yaml" + // DefaultModelCatalogModelScopeURL is the ModelScope mirror of the catalog. + DefaultModelCatalogModelScopeURL = "https://www.cloudpods.org/model-catalog-modelscope.yaml" // DefaultLLMCatalogRefreshInterval is how often the LLM service polls the // upstream source in the background. Local file sources are also re-read // on each tick (cheap) so an admin edit is picked up without restart. @@ -80,6 +82,7 @@ func (m *SLLMModelSetManager) Start(ctx context.Context, source string, interval if source == "" { source = DefaultModelCatalogURL } + source = resolveModelCatalogSource(source) m.mu.Lock() m.source = source m.mu.Unlock() @@ -234,8 +237,7 @@ func slugify(s string) string { return out } -// loadSource reads catalog YAML bytes from either an http(s) URL or a local -// file path, picking the loader based on the source prefix. +// loadSource reads catalog YAML bytes from either an http(s) URL or a local file path. func (m *SLLMModelSetManager) loadSource(ctx context.Context, source string) ([]byte, error) { lower := strings.ToLower(source) if strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://") { @@ -490,3 +492,40 @@ func unionOllamaCapabilities(tags []api.SOllamaTag) []string { } return out } + +var defaultHuggingFaceCatalogURLs = map[string]struct{}{ + DefaultModelCatalogURL: {}, + "https://www.cloudpods.org/model-catalog.yaml": {}, +} + +// resolveModelCatalogSource switches to the ModelScope catalog URL when the +// configured source is a built-in HuggingFace URL and HF is unreachable while +// ModelScope is reachable. Mirrors GPUStack's get_builtin_model_catalog_file. +func resolveModelCatalogSource(source string) string { + source = strings.TrimSpace(source) + if _, ok := defaultHuggingFaceCatalogURLs[source]; !ok { + return source + } + hfURL := "https://huggingface.co" + msURL := "https://www.modelscope.cn" + if canAccessCatalogURL(hfURL) || !canAccessCatalogURL(msURL) { + return source + } + log.Infof("Cannot access %s, using ModelScope model catalog at %s", hfURL, DefaultModelCatalogModelScopeURL) + return DefaultModelCatalogModelScopeURL +} + +func canAccessCatalogURL(rawURL string) bool { + ctx, cancel := context.WithTimeout(context.Background(), catalogFetchTimeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return false + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return false + } + defer resp.Body.Close() + return resp.StatusCode >= 200 && resp.StatusCode < 300 +} diff --git a/pkg/llm/models/llm_sku_catalog_import.go b/pkg/llm/models/llm_sku_catalog_import.go index ce9cfddd2e..022bc88140 100644 --- a/pkg/llm/models/llm_sku_catalog_import.go +++ b/pkg/llm/models/llm_sku_catalog_import.go @@ -64,6 +64,27 @@ func buildLLMSkuImportFromModelSpec(input *api.LLMSkuCreateInput) (*api.InstantM importInput.ModelTag = revision input.Source = api.LLM_MODEL_SOURCE_HUGGINGFACE input.HuggingfaceRepoId = repoID + case api.InstantModelSourceModelScope: + if repoID == "" { + repoID = strings.TrimSpace(importInput.ModelName) + } + revision := strings.TrimSpace(importInput.Revision) + if revision == "" { + revision = strings.TrimSpace(importInput.ModelTag) + } + if revision == "" { + revision = defaultModelScopeRevision + } + filePath := strings.TrimSpace(importInput.FilePath) + importInput.Source = api.InstantModelSourceModelScope + importInput.RepoId = repoID + importInput.Revision = revision + importInput.ModelName = repoID + importInput.ModelTag = revision + importInput.FilePath = filePath + input.Source = api.LLM_MODEL_SOURCE_MODEL_SCOPE + input.ModelScopeModelId = repoID + input.ModelScopeFilePath = filePath case "ollama": importInput.Source = source input.Source = source diff --git a/pkg/llm/models/llm_sku_catalog_import_test.go b/pkg/llm/models/llm_sku_catalog_import_test.go new file mode 100644 index 0000000000..08f63d1b27 --- /dev/null +++ b/pkg/llm/models/llm_sku_catalog_import_test.go @@ -0,0 +1,58 @@ +package models + +import ( + "testing" + + api "yunion.io/x/onecloud/pkg/apis/llm" +) + +func TestBuildLLMSkuImportFromModelSpecModelScope(t *testing.T) { + input := &api.LLMSkuCreateInput{ + LLMType: "vllm", + ModelSpec: &api.InstantModelImportInput{ + ModelName: "Qwen/Qwen3-0.6B", + ModelTag: "master", + LlmType: api.LLM_CONTAINER_VLLM, + Source: api.InstantModelSourceModelScope, + RepoId: "Qwen/Qwen3-0.6B", + Revision: "master", + FilePath: "*.safetensors", + }, + } + out, err := buildLLMSkuImportFromModelSpec(input) + if err != nil { + t.Fatalf("buildLLMSkuImportFromModelSpec: %v", err) + } + if out.Source != api.InstantModelSourceModelScope { + t.Fatalf("expected source model_scope, got %q", out.Source) + } + if input.Source != api.LLM_MODEL_SOURCE_MODEL_SCOPE { + t.Fatalf("expected sku source model_scope, got %q", input.Source) + } + if input.ModelScopeModelId != "Qwen/Qwen3-0.6B" { + t.Fatalf("unexpected model_scope_model_id: %q", input.ModelScopeModelId) + } + if input.ModelScopeFilePath != "*.safetensors" { + t.Fatalf("unexpected model_scope_file_path: %q", input.ModelScopeFilePath) + } + if out.FilePath != "*.safetensors" { + t.Fatalf("unexpected file_path: %q", out.FilePath) + } +} + +func TestResolveImportRepoAndRevisionModelScope(t *testing.T) { + source, repo, rev := resolveImportRepoAndRevision(api.InstantModelImportInput{ + Source: api.InstantModelSourceModelScope, + RepoId: "Qwen/Qwen3-8B", + ModelName: "Qwen/Qwen3-8B", + }) + if source != api.InstantModelSourceModelScope { + t.Fatalf("source=%q", source) + } + if repo != "Qwen/Qwen3-8B" { + t.Fatalf("repo=%q", repo) + } + if rev != defaultModelScopeRevision { + t.Fatalf("rev=%q want %q", rev, defaultModelScopeRevision) + } +} diff --git a/pkg/llm/options/option.go b/pkg/llm/options/option.go index d643094e45..b87d9c2560 100644 --- a/pkg/llm/options/option.go +++ b/pkg/llm/options/option.go @@ -47,6 +47,9 @@ type LLMOptions struct { // the dashboard for model browsing). Mirrors GPUStack's /v1/proxy design. HuggingFaceEndpoint string `help:"Replacement endpoint for huggingface.co (e.g., https://hf-mirror.com); empty means no substitution"` HuggingFaceToken string `help:"Optional HuggingFace bearer token; injected as Authorization header on huggingface.co requests"` + + ModelScopeEndpoint string `help:"ModelScope API endpoint (e.g., https://www.modelscope.cn)" default:"https://www.modelscope.cn"` + ModelScopeToken string `help:"Optional ModelScope bearer token; injected as Authorization header on modelscope.cn requests"` } var ( diff --git a/pkg/llm/tasks/llm/llm_instant_model_import_task.go b/pkg/llm/tasks/llm/llm_instant_model_import_task.go index c7af661660..3c16ce5048 100644 --- a/pkg/llm/tasks/llm/llm_instant_model_import_task.go +++ b/pkg/llm/tasks/llm/llm_instant_model_import_task.go @@ -99,9 +99,7 @@ func (task *LLMInstantModelImportTask) OnImportComplete(ctx context.Context, obj task.SetStageComplete(ctx, nil) } -// fetchWeightSizeForImport dispatches by import source. Only HuggingFace is -// supported in this phase; ModelScope / local_path / ollama silently return 0 -// (left as TODO; UI handles the unknown case gracefully). +// fetchWeightSizeForImport dispatches by import source. func fetchWeightSizeForImport(ctx context.Context, input apis.InstantModelImportInput) int64 { if input.Source == apis.InstantModelSourceHuggingFace && input.RepoId != "" { rev := input.Revision @@ -115,6 +113,18 @@ func fetchWeightSizeForImport(ctx context.Context, input apis.InstantModelImport } return w } + if input.Source == apis.InstantModelSourceModelScope && input.RepoId != "" { + rev := input.Revision + if rev == "" { + rev = "master" + } + w, err := models.FetchModelScopeWeightSize(ctx, input.RepoId, rev) + if err != nil { + log.Warningf("LLMInstantModelImportTask: fetch ModelScope weight size for %s@%s: %s", input.RepoId, rev, err) + return 0 + } + return w + } return 0 }