diff --git a/cmd/climc/shell/llm/instantmodel.go b/cmd/climc/shell/llm/instantmodel.go index 3fd1a2dc82..698f7ddbde 100644 --- a/cmd/climc/shell/llm/instantmodel.go +++ b/cmd/climc/shell/llm/instantmodel.go @@ -21,5 +21,7 @@ func init() { cmd.Perform("public", new(commonoptions.BasePublicOptions)) cmd.Perform("private", new(commonoptions.BaseIdOptions)) cmd.PerformClass("import", new(options.LLMInstantModelImportOptions)) + cmd.GetProperty(new(options.LLMInstantModelHuggingFaceSearchOptions)) + cmd.GetProperty(new(options.LLMInstantModelHuggingFaceRepoInfoOptions)) cmd.GetProperty(new(options.LLMInstantModelCommunityRegistryOptions)) } diff --git a/pkg/apis/llm/instantmodel_huggingface.go b/pkg/apis/llm/instantmodel_huggingface.go new file mode 100644 index 0000000000..75d615adf6 --- /dev/null +++ b/pkg/apis/llm/instantmodel_huggingface.go @@ -0,0 +1,46 @@ +package llm + +type InstantModelHuggingFaceSearchInput struct { + Q string `json:"q"` + Author string `json:"author,omitempty"` + Filter []string `json:"filter,omitempty"` + Direction int `json:"direction,omitempty"` + Limit int `json:"limit,omitempty"` + Sort string `json:"sort,omitempty"` +} + +type InstantModelHuggingFaceRepoInfoInput struct { + RepoId string `json:"repo_id"` + Revision string `json:"revision,omitempty"` +} + +type InstantModelHuggingFaceSearchResult struct { + RepoId string `json:"repo_id"` + Author string `json:"author,omitempty"` + Sha string `json:"sha,omitempty"` + LastModified string `json:"last_modified,omitempty"` + Downloads int64 `json:"downloads,omitempty"` + Likes int64 `json:"likes,omitempty"` + PipelineTag string `json:"pipeline_tag,omitempty"` + Tags []string `json:"tags,omitempty"` + Private bool `json:"private,omitempty"` + Gated bool `json:"gated,omitempty"` + Disabled bool `json:"disabled,omitempty"` + Supported bool `json:"supported"` + UnsupportedReason string `json:"unsupported_reason,omitempty"` +} + +type InstantModelHuggingFaceRepoInfo struct { + RepoId string `json:"repo_id"` + RequestedRevision string `json:"requested_revision,omitempty"` + ResolvedRevision string `json:"resolved_revision,omitempty"` + Siblings []string `json:"siblings,omitempty"` + ConfigPresent bool `json:"config_present"` + SafetensorsPresent bool `json:"safetensors_present"` + GgufPresent bool `json:"gguf_present"` + ReadmePresent bool `json:"readme_present"` + SizeBytes int64 `json:"size_bytes,omitempty"` + Supported bool `json:"supported"` + UnsupportedReason string `json:"unsupported_reason,omitempty"` + ImportMode string `json:"import_mode,omitempty"` +} diff --git a/pkg/apis/llm/vllm_const.go b/pkg/apis/llm/vllm_const.go index 82454d86ac..25033d3a34 100644 --- a/pkg/apis/llm/vllm_const.go +++ b/pkg/apis/llm/vllm_const.go @@ -5,13 +5,13 @@ import "time" const ( LLM_VLLM = "vllm" LLM_VLLM_DEFAULT_PORT = 8000 - LLM_VLLM_EXEC_PATH = "python3 -m vllm.entrypoints.openai.api_server" + LLM_VLLM_EXEC_PATH = "vllm serve" LLM_VLLM_HF_ENDPOINT = "https://hf-mirror.com" LLM_VLLM_CACHE_DIR = "/root/.cache/huggingface" LLM_VLLM_BASE_PATH = "/data/models" LLM_VLLM_MODELS_PATH = "/data/models/huggingface" - LLM_VLLM_HEALTH_CHECK_TIMEOUT = 120 * time.Second + LLM_VLLM_HEALTH_CHECK_TIMEOUT = 180 * time.Second LLM_VLLM_HEALTH_CHECK_INTERVAL = 10 * time.Second ) diff --git a/pkg/llm/drivers/llm_container/vllm.go b/pkg/llm/drivers/llm_container/vllm.go index 858c7f97d2..72ca1aaf8a 100644 --- a/pkg/llm/drivers/llm_container/vllm.go +++ b/pkg/llm/drivers/llm_container/vllm.go @@ -422,19 +422,21 @@ func buildVLLMHealthCheckURL(networkType, llmIP, hostAccessIP string, accessInfo if len(llmIP) == 0 { return "", errors.Error("LLM IP is empty for guest network") } - return fmt.Sprintf("http://%s:%d/health", llmIP, api.LLM_VLLM_DEFAULT_PORT), nil + return fmt.Sprintf("http://%s:%d/ping", llmIP, api.LLM_VLLM_DEFAULT_PORT), nil + } + if accessInfo != nil && accessInfo.AccessPort > 0 { + if len(hostAccessIP) == 0 { + return "", errors.Error("host access IP is empty") + } + return fmt.Sprintf("http://%s:%d/ping", hostAccessIP, accessInfo.AccessPort), nil } if len(llmIP) > 0 { - return fmt.Sprintf("http://%s:%d/health", llmIP, api.LLM_VLLM_DEFAULT_PORT), nil + return fmt.Sprintf("http://%s:%d/ping", llmIP, api.LLM_VLLM_DEFAULT_PORT), nil } if len(hostAccessIP) == 0 { return "", errors.Error("host access IP is empty") } - port := api.LLM_VLLM_DEFAULT_PORT - if accessInfo != nil && accessInfo.AccessPort > 0 { - port = accessInfo.AccessPort - } - return fmt.Sprintf("http://%s:%d/health", hostAccessIP, port), nil + return fmt.Sprintf("http://%s:%d/ping", hostAccessIP, api.LLM_VLLM_DEFAULT_PORT), nil } // resolveModelPath resolves the model directory inside the container. diff --git a/pkg/llm/models/instantmodel_huggingface.go b/pkg/llm/models/instantmodel_huggingface.go new file mode 100644 index 0000000000..f02e2f3222 --- /dev/null +++ b/pkg/llm/models/instantmodel_huggingface.go @@ -0,0 +1,289 @@ +package models + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "path/filepath" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + apis "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" +) + +const ( + huggingFaceMirrorEndpoint = "https://hf-mirror.com" + huggingFaceImportMode = "snapshot" +) + +type huggingFaceSearchItem struct { + ID string `json:"id"` + Author string `json:"author"` + Sha string `json:"sha"` + LastModified string `json:"lastModified"` + Downloads int64 `json:"downloads"` + Likes int64 `json:"likes"` + PipelineTag string `json:"pipeline_tag"` + Tags []string `json:"tags"` + Private bool `json:"private"` + Gated interface{} `json:"gated"` + Disabled bool `json:"disabled"` +} + +type huggingFaceRepoSibling struct { + RFilename string `json:"rfilename"` + Size int64 `json:"size"` +} + +type huggingFaceRepoInfoResponse struct { + ID string `json:"id"` + Sha string `json:"sha"` + Siblings []huggingFaceRepoSibling `json:"siblings"` +} + +func (man *SInstantModelManager) GetPropertyHuggingfaceSearch(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + return man.getPropertyHuggingFaceSearch(ctx, userCred, query) +} + +func (man *SInstantModelManager) GetPropertyHuggingFaceSearch(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + return man.getPropertyHuggingFaceSearch(ctx, userCred, query) +} + +func (man *SInstantModelManager) getPropertyHuggingFaceSearch(ctx context.Context, _ mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + input := apis.InstantModelHuggingFaceSearchInput{} + if query != nil { + if err := query.Unmarshal(&input); err != nil { + return nil, errors.Wrap(err, "query.Unmarshal") + } + } + input.Q = strings.TrimSpace(input.Q) + if input.Q == "" { + return nil, httperrors.NewMissingParameterError("q") + } + if input.Limit <= 0 { + input.Limit = 20 + } + if input.Limit > 100 { + input.Limit = 100 + } + input.Author = strings.TrimSpace(input.Author) + input.Sort = strings.TrimSpace(input.Sort) + for i := range input.Filter { + input.Filter[i] = strings.TrimSpace(input.Filter[i]) + } + + searchURL := buildHuggingFaceSearchURL(input) + + body, err := huggingFaceHTTPGet(ctx, searchURL) + if err != nil { + return nil, errors.Wrap(err, "huggingFaceHTTPGet") + } + + items := make([]huggingFaceSearchItem, 0) + if err := json.Unmarshal(body, &items); err != nil { + return nil, errors.Wrap(err, "json.Unmarshal") + } + return jsonutils.Marshal(normalizeHuggingFaceSearchResults(items)), nil +} + +func (man *SInstantModelManager) GetPropertyHuggingfaceRepoInfo(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + return man.getPropertyHuggingFaceRepoInfo(ctx, userCred, query) +} + +func (man *SInstantModelManager) GetPropertyHuggingFaceRepoInfo(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + return man.getPropertyHuggingFaceRepoInfo(ctx, userCred, query) +} + +func (man *SInstantModelManager) getPropertyHuggingFaceRepoInfo(ctx context.Context, _ mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + input := apis.InstantModelHuggingFaceRepoInfoInput{} + if query != nil { + if err := query.Unmarshal(&input); err != nil { + return nil, errors.Wrap(err, "query.Unmarshal") + } + } + input.RepoId = strings.TrimSpace(input.RepoId) + input.Revision = strings.TrimSpace(input.Revision) + if input.RepoId == "" { + return nil, httperrors.NewMissingParameterError("repo_id") + } + + resp, err := getHuggingFaceRepoInfo(ctx, input.RepoId, input.Revision) + if err != nil { + return nil, errors.Wrap(err, "getHuggingFaceRepoInfo") + } + return jsonutils.Marshal(buildHuggingFaceRepoInfo(resp, input.Revision)), nil +} + +func normalizeHuggingFaceSearchResults(items []huggingFaceSearchItem) []apis.InstantModelHuggingFaceSearchResult { + results := make([]apis.InstantModelHuggingFaceSearchResult, 0, len(items)) + for _, item := range items { + if item.Private { + continue + } + result := apis.InstantModelHuggingFaceSearchResult{ + RepoId: item.ID, + Author: item.Author, + Sha: item.Sha, + LastModified: item.LastModified, + Downloads: item.Downloads, + Likes: item.Likes, + PipelineTag: item.PipelineTag, + Tags: item.Tags, + Private: item.Private, + Gated: isHuggingFaceGated(item.Gated), + Disabled: item.Disabled, + Supported: true, + } + switch { + case result.Gated: + result.Supported = false + result.UnsupportedReason = "gated repositories are not supported in this phase" + case result.Disabled: + result.Supported = false + result.UnsupportedReason = "disabled repositories are not supported" + case hasTag(item.Tags, "gguf"): + result.Supported = false + result.UnsupportedReason = "gguf repositories are not supported for vllm import" + } + results = append(results, result) + } + return results +} + +func buildHuggingFaceRepoInfo(resp huggingFaceRepoInfoResponse, requestedRevision string) *apis.InstantModelHuggingFaceRepoInfo { + info := &apis.InstantModelHuggingFaceRepoInfo{ + RepoId: resp.ID, + RequestedRevision: requestedRevision, + ResolvedRevision: resp.Sha, + Supported: true, + ImportMode: huggingFaceImportMode, + } + for _, sibling := range resp.Siblings { + name := sibling.RFilename + info.Siblings = append(info.Siblings, name) + info.SizeBytes += sibling.Size + base := strings.ToLower(filepath.Base(name)) + switch { + case base == "config.json": + info.ConfigPresent = true + case base == "readme.md": + info.ReadmePresent = true + case strings.HasSuffix(strings.ToLower(name), ".safetensors"): + info.SafetensorsPresent = true + case strings.HasSuffix(strings.ToLower(name), ".gguf"): + info.GgufPresent = true + } + } + switch { + case info.GgufPresent: + info.Supported = false + info.UnsupportedReason = "gguf repositories are not supported for vllm import" + case !info.ConfigPresent: + info.Supported = false + info.UnsupportedReason = "config.json is required for vllm import" + case !info.SafetensorsPresent: + info.Supported = false + info.UnsupportedReason = "no safetensors weights detected for vllm import" + } + if !info.Supported { + info.ImportMode = "" + } + return info +} + +func buildHuggingFaceSearchURL(input apis.InstantModelHuggingFaceSearchInput) string { + queryParts := []string{fmt.Sprintf("search=%s", url.QueryEscape(input.Q))} + if input.Author != "" { + queryParts = append(queryParts, fmt.Sprintf("author=%s", url.QueryEscape(input.Author))) + } + for _, filter := range input.Filter { + if filter == "" { + continue + } + queryParts = append(queryParts, fmt.Sprintf("filter=%s", url.QueryEscape(filter))) + } + if input.Sort != "" { + queryParts = append(queryParts, fmt.Sprintf("sort=%s", url.QueryEscape(input.Sort))) + } + if input.Direction != 0 { + queryParts = append(queryParts, fmt.Sprintf("direction=%d", input.Direction)) + } + queryParts = append(queryParts, fmt.Sprintf("limit=%d", input.Limit)) + return fmt.Sprintf("%s/api/models?%s", huggingFaceMirrorEndpoint, strings.Join(queryParts, "&")) +} + +func isHuggingFaceGated(v interface{}) bool { + switch value := v.(type) { + case bool: + return value + case string: + return strings.TrimSpace(value) != "" + default: + return false + } +} + +func hasTag(tags []string, target string) bool { + for _, tag := range tags { + if strings.EqualFold(strings.TrimSpace(tag), target) { + return true + } + } + return false +} + +func huggingFaceHTTPGet(ctx context.Context, reqURL string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil) + if err != nil { + return nil, errors.Wrap(err, "http.NewRequestWithContext") + } + client := &http.Client{Timeout: 60 * time.Second} + resp, err := client.Do(req) + if err != nil { + return nil, errors.Wrap(err, "client.Do") + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, errors.Errorf("unexpected status code: %d", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, "io.ReadAll") + } + return body, nil +} + +func getHuggingFaceRepoInfo(ctx context.Context, repoID string, revision string) (huggingFaceRepoInfoResponse, error) { + repoURL := fmt.Sprintf("%s/api/models/%s", huggingFaceMirrorEndpoint, escapeURLPathPreserveSlash(repoID)) + if revision != "" { + repoURL = fmt.Sprintf("%s?revision=%s", repoURL, url.QueryEscape(revision)) + } + body, err := huggingFaceHTTPGet(ctx, repoURL) + if err != nil { + return huggingFaceRepoInfoResponse{}, errors.Wrap(err, "huggingFaceHTTPGet") + } + resp := huggingFaceRepoInfoResponse{} + if err := json.Unmarshal(body, &resp); err != nil { + return huggingFaceRepoInfoResponse{}, errors.Wrap(err, "json.Unmarshal") + } + return resp, nil +} + +func escapeURLPathPreserveSlash(p string) string { + if p == "" { + return "" + } + parts := strings.Split(p, "/") + for i := range parts { + parts[i] = url.PathEscape(parts[i]) + } + return strings.Join(parts, "/") +} diff --git a/pkg/mcclient/options/llm/instantmodel_huggingface.go b/pkg/mcclient/options/llm/instantmodel_huggingface.go new file mode 100644 index 0000000000..68c14d3b7c --- /dev/null +++ b/pkg/mcclient/options/llm/instantmodel_huggingface.go @@ -0,0 +1,33 @@ +package llm + +import "yunion.io/x/jsonutils" + +type LLMInstantModelHuggingFaceSearchOptions struct { + Q string `help:"huggingface query string" json:"q"` + Author string `help:"filter by author or organization" json:"author"` + Filter []string `help:"filter by tags, e.g. text-generation or pytorch" json:"filter"` + Direction int `help:"sort direction, e.g. -1 for descending or 1 for ascending" json:"direction"` + Limit int `help:"max number of search results" json:"limit"` + Sort string `help:"sort order, e.g. downloads|likes|updated" json:"sort"` +} + +func (o *LLMInstantModelHuggingFaceSearchOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(o), nil +} + +func (o *LLMInstantModelHuggingFaceSearchOptions) Property() string { + return "huggingface-search" +} + +type LLMInstantModelHuggingFaceRepoInfoOptions struct { + REPO_ID string `help:"huggingface repo id, e.g. Qwen/Qwen3-8B" json:"repo_id"` + REVISION string `help:"huggingface revision, e.g. main" json:"revision"` +} + +func (o *LLMInstantModelHuggingFaceRepoInfoOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(o), nil +} + +func (o *LLMInstantModelHuggingFaceRepoInfoOptions) Property() string { + return "huggingface-repo-info" +}