mirror of
https://github.com/yunionio/cloudpods.git
synced 2026-09-21 06:09:39 +08:00
Feature/hf-search (#24717)
* llm: add instant-model huggingface query commands * fix(vllm): use ping instead of health
This commit is contained in:
cwz_eikoh
@@ -21,5 +21,7 @@ func init() {
|
||||
cmd.Perform("public", new(commonoptions.BasePublicOptions))
|
||||
cmd.Perform("private", new(commonoptions.BaseIdOptions))
|
||||
cmd.PerformClass("import", new(options.LLMInstantModelImportOptions))
|
||||
cmd.GetProperty(new(options.LLMInstantModelHuggingFaceSearchOptions))
|
||||
cmd.GetProperty(new(options.LLMInstantModelHuggingFaceRepoInfoOptions))
|
||||
cmd.GetProperty(new(options.LLMInstantModelCommunityRegistryOptions))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package llm
|
||||
|
||||
type InstantModelHuggingFaceSearchInput struct {
|
||||
Q string `json:"q"`
|
||||
Author string `json:"author,omitempty"`
|
||||
Filter []string `json:"filter,omitempty"`
|
||||
Direction int `json:"direction,omitempty"`
|
||||
Limit int `json:"limit,omitempty"`
|
||||
Sort string `json:"sort,omitempty"`
|
||||
}
|
||||
|
||||
type InstantModelHuggingFaceRepoInfoInput struct {
|
||||
RepoId string `json:"repo_id"`
|
||||
Revision string `json:"revision,omitempty"`
|
||||
}
|
||||
|
||||
type InstantModelHuggingFaceSearchResult struct {
|
||||
RepoId string `json:"repo_id"`
|
||||
Author string `json:"author,omitempty"`
|
||||
Sha string `json:"sha,omitempty"`
|
||||
LastModified string `json:"last_modified,omitempty"`
|
||||
Downloads int64 `json:"downloads,omitempty"`
|
||||
Likes int64 `json:"likes,omitempty"`
|
||||
PipelineTag string `json:"pipeline_tag,omitempty"`
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
Private bool `json:"private,omitempty"`
|
||||
Gated bool `json:"gated,omitempty"`
|
||||
Disabled bool `json:"disabled,omitempty"`
|
||||
Supported bool `json:"supported"`
|
||||
UnsupportedReason string `json:"unsupported_reason,omitempty"`
|
||||
}
|
||||
|
||||
type InstantModelHuggingFaceRepoInfo struct {
|
||||
RepoId string `json:"repo_id"`
|
||||
RequestedRevision string `json:"requested_revision,omitempty"`
|
||||
ResolvedRevision string `json:"resolved_revision,omitempty"`
|
||||
Siblings []string `json:"siblings,omitempty"`
|
||||
ConfigPresent bool `json:"config_present"`
|
||||
SafetensorsPresent bool `json:"safetensors_present"`
|
||||
GgufPresent bool `json:"gguf_present"`
|
||||
ReadmePresent bool `json:"readme_present"`
|
||||
SizeBytes int64 `json:"size_bytes,omitempty"`
|
||||
Supported bool `json:"supported"`
|
||||
UnsupportedReason string `json:"unsupported_reason,omitempty"`
|
||||
ImportMode string `json:"import_mode,omitempty"`
|
||||
}
|
||||
@@ -5,13 +5,13 @@ import "time"
|
||||
const (
|
||||
LLM_VLLM = "vllm"
|
||||
LLM_VLLM_DEFAULT_PORT = 8000
|
||||
LLM_VLLM_EXEC_PATH = "python3 -m vllm.entrypoints.openai.api_server"
|
||||
LLM_VLLM_EXEC_PATH = "vllm serve"
|
||||
|
||||
LLM_VLLM_HF_ENDPOINT = "https://hf-mirror.com"
|
||||
LLM_VLLM_CACHE_DIR = "/root/.cache/huggingface"
|
||||
LLM_VLLM_BASE_PATH = "/data/models"
|
||||
LLM_VLLM_MODELS_PATH = "/data/models/huggingface"
|
||||
|
||||
LLM_VLLM_HEALTH_CHECK_TIMEOUT = 120 * time.Second
|
||||
LLM_VLLM_HEALTH_CHECK_TIMEOUT = 180 * time.Second
|
||||
LLM_VLLM_HEALTH_CHECK_INTERVAL = 10 * time.Second
|
||||
)
|
||||
|
||||
@@ -422,19 +422,21 @@ func buildVLLMHealthCheckURL(networkType, llmIP, hostAccessIP string, accessInfo
|
||||
if len(llmIP) == 0 {
|
||||
return "", errors.Error("LLM IP is empty for guest network")
|
||||
}
|
||||
return fmt.Sprintf("http://%s:%d/health", llmIP, api.LLM_VLLM_DEFAULT_PORT), nil
|
||||
return fmt.Sprintf("http://%s:%d/ping", llmIP, api.LLM_VLLM_DEFAULT_PORT), nil
|
||||
}
|
||||
if accessInfo != nil && accessInfo.AccessPort > 0 {
|
||||
if len(hostAccessIP) == 0 {
|
||||
return "", errors.Error("host access IP is empty")
|
||||
}
|
||||
return fmt.Sprintf("http://%s:%d/ping", hostAccessIP, accessInfo.AccessPort), nil
|
||||
}
|
||||
if len(llmIP) > 0 {
|
||||
return fmt.Sprintf("http://%s:%d/health", llmIP, api.LLM_VLLM_DEFAULT_PORT), nil
|
||||
return fmt.Sprintf("http://%s:%d/ping", llmIP, api.LLM_VLLM_DEFAULT_PORT), nil
|
||||
}
|
||||
if len(hostAccessIP) == 0 {
|
||||
return "", errors.Error("host access IP is empty")
|
||||
}
|
||||
port := api.LLM_VLLM_DEFAULT_PORT
|
||||
if accessInfo != nil && accessInfo.AccessPort > 0 {
|
||||
port = accessInfo.AccessPort
|
||||
}
|
||||
return fmt.Sprintf("http://%s:%d/health", hostAccessIP, port), nil
|
||||
return fmt.Sprintf("http://%s:%d/ping", hostAccessIP, api.LLM_VLLM_DEFAULT_PORT), nil
|
||||
}
|
||||
|
||||
// resolveModelPath resolves the model directory inside the container.
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
"yunion.io/x/pkg/errors"
|
||||
|
||||
apis "yunion.io/x/onecloud/pkg/apis/llm"
|
||||
"yunion.io/x/onecloud/pkg/httperrors"
|
||||
"yunion.io/x/onecloud/pkg/mcclient"
|
||||
)
|
||||
|
||||
const (
|
||||
huggingFaceMirrorEndpoint = "https://hf-mirror.com"
|
||||
huggingFaceImportMode = "snapshot"
|
||||
)
|
||||
|
||||
type huggingFaceSearchItem struct {
|
||||
ID string `json:"id"`
|
||||
Author string `json:"author"`
|
||||
Sha string `json:"sha"`
|
||||
LastModified string `json:"lastModified"`
|
||||
Downloads int64 `json:"downloads"`
|
||||
Likes int64 `json:"likes"`
|
||||
PipelineTag string `json:"pipeline_tag"`
|
||||
Tags []string `json:"tags"`
|
||||
Private bool `json:"private"`
|
||||
Gated interface{} `json:"gated"`
|
||||
Disabled bool `json:"disabled"`
|
||||
}
|
||||
|
||||
type huggingFaceRepoSibling struct {
|
||||
RFilename string `json:"rfilename"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
type huggingFaceRepoInfoResponse struct {
|
||||
ID string `json:"id"`
|
||||
Sha string `json:"sha"`
|
||||
Siblings []huggingFaceRepoSibling `json:"siblings"`
|
||||
}
|
||||
|
||||
func (man *SInstantModelManager) GetPropertyHuggingfaceSearch(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
|
||||
return man.getPropertyHuggingFaceSearch(ctx, userCred, query)
|
||||
}
|
||||
|
||||
func (man *SInstantModelManager) GetPropertyHuggingFaceSearch(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
|
||||
return man.getPropertyHuggingFaceSearch(ctx, userCred, query)
|
||||
}
|
||||
|
||||
func (man *SInstantModelManager) getPropertyHuggingFaceSearch(ctx context.Context, _ mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
|
||||
input := apis.InstantModelHuggingFaceSearchInput{}
|
||||
if query != nil {
|
||||
if err := query.Unmarshal(&input); err != nil {
|
||||
return nil, errors.Wrap(err, "query.Unmarshal")
|
||||
}
|
||||
}
|
||||
input.Q = strings.TrimSpace(input.Q)
|
||||
if input.Q == "" {
|
||||
return nil, httperrors.NewMissingParameterError("q")
|
||||
}
|
||||
if input.Limit <= 0 {
|
||||
input.Limit = 20
|
||||
}
|
||||
if input.Limit > 100 {
|
||||
input.Limit = 100
|
||||
}
|
||||
input.Author = strings.TrimSpace(input.Author)
|
||||
input.Sort = strings.TrimSpace(input.Sort)
|
||||
for i := range input.Filter {
|
||||
input.Filter[i] = strings.TrimSpace(input.Filter[i])
|
||||
}
|
||||
|
||||
searchURL := buildHuggingFaceSearchURL(input)
|
||||
|
||||
body, err := huggingFaceHTTPGet(ctx, searchURL)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "huggingFaceHTTPGet")
|
||||
}
|
||||
|
||||
items := make([]huggingFaceSearchItem, 0)
|
||||
if err := json.Unmarshal(body, &items); err != nil {
|
||||
return nil, errors.Wrap(err, "json.Unmarshal")
|
||||
}
|
||||
return jsonutils.Marshal(normalizeHuggingFaceSearchResults(items)), nil
|
||||
}
|
||||
|
||||
func (man *SInstantModelManager) GetPropertyHuggingfaceRepoInfo(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
|
||||
return man.getPropertyHuggingFaceRepoInfo(ctx, userCred, query)
|
||||
}
|
||||
|
||||
func (man *SInstantModelManager) GetPropertyHuggingFaceRepoInfo(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
|
||||
return man.getPropertyHuggingFaceRepoInfo(ctx, userCred, query)
|
||||
}
|
||||
|
||||
func (man *SInstantModelManager) getPropertyHuggingFaceRepoInfo(ctx context.Context, _ mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
|
||||
input := apis.InstantModelHuggingFaceRepoInfoInput{}
|
||||
if query != nil {
|
||||
if err := query.Unmarshal(&input); err != nil {
|
||||
return nil, errors.Wrap(err, "query.Unmarshal")
|
||||
}
|
||||
}
|
||||
input.RepoId = strings.TrimSpace(input.RepoId)
|
||||
input.Revision = strings.TrimSpace(input.Revision)
|
||||
if input.RepoId == "" {
|
||||
return nil, httperrors.NewMissingParameterError("repo_id")
|
||||
}
|
||||
|
||||
resp, err := getHuggingFaceRepoInfo(ctx, input.RepoId, input.Revision)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "getHuggingFaceRepoInfo")
|
||||
}
|
||||
return jsonutils.Marshal(buildHuggingFaceRepoInfo(resp, input.Revision)), nil
|
||||
}
|
||||
|
||||
func normalizeHuggingFaceSearchResults(items []huggingFaceSearchItem) []apis.InstantModelHuggingFaceSearchResult {
|
||||
results := make([]apis.InstantModelHuggingFaceSearchResult, 0, len(items))
|
||||
for _, item := range items {
|
||||
if item.Private {
|
||||
continue
|
||||
}
|
||||
result := apis.InstantModelHuggingFaceSearchResult{
|
||||
RepoId: item.ID,
|
||||
Author: item.Author,
|
||||
Sha: item.Sha,
|
||||
LastModified: item.LastModified,
|
||||
Downloads: item.Downloads,
|
||||
Likes: item.Likes,
|
||||
PipelineTag: item.PipelineTag,
|
||||
Tags: item.Tags,
|
||||
Private: item.Private,
|
||||
Gated: isHuggingFaceGated(item.Gated),
|
||||
Disabled: item.Disabled,
|
||||
Supported: true,
|
||||
}
|
||||
switch {
|
||||
case result.Gated:
|
||||
result.Supported = false
|
||||
result.UnsupportedReason = "gated repositories are not supported in this phase"
|
||||
case result.Disabled:
|
||||
result.Supported = false
|
||||
result.UnsupportedReason = "disabled repositories are not supported"
|
||||
case hasTag(item.Tags, "gguf"):
|
||||
result.Supported = false
|
||||
result.UnsupportedReason = "gguf repositories are not supported for vllm import"
|
||||
}
|
||||
results = append(results, result)
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
func buildHuggingFaceRepoInfo(resp huggingFaceRepoInfoResponse, requestedRevision string) *apis.InstantModelHuggingFaceRepoInfo {
|
||||
info := &apis.InstantModelHuggingFaceRepoInfo{
|
||||
RepoId: resp.ID,
|
||||
RequestedRevision: requestedRevision,
|
||||
ResolvedRevision: resp.Sha,
|
||||
Supported: true,
|
||||
ImportMode: huggingFaceImportMode,
|
||||
}
|
||||
for _, sibling := range resp.Siblings {
|
||||
name := sibling.RFilename
|
||||
info.Siblings = append(info.Siblings, name)
|
||||
info.SizeBytes += sibling.Size
|
||||
base := strings.ToLower(filepath.Base(name))
|
||||
switch {
|
||||
case base == "config.json":
|
||||
info.ConfigPresent = true
|
||||
case base == "readme.md":
|
||||
info.ReadmePresent = true
|
||||
case strings.HasSuffix(strings.ToLower(name), ".safetensors"):
|
||||
info.SafetensorsPresent = true
|
||||
case strings.HasSuffix(strings.ToLower(name), ".gguf"):
|
||||
info.GgufPresent = true
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case info.GgufPresent:
|
||||
info.Supported = false
|
||||
info.UnsupportedReason = "gguf repositories are not supported for vllm import"
|
||||
case !info.ConfigPresent:
|
||||
info.Supported = false
|
||||
info.UnsupportedReason = "config.json is required for vllm import"
|
||||
case !info.SafetensorsPresent:
|
||||
info.Supported = false
|
||||
info.UnsupportedReason = "no safetensors weights detected for vllm import"
|
||||
}
|
||||
if !info.Supported {
|
||||
info.ImportMode = ""
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
func buildHuggingFaceSearchURL(input apis.InstantModelHuggingFaceSearchInput) string {
|
||||
queryParts := []string{fmt.Sprintf("search=%s", url.QueryEscape(input.Q))}
|
||||
if input.Author != "" {
|
||||
queryParts = append(queryParts, fmt.Sprintf("author=%s", url.QueryEscape(input.Author)))
|
||||
}
|
||||
for _, filter := range input.Filter {
|
||||
if filter == "" {
|
||||
continue
|
||||
}
|
||||
queryParts = append(queryParts, fmt.Sprintf("filter=%s", url.QueryEscape(filter)))
|
||||
}
|
||||
if input.Sort != "" {
|
||||
queryParts = append(queryParts, fmt.Sprintf("sort=%s", url.QueryEscape(input.Sort)))
|
||||
}
|
||||
if input.Direction != 0 {
|
||||
queryParts = append(queryParts, fmt.Sprintf("direction=%d", input.Direction))
|
||||
}
|
||||
queryParts = append(queryParts, fmt.Sprintf("limit=%d", input.Limit))
|
||||
return fmt.Sprintf("%s/api/models?%s", huggingFaceMirrorEndpoint, strings.Join(queryParts, "&"))
|
||||
}
|
||||
|
||||
func isHuggingFaceGated(v interface{}) bool {
|
||||
switch value := v.(type) {
|
||||
case bool:
|
||||
return value
|
||||
case string:
|
||||
return strings.TrimSpace(value) != ""
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func hasTag(tags []string, target string) bool {
|
||||
for _, tag := range tags {
|
||||
if strings.EqualFold(strings.TrimSpace(tag), target) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func huggingFaceHTTPGet(ctx context.Context, reqURL string) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "http.NewRequestWithContext")
|
||||
}
|
||||
client := &http.Client{Timeout: 60 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "client.Do")
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, errors.Errorf("unexpected status code: %d", resp.StatusCode)
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "io.ReadAll")
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func getHuggingFaceRepoInfo(ctx context.Context, repoID string, revision string) (huggingFaceRepoInfoResponse, error) {
|
||||
repoURL := fmt.Sprintf("%s/api/models/%s", huggingFaceMirrorEndpoint, escapeURLPathPreserveSlash(repoID))
|
||||
if revision != "" {
|
||||
repoURL = fmt.Sprintf("%s?revision=%s", repoURL, url.QueryEscape(revision))
|
||||
}
|
||||
body, err := huggingFaceHTTPGet(ctx, repoURL)
|
||||
if err != nil {
|
||||
return huggingFaceRepoInfoResponse{}, errors.Wrap(err, "huggingFaceHTTPGet")
|
||||
}
|
||||
resp := huggingFaceRepoInfoResponse{}
|
||||
if err := json.Unmarshal(body, &resp); err != nil {
|
||||
return huggingFaceRepoInfoResponse{}, errors.Wrap(err, "json.Unmarshal")
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func escapeURLPathPreserveSlash(p string) string {
|
||||
if p == "" {
|
||||
return ""
|
||||
}
|
||||
parts := strings.Split(p, "/")
|
||||
for i := range parts {
|
||||
parts[i] = url.PathEscape(parts[i])
|
||||
}
|
||||
return strings.Join(parts, "/")
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package llm
|
||||
|
||||
import "yunion.io/x/jsonutils"
|
||||
|
||||
type LLMInstantModelHuggingFaceSearchOptions struct {
|
||||
Q string `help:"huggingface query string" json:"q"`
|
||||
Author string `help:"filter by author or organization" json:"author"`
|
||||
Filter []string `help:"filter by tags, e.g. text-generation or pytorch" json:"filter"`
|
||||
Direction int `help:"sort direction, e.g. -1 for descending or 1 for ascending" json:"direction"`
|
||||
Limit int `help:"max number of search results" json:"limit"`
|
||||
Sort string `help:"sort order, e.g. downloads|likes|updated" json:"sort"`
|
||||
}
|
||||
|
||||
func (o *LLMInstantModelHuggingFaceSearchOptions) Params() (jsonutils.JSONObject, error) {
|
||||
return jsonutils.Marshal(o), nil
|
||||
}
|
||||
|
||||
func (o *LLMInstantModelHuggingFaceSearchOptions) Property() string {
|
||||
return "huggingface-search"
|
||||
}
|
||||
|
||||
type LLMInstantModelHuggingFaceRepoInfoOptions struct {
|
||||
REPO_ID string `help:"huggingface repo id, e.g. Qwen/Qwen3-8B" json:"repo_id"`
|
||||
REVISION string `help:"huggingface revision, e.g. main" json:"revision"`
|
||||
}
|
||||
|
||||
func (o *LLMInstantModelHuggingFaceRepoInfoOptions) Params() (jsonutils.JSONObject, error) {
|
||||
return jsonutils.Marshal(o), nil
|
||||
}
|
||||
|
||||
func (o *LLMInstantModelHuggingFaceRepoInfoOptions) Property() string {
|
||||
return "huggingface-repo-info"
|
||||
}
|
||||
Reference in New Issue
Block a user