diff --git a/pkg/apis/llm/image.go b/pkg/apis/llm/image.go index ae7a84c240..ce8bdef0b6 100644 --- a/pkg/apis/llm/image.go +++ b/pkg/apis/llm/image.go @@ -11,6 +11,7 @@ type LLMImageType string const ( LLM_IMAGE_TYPE_OLLAMA LLMImageType = "ollama" LLM_IMAGE_TYPE_VLLM LLMImageType = "vllm" + LLM_IMAGE_TYPE_SGLANG LLMImageType = "sglang" LLM_IMAGE_TYPE_DIFY LLMImageType = "dify" LLM_IMAGE_TYPE_COMFYUI LLMImageType = "comfyui" LLM_IMAGE_TYPE_OPENCLAW LLMImageType = "openclaw" @@ -21,6 +22,7 @@ var ( LLM_IMAGE_TYPES = sets.NewString( string(LLM_IMAGE_TYPE_OLLAMA), string(LLM_IMAGE_TYPE_VLLM), + string(LLM_IMAGE_TYPE_SGLANG), string(LLM_IMAGE_TYPE_DIFY), string(LLM_IMAGE_TYPE_COMFYUI), string(LLM_IMAGE_TYPE_OPENCLAW), diff --git a/pkg/apis/llm/instantmodel.go b/pkg/apis/llm/instantmodel.go index 8bd64e356a..4a0978d31d 100644 --- a/pkg/apis/llm/instantmodel.go +++ b/pkg/apis/llm/instantmodel.go @@ -15,6 +15,7 @@ type InstantModelListInput struct { ModelName string `json:"model_name"` ModelTag string `json:"model_tag"` ModelId string `json:"model_id"` + LlmType string `json:"llm_type"` Image string `json:"image"` Mounts string `json:"mounts"` diff --git a/pkg/apis/llm/llm_container.go b/pkg/apis/llm/llm_container.go index a70e4724ae..411502c45d 100644 --- a/pkg/apis/llm/llm_container.go +++ b/pkg/apis/llm/llm_container.go @@ -11,6 +11,7 @@ type LLMContainerType string const ( LLM_CONTAINER_OLLAMA LLMContainerType = "ollama" LLM_CONTAINER_VLLM LLMContainerType = "vllm" + LLM_CONTAINER_SGLANG LLMContainerType = "sglang" LLM_CONTAINER_DIFY LLMContainerType = "dify" LLM_CONTAINER_COMFYUI LLMContainerType = "comfyui" LLM_CONTAINER_OPENCLAW LLMContainerType = "openclaw" @@ -21,6 +22,7 @@ var ( LLM_CONTAINER_TYPES = sets.NewString( string(LLM_CONTAINER_OLLAMA), string(LLM_CONTAINER_VLLM), + string(LLM_CONTAINER_SGLANG), string(LLM_CONTAINER_DIFY), string(LLM_CONTAINER_COMFYUI), string(LLM_CONTAINER_OPENCLAW), @@ -29,6 +31,7 @@ var ( LLM_INSTANT_MODEL_TYPES = sets.NewString( string(LLM_CONTAINER_OLLAMA), string(LLM_CONTAINER_VLLM), + string(LLM_CONTAINER_SGLANG), string(LLM_CONTAINER_COMFYUI), string(LLM_CONTAINER_OPENCLAW), ) diff --git a/pkg/apis/llm/llm_spec.go b/pkg/apis/llm/llm_spec.go index c225e947ce..66a8a19e6b 100644 --- a/pkg/apis/llm/llm_spec.go +++ b/pkg/apis/llm/llm_spec.go @@ -25,6 +25,7 @@ import ( type LLMSpec struct { Ollama *LLMSpecOllama `json:"ollama,omitempty"` Vllm *LLMSpecVllm `json:"vllm,omitempty"` + SGLang *LLMSpecSGLang `json:"sglang,omitempty"` Dify *LLMSpecDify `json:"dify,omitempty"` ComfyUI *LLMSpecComfyUI `json:"comfyui,omitempty"` OpenClaw *LLMSpecOpenClaw `json:"openclaw,omitempty"` @@ -39,7 +40,7 @@ func (s *LLMSpec) IsZero() bool { if s == nil { return true } - return s.Ollama == nil && s.Vllm == nil && s.Dify == nil && s.ComfyUI == nil && s.OpenClaw == nil && s.HermesAgent == nil + return s.Ollama == nil && s.Vllm == nil && s.SGLang == nil && s.Dify == nil && s.ComfyUI == nil && s.OpenClaw == nil && s.HermesAgent == nil } // LLMSpecOllama holds type-specific fields for ollama SKUs. @@ -80,6 +81,29 @@ func (s *LLMSpecVllm) IsZero() bool { return s.PreferredModel == "" && len(s.CustomizedArgs) == 0 } +// LLMSpecSGLang holds type-specific fields for SGLang SKUs. +type LLMSpecSGLang struct { + PreferredModel string `json:"preferred_model"` + // On update, a provided customized_args list replaces the previous list. + CustomizedArgs []*SGLangCustomizedArg `json:"customized_args,omitempty"` +} + +type SGLangCustomizedArg struct { + Key string `json:"key"` + Value string `json:"value"` +} + +func (s *LLMSpecSGLang) String() string { + return jsonutils.Marshal(s).String() +} + +func (s *LLMSpecSGLang) IsZero() bool { + if s == nil { + return true + } + return s.PreferredModel == "" && len(s.CustomizedArgs) == 0 +} + // LLMSpecDify holds type-specific fields for Dify SKUs (multiple image ids + customized envs). type LLMSpecDify struct { PostgresImageId string `json:"postgres_image_id"` diff --git a/pkg/apis/llm/sglang_const.go b/pkg/apis/llm/sglang_const.go new file mode 100644 index 0000000000..777838f95f --- /dev/null +++ b/pkg/apis/llm/sglang_const.go @@ -0,0 +1,12 @@ +package llm + +const ( + LLM_SGLANG = "sglang" + LLM_SGLANG_DEFAULT_PORT = 30000 + LLM_SGLANG_EXEC_PATH = "python3 -m sglang.launch_server" + + LLM_SGLANG_HF_ENDPOINT = LLM_VLLM_HF_ENDPOINT + LLM_SGLANG_CACHE_DIR = "/root/.cache/huggingface" + LLM_SGLANG_BASE_PATH = "/data/models" + LLM_SGLANG_MODELS_PATH = "/data/models/huggingface" +) diff --git a/pkg/llm/drivers/llm_container/sglang.go b/pkg/llm/drivers/llm_container/sglang.go new file mode 100644 index 0000000000..32d3053c6a --- /dev/null +++ b/pkg/llm/drivers/llm_container/sglang.go @@ -0,0 +1,648 @@ +package llm_container + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "unicode" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + commonapi "yunion.io/x/onecloud/pkg/apis" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/llm/models" + "yunion.io/x/onecloud/pkg/mcclient" +) + +func init() { + models.RegisterLLMContainerDriver(newSGLang()) +} + +type sglang struct { + baseDriver +} + +func newSGLang() models.ILLMContainerDriver { + return &sglang{baseDriver: newBaseDriver(api.LLM_CONTAINER_SGLANG)} +} + +func (s *sglang) GetSpec(sku *models.SLLMSku) interface{} { + if sku == nil || sku.LLMType != string(api.LLM_CONTAINER_SGLANG) || sku.LLMSpec == nil || sku.LLMSpec.SGLang == nil { + return nil + } + return sku.LLMSpec.SGLang +} + +func (s *sglang) GetEffectiveSpec(llm *models.SLLM, sku *models.SLLMSku) interface{} { + var skuSpec *api.LLMSpecSGLang + if spec := s.GetSpec(sku); spec != nil { + skuSpec = spec.(*api.LLMSpecSGLang) + } + var llmSpec *api.LLMSpecSGLang + if llm != nil && llm.LLMSpec != nil && llm.LLMSpec.SGLang != nil { + llmSpec = llm.LLMSpec.SGLang + } + if skuSpec == nil && llmSpec == nil { + return nil + } + out := &api.LLMSpecSGLang{} + if skuSpec != nil { + out.PreferredModel = skuSpec.PreferredModel + out.CustomizedArgs = skuSpec.CustomizedArgs + } + if llmSpec != nil { + if llmSpec.PreferredModel != "" { + out.PreferredModel = llmSpec.PreferredModel + } + } + mergedArgs, err := normalizeSGLangCustomizedArgs(out.CustomizedArgs) + if err != nil { + log.Errorf("normalize sku sglang customized args: %v", err) + out.CustomizedArgs = nil + } else { + out.CustomizedArgs = mergedArgs + } + if llmSpec != nil { + mergedArgs, err = mergeSGLangCustomizedArgs(out.CustomizedArgs, llmSpec.CustomizedArgs) + if err != nil { + log.Errorf("merge sglang customized args: %v", err) + } else { + out.CustomizedArgs = mergedArgs + } + } + return out +} + +func (s *sglang) ValidateLLMCreateData(ctx context.Context, userCred mcclient.TokenCredential, sku *models.SLLMSku, input *api.LLMCreateInput) (*api.LLMCreateInput, error) { + llmType := string(api.LLM_CONTAINER_SGLANG) + if err := models.ValidateRequireDevices(llmType, input.Devices, nil, sku); err != nil { + return input, err + } + if err := models.ValidateRequireMountedModels(llmType, input.MountedModels, nil, sku); err != nil { + return input, err + } + return input, nil +} + +func (s *sglang) ValidateLLMUpdateData(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, sku *models.SLLMSku, input *api.LLMUpdateInput) (*api.LLMUpdateInput, error) { + llmType := string(api.LLM_CONTAINER_SGLANG) + if err := models.ValidateRequireDevices(llmType, input.Devices, llm.Devices, sku); err != nil { + return input, err + } + if err := models.ValidateRequireMountedModels(llmType, input.MountedModels, llm.MountedModels, sku); err != nil { + return input, err + } + return input, nil +} + +func (s *sglang) ValidateLLMSkuCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.LLMSkuCreateInput) (*api.LLMSkuCreateInput, error) { + input, err := s.baseDriver.ValidateLLMSkuCreateData(ctx, userCred, input) + if err != nil { + return nil, err + } + + spec, err := s.ValidateLLMCreateSpec(ctx, userCred, nil, input.LLMSpec) + if err != nil { + return nil, err + } + if spec == nil { + spec = &api.LLMSpec{SGLang: &api.LLMSpecSGLang{}} + } else if spec.SGLang == nil { + spec.SGLang = &api.LLMSpecSGLang{} + } + input.LLMSpec = spec + return input, nil +} + +func (s *sglang) ValidateLLMSkuUpdateData(ctx context.Context, userCred mcclient.TokenCredential, sku *models.SLLMSku, input *api.LLMSkuUpdateInput) (*api.LLMSkuUpdateInput, error) { + input, err := s.baseDriver.ValidateLLMSkuUpdateData(ctx, userCred, sku, input) + if err != nil { + return nil, err + } + if input.LLMSpec == nil { + return input, nil + } + + fakeLLM := &models.SLLM{LLMSpec: sku.LLMSpec} + spec, err := s.ValidateLLMUpdateSpec(ctx, userCred, fakeLLM, input.LLMSpec) + if err != nil { + return nil, err + } + input.LLMSpec = spec + if input.LLMSpec != nil && input.LLMSpec.SGLang == nil { + input.LLMSpec.SGLang = &api.LLMSpecSGLang{} + } + return input, nil +} + +func (s *sglang) ValidateLLMCreateSpec(ctx context.Context, userCred mcclient.TokenCredential, sku *models.SLLMSku, input *api.LLMSpec) (*api.LLMSpec, error) { + if input == nil { + return nil, nil + } + if input.SGLang == nil { + input.SGLang = &api.LLMSpecSGLang{} + } + + preferred := input.SGLang.PreferredModel + if preferred == "" && sku != nil && sku.LLMSpec != nil && sku.LLMSpec.SGLang != nil { + preferred = sku.LLMSpec.SGLang.PreferredModel + } + + spec := &api.LLMSpecSGLang{} + if sku != nil && sku.LLMSpec != nil && sku.LLMSpec.SGLang != nil { + base := *sku.LLMSpec.SGLang + spec = &base + } + if preferred != "" { + spec.PreferredModel = preferred + } + mergedArgs, err := mergeSGLangCustomizedArgs(spec.CustomizedArgs, input.SGLang.CustomizedArgs) + if err != nil { + return nil, err + } + spec.CustomizedArgs = mergedArgs + + return &api.LLMSpec{SGLang: spec}, nil +} + +func (s *sglang) ValidateLLMUpdateSpec(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, input *api.LLMSpec) (*api.LLMSpec, error) { + if input == nil || input.SGLang == nil { + return input, nil + } + base := &api.LLMSpecSGLang{} + if llm != nil && llm.LLMSpec != nil && llm.LLMSpec.SGLang != nil { + b := *llm.LLMSpec.SGLang + base = &b + } + + if input.SGLang.PreferredModel != "" { + base.PreferredModel = input.SGLang.PreferredModel + } + customizedArgs, err := normalizeSGLangCustomizedArgs(base.CustomizedArgs) + if err != nil { + return nil, err + } + base.CustomizedArgs = customizedArgs + if input.SGLang.CustomizedArgs != nil { + customizedArgs, err = normalizeSGLangCustomizedArgs(input.SGLang.CustomizedArgs) + if err != nil { + return nil, err + } + base.CustomizedArgs = customizedArgs + } + + return &api.LLMSpec{SGLang: base}, nil +} + +func (s *sglang) GetContainerSpec(ctx context.Context, llm *models.SLLM, image *models.SLLMImage, sku *models.SLLMSku, props []string, devices []computeapi.SIsolatedDevice, diskId string) *computeapi.PodContainerCreateInput { + var postOverlays []*commonapi.ContainerVolumeMountDiskPostOverlay + if llm != nil { + var err error + postOverlays, err = llm.GetMountedModelsPostOverlay() + if err != nil { + log.Errorf("GetMountedModelsPostOverlay failed %s", err) + } + } + tensorParallelSize := 1 + if sku != nil && sku.Devices != nil && len(*sku.Devices) > 0 { + tensorParallelSize = len(*sku.Devices) + } + effSpec := (*api.LLMSpecSGLang)(nil) + if eff := s.GetEffectiveSpec(llm, sku); eff != nil { + effSpec = eff.(*api.LLMSpecSGLang) + } + startScript := buildSGLangEntrypointScript(len(postOverlays) > 0, tensorParallelSize, effSpec) + envs := []*commonapi.ContainerKeyValue{ + { + Key: "HUGGING_FACE_HUB_CACHE", + Value: api.LLM_SGLANG_CACHE_DIR, + }, + { + Key: "HF_ENDPOINT", + Value: api.LLM_SGLANG_HF_ENDPOINT, + }, + } + spec := computeapi.ContainerSpec{ + ContainerSpec: commonapi.ContainerSpec{ + Image: image.ToContainerImage(), + ImageCredentialId: image.CredentialId, + Command: []string{"/bin/sh", "-c"}, + Args: []string{startScript}, + EnableLxcfs: true, + AlwaysRestart: true, + Envs: envs, + }, + } + + effDevs := models.GetEffectiveDevices(llm, sku) + if len(devices) == 0 && effDevs != nil && len(*effDevs) > 0 { + for i := range *effDevs { + index := i + spec.Devices = append(spec.Devices, &computeapi.ContainerDevice{ + Type: commonapi.CONTAINER_DEVICE_TYPE_ISOLATED_DEVICE, + IsolatedDevice: &computeapi.ContainerIsolatedDevice{ + Index: &index, + }, + }) + } + } else if len(devices) > 0 { + for i := range devices { + spec.Devices = append(spec.Devices, &computeapi.ContainerDevice{ + Type: commonapi.CONTAINER_DEVICE_TYPE_ISOLATED_DEVICE, + IsolatedDevice: &computeapi.ContainerIsolatedDevice{ + Id: devices[i].Id, + }, + }) + } + } + + diskIndex := 0 + ctrVols := []*commonapi.ContainerVolumeMount{ + { + Disk: &commonapi.ContainerVolumeMountDisk{ + SubDirectory: api.LLM_SGLANG, + Index: &diskIndex, + PostOverlay: postOverlays, + }, + Type: commonapi.CONTAINER_VOLUME_MOUNT_TYPE_DISK, + MountPath: api.LLM_SGLANG_BASE_PATH, + ReadOnly: false, + Propagation: commonapi.MOUNTPROPAGATION_PROPAGATION_HOST_TO_CONTAINER, + }, + { + Disk: &commonapi.ContainerVolumeMountDisk{ + SubDirectory: "cache", + Index: &diskIndex, + }, + Type: commonapi.CONTAINER_VOLUME_MOUNT_TYPE_DISK, + MountPath: "/root/.cache", + ReadOnly: false, + }, + } + spec.VolumeMounts = append(spec.VolumeMounts, ctrVols...) + + return &computeapi.PodContainerCreateInput{ + ContainerSpec: spec, + } +} + +func (s *sglang) GetContainerSpecs(ctx context.Context, llm *models.SLLM, image *models.SLLMImage, sku *models.SLLMSku, props []string, devices []computeapi.SIsolatedDevice, diskId string) []*computeapi.PodContainerCreateInput { + return []*computeapi.PodContainerCreateInput{ + s.GetContainerSpec(ctx, llm, image, sku, props, devices, diskId), + } +} + +func (s *sglang) GetLLMAccessUrlInfo(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, input *models.LLMAccessInfoInput) (*api.LLMAccessUrlInfo, error) { + return models.GetLLMAccessUrlInfo(ctx, userCred, llm, input, "http", api.LLM_SGLANG_DEFAULT_PORT) +} + +func (s *sglang) StartLLM(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM) error { + return nil +} + +func (s *sglang) GetProbedInstantModelsExt(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, mdlIds ...string) (map[string]api.LLMInternalInstantMdlInfo, error) { + lc, err := llm.GetLLMContainer() + if err != nil { + return nil, errors.Wrap(err, "get llm container") + } + + cmd := fmt.Sprintf("du -sk %s/*/", api.LLM_SGLANG_MODELS_PATH) + output, err := exec(ctx, lc.CmpId, cmd, 10) + if err != nil { + return make(map[string]api.LLMInternalInstantMdlInfo), nil + } + + modelsMap := make(map[string]api.LLMInternalInstantMdlInfo) + lines := strings.Split(strings.TrimSpace(output), "\n") + for _, line := range lines { + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + sizeKB, _ := strconv.ParseInt(fields[0], 10, 64) + fullPath := fields[1] + name := path.Base(fullPath) + if name == "" { + continue + } + modelsMap[name] = api.LLMInternalInstantMdlInfo{ + Name: name, + ModelId: name, + Tag: "latest", + Size: sizeKB * 1024, + } + } + return modelsMap, nil +} + +func (s *sglang) DetectModelPaths(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, pkgInfo api.LLMInternalInstantMdlInfo) ([]string, error) { + lc, err := llm.GetLLMContainer() + if err != nil { + return nil, errors.Wrap(err, "get llm container") + } + + modelPath := path.Join(api.LLM_SGLANG_MODELS_PATH, pkgInfo.Name) + checkCmd := fmt.Sprintf("[ -d %s ] && echo 'EXIST' || echo 'MISSING'", shellQuoteSingle(modelPath)) + output, err := exec(ctx, lc.CmpId, checkCmd, 10) + if err != nil { + return nil, errors.Wrap(err, "failed to check file existence") + } + if !strings.Contains(output, "EXIST") { + return nil, errors.Errorf("model directory %s missing", modelPath) + } + return []string{modelPath}, nil +} + +func (s *sglang) GetImageInternalPathMounts(sApp *models.SInstantModel) map[string]string { + res := make(map[string]string) + for _, mount := range sApp.Mounts { + relPath := strings.TrimPrefix(mount, api.LLM_SGLANG_BASE_PATH) + res[relPath] = path.Join(api.LLM_SGLANG, relPath) + } + return res +} + +func (s *sglang) GetSaveDirectories(sApp *models.SInstantModel) (string, []string, error) { + var filteredMounts []string + for _, mount := range sApp.Mounts { + if strings.HasPrefix(mount, api.LLM_SGLANG_BASE_PATH) { + relPath := strings.TrimPrefix(mount, api.LLM_SGLANG_BASE_PATH) + filteredMounts = append(filteredMounts, relPath) + } + } + return "", filteredMounts, nil +} + +func (s *sglang) ValidateMounts(mounts []string, mdlName string, mdlTag string) ([]string, error) { + return mounts, nil +} + +func (s *sglang) CheckDuplicateMounts(errStr string, dupIndex int) string { + return "Duplicate mounts detected" +} + +func (s *sglang) GetInstantModelIdByPostOverlay(postOverlay *commonapi.ContainerVolumeMountDiskPostOverlay, mdlNameToId map[string]string) string { + return "" +} + +func (s *sglang) GetDirPostOverlay(dir api.LLMMountDirInfo) *commonapi.ContainerVolumeMountDiskPostOverlay { + uid := int64(1000) + gid := int64(1000) + ov := dir.ToOverlay() + ov.FsUser = &uid + ov.FsGroup = &gid + return &ov +} + +func (s *sglang) PreInstallModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, instMdl *models.SLLMInstantModel) error { + lc, err := llm.GetLLMContainer() + if err != nil { + return errors.Wrap(err, "get llm container") + } + cmd := fmt.Sprintf("mkdir -p %s", api.LLM_SGLANG_MODELS_PATH) + _, err = exec(ctx, lc.CmpId, cmd, 10) + return err +} + +func (s *sglang) InstallModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, dirs []string, mdlIds []string) error { + return nil +} + +func (s *sglang) UninstallModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, instMdl *models.SLLMInstantModel) error { + return nil +} + +func (s *sglang) DownloadModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, tmpDir string, modelName string, modelTag string) (string, []string, error) { + if strings.TrimSpace(tmpDir) == "" { + return "", nil, errors.Error("tmpDir is empty") + } + if strings.TrimSpace(modelName) == "" { + return "", nil, errors.Error("modelName is empty") + } + + modelBase := filepath.Base(modelName) + localDir := filepath.Join(tmpDir, "huggingface", modelBase) + if err := os.MkdirAll(localDir, 0755); err != nil { + return "", nil, errors.Wrap(err, "mkdir local model dir") + } + if entries, err := os.ReadDir(localDir); err == nil && len(entries) > 0 { + targetDir := path.Join(api.LLM_SGLANG_MODELS_PATH, modelBase) + log.Infof("Model %s already exists in import dir %s", modelName, localDir) + return modelName, []string{targetDir}, nil + } + + rev := resolveHfdRevision(modelTag) + apiURL := fmt.Sprintf("%s/api/models/%s?revision=%s", api.LLM_SGLANG_HF_ENDPOINT, escapeURLPathPreserveSlash(modelName), url.QueryEscape(rev)) + log.Infof("Downloading HF model via HF Mirror API for SGLang: %s", func() string { + b, _ := json.Marshal(map[string]string{ + "model": modelName, + "revision": rev, + "dir": localDir, + "endpoint": api.LLM_SGLANG_HF_ENDPOINT, + "api": apiURL, + }) + return string(b) + }()) + metaBody, err := llm.HttpGet(ctx, apiURL) + if err != nil { + return "", nil, errors.Wrapf(err, "fetch hf model metadata failed: %s", apiURL) + } + meta := hfModelAPIResponse{} + if err := json.Unmarshal(metaBody, &meta); err != nil { + return "", nil, errors.Wrap(err, "unmarshal hf model metadata") + } + if len(meta.Siblings) == 0 { + return "", nil, errors.Errorf("hf model metadata has no siblings: %s", apiURL) + } + + for _, sibling := range meta.Siblings { + rf := strings.TrimSpace(sibling.RFilename) + if rf == "" { + continue + } + dst := filepath.Join(localDir, filepath.FromSlash(rf)) + if isNonEmptyFile(dst) { + continue + } + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return "", nil, errors.Wrapf(err, "mkdir for %s", dst) + } + fileURL := fmt.Sprintf("%s/%s/resolve/%s/%s", api.LLM_SGLANG_HF_ENDPOINT, escapeURLPathPreserveSlash(modelName), url.PathEscape(rev), escapeURLPathPreserveSlash(rf)) + if err := llm.HttpDownloadFile(ctx, fileURL, dst); err != nil { + return "", nil, errors.Wrapf(err, "download file failed: %s -> %s", fileURL, dst) + } + } + + targetDir := path.Join(api.LLM_SGLANG_MODELS_PATH, modelBase) + return modelName, []string{targetDir}, nil +} + +var protectedSGLangArgKeys = map[string]struct{}{ + "model-path": {}, + "served-model-name": {}, + "host": {}, + "port": {}, + "tp-size": {}, +} + +func validateSGLangArgKey(key string) error { + if key == "" { + return errors.Error("sglang arg key is empty") + } + if strings.HasPrefix(key, "--") { + return errors.Errorf("invalid sglang arg key %q: do not include leading --", key) + } + for _, r := range key { + if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '-' || r == '_' { + continue + } + return errors.Errorf("invalid sglang arg key %q", key) + } + if _, ok := protectedSGLangArgKeys[key]; ok { + return errors.Errorf("sglang arg key %q is protected", key) + } + return nil +} + +func normalizeSGLangCustomizedArgs(args []*api.SGLangCustomizedArg) ([]*api.SGLangCustomizedArg, error) { + if len(args) == 0 { + return nil, nil + } + out := make([]*api.SGLangCustomizedArg, 0, len(args)) + indexByKey := make(map[string]int, len(args)) + for _, arg := range args { + if arg == nil { + continue + } + key := strings.TrimSpace(arg.Key) + if err := validateSGLangArgKey(key); err != nil { + return nil, err + } + next := &api.SGLangCustomizedArg{ + Key: key, + Value: arg.Value, + } + if idx, ok := indexByKey[key]; ok { + out[idx] = next + continue + } + indexByKey[key] = len(out) + out = append(out, next) + } + if len(out) == 0 { + return nil, nil + } + return out, nil +} + +func mergeSGLangCustomizedArgs(base, overrides []*api.SGLangCustomizedArg) ([]*api.SGLangCustomizedArg, error) { + out := make([]*api.SGLangCustomizedArg, 0, len(base)+len(overrides)) + indexByKey := make(map[string]int, len(base)+len(overrides)) + appendNormalized := func(items []*api.SGLangCustomizedArg) error { + normalized, err := normalizeSGLangCustomizedArgs(items) + if err != nil { + return err + } + for _, arg := range normalized { + if idx, ok := indexByKey[arg.Key]; ok { + out[idx] = arg + continue + } + indexByKey[arg.Key] = len(out) + out = append(out, arg) + } + return nil + } + if err := appendNormalized(base); err != nil { + return nil, err + } + if err := appendNormalized(overrides); err != nil { + return nil, err + } + if len(out) == 0 { + return nil, nil + } + return out, nil +} + +func appendSGLangCustomizedFlags(flags []string, effSpec *api.LLMSpecSGLang) []string { + if effSpec == nil || len(effSpec.CustomizedArgs) == 0 { + return flags + } + normalizedArgs, err := normalizeSGLangCustomizedArgs(effSpec.CustomizedArgs) + if err != nil { + log.Errorf("normalize sglang customized args: %v", err) + return flags + } + for _, arg := range normalizedArgs { + flagName := "--" + arg.Key + if arg.Value == "" { + flags = append(flags, flagName) + continue + } + flags = append(flags, fmt.Sprintf("%s %s", flagName, shellQuoteSingle(arg.Value))) + } + return flags +} + +func buildSGLangServeFlagsWithModelExpr(modelExpr string, servedModelNameExpr string, tensorParallelSize int, effSpec *api.LLMSpecSGLang) []string { + flags := []string{ + fmt.Sprintf("--model-path %s", modelExpr), + fmt.Sprintf("--served-model-name %s", servedModelNameExpr), + "--host 0.0.0.0", + fmt.Sprintf("--port %d", api.LLM_SGLANG_DEFAULT_PORT), + fmt.Sprintf("--tp-size %d", tensorParallelSize), + } + return appendSGLangCustomizedFlags(flags, effSpec) +} + +func buildSGLangServeFlags(modelPath string, tensorParallelSize int, effSpec *api.LLMSpecSGLang) []string { + modelQuoted := shellQuoteSingle(modelPath) + return buildSGLangServeFlagsWithModelExpr( + modelQuoted, + fmt.Sprintf(`"$(basename %s)"`, modelQuoted), + tensorParallelSize, + effSpec, + ) +} + +func buildSGLangEntrypointScript(hasMountedModels bool, tensorParallelSize int, effSpec *api.LLMSpecSGLang) string { + modelsPath := shellQuoteSingle(api.LLM_SGLANG_MODELS_PATH) + if !hasMountedModels { + return fmt.Sprintf("mkdir -p %s && exec sleep infinity", modelsPath) + } + + preferredPath := "" + if effSpec != nil && strings.TrimSpace(effSpec.PreferredModel) != "" { + preferredPath = path.Join(api.LLM_SGLANG_MODELS_PATH, strings.TrimSpace(effSpec.PreferredModel)) + } + serveCmd := strings.Join(buildSGLangServeFlagsWithModelExpr( + `"$model"`, + `"$(basename "$model")"`, + tensorParallelSize, + effSpec, + ), " ") + return strings.Join([]string{ + "set -e", + fmt.Sprintf("mkdir -p %s", modelsPath), + fmt.Sprintf("preferred=%s", shellQuoteSingle(preferredPath)), + `if [ -n "$preferred" ] && [ -d "$preferred" ]; then`, + ` model="$preferred"`, + "else", + fmt.Sprintf(` model="$(find %s -mindepth 1 -maxdepth 1 -type d | sort | head -n 1)"`, modelsPath), + "fi", + `if [ -z "$model" ]; then`, + ` echo "no mounted SGLang model found" >&2`, + " exit 1", + "fi", + fmt.Sprintf("exec %s %s", api.LLM_SGLANG_EXEC_PATH, serveCmd), + }, "\n") +} diff --git a/pkg/llm/models/instantmodel.go b/pkg/llm/models/instantmodel.go index e82ec267e2..0c2ca671fc 100644 --- a/pkg/llm/models/instantmodel.go +++ b/pkg/llm/models/instantmodel.go @@ -106,6 +106,9 @@ func (man *SInstantModelManager) ListItemFilter( if len(input.ModelId) > 0 { q = q.In("model_id", input.ModelId) } + if len(input.LlmType) > 0 { + q = q.Equals("llm_type", input.LlmType) + } if len(input.Image) > 0 { s := auth.GetSession(ctx, userCred, options.Options.Region) diff --git a/pkg/llm/models/instantmodel_huggingface.go b/pkg/llm/models/instantmodel_huggingface.go index ed27628845..6dec4edb90 100644 --- a/pkg/llm/models/instantmodel_huggingface.go +++ b/pkg/llm/models/instantmodel_huggingface.go @@ -158,7 +158,7 @@ func normalizeHuggingFaceSearchResults(items []huggingFaceSearchItem) []apis.Ins result.UnsupportedReason = "disabled repositories are not supported" case hasTag(item.Tags, "gguf"): result.Supported = false - result.UnsupportedReason = "gguf repositories are not supported for vllm import" + result.UnsupportedReason = "gguf repositories are not supported for hf snapshot import" } results = append(results, result) } @@ -192,13 +192,13 @@ func buildHuggingFaceRepoInfo(resp huggingFaceRepoInfoResponse, requestedRevisio switch { case info.GgufPresent: info.Supported = false - info.UnsupportedReason = "gguf repositories are not supported for vllm import" + info.UnsupportedReason = "gguf repositories are not supported for hf snapshot import" case !info.ConfigPresent: info.Supported = false - info.UnsupportedReason = "config.json is required for vllm import" + info.UnsupportedReason = "config.json is required for hf snapshot import" case !info.SafetensorsPresent: info.Supported = false - info.UnsupportedReason = "no safetensors weights detected for vllm import" + info.UnsupportedReason = "no safetensors weights detected for hf snapshot import" } if !info.Supported { info.ImportMode = "" diff --git a/pkg/mcclient/options/llm/image.go b/pkg/mcclient/options/llm/image.go index 1a128031be..f293a8dfaa 100644 --- a/pkg/mcclient/options/llm/image.go +++ b/pkg/mcclient/options/llm/image.go @@ -18,7 +18,7 @@ func (o *LLMImageShowOptions) Params() (jsonutils.JSONObject, error) { type LLMImageListOptions struct { options.BaseListOptions - LLMType string `json:"llm_type" choices:"ollama|dify|comfyui|vllm" help:"filter by llm type"` + LLMType string `json:"llm_type" choices:"ollama|dify|comfyui|vllm|sglang" help:"filter by llm type"` } func (o *LLMImageListOptions) Params() (jsonutils.JSONObject, error) { @@ -30,7 +30,7 @@ type LLMImageCreateOptions struct { IMAGE_NAME string `json:"image_name"` IMAGE_LABEL string `json:"image_label"` CredentialId string `json:"credential_id"` - LLM_TYPE string `json:"llm_type" choices:"ollama|dify|comfyui|vllm" help:"llm type: ollama, comfyui or dify"` + LLM_TYPE string `json:"llm_type" choices:"ollama|dify|comfyui|vllm|sglang" help:"llm type: ollama, comfyui, vllm, sglang or dify"` } func (o *LLMImageCreateOptions) Params() (jsonutils.JSONObject, error) { @@ -44,7 +44,7 @@ type LLMImageUpdateOptions struct { ImageName string `json:"image_name"` ImageLabel string `json:"image_label"` CredentialId string `json:"credential_id"` - LlmType string `json:"llm_type" choices:"ollama|dify|vllm|comfyui" help:"llm type: ollama, comfyui, vllm or dify"` + LlmType string `json:"llm_type" choices:"ollama|dify|vllm|sglang|comfyui" help:"llm type: ollama, comfyui, vllm, sglang or dify"` } func (o *LLMImageUpdateOptions) GetId() string { diff --git a/pkg/mcclient/options/llm/instantmodel.go b/pkg/mcclient/options/llm/instantmodel.go index aebf8f2e97..5599f9eba6 100644 --- a/pkg/mcclient/options/llm/instantmodel.go +++ b/pkg/mcclient/options/llm/instantmodel.go @@ -12,6 +12,7 @@ type LLMInstantModelListOptions struct { ModelName []string `help:"filter by model name"` ModelTag []string `help:"filter by model tag"` + LLMType string `json:"llm_type" choices:"ollama|vllm|sglang|comfyui" help:"filter by llm type"` } func (o *LLMInstantModelListOptions) Params() (jsonutils.JSONObject, error) { @@ -29,7 +30,7 @@ func (o *LLMInstantModelShowOptions) Params() (jsonutils.JSONObject, error) { type LLMInstantModelCreateOptions struct { options.BaseCreateOptions - LLM_TYPE string `help:"llm instant model type" choices:"ollama|vllm|comfyui" json:"llm_type"` + LLM_TYPE string `help:"llm instant model type" choices:"ollama|vllm|sglang|comfyui" json:"llm_type"` MODEL_NAME string `json:"model_name"` MODEL_TAG string `json:"model_tag"` @@ -66,7 +67,7 @@ func (o *LLMInstantModelDeleteOptions) Params() (jsonutils.JSONObject, error) { } type LLMInstantModelImportOptions struct { - LLM_TYPE string `help:"llm instant model type" choices:"ollama|vllm|comfyui" json:"llm_type"` + LLM_TYPE string `help:"llm instant model type" choices:"ollama|vllm|sglang|comfyui" json:"llm_type"` MODEL_NAME string `help:"model name to import, e.g. qwen3 or Qwen/Qwen3-VL-8B-Instruct" json:"model_name"` MODEL_TAG string `help:"model tag to import, e.g. 8b" json:"model_tag"` REPO_ID string `help:"huggingface repo id, e.g. Qwen/Qwen3-8B" json:"repo_id"` diff --git a/pkg/mcclient/options/llm/llm.go b/pkg/mcclient/options/llm/llm.go index 5fbcf33bfe..0279fcecae 100644 --- a/pkg/mcclient/options/llm/llm.go +++ b/pkg/mcclient/options/llm/llm.go @@ -72,6 +72,9 @@ type LLMCreateOptions struct { LLM_SKU_ID string `help:"llm sku id or name" json:"llm_sku_id"` PreferredModel string `help:"vLLM preferred model dir name under models path (e.g. Qwen/Qwen2-7B)" json:"-"` VllmArg []string `help:"vLLM args in format key=value; use key= for flags without values" json:"-"` + + SGLangPreferredModel string `token:"sglang-preferred-model" help:"SGLang preferred model dir name under models path (e.g. Qwen/Qwen2-7B)" json:"-"` + SGLangArg []string `token:"sglang-arg" help:"SGLang args in format key=value; use key= for flags without values" json:"-"` } func (o *LLMCreateOptions) Params() (jsonutils.JSONObject, error) { @@ -93,10 +96,21 @@ func (o *LLMCreateOptions) Params() (jsonutils.JSONObject, error) { if err != nil { return nil, err } + sglangSpec, err := newSGLangSpecFromArgs(o.SGLangPreferredModel, o.SGLangArg) + if err != nil { + return nil, err + } + if vllmSpec != nil && sglangSpec != nil { + return nil, errors.Error("cannot specify both vLLM and SGLang llm spec args") + } if vllmSpec != nil { spec := &api.LLMSpec{Ollama: nil, Vllm: vllmSpec, Dify: nil} params.Set("llm_spec", jsonutils.Marshal(spec)) } + if sglangSpec != nil { + spec := &api.LLMSpec{SGLang: sglangSpec} + params.Set("llm_spec", jsonutils.Marshal(spec)) + } return params, nil } @@ -110,6 +124,9 @@ type LLMUpdateOptions struct { PreferredModel string `help:"vLLM preferred model dir name under models path (e.g. Qwen/Qwen2-7B); takes effect after pod recreate" json:"-"` VllmArg []string `help:"vLLM args in format key=value; use key= for flags without values" json:"-"` + + SGLangPreferredModel string `token:"sglang-preferred-model" help:"SGLang preferred model dir name under models path (e.g. Qwen/Qwen2-7B); takes effect after pod recreate" json:"-"` + SGLangArg []string `token:"sglang-arg" help:"SGLang args in format key=value; use key= for flags without values" json:"-"` } func (o *LLMUpdateOptions) GetId() string { @@ -125,10 +142,21 @@ func (o *LLMUpdateOptions) Params() (jsonutils.JSONObject, error) { if err != nil { return nil, err } + sglangSpec, err := newSGLangSpecFromArgs(o.SGLangPreferredModel, o.SGLangArg) + if err != nil { + return nil, err + } + if vllmSpec != nil && sglangSpec != nil { + return nil, errors.Error("cannot specify both vLLM and SGLang llm spec args") + } if vllmSpec != nil { spec := &api.LLMSpec{Ollama: nil, Vllm: vllmSpec, Dify: nil} dict.Set("llm_spec", jsonutils.Marshal(spec)) } + if sglangSpec != nil { + spec := &api.LLMSpec{SGLang: sglangSpec} + dict.Set("llm_spec", jsonutils.Marshal(spec)) + } return dict, nil } diff --git a/pkg/mcclient/options/llm/llm_sku.go b/pkg/mcclient/options/llm/llm_sku.go index 33d5bf18d0..a07e4da828 100644 --- a/pkg/mcclient/options/llm/llm_sku.go +++ b/pkg/mcclient/options/llm/llm_sku.go @@ -2,6 +2,7 @@ package llm import ( "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/llm" "yunion.io/x/onecloud/pkg/mcclient/options" @@ -10,7 +11,7 @@ import ( type LLMSkuListOptions struct { options.BaseListOptions - LLMType string `json:"llm_type" choices:"ollama|comfyui|openclaw"` + LLMType string `json:"llm_type" choices:"ollama|vllm|sglang|dify|comfyui|openclaw|hermes-agent"` } func (o *LLMSkuListOptions) Params() (jsonutils.JSONObject, error) { @@ -31,10 +32,13 @@ type LLMSkuCreateOptions struct { MountedModels []string `help:"mounted models, e.g. qwen2:0.5b-dup" json:"mounted_models"` LLM_IMAGE_ID string `json:"llm_image_id"` - LLM_TYPE string `json:"llm_type" choices:"ollama|vllm|comfyui"` + LLM_TYPE string `json:"llm_type" choices:"ollama|vllm|sglang|comfyui"` - PreferredModel string `help:"preferred model (vllm only), sets llm_spec.vllm.preferred_model" json:"-"` + PreferredModel string `help:"preferred model (vllm/sglang), sets llm_spec..preferred_model" json:"-"` VllmArg []string `help:"vLLM args in format key=value; use key= for flags without values" json:"-"` + + SGLangPreferredModel string `token:"sglang-preferred-model" help:"SGLang preferred model; overrides preferred-model when llm_type=sglang" json:"-"` + SGLangArg []string `token:"sglang-arg" help:"SGLang args in format key=value; use key= for flags without values" json:"-"` } func (o *LLMSkuCreateOptions) Params() (jsonutils.JSONObject, error) { @@ -45,17 +49,29 @@ func (o *LLMSkuCreateOptions) Params() (jsonutils.JSONObject, error) { return nil, err } fetchMountedModels(o.MountedModels, dict) - vllmSpec, err := newVLLMSpecFromArgs(o.PreferredModel, o.VllmArg) - if err != nil { - return nil, err - } - if o.LLM_TYPE == string(api.LLM_CONTAINER_VLLM) && vllmSpec != nil { - spec := &api.LLMSpec{ - Ollama: nil, - Vllm: vllmSpec, - Dify: nil, + switch o.LLM_TYPE { + case string(api.LLM_CONTAINER_VLLM): + vllmSpec, err := newVLLMSpecFromArgs(o.PreferredModel, o.VllmArg) + if err != nil { + return nil, err + } + if vllmSpec != nil { + spec := &api.LLMSpec{ + Ollama: nil, + Vllm: vllmSpec, + Dify: nil, + } + dict.Set("llm_spec", jsonutils.Marshal(spec)) + } + case string(api.LLM_CONTAINER_SGLANG): + sglangSpec, err := newSGLangSpecFromArgs(firstNonEmpty(o.SGLangPreferredModel, o.PreferredModel), o.SGLangArg) + if err != nil { + return nil, err + } + if sglangSpec != nil { + spec := &api.LLMSpec{SGLang: sglangSpec} + dict.Set("llm_spec", jsonutils.Marshal(spec)) } - dict.Set("llm_spec", jsonutils.Marshal(spec)) } return dict, nil } @@ -77,11 +93,14 @@ type LLMSkuUpdateOptions struct { MountedModels []string `help:"mounted models, e.g. qwen2:0.5b-dup" json:"mounted_models"` - // For ollama/vllm; backend merges into LLMSpec. Use dify-sku update for dify type. + // For ollama/vllm/sglang; backend merges into LLMSpec. Use dify-sku update for dify type. LlmImageId string `json:"llm_image_id"` PreferredModel string `help:"preferred model (vllm only), sets llm_spec.vllm.preferred_model" json:"-"` VllmArg []string `help:"vLLM args in format key=value; use key= for flags without values" json:"-"` + + SGLangPreferredModel string `token:"sglang-preferred-model" help:"preferred model (SGLang only), sets llm_spec.sglang.preferred_model" json:"-"` + SGLangArg []string `token:"sglang-arg" help:"SGLang args in format key=value; use key= for flags without values" json:"-"` } func (o *LLMSkuUpdateOptions) GetId() string { @@ -100,6 +119,13 @@ func (o *LLMSkuUpdateOptions) Params() (jsonutils.JSONObject, error) { if err != nil { return nil, err } + sglangSpec, err := newSGLangSpecFromArgs(o.SGLangPreferredModel, o.SGLangArg) + if err != nil { + return nil, err + } + if vllmSpec != nil && sglangSpec != nil { + return nil, errors.Error("cannot specify both vLLM and SGLang llm spec args") + } if vllmSpec != nil { spec := &api.LLMSpec{ Ollama: nil, @@ -108,5 +134,9 @@ func (o *LLMSkuUpdateOptions) Params() (jsonutils.JSONObject, error) { } dict.Set("llm_spec", jsonutils.Marshal(spec)) } + if sglangSpec != nil { + spec := &api.LLMSpec{SGLang: sglangSpec} + dict.Set("llm_spec", jsonutils.Marshal(spec)) + } return dict, nil } diff --git a/pkg/mcclient/options/llm/vllm_args.go b/pkg/mcclient/options/llm/vllm_args.go index 966c686284..d48afb4074 100644 --- a/pkg/mcclient/options/llm/vllm_args.go +++ b/pkg/mcclient/options/llm/vllm_args.go @@ -42,3 +42,48 @@ func newVLLMSpecFromArgs(preferredModel string, items []string) (*api.LLMSpecVll CustomizedArgs: customizedArgs, }, nil } + +func parseSGLangCustomizedArgs(items []string) ([]*api.SGLangCustomizedArg, error) { + if len(items) == 0 { + return nil, nil + } + out := make([]*api.SGLangCustomizedArg, 0, len(items)) + for _, item := range items { + idx := strings.Index(item, "=") + if idx <= 0 { + return nil, fmt.Errorf("invalid sglang arg %q, expected key=value", item) + } + key := strings.TrimSpace(item[:idx]) + if key == "" { + return nil, fmt.Errorf("invalid sglang arg %q, empty key", item) + } + out = append(out, &api.SGLangCustomizedArg{ + Key: key, + Value: item[idx+1:], + }) + } + return out, nil +} + +func newSGLangSpecFromArgs(preferredModel string, items []string) (*api.LLMSpecSGLang, error) { + customizedArgs, err := parseSGLangCustomizedArgs(items) + if err != nil { + return nil, err + } + if preferredModel == "" && len(customizedArgs) == 0 { + return nil, nil + } + return &api.LLMSpecSGLang{ + PreferredModel: preferredModel, + CustomizedArgs: customizedArgs, + }, nil +} + +func firstNonEmpty(items ...string) string { + for _, item := range items { + if item != "" { + return item + } + } + return "" +}