diff --git a/cmd/climc/shell/llm/llm_model.go b/cmd/climc/shell/llm/dify_sku.go similarity index 51% rename from cmd/climc/shell/llm/llm_model.go rename to cmd/climc/shell/llm/dify_sku.go index 6472c74c0e..9623ee666d 100644 --- a/cmd/climc/shell/llm/llm_model.go +++ b/cmd/climc/shell/llm/dify_sku.go @@ -8,13 +8,13 @@ import ( ) func init() { - cmd := shell.NewResourceCmd(&modules.LLMModel) - cmd.List(new(options.LLMModelListOptions)) - cmd.Show(new(options.LLMModelShowOptions)) - cmd.Update(new(options.LLMModelUpdateOptions)) - cmd.Create(new(options.LLMModelCreateOptions)) - cmd.Delete(new(options.LLMModelDeleteOptions)) + cmd := shell.NewResourceCmd(&modules.DifySku) + cmd.List(new(options.DifySkuListOptions)) + cmd.Show(new(options.DifySkuShowOptions)) + cmd.Update(new(options.DifySkuUpdateOptions)) + cmd.Create(new(options.DifySkuCreateOptions)) + cmd.Delete(new(options.DifySkuDeleteOptions)) cmd.Perform("public", &base_options.BasePublicOptions{}) cmd.Perform("private", &base_options.BaseIdOptions{}) - // cmd.Perform("clone", new(options.DesktopModelCloneOptions)) + // cmd.Perform("clone", new(options.DesktopSkuCloneOptions)) } diff --git a/cmd/climc/shell/llm/instantmodel.go b/cmd/climc/shell/llm/instantmodel.go new file mode 100644 index 0000000000..d505ea7e90 --- /dev/null +++ b/cmd/climc/shell/llm/instantmodel.go @@ -0,0 +1,25 @@ +package llm + +import ( + "yunion.io/x/onecloud/cmd/climc/shell" + + modules "yunion.io/x/onecloud/pkg/mcclient/modules/llm" + commonoptions "yunion.io/x/onecloud/pkg/mcclient/options" + options "yunion.io/x/onecloud/pkg/mcclient/options/llm" +) + +func init() { + cmd := shell.NewResourceCmd(&modules.LLMInstantModel) + cmd.List(new(options.LLMInstantModelListOptions)) + cmd.Show(new(options.LLMInstantModelShowOptions)) + // cmd.Update(new(options.InstantAppUpdateOptions)) + cmd.Create(new(options.LLMInstantModelCreateOptions)) + // cmd.Delete(new(options.InstantAppDeleteOptions)) + cmd.Perform("syncstatus", new(commonoptions.BaseIdOptions)) + cmd.Perform("change-owner", new(commonoptions.ChangeOwnerOptions)) + cmd.Perform("enable", new(commonoptions.BaseIdOptions)) + cmd.Perform("disable", new(commonoptions.BaseIdOptions)) + cmd.Perform("public", new(commonoptions.BasePublicOptions)) + cmd.Perform("private", new(commonoptions.BaseIdOptions)) + // cmd.PerformClass("import", new(options.InstantAppImportOptions)) +} diff --git a/cmd/climc/shell/llm/llm.go b/cmd/climc/shell/llm/llm.go index 6353bbcf11..71ac71efc9 100644 --- a/cmd/climc/shell/llm/llm.go +++ b/cmd/climc/shell/llm/llm.go @@ -13,6 +13,10 @@ func init() { cmd.Show(new(options.LLMShowOptions)) cmd.Delete(new(options.LLMDeleteOptions)) // cmd.Perform("change-model", new(options.LLMChangeModelOptions)) + cmd.Perform("syncstatus", new(options.LLMIdOptions)) cmd.BatchPerform("stop", new(options.LLMStopOptions)) cmd.BatchPerform("start", new(options.LLMStartOptions)) + cmd.Get("probed-models", new(options.LLMIdOptions)) + cmd.Perform("save-instant-model", new(options.LLMSaveInstantModelOptions)) + cmd.Perform("quick-models", new(options.LLMQuickModelsOptions)) } diff --git a/cmd/climc/shell/llm/dify_model.go b/cmd/climc/shell/llm/llm_sku.go similarity index 51% rename from cmd/climc/shell/llm/dify_model.go rename to cmd/climc/shell/llm/llm_sku.go index b37db81a9d..277bc15a30 100644 --- a/cmd/climc/shell/llm/dify_model.go +++ b/cmd/climc/shell/llm/llm_sku.go @@ -8,13 +8,13 @@ import ( ) func init() { - cmd := shell.NewResourceCmd(&modules.DifyModel) - cmd.List(new(options.DifyModelListOptions)) - cmd.Show(new(options.DifyModelShowOptions)) - cmd.Update(new(options.DifyModelUpdateOptions)) - cmd.Create(new(options.DifyModelCreateOptions)) - cmd.Delete(new(options.DifyModelDeleteOptions)) + cmd := shell.NewResourceCmd(&modules.LLMSku) + cmd.List(new(options.LLMSkuListOptions)) + cmd.Show(new(options.LLMSkuShowOptions)) + cmd.Update(new(options.LLMSkuUpdateOptions)) + cmd.Create(new(options.LLMSkuCreateOptions)) + cmd.Delete(new(options.LLMSkuDeleteOptions)) cmd.Perform("public", &base_options.BasePublicOptions{}) cmd.Perform("private", &base_options.BaseIdOptions{}) - // cmd.Perform("clone", new(options.DesktopModelCloneOptions)) + // cmd.Perform("clone", new(options.DesktopSkuCloneOptions)) } diff --git a/pkg/apis/llm/dify.go b/pkg/apis/llm/dify.go index b057581057..2d5fab3798 100644 --- a/pkg/apis/llm/dify.go +++ b/pkg/apis/llm/dify.go @@ -14,11 +14,11 @@ type DifyCustomized struct { type DifyListInput struct { LLMBaseListInput - DifyModel string `json:"dify_model"` + DifySku string `json:"dify_sku"` } type DifyCreateInput struct { LLMBaseCreateInput - DifyModelId string + DifySkuId string } diff --git a/pkg/apis/llm/instantmodel.go b/pkg/apis/llm/instantmodel.go new file mode 100644 index 0000000000..f9995fbb0c --- /dev/null +++ b/pkg/apis/llm/instantmodel.go @@ -0,0 +1,95 @@ +package llm + +import ( + "yunion.io/x/onecloud/pkg/apis" +) + +type InstantModelListInput struct { + apis.SharableVirtualResourceListInput + apis.EnabledResourceBaseListInput + + ModelName string `json:"model_name"` + Tag string `json:"tag"` + ModelId string `json:"model_id"` + Image string `json:"image"` + + Mounts string `json:"mounts"` + + AutoCache *bool `json:"auto_cache"` +} + +type InstantModelCreateInput struct { + apis.SharableVirtualResourceCreateInput + apis.EnabledBaseResourceCreateInput + + LLMType LLMContainerType `json:"llm_type"` + ModelName string `json:"model_name"` + Tag string `json:"tag"` + ImageId string `json:"image_id"` + Size int64 `json:"size"` + ModelId string `json:"model_id"` + + ActualSizeMb int32 `json:"actual_size_mb"` + + Mounts []string `json:"mounts"` +} + +type InstantModelUpdateInput struct { + apis.SharableVirtualResourceBaseUpdateInput + + ImageId string `json:"image_id"` + Size int64 `json:"size"` + + ActualSizeMb int32 `json:"actual_size_mb"` + + Mounts []string `json:"mounts"` +} + +type InstantModelDetails struct { + apis.SharableVirtualResourceDetails + + Image string + + CacheCount int + CachedCount int + + IconBase64 string `json:"icon_base64"` +} + +type InstantModelImportInput struct { + Endpoint string `json:"endpoint"` + AccessKey string `json:"access_key"` + SecretKey string `json:"secret_key"` + Bucket string `json:"bucket"` + Key string `json:"key"` + SignVer string `json:"sign_ver"` +} + +func (input InstantModelImportInput) Invalid() bool { + if len(input.Endpoint) == 0 || len(input.AccessKey) == 0 || len(input.SecretKey) == 0 || len(input.Bucket) == 0 || len(input.Key) == 0 { + return true + } + return false +} + +type InstantModelSyncstatusInput struct { +} + +type InstantAppCacheInput struct { +} + +type InstantModelEnableAutoCacheInput struct { + AutoCache bool `json:"auto_cache"` +} + +type MountedModelResourceListInput struct { + MountedModels []string `json:"mounted_models"` +} + +type MountedModelResourceCreateInput struct { + MountedModels []string `json:"mounted_models"` +} + +type MountedModelResourceUpdateInput struct { + MountedModels []string `json:"mounted_models"` +} diff --git a/pkg/apis/llm/llm.go b/pkg/apis/llm/llm.go index 90e41921cc..01eddd3ba2 100644 --- a/pkg/apis/llm/llm.go +++ b/pkg/apis/llm/llm.go @@ -1,6 +1,9 @@ package llm -import "yunion.io/x/onecloud/pkg/apis" +import ( + "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" +) const ( SERVICE_TYPE = "llm" @@ -19,7 +22,7 @@ type LLMBaseCreateInput struct { type LLMCreateInput struct { LLMBaseCreateInput - LLMModelId string + LLMSkuId string LLMImageId string } @@ -40,6 +43,73 @@ type LLMBaseListInput struct { type LLMListInput struct { LLMBaseListInput - LLMModel string `json:"llm_model"` + LLMSku string `json:"llm_sku"` LLMImage string `json:"llm_image"` } + +type ModelInfo struct { + // 秒装模型ID + Id string `json:"id"` + // 秒装模型 ModelId + ModelId string `json:"model_id"` + // 秒装模型展示的名称，如: Qwen-7B + DisplayName string `json:"display_name"` + // 秒装模型 tag，如: 7b + Tag string `json:"tag"` +} + +type LLMPerformQuickModelsInput struct { + Models []ModelInfo + Method TQuickModelMethod +} + +type LLMBatchPerformOutput struct { + Data []LLMPerformOutput + Task *taskman.STask +} + +type LLMPerformOutput struct { + Id string + Name string + RequestStatus int + Msg string + TaskId string +} + +type LLMSyncModelTaskInput struct { + LLMPerformQuickModelsInput + + LLMStatus string `json:"llm_status"` + InstallModelIds []string `json:"install_model_ids"` + InstallDirs []string `json:"install_dirs"` + UninstallModelIds []string `json:"uninstall_model_ids"` +} + +type LLMMountDirInfo struct { + ImageId string + Host string + Container string +} + +func (info LLMMountDirInfo) ToOverlay() apis.ContainerVolumeMountDiskPostOverlay { + // uid := int64(1000) + // gid := int64(1000) + if len(info.ImageId) > 0 { + return apis.ContainerVolumeMountDiskPostOverlay{ + Image: &apis.ContainerVolumeMountDiskPostImageOverlay{ + Id: info.ImageId, + }, + // FsUser: &uid, + // FsGroup: &gid, + } + } + return apis.ContainerVolumeMountDiskPostOverlay{ + ContainerTargetDir: info.Container, + HostLowerDir: []string{info.Host}, + // FsUser: &uid, + // FsGroup: &gid, + } +} + +type LLMSyncStatusInput struct { +} diff --git a/pkg/apis/llm/llm_const.go b/pkg/apis/llm/llm_const.go index 87ec78f4d4..48f68d3870 100644 --- a/pkg/apis/llm/llm_const.go +++ b/pkg/apis/llm/llm_const.go @@ -16,11 +16,26 @@ const ( /* 停机失败 */ LLM_STATUS_STOP_FAILED = "stop_fail" + /* 开始保存应用 */ + LLM_STATUS_START_SAVE_MODEL = "start_save_model" + /* 正在保存应用 */ + LLM_STATUS_SAVING_MODEL = "saving_model" + /* 保存应用失败 */ + LLM_STATUS_SAVE_MODEL_FAILED = "save_model_failed" + + /* 开始同步状态 */ + LLM_STATUS_START_SYNCSTATUS = "start_syncstatus" + /* 正在同步状态 */ + LLM_STATUS_SYNCSTATUS = "syncstatus" + /* 停机 */ LLM_STATUS_READY = "ready" /* 运行 */ LLM_STATUS_RUNNING = "running" + /* 删除 */ + LLM_STATUS_DELETED = "deleted" + LLM_STATUS_CREATING_POD = "creating_pod" LLM_STATUS_CREAT_POD_FAILED = "creat_pod_failed" LLM_STATUS_PULLING_MODEL = "pulling_model" @@ -37,3 +52,15 @@ const ( LLM_STATUS_DELETING = "deleting" LLM_STATUS_DELETE_FAILED = "delete_fail" ) + +type TQuickModelMethod string + +const ( + QuickModelInstall = TQuickModelMethod("install") + QuickModelUninstall = TQuickModelMethod("uninstall") + QuickModelReinstall = TQuickModelMethod("reinstall") +) + +const ( + LLM_PROBE_INSTANT_MODEl_INTERVAL_SECOND = 120 // 2 minute +) diff --git a/pkg/apis/llm/llm_instant_model.go b/pkg/apis/llm/llm_instant_model.go new file mode 100644 index 0000000000..447658a91c --- /dev/null +++ b/pkg/apis/llm/llm_instant_model.go @@ -0,0 +1,23 @@ +package llm + +import "yunion.io/x/onecloud/pkg/apis" + +type LLMInternalInstantMdlInfo struct { + ModelId string `json:"model_id"` + Name string `json:"name"` + Tag string `json:"tag"` + Size int64 `json:"size"` + // Modified string `json:"modified"` + Blobs []string `json:"blobs"` +} + +type LLMSaveInstantModelInput struct { + apis.ProjectizedResourceCreateInput + + ModelId string `json:"model_id"` + ImageName string `json:"image_name"` + + InstantModelId string `json:"instant_model_id"` + + // AutoRestart bool `json:"auto_restart"` +} diff --git a/pkg/apis/llm/ollama_const.go b/pkg/apis/llm/ollama_const.go index e72135631f..ba9af4a1db 100644 --- a/pkg/apis/llm/ollama_const.go +++ b/pkg/apis/llm/ollama_const.go @@ -11,6 +11,7 @@ const ( ) const ( + LLM_OLLAMA_SAVE_DIR = "/opt/.ollama-models/%s" LLM_OLLAMA_HOST_PATH = "/opt/ollama-models" LLM_OLLAMA_HOST_MANIFESTS_DIR = "/manifests" LLM_OLLAMA_CACHE_DIR = "/.llm_ollama_cache" diff --git a/pkg/apis/llm/model.go b/pkg/apis/llm/sku.go similarity index 90% rename from pkg/apis/llm/model.go rename to pkg/apis/llm/sku.go index 82123c2999..b836220892 100644 --- a/pkg/apis/llm/model.go +++ b/pkg/apis/llm/sku.go @@ -12,14 +12,14 @@ import ( ) var ( - LLM_MODEL_BASE_NETWORK_TYPES = sets.NewString( + LLM_SKU_BASE_NETWORK_TYPES = sets.NewString( string(computeapi.NETWORK_TYPE_HOSTLOCAL), string(computeapi.NETWORK_TYPE_GUEST), ) ) -func IsLLMModelBaseNetworkType(t string) bool { - return LLM_MODEL_BASE_NETWORK_TYPES.Has(t) +func IsLLMSkuBaseNetworkType(t string) bool { + return LLM_SKU_BASE_NETWORK_TYPES.Has(t) } type HostInfo struct { @@ -108,7 +108,7 @@ func (s Envs) IsZero() bool { return len(s) == 0 } -type LLMModelDetails struct { +type LLMSkuDetails struct { apis.SharableVirtualResourceDetails // 当前大模型套餐包含的实例个数。 LLMCapacity int @@ -123,7 +123,7 @@ type LLMModelDetails struct { // MountedApps []string `json:"mounted_apps"` // } -type LLMModelBaseCreateInput struct { +type LLMSKuBaseCreateInput struct { apis.SharableVirtualResourceCreateInput Cpu int `json:"cpu"` @@ -140,7 +140,7 @@ type LLMModelBaseCreateInput struct { Properties map[string]string `json:"properties"` } -type LLMModelBaseUpdateInput struct { +type LLMSkuBaseUpdateInput struct { apis.SharableVirtualResourceBaseUpdateInput // MountedAppResourceUpdateInput @@ -163,23 +163,23 @@ type LLMModelBaseUpdateInput struct { Properties map[string]string `json:"properties"` } -type LLMModelListInput struct { +type LLMSkuListInput struct { apis.SharableVirtualResourceListInput // MountedAppResourceListInput LLMType string `json:"llm_type"` } -type LLMModelCreateInput struct { - LLMModelBaseCreateInput +type LLMSkuCreateInput struct { + LLMSKuBaseCreateInput LLMImageId string `json:"llm_image_id"` LLMType string `json:"llm_type"` LLMModelName string `json:"llm_model_name"` } -type LLMModelUpdateInput struct { - LLMModelBaseUpdateInput +type LLMSkuUpdateInput struct { + LLMSkuBaseUpdateInput LLMImageId string `json:"llm_image_id"` LLMModelName string `json:"llm_model_name"` @@ -193,13 +193,13 @@ type LLMModelUpdateInput struct { // Request bool `json:"request"` // } -type DifyModelListInput struct { +type DifySkulListInput struct { apis.SharableVirtualResourceListInput // MountedAppResourceListInput } -type DifyModelCreateInput struct { - LLMModelBaseCreateInput +type DifySkuCreateInput struct { + LLMSKuBaseCreateInput PostgresImageId string `json:"postgres_image_id"` RedisImageId string `json:"redis_image_id"` @@ -212,8 +212,8 @@ type DifyModelCreateInput struct { DifyWeaviateImageId string `json:"dify_weaviate_image_id"` } -type DifyModelUpdateInput struct { - LLMModelBaseUpdateInput +type DifySkuUpdateInput struct { + LLMSkuBaseUpdateInput PostgresImageId string `json:"postgres_image_id"` RedisImageId string `json:"redis_image_id"` diff --git a/pkg/llm/drivers/llm_container/ollama.go b/pkg/llm/drivers/llm_container/ollama.go index 63d87f8c0c..eb736d16b9 100644 --- a/pkg/llm/drivers/llm_container/ollama.go +++ b/pkg/llm/drivers/llm_container/ollama.go @@ -2,19 +2,21 @@ package llm_container import ( "context" + "encoding/json" "fmt" "path" - "regexp" "strings" "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/apis" + commonapi "yunion.io/x/onecloud/pkg/apis" computeapi "yunion.io/x/onecloud/pkg/apis/compute" api "yunion.io/x/onecloud/pkg/apis/llm" "yunion.io/x/onecloud/pkg/llm/models" - "yunion.io/x/onecloud/pkg/llm/utils" + llmutil "yunion.io/x/onecloud/pkg/llm/utils" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -33,7 +35,7 @@ func (o *ollama) GetType() api.LLMContainerType { return api.LLM_CONTAINER_OLLAMA } -func (o *ollama) GetContainerSpec(ctx context.Context, llm *models.SLLM, image *models.SLLMImage, sku *models.SLLMModel, props []string, devices []computeapi.SIsolatedDevice, diskId string) *computeapi.PodContainerCreateInput { +func (o *ollama) GetContainerSpec(ctx context.Context, llm *models.SLLM, image *models.SLLMImage, sku *models.SLLMSku, props []string, devices []computeapi.SIsolatedDevice, diskId string) *computeapi.PodContainerCreateInput { spec := computeapi.ContainerSpec{ ContainerSpec: apis.ContainerSpec{ Image: image.ToContainerImage(), @@ -94,27 +96,20 @@ func (o *ollama) GetContainerSpec(ctx context.Context, llm *models.SLLM, image * // vols = append(spec.VolumeMounts, GetDiskVolumeMounts(sku.Volumes, appVolIndex, postOverlays)...) // udevPath := filepath.Join(GetTmpSocketsHostPath(d.GetName()), "udev") - modelName, modelTag, _ := llm.GetLargeLanguageModelName() + diskIndex := 0 ctrVols := []*apis.ContainerVolumeMount{ { - UniqueName: "manifests", - Type: apis.CONTAINER_VOLUME_MOUNT_TYPE_HOST_PATH, - MountPath: getManifestsPath(modelName, modelTag), - HostPath: &apis.ContainerVolumeMountHostPath{ - Type: apis.CONTAINER_VOLUME_MOUNT_HOST_PATH_TYPE_FILE, - Path: path.Join(api.LLM_OLLAMA_HOST_PATH, api.LLM_OLLAMA_HOST_MANIFESTS_DIR, modelName+"-"+modelTag), + Disk: &apis.ContainerVolumeMountDisk{ + SubDirectory: api.LLM_OLLAMA, + Overlay: &apis.ContainerVolumeMountDiskOverlay{ + LowerDir: []string{api.LLM_OLLAMA_HOST_PATH}, + }, + Index: &diskIndex, }, - ReadOnly: true, - }, - { - UniqueName: "blobs", - Type: apis.CONTAINER_VOLUME_MOUNT_TYPE_HOST_PATH, - MountPath: path.Join(api.LLM_OLLAMA_BASE_PATH, api.LLM_OLLAMA_BLOBS_DIR), - HostPath: &apis.ContainerVolumeMountHostPath{ - Type: apis.CONTAINER_VOLUME_MOUNT_HOST_PATH_TYPE_DIRECTORY, - Path: path.Join(api.LLM_OLLAMA_HOST_PATH, api.LLM_OLLAMA_BLOBS_DIR), - }, - ReadOnly: true, + Type: apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK, + MountPath: api.LLM_OLLAMA_BASE_PATH, + ReadOnly: false, + Propagation: apis.MOUNTPROPAGATION_PROPAGATION_HOST_TO_CONTAINER, }, } vols = append(vols, ctrVols...) @@ -160,29 +155,211 @@ func (o *ollama) GetContainerSpec(ctx context.Context, llm *models.SLLM, image * // return err // } -func (o *ollama) CopyBlobs(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM) error { - ctr, _ := llm.GetLLMContainer() - modelName, modelTag, _ := llm.GetLargeLanguageModelName() - blobs, _ := fetchBlobs(ctx, ctr.CmpId, modelName, modelTag) - - blobsTargetDir := path.Join(api.LLM_OLLAMA_BASE_PATH, api.LLM_OLLAMA_BLOBS_DIR) - blobsSrcDir := path.Join(api.LLM_OLLAMA_CACHE_MOUNT_PATH, api.LLM_OLLAMA_CACHE_DIR) - - var commands []string - commands = append(commands, fmt.Sprintf("mkdir -p %s", blobsTargetDir)) - for _, blob := range blobs { - src := path.Join(blobsSrcDir, blob) - target := path.Join(blobsTargetDir, blob) - commands = append(commands, fmt.Sprintf("cp %s %s", src, target)) +func (o *ollama) PreInstallModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, instMdl *models.SLLMInstantModel) error { + // before mount, make sure the manifests dir is ready + lc, err := llm.GetLLMContainer() + if err != nil { + return errors.Wrap(err, "get llm container") } - cmd := strings.Join(commands, " && ") - if _, err := exec(ctx, ctr.CmpId, "/bin/sh", "-c", cmd); err != nil { - return errors.Wrapf(err, "failed to copy blobs to container") + // mkdir llm-registry-base-path / modelname + mkdirReigtryBasePaht := fmt.Sprintf("mkdir -p %s", path.Join(api.LLM_OLLAMA_BASE_PATH, api.LLM_OLLAMA_MANIFESTS_BASE_PATH, instMdl.ModelName)) + _, err = exec(ctx, lc.CmpId, mkdirReigtryBasePaht, 10) + if err != nil { + return errors.Wrap(err, "failed to mkdir llm-registry-base-path / modelname") } + return nil } +func (o *ollama) InstallModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, dirs []string, mdlIds []string) error { + ///* TODO + return nil +} + +func (o *ollama) GetModelMountPaths(ctx context.Context, userCred mcclient.TokenCredential, llmInstMdl *models.SLLMInstantModel) ([]string, error) { + instMdl, _ := llmInstMdl.FindInstantModel(false) + return instMdl.Mounts, nil +} + +func (o *ollama) GetDirPostOverlay(dir api.LLMMountDirInfo) *commonapi.ContainerVolumeMountDiskPostOverlay { + uid := int64(1000) + gid := int64(1000) + ov := dir.ToOverlay() + ov.FsUser = &uid + ov.FsGroup = &gid + return &ov +} + +func (o *ollama) UninstallModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, llmInstMdl *models.SLLMInstantModel) error { + // don't rm file + + // mounts, err := o.GetModelMountPaths(ctx, userCred, llmInstMdl) + // if err != nil { + // return errors.Wrap(err, "GetModelMountPaths") + // } + // ctr, err := llm.GetLLMSContainer(ctx) + // if err != nil { + // return errors.Wrap(err, "GetSContainer") + // } + // _, err = exec(ctx, ctr.Id, fmt.Sprintf("rm -rf %s", strings.Join(mounts, " ")), 10) + // if err != nil { + // return errors.Wrapf(err, "run cmd to remove model mounts of model %s", jsonutils.Marshal(llmInstMdl)) + // } + return nil +} + +func (o *ollama) GetInstantModelIdByPostOverlay(postOverlay *commonapi.ContainerVolumeMountDiskPostOverlay, mdlNameToId map[string]string) string { + if postOverlay.Image != nil { + for k := range postOverlay.Image.PathMap { + idx := strings.Index(k, api.LLM_OLLAMA_MANIFESTS_BASE_PATH) + if idx != -1 { + suffix := k[idx+len(api.LLM_OLLAMA_MANIFESTS_BASE_PATH):] + parts := strings.Split(strings.Trim(suffix, "/"), "/") + if len(parts) >= 2 { + modelName := parts[len(parts)-2] + modelTag := parts[len(parts)-1] + log.Infof("In GetInstantModelIdByPostOverlay, Extracted modelName: %s, modelTag: %s, Got modelId: %s", modelName, modelTag, mdlNameToId[modelName+":"+modelTag]) + return mdlNameToId[modelName+":"+modelTag] + } + } + } + } + return "" +} + +func (o *ollama) DetectModelPaths(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, pkgInfo api.LLMInternalInstantMdlInfo) ([]string, error) { + lc, err := llm.GetLLMContainer() + if err != nil { + return nil, errors.Wrap(err, "get llm container") + } + // check file exists + originBlobs := make([]string, len(pkgInfo.Blobs)) + for idx, blob := range pkgInfo.Blobs { + originBlobs[idx] = path.Join(api.LLM_OLLAMA_BASE_PATH, api.LLM_OLLAMA_BLOBS_DIR, blob) + } + originManifests := path.Join(api.LLM_OLLAMA_BASE_PATH, api.LLM_OLLAMA_MANIFESTS_BASE_PATH, pkgInfo.Name, pkgInfo.Tag) + + var checks []string + for _, blob := range originBlobs { + checks = append(checks, fmt.Sprintf("[ -f '%s' ]", blob)) + } + checks = append(checks, fmt.Sprintf("[ -f '%s' ]", originManifests)) + + checkCmd := strings.Join(checks, " && ") + " && echo 'ALL_EXIST' || echo 'SOME_MISSING'" + output, err := exec(ctx, lc.CmpId, checkCmd, 10) + if err != nil { + return nil, errors.Wrap(err, "failed to check file existence") + } + + if !strings.Contains(output, "ALL_EXIST") { + log.Infof("Some files are missing for model %s:%s, blobs: %v, manifest: %s, checkCmd: %s", + pkgInfo.Name, pkgInfo.Tag, originBlobs, originManifests, checkCmd) + return nil, errors.Errorf("required model files are missing") + } + + // // mkdir + // savePath := fmt.Sprintf(api.LLM_OLLAMA_SAVE_DIR, pkgInfo.Name+"-"+pkgInfo.Tag+"-"+pkgInfo.ModelId) + // mkSaveDir := fmt.Sprintf("mkdir -p %s %s %s", savePath, path.Join(savePath, api.LLM_OLLAMA_BLOBS_DIR), path.Join(savePath, api.LLM_OLLAMA_HOST_MANIFESTS_DIR)) + // _, err = exec(ctx, ctr.CmpId, mkSaveDir, 10) + // if err != nil { + // return a, filtenil, errors.Wrap(err, "mkdir savedir") + // } + + // // cp file + // for _, blob := range originBlobs { + // cpBlob := fmt.Sprintf("cp %s %s", blob, path.Join(savePath, api.LLM_OLLAMA_BLOBS_DIR)) + // _, err = exec(ctx, ctr.CmpId, cpBlob, 60) + // if err != nil { + // return nil, errors.Wrap(err, "copy files") + // } + // } + // cpManifest := "cp " + originManifests + " " + path.Join(savePath, api.LLM_OLLAMA_HOST_MANIFESTS_DIR) + // _, err = exec(ctx, ctr.CmpId, cpManifest, 20) + // if err != nil { + // return nil, errors.Wrap(err, "copy files") + // } + + return append(originBlobs, originManifests), nil +} + +func (o *ollama) GetImageInternalPathMounts(sMdl *models.SInstantModel) map[string]string { + imageToContainer := make(map[string]string) + + for _, mount := range sMdl.Mounts { + imgPath := strings.TrimPrefix(mount, api.LLM_OLLAMA_BASE_PATH) + imageToContainer[imgPath] = path.Join(api.LLM_OLLAMA, imgPath) + } + + return imageToContainer +} + +func (o *ollama) GetSaveDirectories(sApp *models.SInstantModel) (string, []string, error) { + var filteredMounts []string + + for _, mount := range sApp.Mounts { + if strings.HasPrefix(mount, api.LLM_OLLAMA_BASE_PATH) { + relPath := strings.TrimPrefix(mount, api.LLM_OLLAMA_BASE_PATH) + filteredMounts = append(filteredMounts, relPath) + } + } + + return "", filteredMounts, nil +} + +func (o *ollama) GetProbedInstantModelsExt(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, mdlIds ...string) (map[string]api.LLMInternalInstantMdlInfo, error) { + lc, err := llm.GetLLMContainer() + if err != nil { + return nil, errors.Wrap(err, "get llm container") + } + + // get all effective models + getModels := "ollama list" // NAME ID SIZE MODIFIED + modelsOutput, err := exec(ctx, lc.CmpId, getModels, 10) + if err != nil { + return nil, errors.Wrap(err, "get models") + } + lines := strings.Split(strings.TrimSpace(modelsOutput), "\n") + + models := make(map[string]api.LLMInternalInstantMdlInfo, len(lines)-1) + for i := 1; i < len(lines); i++ { + fields := strings.Fields(lines[i]) + if len(fields) > 2 { + if len(mdlIds) > 0 && !utils.IsInStringArray(fields[1], mdlIds) { + continue + } + modelName, modelTag, _ := llm.GetLargeLanguageModelName(fields[0]) + models[fields[1]] = api.LLMInternalInstantMdlInfo{ + Name: modelName, + Tag: modelTag, + ModelId: fields[1], + // Modified: fields[3], + } + } + } + + // for each model, get manifests file, find blobs, calculate size + for modelId, model := range models { + manifests, err := getManifests(ctx, lc.CmpId, model.Name, model.Tag) + if err != nil { + return nil, errors.Wrap(err, "get manifests") + } + model.Size = manifests.Config.Size + model.Blobs = append(model.Blobs, manifests.Config.Digest) + for _, layer := range manifests.Layers { + model.Size += layer.Size + model.Blobs = append(model.Blobs, layer.Digest) + } + models[modelId] = model + } + + return models, nil +} + +func (o *ollama) ValidateMounts(mounts []string, mdlName string, mdlTag string) ([]string, error) { + return mounts, nil +} + // func download(ctx context.Context, userCred mcclient.TokenCredential, containerId string, taskId string, webUrl string, path string) error { // input := &computeapi.ContainerDownloadFileInput{ // WebUrl: webUrl, @@ -193,13 +370,39 @@ func (o *ollama) CopyBlobs(ctx context.Context, userCred mcclient.TokenCredentia // return err // } -func exec(ctx context.Context, containerId string, command ...string) (string, error) { - // exec command +// func (o *ollama) CopyBlobs(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM) error { +// ctr, _ := llm.GetLLMContainer() +// modelName, modelTag, _ := llm.GetLargeLanguageModelName("") +// manifests, _ := getManifests(ctx, ctr.CmpId, modelName, modelTag) +// blobsTargetDir := path.Join(api.LLM_OLLAMA_BASE_PATH, api.LLM_OLLAMA_BLOBS_DIR) +// blobsSrcDir := path.Join(api.LLM_OLLAMA_CACHE_MOUNT_PATH, api.LLM_OLLAMA_CACHE_DIR) + +// var commands []string +// commands = append(commands, fmt.Sprintf("mkdir -p %s", blobsTargetDir)) +// blob := manifests.Config.Digest +// commands = append(commands, fmt.Sprintf("cp %s %s", path.Join(blobsSrcDir, blob), path.Join(blobsTargetDir, blob))) +// for _, layer := range manifests.Layers { +// blob = layer.Digest +// src := path.Join(blobsSrcDir, blob) +// target := path.Join(blobsTargetDir, blob) +// commands = append(commands, fmt.Sprintf("cp %s %s", src, target)) +// } + +// cmd := strings.Join(commands, " && ") +// if _, err := exec(ctx, ctr.CmpId, cmd, 120); err != nil { +// return errors.Wrapf(err, "failed to copy blobs to container") +// } +// return nil +// } + +func exec(ctx context.Context, containerId string, cmd string, timeoutSec int64) (string, error) { + // exec command input := &computeapi.ContainerExecSyncInput{ - Command: command, + Command: []string{"sh", "-c", cmd}, + Timeout: timeoutSec, } - resp, err := utils.ExecSyncContainer(ctx, containerId, input) + resp, err := llmutil.ExecSyncContainer(ctx, containerId, input) // check error and return result var rst string @@ -215,23 +418,36 @@ func getManifestsPath(modelName, modelTag string) string { return path.Join(api.LLM_OLLAMA_BASE_PATH, api.LLM_OLLAMA_MANIFESTS_BASE_PATH, modelName, modelTag) } -func fetchBlobs(ctx context.Context, containerId string, modelName string, modelTag string) ([]string, error) { - manifestContent, err := exec(ctx, containerId, "cat", getManifestsPath(modelName, modelTag)) +type Layer struct { + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + Size int64 `json:"size"` +} + +type Manifest struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Config Layer `json:"config"` + Layers []Layer `json:"layers"` +} + +func getManifests(ctx context.Context, containerId string, modelName string, modelTag string) (*Manifest, error) { + manifestContent, err := exec(ctx, containerId, "cat "+getManifestsPath(modelName, modelTag), 10) if err != nil { return nil, errors.Wrapf(err, "failed to read manifests from container") } // find all blobs - var results []string - re := regexp.MustCompile(`"digest":"(sha256:[^"]*)"`) - matches := re.FindAllStringSubmatch(manifestContent, -1) - for _, match := range matches { - if len(match) > 1 { - digest := match[1] - processedDigest := strings.Replace(digest, "sha256:", "sha256-", 1) - results = append(results, processedDigest) - } + manifests := &Manifest{} + if err = json.Unmarshal([]byte(manifestContent), manifests); err != nil { + return nil, errors.Wrapf(err, "failed to parse manifests") + } + // log.Infof("manifests: %v", manifests) + + manifests.Config.Digest = strings.Replace(manifests.Config.Digest, ":", "-", 1) + for idx, layer := range manifests.Layers { + manifests.Layers[idx].Digest = strings.Replace(layer.Digest, ":", "-", 1) } - return results, nil + return manifests, nil } diff --git a/pkg/llm/models/dify.go b/pkg/llm/models/dify.go index 6743595672..3cd90effb9 100644 --- a/pkg/llm/models/dify.go +++ b/pkg/llm/models/dify.go @@ -49,7 +49,7 @@ type SDifyManager struct { type SDify struct { SLLMBase - DifyModelId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` + DifySkuId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` } func (dm *SDifyManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.DifyCreateInput) (*api.DifyCreateInput, error) { @@ -58,12 +58,12 @@ func (dm *SDifyManager) ValidateCreateData(ctx context.Context, userCred mcclien if err != nil { return input, errors.Wrap(err, "validate VirtualResourceCreateInput") } - model, err := GetDifyModelManager().FetchByIdOrName(ctx, userCred, input.DifyModelId) + sku, err := GetDifySkuManager().FetchByIdOrName(ctx, userCred, input.DifySkuId) if err != nil { - return input, errors.Wrap(err, "fetch DifyModel") + return input, errors.Wrap(err, "fetch DifySku") } - dModel := model.(*SDifyModel) - input.DifyModelId = dModel.Id + dSku := sku.(*SDifySku) + input.DifySkuId = dSku.Id return input, nil } @@ -92,16 +92,16 @@ func (dm *SDifyManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, if err != nil { return q, errors.Wrap(err, "VirtualResourceBaseManager.ListItemFilter") } - if len(input.DifyModel) > 0 { - modelObj, err := GetDifyModelManager().FetchByIdOrName(ctx, userCred, input.DifyModel) + if len(input.DifySku) > 0 { + skuObj, err := GetDifySkuManager().FetchByIdOrName(ctx, userCred, input.DifySku) if err != nil { if errors.Cause(err) == sql.ErrNoRows { - return nil, httperrors.NewResourceNotFoundError2(GetDifyModelManager().KeywordPlural(), input.DifyModel) + return nil, httperrors.NewResourceNotFoundError2(GetDifySkuManager().KeywordPlural(), input.DifySku) } else { - return nil, errors.Wrap(err, "DifyModelManager.FetchByIdOrName") + return nil, errors.Wrap(err, "GetDifySkuManager.FetchByIdOrName") } } - q = q.Equals("dify_model_id", modelObj.GetId()) + q = q.Equals("dify_sku_id", skuObj.GetId()) } return q, nil } @@ -110,15 +110,15 @@ func (dify *SDify) CustomizeDelete(ctx context.Context, userCred mcclient.TokenC return dify.StartDeleteTask(ctx, userCred, "") } -func (dify *SDify) GetDifyModel(modelId string) (*SDifyModel, error) { - if len(modelId) == 0 { - modelId = dify.DifyModelId +func (dify *SDify) GetDifySku(skuId string) (*SDifySku, error) { + if len(skuId) == 0 { + skuId = dify.DifySkuId } - model, err := GetDifyModelManager().FetchById(modelId) + sku, err := GetDifySkuManager().FetchById(skuId) if err != nil { - return nil, errors.Wrap(err, "fetch DifyModel") + return nil, errors.Wrap(err, "fetch DifySku") } - return model.(*SDifyModel), nil + return sku.(*SDifySku), nil } func (dify *SDify) GetDifyContainers() []*computeapi.PodContainerCreateInput { @@ -172,12 +172,12 @@ func (dify *SDify) StartDeleteTask(ctx context.Context, userCred mcclient.TokenC } func (dify *SDify) ServerCreate(ctx context.Context, userCred mcclient.TokenCredential, s *mcclient.ClientSession, input *api.DifyCreateInput) (string, error) { - model, err := dify.GetDifyModel(dify.DifyModelId) + sku, err := dify.GetDifySku(dify.DifySkuId) if nil != err { - return "", errors.Wrap(err, "GetDifyModel") + return "", errors.Wrap(err, "GetDifySku") } - data, err := GetDifyPodCreateInput(ctx, userCred, input, dify, model, "") + data, err := GetDifyPodCreateInput(ctx, userCred, input, dify, sku, "") if nil != err { return "", errors.Wrap(err, "GetDifyPodCreateInput") } @@ -207,11 +207,11 @@ func (dify *SDify) ServerCreate(ctx context.Context, userCred mcclient.TokenCred // } func (dify *SDify) getDifyContainerByContainerKey(containerKey string) (*computeapi.PodContainerCreateInput, error) { - model, err := dify.GetDifyModel("") + sku, err := dify.GetDifySku("") if nil != err { return nil, err } - container, err := getDifyContainersManager().GetContainer(dify.GetName(), containerKey, model) + container, err := getDifyContainersManager().GetContainer(dify.GetName(), containerKey, sku) if nil != err { return nil, err } diff --git a/pkg/llm/models/dify_containers.go b/pkg/llm/models/dify_containers.go index 717dd9e4ba..603a879385 100644 --- a/pkg/llm/models/dify_containers.go +++ b/pkg/llm/models/dify_containers.go @@ -77,7 +77,7 @@ func _getRegistryImage(imageId string) string { return image.(*SLLMImage).ToContainerImage() } -func (m *DifyContainersManager) GetContainer(name, containerKey string, sku *SDifyModel) (*computeapi.PodContainerCreateInput, error) { +func (m *DifyContainersManager) GetContainer(name, containerKey string, sku *SDifySku) (*computeapi.PodContainerCreateInput, error) { switch containerKey { case api.DIFY_REDIS_KEY: return m._getRedisContainer(name, containerKey, _getRegistryImage(sku.RedisImageId)), nil diff --git a/pkg/llm/models/dify_pod.go b/pkg/llm/models/dify_pod.go index 2970719e38..cda5cb0a1c 100644 --- a/pkg/llm/models/dify_pod.go +++ b/pkg/llm/models/dify_pod.go @@ -15,10 +15,10 @@ func GetDifyPodCreateInput( userCred mcclient.TokenCredential, input *api.DifyCreateInput, dify *SDify, - sku *SDifyModel, + sku *SDifySku, eip string, ) (*computeapi.ServerCreateInput, error) { - data, err := GetLLMBasePodCreateInput(ctx, userCred, &input.LLMBaseCreateInput, &dify.SLLMBase, &sku.SLLMModelBase, eip) + data, err := GetLLMBasePodCreateInput(ctx, userCred, &input.LLMBaseCreateInput, &dify.SLLMBase, &sku.SLLMSkuBase, eip) if err != nil { return nil, errors.Wrap(err, "GetLLMBasePodCreateInput: ") } diff --git a/pkg/llm/models/dify_model.go b/pkg/llm/models/dify_sku.go similarity index 59% rename from pkg/llm/models/dify_model.go rename to pkg/llm/models/dify_sku.go index baaad9b740..c45416b8e1 100644 --- a/pkg/llm/models/dify_model.go +++ b/pkg/llm/models/dify_sku.go @@ -13,33 +13,33 @@ import ( ) func init() { - GetDifyModelManager() + GetDifySkuManager() } -var difyModelManager *SDifyModelManager +var difySkuManager *SDifySkuManager -func GetDifyModelManager() *SDifyModelManager { - if difyModelManager != nil { - return difyModelManager +func GetDifySkuManager() *SDifySkuManager { + if difySkuManager != nil { + return difySkuManager } - difyModelManager = &SDifyModelManager{ - SLLMModelBaseManager: NewSLLMModelBaseManager( - SDifyModel{}, - "dify_models_tbl", - "dify_model", - "dify_models", + difySkuManager = &SDifySkuManager{ + SLLMSkuBaseManager: NewSLLMSkuBaseManager( + SDifySku{}, + "dify_skus_tbl", + "dify_sku", + "dify_skus", ), } - difyModelManager.SetVirtualObject(difyModelManager) - return difyModelManager + difySkuManager.SetVirtualObject(difySkuManager) + return difySkuManager } -type SDifyModelManager struct { - SLLMModelBaseManager +type SDifySkuManager struct { + SLLMSkuBaseManager } -type SDifyModel struct { - SLLMModelBase +type SDifySku struct { + SLLMSkuBase PostgresImageId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` RedisImageId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` @@ -52,16 +52,16 @@ type SDifyModel struct { DifyWeaviateImageId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` } -func (man *SDifyModelManager) ListItemFilter( +func (man *SDifySkuManager) ListItemFilter( ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, - input api.DifyModelListInput, + input api.DifySkulListInput, ) (*sqlchemy.SQuery, error) { var err error - q, err = man.SLLMModelBaseManager.ListItemFilter(ctx, q, userCred, input.SharableVirtualResourceListInput) + q, err = man.SLLMSkuBaseManager.ListItemFilter(ctx, q, userCred, input.SharableVirtualResourceListInput) if err != nil { - return nil, errors.Wrapf(err, "SLLMModelBaseManager.ListItemFilter") + return nil, errors.Wrapf(err, "SLLMSkuBaseManager.ListItemFilter") } return q, nil } @@ -77,11 +77,11 @@ func (man *SDifyModelManager) ListItemFilter( // } -func (man *SDifyModelManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.DifyModelCreateInput) (*api.DifyModelCreateInput, error) { +func (man *SDifySkuManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.DifySkuCreateInput) (*api.DifySkuCreateInput, error) { var err error - input.LLMModelBaseCreateInput, err = man.SLLMModelBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.LLMModelBaseCreateInput) + input.LLMSKuBaseCreateInput, err = man.SLLMSkuBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.LLMSKuBaseCreateInput) if err != nil { - return nil, errors.Wrap(err, "SLLMModelBaseManager.ValidateCreateData") + return nil, errors.Wrap(err, "SLLMSkuBaseManager.ValidateCreateData") } for _, imgId := range []*string{&input.PostgresImageId, &input.RedisImageId, &input.NginxImageId, &input.DifyApiImageId, &input.DifyPluginImageId, &input.DifyWebImageId, &input.DifySandboxImageId, &input.DifySSRFImageId, &input.DifyWeaviateImageId} { @@ -94,11 +94,11 @@ func (man *SDifyModelManager) ValidateCreateData(ctx context.Context, userCred m return input, nil } -func (model *SDifyModel) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.DifyModelUpdateInput) (api.DifyModelUpdateInput, error) { +func (sku *SDifySku) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.DifySkuUpdateInput) (api.DifySkuUpdateInput, error) { var err error - input.LLMModelBaseUpdateInput, err = model.SLLMModelBase.ValidateUpdateData(ctx, userCred, query, input.LLMModelBaseUpdateInput) + input.LLMSkuBaseUpdateInput, err = sku.SLLMSkuBase.ValidateUpdateData(ctx, userCred, query, input.LLMSkuBaseUpdateInput) if err != nil { - return input, errors.Wrap(err, "validate LLMModelBaseUpdateInput") + return input, errors.Wrap(err, "validate LLMSkuBaseUpdateInput") } for _, imgId := range []*string{&input.PostgresImageId, &input.RedisImageId, &input.NginxImageId, &input.DifyApiImageId, &input.DifyPluginImageId, &input.DifyWebImageId, &input.DifySandboxImageId, &input.DifySSRFImageId, &input.DifyWeaviateImageId} { @@ -113,13 +113,13 @@ func (model *SDifyModel) ValidateUpdateData(ctx context.Context, userCred mcclie return input, nil } -func (model *SDifyModel) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error { - count, err := GetDifyManager().Query().Equals("dify_model_id", model.Id).CountWithError() +func (sku *SDifySku) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error { + count, err := GetDifyManager().Query().Equals("dify_sku_id", sku.Id).CountWithError() if nil != err { return errors.Wrap(err, "fetch dify") } if count > 0 { - return errors.Wrap(errors.ErrNotSupported, "This model is currently in use") + return errors.Wrap(errors.ErrNotSupported, "This sku is currently in use") } return nil } diff --git a/pkg/llm/models/image.go b/pkg/llm/models/image.go index 5920414f95..d883789df7 100644 --- a/pkg/llm/models/image.go +++ b/pkg/llm/models/image.go @@ -137,7 +137,7 @@ func (image *SLLMImage) ValidateDeleteCondition(ctx context.Context, info jsonut if count > 0 { return errors.Wrapf(errors.ErrNotSupported, "This image is currently in use by %s in llms", field) } - count, err = GetLLMModelManager().Query().Equals("llm_image_id", image.Id).CountWithError() + count, err = GetLLMSkuManager().Query().Equals("llm_image_id", image.Id).CountWithError() if err != nil { return errors.Wrap(err, "fetch llm models") } diff --git a/pkg/llm/models/instantmodel.go b/pkg/llm/models/instantmodel.go new file mode 100644 index 0000000000..7515860586 --- /dev/null +++ b/pkg/llm/models/instantmodel.go @@ -0,0 +1,936 @@ +package models + +import ( + "context" + "fmt" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/httputils" + "yunion.io/x/pkg/utils" + "yunion.io/x/sqlchemy" + + commonapis "yunion.io/x/onecloud/pkg/apis" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + imageapi "yunion.io/x/onecloud/pkg/apis/image" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + computemodules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" + imagemodules "yunion.io/x/onecloud/pkg/mcclient/modules/image" + commonoptions "yunion.io/x/onecloud/pkg/mcclient/options" + "yunion.io/x/onecloud/pkg/util/logclient" + + apis "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/llm/options" +) + +var instantModelManager *SInstantModelManager + +func init() { + GetInstantModelManager() +} + +type SInstantModelManager struct { + db.SSharableVirtualResourceBaseManager + db.SEnabledResourceBaseManager +} + +func GetInstantModelManager() *SInstantModelManager { + if instantModelManager != nil { + return instantModelManager + } + instantModelManager = &SInstantModelManager{ + SSharableVirtualResourceBaseManager: db.NewSharableVirtualResourceBaseManager( + SInstantModel{}, + "instant_models_tbl", + "llm_instant_model", + "llm_instant_models", + ), + } + instantModelManager.SetVirtualObject(instantModelManager) + return instantModelManager +} + +type SInstantModel struct { + db.SSharableVirtualResourceBase + db.SEnabledResourceBase + + LlmType string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` + ModelId string `width:"128" charset:"ascii" list:"user" create:"optional"` + ModelName string `width:"128" charset:"ascii" list:"user" create:"required"` + Tag string `width:"64" charset:"ascii" list:"user" create:"required"` + + ImageId string `width:"128" charset:"ascii" list:"user" create:"optional" update:"user"` + + Mounts []string `charset:"ascii" list:"user" create:"optional" update:"user"` + + Size int64 `nullable:"true" list:"user" create:"optional"` + + ActualSizeMb int32 `nullable:"true" list:"user" update:"user"` + + AutoCache bool `list:"user"` +} + +// climc instant-app-list +func (man *SInstantModelManager) ListItemFilter( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + input apis.InstantModelListInput, +) (*sqlchemy.SQuery, error) { + var err error + q, err = man.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, input.SharableVirtualResourceListInput) + if err != nil { + return nil, errors.Wrap(err, "SSharableBaseResourceManager.ListItemFilter") + } + q, err = man.SEnabledResourceBaseManager.ListItemFilter(ctx, q, userCred, input.EnabledResourceBaseListInput) + if err != nil { + return nil, errors.Wrap(err, "SEnabledResourceBaseManager.ListItemFilter") + } + + if len(input.ModelName) > 0 { + q = q.In("model_name", input.ModelName) + } + if len(input.Tag) > 0 { + q = q.In("tag", input.Tag) + } + if len(input.ModelId) > 0 { + q = q.In("model_id", input.ModelId) + } + + if len(input.Image) > 0 { + s := auth.GetSession(ctx, userCred, options.Options.Region) + params := commonoptions.BaseListOptions{} + params.Scope = "max" + boolFalse := false + params.Details = &boolFalse + limit := 2048 + params.Limit = &limit + params.Filter = []string{fmt.Sprintf("name.contains(%s)", input.Image)} + results, err := imagemodules.Images.List(s, jsonutils.Marshal(params)) + if err != nil { + return nil, errors.Wrap(err, "List") + } + imageIds := make([]string, 0) + for i := range results.Data { + idstr, _ := results.Data[i].GetString("id") + imageIds = append(imageIds, idstr) + } + q = q.In("image_id", imageIds) + } + + if len(input.Mounts) > 0 { + q = q.Contains("mounts", input.Mounts) + } + + if input.AutoCache != nil { + q = q.Equals("auto_cache", *input.AutoCache) + } + + return q, nil +} + +// func (man *SInstantAppManager) FetchCustomizeColumns( +// ctx context.Context, +// userCred mcclient.TokenCredential, +// query jsonutils.JSONObject, +// objs []interface{}, +// fields stringutils2.SSortedStrings, +// isList bool, +// ) []apis.InstantAppDetails { +// res := make([]apis.InstantAppDetails, len(objs)) + +// imageIds := make([]string, 0) +// mdlNames := make([]string, 0) + +// virows := man.SSharableVirtualResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) +// for i := range res { +// res[i].SharableVirtualResourceDetails = virows[i] +// instApp := objs[i].(*SInstantApp) +// if len(instApp.ImageId) > 0 { +// imageIds = append(imageIds, instApp.ImageId) +// } +// if len(instApp.ModelName) > 0 { +// mdlNames = append(mdlNames, instApp.ModelName) +// } +// } + +// s := auth.GetSession(ctx, userCred, options.Options.Region) +// imageMap := make(map[string]imageapi.ImageDetails) +// if len(imageIds) > 0 { +// params := imageapi.ImageListInput{} +// params.Ids = imageIds +// params.VirtualResourceListInput.Scope = "max" +// details := false +// params.Details = &details +// limit := len(imageIds) +// params.Limit = &limit +// params.Field = []string{"id", "name"} +// imageList, err := imagemodules.Images.List(s, jsonutils.Marshal(params)) +// if err != nil { +// log.Errorf("list image fail %s", err) +// } else { +// for i := range imageList.Data { +// imgDetails := imageapi.ImageDetails{} +// err := imageList.Data[i].Unmarshal(&imgDetails) +// if err != nil { +// log.Errorf("unmarshal image info %s fail %s", imageList.Data[i], err) +// } else { +// imageMap[imgDetails.Id] = imgDetails +// } +// } +// } +// } +// type imageCacheStatus struct { +// CachedCount int +// CacheCount int +// } +// imageCacheStatusTbl := make(map[string]*imageCacheStatus) +// if len(imageIds) > 0 { +// params := commonoptions.BaseListOptions{} +// params.Scope = "max" +// params.Filter = []string{fmt.Sprintf("cachedimage_id.in(%s)", strings.Join(imageIds, ","))} +// details := false +// params.Details = &details +// limit := 1024 +// params.Limit = &limit +// params.Field = []string{"storagecache_id", "cachedimage_id", "status"} +// offset := -1 +// total := 0 +// for offset < 0 || offset < total { +// if offset > 0 { +// params.Offset = &offset +// } else { +// offset = 0 +// } +// resp, err := computemodules.Storagecachedimages.List(s, jsonutils.Marshal(params)) +// if err != nil { +// log.Errorf("Storagecachedimages.List fail %s", err) +// break +// } +// for i := range resp.Data { +// sci := computeapi.StoragecachedimageDetails{} +// err := resp.Data[i].Unmarshal(&sci) +// if err != nil { +// log.Errorf("unmarshal image info %s fail %s", resp.Data[i], err) +// } else { +// if _, ok := imageCacheStatusTbl[sci.CachedimageId]; !ok { +// imageCacheStatusTbl[sci.CachedimageId] = &imageCacheStatus{} +// } +// if sci.Status == computeapi.CACHED_IMAGE_STATUS_ACTIVE { +// imageCacheStatusTbl[sci.CachedimageId].CachedCount++ +// } +// imageCacheStatusTbl[sci.CachedimageId].CacheCount++ +// } +// } +// offset += len(resp.Data) +// total = resp.Total +// } + +// } +// for i := range res { +// instApp := objs[i].(*SInstantApp) +// if img, ok := imageMap[instApp.ImageId]; ok { +// res[i].Image = img.Name +// } +// if status, ok := imageCacheStatusTbl[instApp.ImageId]; ok { +// res[i].CacheCount = status.CacheCount +// res[i].CachedCount = status.CachedCount +// } +// } +// return res +// } + +func (man *SInstantModelManager) GetLLMContainerDriver(llmType apis.LLMContainerType) ILLMContainerDriver { + return GetLLMContainerDriver(llmType) +} + +func (man *SInstantModelManager) ValidateCreateData( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + query jsonutils.JSONObject, + input apis.InstantModelCreateInput, +) (apis.InstantModelCreateInput, error) { + var err error + input.SharableVirtualResourceCreateInput, err = man.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.SharableVirtualResourceCreateInput) + if err != nil { + return input, errors.Wrap(err, "SSharableVirtualResourceBaseManager.ValidateCreateData") + } + + if !apis.IsLLMContainerType(string(input.LLMType)) { + return input, errors.Wrapf(httperrors.ErrInvalidFormat, "invalid llm_type %s", input.LLMType) + } + + if len(input.ImageId) > 0 { + img, err := fetchImage(ctx, userCred, input.ImageId) + if err != nil { + return input, errors.Wrapf(err, "fetchImage %s", input.ImageId) + } + if img.DiskFormat != imageapi.IMAGE_DISK_FORMAT_TGZ { + return input, errors.Wrapf(errors.ErrInvalidFormat, "cannot use image as template of format %s", img.DiskFormat) + } + + { + mdl, err := man.findInstantModelByImageId(img.Id) + if err != nil { + return input, errors.Wrap(err, "findInstantModelByImageId") + } + if mdl != nil { + return input, errors.Wrapf(httperrors.ErrConflict, "image %s has been used by other model", input.ImageId) + } + } + + input.ImageId = img.Id + input.Size = img.Size + input.Status = img.Status + input.ActualSizeMb = img.MinDiskMB + } + if len(input.Mounts) > 0 { + drv := man.GetLLMContainerDriver(input.LLMType) + _, err = drv.ValidateMounts(input.Mounts, input.ModelName, input.Tag) + if err != nil { + return input, errors.Wrap(err, "validateMounts") + } + } + + input.Enabled = nil + return input, nil +} + +// func (app *SInstantApp) ValidateUpdateData( +// ctx context.Context, +// userCred mcclient.TokenCredential, +// query jsonutils.JSONObject, +// input apis.InstantAppUpdateInput, +// ) (apis.InstantAppUpdateInput, error) { +// var err error +// input.SharableVirtualResourceBaseUpdateInput, err = app.SSharableVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, input.SharableVirtualResourceBaseUpdateInput) +// if err != nil { +// return input, errors.Wrap(err, "SSharableVirtualResourceBase.ValidateUpdateData") +// } + +// if len(input.ImageId) > 0 { +// img, err := fetchImage(ctx, userCred, input.ImageId) +// if err != nil { +// return input, errors.Wrapf(err, "fetchImage %s", input.ImageId) +// } +// if img.DiskFormat != imageapi.IMAGE_DISK_FORMAT_TGZ { +// return input, errors.Wrapf(errors.ErrInvalidFormat, "cannot use image as template of format %s", img.DiskFormat) +// } + +// { +// findApp, err := GetInstantAppManager().findInstantAppByImageId(img.Id) +// if err != nil { +// return input, errors.Wrap(err, "findInstantAppByImageId") +// } +// if findApp != nil && findApp.Id != app.Id { +// return input, errors.Wrapf(httperrors.ErrConflict, "image %s has been used by other app", input.ImageId) +// } +// } + +// input.ImageId = img.Id +// input.Size = img.Size +// input.ActualSizeMb = img.MinDiskMB +// } +// if len(input.Mounts) > 0 { +// drv := GetInstantAppManager().GetLLMContainerDriver(apis.LLMContainerType(app.LlmType)) +// input.Mounts, err = drv.ValidateMounts(input.Mounts, app.Package) +// if err != nil { +// return input, errors.Wrap(err, "validateMounts") +// } +// if len(input.Mounts) == 0 { +// return input, errors.Wrap(errors.ErrEmpty, "empty mounts") +// } +// } +// return input, nil +// } + +func (model *SInstantModel) PostCreate( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + query jsonutils.JSONObject, + data jsonutils.JSONObject, +) { + model.syncImagePathMap(ctx, userCred) +} + +// func (app *SInstantApp) PostUpdate( +// ctx context.Context, +// userCred mcclient.TokenCredential, +// query jsonutils.JSONObject, +// data jsonutils.JSONObject, +// ) { +// app.syncImagePathMap(ctx, userCred) +// } + +func (model *SInstantModel) getImagePaths() map[string]string { + drv := GetInstantModelManager().GetLLMContainerDriver(apis.LLMContainerType(model.LlmType)) + return drv.GetImageInternalPathMounts(model) +} + +func (model *SInstantModel) syncImagePathMap(ctx context.Context, userCred mcclient.TokenCredential) error { + if len(model.ImageId) == 0 { + return nil + } + imgPaths := model.getImagePaths() + if len(imgPaths) == 0 { + return nil + } + s := auth.GetSession(ctx, userCred, options.Options.Region) + params := imageapi.ImageUpdateInput{ + Properties: map[string]string{ + "internal_path_map": jsonutils.Marshal(imgPaths).String(), + "used_by_post_overlay": "true", + }, + } + _, err := imagemodules.Images.Update(s, model.ImageId, jsonutils.Marshal(params)) + if err != nil { + return errors.Wrap(err, "Update") + } + return nil +} + +func (model *SInstantModel) PerformSyncstatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.InstantModelSyncstatusInput) (jsonutils.JSONObject, error) { + err := model.syncImageStatus(ctx, userCred) + if err != nil { + return nil, errors.Wrap(err, "syncImageStatus") + } + return nil, nil +} + +func (model *SInstantModel) saveImageId(ctx context.Context, userCred mcclient.TokenCredential, imageId string) error { + _, err := db.Update(model, func() error { + model.ImageId = imageId + return nil + }) + if err != nil { + logclient.AddActionLogWithContext(ctx, model, logclient.ACT_SAVE_IMAGE, err, userCred, false) + return errors.Wrap(err, "update image_id") + } + logclient.AddActionLogWithContext(ctx, model, logclient.ACT_SAVE_IMAGE, imageId, userCred, true) + return nil +} + +func (model *SInstantModel) syncImageStatus(ctx context.Context, userCred mcclient.TokenCredential) error { + img, err := fetchImage(ctx, userCred, model.ImageId) + if err != nil { + if httputils.ErrorCode(err) == 404 { + model.SetStatus(ctx, userCred, imageapi.IMAGE_STATUS_DELETED, "not found") + return nil + } + return errors.Wrapf(err, "fetchImage %s", model.ImageId) + } + model.SetStatus(ctx, userCred, img.Status, "syncStatus") + if img.Status == imageapi.IMAGE_STATUS_ACTIVE && (model.Size != img.Size || model.ActualSizeMb != img.MinDiskMB) { + _, err := db.Update(model, func() error { + model.Size = img.Size + model.ActualSizeMb = img.MinDiskMB + return nil + }) + if err != nil { + return errors.Wrap(err, "update size") + } + } + { + err := model.syncImagePathMap(ctx, userCred) + if err != nil { + return errors.Wrap(err, "syncImagePathMap") + } + } + return nil +} + +func (man *SInstantModelManager) findInstantModelByImageId(imageId string) (*SInstantModel, error) { + q := man.Query().Equals("image_id", imageId) + + mdls := make([]SInstantModel, 0) + err := db.FetchModelObjects(man, q, &mdls) + if err != nil { + return nil, errors.Wrap(err, "FetchModelObjects") + } + if len(mdls) == 0 { + return nil, nil + } + return &mdls[0], nil +} + +func (man *SInstantModelManager) GetInstantModelById(id string) (*SInstantModel, error) { + obj, err := man.FetchById(id) + if err != nil { + return nil, errors.Wrap(err, "FetchById") + } + return obj.(*SInstantModel), nil +} + +func (man *SInstantModelManager) findInstantModel(mdlId, tag string, isEnabled bool) (*SInstantModel, error) { + q := man.Query().Equals("model_id", mdlId).Equals("status", imageapi.IMAGE_STATUS_ACTIVE) + if isEnabled { + q = q.IsTrue("enabled") + } + q = q.Desc("created_at") + + mdls := make([]SInstantModel, 0) + err := db.FetchModelObjects(man, q, &mdls) + if err != nil { + return nil, errors.Wrap(err, "FetchModelObjects") + } + if len(mdls) == 0 { + return nil, nil + } + if len(tag) > 0 { + for i := range mdls { + if mdls[i].Tag == tag { + return &mdls[i], nil + } + } + } + return &mdls[0], nil +} + +func (model *SInstantModel) PerformEnable( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input commonapis.PerformEnableInput, +) (jsonutils.JSONObject, error) { + if len(model.ImageId) == 0 { + return nil, errors.Wrap(errors.ErrInvalidStatus, "empty image_id") + } + if len(model.Mounts) == 0 { + return nil, errors.Wrap(errors.ErrInvalidStatus, "empty mounts") + } + { + err := model.syncImageStatus(ctx, userCred) + if err != nil { + return nil, errors.Wrap(err, "syncImageStatus") + } + } + if model.Status != imageapi.IMAGE_STATUS_ACTIVE { + return nil, errors.Wrapf(errors.ErrInvalidStatus, "cannot enable model of status %s", model.Status) + } + // check duplicate + { + existing, err := GetInstantModelManager().findInstantModel(model.ModelId, model.Tag, true) + if err != nil { + return nil, errors.Wrap(err, "findInstantModel") + } + if existing != nil && existing.Id != model.Id { + return nil, errors.Wrapf(errors.ErrDuplicateId, "model of modelId %s tag %s has been enabled", model.ModelId, model.Tag) + } + } + _, err := db.Update(model, func() error { + model.SEnabledResourceBase.SetEnabled(true) + return nil + }) + if err != nil { + return nil, errors.Wrap(err, "update") + } + return nil, nil +} + +func (model *SInstantModel) PerformDisable( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input commonapis.PerformDisableInput, +) (jsonutils.JSONObject, error) { + _, err := db.Update(model, func() error { + model.SEnabledResourceBase.SetEnabled(false) + if model.AutoCache { + model.AutoCache = false + } + return nil + }) + if err != nil { + return nil, errors.Wrap(err, "update") + } + return nil, nil +} + +func (model *SInstantModel) PerformChangeOwner( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input commonapis.PerformChangeProjectOwnerInput, +) (jsonutils.JSONObject, error) { + // perform disk change owner + if len(model.ImageId) > 0 { + s := auth.GetSession(ctx, userCred, options.Options.Region) + _, err := imagemodules.Images.PerformAction(s, model.ImageId, "change-owner", jsonutils.Marshal(input)) + if err != nil { + return nil, errors.Wrap(err, "image change-owner") + } + } + return model.SSharableVirtualResourceBase.PerformChangeOwner(ctx, userCred, query, input) +} + +func (model *SInstantModel) PerformPublic( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input commonapis.PerformPublicProjectInput, +) (jsonutils.JSONObject, error) { + if len(model.ImageId) > 0 { + s := auth.GetSession(ctx, userCred, options.Options.Region) + _, err := imagemodules.Images.PerformAction(s, model.ImageId, "public", jsonutils.Marshal(input)) + if err != nil { + return nil, errors.Wrap(err, "image public") + } + } + return model.SSharableVirtualResourceBase.PerformPublic(ctx, userCred, query, input) +} + +func (model *SInstantModel) PerformPrivate( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input commonapis.PerformPrivateInput, +) (jsonutils.JSONObject, error) { + if len(model.ImageId) > 0 { + s := auth.GetSession(ctx, userCred, options.Options.Region) + _, err := imagemodules.Images.PerformAction(s, model.ImageId, "private", jsonutils.Marshal(input)) + if err != nil { + return nil, errors.Wrap(err, "image private") + } + } + return model.SSharableVirtualResourceBase.PerformPrivate(ctx, userCred, query, input) +} + +// func (app *SInstantApp) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error { +// if app.Enabled.IsTrue() { +// return errors.Wrap(errors.ErrInvalidStatus, "cannot delete when enabled") +// } + +// for _, man := range []MountedAppModelManager{GetDesktopModelManager(), GetVolumeManager()} { +// used, err := man.IsPremountedPackageName(app.Package) +// if err != nil { +// return errors.Wrap(err, "IsPremountedPackageName") +// } +// if used { +// return errors.Wrap(errors.ErrInvalidStatus, "cannot delete when package is used by other resources") +// } +// } + +// return nil +// } + +// func (app *SInstantApp) ValidateUpdateCondition(ctx context.Context) error { +// if app.Enabled.IsTrue() { +// return errors.Wrap(errors.ErrInvalidStatus, "cannot update when enabled") +// } +// return nil +// } + +func (model *SInstantModel) PerformEnableAutoCache( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input apis.InstantModelEnableAutoCacheInput, +) (jsonutils.JSONObject, error) { + if input.AutoCache && model.Enabled.IsFalse() { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "cannot enable auto_cache for disabled app") + } + _, err := db.Update(model, func() error { + model.AutoCache = input.AutoCache + return nil + }) + if err != nil { + return nil, errors.Wrap(err, "update auto_cache") + } + if model.AutoCache { + err := model.doCache(ctx, userCred) + if err != nil { + return nil, errors.Wrap(err, "doCache") + } + } + return nil, nil +} + +func (model *SInstantModel) doCache(ctx context.Context, userCred mcclient.TokenCredential) error { + input := computeapi.CachedImageManagerCacheImageInput{} + input.ImageId = model.ImageId + input.AutoCache = true + input.HostType = []string{computeapi.HOST_TYPE_CONTAINER} + s := auth.GetSession(ctx, userCred, options.Options.Region) + _, err := computemodules.Cachedimages.PerformClassAction(s, "cache-image", jsonutils.Marshal(input)) + if err != nil { + return errors.Wrap(err, "PerformClassAction cache-image") + } + return nil +} + +// func (manager *SInstantAppManager) PerformImport( +// ctx context.Context, +// userCred mcclient.TokenCredential, +// query jsonutils.JSONObject, +// input apis.InstantAppImportInput, +// ) (*SInstantApp, error) { +// if input.Invalid() { +// return nil, httperrors.NewInputParameterError("invalid input: %s", jsonutils.Marshal(input).String()) +// } +// // first create a temporary instant-app +// tempApp := &SInstantApp{} +// tempApp.SetModelManager(manager, &SInstantApp{}) +// tempApp.Name = fmt.Sprintf("tmp-instant-app-%s.%s", timeutils.CompactTime(time.Now()), utils.GenRequestId(6)) +// tempApp.Package = "temp" +// tempApp.Version = "0.0.1" +// tempApp.ProjectId = userCred.GetProjectId() + +// err := manager.TableSpec().Insert(ctx, tempApp) +// if err != nil { +// return nil, errors.Wrap(err, "Insert") +// } + +// err = tempApp.startImportTask(ctx, userCred, input) +// if err != nil { +// return nil, errors.Wrap(err, "startImportTask") +// } + +// return tempApp, nil +// } + +// func (instantapp *SInstantApp) startImportTask(ctx context.Context, userCred mcclient.TokenCredential, input apis.InstantAppImportInput) error { +// params := jsonutils.NewDict() +// params.Add(jsonutils.Marshal(input), "import_input") + +// task, err := taskman.TaskManager.NewTask(ctx, "InstantAppImportTask", instantapp, userCred, params, "", "") +// if err != nil { +// return errors.Wrap(err, "NewTask") +// } +// task.ScheduleRun(nil) + +// return nil +// } + +// func (instantapp *SInstantApp) DoImport(ctx context.Context, userCred mcclient.TokenCredential, input apis.InstantAppImportInput) error { +// // first download the image +// cfg := objectstore.NewObjectStoreClientConfig(input.Endpoint, input.AccessKey, input.SecretKey) +// if len(input.SignVer) > 0 { +// cfg.SignVersion(objectstore.S3SignVersion(input.SignVer)) +// } +// minioClient, err := objectstore.NewObjectStoreClient(cfg) +// if err != nil { +// return errors.Wrap(err, "new minio client") +// } + +// bucket, err := minioClient.GetIBucketByName(input.Bucket) +// if err != nil { +// return errors.Wrap(err, "GetIBucketByName") +// } + +// tmpDir, err := os.MkdirTemp(options.Options.AdbWorkingDirectory, "instant-app-*") +// if err != nil { +// return errors.Wrap(err, "CreateTemp") +// } +// defer func() { +// os.RemoveAll(tmpDir) +// }() + +// tmpFileName := filepath.Join(tmpDir, "instant-app.tar") + +// // download the object +// err = func() error { +// tmpFile, err := os.Create(tmpFileName) +// if err != nil { +// return errors.Wrap(err, "Create") +// } +// defer tmpFile.Close() + +// _, err = cloudprovider.DownloadObjectParallelWithProgress(ctx, bucket, input.Key, nil, tmpFile, 0, 1024*1024*10, false, 3, func(progress float64, progressMbps float64, totalSizeMb int64) { +// log.Infof("DownloadObjectParallelWithProgress progress: %f, progressMbps: %f, totalSizeMb: %d", progress, progressMbps, totalSizeMb) +// }) +// if err != nil { +// return errors.Wrap(err, "DownloadObjectParallel") +// } +// return nil +// }() +// if err != nil { +// return errors.Wrap(err, "download object") +// } + +// // untar the object +// err = procutils.NewCommand("tar", "xf", tmpFileName, "-C", tmpDir, "--strip-components=1").Run() +// if err != nil { +// return errors.Wrapf(err, "untar %s", tmpFileName) +// } + +// scriptPath := filepath.Join(tmpDir, "scripts") +// imagePath := filepath.Join(tmpDir, "image") + +// params, err := decodeParams(scriptPath) +// if err != nil { +// return errors.Wrap(err, "decodeParams") +// } + +// s := auth.GetSession(ctx, userCred, options.Options.Region) + +// // upload the image +// imageId, err := func() (string, error) { +// imgFile, err := os.Open(imagePath) +// if err != nil { +// return "", errors.Wrap(err, "Open") +// } +// defer imgFile.Close() + +// imgFileStat, err := imgFile.Stat() +// if err != nil { +// return "", errors.Wrap(err, "Stat") +// } +// imgFileSize := imgFileStat.Size() + +// imgParams := imageapi.ImageCreateInput{} +// imgParams.GenerateName = params.ImageName +// imgParams.DiskFormat = "tgz" +// imgParams.Size = &imgFileSize +// imgParams.Properties = map[string]string{ +// "os_arch": "aarch64", +// } + +// // upload the image +// imageObj, err := imagemodules.Images.Upload(s, jsonutils.Marshal(imgParams), imgFile, imgFileSize) +// if err != nil { +// return "", errors.Wrap(err, "Create") +// } +// imageId, err := imageObj.GetString("id") +// if err != nil { +// return "", errors.Wrap(err, "GetId") +// } + +// return imageId, nil +// }() +// if err != nil { +// return errors.Wrap(err, "upload image") +// } + +// newName, err := db.GenerateAlterName(instantapp, params.InstantAppName) +// if err != nil { +// return errors.Wrap(err, "GenerateAlterName") +// } +// // update the instant-app +// _, err = db.Update(instantapp, func() error { +// instantapp.Name = newName +// instantapp.Package = params.Package +// instantapp.Version = params.Version +// instantapp.ImageId = imageId +// instantapp.Mounts = []string{params.AppMount, params.DataMount} +// if len(params.MediaMount) > 0 { +// instantapp.Mounts = append(instantapp.Mounts, params.MediaMount) +// } +// instantapp.Status = imageapi.IMAGE_STATUS_SAVING +// return nil +// }) +// if err != nil { +// return errors.Wrap(err, "update instant-app") +// } + +// // wait image to be active +// imgDetails, err := instantapp.WaitImageStatus(ctx, userCred, []string{imageapi.IMAGE_STATUS_ACTIVE}, 1800) +// if err != nil { +// log.Errorf("WaitImageStatus failed: %s", err) +// } + +// // sync image status +// err = instantapp.syncImageStatus(ctx, userCred) +// if err != nil { +// return errors.Wrap(err, "syncImageStatus") +// } + +// if imgDetails.Status == imageapi.IMAGE_STATUS_KILLED || imgDetails.Status == imageapi.IMAGE_STATUS_DEACTIVATED { +// return errors.Wrapf(httperrors.ErrInvalidStatus, "image status: %s", imgDetails.Status) +// } + +// return nil +// } + +// type sInstantAppImportParams struct { +// ImageName string +// InstantAppName string +// Package string +// Version string +// AppMount string +// DataMount string +// MediaMount string +// } + +// func decodeParams(fileDir string) (*sInstantAppImportParams, error) { +// content, err := os.ReadFile(fileDir) +// if err != nil { +// return nil, errors.Wrap(err, "ReadFile") +// } +// return decodeParamsString(string(content)) +// } + +// const scriptFormat = `/root/climc image-upload --format tgz --os-arch aarch64 (?P.*) \./image +// /root/climc instant-app-create (?P.*) (?P.*) (?P.*) \\ +// --mounts "(?P.*)" \\ +// --mounts "(?P.*)" \\ +// (--mounts "(?P.*)")?` + +// var ( +// scriptRE = regexp.MustCompile(scriptFormat) +// ) + +// func decodeParamsString(content string) (*sInstantAppImportParams, error) { +// params := sInstantAppImportParams{} +// matches := scriptRE.FindStringSubmatch(content) + +// log.Debugf("decodeParamsString matches: %v", jsonutils.Marshal(matches)) + +// if len(matches) > 6 { +// params.ImageName = matches[1] +// params.InstantAppName = matches[2] +// params.Package = matches[3] +// params.Version = matches[4] +// params.AppMount = matches[5] +// params.DataMount = matches[6] +// if len(matches) > 8 { +// params.MediaMount = matches[8] +// } +// } +// return ¶ms, nil +// } + +func (model *SInstantModel) GetImage(ctx context.Context, userCred mcclient.TokenCredential) (*imageapi.ImageDetails, error) { + s := auth.GetSession(ctx, userCred, options.Options.Region) + imageObj, err := imagemodules.Images.Get(s, model.ImageId, nil) + if err != nil { + return nil, errors.Wrap(err, "Get") + } + imgDetail := imageapi.ImageDetails{} + err = imageObj.Unmarshal(&imgDetail) + if err != nil { + return nil, errors.Wrap(err, "Unmarshal") + } + return &imgDetail, nil +} + +func (model *SInstantModel) WaitImageStatus(ctx context.Context, userCred mcclient.TokenCredential, targetStatus []string, timeoutSecs int) (*imageapi.ImageDetails, error) { + expire := time.Now().Add(time.Second * time.Duration(timeoutSecs)) + for time.Now().Before(expire) { + img, err := model.GetImage(ctx, userCred) + if err != nil { + return nil, errors.Wrap(err, "GetImage") + } + if utils.IsInArray(img.Status, targetStatus) { + return img, nil + } + if strings.Contains(img.Status, "fail") || img.Status == imageapi.IMAGE_STATUS_KILLED || img.Status == imageapi.IMAGE_STATUS_DEACTIVATED { + return nil, errors.Wrap(errors.ErrInvalidStatus, img.Status) + } + time.Sleep(2 * time.Second) + } + return nil, errors.Wrapf(httperrors.ErrTimeout, "wait image status %s timeout", targetStatus) +} + +func (model *SInstantModel) GetActualSizeMb() int32 { + if model.ActualSizeMb > 0 { + return model.ActualSizeMb + } + return int32(model.Size / 1024 / 1024) +} diff --git a/pkg/llm/models/llm.go b/pkg/llm/models/llm.go index 14cc7220e6..51671920eb 100644 --- a/pkg/llm/models/llm.go +++ b/pkg/llm/models/llm.go @@ -17,6 +17,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/httperrors" + llmutils "yunion.io/x/onecloud/pkg/llm/utils" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/modules/compute" ) @@ -50,8 +51,11 @@ type SLLMManager struct { type SLLM struct { SLLMBase - LLMModelId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` + LLMSkuId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` LLMImageId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` + + // 秒装应用配额（可安装的总容量限制） + InstantModelQuotaGb int `list:"user" update:"user" create:"optional" default:"0" nullable:"false"` } func (man *SLLMManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.LLMCreateInput) (*api.LLMCreateInput, error) { @@ -60,13 +64,13 @@ func (man *SLLMManager) ValidateCreateData(ctx context.Context, userCred mcclien if err != nil { return input, errors.Wrap(err, "validate LLMBaseCreateInput") } - model, err := GetLLMModelManager().FetchByIdOrName(ctx, userCred, input.LLMModelId) + sku, err := GetLLMSkuManager().FetchByIdOrName(ctx, userCred, input.LLMSkuId) if err != nil { - return input, errors.Wrap(err, "fetch LLMModel") + return input, errors.Wrap(err, "fetch LLMSku") } - lModel := model.(*SLLMModel) - input.LLMModelId = lModel.Id - input.LLMImageId = lModel.LLMImageId + lSku := sku.(*SLLMSku) + input.LLMSkuId = lSku.Id + input.LLMImageId = lSku.LLMImageId return input, nil } @@ -85,16 +89,16 @@ func (man *SLLMManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, return q, errors.Wrap(err, "VirtualResourceBaseManager.ListItemFilter") } - if len(input.LLMModel) > 0 { - modelObj, err := GetLLMModelManager().FetchByIdOrName(ctx, userCred, input.LLMModel) + if len(input.LLMSku) > 0 { + skuObj, err := GetLLMSkuManager().FetchByIdOrName(ctx, userCred, input.LLMSku) if err != nil { if errors.Cause(err) == sql.ErrNoRows { - return nil, httperrors.NewResourceNotFoundError2(GetLLMModelManager().KeywordPlural(), input.LLMModel) + return nil, httperrors.NewResourceNotFoundError2(GetLLMSkuManager().KeywordPlural(), input.LLMSku) } else { - return nil, errors.Wrap(err, "LLMModelManager.FetchByIdOrName") + return nil, errors.Wrap(err, "GetLLMSkuManager.FetchByIdOrName") } } - q = q.Equals("llm_model_id", modelObj.GetId()) + q = q.Equals("llm_sku_id", skuObj.GetId()) } if len(input.LLMImage) > 0 { imgObj, err := GetLLMImageManager().FetchByIdOrName(ctx, userCred, input.LLMImage) @@ -158,27 +162,29 @@ func (llm *SLLM) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCre return llm.StartDeleteTask(ctx, userCred, "") } -func (llm *SLLM) GetLLMModel(modelId string) (*SLLMModel, error) { - if len(modelId) == 0 { - modelId = llm.LLMModelId +func (llm *SLLM) GetLLMSku(skuId string) (*SLLMSku, error) { + if len(skuId) == 0 { + skuId = llm.LLMSkuId } - model, err := GetLLMModelManager().FetchById(modelId) + sku, err := GetLLMSkuManager().FetchById(skuId) if err != nil { - return nil, errors.Wrap(err, "fetch LLMModel") + return nil, errors.Wrap(err, "fetch LLMSku") } - return model.(*SLLMModel), nil + return sku.(*SLLMSku), nil } -func (llm *SLLM) GetLargeLanguageModelName() (modelName string, modelTag string, err error) { - model, err := llm.GetLLMModel("") - if err != nil { - return "", "", err +func (llm *SLLM) GetLargeLanguageModelName(name string) (modelName string, modelTag string, err error) { + if name == "" { + sku, err := llm.GetLLMSku("") + if err != nil { + return "", "", err + } + name = sku.LLMModelName } - name := model.LLMModelName parts := strings.Split(name, ":") modelName = parts[0] modelTag = "latest" - if len(parts) > 1 { + if len(parts) == 2 { modelTag = parts[1] } return @@ -188,13 +194,21 @@ func (llm *SLLM) GetLLMImage() (*SLLMImage, error) { return llm.getImage(llm.LLMImageId) } +func (llm *SLLM) GetLLMSContainer(ctx context.Context) (*computeapi.SContainer, error) { + llmCtr, err := llm.GetLLMContainer() + if err != nil { + return nil, errors.Wrap(err, "GetLLMContainer") + } + return llmCtr.GetSContainer(ctx) +} + func (llm *SLLM) GetLLMContainer() (*SLLMContainer, error) { return GetLLMContainerManager().FetchByLLMId(llm.Id) } func (llm *SLLM) GetLLMContainerDriver() ILLMContainerDriver { - model, _ := llm.GetLLMModel(llm.LLMModelId) - return model.GetLLMContainerDriver() + sku, _ := llm.GetLLMSku(llm.LLMSkuId) + return sku.GetLLMContainerDriver() } func (llm *SLLM) StartCreateTask(ctx context.Context, userCred mcclient.TokenCredential, input api.LLMCreateInput, parentTaskId string) error { @@ -232,16 +246,16 @@ func (llm *SLLM) StartDeleteTask(ctx context.Context, userCred mcclient.TokenCre } func (llm *SLLM) ServerCreate(ctx context.Context, userCred mcclient.TokenCredential, s *mcclient.ClientSession, input *api.LLMCreateInput) (string, error) { - model, err := llm.GetLLMModel(llm.LLMModelId) + sku, err := llm.GetLLMSku(llm.LLMSkuId) if nil != err { - return "", errors.Wrap(err, "GetLLMModel") + return "", errors.Wrap(err, "GetLLMSku") } llmImage, err := llm.GetLLMImage() if nil != err { return "", errors.Wrap(err, "GetLLMImage") } - data, err := GetLLMPodCreateInput(ctx, userCred, input, llm, model, llmImage, "") + data, err := GetLLMPodCreateInput(ctx, userCred, input, llm, sku, llmImage, "") if nil != err { return "", errors.Wrap(err, "GetPodCreateInput") } @@ -314,6 +328,31 @@ func (llm *SLLM) StartLLMStopTask(ctx context.Context, userCred mcclient.TokenCr // return nil // } -// func (llm *SLLM) WaitContainerStatus(ctx context.Context, userCred mcclient.TokenCredential, targetStatus []string, timeoutSecs int) (*computeapi.SContainer, error) { -// return nil, nil -// } +func (llm *SLLM) WaitContainerStatus(ctx context.Context, userCred mcclient.TokenCredential, targetStatus []string, timeoutSecs int) (*computeapi.SContainer, error) { + llmCtr, err := llm.GetLLMContainer() + if err != nil { + return nil, errors.Wrap(err, "GetLLMContainer") + } + return llmutils.WaitContainerStatus(ctx, llmCtr.CmpId, targetStatus, timeoutSecs) +} + +func (llm *SLLM) PerformSyncstatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data api.LLMSyncStatusInput) (jsonutils.JSONObject, error) { + llm.SetStatus(ctx, userCred, api.LLM_STATUS_START_SYNCSTATUS, "perform syncstatus") + err := llm.StartSyncStatusTask(ctx, userCred, "") + if err != nil { + return nil, errors.Wrap(err, "StartSyncStatusTask") + } + return nil, nil +} + +func (llm *SLLM) StartSyncStatusTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { + task, err := taskman.TaskManager.NewTask(ctx, "LLMSyncStatusTask", llm, userCred, nil, parentTaskId, "") + if err != nil { + return errors.Wrap(err, "NewTask") + } + err = task.ScheduleRun(nil) + if err != nil { + return errors.Wrap(err, "ScheduleRun") + } + return nil +} diff --git a/pkg/llm/models/llm_base.go b/pkg/llm/models/llm_base.go index d333ae43cb..5319333833 100644 --- a/pkg/llm/models/llm_base.go +++ b/pkg/llm/models/llm_base.go @@ -48,7 +48,7 @@ type SLLMBase struct { Priority int `nullable:"false" default:"100" list:"user"` BandwidthMb int `nullable:"true" list:"user" create:"admin_optional"` - LastAppProbe time.Time `nullable:"true" list:"user" create:"admin_optional"` + LastInstantModelProbe time.Time `nullable:"true" list:"user" create:"admin_optional"` // 是否请求同步更新镜像 SyncImageRequest bool `default:"false" nullable:"false" list:"user" update:"user"` diff --git a/pkg/llm/models/llm_base_pod.go b/pkg/llm/models/llm_base_pod.go index 6bc7f3078b..2f6fb69f78 100644 --- a/pkg/llm/models/llm_base_pod.go +++ b/pkg/llm/models/llm_base_pod.go @@ -25,7 +25,7 @@ func GetLLMBasePodCreateInput( userCred mcclient.TokenCredential, input *api.LLMBaseCreateInput, llmBase *SLLMBase, - skuBase *SLLMModelBase, + skuBase *SLLMSkuBase, eip string, ) (*computeapi.ServerCreateInput, error) { data := computeapi.ServerCreateInput{} diff --git a/pkg/llm/models/llm_container.go b/pkg/llm/models/llm_container.go index 9b97dbaae2..526dc5f68c 100644 --- a/pkg/llm/models/llm_container.go +++ b/pkg/llm/models/llm_container.go @@ -7,9 +7,11 @@ import ( "yunion.io/x/pkg/errors" commonapi "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/apis/compute" api "yunion.io/x/onecloud/pkg/apis/llm" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/llm/utils" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -76,6 +78,10 @@ func (m *SLLMContainerManager) FetchByLLMId( return lc, nil } +func (lc *SLLMContainer) GetSContainer(ctx context.Context) (*compute.SContainer, error) { + return utils.GetContainer(ctx, lc.CmpId) +} + func (lc *SLLMContainer) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error { return lc.SVirtualResourceBase.Delete(ctx, userCred) } diff --git a/pkg/llm/models/llm_container_driver.go b/pkg/llm/models/llm_container_driver.go index c1317f79e5..bcead16f58 100644 --- a/pkg/llm/models/llm_container_driver.go +++ b/pkg/llm/models/llm_container_driver.go @@ -4,6 +4,7 @@ import ( "context" "sync" + commonapi "yunion.io/x/onecloud/pkg/apis" computeapi "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/apis/llm" "yunion.io/x/onecloud/pkg/httperrors" @@ -66,11 +67,29 @@ type ILLMContainerPullModel interface { CopyBlobs(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM) error } +type ILLMContainerInstantApp interface { + GetProbedInstantModelsExt(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, mdlIds ...string) (map[string]llm.LLMInternalInstantMdlInfo, error) + DetectModelPaths(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, pkgInfo llm.LLMInternalInstantMdlInfo) ([]string, error) + + GetImageInternalPathMounts(sApp *SInstantModel) map[string]string + GetSaveDirectories(sApp *SInstantModel) (string, []string, error) + + ValidateMounts(mounts []string, mdlName string, mdlTag string) ([]string, error) + GetInstantModelIdByPostOverlay(postOverlay *commonapi.ContainerVolumeMountDiskPostOverlay, mdlNameToId map[string]string) string + GetDirPostOverlay(dir llm.LLMMountDirInfo) *commonapi.ContainerVolumeMountDiskPostOverlay + + PreInstallModel(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, instMdl *SLLMInstantModel) error + InstallModel(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, dirs []string, mdlIds []string) error + UninstallModel(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, instMdl *SLLMInstantModel) error +} + type ILLMContainerDriver interface { GetType() llm.LLMContainerType - GetContainerSpec(ctx context.Context, llm *SLLM, image *SLLMImage, sku *SLLMModel, props []string, devices []computeapi.SIsolatedDevice, diskId string) *computeapi.PodContainerCreateInput + GetContainerSpec(ctx context.Context, llm *SLLM, image *SLLMImage, sku *SLLMSku, props []string, devices []computeapi.SIsolatedDevice, diskId string) *computeapi.PodContainerCreateInput // ILLMContainerPullModel + + ILLMContainerInstantApp } var ( diff --git a/pkg/llm/models/llm_instant_model.go b/pkg/llm/models/llm_instant_model.go new file mode 100644 index 0000000000..cccaf4103c --- /dev/null +++ b/pkg/llm/models/llm_instant_model.go @@ -0,0 +1,226 @@ +package models + +import ( + "context" + "database/sql" + "fmt" + "time" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/sqlchemy" + + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" +) + +var llmInstantModelManager *SLLMInstantModelManager + +func init() { + GetLLMInstantModelManager() +} + +func GetLLMInstantModelManager() *SLLMInstantModelManager { + if llmInstantModelManager == nil { + llmInstantModelManager = &SLLMInstantModelManager{ + SResourceBaseManager: db.NewResourceBaseManager( + SLLMInstantModel{}, + "llm_instant_models_tbl", + "llm_with_instant_model", + "llm_with_instant_models", + ), + } + llmInstantModelManager.SetVirtualObject(llmInstantModelManager) + } + return llmInstantModelManager +} + +type SLLMInstantModelManager struct { + db.SResourceBaseManager + db.SStatusResourceBaseManager +} + +type SLLMInstantModel struct { + db.SResourceBase + db.SStatusResourceBase + + // InstantModelId string `name:"model_id" width:"128" charset:"ascii" nullable:"false" list:"user" primary:"true"` + LlmId string `width:"128" charset:"ascii" nullable:"false" list:"user" primary:"true"` + // Model ID, large language model's ID, referring to special model, such as qwen3:8b + ModelId string `name:"model_id" width:"128" charset:"ascii" nullable:"false" list:"user" primary:"true"` + + // Model Tag + Tag string `width:"64" charset:"utf8" nullable:"true" list:"user"` + // Model Name + ModelName string `width:"128" charset:"utf8" nullable:"false" list:"user"` + IsProbed bool `list:"user"` + IsMounted bool `list:"user"` + // IsSystem tristate.TriState `list:"user"` +} + +func (man *SLLMInstantModelManager) fetchLLMInstantModel(llmId string, mdlId string) (*SLLMInstantModel, error) { + q := man.RawQuery().Equals("llm_id", llmId).Equals("model_id", mdlId) + llmInstantModel := SLLMInstantModel{} + err := q.First(&llmInstantModel) + if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return nil, errors.ErrNotFound + } + return nil, errors.Wrap(err, "Query") + } + llmInstantModel.SetModelManager(man, &llmInstantModel) + return &llmInstantModel, nil +} + +func (man *SLLMInstantModelManager) updateInstantModel(ctx context.Context, llmId string, mdlId string, mdlName string, tag string, probed, mounted *bool) (*SLLMInstantModel, error) { + mdl, err := man.fetchLLMInstantModel(llmId, mdlId) + if err != nil && errors.Cause(err) != errors.ErrNotFound { + return nil, errors.Wrap(err, "updateInstantModel") + } + + mountStr := "nil" + probedStr := "nil" + if mounted != nil { + mountStr = fmt.Sprintf("%v", *mounted) + } + if probed != nil { + probedStr = fmt.Sprintf("%v", *probed) + } + log.Debugf("=======updateInstantModel %#v to mounted %s, probed %s", mdl, mountStr, probedStr) + + if mdl == nil { + // if no such app + mdl = &SLLMInstantModel{ + LlmId: llmId, + ModelId: mdlId, + ModelName: mdlName, + // IsSystem: tristate.None, + // Entry: entry, + } + // if isSys != nil { + // if *isSys { + // mdl.IsSystem = tristate.True + // } else { + // mdl.IsSystem = tristate.False + // } + // } + mdl.Tag = tag + if probed != nil { + mdl.IsProbed = *probed + } + if mounted != nil { + mdl.IsMounted = *mounted + } + mdl.syncStatus() + err := man.TableSpec().Insert(ctx, mdl) + if err != nil { + return nil, errors.Wrap(err, "Insert") + } + return mdl, nil + } else { + _, err := db.Update(mdl, func() error { + if len(tag) > 0 { + mdl.Tag = tag + } + if len(mdlName) > 0 { + mdl.ModelName = mdlName + } + // if isSys != nil { + // if *isSys { + // mdl.IsSystem = tristate.True + // } else { + // mdl.IsSystem = tristate.False + // } + // } + // if len(entry) > 0 { + // app.Entry = entry + // } + if probed != nil { + mdl.IsProbed = *probed + if mdl.IsProbed { + mdl.Status = api.LLM_STATUS_READY + } else { + mdl.Status = api.LLM_STATUS_DELETED + } + } + + if mounted != nil { + mdl.IsMounted = *mounted + } + mdl.syncStatus() + return nil + }) + if err != nil { + return nil, errors.Wrap(err, "Update") + } + } + return mdl, nil +} + +func (man *SLLMInstantModelManager) filterModels(q *sqlchemy.SQuery, isProbed, isMounted, isSystem *bool) *sqlchemy.SQuery { + if isProbed != nil { + if *isProbed { + q = q.IsTrue("is_probed") + } else { + q = q.IsFalse("is_probed") + } + } + if isMounted != nil { + if *isMounted { + q = q.IsTrue("is_mounted") + } else { + q = q.IsFalse("is_mounted") + } + } + if isSystem != nil { + if *isSystem { + q = q.IsTrue("is_system") + } else { + q = q.Filter(sqlchemy.OR( + sqlchemy.IsNull(q.Field("is_system")), + sqlchemy.IsFalse(q.Field("is_system")), + )) + } + } + return q +} + +func (mdl *SLLMInstantModel) FindInstantModel(isInstall bool) (*SInstantModel, error) { + instMdl, err := GetInstantModelManager().findInstantModel(mdl.ModelId, mdl.Tag, isInstall) + if err != nil { + return nil, errors.Wrap(err, "FindInstantModel") + } + return instMdl, nil +} + +func (mdl *SLLMInstantModel) syncStatus() { + if !mdl.IsProbed && !mdl.IsMounted { + mdl.MarkDelete() + } else { + mdl.Deleted = false + mdl.DeletedAt = time.Time{} + } +} + +func (mdl *SLLMInstantModel) getMountPaths(isInstall bool) ([]api.LLMMountDirInfo, error) { + info, err := mdl.getMountPathsFromImage(isInstall) + if err != nil { + return nil, errors.Wrap(err, "getMountPathsFromImage") + } + return info, nil +} + +func (mdl *SLLMInstantModel) getMountPathsFromImage(isInstall bool) ([]api.LLMMountDirInfo, error) { + instMdl, err := mdl.FindInstantModel(isInstall) + if err != nil { + return nil, errors.Wrap(err, "findInstantApp") + } + if instMdl == nil { + return nil, nil + } + info := make([]api.LLMMountDirInfo, 0) + info = append(info, api.LLMMountDirInfo{ + ImageId: instMdl.ImageId, + }) + return info, nil +} diff --git a/pkg/llm/models/llm_instant_model_quota.go b/pkg/llm/models/llm_instant_model_quota.go new file mode 100644 index 0000000000..f6e2aeb769 --- /dev/null +++ b/pkg/llm/models/llm_instant_model_quota.go @@ -0,0 +1,59 @@ +package models + +type LLMInstanceModelPendingTask struct { + LLMId string + TaskId string + SizeGb float64 +} + +var llmPendingInstantModelTasks []LLMInstanceModelPendingTask + +func init() { + llmPendingInstantModelTasks = make([]LLMInstanceModelPendingTask, 0) +} + +func (llm *SLLM) GetInstantModelSizeGb() float64 { + boolTrue := true + models, err := llm.FetchModels(nil, &boolTrue, nil) + if err != nil { + return 0 + } + totalSizeGb := 0.0 + for _, model := range models { + instModel, _ := GetInstantModelManager().findInstantModel(model.ModelId, model.Tag, false) + if instModel == nil { + continue + } + totalSizeGb += float64(instModel.GetActualSizeMb()) * 1024 * 1024 / 1000 / 1000 / 1000 + } + return totalSizeGb +} + +func (llm *SLLM) GetPendingInstantModelSizeGb() float64 { + var totalSizeGb float64 + for _, task := range llmPendingInstantModelTasks { + totalSizeGb += task.SizeGb + } + return totalSizeGb +} + +func (llm *SLLM) GetTotalInstantModelSizeGb() float64 { + return llm.GetInstantModelSizeGb() + llm.GetPendingInstantModelSizeGb() +} + +func (llm *SLLM) insertPendingInstantModelQuota(taskId string, sizeGb float64) { + llmPendingInstantModelTasks = append(llmPendingInstantModelTasks, LLMInstanceModelPendingTask{ + LLMId: llm.Id, + TaskId: taskId, + SizeGb: sizeGb, + }) +} + +func (llm *SLLM) ClearPendingInstantModelQuota(taskId string) { + for i := range llmPendingInstantModelTasks { + if llmPendingInstantModelTasks[i].TaskId == taskId { + llmPendingInstantModelTasks = append(llmPendingInstantModelTasks[:i], llmPendingInstantModelTasks[i+1:]...) + break + } + } +} diff --git a/pkg/llm/models/llm_instant_model_sync.go b/pkg/llm/models/llm_instant_model_sync.go new file mode 100644 index 0000000000..d293049fb2 --- /dev/null +++ b/pkg/llm/models/llm_instant_model_sync.go @@ -0,0 +1,671 @@ +package models + +import ( + "context" + "database/sql" + "net/http" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" + + commonapi "yunion.io/x/onecloud/pkg/apis" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + apis "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/compute" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +func (llm *SLLM) getMountedInstantModels(ctx context.Context, probedExt map[string]apis.LLMInternalInstantMdlInfo) (map[string]struct{}, error) { + container, err := llm.GetLLMSContainer(ctx) + if err != nil { + return nil, errors.Wrap(err, "GetSContainer") + } + if container.Spec == nil { + return nil, errors.Wrap(errors.ErrEmpty, "no Spec") + } + if len(container.Spec.VolumeMounts) == 0 { + return nil, errors.Wrap(errors.ErrEmpty, "no VolumeMounts") + } + if container.Spec.VolumeMounts[0].Disk == nil { + return nil, errors.Wrap(errors.ErrEmpty, "no Disk") + } + if len(container.Spec.VolumeMounts[0].Disk.PostOverlay) == 0 { + return nil, nil + } + mdlNameToId := make(map[string]string) + for mdlId, model := range probedExt { + mdlNameToId[model.Name+":"+model.Tag] = mdlId + } + mdlMap := make(map[string]struct{}) + postOverlays := container.Spec.VolumeMounts[0].Disk.PostOverlay + drv := llm.GetLLMContainerDriver() + for i := range postOverlays { + postOverlay := postOverlays[i] + mdlId := drv.GetInstantModelIdByPostOverlay(postOverlay, mdlNameToId) + if mdlId != "" { + mdlMap[mdlId] = struct{}{} + } + } + return mdlMap, nil +} + +func (llm *SLLM) getProbedInstantModelsExt(ctx context.Context, userCred mcclient.TokenCredential, instantModelIds ...string) (map[string]apis.LLMInternalInstantMdlInfo, error) { + drv := llm.GetLLMContainerDriver() + return drv.GetProbedInstantModelsExt(ctx, userCred, llm, instantModelIds...) +} + +type sInstantModelStatus struct { + apis.LLMInternalInstantMdlInfo + + Probed bool + Mounted bool +} + +func (llm *SLLM) getProbedMountedInstantModels(ctx context.Context, userCred mcclient.TokenCredential) (map[string]*sInstantModelStatus, error) { + mdlMap := make(map[string]*sInstantModelStatus) + probedExt, errExt := llm.getProbedInstantModelsExt(ctx, userCred) + if errExt != nil { + return nil, errors.Wrap(errExt, "getProbedInstantModelsExt") + } + + for modelId := range probedExt { + mdlMap[modelId] = &sInstantModelStatus{ + LLMInternalInstantMdlInfo: probedExt[modelId], + Probed: true, + } + } + + mounted, err := llm.getMountedInstantModels(ctx, probedExt) + if err != nil { + return nil, errors.Wrap(err, "llm.getMountedInstantModels") + } + for mdlId := range mounted { + if _, ok := mdlMap[mdlId]; ok { + mdlMap[mdlId].Mounted = true + } else { + mdlMap[mdlId] = &sInstantModelStatus{ + LLMInternalInstantMdlInfo: apis.LLMInternalInstantMdlInfo{ + ModelId: mdlId, + }, + Mounted: true, + } + } + } + return mdlMap, nil +} + +func (llm *SLLM) uninstallInstantModel(ctx context.Context, userCred mcclient.TokenCredential, mdlId string) error { + boolFalse := false + // uninstalled + probed := &boolFalse + mounted := &boolFalse + _, err := GetLLMInstantModelManager().updateInstantModel(ctx, llm.Id, mdlId, "", "", probed, mounted) + if err != nil { + return errors.Wrap(err, "uninstallPackage") + } + return nil +} + +func findInstantModelWithModelInfo(allModels []SLLMInstantModel, mdl apis.ModelInfo) *SLLMInstantModel { + for i := range allModels { + if allModels[i].ModelId == mdl.ModelId { + return &allModels[i] + } + } + return nil +} + +func findModelsToUninstall(allModels []SLLMInstantModel, input apis.LLMSyncModelTaskInput) []SLLMInstantModel { + ret := make([]SLLMInstantModel, 0) + for i := range input.Models { + existingModel := findInstantModelWithModelInfo(allModels, input.Models[i]) + if existingModel != nil && existingModel.IsMounted && (input.Method == apis.QuickModelUninstall || input.Method == apis.QuickModelReinstall || (!existingModel.IsProbed && input.Method == apis.QuickModelInstall)) { + ret = append(ret, *existingModel) + } + } + return ret +} + +func findModelsToUnmount(allModels []SLLMInstantModel, input apis.LLMSyncModelTaskInput) []SLLMInstantModel { + ret := make([]SLLMInstantModel, 0) + for i := range input.Models { + existingModel := findInstantModelWithModelInfo(allModels, input.Models[i]) + if existingModel != nil && existingModel.IsMounted && (input.Method == apis.QuickModelUninstall || input.Method == apis.QuickModelReinstall || (!existingModel.IsProbed && input.Method == apis.QuickModelInstall)) { + ret = append(ret, *existingModel) + } + } + return ret +} + +func isImageInUnmountModels(imageId string, mdls []SLLMInstantModel) (bool, error) { + instMdl, err := GetInstantModelManager().findInstantModelByImageId(imageId) + if err != nil { + return false, errors.Wrap(err, "findInstantAppByImageId") + } + if instMdl == nil { + return false, nil + } + for i := range mdls { + if mdls[i].ModelId == instMdl.ModelId { + return true, nil + } + } + return false, nil +} + +func (llm *SLLM) RefreshInstantModels(ctx context.Context, userCred mcclient.TokenCredential, refresh bool) error { + lockman.LockObject(ctx, llm) + defer lockman.ReleaseObject(ctx, llm) + + if !llm.LastInstantModelProbe.IsZero() && time.Since(llm.LastInstantModelProbe) < apis.LLM_PROBE_INSTANT_MODEl_INTERVAL_SECOND*time.Second && (!refresh || time.Since(llm.LastInstantModelProbe) < apis.LLM_PROBE_INSTANT_MODEl_INTERVAL_SECOND*time.Second) { + // already probed, null operation + return nil + } + + mdlMap, err := llm.getProbedMountedInstantModels(ctx, userCred) + if err != nil { + return errors.Wrap(err, "getProbedMountedInstantModels") + } + + models, err := llm.FetchModels(nil, nil, nil) + if err != nil { + return errors.Wrap(err, "FetchModels") + } + + var errs []error + for i := range models { + mdl := models[i] + var probed *bool + var mounted *bool + if status, ok := mdlMap[mdl.ModelId]; ok { + // probed + probed = &status.Probed + mounted = &status.Mounted + _, err = GetLLMInstantModelManager().updateInstantModel(ctx, llm.Id, mdl.ModelId, status.Name, status.Tag, probed, mounted) + delete(mdlMap, mdl.ModelId) + } else { + // uninstalled + err = llm.uninstallInstantModel(ctx, userCred, mdl.ModelId) + } + if err != nil { + errs = append(errs, err) + } + } + + if len(mdlMap) > 0 { + for mdlId, status := range mdlMap { + _, err := GetLLMInstantModelManager().updateInstantModel(ctx, llm.Id, mdlId, status.Name, status.Tag, &status.Probed, &status.Mounted) + if err != nil { + errs = append(errs, err) + } + } + } + + if len(errs) > 0 { + return errors.NewAggregate(errs) + } + // update timer + _, err = db.Update(llm, func() error { + llm.LastInstantModelProbe = time.Now() + return nil + }) + if err != nil { + return errors.Wrap(err, "update last_instant_model_probe") + } + + return nil +} + +func (llm *SLLM) PerformQuickModels(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.LLMPerformQuickModelsInput) (*apis.LLMBatchPerformOutput, error) { + if !utils.IsInArray(llm.Status, []string{apis.LLM_STATUS_RUNNING, apis.LLM_STATUS_READY}) { + return nil, errors.Wrapf(errors.ErrInvalidStatus, "llm:%s(%s) status:%s", llm.Name, llm.Id, llm.Status) + } + + llmStatus := llm.Status + if len(input.Method) == 0 { + input.Method = apis.QuickModelInstall + } + + var toInstallSizeGb float64 + var errs []error + for i := range input.Models { + // specified by ID + if len(input.Models[i].Id) > 0 { + instModelObj, err := GetInstantModelManager().FetchByIdOrName(ctx, userCred, input.Models[i].Id) + if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + errs = append(errs, httperrors.NewResourceNotFoundError2(GetInstantModelManager().Keyword(), input.Models[i].Id)) + } else { + errs = append(errs, errors.Wrap(err, "FetchByIdOrName")) + } + } else { + instApp := instModelObj.(*SInstantModel) + input.Models[i].Id = instApp.Id + input.Models[i].ModelId = instApp.ModelId + input.Models[i].Tag = instApp.Tag + if input.Method == apis.QuickModelInstall { + toInstallSizeGb += float64(instApp.GetActualSizeMb()) * 1024 * 1024 / 1000 / 1000 / 1000 + } + } + } else { + mdl, err := GetInstantModelManager().findInstantModel(input.Models[i].ModelId, input.Models[i].Tag, true) + if err != nil { + return nil, errors.Wrapf(err, "findInstantModel %s %s", input.Models[i].ModelId, input.Models[i].Tag) + } + if mdl == nil { + errs = append(errs, httperrors.NewResourceNotFoundError2(GetInstantModelManager().Keyword(), input.Models[i].ModelId)) + } else { + input.Models[i].Id = mdl.Id + input.Models[i].Tag = mdl.Tag + input.Models[i].ModelId = mdl.ModelId + } + } + } + if len(errs) > 0 { + return nil, errors.NewAggregate(errs) + } + if input.Method == apis.QuickModelInstall { + if llm.InstantModelQuotaGb > 0 && toInstallSizeGb > float64(llm.InstantModelQuotaGb)-llm.GetTotalInstantModelSizeGb() { + return nil, errors.Wrapf(httperrors.ErrOutOfQuota, "toInstallSizeGb %f > InstantAppQuotaGb %d - total %f Gb", toInstallSizeGb, llm.InstantModelQuotaGb, llm.GetTotalInstantModelSizeGb()) + } + } + + task, err := llm.StartLLMInstantModelsSyncTask(ctx, userCred, llmStatus, input, "") + if err != nil { + return nil, errors.Wrap(err, "StartLLMInstantModelsSyncTask") + } + + if input.Method == apis.QuickModelInstall { + // save pending quota + llm.insertPendingInstantModelQuota(task.Id, toInstallSizeGb) + } + + output := apis.LLMBatchPerformOutput{ + Data: []apis.LLMPerformOutput{ + { + Id: llm.Id, + Name: llm.Name, + RequestStatus: http.StatusOK, + TaskId: task.Id, + }, + }, + Task: task, + } + return &output, nil +} + +func (llm *SLLM) StartLLMInstantModelsSyncTask(ctx context.Context, userCred mcclient.TokenCredential, llmStatus string, input apis.LLMPerformQuickModelsInput, parentTaskid string) (*taskman.STask, error) { + if !utils.IsInArray(llmStatus, []string{apis.LLM_STATUS_RUNNING, apis.LLM_STATUS_READY}) { + return nil, errors.Wrapf(errors.ErrInvalidStatus, "cannot sync models in status %s", llmStatus) + } + taskInput := apis.LLMSyncModelTaskInput{ + LLMPerformQuickModelsInput: input, + LLMStatus: llmStatus, + } + task, err := taskman.TaskManager.NewTask(ctx, "LLMInstantModelsSyncTask", llm, userCred, jsonutils.Marshal(taskInput).(*jsonutils.JSONDict), parentTaskid, "") + if err != nil { + return nil, errors.Wrap(err, "NewTask") + } + err = task.ScheduleRun(nil) + if err != nil { + return nil, errors.Wrap(err, "ScheduleRun") + } + return task, nil +} + +func (llm *SLLM) FetchModels(isProbed, isMounted, isSystem *bool) ([]SLLMInstantModel, error) { + q := GetLLMInstantModelManager().Query().Equals("llm_id", llm.Id) + q = GetLLMInstantModelManager().filterModels(q, isProbed, isMounted, isSystem) + + models := make([]SLLMInstantModel, 0) + err := db.FetchModelObjects(GetLLMInstantModelManager(), q, &models) + if err != nil { + return nil, errors.Wrap(err, "FetchModelObjects") + } + return models, nil +} + +func (llm *SLLM) FetchModelsFullName(isProbed, isMounted *bool) ([]string, error) { + models, err := llm.FetchModels(isProbed, isMounted, nil) + if err != nil { + return nil, errors.Wrap(err, "FetchModels") + } + mdlFullNames := make([]string, len(models)) + for idx, mdl := range models { + mdlFullNames[idx] = mdl.ModelName + ":" + mdl.Tag + "-" + mdl.ModelId + } + return mdlFullNames, nil +} + +func (llm *SLLM) FetchMountedModelFullName() ([]string, error) { + boolTrue := true + return llm.FetchModelsFullName(nil, &boolTrue) +} + +func (llm *SLLM) RequestUnmountModel(ctx context.Context, userCred mcclient.TokenCredential, input apis.LLMSyncModelTaskInput) ([]string, []*commonapi.ContainerVolumeMountDiskPostOverlay, error) { + if input.LLMStatus == apis.LLM_STATUS_RUNNING { + err := llm.RefreshInstantModels(ctx, userCred, true) + if err != nil { + return nil, nil, errors.Wrap(err, "RefreshInstantModels") + } + } + allModels, err := llm.FetchModels(nil, nil, nil) + if err != nil { + return nil, nil, errors.Wrap(err, "FetchModels") + } + drv := llm.GetLLMContainerDriver() + + if input.LLMStatus == apis.LLM_STATUS_RUNNING { + uninstallModels := findModelsToUninstall(allModels, input) + for i := range uninstallModels { + err := drv.UninstallModel(ctx, userCred, llm, &uninstallModels[i]) + if err != nil { + log.Errorf("fail to uninstall %s", err) + continue + } + } + } + + // next found out models that need to unmount + unmountModels := findModelsToUnmount(allModels, input) + if len(unmountModels) == 0 { + return nil, nil, nil + } + + container, err := llm.GetLLMSContainer(ctx) + if err != nil { + return nil, nil, errors.Wrap(err, "GetContainer") + } + + var unmountOverlays []*commonapi.ContainerVolumeMountDiskPostOverlay + existingOverlays := container.Spec.VolumeMounts[0].Disk.PostOverlay + + for i := range existingOverlays { + eOverlay := existingOverlays[i] + if eOverlay.Image != nil && len(eOverlay.Image.Id) > 0 { + find, err := isImageInUnmountModels(eOverlay.Image.Id, unmountModels) + if err != nil { + return nil, nil, errors.Wrap(err, "isImageInUnmountModels") + } + if find { + unmountOverlays = append(unmountOverlays, eOverlay) + } + + } + } + + var modelIds []string + for i := range unmountModels { + modelIds = append(modelIds, unmountModels[i].ModelId) + } + return modelIds, unmountOverlays, nil +} + +func (llm *SLLM) RequestMountModels(ctx context.Context, userCred mcclient.TokenCredential, input apis.LLMSyncModelTaskInput) ([]string, []string, []*commonapi.ContainerVolumeMountDiskPostOverlay, error) { + if input.LLMStatus == apis.LLM_STATUS_RUNNING { + err := llm.RefreshInstantModels(ctx, userCred, true) + if err != nil { + return nil, nil, nil, errors.Wrap(err, "RefreshApps") + } + } + existingMdls, err := llm.FetchModels(nil, nil, nil) + if err != nil { + return nil, nil, nil, errors.Wrap(err, "FetchApps") + } + log.Debugf("=======RequestMountModels input: %s", jsonutils.Marshal(input).PrettyString()) + models, overlays, err := llm.getMountingModelsPostOverlay(ctx, input, existingMdls) + if err != nil { + return nil, nil, nil, errors.Wrap(err, "getMountingModelsPostOverlay") + } + drv := llm.GetLLMContainerDriver() + var mdlIds []string + for i := range models { + model := models[i] + if input.LLMStatus == apis.LLM_STATUS_RUNNING { + err := drv.PreInstallModel(ctx, userCred, llm, &model) + if err != nil { + log.Errorf("preinstallPackage fail %s", err) + } + } + mdlIds = append(mdlIds, model.ModelId) + } + targetDirs := make([]string, 0) + for i := range overlays { + if len(overlays[i].ContainerTargetDir) > 0 { + targetDirs = append(targetDirs, overlays[i].ContainerTargetDir) + } + } + return mdlIds, targetDirs, overlays, nil +} + +func (llm *SLLM) TryContainerUnmountPaths(ctx context.Context, userCred mcclient.TokenCredential, s *mcclient.ClientSession, overlays []*commonapi.ContainerVolumeMountDiskPostOverlay, waitSecs int) error { + start := time.Now() + for time.Since(start) < time.Second*time.Duration(waitSecs) { + err := llm.containerUnmountPaths(ctx, userCred, s, overlays) + if err != nil { + if strings.Contains(err.Error(), string(errors.ErrInvalidStatus)) { + // wait + time.Sleep(5 * time.Second) + } else { + return errors.Wrap(err, "containerMountPaths") + } + } else { + // success + return nil + } + } + return errors.ErrTimeout +} + +func (llm *SLLM) containerUnmountPaths(ctx context.Context, userCred mcclient.TokenCredential, s *mcclient.ClientSession, overlays []*commonapi.ContainerVolumeMountDiskPostOverlay) error { + ctr, err := llm.GetLLMSContainer(ctx) + if err != nil { + return errors.Wrap(err, "GetSContainer") + } + + if !computeapi.ContainerFinalStatus.Has(ctr.Status) { + return errors.Wrapf(errors.ErrInvalidStatus, "cannot unmount post path in status %s", ctr.Status) + } + + params := computeapi.ContainerVolumeMountRemovePostOverlayInput{ + Index: 0, + PostOverlay: overlays, + UseLazy: true, + ClearLayers: true, + } + _, err = compute.Containers.PerformAction(s, ctr.Id, "remove-volume-mount-post-overlay", jsonutils.Marshal(params)) + if err != nil { + return errors.Wrap(err, "PerformAction remove-volume-mount-post-overlay") + } + return nil +} + +func (llm *SLLM) TryContainerMountPaths(ctx context.Context, userCred mcclient.TokenCredential, s *mcclient.ClientSession, overlays []*commonapi.ContainerVolumeMountDiskPostOverlay, waitSecs int) error { + start := time.Now() + for time.Since(start) < time.Second*time.Duration(waitSecs) { + err := llm.containerMountPaths(ctx, userCred, s, overlays) + if err != nil { + if strings.Contains(err.Error(), string(errors.ErrInvalidStatus)) { + log.Errorf("containerMountPaths error %s, retry", err) + // retry + time.Sleep(5 * time.Second) + } else { + return errors.Wrap(err, "containerMountPaths") + } + } else { + // success + return nil + } + } + return errors.ErrTimeout +} + +func (llm *SLLM) containerMountPaths(ctx context.Context, userCred mcclient.TokenCredential, s *mcclient.ClientSession, overlays []*commonapi.ContainerVolumeMountDiskPostOverlay) error { + ctr, err := llm.GetLLMSContainer(ctx) + if err != nil { + return errors.Wrap(err, "GetLLMSContainer") + } + + if !computeapi.ContainerFinalStatus.Has(ctr.Status) { + return errors.Wrapf(errors.ErrInvalidStatus, "cannot mount post path in status %s", ctr.Status) + } + params := computeapi.ContainerVolumeMountAddPostOverlayInput{ + Index: 0, + PostOverlay: overlays, + } + _, err = compute.Containers.PerformAction(s, ctr.Id, "add-volume-mount-post-overlay", jsonutils.Marshal(params)) + if err != nil { + return errors.Wrap(err, "PerformAction add-volume-mount-post-overlay") + } + return nil +} + +func (llm *SLLM) MarkInstantModelsUnmounted(ctx context.Context, userCred mcclient.TokenCredential, llmStatus string, mdlIds []string) error { + return llm.markInstantModelsMounted(ctx, userCred, llmStatus, mdlIds, false) +} + +func (llm *SLLM) MarkInstantModelsMounted(ctx context.Context, userCred mcclient.TokenCredential, llmStatus string, mdlIds []string) error { + return llm.markInstantModelsMounted(ctx, userCred, llmStatus, mdlIds, true) +} + +func (llm *SLLM) markInstantModelsMounted(ctx context.Context, userCred mcclient.TokenCredential, llmStatus string, mdlIds []string, mounted bool) error { + boolFalse := false + boolTrue := true + var isProbed *bool + if !mounted { + isProbed = &boolFalse + } else { + isProbed = &boolTrue + } + + var errs []error + for i := range mdlIds { + _, err := GetLLMInstantModelManager().updateInstantModel(ctx, llm.Id, mdlIds[i], "", "", isProbed, &mounted) + if err != nil { + errs = append(errs, err) + } + } + if len(errs) > 0 { + return errors.NewAggregate(errs) + } + if llmStatus == apis.LLM_STATUS_RUNNING { + err := llm.RefreshInstantModels(ctx, userCred, true) + if err != nil { + return errors.Wrap(err, "RefreshApps") + } + } + mountedModelsFullName, err := llm.FetchMountedModelFullName() + if err != nil { + return errors.Wrap(err, "FetchMountedModelFullName") + } + { + // save mounted apps to volume + err := llm.UpdateVolumeMountedModelFullNames(mountedModelsFullName) + if err != nil { + return errors.Wrap(err, "UpdateVolumeMountedModelFullNames") + } + } + logclient.AddActionLogWithContext(ctx, llm, logclient.ACT_UPDATE, mountedModelsFullName, userCred, true) + return nil +} + +func (llm *SLLM) UpdateVolumeMountedModelFullNames(mdlFullNames []string) error { + volume, err := llm.GetVolume() + if err != nil { + return errors.Wrap(err, "GetVolume") + } + return volume.UpdateMountedModelFullNames(mdlFullNames) +} + +func (llm *SLLM) getMountingModelsPostOverlay(ctx context.Context, input apis.LLMSyncModelTaskInput, existingMdls []SLLMInstantModel) ([]SLLMInstantModel, []*commonapi.ContainerVolumeMountDiskPostOverlay, error) { + var models []SLLMInstantModel + for i := range input.Models { + if input.Method == apis.QuickModelInstall || input.Method == apis.QuickModelReinstall { + mdl := input.Models[i] + if input.Method == apis.QuickModelInstall { + existingModel := findInstantModelWithModelInfo(existingMdls, mdl) + if existingModel != nil && (existingModel.IsProbed || existingModel.IsMounted) { + // if the model is already probed or mounted, skip mount + continue + } + } + model, err := GetLLMInstantModelManager().updateInstantModel(ctx, llm.Id, mdl.ModelId, mdl.DisplayName, mdl.Tag, nil, nil) + if err != nil { + return nil, nil, errors.Wrapf(err, "updateInstantModel %s", mdl.ModelId) + } + models = append(models, *model) + } + } + if len(models) == 0 { + return nil, nil, nil + } + drv := llm.GetLLMContainerDriver() + overlays, err := models2overlays(drv, models, true) + if err != nil { + return nil, nil, errors.Wrap(err, "models2overlays") + } + return models, overlays, nil +} + +func models2overlays(drv ILLMContainerDriver, models []SLLMInstantModel, isInstall bool) ([]*commonapi.ContainerVolumeMountDiskPostOverlay, error) { + var errs []error + var allDirs []apis.LLMMountDirInfo + for i := range models { + mdlDirs, err := models[i].getMountPaths(isInstall) + if err != nil { + errs = append(errs, err) + continue + } + allDirs = append(allDirs, mdlDirs...) + } + if len(allDirs) == 0 { + if len(errs) > 0 { + return nil, errors.Wrap(errors.NewAggregate(errs), "getMountPaths") + } + return nil, nil + } + if len(errs) > 0 { + log.Errorf("models2overlays getMountPaths error %s", errors.NewAggregate(errs)) + } + var overlays []*commonapi.ContainerVolumeMountDiskPostOverlay + for i := range allDirs { + overlay := drv.GetDirPostOverlay(allDirs[i]) + overlays = append(overlays, overlay) + } + return overlays, nil +} + +func (llm *SLLM) InstallInstantModels(ctx context.Context, userCred mcclient.TokenCredential, dirs []string, mdlIds []string) error { + drv := llm.GetLLMContainerDriver() + return drv.InstallModel(ctx, userCred, llm, dirs, mdlIds) +} + +func (llm *SLLM) EnsureInstantModelsInstalled(ctx context.Context, userCred mcclient.TokenCredential, mdlIds []string) error { + mdlMap, err := llm.getProbedInstantModelsExt(ctx, userCred, mdlIds...) + if err != nil { + return errors.Wrap(err, "FetchApps") + } + var errs []error + for _, mdlId := range mdlIds { + if _, ok := mdlMap[mdlId]; ok { + // probed + // errs = append(errs, errors.Wrap(errors.ErrInvalidStatus, pkg)) + } else { + // not mounted and not probed + errs = append(errs, errors.Wrapf(errors.ErrNotFound, "mdlId %s", mdlId)) + } + } + if len(errs) > 0 { + return errors.NewAggregate(errs) + } + return nil +} diff --git a/pkg/llm/models/llm_pod.go b/pkg/llm/models/llm_pod.go index 91bc4483ee..c95955c99d 100644 --- a/pkg/llm/models/llm_pod.go +++ b/pkg/llm/models/llm_pod.go @@ -15,11 +15,11 @@ func GetLLMPodCreateInput( userCred mcclient.TokenCredential, input *api.LLMCreateInput, llm *SLLM, - sku *SLLMModel, + sku *SLLMSku, llmImage *SLLMImage, eip string, ) (*computeapi.ServerCreateInput, error) { - data, err := GetLLMBasePodCreateInput(ctx, userCred, &input.LLMBaseCreateInput, &llm.SLLMBase, &sku.SLLMModelBase, eip) + data, err := GetLLMBasePodCreateInput(ctx, userCred, &input.LLMBaseCreateInput, &llm.SLLMBase, &sku.SLLMSkuBase, eip) if err != nil { return nil, errors.Wrap(err, "GetLLMBasePodCreateInput: ") } diff --git a/pkg/llm/models/llm_save_instant_model.go b/pkg/llm/models/llm_save_instant_model.go new file mode 100644 index 0000000000..f824fd6416 --- /dev/null +++ b/pkg/llm/models/llm_save_instant_model.go @@ -0,0 +1,181 @@ +package models + +import ( + "context" + "fmt" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + hostapi "yunion.io/x/onecloud/pkg/apis/host" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/compute" +) + +func (llm *SLLM) GetDetailsProbedModels(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + mdlInfos, err := llm.getProbedInstantModelsExt(ctx, userCred) + if err != nil { + return nil, errors.Wrap(err, "getProbedPackagesExt") + } + return jsonutils.Marshal(mdlInfos), nil +} + +func (llm *SLLM) PerformSaveInstantModel( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input api.LLMSaveInstantModelInput, +) (jsonutils.JSONObject, error) { + if llm.Status != api.LLM_STATUS_RUNNING { + return nil, httperrors.NewInvalidStatusError("LLM is not running") + } + + mdlInfos, err := llm.getProbedInstantModelsExt(ctx, userCred, input.ModelId) + if err != nil { + return nil, errors.Wrap(err, "getProbedPackagesExt") + } + + mdlInfo, ok := mdlInfos[input.ModelId] + if !ok { + return nil, httperrors.NewBadRequestError("ModelId %s not found", input.ModelId) + } + + mountDirs, err := llm.detectModelPaths(ctx, userCred, mdlInfo) + if err != nil { + return nil, errors.Wrap(err, "detectModelPaths") + } + + if len(input.ImageName) == 0 { + input.ImageName = fmt.Sprintf("%s-%s", mdlInfo.Name+":"+mdlInfo.Tag, time.Now().Format("060102")) + } + + var ownerId mcclient.IIdentityProvider + if len(input.TenantId) > 0 { + domainId := input.ProjectDomainId + if len(domainId) == 0 { + domainId = userCred.GetProjectDomainId() + } else { + domain, err := db.TenantCacheManager.FetchDomainByIdOrName(ctx, domainId) + if err != nil { + return nil, errors.Wrap(err, "TenantCache.FetchDomainByIdOrName") + } + domainId = domain.GetId() + } + tenant, err := db.TenantCacheManager.FetchTenantByIdOrNameInDomain(ctx, input.TenantId, domainId) + if err != nil { + return nil, errors.Wrap(err, "TenantCache.FetchById") + } + ownerId = &db.SOwnerId{ + DomainId: domainId, + Domain: tenant.Domain, + ProjectId: tenant.Id, + Project: tenant.Name, + } + } else { + ownerId = userCred + } + + input.ProjectId = ownerId.GetProjectId() + input.ProjectDomainId = ownerId.GetProjectDomainId() + + drv := llm.GetLLMContainerDriver() + instantModelCreateInput := api.InstantModelCreateInput{ + LLMType: drv.GetType(), + ModelId: mdlInfo.ModelId, + ModelName: mdlInfo.Name, + Tag: mdlInfo.Tag, + Mounts: mountDirs, + } + instantModelCreateInput.Name = input.ImageName + log.Debugf("instantModelCreateInput: %s", jsonutils.Marshal(instantModelCreateInput)) + + instantMdlObj, err := db.DoCreate(GetInstantModelManager(), ctx, userCred, nil, jsonutils.Marshal(instantModelCreateInput), ownerId) + if err != nil { + return nil, errors.Wrap(err, "GetInstantModelManager.DoCreate") + } + + instantMdl := instantMdlObj.(*SInstantModel) + + input.InstantModelId = instantMdl.Id + + _, err = llm.StartSaveModelImageTask(ctx, userCred, input) + if err != nil { + return nil, errors.Wrap(err, "StartSaveAppImageTask") + } + + return jsonutils.Marshal(instantMdl), nil +} + +func (llm *SLLM) DoSaveModelImage(ctx context.Context, userCred mcclient.TokenCredential, session *mcclient.ClientSession, input api.LLMSaveInstantModelInput) error { + llm.SetStatus(ctx, userCred, api.LLM_STATUS_SAVING_MODEL, "DoSaveModelImage") + + instantModelObj, err := GetInstantModelManager().FetchById(input.InstantModelId) + if err != nil { + return errors.Wrap(err, "GetInstantModelManager.FetchById") + } + instantModel := instantModelObj.(*SInstantModel) + + drv := llm.GetLLMContainerDriver() + prefix, saveDirs, err := drv.GetSaveDirectories(instantModel) + if err != nil { + return errors.Wrap(err, "GetSaveDirectories") + } + + saveImageInput := computeapi.ContainerSaveVolumeMountToImageInput{ + GenerateName: input.ImageName, + Notes: fmt.Sprintf("instance model image for %s(%s)", input.ModelId, instantModel.ModelName+":"+instantModel.Tag), + Index: 0, + Dirs: saveDirs, + UsedByPostOverlay: true, + DirPrefix: prefix, + } + + lc, err := llm.GetLLMContainer() + if err != nil { + return errors.Wrap(err, "GetLLMContainer") + } + + result, err := compute.Containers.PerformAction(session, lc.CmpId, "save-volume-mount-image", jsonutils.Marshal(saveImageInput)) + if err != nil { + return errors.Wrap(err, "compute.Containers.PerformAction") + } + log.Debugf("container save-volume-mount-image result: %s", result) + saveImageOutput := hostapi.ContainerSaveVolumeMountToImageInput{} + err = result.Unmarshal(&saveImageOutput) + if err != nil { + return errors.Wrap(err, "save-volume-mount-image.result.Unmarshal") + } + + err = instantModel.saveImageId(ctx, userCred, saveImageOutput.ImageId) + if err != nil { + return errors.Wrap(err, "saveImageId") + } + + return nil +} + +func (llm *SLLM) StartSaveModelImageTask(ctx context.Context, userCred mcclient.TokenCredential, input api.LLMSaveInstantModelInput) (*taskman.STask, error) { + llm.SetStatus(ctx, userCred, api.LLM_STATUS_START_SAVE_MODEL, "StartSaveModelImageTask") + + params := jsonutils.Marshal(input) + task, err := taskman.TaskManager.NewTask(ctx, "LLMStartSaveModelImageTask", llm, userCred, params.(*jsonutils.JSONDict), "", "") + if err != nil { + return nil, errors.Wrap(err, "taskman.TaskManager.NewTask") + } + err = task.ScheduleRun(nil) + if err != nil { + return nil, errors.Wrap(err, "task.ScheduleRun") + } + return task, nil +} + +func (llm *SLLM) detectModelPaths(ctx context.Context, userCred mcclient.TokenCredential, pkgInfo api.LLMInternalInstantMdlInfo) ([]string, error) { + return llm.GetLLMContainerDriver().DetectModelPaths(ctx, userCred, llm, pkgInfo) +} diff --git a/pkg/llm/models/llm_model.go b/pkg/llm/models/llm_sku.go similarity index 65% rename from pkg/llm/models/llm_model.go rename to pkg/llm/models/llm_sku.go index 6450bae80b..9895040b8c 100644 --- a/pkg/llm/models/llm_model.go +++ b/pkg/llm/models/llm_sku.go @@ -18,33 +18,33 @@ import ( ) func init() { - GetLLMModelManager() + GetLLMSkuManager() } -var llmModelManager *SLLMModelManager +var llmSkuManager *SLLMSkuManager -func GetLLMModelManager() *SLLMModelManager { - if llmModelManager != nil { - return llmModelManager +func GetLLMSkuManager() *SLLMSkuManager { + if llmSkuManager != nil { + return llmSkuManager } - llmModelManager = &SLLMModelManager{ - SLLMModelBaseManager: NewSLLMModelBaseManager( - SLLMModel{}, - "llm_models_tbl", - "llm_model", - "llm_models", + llmSkuManager = &SLLMSkuManager{ + SLLMSkuBaseManager: NewSLLMSkuBaseManager( + SLLMSku{}, + "llm_skus_tbl", + "llm_sku", + "llm_skus", ), } - llmModelManager.SetVirtualObject(llmModelManager) - return llmModelManager + llmSkuManager.SetVirtualObject(llmSkuManager) + return llmSkuManager } -type SLLMModelManager struct { - SLLMModelBaseManager +type SLLMSkuManager struct { + SLLMSkuBaseManager } -type SLLMModel struct { - SLLMModelBase +type SLLMSku struct { + SLLMSkuBase // SMountedAppsResource LLMImageId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` @@ -52,16 +52,16 @@ type SLLMModel struct { LLMModelName string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` } -func (man *SLLMModelManager) ListItemFilter( +func (man *SLLMSkuManager) ListItemFilter( ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, - input api.LLMModelListInput, + input api.LLMSkuListInput, ) (*sqlchemy.SQuery, error) { var err error - q, err = man.SLLMModelBaseManager.ListItemFilter(ctx, q, userCred, input.SharableVirtualResourceListInput) + q, err = man.SLLMSkuBaseManager.ListItemFilter(ctx, q, userCred, input.SharableVirtualResourceListInput) if err != nil { - return nil, errors.Wrapf(err, "SLLMModelBaseManager.ListItemFilter") + return nil, errors.Wrapf(err, "SLLMSkuBaseManager.ListItemFilter") } if len(input.LLMType) > 0 { q = q.Equals("llm_type", input.LLMType) @@ -73,19 +73,19 @@ func (man *SLLMModelManager) ListItemFilter( return q, nil } -func (manager *SLLMModelManager) FetchCustomizeColumns( +func (manager *SLLMSkuManager) FetchCustomizeColumns( ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, objs []interface{}, fields stringutils2.SSortedStrings, isList bool, -) []api.LLMModelDetails { +) []api.LLMSkuDetails { // skuIds := []string{} imageIds := []string{} // templateIds := []string{} - skus := []SLLMModel{} + skus := []SLLMSku{} jsonutils.Update(&skus, objs) virows := manager.SSharableVirtualResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) for _, sku := range skus { @@ -104,7 +104,7 @@ func (manager *SLLMModelManager) FetchCustomizeColumns( // LLMCapacity int // }{} // q.All(&details) - res := make([]api.LLMModelDetails, len(objs)) + res := make([]api.LLMSkuDetails, len(objs)) for i := range skus { res[i].SharableVirtualResourceDetails = virows[i] // for _, v := range details { @@ -146,11 +146,11 @@ func (manager *SLLMModelManager) FetchCustomizeColumns( return res } -func (man *SLLMModelManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.LLMModelCreateInput) (*api.LLMModelCreateInput, error) { +func (man *SLLMSkuManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.LLMSkuCreateInput) (*api.LLMSkuCreateInput, error) { var err error - input.LLMModelBaseCreateInput, err = man.SLLMModelBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.LLMModelBaseCreateInput) + input.LLMSKuBaseCreateInput, err = man.SLLMSkuBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.LLMSKuBaseCreateInput) if err != nil { - return nil, errors.Wrap(err, "SLLMModelBaseManager.ValidateCreateData") + return nil, errors.Wrap(err, "SLLMSkuBaseManager.ValidateCreateData") } if !api.IsLLMContainerType(input.LLMType) { return input, errors.Wrap(httperrors.ErrInputParameter, "llm_type must be one of "+strings.Join(api.LLM_CONTAINER_TYPES.List(), ",")) @@ -165,15 +165,15 @@ func (man *SLLMModelManager) ValidateCreateData(ctx context.Context, userCred mc return input, nil } -func (model *SLLMModel) GetLLMContainerDriver() ILLMContainerDriver { - return GetLLMContainerDriver(api.LLMContainerType(model.LLMType)) +func (sku *SLLMSku) GetLLMContainerDriver() ILLMContainerDriver { + return GetLLMContainerDriver(api.LLMContainerType(sku.LLMType)) } -func (model *SLLMModel) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.LLMModelUpdateInput) (api.LLMModelUpdateInput, error) { +func (sku *SLLMSku) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.LLMSkuUpdateInput) (api.LLMSkuUpdateInput, error) { var err error - input.LLMModelBaseUpdateInput, err = model.SLLMModelBase.ValidateUpdateData(ctx, userCred, query, input.LLMModelBaseUpdateInput) + input.LLMSkuBaseUpdateInput, err = sku.SLLMSkuBase.ValidateUpdateData(ctx, userCred, query, input.LLMSkuBaseUpdateInput) if err != nil { - return input, errors.Wrap(err, "validate LLMModelBaseUpdateInput") + return input, errors.Wrap(err, "validate LLMSkuBaseUpdateInput") } if input.LLMImageId != "" { @@ -187,13 +187,13 @@ func (model *SLLMModel) ValidateUpdateData(ctx context.Context, userCred mcclien return input, nil } -func (model *SLLMModel) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error { - count, err := GetLLMManager().Query().Equals("llm_model_id", model.Id).CountWithError() +func (sku *SLLMSku) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error { + count, err := GetLLMManager().Query().Equals("llm_sku_id", sku.Id).CountWithError() if nil != err { return errors.Wrap(err, "fetch llm") } if count > 0 { - return errors.Wrap(errors.ErrNotSupported, "This model is currently in use") + return errors.Wrap(errors.ErrNotSupported, "This sku is currently in use") } return nil } diff --git a/pkg/llm/models/mounted_models_resources.go b/pkg/llm/models/mounted_models_resources.go new file mode 100644 index 0000000000..4178736693 --- /dev/null +++ b/pkg/llm/models/mounted_models_resources.go @@ -0,0 +1,50 @@ +package models + +import ( + "context" + "fmt" + + "yunion.io/x/pkg/errors" + "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/mcclient" + + apis "yunion.io/x/onecloud/pkg/apis/llm" +) + +type SMountedModelsResourceManager struct { +} + +type SMountedModelsResource struct { + MountedModels []string `charset:"utf8" list:"user" update:"user" create:"optional"` +} + +func (manager *SMountedModelsResourceManager) ListItemFilter( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + input apis.MountedModelResourceListInput, +) (*sqlchemy.SQuery, error) { + if len(input.MountedModels) > 0 { + q = q.ContainsAny("mounted_models", input.MountedModels) + } + return q, nil +} + +type MountedModelModelManager interface { + IsPremountedModelName(fullModelName string) (bool, error) +} + +func (manager *SVolumeManager) IsPremountedModelName(fullModelName string) (bool, error) { + return isPremountedModelName(manager, fullModelName) +} + +func isPremountedModelName(manager db.IModelManager, fullModelName string) (bool, error) { + q := manager.Query().Contains("mounted_models", fmt.Sprintf("%q", fullModelName)) + cnt, err := q.CountWithError() + if err != nil { + return false, errors.Wrap(err, "CountWithError") + } + return cnt > 0, nil +} diff --git a/pkg/llm/models/model.go b/pkg/llm/models/sku.go similarity index 80% rename from pkg/llm/models/model.go rename to pkg/llm/models/sku.go index 783de18e32..46a6348a05 100644 --- a/pkg/llm/models/model.go +++ b/pkg/llm/models/sku.go @@ -16,8 +16,8 @@ import ( compute "yunion.io/x/onecloud/pkg/mcclient/modules/compute" ) -func NewSLLMModelBaseManager(dt interface{}, tableName string, keyword string, keywordPlural string) SLLMModelBaseManager { - return SLLMModelBaseManager{ +func NewSLLMSkuBaseManager(dt interface{}, tableName string, keyword string, keywordPlural string) SLLMSkuBaseManager { + return SLLMSkuBaseManager{ SSharableVirtualResourceBaseManager: db.NewSharableVirtualResourceBaseManager( dt, tableName, @@ -27,11 +27,11 @@ func NewSLLMModelBaseManager(dt interface{}, tableName string, keyword string, k } } -type SLLMModelBaseManager struct { +type SLLMSkuBaseManager struct { db.SSharableVirtualResourceBaseManager } -type SLLMModelBase struct { +type SLLMSkuBase struct { db.SSharableVirtualResourceBase BandwidthMb int `nullable:"false" default:"0" create:"optional" list:"user" update:"user"` @@ -48,7 +48,7 @@ type SLLMModelBase struct { NetworkId string `charset:"utf8" nullable:"true" list:"user" update:"user" create:"optional"` } -func (man *SLLMModelBaseManager) ListItemFilter( +func (man *SLLMSkuBaseManager) ListItemFilter( ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, @@ -62,7 +62,7 @@ func (man *SLLMModelBaseManager) ListItemFilter( return q, nil } -func (man *SLLMModelBaseManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.LLMModelBaseCreateInput) (api.LLMModelBaseCreateInput, error) { +func (man *SLLMSkuBaseManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.LLMSKuBaseCreateInput) (api.LLMSKuBaseCreateInput, error) { var err error input.SharableVirtualResourceCreateInput, err = man.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.SharableVirtualResourceCreateInput) if err != nil { @@ -78,7 +78,7 @@ func (man *SLLMModelBaseManager) ValidateCreateData(ctx context.Context, userCre return input, errors.Wrap(httperrors.ErrInputParameter, "volumes cannot be empty") } - if !api.IsLLMModelBaseNetworkType(input.NetworkType) { + if !api.IsLLMSkuBaseNetworkType(input.NetworkType) { return input, errors.Wrapf(httperrors.ErrInputParameter, "invalid network type %s", input.NetworkType) } @@ -96,15 +96,15 @@ func (man *SLLMModelBaseManager) ValidateCreateData(ctx context.Context, userCre return input, nil } -func (modelBase *SLLMModelBase) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.LLMModelBaseUpdateInput) (api.LLMModelBaseUpdateInput, error) { +func (skuBase *SLLMSkuBase) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.LLMSkuBaseUpdateInput) (api.LLMSkuBaseUpdateInput, error) { var err error - input.SharableVirtualResourceBaseUpdateInput, err = modelBase.SSharableVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, input.SharableVirtualResourceBaseUpdateInput) + input.SharableVirtualResourceBaseUpdateInput, err = skuBase.SSharableVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, input.SharableVirtualResourceBaseUpdateInput) if err != nil { return input, errors.Wrap(err, "validate SharableVirtualResourceBaseUpdateInput") } volumes := []api.Volume{} - if err := jsonutils.Marshal(modelBase.Volumes).Unmarshal(&volumes); err != nil { + if err := jsonutils.Marshal(skuBase.Volumes).Unmarshal(&volumes); err != nil { return input, errors.Wrapf(err, "Unmarshal Volumes") } for i, volume := range volumes { @@ -130,7 +130,7 @@ func (modelBase *SLLMModelBase) ValidateUpdateData(ctx context.Context, userCred } input.Volumes = (*api.Volumes)(&volumes) - if input.NetworkType != nil && !api.IsLLMModelBaseNetworkType(*input.NetworkType) { + if input.NetworkType != nil && !api.IsLLMSkuBaseNetworkType(*input.NetworkType) { return input, errors.Wrapf(httperrors.ErrInputParameter, "invalid network type %s", *input.NetworkType) } diff --git a/pkg/llm/models/volume.go b/pkg/llm/models/volume.go index ca5d122e02..e559e80413 100644 --- a/pkg/llm/models/volume.go +++ b/pkg/llm/models/volume.go @@ -2,12 +2,26 @@ package models import ( "context" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/httputils" + "yunion.io/x/pkg/utils" commonapi "yunion.io/x/onecloud/pkg/apis" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + imageapi "yunion.io/x/onecloud/pkg/apis/image" api "yunion.io/x/onecloud/pkg/apis/llm" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/llm/options" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules/compute" + "yunion.io/x/onecloud/pkg/mcclient/modules/image" ) func init() { @@ -34,12 +48,12 @@ func GetVolumeManager() *SVolumeManager { type SVolumeManager struct { db.SVirtualResourceBaseManager - // SMountedAppsResourceManager + SMountedModelsResourceManager } type SVolume struct { db.SVirtualResourceBase - // SMountedAppsResource + SMountedModelsResource LLMId string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"admin_optional" update:"user"` // 存储类型 @@ -65,3 +79,65 @@ func (volume *SVolume) StartDeleteTask(ctx context.Context, userCred mcclient.To func (volume *SVolume) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error { return volume.SVirtualResourceBase.Delete(ctx, userCred) } + +func fetchImage(ctx context.Context, userCred mcclient.TokenCredential, imageId string) (*imageapi.ImageDetails, error) { + s := auth.GetSession(ctx, userCred, options.Options.Region) + imgObj, err := image.Images.Get(s, imageId, nil) + if err != nil { + return nil, errors.Wrapf(err, "Image.Get %s", imageId) + } + img := imageapi.ImageDetails{} + err = imgObj.Unmarshal(&img) + if err != nil { + return nil, errors.Wrap(err, "Unmarshal") + } + return &img, nil +} + +func (volume *SVolume) UpdateMountedModelFullNames(mountModels []string) error { + _, err := db.Update(volume, func() error { + volume.MountedModels = mountModels + return nil + }) + if err != nil { + return errors.Wrap(err, "update volume mounted_apps") + } + return nil +} + +func (volume *SVolume) GetDisk(ctx context.Context) (*computeapi.DiskDetails, error) { + if len(volume.SvrId) == 0 { + return nil, errors.ErrInvalidStatus + } + s := auth.GetAdminSession(ctx, "") + disk := computeapi.DiskDetails{} + resp, err := compute.Disks.GetById(s, volume.SvrId, jsonutils.Marshal(map[string]interface{}{ + "scope": "max", + })) + if err != nil { + if httputils.ErrorCode(err) == 404 { + return nil, errors.Wrapf(errors.ErrNotFound, "GetById %s", volume.SvrId) + } + return nil, errors.Wrap(err, "fetch disk") + } + resp.Unmarshal(&disk) + return &disk, nil +} + +func (volume *SVolume) WaitDiskStatus(ctx context.Context, userCred mcclient.TokenCredential, targetStatus []string, timeoutSecs int) (*computeapi.DiskDetails, error) { + expire := time.Now().Add(time.Second * time.Duration(timeoutSecs)) + for time.Now().Before(expire) { + disk, err := volume.GetDisk(ctx) + if err != nil { + return nil, errors.Wrap(err, "GetDisk") + } + if utils.IsInArray(disk.Status, targetStatus) { + return disk, nil + } + if strings.Contains(disk.Status, "fail") { + return nil, errors.Wrap(errors.ErrInvalidStatus, disk.Status) + } + time.Sleep(2 * time.Second) + } + return nil, errors.Wrapf(httperrors.ErrTimeout, "wait disk status %s timeout", targetStatus) +} diff --git a/pkg/llm/options/option.go b/pkg/llm/options/option.go index 98a748dd74..f9e24d8d84 100644 --- a/pkg/llm/options/option.go +++ b/pkg/llm/options/option.go @@ -20,6 +20,9 @@ type LLMOptions struct { common_options.CommonOptions common_options.DBOptions + InstantModelSyncTaskWorkerCount int `help:"backup task worker count" default:"128"` + ModelSyncTaskWaitSecs int `help:"model sync task wait seconds" default:"30"` + StartTaskWorkerCount int `help:"start task worker count" default:"128"` } diff --git a/pkg/llm/service/handler.go b/pkg/llm/service/handler.go index 55ee3add22..665843c1e8 100644 --- a/pkg/llm/service/handler.go +++ b/pkg/llm/service/handler.go @@ -32,13 +32,15 @@ func InitHandlers(app *appsrv.Application) { db.Metadata, models.GetLLMImageManager(), - models.GetLLMModelManager(), - models.GetDifyModelManager(), + models.GetLLMSkuManager(), + models.GetDifySkuManager(), models.GetVolumeManager(), models.GetAccessInfoManager(), models.GetLLMContainerManager(), models.GetLLMManager(), models.GetDifyManager(), + models.GetInstantModelManager(), + models.GetLLMInstantModelManager(), } { db.RegisterModelManager(manager) handler := db.NewModelHandler(manager) diff --git a/pkg/llm/service/service.go b/pkg/llm/service/service.go index 9cdd8c7573..0d59d4ec18 100644 --- a/pkg/llm/service/service.go +++ b/pkg/llm/service/service.go @@ -15,6 +15,7 @@ import ( "yunion.io/x/onecloud/pkg/llm/models" "yunion.io/x/onecloud/pkg/llm/options" _ "yunion.io/x/onecloud/pkg/llm/tasks" + llmTask "yunion.io/x/onecloud/pkg/llm/tasks/llm" ) // StartService the main service starts @@ -25,6 +26,7 @@ func StartService() { baseOpts := &opts.BaseOptions common_options.ParseOptions(opts, os.Args, "llm.conf", api.SERVICE_TYPE) + llmTask.InitInstantModelSyncTaskManager() app_common.InitAuth(commonOpts, func() { log.Infof("Auth complete!!") }) diff --git a/pkg/llm/tasks/llm/llm_instant_models_sync_task.go b/pkg/llm/tasks/llm/llm_instant_models_sync_task.go new file mode 100644 index 0000000000..4e771e97c7 --- /dev/null +++ b/pkg/llm/tasks/llm/llm_instant_models_sync_task.go @@ -0,0 +1,201 @@ +package llm + +import ( + "context" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + apis "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/appsrv" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + models "yunion.io/x/onecloud/pkg/llm/models" + options "yunion.io/x/onecloud/pkg/llm/options" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +var ( + instantModelSyncTaskWorkerMan *appsrv.SHashedWorkerManager +) + +type LLMInstantModelsSyncTask struct { + taskman.STask +} + +func InitInstantModelSyncTaskManager() { + instantModelSyncTaskWorkerMan = appsrv.NewHashWorkerManager("InstantModelSyncTaskManager", options.Options.InstantModelSyncTaskWorkerCount, 1, 1024, true) + taskman.RegisterTaskAndHashedWorkerManager(LLMInstantModelsSyncTask{}, instantModelSyncTaskWorkerMan) +} + +func (task *LLMInstantModelsSyncTask) taskFailed(ctx context.Context, llm *models.SLLM, err string) { + defer llm.ClearPendingInstantModelQuota(task.Id) + input := apis.LLMSyncModelTaskInput{} + task.Params.Unmarshal(&input) + + db.OpsLog.LogEvent(llm, db.ACT_SYNC_CONF_FAIL, err, task.UserCred) + logclient.AddActionLogWithStartable(task, llm, logclient.ACT_SYNC_CONF, err, task.UserCred, false) + task.SetStageFailed(ctx, jsonutils.NewString(err)) +} + +func (task *LLMInstantModelsSyncTask) taskComplete(ctx context.Context, llm *models.SLLM) { + defer llm.ClearPendingInstantModelQuota(task.Id) + task.SetStageComplete(ctx, nil) +} + +func (task *LLMInstantModelsSyncTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + llm := obj.(*models.SLLM) + input := apis.LLMSyncModelTaskInput{} + err := task.Params.Unmarshal(&input) + if err != nil { + task.taskFailed(ctx, llm, err.Error()) + return + } + + removedModelIds, unmountOverlays, err := llm.RequestUnmountModel(ctx, task.UserCred, input) + if err != nil { + task.taskFailed(ctx, llm, err.Error()) + return + } + if len(removedModelIds) > 0 || len(unmountOverlays) > 0 { + input.UninstallModelIds = removedModelIds + task.SetStage("OnModelsUnmountComplete", jsonutils.Marshal(input).(*jsonutils.JSONDict)) + if len(unmountOverlays) > 0 { + // try unmount post_overlay + s := auth.GetSession(ctx, task.GetUserCred(), options.Options.Region) + err := s.WithTaskCallback(task.GetId(), func() error { + return llm.TryContainerUnmountPaths(ctx, task.UserCred, s, unmountOverlays, 7200) + }) + if err != nil { + task.OnModelsUnmountCompleteFailed(ctx, llm, jsonutils.NewString(errors.Wrap(err, "TryContainerUnmountPaths").Error())) + } + } + } else { + task.OnModelsUnmountComplete(ctx, llm, nil) + } +} + +func (task *LLMInstantModelsSyncTask) OnModelsUnmountComplete(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + llm := obj.(*models.SLLM) + input := apis.LLMSyncModelTaskInput{} + err := task.Params.Unmarshal(&input) + if err != nil { + task.taskFailed(ctx, llm, err.Error()) + return + } + + if len(input.UninstallModelIds) > 0 { + // did uninstall models, clear the flags + log.Debugf("uninstall_models %s", jsonutils.Marshal(input.UninstallModelIds).PrettyString()) + err := llm.MarkInstantModelsUnmounted(ctx, task.UserCred, input.LLMStatus, input.UninstallModelIds) + if err != nil { + task.taskFailed(ctx, llm, errors.Wrap(err, "MarkModelsUnmounted").Error()) + return + } + } + + mdlIds, installDirs, overlays, err := llm.RequestMountModels(ctx, task.UserCred, input) + if err != nil { + task.taskFailed(ctx, llm, err.Error()) + return + } + log.Debugf("=======RequestMountModels: %s, overlays: %s", jsonutils.Marshal(mdlIds).PrettyString(), jsonutils.Marshal(overlays).PrettyString()) + if len(mdlIds) == 0 { + task.taskComplete(ctx, llm) + return + } + + input.InstallModelIds = mdlIds + input.InstallDirs = installDirs + if len(overlays) > 0 { + task.SetStage("OnModelsMountComplete", jsonutils.Marshal(input).(*jsonutils.JSONDict)) + + s := auth.GetSession(ctx, task.GetUserCred(), options.Options.Region) + err := s.WithTaskCallback(task.GetId(), func() error { + return llm.TryContainerMountPaths(ctx, task.UserCred, s, overlays, 7200) + }) + if err != nil { + task.OnModelsUnmountCompleteFailed(ctx, llm, jsonutils.NewString(errors.Wrap(err, "TryContainerMountPaths").Error())) + } + } else { + task.OnModelsMountComplete(ctx, llm, nil) + } +} + +func (task *LLMInstantModelsSyncTask) OnModelsUnmountCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) { + llm := obj.(*models.SLLM) + task.taskFailed(ctx, llm, err.String()) +} + +func (task *LLMInstantModelsSyncTask) OnModelsMountComplete(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + llm := obj.(*models.SLLM) + input := apis.LLMSyncModelTaskInput{} + err := task.Params.Unmarshal(&input) + if err != nil { + task.taskFailed(ctx, llm, err.Error()) + return + } + + log.Debugf("install_models %s", jsonutils.Marshal(input.InstallModelIds).PrettyString()) + err = llm.MarkInstantModelsMounted(ctx, task.UserCred, input.LLMStatus, input.InstallModelIds) + if err != nil { + task.taskFailed(ctx, llm, errors.Wrap(err, "MarkAppsMounted").Error()) + return + } + + if input.LLMStatus == apis.LLM_STATUS_RUNNING { + err := llm.InstallInstantModels(ctx, task.UserCred, input.InstallDirs, input.InstallModelIds) + if err != nil { + task.taskFailed(ctx, llm, errors.Wrap(err, "llm.InstallInstantModels").Error()) + return + } + task.SetStage("OnWaitModelsMountedComplete", jsonutils.Marshal(input).(*jsonutils.JSONDict)) + ModelSyncTaskRun(task, llm.Id, func() (jsonutils.JSONObject, error) { + tried := 0 + const intvSecs = 2 + waitSecs := options.Options.ModelSyncTaskWaitSecs + var errs []error + succ := false + for tried < waitSecs/intvSecs && !succ { + tried++ + time.Sleep(intvSecs * time.Second) + if err := llm.EnsureInstantModelsInstalled(ctx, task.UserCred, input.InstallModelIds); err != nil { + errs = append(errs, err) + } else { + succ = true + } + } + if !succ && len(errs) > 0 { + return nil, errors.NewAggregate(errs) + } + err := llm.RefreshInstantModels(ctx, task.UserCred, true) + if err != nil { + return nil, errors.Wrap(err, "RefreshInstantModels") + } + return nil, nil + }) + } else { + task.taskComplete(ctx, llm) + } +} + +func (task *LLMInstantModelsSyncTask) OnModelsMountCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) { + llm := obj.(*models.SLLM) + task.taskFailed(ctx, llm, err.String()) + + // sync status to clear failed status of container + llm.StartSyncStatusTask(ctx, task.UserCred, "") +} + +func (task *LLMInstantModelsSyncTask) OnWaitModelsMountedComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + llm := obj.(*models.SLLM) + task.taskComplete(ctx, llm) +} + +func (task *LLMInstantModelsSyncTask) OnWaitModelsMountedCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) { + llm := obj.(*models.SLLM) + task.taskFailed(ctx, llm, err.String()) +} diff --git a/pkg/llm/tasks/llm/llm_start_save_model_image_task.go b/pkg/llm/tasks/llm/llm_start_save_model_image_task.go new file mode 100644 index 0000000000..3e82d29f63 --- /dev/null +++ b/pkg/llm/tasks/llm/llm_start_save_model_image_task.go @@ -0,0 +1,109 @@ +package llm + +import ( + "context" + + "yunion.io/x/jsonutils" + + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/llm/models" + "yunion.io/x/onecloud/pkg/llm/options" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type LLMStartSaveModelImageTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(LLMStartSaveModelImageTask{}) +} + +func (task *LLMStartSaveModelImageTask) taskFailed(ctx context.Context, llm *models.SLLM, err string) { + llm.SetStatus(ctx, task.UserCred, api.LLM_STATUS_SAVE_MODEL_FAILED, err) + db.OpsLog.LogEvent(llm, db.ACT_SAVE_FAIL, err, task.UserCred) + logclient.AddActionLogWithStartable(task, llm, logclient.ACT_SAVE_IMAGE, err, task.UserCred, false) + task.SetStageFailed(ctx, jsonutils.NewString(err)) +} + +func (task *LLMStartSaveModelImageTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + llm := obj.(*models.SLLM) + + // first stop the desktop + task.SetStage("OnStopLLMComplete", nil) + err := llm.StartLLMStopTask(ctx, task.UserCred, task.GetTaskId()) + if err != nil { + task.taskFailed(ctx, llm, err.Error()) + return + } +} + +func (task *LLMStartSaveModelImageTask) OnStopLLMCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) { + llm := obj.(*models.SLLM) + task.taskFailed(ctx, llm, err.String()) +} + +func (task *LLMStartSaveModelImageTask) OnStopLLMComplete(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + llm := obj.(*models.SLLM) + + input := api.LLMSaveInstantModelInput{} + err := task.Params.Unmarshal(&input) + if err != nil { + task.taskFailed(ctx, llm, err.Error()) + return + } + + task.SetStage("OnSaveModelImageComplete", nil) + s := auth.GetSession(ctx, task.GetUserCred(), options.Options.Region) + s.WithTaskCallback(task.GetId(), func() error { + return llm.DoSaveModelImage(ctx, task.UserCred, s, input) + }) +} + +func (task *LLMStartSaveModelImageTask) OnSaveModelImageComplete(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + llm := obj.(*models.SLLM) + + input := api.LLMSaveInstantModelInput{} + + err := task.Params.Unmarshal(&input) + if err != nil { + task.taskFailed(ctx, llm, err.Error()) + return + } + + instantModelObj, err := models.GetInstantModelManager().FetchById(input.InstantModelId) + if err != nil { + task.taskFailed(ctx, llm, err.Error()) + return + } + instantModel := instantModelObj.(*models.SInstantModel) + + _, err = instantModel.PerformSyncstatus(ctx, task.GetUserCred(), nil, api.InstantModelSyncstatusInput{}) + if err != nil { + task.taskFailed(ctx, llm, err.Error()) + } + + db.OpsLog.LogEvent(llm, db.ACT_SAVE, instantModel.GetShortDesc(ctx), task.UserCred) + logclient.AddActionLogWithStartable(task, llm, logclient.ACT_SAVE_IMAGE, instantModel.GetShortDesc(ctx), task.UserCred, false) + db.OpsLog.LogEvent(instantModel, db.ACT_SAVE, llm.GetShortDesc(ctx), task.UserCred) + logclient.AddActionLogWithStartable(task, instantModel, logclient.ACT_SAVE_IMAGE, llm.GetShortDesc(ctx), task.UserCred, true) + + task.SetStageComplete(ctx, nil) + + // if input.AutoRestart { + // llm.StartRestartTask(ctx, task.UserCred, api.DesktopRestartTaskInput{ + // DesktopId: llm.Id, + // DesktopStatus: api.LLM_STATUS_READY, + // }, "") + // } else { + // llm.SetStatus(ctx, task.UserCred, api.LLM_STATUS_READY, "OnSaveModelImageComplete") + // } +} + +func (task *LLMStartSaveModelImageTask) OnSaveModelImageCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) { + d := obj.(*models.SLLM) + task.taskFailed(ctx, d, err.String()) +} diff --git a/pkg/llm/tasks/llm/llm_sync_status_task.go b/pkg/llm/tasks/llm/llm_sync_status_task.go new file mode 100644 index 0000000000..45848b2ba6 --- /dev/null +++ b/pkg/llm/tasks/llm/llm_sync_status_task.go @@ -0,0 +1,143 @@ +package llm + +import ( + "context" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + apis "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/llm/models" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules/compute" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type LLMSyncStatusTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(LLMSyncStatusTask{}) +} + +func (task *LLMSyncStatusTask) setLLMStatus(ctx context.Context, llm *models.SLLM, status string, reason string) { + if !task.HasParentTask() { + llm.SetStatus(ctx, task.UserCred, status, reason) + } +} + +func (task *LLMSyncStatusTask) taskFailed(ctx context.Context, llm *models.SLLM, err string) { + task.setLLMStatus(ctx, llm, computeapi.VM_SYNC_FAIL, err) + db.OpsLog.LogEvent(llm, db.ACT_SYNC_STATUS, err, task.UserCred) + logclient.AddActionLogWithStartable(task, llm, logclient.ACT_SYNC_STATUS, err, task.UserCred, false) + // llm.NotifyRequest(ctx, task.GetUserCred(), notify.ActionStart, nil, false) + task.SetStageFailed(ctx, jsonutils.NewString(err)) +} + +func (task *LLMSyncStatusTask) taskComplete(ctx context.Context, _ *models.SLLM) { + // phone.SyncStatus(ctx, task.UserCred) + task.SetStageComplete(ctx, nil) +} + +func (task *LLMSyncStatusTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + llm := obj.(*models.SLLM) + + task.setLLMStatus(ctx, llm, apis.LLM_STATUS_SYNCSTATUS, "LLMSyncStatusTask.OnInit") + + s := auth.GetSession(ctx, task.UserCred, "") + _, err := compute.Servers.PerformAction(s, llm.SvrId, "syncstatus", nil) + if err != nil { + task.taskFailed(ctx, llm, err.Error()) + return + } + task.SetStage("OnSyncStatusComplete", nil) + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + _, err := llm.WaitServerStatus(ctx, task.UserCred, []string{ + computeapi.VM_RUNNING, + computeapi.VM_READY, + computeapi.VM_UNKNOWN, + computeapi.POD_STATUS_CRASH_LOOP_BACK_OFF, + computeapi.POD_STATUS_CONTAINER_EXITED, + computeapi.POD_STATUS_UPLOADING_STATUS_FAILED, + }, 1800) + if err != nil { + return nil, errors.Wrap(err, "WaitServerStatus") + } + + time.Sleep(1 * time.Second) + + // 有可能 server 变为 ready 之后 又变为 sync_container_status + srv, err := llm.WaitServerStatus(ctx, task.UserCred, []string{ + computeapi.VM_RUNNING, + computeapi.VM_READY, + computeapi.VM_UNKNOWN, + computeapi.POD_STATUS_CRASH_LOOP_BACK_OFF, + computeapi.POD_STATUS_CONTAINER_EXITED, + computeapi.POD_STATUS_UPLOADING_STATUS_FAILED, + }, 1800) + if err != nil { + return nil, errors.Wrap(err, "WaitServerStatus") + } + + if utils.IsInArray(srv.Status, []string{ + computeapi.POD_STATUS_CRASH_LOOP_BACK_OFF, + computeapi.POD_STATUS_CONTAINER_EXITED, + }) { + params := computeapi.ServerStopInput{ + IsForce: true, + TimeoutSecs: 10, + } + _, err := compute.Servers.PerformAction(s, llm.SvrId, "stop", jsonutils.Marshal(params)) + if err != nil { + return nil, errors.Wrap(err, "ServerStop") + } + srv, err := llm.WaitServerStatus(ctx, task.UserCred, []string{ + computeapi.VM_READY, + }, 1800) + if err != nil { + return nil, errors.Wrap(err, "WaitServerStatus") + } + task.setLLMStatus(ctx, llm, srv.Status, "stop server") + } else { + task.setLLMStatus(ctx, llm, srv.Status, "WaitServerStatus") + } + + volume, _ := llm.GetVolume() + if volume != nil { + disk, err := volume.GetDisk(ctx) + if err != nil { + task.setLLMStatus(ctx, llm, computeapi.VM_DISK_RESET_FAIL, errors.Wrap(err, "GetDisk").Error()) + return nil, errors.Wrap(err, "GetDisk") + } + if disk.Status != computeapi.DISK_READY { + // do disk syncstatus + _, err := compute.Disks.PerformAction(s, disk.Id, "syncstatus", nil) + if err != nil { + return nil, errors.Wrap(err, "disk perform action syncstatus") + } + _, err = volume.WaitDiskStatus(ctx, task.UserCred, []string{computeapi.DISK_READY}, 1800) + if err != nil { + return nil, errors.Wrap(err, "volume.WaitDiskStatus") + } + } + } + + return nil, nil + }) +} + +func (task *LLMSyncStatusTask) OnSyncStatusComplete(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + d := obj.(*models.SLLM) + task.taskComplete(ctx, d) +} + +func (task *LLMSyncStatusTask) OnSyncStatusCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) { + d := obj.(*models.SLLM) + task.taskFailed(ctx, d, err.String()) +} diff --git a/pkg/llm/tasks/llm/modelsyncworker.go b/pkg/llm/tasks/llm/modelsyncworker.go new file mode 100644 index 0000000000..5a655addad --- /dev/null +++ b/pkg/llm/tasks/llm/modelsyncworker.go @@ -0,0 +1,11 @@ +package llm + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" +) + +func ModelSyncTaskRun(task taskman.ITask, llmId string, proc func() (jsonutils.JSONObject, error)) { + taskman.LocalTaskRunWithWorkers(task, proc, instantModelSyncTaskWorkerMan.GetWorkerManager(llmId)) +} diff --git a/pkg/mcclient/modules/llm/mod_llm_model.go b/pkg/mcclient/modules/llm/mod_dify_sku.go similarity index 53% rename from pkg/mcclient/modules/llm/mod_llm_model.go rename to pkg/mcclient/modules/llm/mod_dify_sku.go index 3f88b1c84c..aca51d1e8b 100644 --- a/pkg/mcclient/modules/llm/mod_llm_model.go +++ b/pkg/mcclient/modules/llm/mod_dify_sku.go @@ -5,20 +5,20 @@ import ( "yunion.io/x/onecloud/pkg/mcclient/modules" ) -type LLMModelManager struct { +type DifySkuManager struct { modulebase.ResourceManager } var ( - LLMModel LLMModelManager + DifySku DifySkuManager ) func init() { - LLMModel = LLMModelManager{ - ResourceManager: modules.NewLLMManager("llm_model", "llm_models", + DifySku = DifySkuManager{ + ResourceManager: modules.NewLLMManager("dify_sku", "dify_skus", []string{}, []string{}, ), } - modules.Register(&LLMModel) + modules.Register(&DifySku) } diff --git a/pkg/mcclient/modules/llm/mod_instantmodel.go b/pkg/mcclient/modules/llm/mod_instantmodel.go new file mode 100644 index 0000000000..5402f3db70 --- /dev/null +++ b/pkg/mcclient/modules/llm/mod_instantmodel.go @@ -0,0 +1,23 @@ +package llm + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var ( + LLMInstantModel LLMInstantModelManager +) + +func init() { + LLMInstantModel = LLMInstantModelManager{ + modules.NewLLMManager("llm_instant_model", "llm_instant_models", + []string{}, + []string{}), + } + modules.Register(&LLMInstantModel) +} + +type LLMInstantModelManager struct { + modulebase.ResourceManager +} diff --git a/pkg/mcclient/modules/llm/mod_dify_model.go b/pkg/mcclient/modules/llm/mod_llm_sku.go similarity index 52% rename from pkg/mcclient/modules/llm/mod_dify_model.go rename to pkg/mcclient/modules/llm/mod_llm_sku.go index 532e6b8bcb..b4076fb257 100644 --- a/pkg/mcclient/modules/llm/mod_dify_model.go +++ b/pkg/mcclient/modules/llm/mod_llm_sku.go @@ -5,20 +5,20 @@ import ( "yunion.io/x/onecloud/pkg/mcclient/modules" ) -type DifyModelManager struct { +type LLMSkuManager struct { modulebase.ResourceManager } var ( - DifyModel DifyModelManager + LLMSku LLMSkuManager ) func init() { - DifyModel = DifyModelManager{ - ResourceManager: modules.NewLLMManager("dify_model", "dify_models", + LLMSku = LLMSkuManager{ + ResourceManager: modules.NewLLMManager("llm_sku", "llm_skus", []string{}, []string{}, ), } - modules.Register(&DifyModel) + modules.Register(&LLMSku) } diff --git a/pkg/mcclient/options/llm/dify.go b/pkg/mcclient/options/llm/dify.go index a36d1a2a8a..b53427b185 100644 --- a/pkg/mcclient/options/llm/dify.go +++ b/pkg/mcclient/options/llm/dify.go @@ -9,7 +9,7 @@ import ( type DifyListOptions struct { LLMBaseListOptions - DifyModel string `help:"filter by dify model"` + DifySku string `help:"filter by dify sku"` } func (o *DifyListOptions) Params() (jsonutils.JSONObject, error) { @@ -34,7 +34,7 @@ func (o *DifyShowOptions) Params() (jsonutils.JSONObject, error) { type DifyCreateOptions struct { LLMBaseCreateOptions - DIFY_MODEL_ID string `help:"dify model id or name" json:"dify_model_id"` + DIFY_SKU_ID string `help:"dify sku id or name" json:"dify_sku_id"` } func (o *DifyCreateOptions) Params() (jsonutils.JSONObject, error) { diff --git a/pkg/mcclient/options/llm/dify_model.go b/pkg/mcclient/options/llm/dify_model.go deleted file mode 100644 index 0c060e8b5e..0000000000 --- a/pkg/mcclient/options/llm/dify_model.go +++ /dev/null @@ -1,78 +0,0 @@ -package llm - -import ( - "yunion.io/x/jsonutils" - - "yunion.io/x/onecloud/pkg/mcclient/options" -) - -type DifyModelListOptions struct { - options.BaseListOptions -} - -func (o *DifyModelListOptions) Params() (jsonutils.JSONObject, error) { - return options.ListStructToParams(o) -} - -type DifyModelShowOptions struct { - options.BaseShowOptions -} - -func (o *DifyModelShowOptions) Params() (jsonutils.JSONObject, error) { - return options.StructToParams(o) -} - -type DifyModelCreateOptions struct { - LLMModelBaseCreateOptions - - POSTGRES_IMAGE_ID string `json:"postgres_image_id"` - REDIS_IMAGE_ID string `json:"redis_image_id"` - NGINX_IMAGE_ID string `json:"nginx_image_id"` - DIFY_API_IMAGE_ID string `json:"dify_api_image_id"` - DIFY_PLUGIN_IMAGE_ID string `json:"dify_plugin_image_id"` - DIFY_WEB_IMAGE_ID string `json:"dify_web_image_id"` - DIFY_SANDBOX_IMAGE_ID string `json:"dify_sandbox_image_id"` - DIFY_SSRF_IMAGE_ID string `json:"dify_ssrf_image_id"` - DIFY_WEAVIATE_IMAGE_ID string `json:"dify_weaviate_image_id"` -} - -func (o *DifyModelCreateOptions) Params() (jsonutils.JSONObject, error) { - dict := jsonutils.NewDict() - obj := jsonutils.Marshal(o) - obj.Unmarshal(dict) - - o.LLMModelBaseCreateOptions.Params(dict) - return dict, nil -} - -type DifyModelDeleteOptions struct { - options.BaseIdOptions -} - -func (o *DifyModelDeleteOptions) GetId() string { - return o.ID -} - -func (o *DifyModelDeleteOptions) Params() (jsonutils.JSONObject, error) { - return options.StructToParams(o) -} - -type DifyModelUpdateOptions struct { - LLMModelBaseUpdateOptions - - LlmImageId string - LlmModelName string -} - -func (o *DifyModelUpdateOptions) GetId() string { - return o.ID -} - -func (o *DifyModelUpdateOptions) Params() (jsonutils.JSONObject, error) { - dict := jsonutils.NewDict() - obj := jsonutils.Marshal(o) - obj.Unmarshal(dict) - - o.LLMModelBaseUpdateOptions.Params(dict) - return dict, nil -} diff --git a/pkg/mcclient/options/llm/dify_sku.go b/pkg/mcclient/options/llm/dify_sku.go new file mode 100644 index 0000000000..f2ca70279d --- /dev/null +++ b/pkg/mcclient/options/llm/dify_sku.go @@ -0,0 +1,85 @@ +package llm + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +type DifySkuListOptions struct { + options.BaseListOptions +} + +func (o *DifySkuListOptions) Params() (jsonutils.JSONObject, error) { + return options.ListStructToParams(o) +} + +type DifySkuShowOptions struct { + options.BaseShowOptions +} + +func (o *DifySkuShowOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type DifySkuCreateOptions struct { + LLMSkuBaseCreateOptions + + POSTGRES_IMAGE_ID string `json:"postgres_image_id"` + REDIS_IMAGE_ID string `json:"redis_image_id"` + NGINX_IMAGE_ID string `json:"nginx_image_id"` + DIFY_API_IMAGE_ID string `json:"dify_api_image_id"` + DIFY_PLUGIN_IMAGE_ID string `json:"dify_plugin_image_id"` + DIFY_WEB_IMAGE_ID string `json:"dify_web_image_id"` + DIFY_SANDBOX_IMAGE_ID string `json:"dify_sandbox_image_id"` + DIFY_SSRF_IMAGE_ID string `json:"dify_ssrf_image_id"` + DIFY_WEAVIATE_IMAGE_ID string `json:"dify_weaviate_image_id"` +} + +func (o *DifySkuCreateOptions) Params() (jsonutils.JSONObject, error) { + dict := jsonutils.NewDict() + obj := jsonutils.Marshal(o) + obj.Unmarshal(dict) + + o.LLMSkuBaseCreateOptions.Params(dict) + return dict, nil +} + +type DifySkuDeleteOptions struct { + options.BaseIdOptions +} + +func (o *DifySkuDeleteOptions) GetId() string { + return o.ID +} + +func (o *DifySkuDeleteOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type DifySkuUpdateOptions struct { + LLMSkuBaseUpdateOptions + + PostgresImageID string `json:"postgres_image_id"` + RedisImageID string `json:"redis_image_id"` + NginxImageID string `json:"nginx_image_id"` + DifyApiImageID string `json:"dify_api_image_id"` + DifyPluginImageID string `json:"dify_plugin_image_id"` + DifyWebImageID string `json:"dify_web_image_id"` + DifySandboxImageID string `json:"dify_sandbox_image_id"` + DifySsrfImageID string `json:"dify_ssrf_image_id"` + DifyWeaviateImageID string `json:"dify_weaviate_image_id"` +} + +func (o *DifySkuUpdateOptions) GetId() string { + return o.ID +} + +func (o *DifySkuUpdateOptions) Params() (jsonutils.JSONObject, error) { + dict := jsonutils.NewDict() + obj := jsonutils.Marshal(o) + obj.Unmarshal(dict) + + o.LLMSkuBaseUpdateOptions.Params(dict) + return dict, nil +} diff --git a/pkg/mcclient/options/llm/instantmodel.go b/pkg/mcclient/options/llm/instantmodel.go new file mode 100644 index 0000000000..ddf8697755 --- /dev/null +++ b/pkg/mcclient/options/llm/instantmodel.go @@ -0,0 +1,40 @@ +package llm + +import ( + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +type LLMInstantModelListOptions struct { + options.BaseListOptions + + ModelName []string `help:"filter by model name"` + Tag []string `help:"filter by model tag"` +} + +func (o *LLMInstantModelListOptions) Params() (jsonutils.JSONObject, error) { + return options.ListStructToParams(o) +} + +type LLMInstantModelShowOptions struct { + options.BaseShowOptions +} + +func (o *LLMInstantModelShowOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type LLMInstantModelCreateOptions struct { + options.BaseCreateOptions + + MODEL_NAME string `json:"model_name"` + TAG string `json:"tag"` + + ImageId string `json:"image_id"` + + Mounts []string `json:"mounts"` +} + +func (o *LLMInstantModelCreateOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(o), nil +} diff --git a/pkg/mcclient/options/llm/llm.go b/pkg/mcclient/options/llm/llm.go index 26de89572b..f177d75f72 100644 --- a/pkg/mcclient/options/llm/llm.go +++ b/pkg/mcclient/options/llm/llm.go @@ -1,8 +1,12 @@ package llm import ( - "yunion.io/x/jsonutils" + "strings" + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/util/regutils" + + api "yunion.io/x/onecloud/pkg/apis/llm" "yunion.io/x/onecloud/pkg/mcclient/options" ) @@ -21,7 +25,7 @@ type LLMBaseListOptions struct { type LLMListOptions struct { LLMBaseListOptions - LlmModel string `help:"filter by llm model"` + LlmSku string `help:"filter by llm sku"` LlmImage string `help:"filter by llm image"` } @@ -59,7 +63,7 @@ type LLMBaseCreateOptions struct { type LLMCreateOptions struct { LLMBaseCreateOptions - LLM_MODEL_ID string `help:"llm model id or name" json:"llm_model_id"` + LLM_SKU_ID string `help:"llm sku id or name" json:"llm_sku_id"` } func (o *LLMCreateOptions) Params() (jsonutils.JSONObject, error) { @@ -97,3 +101,83 @@ type LLMStopOptions struct { func (o *LLMStopOptions) Params() (jsonutils.JSONObject, error) { return jsonutils.Marshal(o), nil } + +type LLMIdOptions struct { + ID string `help:"llm id" json:"-"` +} + +func (opts *LLMIdOptions) GetId() string { + return opts.ID +} + +func (opts *LLMIdOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(opts), nil +} + +type LLMSaveInstantModelOptions struct { + LLMIdOptions + + MODEL_ID string `help:"llm model id, e.g. 500a1f067a9f"` + Name string `help:"instant app name, e.g. qwen3:8b"` + + // AutoRestart bool +} + +func (opts *LLMSaveInstantModelOptions) Params() (jsonutils.JSONObject, error) { + input := api.LLMSaveInstantModelInput{ + ModelId: opts.MODEL_ID, + ImageName: opts.Name, + // AutoRestart: opts.AutoRestart, + } + return jsonutils.Marshal(input), nil +} + +type LLMQuickModelsOptions struct { + LLMIdOptions + + MODEL []string `help:"model id and optional display name in the format of modelId[@modelName:modelTag], e.g. 6f48b936a09f or 6f48b936a09f@qwen2:0.5b"` + + Method string `help:"install or uninstall" choices:"install|uninstall"` +} + +func (opts *LLMQuickModelsOptions) Params() (jsonutils.JSONObject, error) { + params := api.LLMPerformQuickModelsInput{} + for _, mdlFul := range opts.MODEL { + var mdl api.ModelInfo + + var idPart string + var nameAndTagPart string + + if idx := strings.Index(mdlFul, "@"); idx >= 0 { + idPart = mdlFul[:idx] + nameAndTagPart = mdlFul[idx+1:] + + if idxTag := strings.LastIndex(nameAndTagPart, ":"); idxTag >= 0 { + mdl.DisplayName = nameAndTagPart[:idxTag] + mdl.Tag = nameAndTagPart[idxTag+1:] + } else { + mdl.DisplayName = nameAndTagPart + } + } else { + idPart = mdlFul + + if idxTag := strings.LastIndex(idPart, ":"); idxTag >= 0 { + mdl.Tag = idPart[idxTag+1:] + idPart = idPart[:idxTag] + } + } + + if regutils.MatchUUID(idPart) { + mdl.Id = idPart + } else { + mdl.ModelId = idPart + } + + params.Models = append(params.Models, mdl) + } + + if len(opts.Method) > 0 { + params.Method = api.TQuickModelMethod(opts.Method) + } + return jsonutils.Marshal(params), nil +} diff --git a/pkg/mcclient/options/llm/llm_model.go b/pkg/mcclient/options/llm/llm_sku.go similarity index 53% rename from pkg/mcclient/options/llm/llm_model.go rename to pkg/mcclient/options/llm/llm_sku.go index 06cfd96084..7d9734b225 100644 --- a/pkg/mcclient/options/llm/llm_model.go +++ b/pkg/mcclient/options/llm/llm_sku.go @@ -6,69 +6,69 @@ import ( "yunion.io/x/onecloud/pkg/mcclient/options" ) -type LLMModelListOptions struct { +type LLMSkuListOptions struct { options.BaseListOptions LLMType string `json:"llm_type" choices:"ollama"` } -func (o *LLMModelListOptions) Params() (jsonutils.JSONObject, error) { +func (o *LLMSkuListOptions) Params() (jsonutils.JSONObject, error) { return options.ListStructToParams(o) } -type LLMModelShowOptions struct { +type LLMSkuShowOptions struct { options.BaseShowOptions } -func (o *LLMModelShowOptions) Params() (jsonutils.JSONObject, error) { +func (o *LLMSkuShowOptions) Params() (jsonutils.JSONObject, error) { return options.StructToParams(o) } -type LLMModelCreateOptions struct { - LLMModelBaseCreateOptions +type LLMSkuCreateOptions struct { + LLMSkuBaseCreateOptions LLM_IMAGE_ID string `json:"llm_image_id"` LLM_TYPE string `json:"llm_type" choices:"ollama"` LLM_MODEL_NAME string `help:"specific model of large language model, for example: qwen3:32b" json:"llm_model_name"` } -func (o *LLMModelCreateOptions) Params() (jsonutils.JSONObject, error) { +func (o *LLMSkuCreateOptions) Params() (jsonutils.JSONObject, error) { dict := jsonutils.NewDict() obj := jsonutils.Marshal(o) obj.Unmarshal(dict) - o.LLMModelBaseCreateOptions.Params(dict) + o.LLMSkuBaseCreateOptions.Params(dict) return dict, nil } -type LLMModelDeleteOptions struct { +type LLMSkuDeleteOptions struct { options.BaseIdOptions } -func (o *LLMModelDeleteOptions) GetId() string { +func (o *LLMSkuDeleteOptions) GetId() string { return o.ID } -func (o *LLMModelDeleteOptions) Params() (jsonutils.JSONObject, error) { +func (o *LLMSkuDeleteOptions) Params() (jsonutils.JSONObject, error) { return options.StructToParams(o) } -type LLMModelUpdateOptions struct { - LLMModelBaseUpdateOptions +type LLMSkuUpdateOptions struct { + LLMSkuBaseUpdateOptions LlmImageId string LlmModelName string } -func (o *LLMModelUpdateOptions) GetId() string { +func (o *LLMSkuUpdateOptions) GetId() string { return o.ID } -func (o *LLMModelUpdateOptions) Params() (jsonutils.JSONObject, error) { +func (o *LLMSkuUpdateOptions) Params() (jsonutils.JSONObject, error) { dict := jsonutils.NewDict() obj := jsonutils.Marshal(o) obj.Unmarshal(dict) - o.LLMModelBaseUpdateOptions.Params(dict) + o.LLMSkuBaseUpdateOptions.Params(dict) return dict, nil } diff --git a/pkg/mcclient/options/llm/llm_model_base.go b/pkg/mcclient/options/llm/llm_sku_base.go similarity index 96% rename from pkg/mcclient/options/llm/llm_model_base.go rename to pkg/mcclient/options/llm/llm_sku_base.go index b091e4731f..304cce8835 100644 --- a/pkg/mcclient/options/llm/llm_model_base.go +++ b/pkg/mcclient/options/llm/llm_sku_base.go @@ -12,7 +12,7 @@ import ( api "yunion.io/x/onecloud/pkg/apis/llm" ) -type LLMModelBaseCreateOptions struct { +type LLMSkuBaseCreateOptions struct { apis.SharableVirtualResourceCreateInput CPU int @@ -36,7 +36,7 @@ type LLMModelBaseCreateOptions struct { Entrypoint string `help:"entrypoint"` } -func (o *LLMModelBaseCreateOptions) Params(dict *jsonutils.JSONDict) error { +func (o *LLMSkuBaseCreateOptions) Params(dict *jsonutils.JSONDict) error { vol := api.Volume{ SizeMB: o.DISK_SIZE, TemplateId: o.TemplateId, @@ -52,7 +52,7 @@ func (o *LLMModelBaseCreateOptions) Params(dict *jsonutils.JSONDict) error { return nil } -type LLMModelBaseUpdateOptions struct { +type LLMSkuBaseUpdateOptions struct { apis.SharableVirtualResourceBaseUpdateInput ID string @@ -81,7 +81,7 @@ type LLMModelBaseUpdateOptions struct { Entrypoint string `help:"entrypoint"` } -func (o *LLMModelBaseUpdateOptions) Params(dict *jsonutils.JSONDict) error { +func (o *LLMSkuBaseUpdateOptions) Params(dict *jsonutils.JSONDict) error { if o.NoTemplate { dict.Set("template_id", jsonutils.NewString("")) }