Automated cherry pick of #23769: [Feature] llm model 使用 post_overlay 功能实现秒装挂载模型 (#23888)

* feat(llm): add probed-package

* feat(llm): add llm-save-instant-app

* feat(llm): add some llm-instant-app options

* fix(llm): change app to model

* fix(llm): rename origin model to sku

* feat(llm): add llm-quick-models

* fix(llm): saved paths for instant model

* fix(llm): fix quick-install-model

---------

Co-authored-by: Zexi Li <zexi.li@icloud.com>

cmd/climc/shell/llm/dify_sku.go

@@ -8,13 +8,13 @@ import (
 )
 
 func init() {
-	cmd := shell.NewResourceCmd(&modules.LLMModel)
-	cmd.List(new(options.LLMModelListOptions))
-	cmd.Show(new(options.LLMModelShowOptions))
-	cmd.Update(new(options.LLMModelUpdateOptions))
-	cmd.Create(new(options.LLMModelCreateOptions))
-	cmd.Delete(new(options.LLMModelDeleteOptions))
+	cmd := shell.NewResourceCmd(&modules.DifySku)
+	cmd.List(new(options.DifySkuListOptions))
+	cmd.Show(new(options.DifySkuShowOptions))
+	cmd.Update(new(options.DifySkuUpdateOptions))
+	cmd.Create(new(options.DifySkuCreateOptions))
+	cmd.Delete(new(options.DifySkuDeleteOptions))
 	cmd.Perform("public", &base_options.BasePublicOptions{})
 	cmd.Perform("private", &base_options.BaseIdOptions{})
-	// cmd.Perform("clone", new(options.DesktopModelCloneOptions))
+	// cmd.Perform("clone", new(options.DesktopSkuCloneOptions))
 }

cmd/climc/shell/llm/instantmodel.go

@@ -0,0 +1,25 @@
+package llm
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+
+	modules "yunion.io/x/onecloud/pkg/mcclient/modules/llm"
+	commonoptions "yunion.io/x/onecloud/pkg/mcclient/options"
+	options "yunion.io/x/onecloud/pkg/mcclient/options/llm"
+)
+
+func init() {
+	cmd := shell.NewResourceCmd(&modules.LLMInstantModel)
+	cmd.List(new(options.LLMInstantModelListOptions))
+	cmd.Show(new(options.LLMInstantModelShowOptions))
+	// cmd.Update(new(options.InstantAppUpdateOptions))
+	cmd.Create(new(options.LLMInstantModelCreateOptions))
+	// cmd.Delete(new(options.InstantAppDeleteOptions))
+	cmd.Perform("syncstatus", new(commonoptions.BaseIdOptions))
+	cmd.Perform("change-owner", new(commonoptions.ChangeOwnerOptions))
+	cmd.Perform("enable", new(commonoptions.BaseIdOptions))
+	cmd.Perform("disable", new(commonoptions.BaseIdOptions))
+	cmd.Perform("public", new(commonoptions.BasePublicOptions))
+	cmd.Perform("private", new(commonoptions.BaseIdOptions))
+	// cmd.PerformClass("import", new(options.InstantAppImportOptions))
+}

cmd/climc/shell/llm/llm.go

@@ -13,6 +13,10 @@ func init() {
 	cmd.Show(new(options.LLMShowOptions))
 	cmd.Delete(new(options.LLMDeleteOptions))
 	// cmd.Perform("change-model", new(options.LLMChangeModelOptions))
+	cmd.Perform("syncstatus", new(options.LLMIdOptions))
 	cmd.BatchPerform("stop", new(options.LLMStopOptions))
 	cmd.BatchPerform("start", new(options.LLMStartOptions))
+	cmd.Get("probed-models", new(options.LLMIdOptions))
+	cmd.Perform("save-instant-model", new(options.LLMSaveInstantModelOptions))
+	cmd.Perform("quick-models", new(options.LLMQuickModelsOptions))
 }

cmd/climc/shell/llm/llm_sku.go

@@ -8,13 +8,13 @@ import (
 )
 
 func init() {
-	cmd := shell.NewResourceCmd(&modules.DifyModel)
-	cmd.List(new(options.DifyModelListOptions))
-	cmd.Show(new(options.DifyModelShowOptions))
-	cmd.Update(new(options.DifyModelUpdateOptions))
-	cmd.Create(new(options.DifyModelCreateOptions))
-	cmd.Delete(new(options.DifyModelDeleteOptions))
+	cmd := shell.NewResourceCmd(&modules.LLMSku)
+	cmd.List(new(options.LLMSkuListOptions))
+	cmd.Show(new(options.LLMSkuShowOptions))
+	cmd.Update(new(options.LLMSkuUpdateOptions))
+	cmd.Create(new(options.LLMSkuCreateOptions))
+	cmd.Delete(new(options.LLMSkuDeleteOptions))
 	cmd.Perform("public", &base_options.BasePublicOptions{})
 	cmd.Perform("private", &base_options.BaseIdOptions{})
-	// cmd.Perform("clone", new(options.DesktopModelCloneOptions))
+	// cmd.Perform("clone", new(options.DesktopSkuCloneOptions))
 }

pkg/apis/llm/dify.go

@@ -14,11 +14,11 @@ type DifyCustomized struct {
 type DifyListInput struct {
 	LLMBaseListInput
 
-	DifyModel string `json:"dify_model"`
+	DifySku string `json:"dify_sku"`
 }
 
 type DifyCreateInput struct {
 	LLMBaseCreateInput
 
-	DifyModelId string
+	DifySkuId string
 }

pkg/apis/llm/instantmodel.go

@@ -0,0 +1,95 @@
+package llm
+
+import (
+	"yunion.io/x/onecloud/pkg/apis"
+)
+
+type InstantModelListInput struct {
+	apis.SharableVirtualResourceListInput
+	apis.EnabledResourceBaseListInput
+
+	ModelName string `json:"model_name"`
+	Tag       string `json:"tag"`
+	ModelId   string `json:"model_id"`
+	Image     string `json:"image"`
+
+	Mounts string `json:"mounts"`
+
+	AutoCache *bool `json:"auto_cache"`
+}
+
+type InstantModelCreateInput struct {
+	apis.SharableVirtualResourceCreateInput
+	apis.EnabledBaseResourceCreateInput
+
+	LLMType   LLMContainerType `json:"llm_type"`
+	ModelName string           `json:"model_name"`
+	Tag       string           `json:"tag"`
+	ImageId   string           `json:"image_id"`
+	Size      int64            `json:"size"`
+	ModelId   string           `json:"model_id"`
+
+	ActualSizeMb int32 `json:"actual_size_mb"`
+
+	Mounts []string `json:"mounts"`
+}
+
+type InstantModelUpdateInput struct {
+	apis.SharableVirtualResourceBaseUpdateInput
+
+	ImageId string `json:"image_id"`
+	Size    int64  `json:"size"`
+
+	ActualSizeMb int32 `json:"actual_size_mb"`
+
+	Mounts []string `json:"mounts"`
+}
+
+type InstantModelDetails struct {
+	apis.SharableVirtualResourceDetails
+
+	Image string
+
+	CacheCount  int
+	CachedCount int
+
+	IconBase64 string `json:"icon_base64"`
+}
+
+type InstantModelImportInput struct {
+	Endpoint  string `json:"endpoint"`
+	AccessKey string `json:"access_key"`
+	SecretKey string `json:"secret_key"`
+	Bucket    string `json:"bucket"`
+	Key       string `json:"key"`
+	SignVer   string `json:"sign_ver"`
+}
+
+func (input InstantModelImportInput) Invalid() bool {
+	if len(input.Endpoint) == 0 || len(input.AccessKey) == 0 || len(input.SecretKey) == 0 || len(input.Bucket) == 0 || len(input.Key) == 0 {
+		return true
+	}
+	return false
+}
+
+type InstantModelSyncstatusInput struct {
+}
+
+type InstantAppCacheInput struct {
+}
+
+type InstantModelEnableAutoCacheInput struct {
+	AutoCache bool `json:"auto_cache"`
+}
+
+type MountedModelResourceListInput struct {
+	MountedModels []string `json:"mounted_models"`
+}
+
+type MountedModelResourceCreateInput struct {
+	MountedModels []string `json:"mounted_models"`
+}
+
+type MountedModelResourceUpdateInput struct {
+	MountedModels []string `json:"mounted_models"`
+}

pkg/apis/llm/llm.go

@@ -1,6 +1,9 @@
 package llm
 
-import "yunion.io/x/onecloud/pkg/apis"
+import (
+	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+)
 
 const (
 	SERVICE_TYPE = "llm"
@@ -19,7 +22,7 @@ type LLMBaseCreateInput struct {
 type LLMCreateInput struct {
 	LLMBaseCreateInput
 
-	LLMModelId string
+	LLMSkuId   string
 	LLMImageId string
 }
 
@@ -40,6 +43,73 @@ type LLMBaseListInput struct {
 type LLMListInput struct {
 	LLMBaseListInput
 
-	LLMModel string `json:"llm_model"`
+	LLMSku   string `json:"llm_sku"`
 	LLMImage string `json:"llm_image"`
 }
+
+type ModelInfo struct {
+	// 秒装模型ID
+	Id string `json:"id"`
+	// 秒装模型 ModelId
+	ModelId string `json:"model_id"`
+	// 秒装模型展示的名称，如: Qwen-7B
+	DisplayName string `json:"display_name"`
+	// 秒装模型 tag，如: 7b
+	Tag string `json:"tag"`
+}
+
+type LLMPerformQuickModelsInput struct {
+	Models []ModelInfo
+	Method TQuickModelMethod
+}
+
+type LLMBatchPerformOutput struct {
+	Data []LLMPerformOutput
+	Task *taskman.STask
+}
+
+type LLMPerformOutput struct {
+	Id            string
+	Name          string
+	RequestStatus int
+	Msg           string
+	TaskId        string
+}
+
+type LLMSyncModelTaskInput struct {
+	LLMPerformQuickModelsInput
+
+	LLMStatus         string   `json:"llm_status"`
+	InstallModelIds   []string `json:"install_model_ids"`
+	InstallDirs       []string `json:"install_dirs"`
+	UninstallModelIds []string `json:"uninstall_model_ids"`
+}
+
+type LLMMountDirInfo struct {
+	ImageId   string
+	Host      string
+	Container string
+}
+
+func (info LLMMountDirInfo) ToOverlay() apis.ContainerVolumeMountDiskPostOverlay {
+	// uid := int64(1000)
+	// gid := int64(1000)
+	if len(info.ImageId) > 0 {
+		return apis.ContainerVolumeMountDiskPostOverlay{
+			Image: &apis.ContainerVolumeMountDiskPostImageOverlay{
+				Id: info.ImageId,
+			},
+			// FsUser:  &uid,
+			// FsGroup: &gid,
+		}
+	}
+	return apis.ContainerVolumeMountDiskPostOverlay{
+		ContainerTargetDir: info.Container,
+		HostLowerDir:       []string{info.Host},
+		// FsUser:             &uid,
+		// FsGroup:            &gid,
+	}
+}
+
+type LLMSyncStatusInput struct {
+}

pkg/apis/llm/llm_const.go

@@ -16,11 +16,26 @@ const (
 	/* 停机失败 */
 	LLM_STATUS_STOP_FAILED = "stop_fail"
 
+	/* 开始保存应用 */
+	LLM_STATUS_START_SAVE_MODEL = "start_save_model"
+	/* 正在保存应用 */
+	LLM_STATUS_SAVING_MODEL = "saving_model"
+	/* 保存应用失败 */
+	LLM_STATUS_SAVE_MODEL_FAILED = "save_model_failed"
+
+	/* 开始同步状态 */
+	LLM_STATUS_START_SYNCSTATUS = "start_syncstatus"
+	/* 正在同步状态 */
+	LLM_STATUS_SYNCSTATUS = "syncstatus"
+
 	/* 停机 */
 	LLM_STATUS_READY = "ready"
 	/* 运行 */
 	LLM_STATUS_RUNNING = "running"
 
+	/* 删除 */
+	LLM_STATUS_DELETED = "deleted"
+
 	LLM_STATUS_CREATING_POD             = "creating_pod"
 	LLM_STATUS_CREAT_POD_FAILED         = "creat_pod_failed"
 	LLM_STATUS_PULLING_MODEL            = "pulling_model"
@@ -37,3 +52,15 @@ const (
 	LLM_STATUS_DELETING                 = "deleting"
 	LLM_STATUS_DELETE_FAILED            = "delete_fail"
 )
+
+type TQuickModelMethod string
+
+const (
+	QuickModelInstall   = TQuickModelMethod("install")
+	QuickModelUninstall = TQuickModelMethod("uninstall")
+	QuickModelReinstall = TQuickModelMethod("reinstall")
+)
+
+const (
+	LLM_PROBE_INSTANT_MODEl_INTERVAL_SECOND = 120 // 2 minute
+)

pkg/apis/llm/llm_instant_model.go

@@ -0,0 +1,23 @@
+package llm
+
+import "yunion.io/x/onecloud/pkg/apis"
+
+type LLMInternalInstantMdlInfo struct {
+	ModelId string `json:"model_id"`
+	Name    string `json:"name"`
+	Tag     string `json:"tag"`
+	Size    int64  `json:"size"`
+	// Modified string   `json:"modified"`
+	Blobs []string `json:"blobs"`
+}
+
+type LLMSaveInstantModelInput struct {
+	apis.ProjectizedResourceCreateInput
+
+	ModelId   string `json:"model_id"`
+	ImageName string `json:"image_name"`
+
+	InstantModelId string `json:"instant_model_id"`
+
+	// AutoRestart bool `json:"auto_restart"`
+}

pkg/apis/llm/ollama_const.go

@@ -11,6 +11,7 @@ const (
 )
 
 const (
+	LLM_OLLAMA_SAVE_DIR            = "/opt/.ollama-models/%s"
 	LLM_OLLAMA_HOST_PATH           = "/opt/ollama-models"
 	LLM_OLLAMA_HOST_MANIFESTS_DIR  = "/manifests"
 	LLM_OLLAMA_CACHE_DIR           = "/.llm_ollama_cache"

pkg/apis/llm/sku.go

@@ -12,14 +12,14 @@ import (
 )
 
 var (
-	LLM_MODEL_BASE_NETWORK_TYPES = sets.NewString(
+	LLM_SKU_BASE_NETWORK_TYPES = sets.NewString(
 		string(computeapi.NETWORK_TYPE_HOSTLOCAL),
 		string(computeapi.NETWORK_TYPE_GUEST),
 	)
 )
 
-func IsLLMModelBaseNetworkType(t string) bool {
-	return LLM_MODEL_BASE_NETWORK_TYPES.Has(t)
+func IsLLMSkuBaseNetworkType(t string) bool {
+	return LLM_SKU_BASE_NETWORK_TYPES.Has(t)
 }
 
 type HostInfo struct {
@@ -108,7 +108,7 @@ func (s Envs) IsZero() bool {
 	return len(s) == 0
 }
 
-type LLMModelDetails struct {
+type LLMSkuDetails struct {
 	apis.SharableVirtualResourceDetails
 	// 当前大模型套餐包含的实例个数。
 	LLMCapacity int
@@ -123,7 +123,7 @@ type LLMModelDetails struct {
 // 	MountedApps []string `json:"mounted_apps"`
 // }
 
-type LLMModelBaseCreateInput struct {
+type LLMSKuBaseCreateInput struct {
 	apis.SharableVirtualResourceCreateInput
 
 	Cpu    int `json:"cpu"`
@@ -140,7 +140,7 @@ type LLMModelBaseCreateInput struct {
 	Properties   map[string]string `json:"properties"`
 }
 
-type LLMModelBaseUpdateInput struct {
+type LLMSkuBaseUpdateInput struct {
 	apis.SharableVirtualResourceBaseUpdateInput
 	// MountedAppResourceUpdateInput
 
@@ -163,23 +163,23 @@ type LLMModelBaseUpdateInput struct {
 	Properties   map[string]string `json:"properties"`
 }
 
-type LLMModelListInput struct {
+type LLMSkuListInput struct {
 	apis.SharableVirtualResourceListInput
 	// MountedAppResourceListInput
 
 	LLMType string `json:"llm_type"`
 }
 
-type LLMModelCreateInput struct {
-	LLMModelBaseCreateInput
+type LLMSkuCreateInput struct {
+	LLMSKuBaseCreateInput
 
 	LLMImageId   string `json:"llm_image_id"`
 	LLMType      string `json:"llm_type"`
 	LLMModelName string `json:"llm_model_name"`
 }
 
-type LLMModelUpdateInput struct {
-	LLMModelBaseUpdateInput
+type LLMSkuUpdateInput struct {
+	LLMSkuBaseUpdateInput
 
 	LLMImageId   string `json:"llm_image_id"`
 	LLMModelName string `json:"llm_model_name"`
@@ -193,13 +193,13 @@ type LLMModelUpdateInput struct {
 // 	Request bool `json:"request"`
 // }
 
-type DifyModelListInput struct {
+type DifySkulListInput struct {
 	apis.SharableVirtualResourceListInput
 	// MountedAppResourceListInput
 }
 
-type DifyModelCreateInput struct {
-	LLMModelBaseCreateInput
+type DifySkuCreateInput struct {
+	LLMSKuBaseCreateInput
 
 	PostgresImageId     string `json:"postgres_image_id"`
 	RedisImageId        string `json:"redis_image_id"`
@@ -212,8 +212,8 @@ type DifyModelCreateInput struct {
 	DifyWeaviateImageId string `json:"dify_weaviate_image_id"`
 }
 
-type DifyModelUpdateInput struct {
-	LLMModelBaseUpdateInput
+type DifySkuUpdateInput struct {
+	LLMSkuBaseUpdateInput
 
 	PostgresImageId     string `json:"postgres_image_id"`
 	RedisImageId        string `json:"redis_image_id"`

pkg/llm/drivers/llm_container/ollama.go

@@ -2,19 +2,21 @@ package llm_container
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"path"
-	"regexp"
 	"strings"
 
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/apis"
+	commonapi "yunion.io/x/onecloud/pkg/apis"
 	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
 	api "yunion.io/x/onecloud/pkg/apis/llm"
 	"yunion.io/x/onecloud/pkg/llm/models"
-	"yunion.io/x/onecloud/pkg/llm/utils"
+	llmutil "yunion.io/x/onecloud/pkg/llm/utils"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
@@ -33,7 +35,7 @@ func (o *ollama) GetType() api.LLMContainerType {
 	return api.LLM_CONTAINER_OLLAMA
 }
 
-func (o *ollama) GetContainerSpec(ctx context.Context, llm *models.SLLM, image *models.SLLMImage, sku *models.SLLMModel, props []string, devices []computeapi.SIsolatedDevice, diskId string) *computeapi.PodContainerCreateInput {
+func (o *ollama) GetContainerSpec(ctx context.Context, llm *models.SLLM, image *models.SLLMImage, sku *models.SLLMSku, props []string, devices []computeapi.SIsolatedDevice, diskId string) *computeapi.PodContainerCreateInput {
 	spec := computeapi.ContainerSpec{
 		ContainerSpec: apis.ContainerSpec{
 			Image:             image.ToContainerImage(),
@@ -94,27 +96,20 @@ func (o *ollama) GetContainerSpec(ctx context.Context, llm *models.SLLM, image *
 	// vols = append(spec.VolumeMounts, GetDiskVolumeMounts(sku.Volumes, appVolIndex, postOverlays)...)
 
 	// udevPath := filepath.Join(GetTmpSocketsHostPath(d.GetName()), "udev")
-	modelName, modelTag, _ := llm.GetLargeLanguageModelName()
+	diskIndex := 0
 	ctrVols := []*apis.ContainerVolumeMount{
 		{
-			UniqueName: "manifests",
-			Type:       apis.CONTAINER_VOLUME_MOUNT_TYPE_HOST_PATH,
-			MountPath:  getManifestsPath(modelName, modelTag),
-			HostPath: &apis.ContainerVolumeMountHostPath{
-				Type: apis.CONTAINER_VOLUME_MOUNT_HOST_PATH_TYPE_FILE,
-				Path: path.Join(api.LLM_OLLAMA_HOST_PATH, api.LLM_OLLAMA_HOST_MANIFESTS_DIR, modelName+"-"+modelTag),
+			Disk: &apis.ContainerVolumeMountDisk{
+				SubDirectory: api.LLM_OLLAMA,
+				Overlay: &apis.ContainerVolumeMountDiskOverlay{
+					LowerDir: []string{api.LLM_OLLAMA_HOST_PATH},
+				},
+				Index: &diskIndex,
 			},
-			ReadOnly: true,
-		},
-		{
-			UniqueName: "blobs",
-			Type:       apis.CONTAINER_VOLUME_MOUNT_TYPE_HOST_PATH,
-			MountPath:  path.Join(api.LLM_OLLAMA_BASE_PATH, api.LLM_OLLAMA_BLOBS_DIR),
-			HostPath: &apis.ContainerVolumeMountHostPath{
-				Type: apis.CONTAINER_VOLUME_MOUNT_HOST_PATH_TYPE_DIRECTORY,
-				Path: path.Join(api.LLM_OLLAMA_HOST_PATH, api.LLM_OLLAMA_BLOBS_DIR),
-			},
-			ReadOnly: true,
+			Type:        apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK,
+			MountPath:   api.LLM_OLLAMA_BASE_PATH,
+			ReadOnly:    false,
+			Propagation: apis.MOUNTPROPAGATION_PROPAGATION_HOST_TO_CONTAINER,
 		},
 	}
 	vols = append(vols, ctrVols...)
@@ -160,29 +155,211 @@ func (o *ollama) GetContainerSpec(ctx context.Context, llm *models.SLLM, image *
 // 	return err
 // }
 
-func (o *ollama) CopyBlobs(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM) error {
-	ctr, _ := llm.GetLLMContainer()
-	modelName, modelTag, _ := llm.GetLargeLanguageModelName()
-	blobs, _ := fetchBlobs(ctx, ctr.CmpId, modelName, modelTag)
-
-	blobsTargetDir := path.Join(api.LLM_OLLAMA_BASE_PATH, api.LLM_OLLAMA_BLOBS_DIR)
-	blobsSrcDir := path.Join(api.LLM_OLLAMA_CACHE_MOUNT_PATH, api.LLM_OLLAMA_CACHE_DIR)
-
-	var commands []string
-	commands = append(commands, fmt.Sprintf("mkdir -p %s", blobsTargetDir))
-	for _, blob := range blobs {
-		src := path.Join(blobsSrcDir, blob)
-		target := path.Join(blobsTargetDir, blob)
-		commands = append(commands, fmt.Sprintf("cp %s %s", src, target))
+func (o *ollama) PreInstallModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, instMdl *models.SLLMInstantModel) error {
+	// before mount, make sure the manifests dir is ready
+	lc, err := llm.GetLLMContainer()
+	if err != nil {
+		return errors.Wrap(err, "get llm container")
 	}
 
-	cmd := strings.Join(commands, " && ")
-	if _, err := exec(ctx, ctr.CmpId, "/bin/sh", "-c", cmd); err != nil {
-		return errors.Wrapf(err, "failed to copy blobs to container")
+	// mkdir llm-registry-base-path / modelname
+	mkdirReigtryBasePaht := fmt.Sprintf("mkdir -p %s", path.Join(api.LLM_OLLAMA_BASE_PATH, api.LLM_OLLAMA_MANIFESTS_BASE_PATH, instMdl.ModelName))
+	_, err = exec(ctx, lc.CmpId, mkdirReigtryBasePaht, 10)
+	if err != nil {
+		return errors.Wrap(err, "failed to mkdir llm-registry-base-path / modelname")
 	}
+
 	return nil
 }
 
+func (o *ollama) InstallModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, dirs []string, mdlIds []string) error {
+	///* TODO
+	return nil
+}
+
+func (o *ollama) GetModelMountPaths(ctx context.Context, userCred mcclient.TokenCredential, llmInstMdl *models.SLLMInstantModel) ([]string, error) {
+	instMdl, _ := llmInstMdl.FindInstantModel(false)
+	return instMdl.Mounts, nil
+}
+
+func (o *ollama) GetDirPostOverlay(dir api.LLMMountDirInfo) *commonapi.ContainerVolumeMountDiskPostOverlay {
+	uid := int64(1000)
+	gid := int64(1000)
+	ov := dir.ToOverlay()
+	ov.FsUser = &uid
+	ov.FsGroup = &gid
+	return &ov
+}
+
+func (o *ollama) UninstallModel(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, llmInstMdl *models.SLLMInstantModel) error {
+	// don't rm file
+
+	// mounts, err := o.GetModelMountPaths(ctx, userCred, llmInstMdl)
+	// if err != nil {
+	// 	return errors.Wrap(err, "GetModelMountPaths")
+	// }
+	// ctr, err := llm.GetLLMSContainer(ctx)
+	// if err != nil {
+	// 	return errors.Wrap(err, "GetSContainer")
+	// }
+	// _, err = exec(ctx, ctr.Id, fmt.Sprintf("rm -rf %s", strings.Join(mounts, " ")), 10)
+	// if err != nil {
+	// 	return errors.Wrapf(err, "run cmd to remove model mounts of model %s", jsonutils.Marshal(llmInstMdl))
+	// }
+	return nil
+}
+
+func (o *ollama) GetInstantModelIdByPostOverlay(postOverlay *commonapi.ContainerVolumeMountDiskPostOverlay, mdlNameToId map[string]string) string {
+	if postOverlay.Image != nil {
+		for k := range postOverlay.Image.PathMap {
+			idx := strings.Index(k, api.LLM_OLLAMA_MANIFESTS_BASE_PATH)
+			if idx != -1 {
+				suffix := k[idx+len(api.LLM_OLLAMA_MANIFESTS_BASE_PATH):]
+				parts := strings.Split(strings.Trim(suffix, "/"), "/")
+				if len(parts) >= 2 {
+					modelName := parts[len(parts)-2]
+					modelTag := parts[len(parts)-1]
+					log.Infof("In GetInstantModelIdByPostOverlay, Extracted modelName: %s, modelTag: %s, Got modelId: %s", modelName, modelTag, mdlNameToId[modelName+":"+modelTag])
+					return mdlNameToId[modelName+":"+modelTag]
+				}
+			}
+		}
+	}
+	return ""
+}
+
+func (o *ollama) DetectModelPaths(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, pkgInfo api.LLMInternalInstantMdlInfo) ([]string, error) {
+	lc, err := llm.GetLLMContainer()
+	if err != nil {
+		return nil, errors.Wrap(err, "get llm container")
+	}
+	// check file exists
+	originBlobs := make([]string, len(pkgInfo.Blobs))
+	for idx, blob := range pkgInfo.Blobs {
+		originBlobs[idx] = path.Join(api.LLM_OLLAMA_BASE_PATH, api.LLM_OLLAMA_BLOBS_DIR, blob)
+	}
+	originManifests := path.Join(api.LLM_OLLAMA_BASE_PATH, api.LLM_OLLAMA_MANIFESTS_BASE_PATH, pkgInfo.Name, pkgInfo.Tag)
+
+	var checks []string
+	for _, blob := range originBlobs {
+		checks = append(checks, fmt.Sprintf("[ -f '%s' ]", blob))
+	}
+	checks = append(checks, fmt.Sprintf("[ -f '%s' ]", originManifests))
+
+	checkCmd := strings.Join(checks, " && ") + " && echo 'ALL_EXIST' || echo 'SOME_MISSING'"
+	output, err := exec(ctx, lc.CmpId, checkCmd, 10)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to check file existence")
+	}
+
+	if !strings.Contains(output, "ALL_EXIST") {
+		log.Infof("Some files are missing for model %s:%s, blobs: %v, manifest: %s, checkCmd: %s",
+			pkgInfo.Name, pkgInfo.Tag, originBlobs, originManifests, checkCmd)
+		return nil, errors.Errorf("required model files are missing")
+	}
+
+	// // mkdir
+	// savePath := fmt.Sprintf(api.LLM_OLLAMA_SAVE_DIR, pkgInfo.Name+"-"+pkgInfo.Tag+"-"+pkgInfo.ModelId)
+	// mkSaveDir := fmt.Sprintf("mkdir -p %s %s %s", savePath, path.Join(savePath, api.LLM_OLLAMA_BLOBS_DIR), path.Join(savePath, api.LLM_OLLAMA_HOST_MANIFESTS_DIR))
+	// _, err = exec(ctx, ctr.CmpId, mkSaveDir, 10)
+	// if err != nil {
+	// 	return a, filtenil, errors.Wrap(err, "mkdir savedir")
+	// }
+
+	// // cp file
+	// for _, blob := range originBlobs {
+	// 	cpBlob := fmt.Sprintf("cp %s %s", blob, path.Join(savePath, api.LLM_OLLAMA_BLOBS_DIR))
+	// 	_, err = exec(ctx, ctr.CmpId, cpBlob, 60)
+	// 	if err != nil {
+	// 		return nil, errors.Wrap(err, "copy files")
+	// 	}
+	// }
+	// cpManifest := "cp " + originManifests + " " + path.Join(savePath, api.LLM_OLLAMA_HOST_MANIFESTS_DIR)
+	// _, err = exec(ctx, ctr.CmpId, cpManifest, 20)
+	// if err != nil {
+	// 	return nil, errors.Wrap(err, "copy files")
+	// }
+
+	return append(originBlobs, originManifests), nil
+}
+
+func (o *ollama) GetImageInternalPathMounts(sMdl *models.SInstantModel) map[string]string {
+	imageToContainer := make(map[string]string)
+
+	for _, mount := range sMdl.Mounts {
+		imgPath := strings.TrimPrefix(mount, api.LLM_OLLAMA_BASE_PATH)
+		imageToContainer[imgPath] = path.Join(api.LLM_OLLAMA, imgPath)
+	}
+
+	return imageToContainer
+}
+
+func (o *ollama) GetSaveDirectories(sApp *models.SInstantModel) (string, []string, error) {
+	var filteredMounts []string
+
+	for _, mount := range sApp.Mounts {
+		if strings.HasPrefix(mount, api.LLM_OLLAMA_BASE_PATH) {
+			relPath := strings.TrimPrefix(mount, api.LLM_OLLAMA_BASE_PATH)
+			filteredMounts = append(filteredMounts, relPath)
+		}
+	}
+
+	return "", filteredMounts, nil
+}
+
+func (o *ollama) GetProbedInstantModelsExt(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, mdlIds ...string) (map[string]api.LLMInternalInstantMdlInfo, error) {
+	lc, err := llm.GetLLMContainer()
+	if err != nil {
+		return nil, errors.Wrap(err, "get llm container")
+	}
+
+	// get all effective models
+	getModels := "ollama list" // NAME ID SIZE MODIFIED
+	modelsOutput, err := exec(ctx, lc.CmpId, getModels, 10)
+	if err != nil {
+		return nil, errors.Wrap(err, "get models")
+	}
+	lines := strings.Split(strings.TrimSpace(modelsOutput), "\n")
+
+	models := make(map[string]api.LLMInternalInstantMdlInfo, len(lines)-1)
+	for i := 1; i < len(lines); i++ {
+		fields := strings.Fields(lines[i])
+		if len(fields) > 2 {
+			if len(mdlIds) > 0 && !utils.IsInStringArray(fields[1], mdlIds) {
+				continue
+			}
+			modelName, modelTag, _ := llm.GetLargeLanguageModelName(fields[0])
+			models[fields[1]] = api.LLMInternalInstantMdlInfo{
+				Name:    modelName,
+				Tag:     modelTag,
+				ModelId: fields[1],
+				// Modified: fields[3],
+			}
+		}
+	}
+
+	// for each model, get manifests file, find blobs, calculate size
+	for modelId, model := range models {
+		manifests, err := getManifests(ctx, lc.CmpId, model.Name, model.Tag)
+		if err != nil {
+			return nil, errors.Wrap(err, "get manifests")
+		}
+		model.Size = manifests.Config.Size
+		model.Blobs = append(model.Blobs, manifests.Config.Digest)
+		for _, layer := range manifests.Layers {
+			model.Size += layer.Size
+			model.Blobs = append(model.Blobs, layer.Digest)
+		}
+		models[modelId] = model
+	}
+
+	return models, nil
+}
+
+func (o *ollama) ValidateMounts(mounts []string, mdlName string, mdlTag string) ([]string, error) {
+	return mounts, nil
+}
+
 // func download(ctx context.Context, userCred mcclient.TokenCredential, containerId string, taskId string, webUrl string, path string) error {
 // 	input := &computeapi.ContainerDownloadFileInput{
 // 		WebUrl: webUrl,
@@ -193,13 +370,39 @@ func (o *ollama) CopyBlobs(ctx context.Context, userCred mcclient.TokenCredentia
 // 	return err
 // }
 
-func exec(ctx context.Context, containerId string, command ...string) (string, error) {
-	// exec command
+// func (o *ollama) CopyBlobs(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM) error {
+// 	ctr, _ := llm.GetLLMContainer()
+// 	modelName, modelTag, _ := llm.GetLargeLanguageModelName("")
+// 	manifests, _ := getManifests(ctx, ctr.CmpId, modelName, modelTag)
 
+// 	blobsTargetDir := path.Join(api.LLM_OLLAMA_BASE_PATH, api.LLM_OLLAMA_BLOBS_DIR)
+// 	blobsSrcDir := path.Join(api.LLM_OLLAMA_CACHE_MOUNT_PATH, api.LLM_OLLAMA_CACHE_DIR)
+
+// 	var commands []string
+// 	commands = append(commands, fmt.Sprintf("mkdir -p %s", blobsTargetDir))
+// 	blob := manifests.Config.Digest
+// 	commands = append(commands, fmt.Sprintf("cp %s %s", path.Join(blobsSrcDir, blob), path.Join(blobsTargetDir, blob)))
+// 	for _, layer := range manifests.Layers {
+// 		blob = layer.Digest
+// 		src := path.Join(blobsSrcDir, blob)
+// 		target := path.Join(blobsTargetDir, blob)
+// 		commands = append(commands, fmt.Sprintf("cp %s %s", src, target))
+// 	}
+
+// 	cmd := strings.Join(commands, " && ")
+// 	if _, err := exec(ctx, ctr.CmpId, cmd, 120); err != nil {
+// 		return errors.Wrapf(err, "failed to copy blobs to container")
+// 	}
+// 	return nil
+// }
+
+func exec(ctx context.Context, containerId string, cmd string, timeoutSec int64) (string, error) {
+	// exec command
 	input := &computeapi.ContainerExecSyncInput{
-		Command: command,
+		Command: []string{"sh", "-c", cmd},
+		Timeout: timeoutSec,
 	}
-	resp, err := utils.ExecSyncContainer(ctx, containerId, input)
+	resp, err := llmutil.ExecSyncContainer(ctx, containerId, input)
 
 	// check error and return result
 	var rst string
@@ -215,23 +418,36 @@ func getManifestsPath(modelName, modelTag string) string {
 	return path.Join(api.LLM_OLLAMA_BASE_PATH, api.LLM_OLLAMA_MANIFESTS_BASE_PATH, modelName, modelTag)
 }
 
-func fetchBlobs(ctx context.Context, containerId string, modelName string, modelTag string) ([]string, error) {
-	manifestContent, err := exec(ctx, containerId, "cat", getManifestsPath(modelName, modelTag))
+type Layer struct {
+	MediaType string `json:"mediaType"`
+	Digest    string `json:"digest"`
+	Size      int64  `json:"size"`
+}
+
+type Manifest struct {
+	SchemaVersion int     `json:"schemaVersion"`
+	MediaType     string  `json:"mediaType"`
+	Config        Layer   `json:"config"`
+	Layers        []Layer `json:"layers"`
+}
+
+func getManifests(ctx context.Context, containerId string, modelName string, modelTag string) (*Manifest, error) {
+	manifestContent, err := exec(ctx, containerId, "cat "+getManifestsPath(modelName, modelTag), 10)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to read manifests from container")
 	}
 
 	// find all blobs
-	var results []string
-	re := regexp.MustCompile(`"digest":"(sha256:[^"]*)"`)
-	matches := re.FindAllStringSubmatch(manifestContent, -1)
-	for _, match := range matches {
-		if len(match) > 1 {
-			digest := match[1]
-			processedDigest := strings.Replace(digest, "sha256:", "sha256-", 1)
-			results = append(results, processedDigest)
-		}
+	manifests := &Manifest{}
+	if err = json.Unmarshal([]byte(manifestContent), manifests); err != nil {
+		return nil, errors.Wrapf(err, "failed to parse manifests")
+	}
+	// log.Infof("manifests: %v", manifests)
+
+	manifests.Config.Digest = strings.Replace(manifests.Config.Digest, ":", "-", 1)
+	for idx, layer := range manifests.Layers {
+		manifests.Layers[idx].Digest = strings.Replace(layer.Digest, ":", "-", 1)
 	}
 
-	return results, nil
+	return manifests, nil
 }

pkg/llm/models/dify.go

@@ -49,7 +49,7 @@ type SDifyManager struct {
 type SDify struct {
 	SLLMBase
 
-	DifyModelId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"`
+	DifySkuId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"`
 }
 
 func (dm *SDifyManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.DifyCreateInput) (*api.DifyCreateInput, error) {
@@ -58,12 +58,12 @@ func (dm *SDifyManager) ValidateCreateData(ctx context.Context, userCred mcclien
 	if err != nil {
 		return input, errors.Wrap(err, "validate VirtualResourceCreateInput")
 	}
-	model, err := GetDifyModelManager().FetchByIdOrName(ctx, userCred, input.DifyModelId)
+	sku, err := GetDifySkuManager().FetchByIdOrName(ctx, userCred, input.DifySkuId)
 	if err != nil {
-		return input, errors.Wrap(err, "fetch DifyModel")
+		return input, errors.Wrap(err, "fetch DifySku")
 	}
-	dModel := model.(*SDifyModel)
-	input.DifyModelId = dModel.Id
+	dSku := sku.(*SDifySku)
+	input.DifySkuId = dSku.Id
 
 	return input, nil
 }
@@ -92,16 +92,16 @@ func (dm *SDifyManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery,
 	if err != nil {
 		return q, errors.Wrap(err, "VirtualResourceBaseManager.ListItemFilter")
 	}
-	if len(input.DifyModel) > 0 {
-		modelObj, err := GetDifyModelManager().FetchByIdOrName(ctx, userCred, input.DifyModel)
+	if len(input.DifySku) > 0 {
+		skuObj, err := GetDifySkuManager().FetchByIdOrName(ctx, userCred, input.DifySku)
 		if err != nil {
 			if errors.Cause(err) == sql.ErrNoRows {
-				return nil, httperrors.NewResourceNotFoundError2(GetDifyModelManager().KeywordPlural(), input.DifyModel)
+				return nil, httperrors.NewResourceNotFoundError2(GetDifySkuManager().KeywordPlural(), input.DifySku)
 			} else {
-				return nil, errors.Wrap(err, "DifyModelManager.FetchByIdOrName")
+				return nil, errors.Wrap(err, "GetDifySkuManager.FetchByIdOrName")
 			}
 		}
-		q = q.Equals("dify_model_id", modelObj.GetId())
+		q = q.Equals("dify_sku_id", skuObj.GetId())
 	}
 	return q, nil
 }
@@ -110,15 +110,15 @@ func (dify *SDify) CustomizeDelete(ctx context.Context, userCred mcclient.TokenC
 	return dify.StartDeleteTask(ctx, userCred, "")
 }
 
-func (dify *SDify) GetDifyModel(modelId string) (*SDifyModel, error) {
-	if len(modelId) == 0 {
-		modelId = dify.DifyModelId
+func (dify *SDify) GetDifySku(skuId string) (*SDifySku, error) {
+	if len(skuId) == 0 {
+		skuId = dify.DifySkuId
 	}
-	model, err := GetDifyModelManager().FetchById(modelId)
+	sku, err := GetDifySkuManager().FetchById(skuId)
 	if err != nil {
-		return nil, errors.Wrap(err, "fetch DifyModel")
+		return nil, errors.Wrap(err, "fetch DifySku")
 	}
-	return model.(*SDifyModel), nil
+	return sku.(*SDifySku), nil
 }
 
 func (dify *SDify) GetDifyContainers() []*computeapi.PodContainerCreateInput {
@@ -172,12 +172,12 @@ func (dify *SDify) StartDeleteTask(ctx context.Context, userCred mcclient.TokenC
 }
 
 func (dify *SDify) ServerCreate(ctx context.Context, userCred mcclient.TokenCredential, s *mcclient.ClientSession, input *api.DifyCreateInput) (string, error) {
-	model, err := dify.GetDifyModel(dify.DifyModelId)
+	sku, err := dify.GetDifySku(dify.DifySkuId)
 	if nil != err {
-		return "", errors.Wrap(err, "GetDifyModel")
+		return "", errors.Wrap(err, "GetDifySku")
 	}
 
-	data, err := GetDifyPodCreateInput(ctx, userCred, input, dify, model, "")
+	data, err := GetDifyPodCreateInput(ctx, userCred, input, dify, sku, "")
 	if nil != err {
 		return "", errors.Wrap(err, "GetDifyPodCreateInput")
 	}
@@ -207,11 +207,11 @@ func (dify *SDify) ServerCreate(ctx context.Context, userCred mcclient.TokenCred
 // }
 
 func (dify *SDify) getDifyContainerByContainerKey(containerKey string) (*computeapi.PodContainerCreateInput, error) {
-	model, err := dify.GetDifyModel("")
+	sku, err := dify.GetDifySku("")
 	if nil != err {
 		return nil, err
 	}
-	container, err := getDifyContainersManager().GetContainer(dify.GetName(), containerKey, model)
+	container, err := getDifyContainersManager().GetContainer(dify.GetName(), containerKey, sku)
 	if nil != err {
 		return nil, err
 	}

pkg/llm/models/dify_containers.go

@@ -77,7 +77,7 @@ func _getRegistryImage(imageId string) string {
 	return image.(*SLLMImage).ToContainerImage()
 }
 
-func (m *DifyContainersManager) GetContainer(name, containerKey string, sku *SDifyModel) (*computeapi.PodContainerCreateInput, error) {
+func (m *DifyContainersManager) GetContainer(name, containerKey string, sku *SDifySku) (*computeapi.PodContainerCreateInput, error) {
 	switch containerKey {
 	case api.DIFY_REDIS_KEY:
 		return m._getRedisContainer(name, containerKey, _getRegistryImage(sku.RedisImageId)), nil

pkg/llm/models/dify_pod.go

@@ -15,10 +15,10 @@ func GetDifyPodCreateInput(
 	userCred mcclient.TokenCredential,
 	input *api.DifyCreateInput,
 	dify *SDify,
-	sku *SDifyModel,
+	sku *SDifySku,
 	eip string,
 ) (*computeapi.ServerCreateInput, error) {
-	data, err := GetLLMBasePodCreateInput(ctx, userCred, &input.LLMBaseCreateInput, &dify.SLLMBase, &sku.SLLMModelBase, eip)
+	data, err := GetLLMBasePodCreateInput(ctx, userCred, &input.LLMBaseCreateInput, &dify.SLLMBase, &sku.SLLMSkuBase, eip)
 	if err != nil {
 		return nil, errors.Wrap(err, "GetLLMBasePodCreateInput: ")
 	}

pkg/llm/models/dify_sku.go

@@ -13,33 +13,33 @@ import (
 )
 
 func init() {
-	GetDifyModelManager()
+	GetDifySkuManager()
 }
 
-var difyModelManager *SDifyModelManager
+var difySkuManager *SDifySkuManager
 
-func GetDifyModelManager() *SDifyModelManager {
-	if difyModelManager != nil {
-		return difyModelManager
+func GetDifySkuManager() *SDifySkuManager {
+	if difySkuManager != nil {
+		return difySkuManager
 	}
-	difyModelManager = &SDifyModelManager{
-		SLLMModelBaseManager: NewSLLMModelBaseManager(
-			SDifyModel{},
-			"dify_models_tbl",
-			"dify_model",
-			"dify_models",
+	difySkuManager = &SDifySkuManager{
+		SLLMSkuBaseManager: NewSLLMSkuBaseManager(
+			SDifySku{},
+			"dify_skus_tbl",
+			"dify_sku",
+			"dify_skus",
 		),
 	}
-	difyModelManager.SetVirtualObject(difyModelManager)
-	return difyModelManager
+	difySkuManager.SetVirtualObject(difySkuManager)
+	return difySkuManager
 }
 
-type SDifyModelManager struct {
-	SLLMModelBaseManager
+type SDifySkuManager struct {
+	SLLMSkuBaseManager
 }
 
-type SDifyModel struct {
-	SLLMModelBase
+type SDifySku struct {
+	SLLMSkuBase
 
 	PostgresImageId     string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"`
 	RedisImageId        string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"`
@@ -52,16 +52,16 @@ type SDifyModel struct {
 	DifyWeaviateImageId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"`
 }
 
-func (man *SDifyModelManager) ListItemFilter(
+func (man *SDifySkuManager) ListItemFilter(
 	ctx context.Context,
 	q *sqlchemy.SQuery,
 	userCred mcclient.TokenCredential,
-	input api.DifyModelListInput,
+	input api.DifySkulListInput,
 ) (*sqlchemy.SQuery, error) {
 	var err error
-	q, err = man.SLLMModelBaseManager.ListItemFilter(ctx, q, userCred, input.SharableVirtualResourceListInput)
+	q, err = man.SLLMSkuBaseManager.ListItemFilter(ctx, q, userCred, input.SharableVirtualResourceListInput)
 	if err != nil {
-		return nil, errors.Wrapf(err, "SLLMModelBaseManager.ListItemFilter")
+		return nil, errors.Wrapf(err, "SLLMSkuBaseManager.ListItemFilter")
 	}
 	return q, nil
 }
@@ -77,11 +77,11 @@ func (man *SDifyModelManager) ListItemFilter(
 
 // }
 
-func (man *SDifyModelManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.DifyModelCreateInput) (*api.DifyModelCreateInput, error) {
+func (man *SDifySkuManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.DifySkuCreateInput) (*api.DifySkuCreateInput, error) {
 	var err error
-	input.LLMModelBaseCreateInput, err = man.SLLMModelBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.LLMModelBaseCreateInput)
+	input.LLMSKuBaseCreateInput, err = man.SLLMSkuBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.LLMSKuBaseCreateInput)
 	if err != nil {
-		return nil, errors.Wrap(err, "SLLMModelBaseManager.ValidateCreateData")
+		return nil, errors.Wrap(err, "SLLMSkuBaseManager.ValidateCreateData")
 	}
 
 	for _, imgId := range []*string{&input.PostgresImageId, &input.RedisImageId, &input.NginxImageId, &input.DifyApiImageId, &input.DifyPluginImageId, &input.DifyWebImageId, &input.DifySandboxImageId, &input.DifySSRFImageId, &input.DifyWeaviateImageId} {
@@ -94,11 +94,11 @@ func (man *SDifyModelManager) ValidateCreateData(ctx context.Context, userCred m
 	return input, nil
 }
 
-func (model *SDifyModel) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.DifyModelUpdateInput) (api.DifyModelUpdateInput, error) {
+func (sku *SDifySku) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.DifySkuUpdateInput) (api.DifySkuUpdateInput, error) {
 	var err error
-	input.LLMModelBaseUpdateInput, err = model.SLLMModelBase.ValidateUpdateData(ctx, userCred, query, input.LLMModelBaseUpdateInput)
+	input.LLMSkuBaseUpdateInput, err = sku.SLLMSkuBase.ValidateUpdateData(ctx, userCred, query, input.LLMSkuBaseUpdateInput)
 	if err != nil {
-		return input, errors.Wrap(err, "validate LLMModelBaseUpdateInput")
+		return input, errors.Wrap(err, "validate LLMSkuBaseUpdateInput")
 	}
 
 	for _, imgId := range []*string{&input.PostgresImageId, &input.RedisImageId, &input.NginxImageId, &input.DifyApiImageId, &input.DifyPluginImageId, &input.DifyWebImageId, &input.DifySandboxImageId, &input.DifySSRFImageId, &input.DifyWeaviateImageId} {
@@ -113,13 +113,13 @@ func (model *SDifyModel) ValidateUpdateData(ctx context.Context, userCred mcclie
 	return input, nil
 }
 
-func (model *SDifyModel) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
-	count, err := GetDifyManager().Query().Equals("dify_model_id", model.Id).CountWithError()
+func (sku *SDifySku) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
+	count, err := GetDifyManager().Query().Equals("dify_sku_id", sku.Id).CountWithError()
 	if nil != err {
 		return errors.Wrap(err, "fetch dify")
 	}
 	if count > 0 {
-		return errors.Wrap(errors.ErrNotSupported, "This model is currently in use")
+		return errors.Wrap(errors.ErrNotSupported, "This sku is currently in use")
 	}
 	return nil
 }

pkg/llm/models/image.go

@@ -137,7 +137,7 @@ func (image *SLLMImage) ValidateDeleteCondition(ctx context.Context, info jsonut
 		if count > 0 {
 			return errors.Wrapf(errors.ErrNotSupported, "This image is currently in use by %s in llms", field)
 		}
-		count, err = GetLLMModelManager().Query().Equals("llm_image_id", image.Id).CountWithError()
+		count, err = GetLLMSkuManager().Query().Equals("llm_image_id", image.Id).CountWithError()
 		if err != nil {
 			return errors.Wrap(err, "fetch llm models")
 		}

pkg/llm/models/instantmodel.go

@@ -0,0 +1,936 @@
+package models
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/httputils"
+	"yunion.io/x/pkg/utils"
+	"yunion.io/x/sqlchemy"
+
+	commonapis "yunion.io/x/onecloud/pkg/apis"
+	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
+	imageapi "yunion.io/x/onecloud/pkg/apis/image"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	computemodules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
+	imagemodules "yunion.io/x/onecloud/pkg/mcclient/modules/image"
+	commonoptions "yunion.io/x/onecloud/pkg/mcclient/options"
+	"yunion.io/x/onecloud/pkg/util/logclient"
+
+	apis "yunion.io/x/onecloud/pkg/apis/llm"
+	"yunion.io/x/onecloud/pkg/llm/options"
+)
+
+var instantModelManager *SInstantModelManager
+
+func init() {
+	GetInstantModelManager()
+}
+
+type SInstantModelManager struct {
+	db.SSharableVirtualResourceBaseManager
+	db.SEnabledResourceBaseManager
+}
+
+func GetInstantModelManager() *SInstantModelManager {
+	if instantModelManager != nil {
+		return instantModelManager
+	}
+	instantModelManager = &SInstantModelManager{
+		SSharableVirtualResourceBaseManager: db.NewSharableVirtualResourceBaseManager(
+			SInstantModel{},
+			"instant_models_tbl",
+			"llm_instant_model",
+			"llm_instant_models",
+		),
+	}
+	instantModelManager.SetVirtualObject(instantModelManager)
+	return instantModelManager
+}
+
+type SInstantModel struct {
+	db.SSharableVirtualResourceBase
+	db.SEnabledResourceBase
+
+	LlmType   string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"`
+	ModelId   string `width:"128" charset:"ascii" list:"user" create:"optional"`
+	ModelName string `width:"128" charset:"ascii" list:"user" create:"required"`
+	Tag       string `width:"64" charset:"ascii" list:"user" create:"required"`
+
+	ImageId string `width:"128" charset:"ascii" list:"user" create:"optional" update:"user"`
+
+	Mounts []string `charset:"ascii" list:"user" create:"optional" update:"user"`
+
+	Size int64 `nullable:"true" list:"user" create:"optional"`
+
+	ActualSizeMb int32 `nullable:"true" list:"user" update:"user"`
+
+	AutoCache bool `list:"user"`
+}
+
+// climc instant-app-list
+func (man *SInstantModelManager) ListItemFilter(
+	ctx context.Context,
+	q *sqlchemy.SQuery,
+	userCred mcclient.TokenCredential,
+	input apis.InstantModelListInput,
+) (*sqlchemy.SQuery, error) {
+	var err error
+	q, err = man.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, input.SharableVirtualResourceListInput)
+	if err != nil {
+		return nil, errors.Wrap(err, "SSharableBaseResourceManager.ListItemFilter")
+	}
+	q, err = man.SEnabledResourceBaseManager.ListItemFilter(ctx, q, userCred, input.EnabledResourceBaseListInput)
+	if err != nil {
+		return nil, errors.Wrap(err, "SEnabledResourceBaseManager.ListItemFilter")
+	}
+
+	if len(input.ModelName) > 0 {
+		q = q.In("model_name", input.ModelName)
+	}
+	if len(input.Tag) > 0 {
+		q = q.In("tag", input.Tag)
+	}
+	if len(input.ModelId) > 0 {
+		q = q.In("model_id", input.ModelId)
+	}
+
+	if len(input.Image) > 0 {
+		s := auth.GetSession(ctx, userCred, options.Options.Region)
+		params := commonoptions.BaseListOptions{}
+		params.Scope = "max"
+		boolFalse := false
+		params.Details = &boolFalse
+		limit := 2048
+		params.Limit = &limit
+		params.Filter = []string{fmt.Sprintf("name.contains(%s)", input.Image)}
+		results, err := imagemodules.Images.List(s, jsonutils.Marshal(params))
+		if err != nil {
+			return nil, errors.Wrap(err, "List")
+		}
+		imageIds := make([]string, 0)
+		for i := range results.Data {
+			idstr, _ := results.Data[i].GetString("id")
+			imageIds = append(imageIds, idstr)
+		}
+		q = q.In("image_id", imageIds)
+	}
+
+	if len(input.Mounts) > 0 {
+		q = q.Contains("mounts", input.Mounts)
+	}
+
+	if input.AutoCache != nil {
+		q = q.Equals("auto_cache", *input.AutoCache)
+	}
+
+	return q, nil
+}
+
+// func (man *SInstantAppManager) FetchCustomizeColumns(
+// 	ctx context.Context,
+// 	userCred mcclient.TokenCredential,
+// 	query jsonutils.JSONObject,
+// 	objs []interface{},
+// 	fields stringutils2.SSortedStrings,
+// 	isList bool,
+// ) []apis.InstantAppDetails {
+// 	res := make([]apis.InstantAppDetails, len(objs))
+
+// 	imageIds := make([]string, 0)
+// 	mdlNames := make([]string, 0)
+
+// 	virows := man.SSharableVirtualResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
+// 	for i := range res {
+// 		res[i].SharableVirtualResourceDetails = virows[i]
+// 		instApp := objs[i].(*SInstantApp)
+// 		if len(instApp.ImageId) > 0 {
+// 			imageIds = append(imageIds, instApp.ImageId)
+// 		}
+// 		if len(instApp.ModelName) > 0 {
+// 			mdlNames = append(mdlNames, instApp.ModelName)
+// 		}
+// 	}
+
+// 	s := auth.GetSession(ctx, userCred, options.Options.Region)
+// 	imageMap := make(map[string]imageapi.ImageDetails)
+// 	if len(imageIds) > 0 {
+// 		params := imageapi.ImageListInput{}
+// 		params.Ids = imageIds
+// 		params.VirtualResourceListInput.Scope = "max"
+// 		details := false
+// 		params.Details = &details
+// 		limit := len(imageIds)
+// 		params.Limit = &limit
+// 		params.Field = []string{"id", "name"}
+// 		imageList, err := imagemodules.Images.List(s, jsonutils.Marshal(params))
+// 		if err != nil {
+// 			log.Errorf("list image fail %s", err)
+// 		} else {
+// 			for i := range imageList.Data {
+// 				imgDetails := imageapi.ImageDetails{}
+// 				err := imageList.Data[i].Unmarshal(&imgDetails)
+// 				if err != nil {
+// 					log.Errorf("unmarshal image info %s fail %s", imageList.Data[i], err)
+// 				} else {
+// 					imageMap[imgDetails.Id] = imgDetails
+// 				}
+// 			}
+// 		}
+// 	}
+// 	type imageCacheStatus struct {
+// 		CachedCount int
+// 		CacheCount  int
+// 	}
+// 	imageCacheStatusTbl := make(map[string]*imageCacheStatus)
+// 	if len(imageIds) > 0 {
+// 		params := commonoptions.BaseListOptions{}
+// 		params.Scope = "max"
+// 		params.Filter = []string{fmt.Sprintf("cachedimage_id.in(%s)", strings.Join(imageIds, ","))}
+// 		details := false
+// 		params.Details = &details
+// 		limit := 1024
+// 		params.Limit = &limit
+// 		params.Field = []string{"storagecache_id", "cachedimage_id", "status"}
+// 		offset := -1
+// 		total := 0
+// 		for offset < 0 || offset < total {
+// 			if offset > 0 {
+// 				params.Offset = &offset
+// 			} else {
+// 				offset = 0
+// 			}
+// 			resp, err := computemodules.Storagecachedimages.List(s, jsonutils.Marshal(params))
+// 			if err != nil {
+// 				log.Errorf("Storagecachedimages.List fail %s", err)
+// 				break
+// 			}
+// 			for i := range resp.Data {
+// 				sci := computeapi.StoragecachedimageDetails{}
+// 				err := resp.Data[i].Unmarshal(&sci)
+// 				if err != nil {
+// 					log.Errorf("unmarshal image info %s fail %s", resp.Data[i], err)
+// 				} else {
+// 					if _, ok := imageCacheStatusTbl[sci.CachedimageId]; !ok {
+// 						imageCacheStatusTbl[sci.CachedimageId] = &imageCacheStatus{}
+// 					}
+// 					if sci.Status == computeapi.CACHED_IMAGE_STATUS_ACTIVE {
+// 						imageCacheStatusTbl[sci.CachedimageId].CachedCount++
+// 					}
+// 					imageCacheStatusTbl[sci.CachedimageId].CacheCount++
+// 				}
+// 			}
+// 			offset += len(resp.Data)
+// 			total = resp.Total
+// 		}
+
+// 	}
+// 	for i := range res {
+// 		instApp := objs[i].(*SInstantApp)
+// 		if img, ok := imageMap[instApp.ImageId]; ok {
+// 			res[i].Image = img.Name
+// 		}
+// 		if status, ok := imageCacheStatusTbl[instApp.ImageId]; ok {
+// 			res[i].CacheCount = status.CacheCount
+// 			res[i].CachedCount = status.CachedCount
+// 		}
+// 	}
+// 	return res
+// }
+
+func (man *SInstantModelManager) GetLLMContainerDriver(llmType apis.LLMContainerType) ILLMContainerDriver {
+	return GetLLMContainerDriver(llmType)
+}
+
+func (man *SInstantModelManager) ValidateCreateData(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	ownerId mcclient.IIdentityProvider,
+	query jsonutils.JSONObject,
+	input apis.InstantModelCreateInput,
+) (apis.InstantModelCreateInput, error) {
+	var err error
+	input.SharableVirtualResourceCreateInput, err = man.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.SharableVirtualResourceCreateInput)
+	if err != nil {
+		return input, errors.Wrap(err, "SSharableVirtualResourceBaseManager.ValidateCreateData")
+	}
+
+	if !apis.IsLLMContainerType(string(input.LLMType)) {
+		return input, errors.Wrapf(httperrors.ErrInvalidFormat, "invalid llm_type %s", input.LLMType)
+	}
+
+	if len(input.ImageId) > 0 {
+		img, err := fetchImage(ctx, userCred, input.ImageId)
+		if err != nil {
+			return input, errors.Wrapf(err, "fetchImage %s", input.ImageId)
+		}
+		if img.DiskFormat != imageapi.IMAGE_DISK_FORMAT_TGZ {
+			return input, errors.Wrapf(errors.ErrInvalidFormat, "cannot use image as template of format %s", img.DiskFormat)
+		}
+
+		{
+			mdl, err := man.findInstantModelByImageId(img.Id)
+			if err != nil {
+				return input, errors.Wrap(err, "findInstantModelByImageId")
+			}
+			if mdl != nil {
+				return input, errors.Wrapf(httperrors.ErrConflict, "image %s has been used by other model", input.ImageId)
+			}
+		}
+
+		input.ImageId = img.Id
+		input.Size = img.Size
+		input.Status = img.Status
+		input.ActualSizeMb = img.MinDiskMB
+	}
+	if len(input.Mounts) > 0 {
+		drv := man.GetLLMContainerDriver(input.LLMType)
+		_, err = drv.ValidateMounts(input.Mounts, input.ModelName, input.Tag)
+		if err != nil {
+			return input, errors.Wrap(err, "validateMounts")
+		}
+	}
+
+	input.Enabled = nil
+	return input, nil
+}
+
+// func (app *SInstantApp) ValidateUpdateData(
+// 	ctx context.Context,
+// 	userCred mcclient.TokenCredential,
+// 	query jsonutils.JSONObject,
+// 	input apis.InstantAppUpdateInput,
+// ) (apis.InstantAppUpdateInput, error) {
+// 	var err error
+// 	input.SharableVirtualResourceBaseUpdateInput, err = app.SSharableVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, input.SharableVirtualResourceBaseUpdateInput)
+// 	if err != nil {
+// 		return input, errors.Wrap(err, "SSharableVirtualResourceBase.ValidateUpdateData")
+// 	}
+
+// 	if len(input.ImageId) > 0 {
+// 		img, err := fetchImage(ctx, userCred, input.ImageId)
+// 		if err != nil {
+// 			return input, errors.Wrapf(err, "fetchImage %s", input.ImageId)
+// 		}
+// 		if img.DiskFormat != imageapi.IMAGE_DISK_FORMAT_TGZ {
+// 			return input, errors.Wrapf(errors.ErrInvalidFormat, "cannot use image as template of format %s", img.DiskFormat)
+// 		}
+
+// 		{
+// 			findApp, err := GetInstantAppManager().findInstantAppByImageId(img.Id)
+// 			if err != nil {
+// 				return input, errors.Wrap(err, "findInstantAppByImageId")
+// 			}
+// 			if findApp != nil && findApp.Id != app.Id {
+// 				return input, errors.Wrapf(httperrors.ErrConflict, "image %s has been used by other app", input.ImageId)
+// 			}
+// 		}
+
+// 		input.ImageId = img.Id
+// 		input.Size = img.Size
+// 		input.ActualSizeMb = img.MinDiskMB
+// 	}
+// 	if len(input.Mounts) > 0 {
+// 		drv := GetInstantAppManager().GetLLMContainerDriver(apis.LLMContainerType(app.LlmType))
+// 		input.Mounts, err = drv.ValidateMounts(input.Mounts, app.Package)
+// 		if err != nil {
+// 			return input, errors.Wrap(err, "validateMounts")
+// 		}
+// 		if len(input.Mounts) == 0 {
+// 			return input, errors.Wrap(errors.ErrEmpty, "empty mounts")
+// 		}
+// 	}
+// 	return input, nil
+// }
+
+func (model *SInstantModel) PostCreate(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	ownerId mcclient.IIdentityProvider,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject,
+) {
+	model.syncImagePathMap(ctx, userCred)
+}
+
+// func (app *SInstantApp) PostUpdate(
+// 	ctx context.Context,
+// 	userCred mcclient.TokenCredential,
+// 	query jsonutils.JSONObject,
+// 	data jsonutils.JSONObject,
+// ) {
+// 	app.syncImagePathMap(ctx, userCred)
+// }
+
+func (model *SInstantModel) getImagePaths() map[string]string {
+	drv := GetInstantModelManager().GetLLMContainerDriver(apis.LLMContainerType(model.LlmType))
+	return drv.GetImageInternalPathMounts(model)
+}
+
+func (model *SInstantModel) syncImagePathMap(ctx context.Context, userCred mcclient.TokenCredential) error {
+	if len(model.ImageId) == 0 {
+		return nil
+	}
+	imgPaths := model.getImagePaths()
+	if len(imgPaths) == 0 {
+		return nil
+	}
+	s := auth.GetSession(ctx, userCred, options.Options.Region)
+	params := imageapi.ImageUpdateInput{
+		Properties: map[string]string{
+			"internal_path_map":    jsonutils.Marshal(imgPaths).String(),
+			"used_by_post_overlay": "true",
+		},
+	}
+	_, err := imagemodules.Images.Update(s, model.ImageId, jsonutils.Marshal(params))
+	if err != nil {
+		return errors.Wrap(err, "Update")
+	}
+	return nil
+}
+
+func (model *SInstantModel) PerformSyncstatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.InstantModelSyncstatusInput) (jsonutils.JSONObject, error) {
+	err := model.syncImageStatus(ctx, userCred)
+	if err != nil {
+		return nil, errors.Wrap(err, "syncImageStatus")
+	}
+	return nil, nil
+}
+
+func (model *SInstantModel) saveImageId(ctx context.Context, userCred mcclient.TokenCredential, imageId string) error {
+	_, err := db.Update(model, func() error {
+		model.ImageId = imageId
+		return nil
+	})
+	if err != nil {
+		logclient.AddActionLogWithContext(ctx, model, logclient.ACT_SAVE_IMAGE, err, userCred, false)
+		return errors.Wrap(err, "update image_id")
+	}
+	logclient.AddActionLogWithContext(ctx, model, logclient.ACT_SAVE_IMAGE, imageId, userCred, true)
+	return nil
+}
+
+func (model *SInstantModel) syncImageStatus(ctx context.Context, userCred mcclient.TokenCredential) error {
+	img, err := fetchImage(ctx, userCred, model.ImageId)
+	if err != nil {
+		if httputils.ErrorCode(err) == 404 {
+			model.SetStatus(ctx, userCred, imageapi.IMAGE_STATUS_DELETED, "not found")
+			return nil
+		}
+		return errors.Wrapf(err, "fetchImage %s", model.ImageId)
+	}
+	model.SetStatus(ctx, userCred, img.Status, "syncStatus")
+	if img.Status == imageapi.IMAGE_STATUS_ACTIVE && (model.Size != img.Size || model.ActualSizeMb != img.MinDiskMB) {
+		_, err := db.Update(model, func() error {
+			model.Size = img.Size
+			model.ActualSizeMb = img.MinDiskMB
+			return nil
+		})
+		if err != nil {
+			return errors.Wrap(err, "update size")
+		}
+	}
+	{
+		err := model.syncImagePathMap(ctx, userCred)
+		if err != nil {
+			return errors.Wrap(err, "syncImagePathMap")
+		}
+	}
+	return nil
+}
+
+func (man *SInstantModelManager) findInstantModelByImageId(imageId string) (*SInstantModel, error) {
+	q := man.Query().Equals("image_id", imageId)
+
+	mdls := make([]SInstantModel, 0)
+	err := db.FetchModelObjects(man, q, &mdls)
+	if err != nil {
+		return nil, errors.Wrap(err, "FetchModelObjects")
+	}
+	if len(mdls) == 0 {
+		return nil, nil
+	}
+	return &mdls[0], nil
+}
+
+func (man *SInstantModelManager) GetInstantModelById(id string) (*SInstantModel, error) {
+	obj, err := man.FetchById(id)
+	if err != nil {
+		return nil, errors.Wrap(err, "FetchById")
+	}
+	return obj.(*SInstantModel), nil
+}
+
+func (man *SInstantModelManager) findInstantModel(mdlId, tag string, isEnabled bool) (*SInstantModel, error) {
+	q := man.Query().Equals("model_id", mdlId).Equals("status", imageapi.IMAGE_STATUS_ACTIVE)
+	if isEnabled {
+		q = q.IsTrue("enabled")
+	}
+	q = q.Desc("created_at")
+
+	mdls := make([]SInstantModel, 0)
+	err := db.FetchModelObjects(man, q, &mdls)
+	if err != nil {
+		return nil, errors.Wrap(err, "FetchModelObjects")
+	}
+	if len(mdls) == 0 {
+		return nil, nil
+	}
+	if len(tag) > 0 {
+		for i := range mdls {
+			if mdls[i].Tag == tag {
+				return &mdls[i], nil
+			}
+		}
+	}
+	return &mdls[0], nil
+}
+
+func (model *SInstantModel) PerformEnable(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	input commonapis.PerformEnableInput,
+) (jsonutils.JSONObject, error) {
+	if len(model.ImageId) == 0 {
+		return nil, errors.Wrap(errors.ErrInvalidStatus, "empty image_id")
+	}
+	if len(model.Mounts) == 0 {
+		return nil, errors.Wrap(errors.ErrInvalidStatus, "empty mounts")
+	}
+	{
+		err := model.syncImageStatus(ctx, userCred)
+		if err != nil {
+			return nil, errors.Wrap(err, "syncImageStatus")
+		}
+	}
+	if model.Status != imageapi.IMAGE_STATUS_ACTIVE {
+		return nil, errors.Wrapf(errors.ErrInvalidStatus, "cannot enable model of status %s", model.Status)
+	}
+	// check duplicate
+	{
+		existing, err := GetInstantModelManager().findInstantModel(model.ModelId, model.Tag, true)
+		if err != nil {
+			return nil, errors.Wrap(err, "findInstantModel")
+		}
+		if existing != nil && existing.Id != model.Id {
+			return nil, errors.Wrapf(errors.ErrDuplicateId, "model of modelId %s tag %s has been enabled", model.ModelId, model.Tag)
+		}
+	}
+	_, err := db.Update(model, func() error {
+		model.SEnabledResourceBase.SetEnabled(true)
+		return nil
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "update")
+	}
+	return nil, nil
+}
+
+func (model *SInstantModel) PerformDisable(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	input commonapis.PerformDisableInput,
+) (jsonutils.JSONObject, error) {
+	_, err := db.Update(model, func() error {
+		model.SEnabledResourceBase.SetEnabled(false)
+		if model.AutoCache {
+			model.AutoCache = false
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "update")
+	}
+	return nil, nil
+}
+
+func (model *SInstantModel) PerformChangeOwner(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	input commonapis.PerformChangeProjectOwnerInput,
+) (jsonutils.JSONObject, error) {
+	// perform disk change owner
+	if len(model.ImageId) > 0 {
+		s := auth.GetSession(ctx, userCred, options.Options.Region)
+		_, err := imagemodules.Images.PerformAction(s, model.ImageId, "change-owner", jsonutils.Marshal(input))
+		if err != nil {
+			return nil, errors.Wrap(err, "image change-owner")
+		}
+	}
+	return model.SSharableVirtualResourceBase.PerformChangeOwner(ctx, userCred, query, input)
+}
+
+func (model *SInstantModel) PerformPublic(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	input commonapis.PerformPublicProjectInput,
+) (jsonutils.JSONObject, error) {
+	if len(model.ImageId) > 0 {
+		s := auth.GetSession(ctx, userCred, options.Options.Region)
+		_, err := imagemodules.Images.PerformAction(s, model.ImageId, "public", jsonutils.Marshal(input))
+		if err != nil {
+			return nil, errors.Wrap(err, "image public")
+		}
+	}
+	return model.SSharableVirtualResourceBase.PerformPublic(ctx, userCred, query, input)
+}
+
+func (model *SInstantModel) PerformPrivate(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	input commonapis.PerformPrivateInput,
+) (jsonutils.JSONObject, error) {
+	if len(model.ImageId) > 0 {
+		s := auth.GetSession(ctx, userCred, options.Options.Region)
+		_, err := imagemodules.Images.PerformAction(s, model.ImageId, "private", jsonutils.Marshal(input))
+		if err != nil {
+			return nil, errors.Wrap(err, "image private")
+		}
+	}
+	return model.SSharableVirtualResourceBase.PerformPrivate(ctx, userCred, query, input)
+}
+
+// func (app *SInstantApp) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
+// 	if app.Enabled.IsTrue() {
+// 		return errors.Wrap(errors.ErrInvalidStatus, "cannot delete when enabled")
+// 	}
+
+// 	for _, man := range []MountedAppModelManager{GetDesktopModelManager(), GetVolumeManager()} {
+// 		used, err := man.IsPremountedPackageName(app.Package)
+// 		if err != nil {
+// 			return errors.Wrap(err, "IsPremountedPackageName")
+// 		}
+// 		if used {
+// 			return errors.Wrap(errors.ErrInvalidStatus, "cannot delete when package is used by other resources")
+// 		}
+// 	}
+
+// 	return nil
+// }
+
+// func (app *SInstantApp) ValidateUpdateCondition(ctx context.Context) error {
+// 	if app.Enabled.IsTrue() {
+// 		return errors.Wrap(errors.ErrInvalidStatus, "cannot update when enabled")
+// 	}
+// 	return nil
+// }
+
+func (model *SInstantModel) PerformEnableAutoCache(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	input apis.InstantModelEnableAutoCacheInput,
+) (jsonutils.JSONObject, error) {
+	if input.AutoCache && model.Enabled.IsFalse() {
+		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "cannot enable auto_cache for disabled app")
+	}
+	_, err := db.Update(model, func() error {
+		model.AutoCache = input.AutoCache
+		return nil
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "update auto_cache")
+	}
+	if model.AutoCache {
+		err := model.doCache(ctx, userCred)
+		if err != nil {
+			return nil, errors.Wrap(err, "doCache")
+		}
+	}
+	return nil, nil
+}
+
+func (model *SInstantModel) doCache(ctx context.Context, userCred mcclient.TokenCredential) error {
+	input := computeapi.CachedImageManagerCacheImageInput{}
+	input.ImageId = model.ImageId
+	input.AutoCache = true
+	input.HostType = []string{computeapi.HOST_TYPE_CONTAINER}
+	s := auth.GetSession(ctx, userCred, options.Options.Region)
+	_, err := computemodules.Cachedimages.PerformClassAction(s, "cache-image", jsonutils.Marshal(input))
+	if err != nil {
+		return errors.Wrap(err, "PerformClassAction cache-image")
+	}
+	return nil
+}
+
+// func (manager *SInstantAppManager) PerformImport(
+// 	ctx context.Context,
+// 	userCred mcclient.TokenCredential,
+// 	query jsonutils.JSONObject,
+// 	input apis.InstantAppImportInput,
+// ) (*SInstantApp, error) {
+// 	if input.Invalid() {
+// 		return nil, httperrors.NewInputParameterError("invalid input: %s", jsonutils.Marshal(input).String())
+// 	}
+// 	// first create a temporary instant-app
+// 	tempApp := &SInstantApp{}
+// 	tempApp.SetModelManager(manager, &SInstantApp{})
+// 	tempApp.Name = fmt.Sprintf("tmp-instant-app-%s.%s", timeutils.CompactTime(time.Now()), utils.GenRequestId(6))
+// 	tempApp.Package = "temp"
+// 	tempApp.Version = "0.0.1"
+// 	tempApp.ProjectId = userCred.GetProjectId()
+
+// 	err := manager.TableSpec().Insert(ctx, tempApp)
+// 	if err != nil {
+// 		return nil, errors.Wrap(err, "Insert")
+// 	}
+
+// 	err = tempApp.startImportTask(ctx, userCred, input)
+// 	if err != nil {
+// 		return nil, errors.Wrap(err, "startImportTask")
+// 	}
+
+// 	return tempApp, nil
+// }
+
+// func (instantapp *SInstantApp) startImportTask(ctx context.Context, userCred mcclient.TokenCredential, input apis.InstantAppImportInput) error {
+// 	params := jsonutils.NewDict()
+// 	params.Add(jsonutils.Marshal(input), "import_input")
+
+// 	task, err := taskman.TaskManager.NewTask(ctx, "InstantAppImportTask", instantapp, userCred, params, "", "")
+// 	if err != nil {
+// 		return errors.Wrap(err, "NewTask")
+// 	}
+// 	task.ScheduleRun(nil)
+
+// 	return nil
+// }
+
+// func (instantapp *SInstantApp) DoImport(ctx context.Context, userCred mcclient.TokenCredential, input apis.InstantAppImportInput) error {
+// 	// first download the image
+// 	cfg := objectstore.NewObjectStoreClientConfig(input.Endpoint, input.AccessKey, input.SecretKey)
+// 	if len(input.SignVer) > 0 {
+// 		cfg.SignVersion(objectstore.S3SignVersion(input.SignVer))
+// 	}
+// 	minioClient, err := objectstore.NewObjectStoreClient(cfg)
+// 	if err != nil {
+// 		return errors.Wrap(err, "new minio client")
+// 	}
+
+// 	bucket, err := minioClient.GetIBucketByName(input.Bucket)
+// 	if err != nil {
+// 		return errors.Wrap(err, "GetIBucketByName")
+// 	}
+
+// 	tmpDir, err := os.MkdirTemp(options.Options.AdbWorkingDirectory, "instant-app-*")
+// 	if err != nil {
+// 		return errors.Wrap(err, "CreateTemp")
+// 	}
+// 	defer func() {
+// 		os.RemoveAll(tmpDir)
+// 	}()
+
+// 	tmpFileName := filepath.Join(tmpDir, "instant-app.tar")
+
+// 	// download the object
+// 	err = func() error {
+// 		tmpFile, err := os.Create(tmpFileName)
+// 		if err != nil {
+// 			return errors.Wrap(err, "Create")
+// 		}
+// 		defer tmpFile.Close()
+
+// 		_, err = cloudprovider.DownloadObjectParallelWithProgress(ctx, bucket, input.Key, nil, tmpFile, 0, 1024*1024*10, false, 3, func(progress float64, progressMbps float64, totalSizeMb int64) {
+// 			log.Infof("DownloadObjectParallelWithProgress progress: %f, progressMbps: %f, totalSizeMb: %d", progress, progressMbps, totalSizeMb)
+// 		})
+// 		if err != nil {
+// 			return errors.Wrap(err, "DownloadObjectParallel")
+// 		}
+// 		return nil
+// 	}()
+// 	if err != nil {
+// 		return errors.Wrap(err, "download object")
+// 	}
+
+// 	// untar the object
+// 	err = procutils.NewCommand("tar", "xf", tmpFileName, "-C", tmpDir, "--strip-components=1").Run()
+// 	if err != nil {
+// 		return errors.Wrapf(err, "untar %s", tmpFileName)
+// 	}
+
+// 	scriptPath := filepath.Join(tmpDir, "scripts")
+// 	imagePath := filepath.Join(tmpDir, "image")
+
+// 	params, err := decodeParams(scriptPath)
+// 	if err != nil {
+// 		return errors.Wrap(err, "decodeParams")
+// 	}
+
+// 	s := auth.GetSession(ctx, userCred, options.Options.Region)
+
+// 	// upload the image
+// 	imageId, err := func() (string, error) {
+// 		imgFile, err := os.Open(imagePath)
+// 		if err != nil {
+// 			return "", errors.Wrap(err, "Open")
+// 		}
+// 		defer imgFile.Close()
+
+// 		imgFileStat, err := imgFile.Stat()
+// 		if err != nil {
+// 			return "", errors.Wrap(err, "Stat")
+// 		}
+// 		imgFileSize := imgFileStat.Size()
+
+// 		imgParams := imageapi.ImageCreateInput{}
+// 		imgParams.GenerateName = params.ImageName
+// 		imgParams.DiskFormat = "tgz"
+// 		imgParams.Size = &imgFileSize
+// 		imgParams.Properties = map[string]string{
+// 			"os_arch": "aarch64",
+// 		}
+
+// 		// upload the image
+// 		imageObj, err := imagemodules.Images.Upload(s, jsonutils.Marshal(imgParams), imgFile, imgFileSize)
+// 		if err != nil {
+// 			return "", errors.Wrap(err, "Create")
+// 		}
+// 		imageId, err := imageObj.GetString("id")
+// 		if err != nil {
+// 			return "", errors.Wrap(err, "GetId")
+// 		}
+
+// 		return imageId, nil
+// 	}()
+// 	if err != nil {
+// 		return errors.Wrap(err, "upload image")
+// 	}
+
+// 	newName, err := db.GenerateAlterName(instantapp, params.InstantAppName)
+// 	if err != nil {
+// 		return errors.Wrap(err, "GenerateAlterName")
+// 	}
+// 	// update the instant-app
+// 	_, err = db.Update(instantapp, func() error {
+// 		instantapp.Name = newName
+// 		instantapp.Package = params.Package
+// 		instantapp.Version = params.Version
+// 		instantapp.ImageId = imageId
+// 		instantapp.Mounts = []string{params.AppMount, params.DataMount}
+// 		if len(params.MediaMount) > 0 {
+// 			instantapp.Mounts = append(instantapp.Mounts, params.MediaMount)
+// 		}
+// 		instantapp.Status = imageapi.IMAGE_STATUS_SAVING
+// 		return nil
+// 	})
+// 	if err != nil {
+// 		return errors.Wrap(err, "update instant-app")
+// 	}
+
+// 	// wait image to be active
+// 	imgDetails, err := instantapp.WaitImageStatus(ctx, userCred, []string{imageapi.IMAGE_STATUS_ACTIVE}, 1800)
+// 	if err != nil {
+// 		log.Errorf("WaitImageStatus failed: %s", err)
+// 	}
+
+// 	// sync image status
+// 	err = instantapp.syncImageStatus(ctx, userCred)
+// 	if err != nil {
+// 		return errors.Wrap(err, "syncImageStatus")
+// 	}
+
+// 	if imgDetails.Status == imageapi.IMAGE_STATUS_KILLED || imgDetails.Status == imageapi.IMAGE_STATUS_DEACTIVATED {
+// 		return errors.Wrapf(httperrors.ErrInvalidStatus, "image status: %s", imgDetails.Status)
+// 	}
+
+// 	return nil
+// }
+
+// type sInstantAppImportParams struct {
+// 	ImageName      string
+// 	InstantAppName string
+// 	Package        string
+// 	Version        string
+// 	AppMount       string
+// 	DataMount      string
+// 	MediaMount     string
+// }
+
+// func decodeParams(fileDir string) (*sInstantAppImportParams, error) {
+// 	content, err := os.ReadFile(fileDir)
+// 	if err != nil {
+// 		return nil, errors.Wrap(err, "ReadFile")
+// 	}
+// 	return decodeParamsString(string(content))
+// }
+
+// const scriptFormat = `/root/climc image-upload --format tgz --os-arch aarch64 (?P<image_name>.*) \./image
+// /root/climc instant-app-create (?P<instant_app_name>.*) (?P<package>.*) (?P<version>.*) \\
+// --mounts "(?P<app_mount>.*)" \\
+// --mounts "(?P<data_mount>.*)" \\
+// (--mounts "(?P<media_mount>.*)")?`
+
+// var (
+// 	scriptRE = regexp.MustCompile(scriptFormat)
+// )
+
+// func decodeParamsString(content string) (*sInstantAppImportParams, error) {
+// 	params := sInstantAppImportParams{}
+// 	matches := scriptRE.FindStringSubmatch(content)
+
+// 	log.Debugf("decodeParamsString matches: %v", jsonutils.Marshal(matches))
+
+// 	if len(matches) > 6 {
+// 		params.ImageName = matches[1]
+// 		params.InstantAppName = matches[2]
+// 		params.Package = matches[3]
+// 		params.Version = matches[4]
+// 		params.AppMount = matches[5]
+// 		params.DataMount = matches[6]
+// 		if len(matches) > 8 {
+// 			params.MediaMount = matches[8]
+// 		}
+// 	}
+// 	return &params, nil
+// }
+
+func (model *SInstantModel) GetImage(ctx context.Context, userCred mcclient.TokenCredential) (*imageapi.ImageDetails, error) {
+	s := auth.GetSession(ctx, userCred, options.Options.Region)
+	imageObj, err := imagemodules.Images.Get(s, model.ImageId, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "Get")
+	}
+	imgDetail := imageapi.ImageDetails{}
+	err = imageObj.Unmarshal(&imgDetail)
+	if err != nil {
+		return nil, errors.Wrap(err, "Unmarshal")
+	}
+	return &imgDetail, nil
+}
+
+func (model *SInstantModel) WaitImageStatus(ctx context.Context, userCred mcclient.TokenCredential, targetStatus []string, timeoutSecs int) (*imageapi.ImageDetails, error) {
+	expire := time.Now().Add(time.Second * time.Duration(timeoutSecs))
+	for time.Now().Before(expire) {
+		img, err := model.GetImage(ctx, userCred)
+		if err != nil {
+			return nil, errors.Wrap(err, "GetImage")
+		}
+		if utils.IsInArray(img.Status, targetStatus) {
+			return img, nil
+		}
+		if strings.Contains(img.Status, "fail") || img.Status == imageapi.IMAGE_STATUS_KILLED || img.Status == imageapi.IMAGE_STATUS_DEACTIVATED {
+			return nil, errors.Wrap(errors.ErrInvalidStatus, img.Status)
+		}
+		time.Sleep(2 * time.Second)
+	}
+	return nil, errors.Wrapf(httperrors.ErrTimeout, "wait image status %s timeout", targetStatus)
+}
+
+func (model *SInstantModel) GetActualSizeMb() int32 {
+	if model.ActualSizeMb > 0 {
+		return model.ActualSizeMb
+	}
+	return int32(model.Size / 1024 / 1024)
+}

pkg/llm/models/llm.go

@@ -17,6 +17,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/httperrors"
+	llmutils "yunion.io/x/onecloud/pkg/llm/utils"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules/compute"
 )
@@ -50,8 +51,11 @@ type SLLMManager struct {
 type SLLM struct {
 	SLLMBase
 
-	LLMModelId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"`
+	LLMSkuId   string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"`
 	LLMImageId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"`
+
+	// 秒装应用配额（可安装的总容量限制）
+	InstantModelQuotaGb int `list:"user" update:"user" create:"optional" default:"0" nullable:"false"`
 }
 
 func (man *SLLMManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.LLMCreateInput) (*api.LLMCreateInput, error) {
@@ -60,13 +64,13 @@ func (man *SLLMManager) ValidateCreateData(ctx context.Context, userCred mcclien
 	if err != nil {
 		return input, errors.Wrap(err, "validate LLMBaseCreateInput")
 	}
-	model, err := GetLLMModelManager().FetchByIdOrName(ctx, userCred, input.LLMModelId)
+	sku, err := GetLLMSkuManager().FetchByIdOrName(ctx, userCred, input.LLMSkuId)
 	if err != nil {
-		return input, errors.Wrap(err, "fetch LLMModel")
+		return input, errors.Wrap(err, "fetch LLMSku")
 	}
-	lModel := model.(*SLLMModel)
-	input.LLMModelId = lModel.Id
-	input.LLMImageId = lModel.LLMImageId
+	lSku := sku.(*SLLMSku)
+	input.LLMSkuId = lSku.Id
+	input.LLMImageId = lSku.LLMImageId
 
 	return input, nil
 }
@@ -85,16 +89,16 @@ func (man *SLLMManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery,
 		return q, errors.Wrap(err, "VirtualResourceBaseManager.ListItemFilter")
 	}
 
-	if len(input.LLMModel) > 0 {
-		modelObj, err := GetLLMModelManager().FetchByIdOrName(ctx, userCred, input.LLMModel)
+	if len(input.LLMSku) > 0 {
+		skuObj, err := GetLLMSkuManager().FetchByIdOrName(ctx, userCred, input.LLMSku)
 		if err != nil {
 			if errors.Cause(err) == sql.ErrNoRows {
-				return nil, httperrors.NewResourceNotFoundError2(GetLLMModelManager().KeywordPlural(), input.LLMModel)
+				return nil, httperrors.NewResourceNotFoundError2(GetLLMSkuManager().KeywordPlural(), input.LLMSku)
 			} else {
-				return nil, errors.Wrap(err, "LLMModelManager.FetchByIdOrName")
+				return nil, errors.Wrap(err, "GetLLMSkuManager.FetchByIdOrName")
 			}
 		}
-		q = q.Equals("llm_model_id", modelObj.GetId())
+		q = q.Equals("llm_sku_id", skuObj.GetId())
 	}
 	if len(input.LLMImage) > 0 {
 		imgObj, err := GetLLMImageManager().FetchByIdOrName(ctx, userCred, input.LLMImage)
@@ -158,27 +162,29 @@ func (llm *SLLM) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCre
 	return llm.StartDeleteTask(ctx, userCred, "")
 }
 
-func (llm *SLLM) GetLLMModel(modelId string) (*SLLMModel, error) {
-	if len(modelId) == 0 {
-		modelId = llm.LLMModelId
+func (llm *SLLM) GetLLMSku(skuId string) (*SLLMSku, error) {
+	if len(skuId) == 0 {
+		skuId = llm.LLMSkuId
 	}
-	model, err := GetLLMModelManager().FetchById(modelId)
+	sku, err := GetLLMSkuManager().FetchById(skuId)
 	if err != nil {
-		return nil, errors.Wrap(err, "fetch LLMModel")
+		return nil, errors.Wrap(err, "fetch LLMSku")
 	}
-	return model.(*SLLMModel), nil
+	return sku.(*SLLMSku), nil
 }
 
-func (llm *SLLM) GetLargeLanguageModelName() (modelName string, modelTag string, err error) {
-	model, err := llm.GetLLMModel("")
-	if err != nil {
-		return "", "", err
+func (llm *SLLM) GetLargeLanguageModelName(name string) (modelName string, modelTag string, err error) {
+	if name == "" {
+		sku, err := llm.GetLLMSku("")
+		if err != nil {
+			return "", "", err
+		}
+		name = sku.LLMModelName
 	}
-	name := model.LLMModelName
 	parts := strings.Split(name, ":")
 	modelName = parts[0]
 	modelTag = "latest"
-	if len(parts) > 1 {
+	if len(parts) == 2 {
 		modelTag = parts[1]
 	}
 	return
@@ -188,13 +194,21 @@ func (llm *SLLM) GetLLMImage() (*SLLMImage, error) {
 	return llm.getImage(llm.LLMImageId)
 }
 
+func (llm *SLLM) GetLLMSContainer(ctx context.Context) (*computeapi.SContainer, error) {
+	llmCtr, err := llm.GetLLMContainer()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetLLMContainer")
+	}
+	return llmCtr.GetSContainer(ctx)
+}
+
 func (llm *SLLM) GetLLMContainer() (*SLLMContainer, error) {
 	return GetLLMContainerManager().FetchByLLMId(llm.Id)
 }
 
 func (llm *SLLM) GetLLMContainerDriver() ILLMContainerDriver {
-	model, _ := llm.GetLLMModel(llm.LLMModelId)
-	return model.GetLLMContainerDriver()
+	sku, _ := llm.GetLLMSku(llm.LLMSkuId)
+	return sku.GetLLMContainerDriver()
 }
 
 func (llm *SLLM) StartCreateTask(ctx context.Context, userCred mcclient.TokenCredential, input api.LLMCreateInput, parentTaskId string) error {
@@ -232,16 +246,16 @@ func (llm *SLLM) StartDeleteTask(ctx context.Context, userCred mcclient.TokenCre
 }
 
 func (llm *SLLM) ServerCreate(ctx context.Context, userCred mcclient.TokenCredential, s *mcclient.ClientSession, input *api.LLMCreateInput) (string, error) {
-	model, err := llm.GetLLMModel(llm.LLMModelId)
+	sku, err := llm.GetLLMSku(llm.LLMSkuId)
 	if nil != err {
-		return "", errors.Wrap(err, "GetLLMModel")
+		return "", errors.Wrap(err, "GetLLMSku")
 	}
 	llmImage, err := llm.GetLLMImage()
 	if nil != err {
 		return "", errors.Wrap(err, "GetLLMImage")
 	}
 
-	data, err := GetLLMPodCreateInput(ctx, userCred, input, llm, model, llmImage, "")
+	data, err := GetLLMPodCreateInput(ctx, userCred, input, llm, sku, llmImage, "")
 	if nil != err {
 		return "", errors.Wrap(err, "GetPodCreateInput")
 	}
@@ -314,6 +328,31 @@ func (llm *SLLM) StartLLMStopTask(ctx context.Context, userCred mcclient.TokenCr
 // 	return nil
 // }
 
-// func (llm *SLLM) WaitContainerStatus(ctx context.Context, userCred mcclient.TokenCredential, targetStatus []string, timeoutSecs int) (*computeapi.SContainer, error) {
-// 	return nil, nil
-// }
+func (llm *SLLM) WaitContainerStatus(ctx context.Context, userCred mcclient.TokenCredential, targetStatus []string, timeoutSecs int) (*computeapi.SContainer, error) {
+	llmCtr, err := llm.GetLLMContainer()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetLLMContainer")
+	}
+	return llmutils.WaitContainerStatus(ctx, llmCtr.CmpId, targetStatus, timeoutSecs)
+}
+
+func (llm *SLLM) PerformSyncstatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data api.LLMSyncStatusInput) (jsonutils.JSONObject, error) {
+	llm.SetStatus(ctx, userCred, api.LLM_STATUS_START_SYNCSTATUS, "perform syncstatus")
+	err := llm.StartSyncStatusTask(ctx, userCred, "")
+	if err != nil {
+		return nil, errors.Wrap(err, "StartSyncStatusTask")
+	}
+	return nil, nil
+}
+
+func (llm *SLLM) StartSyncStatusTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error {
+	task, err := taskman.TaskManager.NewTask(ctx, "LLMSyncStatusTask", llm, userCred, nil, parentTaskId, "")
+	if err != nil {
+		return errors.Wrap(err, "NewTask")
+	}
+	err = task.ScheduleRun(nil)
+	if err != nil {
+		return errors.Wrap(err, "ScheduleRun")
+	}
+	return nil
+}

pkg/llm/models/llm_base.go

@@ -48,7 +48,7 @@ type SLLMBase struct {
 	Priority    int `nullable:"false" default:"100" list:"user"`
 	BandwidthMb int `nullable:"true" list:"user" create:"admin_optional"`
 
-	LastAppProbe time.Time `nullable:"true" list:"user" create:"admin_optional"`
+	LastInstantModelProbe time.Time `nullable:"true" list:"user" create:"admin_optional"`
 
 	// 是否请求同步更新镜像
 	SyncImageRequest bool `default:"false" nullable:"false" list:"user" update:"user"`

pkg/llm/models/llm_base_pod.go

@@ -25,7 +25,7 @@ func GetLLMBasePodCreateInput(
 	userCred mcclient.TokenCredential,
 	input *api.LLMBaseCreateInput,
 	llmBase *SLLMBase,
-	skuBase *SLLMModelBase,
+	skuBase *SLLMSkuBase,
 	eip string,
 ) (*computeapi.ServerCreateInput, error) {
 	data := computeapi.ServerCreateInput{}

pkg/llm/models/llm_container.go

@@ -7,9 +7,11 @@ import (
 	"yunion.io/x/pkg/errors"
 
 	commonapi "yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/apis/compute"
 	api "yunion.io/x/onecloud/pkg/apis/llm"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/llm/utils"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
@@ -76,6 +78,10 @@ func (m *SLLMContainerManager) FetchByLLMId(
 	return lc, nil
 }
 
+func (lc *SLLMContainer) GetSContainer(ctx context.Context) (*compute.SContainer, error) {
+	return utils.GetContainer(ctx, lc.CmpId)
+}
+
 func (lc *SLLMContainer) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
 	return lc.SVirtualResourceBase.Delete(ctx, userCred)
 }

pkg/llm/models/llm_container_driver.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"sync"
 
+	commonapi "yunion.io/x/onecloud/pkg/apis"
 	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/apis/llm"
 	"yunion.io/x/onecloud/pkg/httperrors"
@@ -66,11 +67,29 @@ type ILLMContainerPullModel interface {
 	CopyBlobs(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM) error
 }
 
+type ILLMContainerInstantApp interface {
+	GetProbedInstantModelsExt(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, mdlIds ...string) (map[string]llm.LLMInternalInstantMdlInfo, error)
+	DetectModelPaths(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, pkgInfo llm.LLMInternalInstantMdlInfo) ([]string, error)
+
+	GetImageInternalPathMounts(sApp *SInstantModel) map[string]string
+	GetSaveDirectories(sApp *SInstantModel) (string, []string, error)
+
+	ValidateMounts(mounts []string, mdlName string, mdlTag string) ([]string, error)
+	GetInstantModelIdByPostOverlay(postOverlay *commonapi.ContainerVolumeMountDiskPostOverlay, mdlNameToId map[string]string) string
+	GetDirPostOverlay(dir llm.LLMMountDirInfo) *commonapi.ContainerVolumeMountDiskPostOverlay
+
+	PreInstallModel(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, instMdl *SLLMInstantModel) error
+	InstallModel(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, dirs []string, mdlIds []string) error
+	UninstallModel(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, instMdl *SLLMInstantModel) error
+}
+
 type ILLMContainerDriver interface {
 	GetType() llm.LLMContainerType
-	GetContainerSpec(ctx context.Context, llm *SLLM, image *SLLMImage, sku *SLLMModel, props []string, devices []computeapi.SIsolatedDevice, diskId string) *computeapi.PodContainerCreateInput
+	GetContainerSpec(ctx context.Context, llm *SLLM, image *SLLMImage, sku *SLLMSku, props []string, devices []computeapi.SIsolatedDevice, diskId string) *computeapi.PodContainerCreateInput
 
 	// ILLMContainerPullModel
+
+	ILLMContainerInstantApp
 }
 
 var (

pkg/llm/models/llm_instant_model.go

@@ -0,0 +1,226 @@
+package models
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"time"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/sqlchemy"
+
+	api "yunion.io/x/onecloud/pkg/apis/llm"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+)
+
+var llmInstantModelManager *SLLMInstantModelManager
+
+func init() {
+	GetLLMInstantModelManager()
+}
+
+func GetLLMInstantModelManager() *SLLMInstantModelManager {
+	if llmInstantModelManager == nil {
+		llmInstantModelManager = &SLLMInstantModelManager{
+			SResourceBaseManager: db.NewResourceBaseManager(
+				SLLMInstantModel{},
+				"llm_instant_models_tbl",
+				"llm_with_instant_model",
+				"llm_with_instant_models",
+			),
+		}
+		llmInstantModelManager.SetVirtualObject(llmInstantModelManager)
+	}
+	return llmInstantModelManager
+}
+
+type SLLMInstantModelManager struct {
+	db.SResourceBaseManager
+	db.SStatusResourceBaseManager
+}
+
+type SLLMInstantModel struct {
+	db.SResourceBase
+	db.SStatusResourceBase
+
+	// InstantModelId string `name:"model_id" width:"128" charset:"ascii" nullable:"false" list:"user" primary:"true"`
+	LlmId string `width:"128" charset:"ascii" nullable:"false" list:"user" primary:"true"`
+	// Model ID, large language model's ID, referring to special model, such as qwen3:8b
+	ModelId string `name:"model_id" width:"128" charset:"ascii" nullable:"false" list:"user" primary:"true"`
+
+	// Model Tag
+	Tag string `width:"64" charset:"utf8" nullable:"true" list:"user"`
+	// Model Name
+	ModelName string `width:"128" charset:"utf8" nullable:"false" list:"user"`
+	IsProbed  bool   `list:"user"`
+	IsMounted bool   `list:"user"`
+	// IsSystem    tristate.TriState `list:"user"`
+}
+
+func (man *SLLMInstantModelManager) fetchLLMInstantModel(llmId string, mdlId string) (*SLLMInstantModel, error) {
+	q := man.RawQuery().Equals("llm_id", llmId).Equals("model_id", mdlId)
+	llmInstantModel := SLLMInstantModel{}
+	err := q.First(&llmInstantModel)
+	if err != nil {
+		if errors.Cause(err) == sql.ErrNoRows {
+			return nil, errors.ErrNotFound
+		}
+		return nil, errors.Wrap(err, "Query")
+	}
+	llmInstantModel.SetModelManager(man, &llmInstantModel)
+	return &llmInstantModel, nil
+}
+
+func (man *SLLMInstantModelManager) updateInstantModel(ctx context.Context, llmId string, mdlId string, mdlName string, tag string, probed, mounted *bool) (*SLLMInstantModel, error) {
+	mdl, err := man.fetchLLMInstantModel(llmId, mdlId)
+	if err != nil && errors.Cause(err) != errors.ErrNotFound {
+		return nil, errors.Wrap(err, "updateInstantModel")
+	}
+
+	mountStr := "nil"
+	probedStr := "nil"
+	if mounted != nil {
+		mountStr = fmt.Sprintf("%v", *mounted)
+	}
+	if probed != nil {
+		probedStr = fmt.Sprintf("%v", *probed)
+	}
+	log.Debugf("=======updateInstantModel %#v to mounted %s, probed %s", mdl, mountStr, probedStr)
+
+	if mdl == nil {
+		// if no such app
+		mdl = &SLLMInstantModel{
+			LlmId:     llmId,
+			ModelId:   mdlId,
+			ModelName: mdlName,
+			// IsSystem:    tristate.None,
+			// Entry:       entry,
+		}
+		// if isSys != nil {
+		// 	if *isSys {
+		// 		mdl.IsSystem = tristate.True
+		// 	} else {
+		// 		mdl.IsSystem = tristate.False
+		// 	}
+		// }
+		mdl.Tag = tag
+		if probed != nil {
+			mdl.IsProbed = *probed
+		}
+		if mounted != nil {
+			mdl.IsMounted = *mounted
+		}
+		mdl.syncStatus()
+		err := man.TableSpec().Insert(ctx, mdl)
+		if err != nil {
+			return nil, errors.Wrap(err, "Insert")
+		}
+		return mdl, nil
+	} else {
+		_, err := db.Update(mdl, func() error {
+			if len(tag) > 0 {
+				mdl.Tag = tag
+			}
+			if len(mdlName) > 0 {
+				mdl.ModelName = mdlName
+			}
+			// if isSys != nil {
+			// 	if *isSys {
+			// 		mdl.IsSystem = tristate.True
+			// 	} else {
+			// 		mdl.IsSystem = tristate.False
+			// 	}
+			// }
+			// if len(entry) > 0 {
+			// 	app.Entry = entry
+			// }
+			if probed != nil {
+				mdl.IsProbed = *probed
+				if mdl.IsProbed {
+					mdl.Status = api.LLM_STATUS_READY
+				} else {
+					mdl.Status = api.LLM_STATUS_DELETED
+				}
+			}
+
+			if mounted != nil {
+				mdl.IsMounted = *mounted
+			}
+			mdl.syncStatus()
+			return nil
+		})
+		if err != nil {
+			return nil, errors.Wrap(err, "Update")
+		}
+	}
+	return mdl, nil
+}
+
+func (man *SLLMInstantModelManager) filterModels(q *sqlchemy.SQuery, isProbed, isMounted, isSystem *bool) *sqlchemy.SQuery {
+	if isProbed != nil {
+		if *isProbed {
+			q = q.IsTrue("is_probed")
+		} else {
+			q = q.IsFalse("is_probed")
+		}
+	}
+	if isMounted != nil {
+		if *isMounted {
+			q = q.IsTrue("is_mounted")
+		} else {
+			q = q.IsFalse("is_mounted")
+		}
+	}
+	if isSystem != nil {
+		if *isSystem {
+			q = q.IsTrue("is_system")
+		} else {
+			q = q.Filter(sqlchemy.OR(
+				sqlchemy.IsNull(q.Field("is_system")),
+				sqlchemy.IsFalse(q.Field("is_system")),
+			))
+		}
+	}
+	return q
+}
+
+func (mdl *SLLMInstantModel) FindInstantModel(isInstall bool) (*SInstantModel, error) {
+	instMdl, err := GetInstantModelManager().findInstantModel(mdl.ModelId, mdl.Tag, isInstall)
+	if err != nil {
+		return nil, errors.Wrap(err, "FindInstantModel")
+	}
+	return instMdl, nil
+}
+
+func (mdl *SLLMInstantModel) syncStatus() {
+	if !mdl.IsProbed && !mdl.IsMounted {
+		mdl.MarkDelete()
+	} else {
+		mdl.Deleted = false
+		mdl.DeletedAt = time.Time{}
+	}
+}
+
+func (mdl *SLLMInstantModel) getMountPaths(isInstall bool) ([]api.LLMMountDirInfo, error) {
+	info, err := mdl.getMountPathsFromImage(isInstall)
+	if err != nil {
+		return nil, errors.Wrap(err, "getMountPathsFromImage")
+	}
+	return info, nil
+}
+
+func (mdl *SLLMInstantModel) getMountPathsFromImage(isInstall bool) ([]api.LLMMountDirInfo, error) {
+	instMdl, err := mdl.FindInstantModel(isInstall)
+	if err != nil {
+		return nil, errors.Wrap(err, "findInstantApp")
+	}
+	if instMdl == nil {
+		return nil, nil
+	}
+	info := make([]api.LLMMountDirInfo, 0)
+	info = append(info, api.LLMMountDirInfo{
+		ImageId: instMdl.ImageId,
+	})
+	return info, nil
+}

pkg/llm/models/llm_instant_model_quota.go

@@ -0,0 +1,59 @@
+package models
+
+type LLMInstanceModelPendingTask struct {
+	LLMId  string
+	TaskId string
+	SizeGb float64
+}
+
+var llmPendingInstantModelTasks []LLMInstanceModelPendingTask
+
+func init() {
+	llmPendingInstantModelTasks = make([]LLMInstanceModelPendingTask, 0)
+}
+
+func (llm *SLLM) GetInstantModelSizeGb() float64 {
+	boolTrue := true
+	models, err := llm.FetchModels(nil, &boolTrue, nil)
+	if err != nil {
+		return 0
+	}
+	totalSizeGb := 0.0
+	for _, model := range models {
+		instModel, _ := GetInstantModelManager().findInstantModel(model.ModelId, model.Tag, false)
+		if instModel == nil {
+			continue
+		}
+		totalSizeGb += float64(instModel.GetActualSizeMb()) * 1024 * 1024 / 1000 / 1000 / 1000
+	}
+	return totalSizeGb
+}
+
+func (llm *SLLM) GetPendingInstantModelSizeGb() float64 {
+	var totalSizeGb float64
+	for _, task := range llmPendingInstantModelTasks {
+		totalSizeGb += task.SizeGb
+	}
+	return totalSizeGb
+}
+
+func (llm *SLLM) GetTotalInstantModelSizeGb() float64 {
+	return llm.GetInstantModelSizeGb() + llm.GetPendingInstantModelSizeGb()
+}
+
+func (llm *SLLM) insertPendingInstantModelQuota(taskId string, sizeGb float64) {
+	llmPendingInstantModelTasks = append(llmPendingInstantModelTasks, LLMInstanceModelPendingTask{
+		LLMId:  llm.Id,
+		TaskId: taskId,
+		SizeGb: sizeGb,
+	})
+}
+
+func (llm *SLLM) ClearPendingInstantModelQuota(taskId string) {
+	for i := range llmPendingInstantModelTasks {
+		if llmPendingInstantModelTasks[i].TaskId == taskId {
+			llmPendingInstantModelTasks = append(llmPendingInstantModelTasks[:i], llmPendingInstantModelTasks[i+1:]...)
+			break
+		}
+	}
+}

pkg/llm/models/llm_instant_model_sync.go

@@ -0,0 +1,671 @@
+package models
+
+import (
+	"context"
+	"database/sql"
+	"net/http"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/utils"
+
+	commonapi "yunion.io/x/onecloud/pkg/apis"
+	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
+	apis "yunion.io/x/onecloud/pkg/apis/llm"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/compute"
+	"yunion.io/x/onecloud/pkg/util/logclient"
+)
+
+func (llm *SLLM) getMountedInstantModels(ctx context.Context, probedExt map[string]apis.LLMInternalInstantMdlInfo) (map[string]struct{}, error) {
+	container, err := llm.GetLLMSContainer(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "GetSContainer")
+	}
+	if container.Spec == nil {
+		return nil, errors.Wrap(errors.ErrEmpty, "no Spec")
+	}
+	if len(container.Spec.VolumeMounts) == 0 {
+		return nil, errors.Wrap(errors.ErrEmpty, "no VolumeMounts")
+	}
+	if container.Spec.VolumeMounts[0].Disk == nil {
+		return nil, errors.Wrap(errors.ErrEmpty, "no Disk")
+	}
+	if len(container.Spec.VolumeMounts[0].Disk.PostOverlay) == 0 {
+		return nil, nil
+	}
+	mdlNameToId := make(map[string]string)
+	for mdlId, model := range probedExt {
+		mdlNameToId[model.Name+":"+model.Tag] = mdlId
+	}
+	mdlMap := make(map[string]struct{})
+	postOverlays := container.Spec.VolumeMounts[0].Disk.PostOverlay
+	drv := llm.GetLLMContainerDriver()
+	for i := range postOverlays {
+		postOverlay := postOverlays[i]
+		mdlId := drv.GetInstantModelIdByPostOverlay(postOverlay, mdlNameToId)
+		if mdlId != "" {
+			mdlMap[mdlId] = struct{}{}
+		}
+	}
+	return mdlMap, nil
+}
+
+func (llm *SLLM) getProbedInstantModelsExt(ctx context.Context, userCred mcclient.TokenCredential, instantModelIds ...string) (map[string]apis.LLMInternalInstantMdlInfo, error) {
+	drv := llm.GetLLMContainerDriver()
+	return drv.GetProbedInstantModelsExt(ctx, userCred, llm, instantModelIds...)
+}
+
+type sInstantModelStatus struct {
+	apis.LLMInternalInstantMdlInfo
+
+	Probed  bool
+	Mounted bool
+}
+
+func (llm *SLLM) getProbedMountedInstantModels(ctx context.Context, userCred mcclient.TokenCredential) (map[string]*sInstantModelStatus, error) {
+	mdlMap := make(map[string]*sInstantModelStatus)
+	probedExt, errExt := llm.getProbedInstantModelsExt(ctx, userCred)
+	if errExt != nil {
+		return nil, errors.Wrap(errExt, "getProbedInstantModelsExt")
+	}
+
+	for modelId := range probedExt {
+		mdlMap[modelId] = &sInstantModelStatus{
+			LLMInternalInstantMdlInfo: probedExt[modelId],
+			Probed:                    true,
+		}
+	}
+
+	mounted, err := llm.getMountedInstantModels(ctx, probedExt)
+	if err != nil {
+		return nil, errors.Wrap(err, "llm.getMountedInstantModels")
+	}
+	for mdlId := range mounted {
+		if _, ok := mdlMap[mdlId]; ok {
+			mdlMap[mdlId].Mounted = true
+		} else {
+			mdlMap[mdlId] = &sInstantModelStatus{
+				LLMInternalInstantMdlInfo: apis.LLMInternalInstantMdlInfo{
+					ModelId: mdlId,
+				},
+				Mounted: true,
+			}
+		}
+	}
+	return mdlMap, nil
+}
+
+func (llm *SLLM) uninstallInstantModel(ctx context.Context, userCred mcclient.TokenCredential, mdlId string) error {
+	boolFalse := false
+	// uninstalled
+	probed := &boolFalse
+	mounted := &boolFalse
+	_, err := GetLLMInstantModelManager().updateInstantModel(ctx, llm.Id, mdlId, "", "", probed, mounted)
+	if err != nil {
+		return errors.Wrap(err, "uninstallPackage")
+	}
+	return nil
+}
+
+func findInstantModelWithModelInfo(allModels []SLLMInstantModel, mdl apis.ModelInfo) *SLLMInstantModel {
+	for i := range allModels {
+		if allModels[i].ModelId == mdl.ModelId {
+			return &allModels[i]
+		}
+	}
+	return nil
+}
+
+func findModelsToUninstall(allModels []SLLMInstantModel, input apis.LLMSyncModelTaskInput) []SLLMInstantModel {
+	ret := make([]SLLMInstantModel, 0)
+	for i := range input.Models {
+		existingModel := findInstantModelWithModelInfo(allModels, input.Models[i])
+		if existingModel != nil && existingModel.IsMounted && (input.Method == apis.QuickModelUninstall || input.Method == apis.QuickModelReinstall || (!existingModel.IsProbed && input.Method == apis.QuickModelInstall)) {
+			ret = append(ret, *existingModel)
+		}
+	}
+	return ret
+}
+
+func findModelsToUnmount(allModels []SLLMInstantModel, input apis.LLMSyncModelTaskInput) []SLLMInstantModel {
+	ret := make([]SLLMInstantModel, 0)
+	for i := range input.Models {
+		existingModel := findInstantModelWithModelInfo(allModels, input.Models[i])
+		if existingModel != nil && existingModel.IsMounted && (input.Method == apis.QuickModelUninstall || input.Method == apis.QuickModelReinstall || (!existingModel.IsProbed && input.Method == apis.QuickModelInstall)) {
+			ret = append(ret, *existingModel)
+		}
+	}
+	return ret
+}
+
+func isImageInUnmountModels(imageId string, mdls []SLLMInstantModel) (bool, error) {
+	instMdl, err := GetInstantModelManager().findInstantModelByImageId(imageId)
+	if err != nil {
+		return false, errors.Wrap(err, "findInstantAppByImageId")
+	}
+	if instMdl == nil {
+		return false, nil
+	}
+	for i := range mdls {
+		if mdls[i].ModelId == instMdl.ModelId {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func (llm *SLLM) RefreshInstantModels(ctx context.Context, userCred mcclient.TokenCredential, refresh bool) error {
+	lockman.LockObject(ctx, llm)
+	defer lockman.ReleaseObject(ctx, llm)
+
+	if !llm.LastInstantModelProbe.IsZero() && time.Since(llm.LastInstantModelProbe) < apis.LLM_PROBE_INSTANT_MODEl_INTERVAL_SECOND*time.Second && (!refresh || time.Since(llm.LastInstantModelProbe) < apis.LLM_PROBE_INSTANT_MODEl_INTERVAL_SECOND*time.Second) {
+		// already probed, null operation
+		return nil
+	}
+
+	mdlMap, err := llm.getProbedMountedInstantModels(ctx, userCred)
+	if err != nil {
+		return errors.Wrap(err, "getProbedMountedInstantModels")
+	}
+
+	models, err := llm.FetchModels(nil, nil, nil)
+	if err != nil {
+		return errors.Wrap(err, "FetchModels")
+	}
+
+	var errs []error
+	for i := range models {
+		mdl := models[i]
+		var probed *bool
+		var mounted *bool
+		if status, ok := mdlMap[mdl.ModelId]; ok {
+			// probed
+			probed = &status.Probed
+			mounted = &status.Mounted
+			_, err = GetLLMInstantModelManager().updateInstantModel(ctx, llm.Id, mdl.ModelId, status.Name, status.Tag, probed, mounted)
+			delete(mdlMap, mdl.ModelId)
+		} else {
+			// uninstalled
+			err = llm.uninstallInstantModel(ctx, userCred, mdl.ModelId)
+		}
+		if err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	if len(mdlMap) > 0 {
+		for mdlId, status := range mdlMap {
+			_, err := GetLLMInstantModelManager().updateInstantModel(ctx, llm.Id, mdlId, status.Name, status.Tag, &status.Probed, &status.Mounted)
+			if err != nil {
+				errs = append(errs, err)
+			}
+		}
+	}
+
+	if len(errs) > 0 {
+		return errors.NewAggregate(errs)
+	}
+	// update timer
+	_, err = db.Update(llm, func() error {
+		llm.LastInstantModelProbe = time.Now()
+		return nil
+	})
+	if err != nil {
+		return errors.Wrap(err, "update last_instant_model_probe")
+	}
+
+	return nil
+}
+
+func (llm *SLLM) PerformQuickModels(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.LLMPerformQuickModelsInput) (*apis.LLMBatchPerformOutput, error) {
+	if !utils.IsInArray(llm.Status, []string{apis.LLM_STATUS_RUNNING, apis.LLM_STATUS_READY}) {
+		return nil, errors.Wrapf(errors.ErrInvalidStatus, "llm:%s(%s) status:%s", llm.Name, llm.Id, llm.Status)
+	}
+
+	llmStatus := llm.Status
+	if len(input.Method) == 0 {
+		input.Method = apis.QuickModelInstall
+	}
+
+	var toInstallSizeGb float64
+	var errs []error
+	for i := range input.Models {
+		// specified by ID
+		if len(input.Models[i].Id) > 0 {
+			instModelObj, err := GetInstantModelManager().FetchByIdOrName(ctx, userCred, input.Models[i].Id)
+			if err != nil {
+				if errors.Cause(err) == sql.ErrNoRows {
+					errs = append(errs, httperrors.NewResourceNotFoundError2(GetInstantModelManager().Keyword(), input.Models[i].Id))
+				} else {
+					errs = append(errs, errors.Wrap(err, "FetchByIdOrName"))
+				}
+			} else {
+				instApp := instModelObj.(*SInstantModel)
+				input.Models[i].Id = instApp.Id
+				input.Models[i].ModelId = instApp.ModelId
+				input.Models[i].Tag = instApp.Tag
+				if input.Method == apis.QuickModelInstall {
+					toInstallSizeGb += float64(instApp.GetActualSizeMb()) * 1024 * 1024 / 1000 / 1000 / 1000
+				}
+			}
+		} else {
+			mdl, err := GetInstantModelManager().findInstantModel(input.Models[i].ModelId, input.Models[i].Tag, true)
+			if err != nil {
+				return nil, errors.Wrapf(err, "findInstantModel %s %s", input.Models[i].ModelId, input.Models[i].Tag)
+			}
+			if mdl == nil {
+				errs = append(errs, httperrors.NewResourceNotFoundError2(GetInstantModelManager().Keyword(), input.Models[i].ModelId))
+			} else {
+				input.Models[i].Id = mdl.Id
+				input.Models[i].Tag = mdl.Tag
+				input.Models[i].ModelId = mdl.ModelId
+			}
+		}
+	}
+	if len(errs) > 0 {
+		return nil, errors.NewAggregate(errs)
+	}
+	if input.Method == apis.QuickModelInstall {
+		if llm.InstantModelQuotaGb > 0 && toInstallSizeGb > float64(llm.InstantModelQuotaGb)-llm.GetTotalInstantModelSizeGb() {
+			return nil, errors.Wrapf(httperrors.ErrOutOfQuota, "toInstallSizeGb %f > InstantAppQuotaGb %d - total %f Gb", toInstallSizeGb, llm.InstantModelQuotaGb, llm.GetTotalInstantModelSizeGb())
+		}
+	}
+
+	task, err := llm.StartLLMInstantModelsSyncTask(ctx, userCred, llmStatus, input, "")
+	if err != nil {
+		return nil, errors.Wrap(err, "StartLLMInstantModelsSyncTask")
+	}
+
+	if input.Method == apis.QuickModelInstall {
+		// save pending quota
+		llm.insertPendingInstantModelQuota(task.Id, toInstallSizeGb)
+	}
+
+	output := apis.LLMBatchPerformOutput{
+		Data: []apis.LLMPerformOutput{
+			{
+				Id:            llm.Id,
+				Name:          llm.Name,
+				RequestStatus: http.StatusOK,
+				TaskId:        task.Id,
+			},
+		},
+		Task: task,
+	}
+	return &output, nil
+}
+
+func (llm *SLLM) StartLLMInstantModelsSyncTask(ctx context.Context, userCred mcclient.TokenCredential, llmStatus string, input apis.LLMPerformQuickModelsInput, parentTaskid string) (*taskman.STask, error) {
+	if !utils.IsInArray(llmStatus, []string{apis.LLM_STATUS_RUNNING, apis.LLM_STATUS_READY}) {
+		return nil, errors.Wrapf(errors.ErrInvalidStatus, "cannot sync models in status %s", llmStatus)
+	}
+	taskInput := apis.LLMSyncModelTaskInput{
+		LLMPerformQuickModelsInput: input,
+		LLMStatus:                  llmStatus,
+	}
+	task, err := taskman.TaskManager.NewTask(ctx, "LLMInstantModelsSyncTask", llm, userCred, jsonutils.Marshal(taskInput).(*jsonutils.JSONDict), parentTaskid, "")
+	if err != nil {
+		return nil, errors.Wrap(err, "NewTask")
+	}
+	err = task.ScheduleRun(nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "ScheduleRun")
+	}
+	return task, nil
+}
+
+func (llm *SLLM) FetchModels(isProbed, isMounted, isSystem *bool) ([]SLLMInstantModel, error) {
+	q := GetLLMInstantModelManager().Query().Equals("llm_id", llm.Id)
+	q = GetLLMInstantModelManager().filterModels(q, isProbed, isMounted, isSystem)
+
+	models := make([]SLLMInstantModel, 0)
+	err := db.FetchModelObjects(GetLLMInstantModelManager(), q, &models)
+	if err != nil {
+		return nil, errors.Wrap(err, "FetchModelObjects")
+	}
+	return models, nil
+}
+
+func (llm *SLLM) FetchModelsFullName(isProbed, isMounted *bool) ([]string, error) {
+	models, err := llm.FetchModels(isProbed, isMounted, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "FetchModels")
+	}
+	mdlFullNames := make([]string, len(models))
+	for idx, mdl := range models {
+		mdlFullNames[idx] = mdl.ModelName + ":" + mdl.Tag + "-" + mdl.ModelId
+	}
+	return mdlFullNames, nil
+}
+
+func (llm *SLLM) FetchMountedModelFullName() ([]string, error) {
+	boolTrue := true
+	return llm.FetchModelsFullName(nil, &boolTrue)
+}
+
+func (llm *SLLM) RequestUnmountModel(ctx context.Context, userCred mcclient.TokenCredential, input apis.LLMSyncModelTaskInput) ([]string, []*commonapi.ContainerVolumeMountDiskPostOverlay, error) {
+	if input.LLMStatus == apis.LLM_STATUS_RUNNING {
+		err := llm.RefreshInstantModels(ctx, userCred, true)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "RefreshInstantModels")
+		}
+	}
+	allModels, err := llm.FetchModels(nil, nil, nil)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "FetchModels")
+	}
+	drv := llm.GetLLMContainerDriver()
+
+	if input.LLMStatus == apis.LLM_STATUS_RUNNING {
+		uninstallModels := findModelsToUninstall(allModels, input)
+		for i := range uninstallModels {
+			err := drv.UninstallModel(ctx, userCred, llm, &uninstallModels[i])
+			if err != nil {
+				log.Errorf("fail to uninstall %s", err)
+				continue
+			}
+		}
+	}
+
+	// next found out models that need to unmount
+	unmountModels := findModelsToUnmount(allModels, input)
+	if len(unmountModels) == 0 {
+		return nil, nil, nil
+	}
+
+	container, err := llm.GetLLMSContainer(ctx)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "GetContainer")
+	}
+
+	var unmountOverlays []*commonapi.ContainerVolumeMountDiskPostOverlay
+	existingOverlays := container.Spec.VolumeMounts[0].Disk.PostOverlay
+
+	for i := range existingOverlays {
+		eOverlay := existingOverlays[i]
+		if eOverlay.Image != nil && len(eOverlay.Image.Id) > 0 {
+			find, err := isImageInUnmountModels(eOverlay.Image.Id, unmountModels)
+			if err != nil {
+				return nil, nil, errors.Wrap(err, "isImageInUnmountModels")
+			}
+			if find {
+				unmountOverlays = append(unmountOverlays, eOverlay)
+			}
+
+		}
+	}
+
+	var modelIds []string
+	for i := range unmountModels {
+		modelIds = append(modelIds, unmountModels[i].ModelId)
+	}
+	return modelIds, unmountOverlays, nil
+}
+
+func (llm *SLLM) RequestMountModels(ctx context.Context, userCred mcclient.TokenCredential, input apis.LLMSyncModelTaskInput) ([]string, []string, []*commonapi.ContainerVolumeMountDiskPostOverlay, error) {
+	if input.LLMStatus == apis.LLM_STATUS_RUNNING {
+		err := llm.RefreshInstantModels(ctx, userCred, true)
+		if err != nil {
+			return nil, nil, nil, errors.Wrap(err, "RefreshApps")
+		}
+	}
+	existingMdls, err := llm.FetchModels(nil, nil, nil)
+	if err != nil {
+		return nil, nil, nil, errors.Wrap(err, "FetchApps")
+	}
+	log.Debugf("=======RequestMountModels input: %s", jsonutils.Marshal(input).PrettyString())
+	models, overlays, err := llm.getMountingModelsPostOverlay(ctx, input, existingMdls)
+	if err != nil {
+		return nil, nil, nil, errors.Wrap(err, "getMountingModelsPostOverlay")
+	}
+	drv := llm.GetLLMContainerDriver()
+	var mdlIds []string
+	for i := range models {
+		model := models[i]
+		if input.LLMStatus == apis.LLM_STATUS_RUNNING {
+			err := drv.PreInstallModel(ctx, userCred, llm, &model)
+			if err != nil {
+				log.Errorf("preinstallPackage fail %s", err)
+			}
+		}
+		mdlIds = append(mdlIds, model.ModelId)
+	}
+	targetDirs := make([]string, 0)
+	for i := range overlays {
+		if len(overlays[i].ContainerTargetDir) > 0 {
+			targetDirs = append(targetDirs, overlays[i].ContainerTargetDir)
+		}
+	}
+	return mdlIds, targetDirs, overlays, nil
+}
+
+func (llm *SLLM) TryContainerUnmountPaths(ctx context.Context, userCred mcclient.TokenCredential, s *mcclient.ClientSession, overlays []*commonapi.ContainerVolumeMountDiskPostOverlay, waitSecs int) error {
+	start := time.Now()
+	for time.Since(start) < time.Second*time.Duration(waitSecs) {
+		err := llm.containerUnmountPaths(ctx, userCred, s, overlays)
+		if err != nil {
+			if strings.Contains(err.Error(), string(errors.ErrInvalidStatus)) {
+				// wait
+				time.Sleep(5 * time.Second)
+			} else {
+				return errors.Wrap(err, "containerMountPaths")
+			}
+		} else {
+			// success
+			return nil
+		}
+	}
+	return errors.ErrTimeout
+}
+
+func (llm *SLLM) containerUnmountPaths(ctx context.Context, userCred mcclient.TokenCredential, s *mcclient.ClientSession, overlays []*commonapi.ContainerVolumeMountDiskPostOverlay) error {
+	ctr, err := llm.GetLLMSContainer(ctx)
+	if err != nil {
+		return errors.Wrap(err, "GetSContainer")
+	}
+
+	if !computeapi.ContainerFinalStatus.Has(ctr.Status) {
+		return errors.Wrapf(errors.ErrInvalidStatus, "cannot unmount post path in status %s", ctr.Status)
+	}
+
+	params := computeapi.ContainerVolumeMountRemovePostOverlayInput{
+		Index:       0,
+		PostOverlay: overlays,
+		UseLazy:     true,
+		ClearLayers: true,
+	}
+	_, err = compute.Containers.PerformAction(s, ctr.Id, "remove-volume-mount-post-overlay", jsonutils.Marshal(params))
+	if err != nil {
+		return errors.Wrap(err, "PerformAction remove-volume-mount-post-overlay")
+	}
+	return nil
+}
+
+func (llm *SLLM) TryContainerMountPaths(ctx context.Context, userCred mcclient.TokenCredential, s *mcclient.ClientSession, overlays []*commonapi.ContainerVolumeMountDiskPostOverlay, waitSecs int) error {
+	start := time.Now()
+	for time.Since(start) < time.Second*time.Duration(waitSecs) {
+		err := llm.containerMountPaths(ctx, userCred, s, overlays)
+		if err != nil {
+			if strings.Contains(err.Error(), string(errors.ErrInvalidStatus)) {
+				log.Errorf("containerMountPaths error %s, retry", err)
+				// retry
+				time.Sleep(5 * time.Second)
+			} else {
+				return errors.Wrap(err, "containerMountPaths")
+			}
+		} else {
+			// success
+			return nil
+		}
+	}
+	return errors.ErrTimeout
+}
+
+func (llm *SLLM) containerMountPaths(ctx context.Context, userCred mcclient.TokenCredential, s *mcclient.ClientSession, overlays []*commonapi.ContainerVolumeMountDiskPostOverlay) error {
+	ctr, err := llm.GetLLMSContainer(ctx)
+	if err != nil {
+		return errors.Wrap(err, "GetLLMSContainer")
+	}
+
+	if !computeapi.ContainerFinalStatus.Has(ctr.Status) {
+		return errors.Wrapf(errors.ErrInvalidStatus, "cannot mount post path in status %s", ctr.Status)
+	}
+	params := computeapi.ContainerVolumeMountAddPostOverlayInput{
+		Index:       0,
+		PostOverlay: overlays,
+	}
+	_, err = compute.Containers.PerformAction(s, ctr.Id, "add-volume-mount-post-overlay", jsonutils.Marshal(params))
+	if err != nil {
+		return errors.Wrap(err, "PerformAction add-volume-mount-post-overlay")
+	}
+	return nil
+}
+
+func (llm *SLLM) MarkInstantModelsUnmounted(ctx context.Context, userCred mcclient.TokenCredential, llmStatus string, mdlIds []string) error {
+	return llm.markInstantModelsMounted(ctx, userCred, llmStatus, mdlIds, false)
+}
+
+func (llm *SLLM) MarkInstantModelsMounted(ctx context.Context, userCred mcclient.TokenCredential, llmStatus string, mdlIds []string) error {
+	return llm.markInstantModelsMounted(ctx, userCred, llmStatus, mdlIds, true)
+}
+
+func (llm *SLLM) markInstantModelsMounted(ctx context.Context, userCred mcclient.TokenCredential, llmStatus string, mdlIds []string, mounted bool) error {
+	boolFalse := false
+	boolTrue := true
+	var isProbed *bool
+	if !mounted {
+		isProbed = &boolFalse
+	} else {
+		isProbed = &boolTrue
+	}
+
+	var errs []error
+	for i := range mdlIds {
+		_, err := GetLLMInstantModelManager().updateInstantModel(ctx, llm.Id, mdlIds[i], "", "", isProbed, &mounted)
+		if err != nil {
+			errs = append(errs, err)
+		}
+	}
+	if len(errs) > 0 {
+		return errors.NewAggregate(errs)
+	}
+	if llmStatus == apis.LLM_STATUS_RUNNING {
+		err := llm.RefreshInstantModels(ctx, userCred, true)
+		if err != nil {
+			return errors.Wrap(err, "RefreshApps")
+		}
+	}
+	mountedModelsFullName, err := llm.FetchMountedModelFullName()
+	if err != nil {
+		return errors.Wrap(err, "FetchMountedModelFullName")
+	}
+	{
+		// save mounted apps to volume
+		err := llm.UpdateVolumeMountedModelFullNames(mountedModelsFullName)
+		if err != nil {
+			return errors.Wrap(err, "UpdateVolumeMountedModelFullNames")
+		}
+	}
+	logclient.AddActionLogWithContext(ctx, llm, logclient.ACT_UPDATE, mountedModelsFullName, userCred, true)
+	return nil
+}
+
+func (llm *SLLM) UpdateVolumeMountedModelFullNames(mdlFullNames []string) error {
+	volume, err := llm.GetVolume()
+	if err != nil {
+		return errors.Wrap(err, "GetVolume")
+	}
+	return volume.UpdateMountedModelFullNames(mdlFullNames)
+}
+
+func (llm *SLLM) getMountingModelsPostOverlay(ctx context.Context, input apis.LLMSyncModelTaskInput, existingMdls []SLLMInstantModel) ([]SLLMInstantModel, []*commonapi.ContainerVolumeMountDiskPostOverlay, error) {
+	var models []SLLMInstantModel
+	for i := range input.Models {
+		if input.Method == apis.QuickModelInstall || input.Method == apis.QuickModelReinstall {
+			mdl := input.Models[i]
+			if input.Method == apis.QuickModelInstall {
+				existingModel := findInstantModelWithModelInfo(existingMdls, mdl)
+				if existingModel != nil && (existingModel.IsProbed || existingModel.IsMounted) {
+					// if the model is already probed or mounted, skip mount
+					continue
+				}
+			}
+			model, err := GetLLMInstantModelManager().updateInstantModel(ctx, llm.Id, mdl.ModelId, mdl.DisplayName, mdl.Tag, nil, nil)
+			if err != nil {
+				return nil, nil, errors.Wrapf(err, "updateInstantModel %s", mdl.ModelId)
+			}
+			models = append(models, *model)
+		}
+	}
+	if len(models) == 0 {
+		return nil, nil, nil
+	}
+	drv := llm.GetLLMContainerDriver()
+	overlays, err := models2overlays(drv, models, true)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "models2overlays")
+	}
+	return models, overlays, nil
+}
+
+func models2overlays(drv ILLMContainerDriver, models []SLLMInstantModel, isInstall bool) ([]*commonapi.ContainerVolumeMountDiskPostOverlay, error) {
+	var errs []error
+	var allDirs []apis.LLMMountDirInfo
+	for i := range models {
+		mdlDirs, err := models[i].getMountPaths(isInstall)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		allDirs = append(allDirs, mdlDirs...)
+	}
+	if len(allDirs) == 0 {
+		if len(errs) > 0 {
+			return nil, errors.Wrap(errors.NewAggregate(errs), "getMountPaths")
+		}
+		return nil, nil
+	}
+	if len(errs) > 0 {
+		log.Errorf("models2overlays getMountPaths error %s", errors.NewAggregate(errs))
+	}
+	var overlays []*commonapi.ContainerVolumeMountDiskPostOverlay
+	for i := range allDirs {
+		overlay := drv.GetDirPostOverlay(allDirs[i])
+		overlays = append(overlays, overlay)
+	}
+	return overlays, nil
+}
+
+func (llm *SLLM) InstallInstantModels(ctx context.Context, userCred mcclient.TokenCredential, dirs []string, mdlIds []string) error {
+	drv := llm.GetLLMContainerDriver()
+	return drv.InstallModel(ctx, userCred, llm, dirs, mdlIds)
+}
+
+func (llm *SLLM) EnsureInstantModelsInstalled(ctx context.Context, userCred mcclient.TokenCredential, mdlIds []string) error {
+	mdlMap, err := llm.getProbedInstantModelsExt(ctx, userCred, mdlIds...)
+	if err != nil {
+		return errors.Wrap(err, "FetchApps")
+	}
+	var errs []error
+	for _, mdlId := range mdlIds {
+		if _, ok := mdlMap[mdlId]; ok {
+			// probed
+			// errs = append(errs, errors.Wrap(errors.ErrInvalidStatus, pkg))
+		} else {
+			// not mounted and not probed
+			errs = append(errs, errors.Wrapf(errors.ErrNotFound, "mdlId %s", mdlId))
+		}
+	}
+	if len(errs) > 0 {
+		return errors.NewAggregate(errs)
+	}
+	return nil
+}

pkg/llm/models/llm_pod.go

@@ -15,11 +15,11 @@ func GetLLMPodCreateInput(
 	userCred mcclient.TokenCredential,
 	input *api.LLMCreateInput,
 	llm *SLLM,
-	sku *SLLMModel,
+	sku *SLLMSku,
 	llmImage *SLLMImage,
 	eip string,
 ) (*computeapi.ServerCreateInput, error) {
-	data, err := GetLLMBasePodCreateInput(ctx, userCred, &input.LLMBaseCreateInput, &llm.SLLMBase, &sku.SLLMModelBase, eip)
+	data, err := GetLLMBasePodCreateInput(ctx, userCred, &input.LLMBaseCreateInput, &llm.SLLMBase, &sku.SLLMSkuBase, eip)
 	if err != nil {
 		return nil, errors.Wrap(err, "GetLLMBasePodCreateInput: ")
 	}

pkg/llm/models/llm_save_instant_model.go

@@ -0,0 +1,181 @@
+package models
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
+	hostapi "yunion.io/x/onecloud/pkg/apis/host"
+	api "yunion.io/x/onecloud/pkg/apis/llm"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/compute"
+)
+
+func (llm *SLLM) GetDetailsProbedModels(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	mdlInfos, err := llm.getProbedInstantModelsExt(ctx, userCred)
+	if err != nil {
+		return nil, errors.Wrap(err, "getProbedPackagesExt")
+	}
+	return jsonutils.Marshal(mdlInfos), nil
+}
+
+func (llm *SLLM) PerformSaveInstantModel(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	input api.LLMSaveInstantModelInput,
+) (jsonutils.JSONObject, error) {
+	if llm.Status != api.LLM_STATUS_RUNNING {
+		return nil, httperrors.NewInvalidStatusError("LLM is not running")
+	}
+
+	mdlInfos, err := llm.getProbedInstantModelsExt(ctx, userCred, input.ModelId)
+	if err != nil {
+		return nil, errors.Wrap(err, "getProbedPackagesExt")
+	}
+
+	mdlInfo, ok := mdlInfos[input.ModelId]
+	if !ok {
+		return nil, httperrors.NewBadRequestError("ModelId %s not found", input.ModelId)
+	}
+
+	mountDirs, err := llm.detectModelPaths(ctx, userCred, mdlInfo)
+	if err != nil {
+		return nil, errors.Wrap(err, "detectModelPaths")
+	}
+
+	if len(input.ImageName) == 0 {
+		input.ImageName = fmt.Sprintf("%s-%s", mdlInfo.Name+":"+mdlInfo.Tag, time.Now().Format("060102"))
+	}
+
+	var ownerId mcclient.IIdentityProvider
+	if len(input.TenantId) > 0 {
+		domainId := input.ProjectDomainId
+		if len(domainId) == 0 {
+			domainId = userCred.GetProjectDomainId()
+		} else {
+			domain, err := db.TenantCacheManager.FetchDomainByIdOrName(ctx, domainId)
+			if err != nil {
+				return nil, errors.Wrap(err, "TenantCache.FetchDomainByIdOrName")
+			}
+			domainId = domain.GetId()
+		}
+		tenant, err := db.TenantCacheManager.FetchTenantByIdOrNameInDomain(ctx, input.TenantId, domainId)
+		if err != nil {
+			return nil, errors.Wrap(err, "TenantCache.FetchById")
+		}
+		ownerId = &db.SOwnerId{
+			DomainId:  domainId,
+			Domain:    tenant.Domain,
+			ProjectId: tenant.Id,
+			Project:   tenant.Name,
+		}
+	} else {
+		ownerId = userCred
+	}
+
+	input.ProjectId = ownerId.GetProjectId()
+	input.ProjectDomainId = ownerId.GetProjectDomainId()
+
+	drv := llm.GetLLMContainerDriver()
+	instantModelCreateInput := api.InstantModelCreateInput{
+		LLMType:   drv.GetType(),
+		ModelId:   mdlInfo.ModelId,
+		ModelName: mdlInfo.Name,
+		Tag:       mdlInfo.Tag,
+		Mounts:    mountDirs,
+	}
+	instantModelCreateInput.Name = input.ImageName
+	log.Debugf("instantModelCreateInput: %s", jsonutils.Marshal(instantModelCreateInput))
+
+	instantMdlObj, err := db.DoCreate(GetInstantModelManager(), ctx, userCred, nil, jsonutils.Marshal(instantModelCreateInput), ownerId)
+	if err != nil {
+		return nil, errors.Wrap(err, "GetInstantModelManager.DoCreate")
+	}
+
+	instantMdl := instantMdlObj.(*SInstantModel)
+
+	input.InstantModelId = instantMdl.Id
+
+	_, err = llm.StartSaveModelImageTask(ctx, userCred, input)
+	if err != nil {
+		return nil, errors.Wrap(err, "StartSaveAppImageTask")
+	}
+
+	return jsonutils.Marshal(instantMdl), nil
+}
+
+func (llm *SLLM) DoSaveModelImage(ctx context.Context, userCred mcclient.TokenCredential, session *mcclient.ClientSession, input api.LLMSaveInstantModelInput) error {
+	llm.SetStatus(ctx, userCred, api.LLM_STATUS_SAVING_MODEL, "DoSaveModelImage")
+
+	instantModelObj, err := GetInstantModelManager().FetchById(input.InstantModelId)
+	if err != nil {
+		return errors.Wrap(err, "GetInstantModelManager.FetchById")
+	}
+	instantModel := instantModelObj.(*SInstantModel)
+
+	drv := llm.GetLLMContainerDriver()
+	prefix, saveDirs, err := drv.GetSaveDirectories(instantModel)
+	if err != nil {
+		return errors.Wrap(err, "GetSaveDirectories")
+	}
+
+	saveImageInput := computeapi.ContainerSaveVolumeMountToImageInput{
+		GenerateName:      input.ImageName,
+		Notes:             fmt.Sprintf("instance model image for %s(%s)", input.ModelId, instantModel.ModelName+":"+instantModel.Tag),
+		Index:             0,
+		Dirs:              saveDirs,
+		UsedByPostOverlay: true,
+		DirPrefix:         prefix,
+	}
+
+	lc, err := llm.GetLLMContainer()
+	if err != nil {
+		return errors.Wrap(err, "GetLLMContainer")
+	}
+
+	result, err := compute.Containers.PerformAction(session, lc.CmpId, "save-volume-mount-image", jsonutils.Marshal(saveImageInput))
+	if err != nil {
+		return errors.Wrap(err, "compute.Containers.PerformAction")
+	}
+	log.Debugf("container save-volume-mount-image result: %s", result)
+	saveImageOutput := hostapi.ContainerSaveVolumeMountToImageInput{}
+	err = result.Unmarshal(&saveImageOutput)
+	if err != nil {
+		return errors.Wrap(err, "save-volume-mount-image.result.Unmarshal")
+	}
+
+	err = instantModel.saveImageId(ctx, userCred, saveImageOutput.ImageId)
+	if err != nil {
+		return errors.Wrap(err, "saveImageId")
+	}
+
+	return nil
+}
+
+func (llm *SLLM) StartSaveModelImageTask(ctx context.Context, userCred mcclient.TokenCredential, input api.LLMSaveInstantModelInput) (*taskman.STask, error) {
+	llm.SetStatus(ctx, userCred, api.LLM_STATUS_START_SAVE_MODEL, "StartSaveModelImageTask")
+
+	params := jsonutils.Marshal(input)
+	task, err := taskman.TaskManager.NewTask(ctx, "LLMStartSaveModelImageTask", llm, userCred, params.(*jsonutils.JSONDict), "", "")
+	if err != nil {
+		return nil, errors.Wrap(err, "taskman.TaskManager.NewTask")
+	}
+	err = task.ScheduleRun(nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "task.ScheduleRun")
+	}
+	return task, nil
+}
+
+func (llm *SLLM) detectModelPaths(ctx context.Context, userCred mcclient.TokenCredential, pkgInfo api.LLMInternalInstantMdlInfo) ([]string, error) {
+	return llm.GetLLMContainerDriver().DetectModelPaths(ctx, userCred, llm, pkgInfo)
+}

pkg/llm/models/llm_sku.go

@@ -18,33 +18,33 @@ import (
 )
 
 func init() {
-	GetLLMModelManager()
+	GetLLMSkuManager()
 }
 
-var llmModelManager *SLLMModelManager
+var llmSkuManager *SLLMSkuManager
 
-func GetLLMModelManager() *SLLMModelManager {
-	if llmModelManager != nil {
-		return llmModelManager
+func GetLLMSkuManager() *SLLMSkuManager {
+	if llmSkuManager != nil {
+		return llmSkuManager
 	}
-	llmModelManager = &SLLMModelManager{
-		SLLMModelBaseManager: NewSLLMModelBaseManager(
-			SLLMModel{},
-			"llm_models_tbl",
-			"llm_model",
-			"llm_models",
+	llmSkuManager = &SLLMSkuManager{
+		SLLMSkuBaseManager: NewSLLMSkuBaseManager(
+			SLLMSku{},
+			"llm_skus_tbl",
+			"llm_sku",
+			"llm_skus",
 		),
 	}
-	llmModelManager.SetVirtualObject(llmModelManager)
-	return llmModelManager
+	llmSkuManager.SetVirtualObject(llmSkuManager)
+	return llmSkuManager
 }
 
-type SLLMModelManager struct {
-	SLLMModelBaseManager
+type SLLMSkuManager struct {
+	SLLMSkuBaseManager
 }
 
-type SLLMModel struct {
-	SLLMModelBase
+type SLLMSku struct {
+	SLLMSkuBase
 	// SMountedAppsResource
 
 	LLMImageId   string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"`
@@ -52,16 +52,16 @@ type SLLMModel struct {
 	LLMModelName string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"`
 }
 
-func (man *SLLMModelManager) ListItemFilter(
+func (man *SLLMSkuManager) ListItemFilter(
 	ctx context.Context,
 	q *sqlchemy.SQuery,
 	userCred mcclient.TokenCredential,
-	input api.LLMModelListInput,
+	input api.LLMSkuListInput,
 ) (*sqlchemy.SQuery, error) {
 	var err error
-	q, err = man.SLLMModelBaseManager.ListItemFilter(ctx, q, userCred, input.SharableVirtualResourceListInput)
+	q, err = man.SLLMSkuBaseManager.ListItemFilter(ctx, q, userCred, input.SharableVirtualResourceListInput)
 	if err != nil {
-		return nil, errors.Wrapf(err, "SLLMModelBaseManager.ListItemFilter")
+		return nil, errors.Wrapf(err, "SLLMSkuBaseManager.ListItemFilter")
 	}
 	if len(input.LLMType) > 0 {
 		q = q.Equals("llm_type", input.LLMType)
@@ -73,19 +73,19 @@ func (man *SLLMModelManager) ListItemFilter(
 	return q, nil
 }
 
-func (manager *SLLMModelManager) FetchCustomizeColumns(
+func (manager *SLLMSkuManager) FetchCustomizeColumns(
 	ctx context.Context,
 	userCred mcclient.TokenCredential,
 	query jsonutils.JSONObject,
 	objs []interface{},
 	fields stringutils2.SSortedStrings,
 	isList bool,
-) []api.LLMModelDetails {
+) []api.LLMSkuDetails {
 	// skuIds := []string{}
 	imageIds := []string{}
 	// templateIds := []string{}
 
-	skus := []SLLMModel{}
+	skus := []SLLMSku{}
 	jsonutils.Update(&skus, objs)
 	virows := manager.SSharableVirtualResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
 	for _, sku := range skus {
@@ -104,7 +104,7 @@ func (manager *SLLMModelManager) FetchCustomizeColumns(
 	// 	LLMCapacity int
 	// }{}
 	// q.All(&details)
-	res := make([]api.LLMModelDetails, len(objs))
+	res := make([]api.LLMSkuDetails, len(objs))
 	for i := range skus {
 		res[i].SharableVirtualResourceDetails = virows[i]
 		// for _, v := range details {
@@ -146,11 +146,11 @@ func (manager *SLLMModelManager) FetchCustomizeColumns(
 	return res
 }
 
-func (man *SLLMModelManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.LLMModelCreateInput) (*api.LLMModelCreateInput, error) {
+func (man *SLLMSkuManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.LLMSkuCreateInput) (*api.LLMSkuCreateInput, error) {
 	var err error
-	input.LLMModelBaseCreateInput, err = man.SLLMModelBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.LLMModelBaseCreateInput)
+	input.LLMSKuBaseCreateInput, err = man.SLLMSkuBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.LLMSKuBaseCreateInput)
 	if err != nil {
-		return nil, errors.Wrap(err, "SLLMModelBaseManager.ValidateCreateData")
+		return nil, errors.Wrap(err, "SLLMSkuBaseManager.ValidateCreateData")
 	}
 	if !api.IsLLMContainerType(input.LLMType) {
 		return input, errors.Wrap(httperrors.ErrInputParameter, "llm_type must be one of "+strings.Join(api.LLM_CONTAINER_TYPES.List(), ","))
@@ -165,15 +165,15 @@ func (man *SLLMModelManager) ValidateCreateData(ctx context.Context, userCred mc
 	return input, nil
 }
 
-func (model *SLLMModel) GetLLMContainerDriver() ILLMContainerDriver {
-	return GetLLMContainerDriver(api.LLMContainerType(model.LLMType))
+func (sku *SLLMSku) GetLLMContainerDriver() ILLMContainerDriver {
+	return GetLLMContainerDriver(api.LLMContainerType(sku.LLMType))
 }
 
-func (model *SLLMModel) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.LLMModelUpdateInput) (api.LLMModelUpdateInput, error) {
+func (sku *SLLMSku) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.LLMSkuUpdateInput) (api.LLMSkuUpdateInput, error) {
 	var err error
-	input.LLMModelBaseUpdateInput, err = model.SLLMModelBase.ValidateUpdateData(ctx, userCred, query, input.LLMModelBaseUpdateInput)
+	input.LLMSkuBaseUpdateInput, err = sku.SLLMSkuBase.ValidateUpdateData(ctx, userCred, query, input.LLMSkuBaseUpdateInput)
 	if err != nil {
-		return input, errors.Wrap(err, "validate LLMModelBaseUpdateInput")
+		return input, errors.Wrap(err, "validate LLMSkuBaseUpdateInput")
 	}
 
 	if input.LLMImageId != "" {
@@ -187,13 +187,13 @@ func (model *SLLMModel) ValidateUpdateData(ctx context.Context, userCred mcclien
 	return input, nil
 }
 
-func (model *SLLMModel) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
-	count, err := GetLLMManager().Query().Equals("llm_model_id", model.Id).CountWithError()
+func (sku *SLLMSku) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
+	count, err := GetLLMManager().Query().Equals("llm_sku_id", sku.Id).CountWithError()
 	if nil != err {
 		return errors.Wrap(err, "fetch llm")
 	}
 	if count > 0 {
-		return errors.Wrap(errors.ErrNotSupported, "This model is currently in use")
+		return errors.Wrap(errors.ErrNotSupported, "This sku is currently in use")
 	}
 	return nil
 }

pkg/llm/models/mounted_models_resources.go

@@ -0,0 +1,50 @@
+package models
+
+import (
+	"context"
+	"fmt"
+
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/mcclient"
+
+	apis "yunion.io/x/onecloud/pkg/apis/llm"
+)
+
+type SMountedModelsResourceManager struct {
+}
+
+type SMountedModelsResource struct {
+	MountedModels []string `charset:"utf8" list:"user" update:"user" create:"optional"`
+}
+
+func (manager *SMountedModelsResourceManager) ListItemFilter(
+	ctx context.Context,
+	q *sqlchemy.SQuery,
+	userCred mcclient.TokenCredential,
+	input apis.MountedModelResourceListInput,
+) (*sqlchemy.SQuery, error) {
+	if len(input.MountedModels) > 0 {
+		q = q.ContainsAny("mounted_models", input.MountedModels)
+	}
+	return q, nil
+}
+
+type MountedModelModelManager interface {
+	IsPremountedModelName(fullModelName string) (bool, error)
+}
+
+func (manager *SVolumeManager) IsPremountedModelName(fullModelName string) (bool, error) {
+	return isPremountedModelName(manager, fullModelName)
+}
+
+func isPremountedModelName(manager db.IModelManager, fullModelName string) (bool, error) {
+	q := manager.Query().Contains("mounted_models", fmt.Sprintf("%q", fullModelName))
+	cnt, err := q.CountWithError()
+	if err != nil {
+		return false, errors.Wrap(err, "CountWithError")
+	}
+	return cnt > 0, nil
+}

pkg/llm/models/sku.go

@@ -16,8 +16,8 @@ import (
 	compute "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
 )
 
-func NewSLLMModelBaseManager(dt interface{}, tableName string, keyword string, keywordPlural string) SLLMModelBaseManager {
-	return SLLMModelBaseManager{
+func NewSLLMSkuBaseManager(dt interface{}, tableName string, keyword string, keywordPlural string) SLLMSkuBaseManager {
+	return SLLMSkuBaseManager{
 		SSharableVirtualResourceBaseManager: db.NewSharableVirtualResourceBaseManager(
 			dt,
 			tableName,
@@ -27,11 +27,11 @@ func NewSLLMModelBaseManager(dt interface{}, tableName string, keyword string, k
 	}
 }
 
-type SLLMModelBaseManager struct {
+type SLLMSkuBaseManager struct {
 	db.SSharableVirtualResourceBaseManager
 }
 
-type SLLMModelBase struct {
+type SLLMSkuBase struct {
 	db.SSharableVirtualResourceBase
 
 	BandwidthMb  int               `nullable:"false" default:"0" create:"optional" list:"user" update:"user"`
@@ -48,7 +48,7 @@ type SLLMModelBase struct {
 	NetworkId   string `charset:"utf8" nullable:"true" list:"user" update:"user" create:"optional"`
 }
 
-func (man *SLLMModelBaseManager) ListItemFilter(
+func (man *SLLMSkuBaseManager) ListItemFilter(
 	ctx context.Context,
 	q *sqlchemy.SQuery,
 	userCred mcclient.TokenCredential,
@@ -62,7 +62,7 @@ func (man *SLLMModelBaseManager) ListItemFilter(
 	return q, nil
 }
 
-func (man *SLLMModelBaseManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.LLMModelBaseCreateInput) (api.LLMModelBaseCreateInput, error) {
+func (man *SLLMSkuBaseManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.LLMSKuBaseCreateInput) (api.LLMSKuBaseCreateInput, error) {
 	var err error
 	input.SharableVirtualResourceCreateInput, err = man.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.SharableVirtualResourceCreateInput)
 	if err != nil {
@@ -78,7 +78,7 @@ func (man *SLLMModelBaseManager) ValidateCreateData(ctx context.Context, userCre
 		return input, errors.Wrap(httperrors.ErrInputParameter, "volumes cannot be empty")
 	}
 
-	if !api.IsLLMModelBaseNetworkType(input.NetworkType) {
+	if !api.IsLLMSkuBaseNetworkType(input.NetworkType) {
 		return input, errors.Wrapf(httperrors.ErrInputParameter, "invalid network type %s", input.NetworkType)
 	}
 
@@ -96,15 +96,15 @@ func (man *SLLMModelBaseManager) ValidateCreateData(ctx context.Context, userCre
 	return input, nil
 }
 
-func (modelBase *SLLMModelBase) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.LLMModelBaseUpdateInput) (api.LLMModelBaseUpdateInput, error) {
+func (skuBase *SLLMSkuBase) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.LLMSkuBaseUpdateInput) (api.LLMSkuBaseUpdateInput, error) {
 	var err error
-	input.SharableVirtualResourceBaseUpdateInput, err = modelBase.SSharableVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, input.SharableVirtualResourceBaseUpdateInput)
+	input.SharableVirtualResourceBaseUpdateInput, err = skuBase.SSharableVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, input.SharableVirtualResourceBaseUpdateInput)
 	if err != nil {
 		return input, errors.Wrap(err, "validate SharableVirtualResourceBaseUpdateInput")
 	}
 
 	volumes := []api.Volume{}
-	if err := jsonutils.Marshal(modelBase.Volumes).Unmarshal(&volumes); err != nil {
+	if err := jsonutils.Marshal(skuBase.Volumes).Unmarshal(&volumes); err != nil {
 		return input, errors.Wrapf(err, "Unmarshal Volumes")
 	}
 	for i, volume := range volumes {
@@ -130,7 +130,7 @@ func (modelBase *SLLMModelBase) ValidateUpdateData(ctx context.Context, userCred
 	}
 	input.Volumes = (*api.Volumes)(&volumes)
 
-	if input.NetworkType != nil && !api.IsLLMModelBaseNetworkType(*input.NetworkType) {
+	if input.NetworkType != nil && !api.IsLLMSkuBaseNetworkType(*input.NetworkType) {
 		return input, errors.Wrapf(httperrors.ErrInputParameter, "invalid network type %s", *input.NetworkType)
 	}
 

pkg/llm/models/volume.go

@@ -2,12 +2,26 @@ package models
 
 import (
 	"context"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/httputils"
+	"yunion.io/x/pkg/utils"
 
 	commonapi "yunion.io/x/onecloud/pkg/apis"
+	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
+	imageapi "yunion.io/x/onecloud/pkg/apis/image"
 	api "yunion.io/x/onecloud/pkg/apis/llm"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/llm/options"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/compute"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/image"
 )
 
 func init() {
@@ -34,12 +48,12 @@ func GetVolumeManager() *SVolumeManager {
 
 type SVolumeManager struct {
 	db.SVirtualResourceBaseManager
-	// SMountedAppsResourceManager
+	SMountedModelsResourceManager
 }
 
 type SVolume struct {
 	db.SVirtualResourceBase
-	// SMountedAppsResource
+	SMountedModelsResource
 
 	LLMId string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"admin_optional" update:"user"`
 	// 存储类型
@@ -65,3 +79,65 @@ func (volume *SVolume) StartDeleteTask(ctx context.Context, userCred mcclient.To
 func (volume *SVolume) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
 	return volume.SVirtualResourceBase.Delete(ctx, userCred)
 }
+
+func fetchImage(ctx context.Context, userCred mcclient.TokenCredential, imageId string) (*imageapi.ImageDetails, error) {
+	s := auth.GetSession(ctx, userCred, options.Options.Region)
+	imgObj, err := image.Images.Get(s, imageId, nil)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Image.Get %s", imageId)
+	}
+	img := imageapi.ImageDetails{}
+	err = imgObj.Unmarshal(&img)
+	if err != nil {
+		return nil, errors.Wrap(err, "Unmarshal")
+	}
+	return &img, nil
+}
+
+func (volume *SVolume) UpdateMountedModelFullNames(mountModels []string) error {
+	_, err := db.Update(volume, func() error {
+		volume.MountedModels = mountModels
+		return nil
+	})
+	if err != nil {
+		return errors.Wrap(err, "update volume mounted_apps")
+	}
+	return nil
+}
+
+func (volume *SVolume) GetDisk(ctx context.Context) (*computeapi.DiskDetails, error) {
+	if len(volume.SvrId) == 0 {
+		return nil, errors.ErrInvalidStatus
+	}
+	s := auth.GetAdminSession(ctx, "")
+	disk := computeapi.DiskDetails{}
+	resp, err := compute.Disks.GetById(s, volume.SvrId, jsonutils.Marshal(map[string]interface{}{
+		"scope": "max",
+	}))
+	if err != nil {
+		if httputils.ErrorCode(err) == 404 {
+			return nil, errors.Wrapf(errors.ErrNotFound, "GetById %s", volume.SvrId)
+		}
+		return nil, errors.Wrap(err, "fetch disk")
+	}
+	resp.Unmarshal(&disk)
+	return &disk, nil
+}
+
+func (volume *SVolume) WaitDiskStatus(ctx context.Context, userCred mcclient.TokenCredential, targetStatus []string, timeoutSecs int) (*computeapi.DiskDetails, error) {
+	expire := time.Now().Add(time.Second * time.Duration(timeoutSecs))
+	for time.Now().Before(expire) {
+		disk, err := volume.GetDisk(ctx)
+		if err != nil {
+			return nil, errors.Wrap(err, "GetDisk")
+		}
+		if utils.IsInArray(disk.Status, targetStatus) {
+			return disk, nil
+		}
+		if strings.Contains(disk.Status, "fail") {
+			return nil, errors.Wrap(errors.ErrInvalidStatus, disk.Status)
+		}
+		time.Sleep(2 * time.Second)
+	}
+	return nil, errors.Wrapf(httperrors.ErrTimeout, "wait disk status %s timeout", targetStatus)
+}

pkg/llm/options/option.go

@@ -20,6 +20,9 @@ type LLMOptions struct {
 	common_options.CommonOptions
 	common_options.DBOptions
 
+	InstantModelSyncTaskWorkerCount int `help:"backup task worker count" default:"128"`
+	ModelSyncTaskWaitSecs           int `help:"model sync task wait seconds" default:"30"`
+
 	StartTaskWorkerCount int `help:"start task worker count" default:"128"`
 }
 

pkg/llm/service/handler.go

@@ -32,13 +32,15 @@ func InitHandlers(app *appsrv.Application) {
 		db.Metadata,
 
 		models.GetLLMImageManager(),
-		models.GetLLMModelManager(),
-		models.GetDifyModelManager(),
+		models.GetLLMSkuManager(),
+		models.GetDifySkuManager(),
 		models.GetVolumeManager(),
 		models.GetAccessInfoManager(),
 		models.GetLLMContainerManager(),
 		models.GetLLMManager(),
 		models.GetDifyManager(),
+		models.GetInstantModelManager(),
+		models.GetLLMInstantModelManager(),
 	} {
 		db.RegisterModelManager(manager)
 		handler := db.NewModelHandler(manager)

pkg/llm/service/service.go

@@ -15,6 +15,7 @@ import (
 	"yunion.io/x/onecloud/pkg/llm/models"
 	"yunion.io/x/onecloud/pkg/llm/options"
 	_ "yunion.io/x/onecloud/pkg/llm/tasks"
+	llmTask "yunion.io/x/onecloud/pkg/llm/tasks/llm"
 )
 
 // StartService the main service starts
@@ -25,6 +26,7 @@ func StartService() {
 	baseOpts := &opts.BaseOptions
 	common_options.ParseOptions(opts, os.Args, "llm.conf", api.SERVICE_TYPE)
 
+	llmTask.InitInstantModelSyncTaskManager()
 	app_common.InitAuth(commonOpts, func() {
 		log.Infof("Auth complete!!")
 	})

pkg/llm/tasks/llm/llm_instant_models_sync_task.go

@@ -0,0 +1,201 @@
+package llm
+
+import (
+	"context"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	apis "yunion.io/x/onecloud/pkg/apis/llm"
+	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	models "yunion.io/x/onecloud/pkg/llm/models"
+	options "yunion.io/x/onecloud/pkg/llm/options"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/util/logclient"
+)
+
+var (
+	instantModelSyncTaskWorkerMan *appsrv.SHashedWorkerManager
+)
+
+type LLMInstantModelsSyncTask struct {
+	taskman.STask
+}
+
+func InitInstantModelSyncTaskManager() {
+	instantModelSyncTaskWorkerMan = appsrv.NewHashWorkerManager("InstantModelSyncTaskManager", options.Options.InstantModelSyncTaskWorkerCount, 1, 1024, true)
+	taskman.RegisterTaskAndHashedWorkerManager(LLMInstantModelsSyncTask{}, instantModelSyncTaskWorkerMan)
+}
+
+func (task *LLMInstantModelsSyncTask) taskFailed(ctx context.Context, llm *models.SLLM, err string) {
+	defer llm.ClearPendingInstantModelQuota(task.Id)
+	input := apis.LLMSyncModelTaskInput{}
+	task.Params.Unmarshal(&input)
+
+	db.OpsLog.LogEvent(llm, db.ACT_SYNC_CONF_FAIL, err, task.UserCred)
+	logclient.AddActionLogWithStartable(task, llm, logclient.ACT_SYNC_CONF, err, task.UserCred, false)
+	task.SetStageFailed(ctx, jsonutils.NewString(err))
+}
+
+func (task *LLMInstantModelsSyncTask) taskComplete(ctx context.Context, llm *models.SLLM) {
+	defer llm.ClearPendingInstantModelQuota(task.Id)
+	task.SetStageComplete(ctx, nil)
+}
+
+func (task *LLMInstantModelsSyncTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	llm := obj.(*models.SLLM)
+	input := apis.LLMSyncModelTaskInput{}
+	err := task.Params.Unmarshal(&input)
+	if err != nil {
+		task.taskFailed(ctx, llm, err.Error())
+		return
+	}
+
+	removedModelIds, unmountOverlays, err := llm.RequestUnmountModel(ctx, task.UserCred, input)
+	if err != nil {
+		task.taskFailed(ctx, llm, err.Error())
+		return
+	}
+	if len(removedModelIds) > 0 || len(unmountOverlays) > 0 {
+		input.UninstallModelIds = removedModelIds
+		task.SetStage("OnModelsUnmountComplete", jsonutils.Marshal(input).(*jsonutils.JSONDict))
+		if len(unmountOverlays) > 0 {
+			// try unmount post_overlay
+			s := auth.GetSession(ctx, task.GetUserCred(), options.Options.Region)
+			err := s.WithTaskCallback(task.GetId(), func() error {
+				return llm.TryContainerUnmountPaths(ctx, task.UserCred, s, unmountOverlays, 7200)
+			})
+			if err != nil {
+				task.OnModelsUnmountCompleteFailed(ctx, llm, jsonutils.NewString(errors.Wrap(err, "TryContainerUnmountPaths").Error()))
+			}
+		}
+	} else {
+		task.OnModelsUnmountComplete(ctx, llm, nil)
+	}
+}
+
+func (task *LLMInstantModelsSyncTask) OnModelsUnmountComplete(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	llm := obj.(*models.SLLM)
+	input := apis.LLMSyncModelTaskInput{}
+	err := task.Params.Unmarshal(&input)
+	if err != nil {
+		task.taskFailed(ctx, llm, err.Error())
+		return
+	}
+
+	if len(input.UninstallModelIds) > 0 {
+		// did uninstall models, clear the flags
+		log.Debugf("uninstall_models %s", jsonutils.Marshal(input.UninstallModelIds).PrettyString())
+		err := llm.MarkInstantModelsUnmounted(ctx, task.UserCred, input.LLMStatus, input.UninstallModelIds)
+		if err != nil {
+			task.taskFailed(ctx, llm, errors.Wrap(err, "MarkModelsUnmounted").Error())
+			return
+		}
+	}
+
+	mdlIds, installDirs, overlays, err := llm.RequestMountModels(ctx, task.UserCred, input)
+	if err != nil {
+		task.taskFailed(ctx, llm, err.Error())
+		return
+	}
+	log.Debugf("=======RequestMountModels: %s, overlays: %s", jsonutils.Marshal(mdlIds).PrettyString(), jsonutils.Marshal(overlays).PrettyString())
+	if len(mdlIds) == 0 {
+		task.taskComplete(ctx, llm)
+		return
+	}
+
+	input.InstallModelIds = mdlIds
+	input.InstallDirs = installDirs
+	if len(overlays) > 0 {
+		task.SetStage("OnModelsMountComplete", jsonutils.Marshal(input).(*jsonutils.JSONDict))
+
+		s := auth.GetSession(ctx, task.GetUserCred(), options.Options.Region)
+		err := s.WithTaskCallback(task.GetId(), func() error {
+			return llm.TryContainerMountPaths(ctx, task.UserCred, s, overlays, 7200)
+		})
+		if err != nil {
+			task.OnModelsUnmountCompleteFailed(ctx, llm, jsonutils.NewString(errors.Wrap(err, "TryContainerMountPaths").Error()))
+		}
+	} else {
+		task.OnModelsMountComplete(ctx, llm, nil)
+	}
+}
+
+func (task *LLMInstantModelsSyncTask) OnModelsUnmountCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) {
+	llm := obj.(*models.SLLM)
+	task.taskFailed(ctx, llm, err.String())
+}
+
+func (task *LLMInstantModelsSyncTask) OnModelsMountComplete(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	llm := obj.(*models.SLLM)
+	input := apis.LLMSyncModelTaskInput{}
+	err := task.Params.Unmarshal(&input)
+	if err != nil {
+		task.taskFailed(ctx, llm, err.Error())
+		return
+	}
+
+	log.Debugf("install_models %s", jsonutils.Marshal(input.InstallModelIds).PrettyString())
+	err = llm.MarkInstantModelsMounted(ctx, task.UserCred, input.LLMStatus, input.InstallModelIds)
+	if err != nil {
+		task.taskFailed(ctx, llm, errors.Wrap(err, "MarkAppsMounted").Error())
+		return
+	}
+
+	if input.LLMStatus == apis.LLM_STATUS_RUNNING {
+		err := llm.InstallInstantModels(ctx, task.UserCred, input.InstallDirs, input.InstallModelIds)
+		if err != nil {
+			task.taskFailed(ctx, llm, errors.Wrap(err, "llm.InstallInstantModels").Error())
+			return
+		}
+		task.SetStage("OnWaitModelsMountedComplete", jsonutils.Marshal(input).(*jsonutils.JSONDict))
+		ModelSyncTaskRun(task, llm.Id, func() (jsonutils.JSONObject, error) {
+			tried := 0
+			const intvSecs = 2
+			waitSecs := options.Options.ModelSyncTaskWaitSecs
+			var errs []error
+			succ := false
+			for tried < waitSecs/intvSecs && !succ {
+				tried++
+				time.Sleep(intvSecs * time.Second)
+				if err := llm.EnsureInstantModelsInstalled(ctx, task.UserCred, input.InstallModelIds); err != nil {
+					errs = append(errs, err)
+				} else {
+					succ = true
+				}
+			}
+			if !succ && len(errs) > 0 {
+				return nil, errors.NewAggregate(errs)
+			}
+			err := llm.RefreshInstantModels(ctx, task.UserCred, true)
+			if err != nil {
+				return nil, errors.Wrap(err, "RefreshInstantModels")
+			}
+			return nil, nil
+		})
+	} else {
+		task.taskComplete(ctx, llm)
+	}
+}
+
+func (task *LLMInstantModelsSyncTask) OnModelsMountCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) {
+	llm := obj.(*models.SLLM)
+	task.taskFailed(ctx, llm, err.String())
+
+	// sync status to clear failed status of container
+	llm.StartSyncStatusTask(ctx, task.UserCred, "")
+}
+
+func (task *LLMInstantModelsSyncTask) OnWaitModelsMountedComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	llm := obj.(*models.SLLM)
+	task.taskComplete(ctx, llm)
+}
+
+func (task *LLMInstantModelsSyncTask) OnWaitModelsMountedCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) {
+	llm := obj.(*models.SLLM)
+	task.taskFailed(ctx, llm, err.String())
+}

pkg/llm/tasks/llm/llm_start_save_model_image_task.go

@@ -0,0 +1,109 @@
+package llm
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+
+	api "yunion.io/x/onecloud/pkg/apis/llm"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/llm/models"
+	"yunion.io/x/onecloud/pkg/llm/options"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/util/logclient"
+)
+
+type LLMStartSaveModelImageTask struct {
+	taskman.STask
+}
+
+func init() {
+	taskman.RegisterTask(LLMStartSaveModelImageTask{})
+}
+
+func (task *LLMStartSaveModelImageTask) taskFailed(ctx context.Context, llm *models.SLLM, err string) {
+	llm.SetStatus(ctx, task.UserCred, api.LLM_STATUS_SAVE_MODEL_FAILED, err)
+	db.OpsLog.LogEvent(llm, db.ACT_SAVE_FAIL, err, task.UserCred)
+	logclient.AddActionLogWithStartable(task, llm, logclient.ACT_SAVE_IMAGE, err, task.UserCred, false)
+	task.SetStageFailed(ctx, jsonutils.NewString(err))
+}
+
+func (task *LLMStartSaveModelImageTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	llm := obj.(*models.SLLM)
+
+	// first stop the desktop
+	task.SetStage("OnStopLLMComplete", nil)
+	err := llm.StartLLMStopTask(ctx, task.UserCred, task.GetTaskId())
+	if err != nil {
+		task.taskFailed(ctx, llm, err.Error())
+		return
+	}
+}
+
+func (task *LLMStartSaveModelImageTask) OnStopLLMCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) {
+	llm := obj.(*models.SLLM)
+	task.taskFailed(ctx, llm, err.String())
+}
+
+func (task *LLMStartSaveModelImageTask) OnStopLLMComplete(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	llm := obj.(*models.SLLM)
+
+	input := api.LLMSaveInstantModelInput{}
+	err := task.Params.Unmarshal(&input)
+	if err != nil {
+		task.taskFailed(ctx, llm, err.Error())
+		return
+	}
+
+	task.SetStage("OnSaveModelImageComplete", nil)
+	s := auth.GetSession(ctx, task.GetUserCred(), options.Options.Region)
+	s.WithTaskCallback(task.GetId(), func() error {
+		return llm.DoSaveModelImage(ctx, task.UserCred, s, input)
+	})
+}
+
+func (task *LLMStartSaveModelImageTask) OnSaveModelImageComplete(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	llm := obj.(*models.SLLM)
+
+	input := api.LLMSaveInstantModelInput{}
+
+	err := task.Params.Unmarshal(&input)
+	if err != nil {
+		task.taskFailed(ctx, llm, err.Error())
+		return
+	}
+
+	instantModelObj, err := models.GetInstantModelManager().FetchById(input.InstantModelId)
+	if err != nil {
+		task.taskFailed(ctx, llm, err.Error())
+		return
+	}
+	instantModel := instantModelObj.(*models.SInstantModel)
+
+	_, err = instantModel.PerformSyncstatus(ctx, task.GetUserCred(), nil, api.InstantModelSyncstatusInput{})
+	if err != nil {
+		task.taskFailed(ctx, llm, err.Error())
+	}
+
+	db.OpsLog.LogEvent(llm, db.ACT_SAVE, instantModel.GetShortDesc(ctx), task.UserCred)
+	logclient.AddActionLogWithStartable(task, llm, logclient.ACT_SAVE_IMAGE, instantModel.GetShortDesc(ctx), task.UserCred, false)
+	db.OpsLog.LogEvent(instantModel, db.ACT_SAVE, llm.GetShortDesc(ctx), task.UserCred)
+	logclient.AddActionLogWithStartable(task, instantModel, logclient.ACT_SAVE_IMAGE, llm.GetShortDesc(ctx), task.UserCred, true)
+
+	task.SetStageComplete(ctx, nil)
+
+	// if input.AutoRestart {
+	// 	llm.StartRestartTask(ctx, task.UserCred, api.DesktopRestartTaskInput{
+	// 		DesktopId:     llm.Id,
+	// 		DesktopStatus: api.LLM_STATUS_READY,
+	// 	}, "")
+	// } else {
+	// 	llm.SetStatus(ctx, task.UserCred, api.LLM_STATUS_READY, "OnSaveModelImageComplete")
+	// }
+}
+
+func (task *LLMStartSaveModelImageTask) OnSaveModelImageCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) {
+	d := obj.(*models.SLLM)
+	task.taskFailed(ctx, d, err.String())
+}

pkg/llm/tasks/llm/llm_sync_status_task.go

@@ -0,0 +1,143 @@
+package llm
+
+import (
+	"context"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/utils"
+
+	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
+	apis "yunion.io/x/onecloud/pkg/apis/llm"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/llm/models"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/compute"
+	"yunion.io/x/onecloud/pkg/util/logclient"
+)
+
+type LLMSyncStatusTask struct {
+	taskman.STask
+}
+
+func init() {
+	taskman.RegisterTask(LLMSyncStatusTask{})
+}
+
+func (task *LLMSyncStatusTask) setLLMStatus(ctx context.Context, llm *models.SLLM, status string, reason string) {
+	if !task.HasParentTask() {
+		llm.SetStatus(ctx, task.UserCred, status, reason)
+	}
+}
+
+func (task *LLMSyncStatusTask) taskFailed(ctx context.Context, llm *models.SLLM, err string) {
+	task.setLLMStatus(ctx, llm, computeapi.VM_SYNC_FAIL, err)
+	db.OpsLog.LogEvent(llm, db.ACT_SYNC_STATUS, err, task.UserCred)
+	logclient.AddActionLogWithStartable(task, llm, logclient.ACT_SYNC_STATUS, err, task.UserCred, false)
+	// llm.NotifyRequest(ctx, task.GetUserCred(), notify.ActionStart, nil, false)
+	task.SetStageFailed(ctx, jsonutils.NewString(err))
+}
+
+func (task *LLMSyncStatusTask) taskComplete(ctx context.Context, _ *models.SLLM) {
+	// phone.SyncStatus(ctx, task.UserCred)
+	task.SetStageComplete(ctx, nil)
+}
+
+func (task *LLMSyncStatusTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	llm := obj.(*models.SLLM)
+
+	task.setLLMStatus(ctx, llm, apis.LLM_STATUS_SYNCSTATUS, "LLMSyncStatusTask.OnInit")
+
+	s := auth.GetSession(ctx, task.UserCred, "")
+	_, err := compute.Servers.PerformAction(s, llm.SvrId, "syncstatus", nil)
+	if err != nil {
+		task.taskFailed(ctx, llm, err.Error())
+		return
+	}
+	task.SetStage("OnSyncStatusComplete", nil)
+	taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
+		_, err := llm.WaitServerStatus(ctx, task.UserCred, []string{
+			computeapi.VM_RUNNING,
+			computeapi.VM_READY,
+			computeapi.VM_UNKNOWN,
+			computeapi.POD_STATUS_CRASH_LOOP_BACK_OFF,
+			computeapi.POD_STATUS_CONTAINER_EXITED,
+			computeapi.POD_STATUS_UPLOADING_STATUS_FAILED,
+		}, 1800)
+		if err != nil {
+			return nil, errors.Wrap(err, "WaitServerStatus")
+		}
+
+		time.Sleep(1 * time.Second)
+
+		// 有可能 server 变为 ready 之后 又变为 sync_container_status
+		srv, err := llm.WaitServerStatus(ctx, task.UserCred, []string{
+			computeapi.VM_RUNNING,
+			computeapi.VM_READY,
+			computeapi.VM_UNKNOWN,
+			computeapi.POD_STATUS_CRASH_LOOP_BACK_OFF,
+			computeapi.POD_STATUS_CONTAINER_EXITED,
+			computeapi.POD_STATUS_UPLOADING_STATUS_FAILED,
+		}, 1800)
+		if err != nil {
+			return nil, errors.Wrap(err, "WaitServerStatus")
+		}
+
+		if utils.IsInArray(srv.Status, []string{
+			computeapi.POD_STATUS_CRASH_LOOP_BACK_OFF,
+			computeapi.POD_STATUS_CONTAINER_EXITED,
+		}) {
+			params := computeapi.ServerStopInput{
+				IsForce:     true,
+				TimeoutSecs: 10,
+			}
+			_, err := compute.Servers.PerformAction(s, llm.SvrId, "stop", jsonutils.Marshal(params))
+			if err != nil {
+				return nil, errors.Wrap(err, "ServerStop")
+			}
+			srv, err := llm.WaitServerStatus(ctx, task.UserCred, []string{
+				computeapi.VM_READY,
+			}, 1800)
+			if err != nil {
+				return nil, errors.Wrap(err, "WaitServerStatus")
+			}
+			task.setLLMStatus(ctx, llm, srv.Status, "stop server")
+		} else {
+			task.setLLMStatus(ctx, llm, srv.Status, "WaitServerStatus")
+		}
+
+		volume, _ := llm.GetVolume()
+		if volume != nil {
+			disk, err := volume.GetDisk(ctx)
+			if err != nil {
+				task.setLLMStatus(ctx, llm, computeapi.VM_DISK_RESET_FAIL, errors.Wrap(err, "GetDisk").Error())
+				return nil, errors.Wrap(err, "GetDisk")
+			}
+			if disk.Status != computeapi.DISK_READY {
+				// do disk syncstatus
+				_, err := compute.Disks.PerformAction(s, disk.Id, "syncstatus", nil)
+				if err != nil {
+					return nil, errors.Wrap(err, "disk perform action syncstatus")
+				}
+				_, err = volume.WaitDiskStatus(ctx, task.UserCred, []string{computeapi.DISK_READY}, 1800)
+				if err != nil {
+					return nil, errors.Wrap(err, "volume.WaitDiskStatus")
+				}
+			}
+		}
+
+		return nil, nil
+	})
+}
+
+func (task *LLMSyncStatusTask) OnSyncStatusComplete(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	d := obj.(*models.SLLM)
+	task.taskComplete(ctx, d)
+}
+
+func (task *LLMSyncStatusTask) OnSyncStatusCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) {
+	d := obj.(*models.SLLM)
+	task.taskFailed(ctx, d, err.String())
+}

pkg/llm/tasks/llm/modelsyncworker.go

@@ -0,0 +1,11 @@
+package llm
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+)
+
+func ModelSyncTaskRun(task taskman.ITask, llmId string, proc func() (jsonutils.JSONObject, error)) {
+	taskman.LocalTaskRunWithWorkers(task, proc, instantModelSyncTaskWorkerMan.GetWorkerManager(llmId))
+}

pkg/mcclient/modules/llm/mod_dify_sku.go

@@ -5,20 +5,20 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 )
 
-type LLMModelManager struct {
+type DifySkuManager struct {
 	modulebase.ResourceManager
 }
 
 var (
-	LLMModel LLMModelManager
+	DifySku DifySkuManager
 )
 
 func init() {
-	LLMModel = LLMModelManager{
-		ResourceManager: modules.NewLLMManager("llm_model", "llm_models",
+	DifySku = DifySkuManager{
+		ResourceManager: modules.NewLLMManager("dify_sku", "dify_skus",
 			[]string{},
 			[]string{},
 		),
 	}
-	modules.Register(&LLMModel)
+	modules.Register(&DifySku)
 }

pkg/mcclient/modules/llm/mod_instantmodel.go

@@ -0,0 +1,23 @@
+package llm
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+var (
+	LLMInstantModel LLMInstantModelManager
+)
+
+func init() {
+	LLMInstantModel = LLMInstantModelManager{
+		modules.NewLLMManager("llm_instant_model", "llm_instant_models",
+			[]string{},
+			[]string{}),
+	}
+	modules.Register(&LLMInstantModel)
+}
+
+type LLMInstantModelManager struct {
+	modulebase.ResourceManager
+}

pkg/mcclient/modules/llm/mod_llm_sku.go

@@ -5,20 +5,20 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 )
 
-type DifyModelManager struct {
+type LLMSkuManager struct {
 	modulebase.ResourceManager
 }
 
 var (
-	DifyModel DifyModelManager
+	LLMSku LLMSkuManager
 )
 
 func init() {
-	DifyModel = DifyModelManager{
-		ResourceManager: modules.NewLLMManager("dify_model", "dify_models",
+	LLMSku = LLMSkuManager{
+		ResourceManager: modules.NewLLMManager("llm_sku", "llm_skus",
 			[]string{},
 			[]string{},
 		),
 	}
-	modules.Register(&DifyModel)
+	modules.Register(&LLMSku)
 }

pkg/mcclient/options/llm/dify.go

@@ -9,7 +9,7 @@ import (
 type DifyListOptions struct {
 	LLMBaseListOptions
 
-	DifyModel string `help:"filter by dify model"`
+	DifySku string `help:"filter by dify sku"`
 }
 
 func (o *DifyListOptions) Params() (jsonutils.JSONObject, error) {
@@ -34,7 +34,7 @@ func (o *DifyShowOptions) Params() (jsonutils.JSONObject, error) {
 type DifyCreateOptions struct {
 	LLMBaseCreateOptions
 
-	DIFY_MODEL_ID string `help:"dify model id or name" json:"dify_model_id"`
+	DIFY_SKU_ID string `help:"dify sku id or name" json:"dify_sku_id"`
 }
 
 func (o *DifyCreateOptions) Params() (jsonutils.JSONObject, error) {

pkg/mcclient/options/llm/dify_model.go

@@ -1,78 +0,0 @@
-package llm
-
-import (
-	"yunion.io/x/jsonutils"
-
-	"yunion.io/x/onecloud/pkg/mcclient/options"
-)
-
-type DifyModelListOptions struct {
-	options.BaseListOptions
-}
-
-func (o *DifyModelListOptions) Params() (jsonutils.JSONObject, error) {
-	return options.ListStructToParams(o)
-}
-
-type DifyModelShowOptions struct {
-	options.BaseShowOptions
-}
-
-func (o *DifyModelShowOptions) Params() (jsonutils.JSONObject, error) {
-	return options.StructToParams(o)
-}
-
-type DifyModelCreateOptions struct {
-	LLMModelBaseCreateOptions
-
-	POSTGRES_IMAGE_ID      string `json:"postgres_image_id"`
-	REDIS_IMAGE_ID         string `json:"redis_image_id"`
-	NGINX_IMAGE_ID         string `json:"nginx_image_id"`
-	DIFY_API_IMAGE_ID      string `json:"dify_api_image_id"`
-	DIFY_PLUGIN_IMAGE_ID   string `json:"dify_plugin_image_id"`
-	DIFY_WEB_IMAGE_ID      string `json:"dify_web_image_id"`
-	DIFY_SANDBOX_IMAGE_ID  string `json:"dify_sandbox_image_id"`
-	DIFY_SSRF_IMAGE_ID     string `json:"dify_ssrf_image_id"`
-	DIFY_WEAVIATE_IMAGE_ID string `json:"dify_weaviate_image_id"`
-}
-
-func (o *DifyModelCreateOptions) Params() (jsonutils.JSONObject, error) {
-	dict := jsonutils.NewDict()
-	obj := jsonutils.Marshal(o)
-	obj.Unmarshal(dict)
-
-	o.LLMModelBaseCreateOptions.Params(dict)
-	return dict, nil
-}
-
-type DifyModelDeleteOptions struct {
-	options.BaseIdOptions
-}
-
-func (o *DifyModelDeleteOptions) GetId() string {
-	return o.ID
-}
-
-func (o *DifyModelDeleteOptions) Params() (jsonutils.JSONObject, error) {
-	return options.StructToParams(o)
-}
-
-type DifyModelUpdateOptions struct {
-	LLMModelBaseUpdateOptions
-
-	LlmImageId   string
-	LlmModelName string
-}
-
-func (o *DifyModelUpdateOptions) GetId() string {
-	return o.ID
-}
-
-func (o *DifyModelUpdateOptions) Params() (jsonutils.JSONObject, error) {
-	dict := jsonutils.NewDict()
-	obj := jsonutils.Marshal(o)
-	obj.Unmarshal(dict)
-
-	o.LLMModelBaseUpdateOptions.Params(dict)
-	return dict, nil
-}

pkg/mcclient/options/llm/dify_sku.go

@@ -0,0 +1,85 @@
+package llm
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+type DifySkuListOptions struct {
+	options.BaseListOptions
+}
+
+func (o *DifySkuListOptions) Params() (jsonutils.JSONObject, error) {
+	return options.ListStructToParams(o)
+}
+
+type DifySkuShowOptions struct {
+	options.BaseShowOptions
+}
+
+func (o *DifySkuShowOptions) Params() (jsonutils.JSONObject, error) {
+	return options.StructToParams(o)
+}
+
+type DifySkuCreateOptions struct {
+	LLMSkuBaseCreateOptions
+
+	POSTGRES_IMAGE_ID      string `json:"postgres_image_id"`
+	REDIS_IMAGE_ID         string `json:"redis_image_id"`
+	NGINX_IMAGE_ID         string `json:"nginx_image_id"`
+	DIFY_API_IMAGE_ID      string `json:"dify_api_image_id"`
+	DIFY_PLUGIN_IMAGE_ID   string `json:"dify_plugin_image_id"`
+	DIFY_WEB_IMAGE_ID      string `json:"dify_web_image_id"`
+	DIFY_SANDBOX_IMAGE_ID  string `json:"dify_sandbox_image_id"`
+	DIFY_SSRF_IMAGE_ID     string `json:"dify_ssrf_image_id"`
+	DIFY_WEAVIATE_IMAGE_ID string `json:"dify_weaviate_image_id"`
+}
+
+func (o *DifySkuCreateOptions) Params() (jsonutils.JSONObject, error) {
+	dict := jsonutils.NewDict()
+	obj := jsonutils.Marshal(o)
+	obj.Unmarshal(dict)
+
+	o.LLMSkuBaseCreateOptions.Params(dict)
+	return dict, nil
+}
+
+type DifySkuDeleteOptions struct {
+	options.BaseIdOptions
+}
+
+func (o *DifySkuDeleteOptions) GetId() string {
+	return o.ID
+}
+
+func (o *DifySkuDeleteOptions) Params() (jsonutils.JSONObject, error) {
+	return options.StructToParams(o)
+}
+
+type DifySkuUpdateOptions struct {
+	LLMSkuBaseUpdateOptions
+
+	PostgresImageID     string `json:"postgres_image_id"`
+	RedisImageID        string `json:"redis_image_id"`
+	NginxImageID        string `json:"nginx_image_id"`
+	DifyApiImageID      string `json:"dify_api_image_id"`
+	DifyPluginImageID   string `json:"dify_plugin_image_id"`
+	DifyWebImageID      string `json:"dify_web_image_id"`
+	DifySandboxImageID  string `json:"dify_sandbox_image_id"`
+	DifySsrfImageID     string `json:"dify_ssrf_image_id"`
+	DifyWeaviateImageID string `json:"dify_weaviate_image_id"`
+}
+
+func (o *DifySkuUpdateOptions) GetId() string {
+	return o.ID
+}
+
+func (o *DifySkuUpdateOptions) Params() (jsonutils.JSONObject, error) {
+	dict := jsonutils.NewDict()
+	obj := jsonutils.Marshal(o)
+	obj.Unmarshal(dict)
+
+	o.LLMSkuBaseUpdateOptions.Params(dict)
+	return dict, nil
+}

pkg/mcclient/options/llm/instantmodel.go

@@ -0,0 +1,40 @@
+package llm
+
+import (
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+type LLMInstantModelListOptions struct {
+	options.BaseListOptions
+
+	ModelName []string `help:"filter by model name"`
+	Tag       []string `help:"filter by model tag"`
+}
+
+func (o *LLMInstantModelListOptions) Params() (jsonutils.JSONObject, error) {
+	return options.ListStructToParams(o)
+}
+
+type LLMInstantModelShowOptions struct {
+	options.BaseShowOptions
+}
+
+func (o *LLMInstantModelShowOptions) Params() (jsonutils.JSONObject, error) {
+	return options.StructToParams(o)
+}
+
+type LLMInstantModelCreateOptions struct {
+	options.BaseCreateOptions
+
+	MODEL_NAME string `json:"model_name"`
+	TAG        string `json:"tag"`
+
+	ImageId string `json:"image_id"`
+
+	Mounts []string `json:"mounts"`
+}
+
+func (o *LLMInstantModelCreateOptions) Params() (jsonutils.JSONObject, error) {
+	return jsonutils.Marshal(o), nil
+}

pkg/mcclient/options/llm/llm.go

@@ -1,8 +1,12 @@
 package llm
 
 import (
-	"yunion.io/x/jsonutils"
+	"strings"
 
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/util/regutils"
+
+	api "yunion.io/x/onecloud/pkg/apis/llm"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
 )
 
@@ -21,7 +25,7 @@ type LLMBaseListOptions struct {
 type LLMListOptions struct {
 	LLMBaseListOptions
 
-	LlmModel string `help:"filter by llm model"`
+	LlmSku   string `help:"filter by llm sku"`
 	LlmImage string `help:"filter by llm image"`
 }
 
@@ -59,7 +63,7 @@ type LLMBaseCreateOptions struct {
 type LLMCreateOptions struct {
 	LLMBaseCreateOptions
 
-	LLM_MODEL_ID string `help:"llm model id or name" json:"llm_model_id"`
+	LLM_SKU_ID string `help:"llm sku id or name" json:"llm_sku_id"`
 }
 
 func (o *LLMCreateOptions) Params() (jsonutils.JSONObject, error) {
@@ -97,3 +101,83 @@ type LLMStopOptions struct {
 func (o *LLMStopOptions) Params() (jsonutils.JSONObject, error) {
 	return jsonutils.Marshal(o), nil
 }
+
+type LLMIdOptions struct {
+	ID string `help:"llm id" json:"-"`
+}
+
+func (opts *LLMIdOptions) GetId() string {
+	return opts.ID
+}
+
+func (opts *LLMIdOptions) Params() (jsonutils.JSONObject, error) {
+	return jsonutils.Marshal(opts), nil
+}
+
+type LLMSaveInstantModelOptions struct {
+	LLMIdOptions
+
+	MODEL_ID string `help:"llm model id, e.g. 500a1f067a9f"`
+	Name     string `help:"instant app name, e.g. qwen3:8b"`
+
+	// AutoRestart bool
+}
+
+func (opts *LLMSaveInstantModelOptions) Params() (jsonutils.JSONObject, error) {
+	input := api.LLMSaveInstantModelInput{
+		ModelId:   opts.MODEL_ID,
+		ImageName: opts.Name,
+		// AutoRestart: opts.AutoRestart,
+	}
+	return jsonutils.Marshal(input), nil
+}
+
+type LLMQuickModelsOptions struct {
+	LLMIdOptions
+
+	MODEL []string `help:"model id and optional display name in the format of modelId[@modelName:modelTag], e.g. 6f48b936a09f or 6f48b936a09f@qwen2:0.5b"`
+
+	Method string `help:"install or uninstall" choices:"install|uninstall"`
+}
+
+func (opts *LLMQuickModelsOptions) Params() (jsonutils.JSONObject, error) {
+	params := api.LLMPerformQuickModelsInput{}
+	for _, mdlFul := range opts.MODEL {
+		var mdl api.ModelInfo
+
+		var idPart string
+		var nameAndTagPart string
+
+		if idx := strings.Index(mdlFul, "@"); idx >= 0 {
+			idPart = mdlFul[:idx]
+			nameAndTagPart = mdlFul[idx+1:]
+
+			if idxTag := strings.LastIndex(nameAndTagPart, ":"); idxTag >= 0 {
+				mdl.DisplayName = nameAndTagPart[:idxTag]
+				mdl.Tag = nameAndTagPart[idxTag+1:]
+			} else {
+				mdl.DisplayName = nameAndTagPart
+			}
+		} else {
+			idPart = mdlFul
+
+			if idxTag := strings.LastIndex(idPart, ":"); idxTag >= 0 {
+				mdl.Tag = idPart[idxTag+1:]
+				idPart = idPart[:idxTag]
+			}
+		}
+
+		if regutils.MatchUUID(idPart) {
+			mdl.Id = idPart
+		} else {
+			mdl.ModelId = idPart
+		}
+
+		params.Models = append(params.Models, mdl)
+	}
+
+	if len(opts.Method) > 0 {
+		params.Method = api.TQuickModelMethod(opts.Method)
+	}
+	return jsonutils.Marshal(params), nil
+}

pkg/mcclient/options/llm/llm_sku.go

@@ -6,69 +6,69 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/options"
 )
 
-type LLMModelListOptions struct {
+type LLMSkuListOptions struct {
 	options.BaseListOptions
 
 	LLMType string `json:"llm_type" choices:"ollama"`
 }
 
-func (o *LLMModelListOptions) Params() (jsonutils.JSONObject, error) {
+func (o *LLMSkuListOptions) Params() (jsonutils.JSONObject, error) {
 	return options.ListStructToParams(o)
 }
 
-type LLMModelShowOptions struct {
+type LLMSkuShowOptions struct {
 	options.BaseShowOptions
 }
 
-func (o *LLMModelShowOptions) Params() (jsonutils.JSONObject, error) {
+func (o *LLMSkuShowOptions) Params() (jsonutils.JSONObject, error) {
 	return options.StructToParams(o)
 }
 
-type LLMModelCreateOptions struct {
-	LLMModelBaseCreateOptions
+type LLMSkuCreateOptions struct {
+	LLMSkuBaseCreateOptions
 
 	LLM_IMAGE_ID   string `json:"llm_image_id"`
 	LLM_TYPE       string `json:"llm_type" choices:"ollama"`
 	LLM_MODEL_NAME string `help:"specific model of large language model, for example: qwen3:32b" json:"llm_model_name"`
 }
 
-func (o *LLMModelCreateOptions) Params() (jsonutils.JSONObject, error) {
+func (o *LLMSkuCreateOptions) Params() (jsonutils.JSONObject, error) {
 	dict := jsonutils.NewDict()
 	obj := jsonutils.Marshal(o)
 	obj.Unmarshal(dict)
 
-	o.LLMModelBaseCreateOptions.Params(dict)
+	o.LLMSkuBaseCreateOptions.Params(dict)
 	return dict, nil
 }
 
-type LLMModelDeleteOptions struct {
+type LLMSkuDeleteOptions struct {
 	options.BaseIdOptions
 }
 
-func (o *LLMModelDeleteOptions) GetId() string {
+func (o *LLMSkuDeleteOptions) GetId() string {
 	return o.ID
 }
 
-func (o *LLMModelDeleteOptions) Params() (jsonutils.JSONObject, error) {
+func (o *LLMSkuDeleteOptions) Params() (jsonutils.JSONObject, error) {
 	return options.StructToParams(o)
 }
 
-type LLMModelUpdateOptions struct {
-	LLMModelBaseUpdateOptions
+type LLMSkuUpdateOptions struct {
+	LLMSkuBaseUpdateOptions
 
 	LlmImageId   string
 	LlmModelName string
 }
 
-func (o *LLMModelUpdateOptions) GetId() string {
+func (o *LLMSkuUpdateOptions) GetId() string {
 	return o.ID
 }
 
-func (o *LLMModelUpdateOptions) Params() (jsonutils.JSONObject, error) {
+func (o *LLMSkuUpdateOptions) Params() (jsonutils.JSONObject, error) {
 	dict := jsonutils.NewDict()
 	obj := jsonutils.Marshal(o)
 	obj.Unmarshal(dict)
 
-	o.LLMModelBaseUpdateOptions.Params(dict)
+	o.LLMSkuBaseUpdateOptions.Params(dict)
 	return dict, nil
 }

pkg/mcclient/options/llm/llm_sku_base.go

@@ -12,7 +12,7 @@ import (
 	api "yunion.io/x/onecloud/pkg/apis/llm"
 )
 
-type LLMModelBaseCreateOptions struct {
+type LLMSkuBaseCreateOptions struct {
 	apis.SharableVirtualResourceCreateInput
 
 	CPU       int
@@ -36,7 +36,7 @@ type LLMModelBaseCreateOptions struct {
 	Entrypoint string `help:"entrypoint"`
 }
 
-func (o *LLMModelBaseCreateOptions) Params(dict *jsonutils.JSONDict) error {
+func (o *LLMSkuBaseCreateOptions) Params(dict *jsonutils.JSONDict) error {
 	vol := api.Volume{
 		SizeMB:      o.DISK_SIZE,
 		TemplateId:  o.TemplateId,
@@ -52,7 +52,7 @@ func (o *LLMModelBaseCreateOptions) Params(dict *jsonutils.JSONDict) error {
 	return nil
 }
 
-type LLMModelBaseUpdateOptions struct {
+type LLMSkuBaseUpdateOptions struct {
 	apis.SharableVirtualResourceBaseUpdateInput
 
 	ID string
@@ -81,7 +81,7 @@ type LLMModelBaseUpdateOptions struct {
 	Entrypoint string `help:"entrypoint"`
 }
 
-func (o *LLMModelBaseUpdateOptions) Params(dict *jsonutils.JSONDict) error {
+func (o *LLMSkuBaseUpdateOptions) Params(dict *jsonutils.JSONDict) error {
 	if o.NoTemplate {
 		dict.Set("template_id", jsonutils.NewString(""))
 	}