diff --git a/pkg/apis/llm/llm.go b/pkg/apis/llm/llm.go index 231f1ed709..c68c779a2a 100644 --- a/pkg/apis/llm/llm.go +++ b/pkg/apis/llm/llm.go @@ -58,7 +58,7 @@ type LLMListDetails struct { LLMSku string `json:"llm_sku"` LLMType string `json:"llm_type"` - MountedModels []MountedModelInfo `json:"mounted_models"` + MountedModelInfos []MountedModelInfo `json:"mounted_model_infos"` } type LLMBaseCreateInput struct { @@ -76,18 +76,30 @@ type LLMBaseCreateInput struct { type LLMCreateInput struct { LLMBaseCreateInput + // MountedModels overrides the SKU's mounted_models when non-empty. + MountedModelResourceCreateInput LLMSkuId string `json:"llm_sku_id"` LLMImageId string `json:"llm_image_id"` LLMSpec *LLMSpec `json:"llm_spec,omitempty"` + + // Devices/HostPaths override the corresponding sku fields when set. + HostPaths *HostPaths `json:"host_paths,omitempty"` + Devices *Devices `json:"devices,omitempty"` } // LLMUpdateInput is the request body for updating an LLM (including llm_spec overrides). type LLMUpdateInput struct { apis.VirtualResourceBaseUpdateInput + // MountedModels overrides the SKU's mounted_models when non-empty. + MountedModelResourceUpdateInput InstantModelQuotaGb *int `json:"instant_model_quota_gb,omitempty"` LLMSpec *LLMSpec `json:"llm_spec,omitempty"` + + // Devices/HostPaths override the corresponding sku fields when set. + HostPaths *HostPaths `json:"host_paths,omitempty"` + Devices *Devices `json:"devices,omitempty"` } type LLMBaseListInput struct { diff --git a/pkg/llm/drivers/llm_container/base_driver.go b/pkg/llm/drivers/llm_container/base_driver.go index 220b440dc4..f4f744e59f 100644 --- a/pkg/llm/drivers/llm_container/base_driver.go +++ b/pkg/llm/drivers/llm_container/base_driver.go @@ -124,3 +124,11 @@ func (b *baseDriver) ValidateLLMCreateSpec(ctx context.Context, userCred mcclien func (b *baseDriver) ValidateLLMUpdateSpec(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, input *api.LLMSpec) (*api.LLMSpec, error) { return input, nil } + +func (b *baseDriver) ValidateLLMCreateData(ctx context.Context, userCred mcclient.TokenCredential, sku *models.SLLMSku, input *api.LLMCreateInput) (*api.LLMCreateInput, error) { + return input, nil +} + +func (b *baseDriver) ValidateLLMUpdateData(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, sku *models.SLLMSku, input *api.LLMUpdateInput) (*api.LLMUpdateInput, error) { + return input, nil +} diff --git a/pkg/llm/drivers/llm_container/comfyui.go b/pkg/llm/drivers/llm_container/comfyui.go index a859a7c1e0..628ff966cb 100644 --- a/pkg/llm/drivers/llm_container/comfyui.go +++ b/pkg/llm/drivers/llm_container/comfyui.go @@ -35,6 +35,20 @@ func newComfyUI() models.ILLMContainerDriver { return &comfyui{baseDriver: newBaseDriver(api.LLM_CONTAINER_COMFYUI)} } +func (c *comfyui) ValidateLLMCreateData(ctx context.Context, userCred mcclient.TokenCredential, sku *models.SLLMSku, input *api.LLMCreateInput) (*api.LLMCreateInput, error) { + if err := models.ValidateRequireDevices(string(api.LLM_CONTAINER_COMFYUI), input.Devices, nil, sku); err != nil { + return input, err + } + return input, nil +} + +func (c *comfyui) ValidateLLMUpdateData(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, sku *models.SLLMSku, input *api.LLMUpdateInput) (*api.LLMUpdateInput, error) { + if err := models.ValidateRequireDevices(string(api.LLM_CONTAINER_COMFYUI), input.Devices, llm.Devices, sku); err != nil { + return input, err + } + return input, nil +} + func (c *comfyui) GetSpec(sku *models.SLLMSku) interface{} { if sku.LLMSpec == nil { return nil diff --git a/pkg/llm/drivers/llm_container/dify.go b/pkg/llm/drivers/llm_container/dify.go index 2d4051663a..c361944d3e 100644 --- a/pkg/llm/drivers/llm_container/dify.go +++ b/pkg/llm/drivers/llm_container/dify.go @@ -159,6 +159,14 @@ func (d *dify) ValidateLLMSkuUpdateData(ctx context.Context, userCred mcclient.T return input, nil } +func (d *dify) ValidateLLMCreateData(ctx context.Context, userCred mcclient.TokenCredential, sku *models.SLLMSku, input *api.LLMCreateInput) (*api.LLMCreateInput, error) { + return input, nil +} + +func (d *dify) ValidateLLMUpdateData(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, sku *models.SLLMSku, input *api.LLMUpdateInput) (*api.LLMUpdateInput, error) { + return input, nil +} + // ValidateLLMCreateSpec implements ILLMContainerDriver. Validates image ids and merges empty fields from SKU spec. func (d *dify) ValidateLLMCreateSpec(ctx context.Context, userCred mcclient.TokenCredential, sku *models.SLLMSku, input *api.LLMSpec) (*api.LLMSpec, error) { if input == nil || input.Dify == nil { diff --git a/pkg/llm/drivers/llm_container/ollama.go b/pkg/llm/drivers/llm_container/ollama.go index 2a256b2e6b..163c94d227 100644 --- a/pkg/llm/drivers/llm_container/ollama.go +++ b/pkg/llm/drivers/llm_container/ollama.go @@ -35,6 +35,28 @@ func newOllama() models.ILLMContainerDriver { return &ollama{baseDriver: newBaseDriver(api.LLM_CONTAINER_OLLAMA)} } +func (o *ollama) ValidateLLMCreateData(ctx context.Context, userCred mcclient.TokenCredential, sku *models.SLLMSku, input *api.LLMCreateInput) (*api.LLMCreateInput, error) { + llmType := string(api.LLM_CONTAINER_OLLAMA) + if err := models.ValidateRequireDevices(llmType, input.Devices, nil, sku); err != nil { + return input, err + } + if err := models.ValidateRequireMountedModels(llmType, input.MountedModels, nil, sku); err != nil { + return input, err + } + return input, nil +} + +func (o *ollama) ValidateLLMUpdateData(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, sku *models.SLLMSku, input *api.LLMUpdateInput) (*api.LLMUpdateInput, error) { + llmType := string(api.LLM_CONTAINER_OLLAMA) + if err := models.ValidateRequireDevices(llmType, input.Devices, llm.Devices, sku); err != nil { + return input, err + } + if err := models.ValidateRequireMountedModels(llmType, input.MountedModels, llm.MountedModels, sku); err != nil { + return input, err + } + return input, nil +} + func (o *ollama) GetSpec(sku *models.SLLMSku) interface{} { if sku.LLMType != string(api.LLM_CONTAINER_OLLAMA) || sku.LLMSpec == nil || sku.LLMSpec.Ollama == nil { return nil diff --git a/pkg/llm/drivers/llm_container/vllm.go b/pkg/llm/drivers/llm_container/vllm.go index 72ca1aaf8a..7f7c980c4b 100644 --- a/pkg/llm/drivers/llm_container/vllm.go +++ b/pkg/llm/drivers/llm_container/vllm.go @@ -216,6 +216,28 @@ func (v *vllm) GetEffectiveSpec(llm *models.SLLM, sku *models.SLLMSku) interface return out } +func (v *vllm) ValidateLLMCreateData(ctx context.Context, userCred mcclient.TokenCredential, sku *models.SLLMSku, input *api.LLMCreateInput) (*api.LLMCreateInput, error) { + llmType := string(api.LLM_CONTAINER_VLLM) + if err := models.ValidateRequireDevices(llmType, input.Devices, nil, sku); err != nil { + return input, err + } + if err := models.ValidateRequireMountedModels(llmType, input.MountedModels, nil, sku); err != nil { + return input, err + } + return input, nil +} + +func (v *vllm) ValidateLLMUpdateData(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, sku *models.SLLMSku, input *api.LLMUpdateInput) (*api.LLMUpdateInput, error) { + llmType := string(api.LLM_CONTAINER_VLLM) + if err := models.ValidateRequireDevices(llmType, input.Devices, llm.Devices, sku); err != nil { + return input, err + } + if err := models.ValidateRequireMountedModels(llmType, input.MountedModels, llm.MountedModels, sku); err != nil { + return input, err + } + return input, nil +} + func (v *vllm) ValidateLLMSkuCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.LLMSkuCreateInput) (*api.LLMSkuCreateInput, error) { input, err := v.baseDriver.ValidateLLMSkuCreateData(ctx, userCred, input) if err != nil { diff --git a/pkg/llm/models/llm.go b/pkg/llm/models/llm.go index b689148bb8..c659acdd4d 100644 --- a/pkg/llm/models/llm.go +++ b/pkg/llm/models/llm.go @@ -59,6 +59,8 @@ type SLLMManager struct { type SLLM struct { SLLMBase + // MountedModels overrides the SKU's mounted_models when non-empty; otherwise SKU's value is used. + SMountedModelsResource LLMSkuId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` LLMImageId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` @@ -78,6 +80,78 @@ func (llm *SLLM) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCre return nil } +// getEffectiveMountedModels returns the effective mounted model ids: llm's override takes priority over sku's when non-empty. +func getEffectiveMountedModels(llm *SLLM, sku *SLLMSku) []string { + if llm != nil && len(llm.MountedModels) > 0 { + return llm.MountedModels + } + if sku != nil { + return sku.GetMountedModels() + } + return nil +} + +func devicesIsEmpty(d *api.Devices) bool { + return d == nil || d.IsZero() +} + +// ValidateRequireDevices errors if neither input nor existing llm nor sku supplies devices. For create, pass nil for llmCurDevices. +func ValidateRequireDevices( + llmType string, + inputDevices *api.Devices, + llmCurDevices *api.Devices, + sku *SLLMSku, +) error { + effectiveDevices := llmCurDevices + if inputDevices != nil { + effectiveDevices = inputDevices + } + if devicesIsEmpty(effectiveDevices) && sku != nil { + effectiveDevices = sku.Devices + } + if devicesIsEmpty(effectiveDevices) { + return errors.Wrapf(httperrors.ErrInputParameter, "devices is required for %s: specify in request or set on sku", llmType) + } + return nil +} + +// ValidateRequireMountedModels errors if neither input nor existing llm nor sku supplies mounted_models. For create, pass nil/empty for llmCurMountedModels. +func ValidateRequireMountedModels( + llmType string, + inputMountedModels []string, + llmCurMountedModels []string, + sku *SLLMSku, +) error { + effectiveModels := llmCurMountedModels + if inputMountedModels != nil { + effectiveModels = inputMountedModels + } + if len(effectiveModels) == 0 && sku != nil { + effectiveModels = sku.GetMountedModels() + } + if len(effectiveModels) == 0 { + return errors.Wrapf(httperrors.ErrInputParameter, "mounted_models is required for %s: specify in request or set on sku", llmType) + } + return nil +} + +// validateMountedModelsAgainstLLMType ensures every id exists and is compatible with the given llm_type, replacing names with ids in-place. +func validateMountedModelsAgainstLLMType(ctx context.Context, userCred mcclient.TokenCredential, mountedModels []string, llmType string) ([]string, error) { + out := make([]string, len(mountedModels)) + for i, mdl := range mountedModels { + instMdl, err := GetInstantModelManager().FetchByIdOrName(ctx, userCred, mdl) + if err != nil { + return nil, errors.Wrapf(err, "validate mounted model %s", mdl) + } + instantModel := instMdl.(*SInstantModel) + if !api.IsLLMInstantModelCompatible(api.LLMContainerType(instantModel.LlmType), api.LLMContainerType(llmType)) { + return nil, errors.Wrapf(httperrors.ErrInvalidStatus, "mounted model %s is not of type %s", mdl, llmType) + } + out[i] = instantModel.GetId() + } + return out, nil +} + func (man *SLLMManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.LLMCreateInput) (*api.LLMCreateInput, error) { var err error input.LLMBaseCreateInput, err = man.SLLMBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.LLMBaseCreateInput) @@ -101,6 +175,20 @@ func (man *SLLMManager) ValidateCreateData(ctx context.Context, userCred mcclien input.LLMSpec = spec } + if len(input.MountedModels) > 0 { + ids, err := validateMountedModelsAgainstLLMType(ctx, userCred, input.MountedModels, lSku.LLMType) + if err != nil { + return input, errors.Wrap(err, "validate mounted_models") + } + input.MountedModels = ids + } + + drv := lSku.GetLLMContainerDriver() + input, err = drv.ValidateLLMCreateData(ctx, userCred, lSku, input) + if err != nil { + return input, errors.Wrap(err, "validate LLM create data") + } + return input, nil } @@ -186,7 +274,7 @@ func (man *SLLMManager) FetchCustomizeColumns( networkIds = append(networkIds, llm.NetworkId) } mountedModelInfo, _ := llm.FetchMountedModelInfo() - res[idx].MountedModels = mountedModelInfo + res[idx].MountedModelInfos = mountedModelInfo res[idx].NetworkType = llm.NetworkType res[idx].NetworkId = llm.NetworkId } @@ -403,14 +491,31 @@ func (llm *SLLM) ValidateUpdateData(ctx context.Context, userCred mcclient.Token return input, errors.Wrap(err, "validate VirtualResourceBaseUpdateInput") } - if input.LLMSpec == nil { - return input, nil - } sku, err := llm.GetLLMSku(llm.LLMSkuId) if err != nil { return input, errors.Wrap(err, "fetch LLMSku") } + + if len(input.MountedModels) > 0 { + ids, err := validateMountedModelsAgainstLLMType(ctx, userCred, input.MountedModels, sku.LLMType) + if err != nil { + return input, errors.Wrap(err, "validate mounted_models") + } + input.MountedModels = ids + } + drv := sku.GetLLMContainerDriver() + out, err := drv.ValidateLLMUpdateData(ctx, userCred, llm, sku, &input) + if err != nil { + return input, errors.Wrap(err, "validate LLM update data") + } + if out != nil { + input = *out + } + + if input.LLMSpec == nil { + return input, nil + } spec, err := drv.ValidateLLMUpdateSpec(ctx, userCred, llm, input.LLMSpec) if err != nil { return input, errors.Wrap(err, "validate LLM update spec") diff --git a/pkg/llm/models/llm_base.go b/pkg/llm/models/llm_base.go index 59c768cf1b..f63741f200 100644 --- a/pkg/llm/models/llm_base.go +++ b/pkg/llm/models/llm_base.go @@ -66,6 +66,30 @@ type SLLMBase struct { NetworkType string `charset:"utf8" list:"user" update:"user" create:"optional"` NetworkId string `charset:"utf8" nullable:"true" list:"user" update:"user" create:"optional"` + + // Devices/HostPaths override the corresponding sku fields when set; sku values are used when nil/empty. + HostPaths *api.HostPaths `charset:"utf8" length:"medium" nullable:"true" list:"user" update:"user" create:"optional"` + Devices *api.Devices `charset:"utf8" length:"medium" nullable:"true" list:"user" update:"user" create:"optional"` +} + +func getEffectiveDevices(llmBase *SLLMBase, skuBase *SLLMSkuBase) *api.Devices { + if llmBase != nil && llmBase.Devices != nil && !llmBase.Devices.IsZero() { + return llmBase.Devices + } + if skuBase != nil { + return skuBase.Devices + } + return nil +} + +func getEffectiveHostPaths(llmBase *SLLMBase, skuBase *SLLMSkuBase) *api.HostPaths { + if llmBase != nil && llmBase.HostPaths != nil && !llmBase.HostPaths.IsZero() { + return llmBase.HostPaths + } + if skuBase != nil { + return skuBase.HostPaths + } + return nil } func (man *SLLMBaseManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.LLMBaseCreateInput) (api.LLMBaseCreateInput, error) { @@ -374,16 +398,17 @@ func GetHostPathVolumeMounts(hostPaths *api.HostPaths, containerIndex int) []*ap return mounts } -func AppendLLMSkuVolumeMounts(containers []*computeapi.PodContainerCreateInput, skuBase *SLLMSkuBase, postOverlays []*apis.ContainerVolumeMountDiskPostOverlay) { +func AppendLLMSkuVolumeMounts(containers []*computeapi.PodContainerCreateInput, llmBase *SLLMBase, skuBase *SLLMSkuBase, postOverlays []*apis.ContainerVolumeMountDiskPostOverlay) { if skuBase == nil { return } + effectiveHostPaths := getEffectiveHostPaths(llmBase, skuBase) for idx := range containers { if containers[idx] == nil { continue } containers[idx].VolumeMounts = append(containers[idx].VolumeMounts, GetDiskVolumeMounts(skuBase.Volumes, idx, postOverlays)...) - containers[idx].VolumeMounts = append(containers[idx].VolumeMounts, GetHostPathVolumeMounts(skuBase.HostPaths, idx)...) + containers[idx].VolumeMounts = append(containers[idx].VolumeMounts, GetHostPathVolumeMounts(effectiveHostPaths, idx)...) } } diff --git a/pkg/llm/models/llm_base_pod.go b/pkg/llm/models/llm_base_pod.go index a777f26dd3..a121df2a51 100644 --- a/pkg/llm/models/llm_base_pod.go +++ b/pkg/llm/models/llm_base_pod.go @@ -62,9 +62,10 @@ func GetLLMBasePodCreateInput( } // isolated devices - if skuBase.Devices != nil && !skuBase.Devices.IsZero() { + effectiveDevices := getEffectiveDevices(llmBase, skuBase) + if effectiveDevices != nil && !effectiveDevices.IsZero() { data.IsolatedDevices = make([]*computeapi.IsolatedDeviceConfig, 0) - devices := *skuBase.Devices + devices := *effectiveDevices for i := 0; i < len(devices); i++ { isolatedDevice := &computeapi.IsolatedDeviceConfig{ DevType: devices[i].DevType, diff --git a/pkg/llm/models/llm_container_driver.go b/pkg/llm/models/llm_container_driver.go index bb374a8060..560d1268be 100644 --- a/pkg/llm/models/llm_container_driver.go +++ b/pkg/llm/models/llm_container_driver.go @@ -103,6 +103,11 @@ type ILLMContainerDriver interface { ValidateLLMCreateSpec(ctx context.Context, userCred mcclient.TokenCredential, sku *SLLMSku, input *llm.LLMSpec) (*llm.LLMSpec, error) ValidateLLMUpdateSpec(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, input *llm.LLMSpec) (*llm.LLMSpec, error) + // ValidateLLMCreateData validates SLLM create input against this driver's requirements (e.g. ollama/vllm require devices + mounted_models from either input or sku). Returns the (possibly modified) input. + ValidateLLMCreateData(ctx context.Context, userCred mcclient.TokenCredential, sku *SLLMSku, input *llm.LLMCreateInput) (*llm.LLMCreateInput, error) + // ValidateLLMUpdateData validates SLLM update input against this driver's requirements; the current SLLM is passed so existing override values can be considered. + ValidateLLMUpdateData(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, sku *SLLMSku, input *llm.LLMUpdateInput) (*llm.LLMUpdateInput, error) + ILLMContainerMCPAgent } @@ -152,7 +157,11 @@ func GetLLMContainerInstantModelDriver(typ llm.LLMContainerType) (ILLMContainerI func GetDriverPodContainers(ctx context.Context, drv ILLMContainerDriver, llm *SLLM, image *SLLMImage, sku *SLLMSku, props []string, devices []computeapi.SIsolatedDevice, diskId string) []*computeapi.PodContainerCreateInput { containers := drv.GetContainerSpecs(ctx, llm, image, sku, props, devices, diskId) if sku != nil { - AppendLLMSkuVolumeMounts(containers, &sku.SLLMSkuBase, nil) + var llmBase *SLLMBase + if llm != nil { + llmBase = &llm.SLLMBase + } + AppendLLMSkuVolumeMounts(containers, llmBase, &sku.SLLMSkuBase, nil) } return containers } diff --git a/pkg/llm/models/llm_instant_model_sync.go b/pkg/llm/models/llm_instant_model_sync.go index c01681cbfd..1f30d3ae25 100644 --- a/pkg/llm/models/llm_instant_model_sync.go +++ b/pkg/llm/models/llm_instant_model_sync.go @@ -691,7 +691,7 @@ func (llm *SLLM) UpdateMountedModelFullNames(ctx context.Context, userCred mccli return errors.Wrap(err, "getDeletedModelIds") } } - mountedModels := sku.GetMountedModels() + mountedModels := getEffectiveMountedModels(llm, sku) for i := range mountedModels { instMdl, err := GetInstantModelManager().FetchByIdOrName(ctx, userCred, mountedModels[i]) if err != nil {