opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-01 15:07:17 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(llm): add SLLM mounted_models override and driver-level validation (#24797)

Browse Source
This commit is contained in:
Zexi Li
2026-05-11 17:41:28 +08:00
committed by GitHub
parent 20612e6f4a
commit b2ec881a68
11 changed files with 237 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/apis/llm/llm.go
+13 -1
View File
@@ -58,7 +58,7 @@ type LLMListDetails struct {
LLMSku string `json:"llm_sku"`
LLMType string `json:"llm_type"`
MountedModels []MountedModelInfo `json:"mounted_models"`
MountedModelInfos []MountedModelInfo `json:"mounted_model_infos"`
}
type LLMBaseCreateInput struct {
@@ -76,18 +76,30 @@ type LLMBaseCreateInput struct {
type LLMCreateInput struct {
LLMBaseCreateInput
// MountedModels overrides the SKU's mounted_models when non-empty.
MountedModelResourceCreateInput
LLMSkuId string `json:"llm_sku_id"`
LLMImageId string `json:"llm_image_id"`
LLMSpec *LLMSpec `json:"llm_spec,omitempty"`
// Devices/HostPaths override the corresponding sku fields when set.
HostPaths *HostPaths `json:"host_paths,omitempty"`
Devices *Devices `json:"devices,omitempty"`
}
// LLMUpdateInput is the request body for updating an LLM (including llm_spec overrides).
type LLMUpdateInput struct {
apis.VirtualResourceBaseUpdateInput
// MountedModels overrides the SKU's mounted_models when non-empty.
MountedModelResourceUpdateInput
InstantModelQuotaGb *int `json:"instant_model_quota_gb,omitempty"`
LLMSpec *LLMSpec `json:"llm_spec,omitempty"`
// Devices/HostPaths override the corresponding sku fields when set.
HostPaths *HostPaths `json:"host_paths,omitempty"`
Devices *Devices `json:"devices,omitempty"`
}
type LLMBaseListInput struct {
pkg/llm/drivers/llm_container/base_driver.go
+8
View File
@@ -124,3 +124,11 @@ func (b *baseDriver) ValidateLLMCreateSpec(ctx context.Context, userCred mcclien
func (b *baseDriver) ValidateLLMUpdateSpec(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, input *api.LLMSpec) (*api.LLMSpec, error) {
return input, nil
}
func (b *baseDriver) ValidateLLMCreateData(ctx context.Context, userCred mcclient.TokenCredential, sku *models.SLLMSku, input *api.LLMCreateInput) (*api.LLMCreateInput, error) {
return input, nil
}
func (b *baseDriver) ValidateLLMUpdateData(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, sku *models.SLLMSku, input *api.LLMUpdateInput) (*api.LLMUpdateInput, error) {
return input, nil
}
pkg/llm/drivers/llm_container/comfyui.go
+14
View File
@@ -35,6 +35,20 @@ func newComfyUI() models.ILLMContainerDriver {
return &comfyui{baseDriver: newBaseDriver(api.LLM_CONTAINER_COMFYUI)}
}
func (c *comfyui) ValidateLLMCreateData(ctx context.Context, userCred mcclient.TokenCredential, sku *models.SLLMSku, input *api.LLMCreateInput) (*api.LLMCreateInput, error) {
if err := models.ValidateRequireDevices(string(api.LLM_CONTAINER_COMFYUI), input.Devices, nil, sku); err != nil {
return input, err
}
return input, nil
}
func (c *comfyui) ValidateLLMUpdateData(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, sku *models.SLLMSku, input *api.LLMUpdateInput) (*api.LLMUpdateInput, error) {
if err := models.ValidateRequireDevices(string(api.LLM_CONTAINER_COMFYUI), input.Devices, llm.Devices, sku); err != nil {
return input, err
}
return input, nil
}
func (c *comfyui) GetSpec(sku *models.SLLMSku) interface{} {
if sku.LLMSpec == nil {
return nil
pkg/llm/drivers/llm_container/dify.go
+8
View File
@@ -159,6 +159,14 @@ func (d *dify) ValidateLLMSkuUpdateData(ctx context.Context, userCred mcclient.T
return input, nil
}
func (d *dify) ValidateLLMCreateData(ctx context.Context, userCred mcclient.TokenCredential, sku *models.SLLMSku, input *api.LLMCreateInput) (*api.LLMCreateInput, error) {
return input, nil
}
func (d *dify) ValidateLLMUpdateData(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, sku *models.SLLMSku, input *api.LLMUpdateInput) (*api.LLMUpdateInput, error) {
return input, nil
}
// ValidateLLMCreateSpec implements ILLMContainerDriver. Validates image ids and merges empty fields from SKU spec.
func (d *dify) ValidateLLMCreateSpec(ctx context.Context, userCred mcclient.TokenCredential, sku *models.SLLMSku, input *api.LLMSpec) (*api.LLMSpec, error) {
if input == nil || input.Dify == nil {
pkg/llm/drivers/llm_container/ollama.go
+22
View File
@@ -35,6 +35,28 @@ func newOllama() models.ILLMContainerDriver {
return &ollama{baseDriver: newBaseDriver(api.LLM_CONTAINER_OLLAMA)}
}
func (o *ollama) ValidateLLMCreateData(ctx context.Context, userCred mcclient.TokenCredential, sku *models.SLLMSku, input *api.LLMCreateInput) (*api.LLMCreateInput, error) {
llmType := string(api.LLM_CONTAINER_OLLAMA)
if err := models.ValidateRequireDevices(llmType, input.Devices, nil, sku); err != nil {
return input, err
}
if err := models.ValidateRequireMountedModels(llmType, input.MountedModels, nil, sku); err != nil {
return input, err
}
return input, nil
}
func (o *ollama) ValidateLLMUpdateData(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, sku *models.SLLMSku, input *api.LLMUpdateInput) (*api.LLMUpdateInput, error) {
llmType := string(api.LLM_CONTAINER_OLLAMA)
if err := models.ValidateRequireDevices(llmType, input.Devices, llm.Devices, sku); err != nil {
return input, err
}
if err := models.ValidateRequireMountedModels(llmType, input.MountedModels, llm.MountedModels, sku); err != nil {
return input, err
}
return input, nil
}
func (o *ollama) GetSpec(sku *models.SLLMSku) interface{} {
if sku.LLMType != string(api.LLM_CONTAINER_OLLAMA) || sku.LLMSpec == nil || sku.LLMSpec.Ollama == nil {
return nil
pkg/llm/drivers/llm_container/vllm.go
+22
View File
@@ -216,6 +216,28 @@ func (v *vllm) GetEffectiveSpec(llm *models.SLLM, sku *models.SLLMSku) interface
return out
}
func (v *vllm) ValidateLLMCreateData(ctx context.Context, userCred mcclient.TokenCredential, sku *models.SLLMSku, input *api.LLMCreateInput) (*api.LLMCreateInput, error) {
llmType := string(api.LLM_CONTAINER_VLLM)
if err := models.ValidateRequireDevices(llmType, input.Devices, nil, sku); err != nil {
return input, err
}
if err := models.ValidateRequireMountedModels(llmType, input.MountedModels, nil, sku); err != nil {
return input, err
}
return input, nil
}
func (v *vllm) ValidateLLMUpdateData(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, sku *models.SLLMSku, input *api.LLMUpdateInput) (*api.LLMUpdateInput, error) {
llmType := string(api.LLM_CONTAINER_VLLM)
if err := models.ValidateRequireDevices(llmType, input.Devices, llm.Devices, sku); err != nil {
return input, err
}
if err := models.ValidateRequireMountedModels(llmType, input.MountedModels, llm.MountedModels, sku); err != nil {
return input, err
}
return input, nil
}
func (v *vllm) ValidateLLMSkuCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.LLMSkuCreateInput) (*api.LLMSkuCreateInput, error) {
input, err := v.baseDriver.ValidateLLMSkuCreateData(ctx, userCred, input)
if err != nil {
pkg/llm/models/llm.go
+109 -4
View File
@@ -59,6 +59,8 @@ type SLLMManager struct {
type SLLM struct {
SLLMBase
// MountedModels overrides the SKU's mounted_models when non-empty; otherwise SKU's value is used.
SMountedModelsResource
LLMSkuId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"`
LLMImageId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"`
@@ -78,6 +80,78 @@ func (llm *SLLM) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCre
return nil
}
// getEffectiveMountedModels returns the effective mounted model ids: llm's override takes priority over sku's when non-empty.
func getEffectiveMountedModels(llm *SLLM, sku *SLLMSku) []string {
if llm != nil && len(llm.MountedModels) > 0 {
return llm.MountedModels
}
if sku != nil {
return sku.GetMountedModels()
}
return nil
}
func devicesIsEmpty(d *api.Devices) bool {
return d == nil || d.IsZero()
}
// ValidateRequireDevices errors if neither input nor existing llm nor sku supplies devices. For create, pass nil for llmCurDevices.
func ValidateRequireDevices(
llmType string,
inputDevices *api.Devices,
llmCurDevices *api.Devices,
sku *SLLMSku,
) error {
effectiveDevices := llmCurDevices
if inputDevices != nil {
effectiveDevices = inputDevices
}
if devicesIsEmpty(effectiveDevices) && sku != nil {
effectiveDevices = sku.Devices
}
if devicesIsEmpty(effectiveDevices) {
return errors.Wrapf(httperrors.ErrInputParameter, "devices is required for %s: specify in request or set on sku", llmType)
}
return nil
}
// ValidateRequireMountedModels errors if neither input nor existing llm nor sku supplies mounted_models. For create, pass nil/empty for llmCurMountedModels.
func ValidateRequireMountedModels(
llmType string,
inputMountedModels []string,
llmCurMountedModels []string,
sku *SLLMSku,
) error {
effectiveModels := llmCurMountedModels
if inputMountedModels != nil {
effectiveModels = inputMountedModels
}
if len(effectiveModels) == 0 && sku != nil {
effectiveModels = sku.GetMountedModels()
}
if len(effectiveModels) == 0 {
return errors.Wrapf(httperrors.ErrInputParameter, "mounted_models is required for %s: specify in request or set on sku", llmType)
}
return nil
}
// validateMountedModelsAgainstLLMType ensures every id exists and is compatible with the given llm_type, replacing names with ids in-place.
func validateMountedModelsAgainstLLMType(ctx context.Context, userCred mcclient.TokenCredential, mountedModels []string, llmType string) ([]string, error) {
out := make([]string, len(mountedModels))
for i, mdl := range mountedModels {
instMdl, err := GetInstantModelManager().FetchByIdOrName(ctx, userCred, mdl)
if err != nil {
return nil, errors.Wrapf(err, "validate mounted model %s", mdl)
}
instantModel := instMdl.(*SInstantModel)
if !api.IsLLMInstantModelCompatible(api.LLMContainerType(instantModel.LlmType), api.LLMContainerType(llmType)) {
return nil, errors.Wrapf(httperrors.ErrInvalidStatus, "mounted model %s is not of type %s", mdl, llmType)
}
out[i] = instantModel.GetId()
}
return out, nil
}
func (man *SLLMManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.LLMCreateInput) (*api.LLMCreateInput, error) {
var err error
input.LLMBaseCreateInput, err = man.SLLMBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.LLMBaseCreateInput)
@@ -101,6 +175,20 @@ func (man *SLLMManager) ValidateCreateData(ctx context.Context, userCred mcclien
input.LLMSpec = spec
}
if len(input.MountedModels) > 0 {
ids, err := validateMountedModelsAgainstLLMType(ctx, userCred, input.MountedModels, lSku.LLMType)
if err != nil {
return input, errors.Wrap(err, "validate mounted_models")
}
input.MountedModels = ids
}
drv := lSku.GetLLMContainerDriver()
input, err = drv.ValidateLLMCreateData(ctx, userCred, lSku, input)
if err != nil {
return input, errors.Wrap(err, "validate LLM create data")
}
return input, nil
}
@@ -186,7 +274,7 @@ func (man *SLLMManager) FetchCustomizeColumns(
networkIds = append(networkIds, llm.NetworkId)
}
mountedModelInfo, _ := llm.FetchMountedModelInfo()
res[idx].MountedModels = mountedModelInfo
res[idx].MountedModelInfos = mountedModelInfo
res[idx].NetworkType = llm.NetworkType
res[idx].NetworkId = llm.NetworkId
}
@@ -403,14 +491,31 @@ func (llm *SLLM) ValidateUpdateData(ctx context.Context, userCred mcclient.Token
return input, errors.Wrap(err, "validate VirtualResourceBaseUpdateInput")
}
if input.LLMSpec == nil {
return input, nil
}
sku, err := llm.GetLLMSku(llm.LLMSkuId)
if err != nil {
return input, errors.Wrap(err, "fetch LLMSku")
}
if len(input.MountedModels) > 0 {
ids, err := validateMountedModelsAgainstLLMType(ctx, userCred, input.MountedModels, sku.LLMType)
if err != nil {
return input, errors.Wrap(err, "validate mounted_models")
}
input.MountedModels = ids
}
drv := sku.GetLLMContainerDriver()
out, err := drv.ValidateLLMUpdateData(ctx, userCred, llm, sku, &input)
if err != nil {
return input, errors.Wrap(err, "validate LLM update data")
}
if out != nil {
input = *out
}
if input.LLMSpec == nil {
return input, nil
}
spec, err := drv.ValidateLLMUpdateSpec(ctx, userCred, llm, input.LLMSpec)
if err != nil {
return input, errors.Wrap(err, "validate LLM update spec")
pkg/llm/models/llm_base.go
+27 -2
View File
@@ -66,6 +66,30 @@ type SLLMBase struct {
NetworkType string `charset:"utf8" list:"user" update:"user" create:"optional"`
NetworkId string `charset:"utf8" nullable:"true" list:"user" update:"user" create:"optional"`
// Devices/HostPaths override the corresponding sku fields when set; sku values are used when nil/empty.
HostPaths *api.HostPaths `charset:"utf8" length:"medium" nullable:"true" list:"user" update:"user" create:"optional"`
Devices *api.Devices `charset:"utf8" length:"medium" nullable:"true" list:"user" update:"user" create:"optional"`
}
func getEffectiveDevices(llmBase *SLLMBase, skuBase *SLLMSkuBase) *api.Devices {
if llmBase != nil && llmBase.Devices != nil && !llmBase.Devices.IsZero() {
return llmBase.Devices
}
if skuBase != nil {
return skuBase.Devices
}
return nil
}
func getEffectiveHostPaths(llmBase *SLLMBase, skuBase *SLLMSkuBase) *api.HostPaths {
if llmBase != nil && llmBase.HostPaths != nil && !llmBase.HostPaths.IsZero() {
return llmBase.HostPaths
}
if skuBase != nil {
return skuBase.HostPaths
}
return nil
}
func (man *SLLMBaseManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.LLMBaseCreateInput) (api.LLMBaseCreateInput, error) {
@@ -374,16 +398,17 @@ func GetHostPathVolumeMounts(hostPaths *api.HostPaths, containerIndex int) []*ap
return mounts
}
func AppendLLMSkuVolumeMounts(containers []*computeapi.PodContainerCreateInput, skuBase *SLLMSkuBase, postOverlays []*apis.ContainerVolumeMountDiskPostOverlay) {
func AppendLLMSkuVolumeMounts(containers []*computeapi.PodContainerCreateInput, llmBase *SLLMBase, skuBase *SLLMSkuBase, postOverlays []*apis.ContainerVolumeMountDiskPostOverlay) {
if skuBase == nil {
return
}
effectiveHostPaths := getEffectiveHostPaths(llmBase, skuBase)
for idx := range containers {
if containers[idx] == nil {
continue
}
containers[idx].VolumeMounts = append(containers[idx].VolumeMounts, GetDiskVolumeMounts(skuBase.Volumes, idx, postOverlays)...)
containers[idx].VolumeMounts = append(containers[idx].VolumeMounts, GetHostPathVolumeMounts(skuBase.HostPaths, idx)...)
containers[idx].VolumeMounts = append(containers[idx].VolumeMounts, GetHostPathVolumeMounts(effectiveHostPaths, idx)...)
}
}
pkg/llm/models/llm_base_pod.go
+3 -2
View File
@@ -62,9 +62,10 @@ func GetLLMBasePodCreateInput(
}
// isolated devices
if skuBase.Devices != nil && !skuBase.Devices.IsZero() {
effectiveDevices := getEffectiveDevices(llmBase, skuBase)
if effectiveDevices != nil && !effectiveDevices.IsZero() {
data.IsolatedDevices = make([]*computeapi.IsolatedDeviceConfig, 0)
devices := *skuBase.Devices
devices := *effectiveDevices
for i := 0; i < len(devices); i++ {
isolatedDevice := &computeapi.IsolatedDeviceConfig{
DevType: devices[i].DevType,
pkg/llm/models/llm_container_driver.go
+10 -1
View File
@@ -103,6 +103,11 @@ type ILLMContainerDriver interface {
ValidateLLMCreateSpec(ctx context.Context, userCred mcclient.TokenCredential, sku *SLLMSku, input *llm.LLMSpec) (*llm.LLMSpec, error)
ValidateLLMUpdateSpec(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, input *llm.LLMSpec) (*llm.LLMSpec, error)
// ValidateLLMCreateData validates SLLM create input against this driver's requirements (e.g. ollama/vllm require devices + mounted_models from either input or sku). Returns the (possibly modified) input.
ValidateLLMCreateData(ctx context.Context, userCred mcclient.TokenCredential, sku *SLLMSku, input *llm.LLMCreateInput) (*llm.LLMCreateInput, error)
// ValidateLLMUpdateData validates SLLM update input against this driver's requirements; the current SLLM is passed so existing override values can be considered.
ValidateLLMUpdateData(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, sku *SLLMSku, input *llm.LLMUpdateInput) (*llm.LLMUpdateInput, error)
ILLMContainerMCPAgent
}
@@ -152,7 +157,11 @@ func GetLLMContainerInstantModelDriver(typ llm.LLMContainerType) (ILLMContainerI
func GetDriverPodContainers(ctx context.Context, drv ILLMContainerDriver, llm *SLLM, image *SLLMImage, sku *SLLMSku, props []string, devices []computeapi.SIsolatedDevice, diskId string) []*computeapi.PodContainerCreateInput {
containers := drv.GetContainerSpecs(ctx, llm, image, sku, props, devices, diskId)
if sku != nil {
AppendLLMSkuVolumeMounts(containers, &sku.SLLMSkuBase, nil)
var llmBase *SLLMBase
if llm != nil {
llmBase = &llm.SLLMBase
}
AppendLLMSkuVolumeMounts(containers, llmBase, &sku.SLLMSkuBase, nil)
}
return containers
}
pkg/llm/models/llm_instant_model_sync.go
+1 -1
View File
@@ -691,7 +691,7 @@ func (llm *SLLM) UpdateMountedModelFullNames(ctx context.Context, userCred mccli
return errors.Wrap(err, "getDeletedModelIds")
}
}
mountedModels := sku.GetMountedModels()
mountedModels := getEffectiveMountedModels(llm, sku)
for i := range mountedModels {
instMdl, err := GetInstantModelManager().FetchByIdOrName(ctx, userCred, mountedModels[i])
if err != nil {