feat(region): release GPU when shutdown_behavior is stop_release_gpu (#20674)

pkg/apis/compute/api.go

@@ -538,7 +538,7 @@ type ServerCreateInput struct {
 
 	// 关机后执行的操作
 	// terminate: 关机后自动删除
-	// emum: stop, terminate
+	// enum: stop, terminate, stop_release_gpu
 	// default: stop
 	ShutdownBehavior string `json:"shutdown_behavior"`
 

pkg/apis/compute/container.go

@@ -39,6 +39,14 @@ const (
 	CONTAINER_DEV_ASCEND_NPU       = "ASCEND_NPU"
 )
 
+var (
+	CONTAINER_GPU_TYPES = []string{
+		CONTAINER_DEV_CPH_AMD_GPU,
+		CONTAINER_DEV_NVIDIA_GPU,
+		CONTAINER_DEV_NVIDIA_MPS,
+	}
+)
+
 const (
 	CONTAINER_STORAGE_LOCAL_RAW = "local_raw"
 )
@@ -66,7 +74,8 @@ const (
 )
 
 const (
-	CONTAINER_METADATA_CRI_ID = "cri_id"
+	CONTAINER_METADATA_CRI_ID           = "cri_id"
+	CONTAINER_METADATA_RELEASED_DEVICES = "released_devices"
 )
 
 type ContainerSpec struct {

pkg/apis/compute/guest_const.go

@@ -14,7 +14,9 @@
 
 package compute
 
-import "yunion.io/x/cloudmux/pkg/apis/compute"
+import (
+	"yunion.io/x/cloudmux/pkg/apis/compute"
+)
 
 const (
 	VM_INIT                  = compute.VM_INIT
@@ -178,8 +180,9 @@ const (
 	VM_QGA_SET_NETWORK        = "qga_set_network"
 	VM_QGA_SET_NETWORK_FAILED = "qga_set_network_failed"
 
-	SHUTDOWN_STOP      = "stop"
-	SHUTDOWN_TERMINATE = "terminate"
+	SHUTDOWN_STOP             = "stop"
+	SHUTDOWN_TERMINATE        = "terminate"
+	SHUTDOWN_STOP_RELEASE_GPU = "stop_release_gpu"
 
 	HYPERVISOR_KVM       = "kvm"
 	HYPERVISOR_POD       = "pod"
@@ -323,6 +326,8 @@ const (
 	VM_METADATA_HOT_REMOVE_NIC      = "hot_remove_nic"
 	VM_METADATA_START_VMEM_MB       = "start_vmem_mb"
 	VM_METADATA_START_VCPU_COUNT    = "start_vcpu_count"
+
+	VM_METADATA_RELEASED_DEVICES = "released_devices"
 )
 
 // windows allow a maximal length of 15

pkg/apis/compute/guests.go

@@ -1335,3 +1335,8 @@ func (conf ServerChangeConfigSettings) AddedDisk() int {
 	}
 	return size
 }
+
+type ServerReleasedIsolatedDevice struct {
+	DevType string `json:"dev_type"`
+	Model   string `json:"model"`
+}

pkg/compute/guestdrivers/base.go

@@ -696,3 +696,11 @@ func (base *SBaseGuestDriver) ValidateGuestChangeConfigInput(ctx context.Context
 func (base *SBaseGuestDriver) ValidateGuestHotChangeConfigInput(ctx context.Context, guest *models.SGuest, confs *api.ServerChangeConfigSettings) (*api.ServerChangeConfigSettings, error) {
 	return confs, nil
 }
+
+func (base *SBaseGuestDriver) BeforeDetachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *models.SGuest, dev *models.SIsolatedDevice) error {
+	return nil
+}
+
+func (base *SBaseGuestDriver) BeforeAttachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *models.SGuest, dev *models.SIsolatedDevice) error {
+	return nil
+}

pkg/compute/guestdrivers/pod.go

@@ -35,6 +35,7 @@ import (
 	hostapi "yunion.io/x/onecloud/pkg/apis/host"
 	"yunion.io/x/onecloud/pkg/apis/image"
 	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/compute/models"
@@ -555,3 +556,106 @@ func (p *SPodDriver) RequestDeleteSnapshot(ctx context.Context, guest *models.SG
 	}*/
 	return p.SKVMGuestDriver.RequestDeleteSnapshot(ctx, guest, task, params)
 }
+
+func (p *SPodDriver) BeforeDetachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, dev *models.SIsolatedDevice) error {
+	ctrs, err := models.GetContainerManager().GetContainersByPod(guest.GetId())
+	if err != nil {
+		return errors.Wrapf(err, "get containers by pod %s", guest.GetId())
+	}
+	for _, ctr := range ctrs {
+		ctrPtr := &ctr
+		spec := ctrPtr.Spec
+		devs := spec.Devices
+		newDevs := make([]*api.ContainerDevice, 0)
+		releasedDevs := make(map[string]models.ContainerReleasedDevice)
+		for _, curDev := range devs {
+			if curDev.IsolatedDevice == nil {
+				continue
+			}
+			if curDev.IsolatedDevice.Id != dev.GetId() {
+				tmpDev := curDev
+				newDevs = append(newDevs, tmpDev)
+			} else {
+				releasedDevs[curDev.IsolatedDevice.Id] = *models.NewContainerReleasedDevice(curDev, dev.DevType, dev.Model)
+			}
+		}
+		if err := ctrPtr.SaveReleasedDevices(ctx, userCred, releasedDevs); err != nil {
+			return errors.Wrapf(err, "save release devices for container %s", ctr.GetId())
+		}
+		if _, err := db.Update(ctrPtr, func() error {
+			ctrPtr.Spec.Devices = newDevs
+			return nil
+		}); err != nil {
+			return errors.Wrapf(err, "update container %s devs", ctrPtr.GetId())
+		}
+	}
+	return nil
+}
+
+func (p *SPodDriver) BeforeAttachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, dev *models.SIsolatedDevice) error {
+	ctrs, err := models.GetContainerManager().GetContainersByPod(guest.GetId())
+	if err != nil {
+		return errors.Wrapf(err, "get containers by pod %s", guest.GetId())
+	}
+	for _, ctr := range ctrs {
+		ctrPtr := &ctr
+		if err := p.attachIsolatedDeviceToContainer(ctx, userCred, ctrPtr, dev); err != nil {
+			return errors.Wrapf(err, "attach isolated device to container %s", ctr.GetId())
+		}
+	}
+	return nil
+}
+
+func (p *SPodDriver) attachIsolatedDeviceToContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrPtr *models.SContainer, dev *models.SIsolatedDevice) error {
+	rlsDevs, err := ctrPtr.GetReleasedDevices(ctx, userCred)
+	if err != nil {
+		return errors.Wrapf(err, "get release devices for container %s", ctrPtr.GetId())
+	}
+	if len(rlsDevs) == 0 {
+		return nil
+	}
+	spec := new(api.ContainerSpec)
+	if err := jsonutils.Marshal(ctrPtr.Spec).Unmarshal(spec); err != nil {
+		return errors.Wrap(err, "deep copy spec")
+	}
+	for id, rlsDev := range rlsDevs {
+		if rlsDev.IsolatedDevice == nil {
+			continue
+		}
+		if rlsDev.DeviceModel == dev.Model && rlsDev.DeviceType == dev.DevType {
+			// attach it
+			if spec.Devices == nil {
+				spec.Devices = make([]*api.ContainerDevice, 0)
+			}
+			shouldUpdate := true
+			for _, curDev := range spec.Devices {
+				if curDev.IsolatedDevice == nil {
+					continue
+				}
+				if curDev.IsolatedDevice.Id == dev.GetId() {
+					shouldUpdate = false
+				}
+			}
+			if shouldUpdate {
+				spec.Devices = append(spec.Devices, &api.ContainerDevice{
+					Type: apis.CONTAINER_DEVICE_TYPE_ISOLATED_DEVICE,
+					IsolatedDevice: &api.ContainerIsolatedDevice{
+						Id: dev.GetId(),
+					},
+				})
+				if _, err := db.Update(ctrPtr, func() error {
+					ctrPtr.Spec = spec
+					return nil
+				}); err != nil {
+					return errors.Wrapf(err, "update container %s devs", ctrPtr.GetId())
+				}
+			}
+			delete(rlsDevs, id)
+			if err := ctrPtr.SaveReleasedDevices(ctx, userCred, rlsDevs); err != nil {
+				return errors.Wrapf(err, "save release devices for container %s", ctrPtr.GetId())
+			}
+			return nil
+		}
+	}
+	return nil
+}

pkg/compute/models/containers.go

@@ -566,3 +566,36 @@ func (c *SContainer) PerformExecSync(ctx context.Context, userCred mcclient.Toke
 	}
 	return c.GetPodDriver().RequestExecSyncContainer(ctx, userCred, c, input)
 }
+
+type ContainerReleasedDevice struct {
+	*api.ContainerDevice
+	DeviceType  string
+	DeviceModel string
+}
+
+func NewContainerReleasedDevice(device *api.ContainerDevice, devType, devModel string) *ContainerReleasedDevice {
+	return &ContainerReleasedDevice{
+		ContainerDevice: device,
+		DeviceType:      devType,
+		DeviceModel:     devModel,
+	}
+}
+
+func (s *SContainer) SaveReleasedDevices(ctx context.Context, userCred mcclient.TokenCredential, devs map[string]ContainerReleasedDevice) error {
+	return s.SetMetadata(ctx, api.CONTAINER_METADATA_RELEASED_DEVICES, devs, userCred)
+}
+
+func (s *SContainer) GetReleasedDevices(ctx context.Context, userCred mcclient.TokenCredential) (map[string]ContainerReleasedDevice, error) {
+	out := make(map[string]ContainerReleasedDevice, 0)
+	if ret := s.GetMetadata(ctx, api.CONTAINER_METADATA_RELEASED_DEVICES, userCred); ret == "" {
+		return out, nil
+	}
+	obj := s.GetMetadataJson(ctx, api.CONTAINER_METADATA_RELEASED_DEVICES, userCred)
+	if obj == nil {
+		return nil, errors.Error("get metadata released devices")
+	}
+	if err := obj.Unmarshal(&out); err != nil {
+		return nil, errors.Wrap(err, "Unmarshal metadata released devices")
+	}
+	return out, nil
+}

pkg/compute/models/guest_actions.go

@@ -2065,6 +2065,66 @@ func (self *SGuest) StartGuestDetachdiskTask(
 	return driver.StartGuestDetachdiskTask(ctx, userCred, self, taskData, parentTaskId)
 }
 
+func (self *SGuest) GetReleasedIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential) ([]api.ServerReleasedIsolatedDevice, error) {
+	devs := make([]api.ServerReleasedIsolatedDevice, 0)
+	if ret := self.GetMetadata(ctx, api.VM_METADATA_RELEASED_DEVICES, userCred); ret == "" {
+		return devs, nil
+	}
+	obj := self.GetMetadataJson(ctx, api.VM_METADATA_RELEASED_DEVICES, userCred)
+	if obj == nil {
+		return nil, errors.Error("get medata json")
+	}
+	if err := obj.Unmarshal(&devs); err != nil {
+		return nil, errors.Wrapf(err, "unmarshal json string: %s", obj.String())
+	}
+	return devs, nil
+}
+
+func (self *SGuest) SetReleasedIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devs []SIsolatedDevice) error {
+	records := make([]api.ServerReleasedIsolatedDevice, 0)
+	for _, dev := range devs {
+		record := api.ServerReleasedIsolatedDevice{
+			DevType: dev.DevType,
+			Model:   dev.Model,
+		}
+		records = append(records, record)
+	}
+	if err := self.SetMetadata(ctx, api.VM_METADATA_RELEASED_DEVICES, records, userCred); err != nil {
+		return errors.Wrap(err, "set metadata")
+	}
+	return nil
+}
+
+func (self *SGuest) DetachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devs []SIsolatedDevice) error {
+	host, _ := self.GetHost()
+	lockman.LockObject(ctx, host)
+	defer lockman.ReleaseObject(ctx, host)
+	for i := 0; i < len(devs); i++ {
+		// check first
+		dev := devs[i]
+		if !utils.IsInStringArray(dev.DevType, api.VALID_ATTACH_TYPES) {
+			if devModel, err := IsolatedDeviceModelManager.GetByDevType(dev.DevType); err != nil {
+				msg := fmt.Sprintf("Can't separately detach dev type %s", dev.DevType)
+				logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false)
+				return httperrors.NewBadRequestError(msg)
+			} else {
+				if !devModel.HotPluggable.Bool() && self.GetStatus() == api.VM_RUNNING {
+					msg := fmt.Sprintf("dev type %s model %s unhotpluggable", dev.DevType, devModel.Model)
+					logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false)
+					return httperrors.NewBadRequestError(msg)
+				}
+			}
+		}
+	}
+	for i := 0; i < len(devs); i++ {
+		err := self.detachIsolateDevice(ctx, userCred, &devs[i])
+		if err != nil {
+			return errors.Wrapf(err, "detach device %s", devs[i].GetId())
+		}
+	}
+	return nil
+}
+
 // 卸载透传设备
 func (self *SGuest) PerformDetachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	if self.Hypervisor != api.HYPERVISOR_KVM && self.Hypervisor != api.HYPERVISOR_POD {
@@ -2077,6 +2137,7 @@ func (self *SGuest) PerformDetachIsolatedDevice(ctx context.Context, userCred mc
 		return nil, httperrors.NewInvalidStatusError(msg)
 	}
 	var detachAllDevice = jsonutils.QueryBoolean(data, "detach_all", false)
+	devs := make([]SIsolatedDevice, 0)
 	if !detachAllDevice {
 		device, err := data.GetString("device")
 		if err != nil {
@@ -2084,40 +2145,21 @@ func (self *SGuest) PerformDetachIsolatedDevice(ctx context.Context, userCred mc
 			logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false)
 			return nil, httperrors.NewBadRequestError(msg)
 		}
-		err = self.startDetachIsolateDeviceWithoutNic(ctx, userCred, device)
+		iDev, err := IsolatedDeviceManager.FetchByIdOrName(ctx, userCred, device)
 		if err != nil {
-			return nil, err
+			msgFmt := "Isolated device %s not found"
+			msg := fmt.Sprintf(msgFmt, device)
+			logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false)
+			return nil, httperrors.NewBadRequestError(msgFmt, device)
 		}
+		devs = append(devs, *iDev.(*SIsolatedDevice))
 	} else {
-		devs, _ := self.GetIsolatedDevices()
-		host, _ := self.GetHost()
-		lockman.LockObject(ctx, host)
-		defer lockman.ReleaseObject(ctx, host)
-		for i := 0; i < len(devs); i++ {
-			// check first
-			dev := devs[i]
-			if !utils.IsInStringArray(dev.DevType, api.VALID_ATTACH_TYPES) {
-				if devModel, err := IsolatedDeviceModelManager.GetByDevType(dev.DevType); err != nil {
-					msg := fmt.Sprintf("Can't separately detach dev type %s", dev.DevType)
-					logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false)
-					return nil, httperrors.NewBadRequestError(msg)
-				} else {
-					if !devModel.HotPluggable.Bool() && self.GetStatus() == api.VM_RUNNING {
-						msg := fmt.Sprintf("dev type %s model %s unhotpluggable", dev.DevType, devModel.Model)
-						logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false)
-						return nil, httperrors.NewBadRequestError(msg)
-					}
-				}
-			}
-		}
-		for i := 0; i < len(devs); i++ {
-			err := self.detachIsolateDevice(ctx, userCred, &devs[i])
-			if err != nil {
-				return nil, err
-			}
-		}
+		devs, _ = self.GetIsolatedDevices()
 	}
-	return nil, self.startIsolatedDevicesSyncTask(ctx, userCred, jsonutils.QueryBoolean(data, "auto_start", false), "")
+	if err := self.DetachIsolatedDevices(ctx, userCred, devs); err != nil {
+		return nil, err
+	}
+	return nil, self.StartIsolatedDevicesSyncTask(ctx, userCred, jsonutils.QueryBoolean(data, "auto_start", false), "")
 }
 
 func (self *SGuest) startDetachIsolateDeviceWithoutNic(ctx context.Context, userCred mcclient.TokenCredential, device string) error {
@@ -2129,25 +2171,7 @@ func (self *SGuest) startDetachIsolateDeviceWithoutNic(ctx context.Context, user
 		return httperrors.NewBadRequestError(msgFmt, device)
 	}
 	dev := iDev.(*SIsolatedDevice)
-	if !utils.IsInStringArray(dev.DevType, api.VALID_ATTACH_TYPES) {
-		if devModel, err := IsolatedDeviceModelManager.GetByDevType(dev.DevType); err != nil {
-			msg := fmt.Sprintf("Can't separately detach dev type %s", dev.DevType)
-			logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false)
-			return httperrors.NewBadRequestError(msg)
-		} else {
-			if !devModel.HotPluggable.Bool() && self.GetStatus() == api.VM_RUNNING {
-				msg := fmt.Sprintf("dev type %s model %s unhotpluggable", dev.DevType, devModel.Model)
-				logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false)
-				return httperrors.NewBadRequestError(msg)
-			}
-		}
-	}
-
-	host, _ := self.GetHost()
-	lockman.LockObject(ctx, host)
-	defer lockman.ReleaseObject(ctx, host)
-	err = self.detachIsolateDevice(ctx, userCred, dev)
-	return err
+	return self.DetachIsolatedDevices(ctx, userCred, []SIsolatedDevice{*dev})
 }
 
 func (self *SGuest) detachIsolateDevice(ctx context.Context, userCred mcclient.TokenCredential, dev *SIsolatedDevice) error {
@@ -2156,6 +2180,10 @@ func (self *SGuest) detachIsolateDevice(ctx context.Context, userCred mcclient.T
 		logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false)
 		return httperrors.NewBadRequestError(msg)
 	}
+	drv, _ := self.GetDriver()
+	if err := drv.BeforeDetachIsolatedDevice(ctx, userCred, self, dev); err != nil {
+		return errors.Wrapf(err, "BeforeDetachIsolatedDevice %s of guest %s", jsonutils.Marshal(dev), self.GetId())
+	}
 	_, err := db.Update(dev, func() error {
 		dev.GuestId = ""
 		dev.NetworkIndex = -1
@@ -2210,61 +2238,67 @@ func (self *SGuest) StartAttachIsolatedDevices(ctx context.Context, userCred mcc
 		return err
 	}
 	// perform post attach task
-	return self.startIsolatedDevicesSyncTask(ctx, userCred, autoStart, "")
+	return self.StartIsolatedDevicesSyncTask(ctx, userCred, autoStart, "")
 }
 
-func (self *SGuest) startAttachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devModel string, count int) error {
+func (self *SGuest) AttachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devModelCount map[string]int) error {
 	host, _ := self.GetHost()
 	lockman.LockObject(ctx, host)
 	defer lockman.ReleaseObject(ctx, host)
-	devs, err := IsolatedDeviceManager.GetUnusedDevsOnHost(host.Id, devModel, count)
-	if err != nil {
-		return httperrors.NewInternalServerError("fetch gpu failed %s", err)
-	}
-	if len(devs) == 0 || len(devs) != count {
-		return httperrors.NewBadRequestError("guest %s host %s isolated device not enough", self.GetName(), host.GetName())
-	}
-	dev := devs[0]
-	if !utils.IsInStringArray(dev.DevType, api.VALID_ATTACH_TYPES) {
-		if devModel, err := IsolatedDeviceModelManager.GetByDevType(dev.DevType); err != nil {
-			return httperrors.NewBadRequestError("Can't separately attach dev type %s", dev.DevType)
-		} else {
-			if !devModel.HotPluggable.Bool() && self.GetStatus() == api.VM_RUNNING {
-				return httperrors.NewBadRequestError("dev type %s model %s unhotpluggable", dev.DevType, devModel.Model)
-			}
-		}
-	}
 
-	if dev.DevType == api.LEGACY_VGPU_TYPE {
-		devs, err := self.GetIsolatedDevices()
+	unusedDevs := []SIsolatedDevice{}
+	for devModel, count := range devModelCount {
+		devs, err := IsolatedDeviceManager.GetUnusedDevsOnHost(host.Id, devModel, count)
 		if err != nil {
-			return errors.Wrap(err, "get isolated devices")
+			return httperrors.NewInternalServerError("fetch gpu failed %s", err)
 		}
-		for i := range devs {
-			if devs[i].DevType == api.LEGACY_VGPU_TYPE {
-				return httperrors.NewBadRequestError("Nvidia vgpu count exceed > 1")
-			} else if utils.IsInStringArray(devs[i].DevType, api.VALID_GPU_TYPES) {
-				return httperrors.NewBadRequestError("Nvidia vgpu can't passthrough with other gpus")
+		if len(devs) == 0 || len(devs) != count {
+			return httperrors.NewBadRequestError("require %d %s isolated device of host %s is not enough", count, devModel, host.GetName())
+		}
+		dev := devs[0]
+		if !utils.IsInStringArray(dev.DevType, api.VALID_ATTACH_TYPES) {
+			if devModel, err := IsolatedDeviceModelManager.GetByDevType(dev.DevType); err != nil {
+				return httperrors.NewBadRequestError("Can't separately attach dev type %s", dev.DevType)
+			} else {
+				if !devModel.HotPluggable.Bool() && self.GetStatus() == api.VM_RUNNING {
+					return httperrors.NewBadRequestError("dev type %s model %s unhotpluggable", dev.DevType, devModel.Model)
+				}
 			}
 		}
+		if dev.DevType == api.LEGACY_VGPU_TYPE {
+			devs, err := self.GetIsolatedDevices()
+			if err != nil {
+				return errors.Wrap(err, "get isolated devices")
+			}
+			for i := range devs {
+				if devs[i].DevType == api.LEGACY_VGPU_TYPE {
+					return httperrors.NewBadRequestError("Nvidia vgpu count exceed > 1")
+				} else if utils.IsInStringArray(devs[i].DevType, api.VALID_GPU_TYPES) {
+					return httperrors.NewBadRequestError("Nvidia vgpu can't passthrough with other gpus")
+				}
+			}
+		}
+		unusedDevs = append(unusedDevs, devs...)
 	}
-
 	defer func() { go host.ClearSchedDescCache() }()
-	for i := 0; i < len(devs); i++ {
-		err = self.attachIsolatedDevice(ctx, userCred, &devs[i], nil, nil)
-		if err != nil {
-			return err
+	for i := 0; i < len(unusedDevs); i++ {
+		if err := self.attachIsolatedDevice(ctx, userCred, &unusedDevs[i], nil, nil); err != nil {
+			return errors.Wrapf(err, "attach device %s", unusedDevs[i].GetId())
 		}
 	}
 	return nil
 }
 
+func (self *SGuest) startAttachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devModel string, count int) error {
+	return self.AttachIsolatedDevices(ctx, userCred, map[string]int{devModel: count})
+}
+
 func (self *SGuest) StartAttachIsolatedDeviceGpuOrUsb(ctx context.Context, userCred mcclient.TokenCredential, device string, autoStart bool) error {
 	if err := self.startAttachIsolatedDevGeneral(ctx, userCred, device); err != nil {
 		return err
 	}
 	// perform post attach task
-	return self.startIsolatedDevicesSyncTask(ctx, userCred, autoStart, "")
+	return self.StartIsolatedDevicesSyncTask(ctx, userCred, autoStart, "")
 }
 
 func (self *SGuest) startAttachIsolatedDevGeneral(ctx context.Context, userCred mcclient.TokenCredential, device string) error {
@@ -2325,7 +2359,11 @@ func (self *SGuest) attachIsolatedDevice(ctx context.Context, userCred mcclient.
 		self.HostId {
 		return fmt.Errorf("Isolated device and guest are not located in the same host")
 	}
-	_, err := db.Update(dev, func() error {
+	drv, _ := self.GetDriver()
+	if err := drv.BeforeAttachIsolatedDevice(ctx, userCred, self, dev); err != nil {
+		return errors.Wrapf(err, "BeforeAttachIsolatedDevice %s of guest %s", jsonutils.Marshal(dev), self.GetId())
+	}
+	if _, err := db.Update(dev, func() error {
 		dev.GuestId = self.Id
 		if networkIndex != nil {
 			dev.NetworkIndex = *networkIndex
@@ -2338,9 +2376,8 @@ func (self *SGuest) attachIsolatedDevice(ctx context.Context, userCred mcclient.
 			dev.DiskIndex = -1
 		}
 		return nil
-	})
-	if err != nil {
-		return err
+	}); err != nil {
+		return errors.Wrap(err, "db.Update")
 	}
 	db.OpsLog.LogEvent(self, db.ACT_GUEST_ATTACH_ISOLATED_DEVICE, dev.GetShortDesc(ctx), userCred)
 	return nil
@@ -2394,17 +2431,17 @@ func (self *SGuest) PerformSetIsolatedDevice(ctx context.Context, userCred mccli
 			return nil, err
 		}
 	}
-	return nil, self.startIsolatedDevicesSyncTask(ctx, userCred, jsonutils.QueryBoolean(data, "auto_start", false), "")
+	return nil, self.StartIsolatedDevicesSyncTask(ctx, userCred, jsonutils.QueryBoolean(data, "auto_start", false), "")
 }
 
-func (self *SGuest) startIsolatedDevicesSyncTask(ctx context.Context, userCred mcclient.TokenCredential, autoStart bool, parenetId string) error {
+func (self *SGuest) StartIsolatedDevicesSyncTask(ctx context.Context, userCred mcclient.TokenCredential, autoStart bool, parentId string) error {
 	if self.GetStatus() == api.VM_RUNNING {
 		autoStart = false
 	}
 	data := jsonutils.Marshal(map[string]interface{}{
 		"auto_start": autoStart,
 	}).(*jsonutils.JSONDict)
-	if task, err := taskman.TaskManager.NewTask(ctx, "GuestIsolatedDeviceSyncTask", self, userCred, data, parenetId, "", nil); err != nil {
+	if task, err := taskman.TaskManager.NewTask(ctx, "GuestIsolatedDeviceSyncTask", self, userCred, data, parentId, "", nil); err != nil {
 		return err
 	} else {
 		return task.ScheduleRun(nil)

pkg/compute/models/guestdrivers.go

@@ -252,6 +252,9 @@ type IGuestDriver interface {
 	ValidateSetOSInfo(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, input *api.ServerSetOSInfoInput) error
 	ValidateSyncOSInfo(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest) error
 	RequestStartRescue(ctx context.Context, task taskman.ITask, body jsonutils.JSONObject, host *SHost, guest *SGuest) error
+
+	BeforeDetachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *SGuest, dev *SIsolatedDevice) error
+	BeforeAttachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *SGuest, dev *SIsolatedDevice) error
 }
 
 var guestDrivers map[string]IGuestDriver

pkg/compute/models/isolated_devices.go

@@ -27,6 +27,7 @@ import (
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/gotypes"
 	"yunion.io/x/pkg/util/rbacscope"
+	"yunion.io/x/pkg/util/sets"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
@@ -468,7 +469,7 @@ func (self *SIsolatedDevice) getVendor() string {
 }
 
 func (self *SIsolatedDevice) IsGPU() bool {
-	return strings.HasPrefix(self.DevType, "GPU")
+	return strings.HasPrefix(self.DevType, "GPU") || sets.NewString(api.CONTAINER_GPU_TYPES...).Has(self.DevType)
 }
 
 func (manager *SIsolatedDeviceManager) parseDeviceInfo(userCred mcclient.TokenCredential, devConfig *api.IsolatedDeviceConfig) (*api.IsolatedDeviceConfig, error) {

pkg/compute/tasks/guest_start_task.go

@@ -19,6 +19,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
@@ -41,10 +42,53 @@ func init() {
 
 func (self *GuestStartTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
 	guest := obj.(*models.SGuest)
+	self.AttachReleasedDevices(ctx, guest)
+}
+
+func (self *GuestStartTask) attachReleasedDevices(ctx context.Context, guest *models.SGuest) error {
+	devs, err := guest.GetReleasedIsolatedDevices(ctx, self.GetUserCred())
+	if err != nil {
+		return errors.Wrap(err, "GetReleasedIsolatedDevices")
+	}
+	if len(devs) == 0 {
+		return self.ScheduleRun(nil)
+	}
+	attachReq := make(map[string]int)
+	for _, dev := range devs {
+		count, ok := attachReq[dev.Model]
+		if !ok {
+			attachReq[dev.Model] = 1
+		} else {
+			attachReq[dev.Model] = count + 1
+		}
+	}
+	if err := guest.AttachIsolatedDevices(ctx, self.GetUserCred(), attachReq); err != nil {
+		return errors.Wrap(err, "attach isolated devices")
+	}
+	return guest.StartIsolatedDevicesSyncTask(ctx, self.GetUserCred(), false, self.GetTaskId())
+}
+
+func (self *GuestStartTask) AttachReleasedDevices(ctx context.Context, guest *models.SGuest) {
+	self.SetStage("OnReleasedDevicesAttached", nil)
+	if guest.ShutdownBehavior != api.SHUTDOWN_STOP_RELEASE_GPU {
+		self.ScheduleRun(nil)
+		return
+	}
+	if err := self.attachReleasedDevices(ctx, guest); err != nil {
+		self.OnStartCompleteFailed(ctx, guest, jsonutils.NewString(errors.Wrap(err, "attach released devices").Error()))
+		return
+	}
+}
+
+func (self *GuestStartTask) OnReleasedDevicesAttached(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
 	db.OpsLog.LogEvent(guest, db.ACT_STARTING, nil, self.UserCred)
 	self.RequestStart(ctx, guest)
 }
 
+func (self *GuestStartTask) OnReleasedDevicesFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
+	self.OnStartCompleteFailed(ctx, guest, data)
+}
+
 func (self *GuestStartTask) RequestStart(ctx context.Context, guest *models.SGuest) {
 	self.SetStage("OnStartComplete", nil)
 	host, _ := guest.GetHost()

pkg/compute/tasks/guest_stop_task.go

@@ -66,15 +66,13 @@ func (self *GuestStopTask) stopGuest(ctx context.Context, guest *models.SGuest)
 
 func (self *GuestStopTask) OnGuestStopTaskComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
 	db.OpsLog.LogEvent(guest, db.ACT_STOP, guest.GetShortDesc(ctx), self.UserCred)
-	models.HostManager.ClearSchedDescCache(guest.HostId)
 	if guest.Status != api.VM_READY && !self.IsSubtask() { // for kvm
 		guest.SetStatus(ctx, self.GetUserCred(), api.VM_READY, "")
 	}
+	models.HostManager.ClearSchedDescCache(guest.HostId)
 	logclient.AddActionLogWithStartable(self, guest, logclient.ACT_VM_STOP, "success", self.UserCred, true)
-	self.SetStageComplete(ctx, nil)
-	if guest.DisableDelete.IsFalse() && guest.ShutdownBehavior == api.SHUTDOWN_TERMINATE {
-		guest.StartAutoDeleteGuestTask(ctx, self.UserCred, "")
-		return
+	if err := self.releaseDevices(ctx, guest); err != nil {
+		self.OnGuestStopTaskCompleteFailed(ctx, guest, jsonutils.NewString(err.Error()))
 	}
 }
 
@@ -87,6 +85,47 @@ func (self *GuestStopTask) OnGuestStopTaskCompleteFailed(ctx context.Context, gu
 	logclient.AddActionLogWithStartable(self, guest, logclient.ACT_VM_STOP, reason.String(), self.UserCred, false)
 }
 
+func (self *GuestStopTask) releaseDevices(ctx context.Context, guest *models.SGuest) error {
+	self.SetStage("OnDevicesReleased", nil)
+	if guest.ShutdownBehavior != api.SHUTDOWN_STOP_RELEASE_GPU {
+		return self.ScheduleRun(nil)
+	}
+	devs, err := guest.GetIsolatedDevices()
+	if err != nil {
+		return errors.Wrapf(err, "GetIsolatedDevices of guest %s", guest.GetId())
+	}
+	gpus := make([]models.SIsolatedDevice, 0)
+	for _, dev := range devs {
+		if dev.IsGPU() {
+			tmpDev := dev
+			gpus = append(gpus, tmpDev)
+		}
+	}
+	if len(gpus) == 0 {
+		return self.ScheduleRun(nil)
+	}
+	if err := guest.SetReleasedIsolatedDevices(ctx, self.GetUserCred(), gpus); err != nil {
+		return errors.Wrapf(err, "SetReleasedIsolatedDevices of guest %s", guest.GetId())
+	}
+
+	if err := guest.DetachIsolatedDevices(ctx, self.GetUserCred(), gpus); err != nil {
+		return errors.Wrapf(err, "DetachIsolatedDevices of guest %s", guest.GetId())
+	}
+	return guest.StartIsolatedDevicesSyncTask(ctx, self.GetUserCred(), false, self.GetTaskId())
+}
+
+func (self *GuestStopTask) OnDevicesReleased(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
+	self.SetStageComplete(ctx, nil)
+	if guest.DisableDelete.IsFalse() && guest.ShutdownBehavior == api.SHUTDOWN_TERMINATE {
+		guest.StartAutoDeleteGuestTask(ctx, self.UserCred, "")
+		return
+	}
+}
+
+func (self *GuestStopTask) OnDevicesReleaseFailed(ctx context.Context, guest *models.SGuest, reason jsonutils.JSONObject) {
+	self.OnGuestStopTaskCompleteFailed(ctx, guest, reason)
+}
+
 type GuestStopAndFreezeTask struct {
 	SGuestBaseTask
 }

pkg/mcclient/options/compute/server_pod.go

@@ -36,10 +36,11 @@ type PodCreateOptions struct {
 	VcpuCount   int    `help:"#CPU cores of VM server, default 1" default:"1" metavar:"<SERVER_CPU_COUNT>" json:"vcpu_count" token:"ncpu"`
 	AllowDelete *bool  `help:"Unlock server to allow deleting" json:"-"`
 	//PortMapping []string `help:"Port mapping of the pod and the format is: host_port=8080,port=80,protocol=<tcp|udp>,host_port_range=<int>-<int>" short-token:"p"`
-	Arch      string `help:"image arch" choices:"aarch64|x86_64"`
-	AutoStart bool   `help:"Auto start server after it is created"`
-	PodUid    int64  `help:"UID of pod" default:"0"`
-	PodGid    int64  `help:"GID of pod" default:"0"`
+	Arch             string `help:"image arch" choices:"aarch64|x86_64"`
+	AutoStart        bool   `help:"Auto start server after it is created"`
+	ShutdownBehavior string `help:"Behavior after VM server shutdown" metavar:"<SHUTDOWN_BEHAVIOR>" choices:"stop|terminate|stop_release_gpu"`
+	PodUid           int64  `help:"UID of pod" default:"0"`
+	PodGid           int64  `help:"GID of pod" default:"0"`
 
 	ContainerCreateCommonOptions
 }
@@ -200,9 +201,10 @@ func (o *PodCreateOptions) Params() (*computeapi.ServerCreateInput, error) {
 	}
 
 	params := &computeapi.ServerCreateInput{
-		ServerConfigs: config,
-		VcpuCount:     o.VcpuCount,
-		AutoStart:     o.AutoStart,
+		ServerConfigs:    config,
+		VcpuCount:        o.VcpuCount,
+		AutoStart:        o.AutoStart,
+		ShutdownBehavior: o.ShutdownBehavior,
 		Pod: &computeapi.PodCreateInput{
 			//PortMappings: portMappings,
 			Containers: []*computeapi.PodContainerCreateInput{

pkg/mcclient/options/compute/servers.go

@@ -464,7 +464,7 @@ type ServerCreateOptionalOptions struct {
 	EnableCloudInit  bool     `help:"Enable cloud-init service"`
 	NoAccountInit    *bool    `help:"Not reset account password"`
 	AllowDelete      *bool    `help:"Unlock server to allow deleting" json:"-"`
-	ShutdownBehavior string   `help:"Behavior after VM server shutdown" metavar:"<SHUTDOWN_BEHAVIOR>" choices:"stop|terminate"`
+	ShutdownBehavior string   `help:"Behavior after VM server shutdown" metavar:"<SHUTDOWN_BEHAVIOR>" choices:"stop|terminate|stop_release_gpu"`
 	AutoStart        bool     `help:"Auto start server after it is created"`
 	Deploy           []string `help:"Specify deploy files in virtual server file system" json:"-"`
 	DeployTelegraf   bool     `help:"Deploy telegraf agent if guest os is supported"`
@@ -695,7 +695,7 @@ type ServerUpdateOptions struct {
 	Desc             string `help:"Description" json:"description"`
 	Boot             string `help:"Boot device" choices:"disk|cdrom"`
 	Delete           string `help:"Lock server to prevent from deleting" choices:"enable|disable" json:"-"`
-	ShutdownBehavior string `help:"Behavior after VM server shutdown" choices:"stop|terminate"`
+	ShutdownBehavior string `help:"Behavior after VM server shutdown" choices:"stop|terminate|stop_release_gpu"`
 	Machine          string `help:"Machine type" choices:"q35|pc"`
 
 	IsDaemon *bool `help:"Daemon server" negative:"no-daemon"`