diff --git a/cmd/climc/shell/compute/containers.go b/cmd/climc/shell/compute/containers.go index 449ee4c76f..aed3a848ef 100644 --- a/cmd/climc/shell/compute/containers.go +++ b/cmd/climc/shell/compute/containers.go @@ -37,6 +37,7 @@ func init() { cmd.Create(new(options.ContainerCreateOptions)) cmd.List(new(options.ContainerListOptions)) cmd.Show(new(options.ContainerShowOptions)) + cmd.GetMetadata(new(options.ServerIdOptions)) cmd.BatchDelete(new(options.ContainerDeleteOptions)) cmd.BatchPerform("stop", new(options.ContainerStopOptions)) cmd.BatchPerform("start", new(options.ContainerStartOptions)) diff --git a/pkg/apis/compute/api.go b/pkg/apis/compute/api.go index aa50254a21..1568d31716 100644 --- a/pkg/apis/compute/api.go +++ b/pkg/apis/compute/api.go @@ -538,7 +538,7 @@ type ServerCreateInput struct { // 关机后执行的操作 // terminate: 关机后自动删除 - // emum: stop, terminate + // enum: stop, terminate, stop_release_gpu // default: stop ShutdownBehavior string `json:"shutdown_behavior"` diff --git a/pkg/apis/compute/container.go b/pkg/apis/compute/container.go index 9badc673bc..5685011885 100644 --- a/pkg/apis/compute/container.go +++ b/pkg/apis/compute/container.go @@ -39,6 +39,14 @@ const ( CONTAINER_DEV_ASCEND_NPU = "ASCEND_NPU" ) +var ( + CONTAINER_GPU_TYPES = []string{ + CONTAINER_DEV_CPH_AMD_GPU, + CONTAINER_DEV_NVIDIA_GPU, + CONTAINER_DEV_NVIDIA_MPS, + } +) + const ( CONTAINER_STORAGE_LOCAL_RAW = "local_raw" ) @@ -66,7 +74,8 @@ const ( ) const ( - CONTAINER_METADATA_CRI_ID = "cri_id" + CONTAINER_METADATA_CRI_ID = "cri_id" + CONTAINER_METADATA_RELEASED_DEVICES = "released_devices" ) type ContainerSpec struct { diff --git a/pkg/apis/compute/guest_const.go b/pkg/apis/compute/guest_const.go index e36642637b..a2cfc1a562 100644 --- a/pkg/apis/compute/guest_const.go +++ b/pkg/apis/compute/guest_const.go @@ -14,7 +14,9 @@ package compute -import "yunion.io/x/cloudmux/pkg/apis/compute" +import ( + "yunion.io/x/cloudmux/pkg/apis/compute" +) const ( VM_INIT = compute.VM_INIT @@ -178,8 +180,9 @@ const ( VM_QGA_SET_NETWORK = "qga_set_network" VM_QGA_SET_NETWORK_FAILED = "qga_set_network_failed" - SHUTDOWN_STOP = "stop" - SHUTDOWN_TERMINATE = "terminate" + SHUTDOWN_STOP = "stop" + SHUTDOWN_TERMINATE = "terminate" + SHUTDOWN_STOP_RELEASE_GPU = "stop_release_gpu" HYPERVISOR_KVM = "kvm" HYPERVISOR_POD = "pod" @@ -323,6 +326,8 @@ const ( VM_METADATA_HOT_REMOVE_NIC = "hot_remove_nic" VM_METADATA_START_VMEM_MB = "start_vmem_mb" VM_METADATA_START_VCPU_COUNT = "start_vcpu_count" + + VM_METADATA_RELEASED_DEVICES = "released_devices" ) // windows allow a maximal length of 15 diff --git a/pkg/apis/compute/guests.go b/pkg/apis/compute/guests.go index 7626a2faac..26bc213b58 100644 --- a/pkg/apis/compute/guests.go +++ b/pkg/apis/compute/guests.go @@ -1335,3 +1335,8 @@ func (conf ServerChangeConfigSettings) AddedDisk() int { } return size } + +type ServerReleasedIsolatedDevice struct { + DevType string `json:"dev_type"` + Model string `json:"model"` +} diff --git a/pkg/compute/guestdrivers/base.go b/pkg/compute/guestdrivers/base.go index 8fb785475b..2480ffb922 100644 --- a/pkg/compute/guestdrivers/base.go +++ b/pkg/compute/guestdrivers/base.go @@ -696,3 +696,11 @@ func (base *SBaseGuestDriver) ValidateGuestChangeConfigInput(ctx context.Context func (base *SBaseGuestDriver) ValidateGuestHotChangeConfigInput(ctx context.Context, guest *models.SGuest, confs *api.ServerChangeConfigSettings) (*api.ServerChangeConfigSettings, error) { return confs, nil } + +func (base *SBaseGuestDriver) BeforeDetachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *models.SGuest, dev *models.SIsolatedDevice) error { + return nil +} + +func (base *SBaseGuestDriver) BeforeAttachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *models.SGuest, dev *models.SIsolatedDevice) error { + return nil +} diff --git a/pkg/compute/guestdrivers/pod.go b/pkg/compute/guestdrivers/pod.go index 87a756e29b..a5fba518da 100644 --- a/pkg/compute/guestdrivers/pod.go +++ b/pkg/compute/guestdrivers/pod.go @@ -35,6 +35,7 @@ import ( hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/apis/image" "yunion.io/x/onecloud/pkg/appsrv" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" @@ -555,3 +556,106 @@ func (p *SPodDriver) RequestDeleteSnapshot(ctx context.Context, guest *models.SG }*/ return p.SKVMGuestDriver.RequestDeleteSnapshot(ctx, guest, task, params) } + +func (p *SPodDriver) BeforeDetachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, dev *models.SIsolatedDevice) error { + ctrs, err := models.GetContainerManager().GetContainersByPod(guest.GetId()) + if err != nil { + return errors.Wrapf(err, "get containers by pod %s", guest.GetId()) + } + for _, ctr := range ctrs { + ctrPtr := &ctr + spec := ctrPtr.Spec + devs := spec.Devices + newDevs := make([]*api.ContainerDevice, 0) + releasedDevs := make(map[string]models.ContainerReleasedDevice) + for _, curDev := range devs { + if curDev.IsolatedDevice == nil { + continue + } + if curDev.IsolatedDevice.Id != dev.GetId() { + tmpDev := curDev + newDevs = append(newDevs, tmpDev) + } else { + releasedDevs[curDev.IsolatedDevice.Id] = *models.NewContainerReleasedDevice(curDev, dev.DevType, dev.Model) + } + } + if err := ctrPtr.SaveReleasedDevices(ctx, userCred, releasedDevs); err != nil { + return errors.Wrapf(err, "save release devices for container %s", ctr.GetId()) + } + if _, err := db.Update(ctrPtr, func() error { + ctrPtr.Spec.Devices = newDevs + return nil + }); err != nil { + return errors.Wrapf(err, "update container %s devs", ctrPtr.GetId()) + } + } + return nil +} + +func (p *SPodDriver) BeforeAttachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, dev *models.SIsolatedDevice) error { + ctrs, err := models.GetContainerManager().GetContainersByPod(guest.GetId()) + if err != nil { + return errors.Wrapf(err, "get containers by pod %s", guest.GetId()) + } + for _, ctr := range ctrs { + ctrPtr := &ctr + if err := p.attachIsolatedDeviceToContainer(ctx, userCred, ctrPtr, dev); err != nil { + return errors.Wrapf(err, "attach isolated device to container %s", ctr.GetId()) + } + } + return nil +} + +func (p *SPodDriver) attachIsolatedDeviceToContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrPtr *models.SContainer, dev *models.SIsolatedDevice) error { + rlsDevs, err := ctrPtr.GetReleasedDevices(ctx, userCred) + if err != nil { + return errors.Wrapf(err, "get release devices for container %s", ctrPtr.GetId()) + } + if len(rlsDevs) == 0 { + return nil + } + spec := new(api.ContainerSpec) + if err := jsonutils.Marshal(ctrPtr.Spec).Unmarshal(spec); err != nil { + return errors.Wrap(err, "deep copy spec") + } + for id, rlsDev := range rlsDevs { + if rlsDev.IsolatedDevice == nil { + continue + } + if rlsDev.DeviceModel == dev.Model && rlsDev.DeviceType == dev.DevType { + // attach it + if spec.Devices == nil { + spec.Devices = make([]*api.ContainerDevice, 0) + } + shouldUpdate := true + for _, curDev := range spec.Devices { + if curDev.IsolatedDevice == nil { + continue + } + if curDev.IsolatedDevice.Id == dev.GetId() { + shouldUpdate = false + } + } + if shouldUpdate { + spec.Devices = append(spec.Devices, &api.ContainerDevice{ + Type: apis.CONTAINER_DEVICE_TYPE_ISOLATED_DEVICE, + IsolatedDevice: &api.ContainerIsolatedDevice{ + Id: dev.GetId(), + }, + }) + if _, err := db.Update(ctrPtr, func() error { + ctrPtr.Spec = spec + return nil + }); err != nil { + return errors.Wrapf(err, "update container %s devs", ctrPtr.GetId()) + } + } + delete(rlsDevs, id) + if err := ctrPtr.SaveReleasedDevices(ctx, userCred, rlsDevs); err != nil { + return errors.Wrapf(err, "save release devices for container %s", ctrPtr.GetId()) + } + return nil + } + } + return nil +} diff --git a/pkg/compute/models/containers.go b/pkg/compute/models/containers.go index 71c848075c..d91455fbde 100644 --- a/pkg/compute/models/containers.go +++ b/pkg/compute/models/containers.go @@ -566,3 +566,36 @@ func (c *SContainer) PerformExecSync(ctx context.Context, userCred mcclient.Toke } return c.GetPodDriver().RequestExecSyncContainer(ctx, userCred, c, input) } + +type ContainerReleasedDevice struct { + *api.ContainerDevice + DeviceType string + DeviceModel string +} + +func NewContainerReleasedDevice(device *api.ContainerDevice, devType, devModel string) *ContainerReleasedDevice { + return &ContainerReleasedDevice{ + ContainerDevice: device, + DeviceType: devType, + DeviceModel: devModel, + } +} + +func (s *SContainer) SaveReleasedDevices(ctx context.Context, userCred mcclient.TokenCredential, devs map[string]ContainerReleasedDevice) error { + return s.SetMetadata(ctx, api.CONTAINER_METADATA_RELEASED_DEVICES, devs, userCred) +} + +func (s *SContainer) GetReleasedDevices(ctx context.Context, userCred mcclient.TokenCredential) (map[string]ContainerReleasedDevice, error) { + out := make(map[string]ContainerReleasedDevice, 0) + if ret := s.GetMetadata(ctx, api.CONTAINER_METADATA_RELEASED_DEVICES, userCred); ret == "" { + return out, nil + } + obj := s.GetMetadataJson(ctx, api.CONTAINER_METADATA_RELEASED_DEVICES, userCred) + if obj == nil { + return nil, errors.Error("get metadata released devices") + } + if err := obj.Unmarshal(&out); err != nil { + return nil, errors.Wrap(err, "Unmarshal metadata released devices") + } + return out, nil +} diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index a7dd3f087a..b51a36d112 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -2065,6 +2065,66 @@ func (self *SGuest) StartGuestDetachdiskTask( return driver.StartGuestDetachdiskTask(ctx, userCred, self, taskData, parentTaskId) } +func (self *SGuest) GetReleasedIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential) ([]api.ServerReleasedIsolatedDevice, error) { + devs := make([]api.ServerReleasedIsolatedDevice, 0) + if ret := self.GetMetadata(ctx, api.VM_METADATA_RELEASED_DEVICES, userCred); ret == "" { + return devs, nil + } + obj := self.GetMetadataJson(ctx, api.VM_METADATA_RELEASED_DEVICES, userCred) + if obj == nil { + return nil, errors.Error("get medata json") + } + if err := obj.Unmarshal(&devs); err != nil { + return nil, errors.Wrapf(err, "unmarshal json string: %s", obj.String()) + } + return devs, nil +} + +func (self *SGuest) SetReleasedIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devs []SIsolatedDevice) error { + records := make([]api.ServerReleasedIsolatedDevice, 0) + for _, dev := range devs { + record := api.ServerReleasedIsolatedDevice{ + DevType: dev.DevType, + Model: dev.Model, + } + records = append(records, record) + } + if err := self.SetMetadata(ctx, api.VM_METADATA_RELEASED_DEVICES, records, userCred); err != nil { + return errors.Wrap(err, "set metadata") + } + return nil +} + +func (self *SGuest) DetachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devs []SIsolatedDevice) error { + host, _ := self.GetHost() + lockman.LockObject(ctx, host) + defer lockman.ReleaseObject(ctx, host) + for i := 0; i < len(devs); i++ { + // check first + dev := devs[i] + if !utils.IsInStringArray(dev.DevType, api.VALID_ATTACH_TYPES) { + if devModel, err := IsolatedDeviceModelManager.GetByDevType(dev.DevType); err != nil { + msg := fmt.Sprintf("Can't separately detach dev type %s", dev.DevType) + logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) + return httperrors.NewBadRequestError(msg) + } else { + if !devModel.HotPluggable.Bool() && self.GetStatus() == api.VM_RUNNING { + msg := fmt.Sprintf("dev type %s model %s unhotpluggable", dev.DevType, devModel.Model) + logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) + return httperrors.NewBadRequestError(msg) + } + } + } + } + for i := 0; i < len(devs); i++ { + err := self.detachIsolateDevice(ctx, userCred, &devs[i]) + if err != nil { + return errors.Wrapf(err, "detach device %s", devs[i].GetId()) + } + } + return nil +} + // 卸载透传设备 func (self *SGuest) PerformDetachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { if self.Hypervisor != api.HYPERVISOR_KVM && self.Hypervisor != api.HYPERVISOR_POD { @@ -2077,6 +2137,7 @@ func (self *SGuest) PerformDetachIsolatedDevice(ctx context.Context, userCred mc return nil, httperrors.NewInvalidStatusError(msg) } var detachAllDevice = jsonutils.QueryBoolean(data, "detach_all", false) + devs := make([]SIsolatedDevice, 0) if !detachAllDevice { device, err := data.GetString("device") if err != nil { @@ -2084,40 +2145,21 @@ func (self *SGuest) PerformDetachIsolatedDevice(ctx context.Context, userCred mc logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) return nil, httperrors.NewBadRequestError(msg) } - err = self.startDetachIsolateDeviceWithoutNic(ctx, userCred, device) + iDev, err := IsolatedDeviceManager.FetchByIdOrName(ctx, userCred, device) if err != nil { - return nil, err + msgFmt := "Isolated device %s not found" + msg := fmt.Sprintf(msgFmt, device) + logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) + return nil, httperrors.NewBadRequestError(msgFmt, device) } + devs = append(devs, *iDev.(*SIsolatedDevice)) } else { - devs, _ := self.GetIsolatedDevices() - host, _ := self.GetHost() - lockman.LockObject(ctx, host) - defer lockman.ReleaseObject(ctx, host) - for i := 0; i < len(devs); i++ { - // check first - dev := devs[i] - if !utils.IsInStringArray(dev.DevType, api.VALID_ATTACH_TYPES) { - if devModel, err := IsolatedDeviceModelManager.GetByDevType(dev.DevType); err != nil { - msg := fmt.Sprintf("Can't separately detach dev type %s", dev.DevType) - logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) - return nil, httperrors.NewBadRequestError(msg) - } else { - if !devModel.HotPluggable.Bool() && self.GetStatus() == api.VM_RUNNING { - msg := fmt.Sprintf("dev type %s model %s unhotpluggable", dev.DevType, devModel.Model) - logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) - return nil, httperrors.NewBadRequestError(msg) - } - } - } - } - for i := 0; i < len(devs); i++ { - err := self.detachIsolateDevice(ctx, userCred, &devs[i]) - if err != nil { - return nil, err - } - } + devs, _ = self.GetIsolatedDevices() } - return nil, self.startIsolatedDevicesSyncTask(ctx, userCred, jsonutils.QueryBoolean(data, "auto_start", false), "") + if err := self.DetachIsolatedDevices(ctx, userCred, devs); err != nil { + return nil, err + } + return nil, self.StartIsolatedDevicesSyncTask(ctx, userCred, jsonutils.QueryBoolean(data, "auto_start", false), "") } func (self *SGuest) startDetachIsolateDeviceWithoutNic(ctx context.Context, userCred mcclient.TokenCredential, device string) error { @@ -2129,25 +2171,7 @@ func (self *SGuest) startDetachIsolateDeviceWithoutNic(ctx context.Context, user return httperrors.NewBadRequestError(msgFmt, device) } dev := iDev.(*SIsolatedDevice) - if !utils.IsInStringArray(dev.DevType, api.VALID_ATTACH_TYPES) { - if devModel, err := IsolatedDeviceModelManager.GetByDevType(dev.DevType); err != nil { - msg := fmt.Sprintf("Can't separately detach dev type %s", dev.DevType) - logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) - return httperrors.NewBadRequestError(msg) - } else { - if !devModel.HotPluggable.Bool() && self.GetStatus() == api.VM_RUNNING { - msg := fmt.Sprintf("dev type %s model %s unhotpluggable", dev.DevType, devModel.Model) - logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) - return httperrors.NewBadRequestError(msg) - } - } - } - - host, _ := self.GetHost() - lockman.LockObject(ctx, host) - defer lockman.ReleaseObject(ctx, host) - err = self.detachIsolateDevice(ctx, userCred, dev) - return err + return self.DetachIsolatedDevices(ctx, userCred, []SIsolatedDevice{*dev}) } func (self *SGuest) detachIsolateDevice(ctx context.Context, userCred mcclient.TokenCredential, dev *SIsolatedDevice) error { @@ -2156,6 +2180,10 @@ func (self *SGuest) detachIsolateDevice(ctx context.Context, userCred mcclient.T logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) return httperrors.NewBadRequestError(msg) } + drv, _ := self.GetDriver() + if err := drv.BeforeDetachIsolatedDevice(ctx, userCred, self, dev); err != nil { + return errors.Wrapf(err, "BeforeDetachIsolatedDevice %s of guest %s", jsonutils.Marshal(dev), self.GetId()) + } _, err := db.Update(dev, func() error { dev.GuestId = "" dev.NetworkIndex = -1 @@ -2210,61 +2238,67 @@ func (self *SGuest) StartAttachIsolatedDevices(ctx context.Context, userCred mcc return err } // perform post attach task - return self.startIsolatedDevicesSyncTask(ctx, userCred, autoStart, "") + return self.StartIsolatedDevicesSyncTask(ctx, userCred, autoStart, "") } -func (self *SGuest) startAttachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devModel string, count int) error { +func (self *SGuest) AttachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devModelCount map[string]int) error { host, _ := self.GetHost() lockman.LockObject(ctx, host) defer lockman.ReleaseObject(ctx, host) - devs, err := IsolatedDeviceManager.GetUnusedDevsOnHost(host.Id, devModel, count) - if err != nil { - return httperrors.NewInternalServerError("fetch gpu failed %s", err) - } - if len(devs) == 0 || len(devs) != count { - return httperrors.NewBadRequestError("guest %s host %s isolated device not enough", self.GetName(), host.GetName()) - } - dev := devs[0] - if !utils.IsInStringArray(dev.DevType, api.VALID_ATTACH_TYPES) { - if devModel, err := IsolatedDeviceModelManager.GetByDevType(dev.DevType); err != nil { - return httperrors.NewBadRequestError("Can't separately attach dev type %s", dev.DevType) - } else { - if !devModel.HotPluggable.Bool() && self.GetStatus() == api.VM_RUNNING { - return httperrors.NewBadRequestError("dev type %s model %s unhotpluggable", dev.DevType, devModel.Model) - } - } - } - if dev.DevType == api.LEGACY_VGPU_TYPE { - devs, err := self.GetIsolatedDevices() + unusedDevs := []SIsolatedDevice{} + for devModel, count := range devModelCount { + devs, err := IsolatedDeviceManager.GetUnusedDevsOnHost(host.Id, devModel, count) if err != nil { - return errors.Wrap(err, "get isolated devices") + return httperrors.NewInternalServerError("fetch gpu failed %s", err) } - for i := range devs { - if devs[i].DevType == api.LEGACY_VGPU_TYPE { - return httperrors.NewBadRequestError("Nvidia vgpu count exceed > 1") - } else if utils.IsInStringArray(devs[i].DevType, api.VALID_GPU_TYPES) { - return httperrors.NewBadRequestError("Nvidia vgpu can't passthrough with other gpus") + if len(devs) == 0 || len(devs) != count { + return httperrors.NewBadRequestError("require %d %s isolated device of host %s is not enough", count, devModel, host.GetName()) + } + dev := devs[0] + if !utils.IsInStringArray(dev.DevType, api.VALID_ATTACH_TYPES) { + if devModel, err := IsolatedDeviceModelManager.GetByDevType(dev.DevType); err != nil { + return httperrors.NewBadRequestError("Can't separately attach dev type %s", dev.DevType) + } else { + if !devModel.HotPluggable.Bool() && self.GetStatus() == api.VM_RUNNING { + return httperrors.NewBadRequestError("dev type %s model %s unhotpluggable", dev.DevType, devModel.Model) + } } } + if dev.DevType == api.LEGACY_VGPU_TYPE { + devs, err := self.GetIsolatedDevices() + if err != nil { + return errors.Wrap(err, "get isolated devices") + } + for i := range devs { + if devs[i].DevType == api.LEGACY_VGPU_TYPE { + return httperrors.NewBadRequestError("Nvidia vgpu count exceed > 1") + } else if utils.IsInStringArray(devs[i].DevType, api.VALID_GPU_TYPES) { + return httperrors.NewBadRequestError("Nvidia vgpu can't passthrough with other gpus") + } + } + } + unusedDevs = append(unusedDevs, devs...) } - defer func() { go host.ClearSchedDescCache() }() - for i := 0; i < len(devs); i++ { - err = self.attachIsolatedDevice(ctx, userCred, &devs[i], nil, nil) - if err != nil { - return err + for i := 0; i < len(unusedDevs); i++ { + if err := self.attachIsolatedDevice(ctx, userCred, &unusedDevs[i], nil, nil); err != nil { + return errors.Wrapf(err, "attach device %s", unusedDevs[i].GetId()) } } return nil } +func (self *SGuest) startAttachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devModel string, count int) error { + return self.AttachIsolatedDevices(ctx, userCred, map[string]int{devModel: count}) +} + func (self *SGuest) StartAttachIsolatedDeviceGpuOrUsb(ctx context.Context, userCred mcclient.TokenCredential, device string, autoStart bool) error { if err := self.startAttachIsolatedDevGeneral(ctx, userCred, device); err != nil { return err } // perform post attach task - return self.startIsolatedDevicesSyncTask(ctx, userCred, autoStart, "") + return self.StartIsolatedDevicesSyncTask(ctx, userCred, autoStart, "") } func (self *SGuest) startAttachIsolatedDevGeneral(ctx context.Context, userCred mcclient.TokenCredential, device string) error { @@ -2325,7 +2359,11 @@ func (self *SGuest) attachIsolatedDevice(ctx context.Context, userCred mcclient. self.HostId { return fmt.Errorf("Isolated device and guest are not located in the same host") } - _, err := db.Update(dev, func() error { + drv, _ := self.GetDriver() + if err := drv.BeforeAttachIsolatedDevice(ctx, userCred, self, dev); err != nil { + return errors.Wrapf(err, "BeforeAttachIsolatedDevice %s of guest %s", jsonutils.Marshal(dev), self.GetId()) + } + if _, err := db.Update(dev, func() error { dev.GuestId = self.Id if networkIndex != nil { dev.NetworkIndex = *networkIndex @@ -2338,9 +2376,8 @@ func (self *SGuest) attachIsolatedDevice(ctx context.Context, userCred mcclient. dev.DiskIndex = -1 } return nil - }) - if err != nil { - return err + }); err != nil { + return errors.Wrap(err, "db.Update") } db.OpsLog.LogEvent(self, db.ACT_GUEST_ATTACH_ISOLATED_DEVICE, dev.GetShortDesc(ctx), userCred) return nil @@ -2394,17 +2431,17 @@ func (self *SGuest) PerformSetIsolatedDevice(ctx context.Context, userCred mccli return nil, err } } - return nil, self.startIsolatedDevicesSyncTask(ctx, userCred, jsonutils.QueryBoolean(data, "auto_start", false), "") + return nil, self.StartIsolatedDevicesSyncTask(ctx, userCred, jsonutils.QueryBoolean(data, "auto_start", false), "") } -func (self *SGuest) startIsolatedDevicesSyncTask(ctx context.Context, userCred mcclient.TokenCredential, autoStart bool, parenetId string) error { +func (self *SGuest) StartIsolatedDevicesSyncTask(ctx context.Context, userCred mcclient.TokenCredential, autoStart bool, parentId string) error { if self.GetStatus() == api.VM_RUNNING { autoStart = false } data := jsonutils.Marshal(map[string]interface{}{ "auto_start": autoStart, }).(*jsonutils.JSONDict) - if task, err := taskman.TaskManager.NewTask(ctx, "GuestIsolatedDeviceSyncTask", self, userCred, data, parenetId, "", nil); err != nil { + if task, err := taskman.TaskManager.NewTask(ctx, "GuestIsolatedDeviceSyncTask", self, userCred, data, parentId, "", nil); err != nil { return err } else { return task.ScheduleRun(nil) diff --git a/pkg/compute/models/guestdrivers.go b/pkg/compute/models/guestdrivers.go index 053b27048d..85777737a8 100644 --- a/pkg/compute/models/guestdrivers.go +++ b/pkg/compute/models/guestdrivers.go @@ -252,6 +252,9 @@ type IGuestDriver interface { ValidateSetOSInfo(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, input *api.ServerSetOSInfoInput) error ValidateSyncOSInfo(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest) error RequestStartRescue(ctx context.Context, task taskman.ITask, body jsonutils.JSONObject, host *SHost, guest *SGuest) error + + BeforeDetachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *SGuest, dev *SIsolatedDevice) error + BeforeAttachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *SGuest, dev *SIsolatedDevice) error } var guestDrivers map[string]IGuestDriver diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index 8fed8ce083..500113d0a5 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -27,6 +27,7 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/util/rbacscope" + "yunion.io/x/pkg/util/sets" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" @@ -468,7 +469,7 @@ func (self *SIsolatedDevice) getVendor() string { } func (self *SIsolatedDevice) IsGPU() bool { - return strings.HasPrefix(self.DevType, "GPU") + return strings.HasPrefix(self.DevType, "GPU") || sets.NewString(api.CONTAINER_GPU_TYPES...).Has(self.DevType) } func (manager *SIsolatedDeviceManager) parseDeviceInfo(userCred mcclient.TokenCredential, devConfig *api.IsolatedDeviceConfig) (*api.IsolatedDeviceConfig, error) { diff --git a/pkg/compute/tasks/guest_start_task.go b/pkg/compute/tasks/guest_start_task.go index c1d8364211..7e06a19271 100644 --- a/pkg/compute/tasks/guest_start_task.go +++ b/pkg/compute/tasks/guest_start_task.go @@ -19,6 +19,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" @@ -41,10 +42,53 @@ func init() { func (self *GuestStartTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { guest := obj.(*models.SGuest) + self.AttachReleasedDevices(ctx, guest) +} + +func (self *GuestStartTask) attachReleasedDevices(ctx context.Context, guest *models.SGuest) error { + devs, err := guest.GetReleasedIsolatedDevices(ctx, self.GetUserCred()) + if err != nil { + return errors.Wrap(err, "GetReleasedIsolatedDevices") + } + if len(devs) == 0 { + return self.ScheduleRun(nil) + } + attachReq := make(map[string]int) + for _, dev := range devs { + count, ok := attachReq[dev.Model] + if !ok { + attachReq[dev.Model] = 1 + } else { + attachReq[dev.Model] = count + 1 + } + } + if err := guest.AttachIsolatedDevices(ctx, self.GetUserCred(), attachReq); err != nil { + return errors.Wrap(err, "attach isolated devices") + } + return guest.StartIsolatedDevicesSyncTask(ctx, self.GetUserCred(), false, self.GetTaskId()) +} + +func (self *GuestStartTask) AttachReleasedDevices(ctx context.Context, guest *models.SGuest) { + self.SetStage("OnReleasedDevicesAttached", nil) + if guest.ShutdownBehavior != api.SHUTDOWN_STOP_RELEASE_GPU { + self.ScheduleRun(nil) + return + } + if err := self.attachReleasedDevices(ctx, guest); err != nil { + self.OnStartCompleteFailed(ctx, guest, jsonutils.NewString(errors.Wrap(err, "attach released devices").Error())) + return + } +} + +func (self *GuestStartTask) OnReleasedDevicesAttached(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { db.OpsLog.LogEvent(guest, db.ACT_STARTING, nil, self.UserCred) self.RequestStart(ctx, guest) } +func (self *GuestStartTask) OnReleasedDevicesFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.OnStartCompleteFailed(ctx, guest, data) +} + func (self *GuestStartTask) RequestStart(ctx context.Context, guest *models.SGuest) { self.SetStage("OnStartComplete", nil) host, _ := guest.GetHost() diff --git a/pkg/compute/tasks/guest_stop_task.go b/pkg/compute/tasks/guest_stop_task.go index 12fd783a2b..0e196beaa9 100644 --- a/pkg/compute/tasks/guest_stop_task.go +++ b/pkg/compute/tasks/guest_stop_task.go @@ -66,15 +66,13 @@ func (self *GuestStopTask) stopGuest(ctx context.Context, guest *models.SGuest) func (self *GuestStopTask) OnGuestStopTaskComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { db.OpsLog.LogEvent(guest, db.ACT_STOP, guest.GetShortDesc(ctx), self.UserCred) - models.HostManager.ClearSchedDescCache(guest.HostId) if guest.Status != api.VM_READY && !self.IsSubtask() { // for kvm guest.SetStatus(ctx, self.GetUserCred(), api.VM_READY, "") } + models.HostManager.ClearSchedDescCache(guest.HostId) logclient.AddActionLogWithStartable(self, guest, logclient.ACT_VM_STOP, "success", self.UserCred, true) - self.SetStageComplete(ctx, nil) - if guest.DisableDelete.IsFalse() && guest.ShutdownBehavior == api.SHUTDOWN_TERMINATE { - guest.StartAutoDeleteGuestTask(ctx, self.UserCred, "") - return + if err := self.releaseDevices(ctx, guest); err != nil { + self.OnGuestStopTaskCompleteFailed(ctx, guest, jsonutils.NewString(err.Error())) } } @@ -87,6 +85,47 @@ func (self *GuestStopTask) OnGuestStopTaskCompleteFailed(ctx context.Context, gu logclient.AddActionLogWithStartable(self, guest, logclient.ACT_VM_STOP, reason.String(), self.UserCred, false) } +func (self *GuestStopTask) releaseDevices(ctx context.Context, guest *models.SGuest) error { + self.SetStage("OnDevicesReleased", nil) + if guest.ShutdownBehavior != api.SHUTDOWN_STOP_RELEASE_GPU { + return self.ScheduleRun(nil) + } + devs, err := guest.GetIsolatedDevices() + if err != nil { + return errors.Wrapf(err, "GetIsolatedDevices of guest %s", guest.GetId()) + } + gpus := make([]models.SIsolatedDevice, 0) + for _, dev := range devs { + if dev.IsGPU() { + tmpDev := dev + gpus = append(gpus, tmpDev) + } + } + if len(gpus) == 0 { + return self.ScheduleRun(nil) + } + if err := guest.SetReleasedIsolatedDevices(ctx, self.GetUserCred(), gpus); err != nil { + return errors.Wrapf(err, "SetReleasedIsolatedDevices of guest %s", guest.GetId()) + } + + if err := guest.DetachIsolatedDevices(ctx, self.GetUserCred(), gpus); err != nil { + return errors.Wrapf(err, "DetachIsolatedDevices of guest %s", guest.GetId()) + } + return guest.StartIsolatedDevicesSyncTask(ctx, self.GetUserCred(), false, self.GetTaskId()) +} + +func (self *GuestStopTask) OnDevicesReleased(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) + if guest.DisableDelete.IsFalse() && guest.ShutdownBehavior == api.SHUTDOWN_TERMINATE { + guest.StartAutoDeleteGuestTask(ctx, self.UserCred, "") + return + } +} + +func (self *GuestStopTask) OnDevicesReleaseFailed(ctx context.Context, guest *models.SGuest, reason jsonutils.JSONObject) { + self.OnGuestStopTaskCompleteFailed(ctx, guest, reason) +} + type GuestStopAndFreezeTask struct { SGuestBaseTask } diff --git a/pkg/mcclient/options/compute/server_pod.go b/pkg/mcclient/options/compute/server_pod.go index 07f8544d74..379f5dcb8a 100644 --- a/pkg/mcclient/options/compute/server_pod.go +++ b/pkg/mcclient/options/compute/server_pod.go @@ -36,10 +36,11 @@ type PodCreateOptions struct { VcpuCount int `help:"#CPU cores of VM server, default 1" default:"1" metavar:"" json:"vcpu_count" token:"ncpu"` AllowDelete *bool `help:"Unlock server to allow deleting" json:"-"` //PortMapping []string `help:"Port mapping of the pod and the format is: host_port=8080,port=80,protocol=,host_port_range=-" short-token:"p"` - Arch string `help:"image arch" choices:"aarch64|x86_64"` - AutoStart bool `help:"Auto start server after it is created"` - PodUid int64 `help:"UID of pod" default:"0"` - PodGid int64 `help:"GID of pod" default:"0"` + Arch string `help:"image arch" choices:"aarch64|x86_64"` + AutoStart bool `help:"Auto start server after it is created"` + ShutdownBehavior string `help:"Behavior after VM server shutdown" metavar:"" choices:"stop|terminate|stop_release_gpu"` + PodUid int64 `help:"UID of pod" default:"0"` + PodGid int64 `help:"GID of pod" default:"0"` ContainerCreateCommonOptions } @@ -200,9 +201,10 @@ func (o *PodCreateOptions) Params() (*computeapi.ServerCreateInput, error) { } params := &computeapi.ServerCreateInput{ - ServerConfigs: config, - VcpuCount: o.VcpuCount, - AutoStart: o.AutoStart, + ServerConfigs: config, + VcpuCount: o.VcpuCount, + AutoStart: o.AutoStart, + ShutdownBehavior: o.ShutdownBehavior, Pod: &computeapi.PodCreateInput{ //PortMappings: portMappings, Containers: []*computeapi.PodContainerCreateInput{ diff --git a/pkg/mcclient/options/compute/servers.go b/pkg/mcclient/options/compute/servers.go index febd9419ce..b7f932209e 100644 --- a/pkg/mcclient/options/compute/servers.go +++ b/pkg/mcclient/options/compute/servers.go @@ -464,7 +464,7 @@ type ServerCreateOptionalOptions struct { EnableCloudInit bool `help:"Enable cloud-init service"` NoAccountInit *bool `help:"Not reset account password"` AllowDelete *bool `help:"Unlock server to allow deleting" json:"-"` - ShutdownBehavior string `help:"Behavior after VM server shutdown" metavar:"" choices:"stop|terminate"` + ShutdownBehavior string `help:"Behavior after VM server shutdown" metavar:"" choices:"stop|terminate|stop_release_gpu"` AutoStart bool `help:"Auto start server after it is created"` Deploy []string `help:"Specify deploy files in virtual server file system" json:"-"` DeployTelegraf bool `help:"Deploy telegraf agent if guest os is supported"` @@ -695,7 +695,7 @@ type ServerUpdateOptions struct { Desc string `help:"Description" json:"description"` Boot string `help:"Boot device" choices:"disk|cdrom"` Delete string `help:"Lock server to prevent from deleting" choices:"enable|disable" json:"-"` - ShutdownBehavior string `help:"Behavior after VM server shutdown" choices:"stop|terminate"` + ShutdownBehavior string `help:"Behavior after VM server shutdown" choices:"stop|terminate|stop_release_gpu"` Machine string `help:"Machine type" choices:"q35|pc"` IsDaemon *bool `help:"Daemon server" negative:"no-daemon"`