From 23a172b082e642b053e5342393f03239785e9a2c Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Fri, 28 Oct 2022 10:49:52 +0800 Subject: [PATCH] feat(gpus): add hotplug support of gpu devices Signed-off-by: wanyaoqi --- pkg/compute/models/guest_actions.go | 4 +- pkg/hostman/guestman/desc/pci.go | 25 +++++ pkg/hostman/guestman/guesttasks.go | 94 +++++++++++++++---- pkg/hostman/guestman/pci.go | 8 +- pkg/hostman/guestman/qemu/generate.go | 38 ++------ pkg/hostman/isolated_device/gpu.go | 53 ++++++++++- .../isolated_device/isolated_device.go | 6 +- pkg/hostman/isolated_device/usb.go | 18 ++-- 8 files changed, 177 insertions(+), 69 deletions(-) diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index b6c0faf71a..a2fbd94553 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -2136,7 +2136,7 @@ func (self *SGuest) startDetachIsolateDevice(ctx context.Context, userCred mccli return httperrors.NewBadRequestError(msgFmt, device) } dev := iDev.(*SIsolatedDevice) - if dev.IsGPU() && self.GetStatus() != api.VM_READY { + if dev.IsGPU() && !utils.IsInStringArray(self.GetStatus(), []string{api.VM_READY, api.VM_RUNNING}) { return httperrors.NewInvalidStatusError("Can't detach GPU when status is %q", self.GetStatus()) } host, _ := self.GetHost() @@ -2243,7 +2243,7 @@ func (self *SGuest) startAttachIsolatedDevice(ctx context.Context, userCred mccl return httperrors.NewBadRequestError(msgFmt, device) } dev := iDev.(*SIsolatedDevice) - if dev.IsGPU() && self.GetStatus() != api.VM_READY { + if dev.IsGPU() && !utils.IsInStringArray(self.GetStatus(), []string{api.VM_READY, api.VM_RUNNING}) { return httperrors.NewInvalidStatusError("Can't attach GPU when status is %q", self.GetStatus()) } host, _ := self.GetHost() diff --git a/pkg/hostman/guestman/desc/pci.go b/pkg/hostman/guestman/desc/pci.go index f6d0758dfd..4d93ea077e 100644 --- a/pkg/hostman/guestman/desc/pci.go +++ b/pkg/hostman/guestman/desc/pci.go @@ -96,6 +96,31 @@ func (d *PCIDevice) MultiFunction() string { } } +func OptionsToString(options map[string]string) string { + var cmd string + for key, value := range options { + if value != "" { + cmd += fmt.Sprintf(",%s=%s", key, value) + } else { + cmd += fmt.Sprintf(",%s", key) + } + } + return cmd +} + +func (d *PCIDevice) OptionsStr() string { + cmd := "" + if d.PCIAddr != nil { + cmd += fmt.Sprintf("bus=%s,addr=%s", d.BusStr(), d.SlotFunc()) + if d.Multi != nil { + cmd += fmt.Sprintf(",%s", d.MultiFunction()) + } + } + + cmd += OptionsToString(d.Options) + return cmd +} + // pvscsi or virtio-scsi-pci type SCSIAddr struct { // The LUN identifies the specific logical unit diff --git a/pkg/hostman/guestman/guesttasks.go b/pkg/hostman/guestman/guesttasks.go index 8cde88d313..bb03b49634 100644 --- a/pkg/hostman/guestman/guesttasks.go +++ b/pkg/hostman/guestman/guesttasks.go @@ -787,8 +787,13 @@ func (t *SGuestIsolatedDeviceSyncTask) removeDevice(dev *desc.SGuestIsolatedDevi var i = 0 for ; i < len(t.guest.Desc.IsolatedDevices); i++ { if t.guest.Desc.IsolatedDevices[i].Id == dev.Id { - if len(t.guest.Desc.IsolatedDevices[i].VfioDevs) > 0 { - //TODO: vfio dev hutplug + for j := 0; j < len(t.guest.Desc.IsolatedDevices[i].VfioDevs); j++ { + pciaddr := t.guest.Desc.IsolatedDevices[i].VfioDevs[j].PCIAddr + if pciaddr != nil { + if e := t.guest.pciAddrs.ReleasePCIAddress(pciaddr); e != nil { + log.Errorf("failed release vfio pci address %s", pciaddr) + } + } } break } @@ -807,7 +812,7 @@ func (t *SGuestIsolatedDeviceSyncTask) removeDevice(dev *desc.SGuestIsolatedDevi return } - opts, err := devObj.GetHotUnplugOptions() + opts, err := devObj.GetHotUnplugOptions(dev) if err != nil { cb(errors.Wrap(err, "GetHotPlugOptions").Error()) return @@ -826,29 +831,86 @@ func (t *SGuestIsolatedDeviceSyncTask) addDevice(dev *desc.SGuestIsolatedDevice) return } - cb := func(res string) { - if len(res) > 0 { - t.errors = append(t.errors, fmt.Errorf("device add failed: %s", res)) - } else { - if dev.DevType == api.USB_TYPE { - dev.Usb = desc.NewUsbDevice("usb-host", - fmt.Sprintf("usb%d", len(t.guest.Desc.IsolatedDevices))) - dev.Usb.Options = devObj.GetPassthroughOptions() - } else { - // TODO: vfio dev hotplug + onFail := func(err error) { + for i := 0; i < len(dev.VfioDevs); i++ { + if dev.VfioDevs[i].PCIAddr != nil { + if eRelease := t.guest.pciAddrs.ReleasePCIAddress(dev.VfioDevs[i].PCIAddr); eRelease != nil { + log.Errorf("failed release pci pci address %s: %s", dev.VfioDevs[i].PCIAddr, eRelease) + } } - t.guest.Desc.IsolatedDevices = append(t.guest.Desc.IsolatedDevices, dev) } + log.Errorln(err) + t.errors = append(t.errors, err) t.syncDevice() + return } - opts, err := devObj.GetHotPlugOptions() + if dev.DevType == api.USB_TYPE { + dev.Usb = desc.NewUsbDevice("usb-host", devObj.GetQemuId()) + dev.Usb.Options = devObj.GetPassthroughOptions() + } else { + pciRoot := t.guest.getHotPlugPciController() + if pciRoot == nil { + log.Errorf("no hotplugable pci controller found") + t.errors = append(t.errors, errors.Errorf("no hotplugable pci controller found")) + t.syncDevice() + return + } + id := devObj.GetQemuId() + dev.VfioDevs = make([]*desc.VFIODevice, 0) + vfioDev := &desc.VFIODevice{ + PCIDevice: desc.NewPCIDevice(pciRoot.CType, "vfio-pci", id), + } + dev.VfioDevs = append(dev.VfioDevs, vfioDev) + dev.VfioDevs[0].HostAddr = devObj.GetAddr() + if devObj.GetDeviceType() == api.GPU_VGA_TYPE { + dev.VfioDevs[0].XVga = true + } + + groupDevAddrs := devObj.GetIOMMUGroupRestAddrs() + for j := 0; j < len(groupDevAddrs); j++ { + gid := fmt.Sprintf("%s-%d", id, j+1) + vfioDev = &desc.VFIODevice{ + PCIDevice: desc.NewPCIDevice(pciRoot.CType, "vfio-pci", gid), + } + vfioDev.HostAddr = groupDevAddrs[j] + dev.VfioDevs = append(dev.VfioDevs, vfioDev) + } + multiFunc := true + err := t.guest.ensureDevicePciAddress(dev.VfioDevs[0].PCIDevice, 0, &multiFunc) + if err != nil { + err = errors.Wrapf(err, "ensure isolated device %s pci address", dev.VfioDevs[0].PCIAddr) + } else { + for j := 1; j < len(dev.VfioDevs); j++ { + dev.VfioDevs[j].PCIAddr = dev.VfioDevs[0].PCIAddr.Copy() + err = t.guest.ensureDevicePciAddress(dev.VfioDevs[j].PCIDevice, j, nil) + if err != nil { + err = errors.Wrapf(err, "ensure isolated device %s pci address", dev.VfioDevs[j].PCIAddr) + break + } + } + } + if err != nil { + onFail(err) + return + } + } + + cb := func(res string) { + if len(res) > 0 { + onFail(fmt.Errorf("device add failed: %s", res)) + } else { + t.guest.Desc.IsolatedDevices = append(t.guest.Desc.IsolatedDevices, dev) + t.syncDevice() + } + } + + opts, err := devObj.GetHotPlugOptions(dev) if err != nil { cb(errors.Wrap(err, "GetHotPlugOptions").Error()) return } - // TODO: support GPU t.addDeviceCallBack(opts, 0, cb) } diff --git a/pkg/hostman/guestman/pci.go b/pkg/hostman/guestman/pci.go index 530c3f0ea1..746f9f250d 100644 --- a/pkg/hostman/guestman/pci.go +++ b/pkg/hostman/guestman/pci.go @@ -299,10 +299,10 @@ func (s *SKVMGuestInstance) initIsolatedDevices(pciRoot, pciBridge *desc.PCICont for i := 0; i < len(s.Desc.IsolatedDevices); i++ { dev := manager.GetDeviceByAddr(s.Desc.IsolatedDevices[i].Addr) if s.Desc.IsolatedDevices[i].DevType == api.USB_TYPE { - s.Desc.IsolatedDevices[i].Usb = desc.NewUsbDevice("usb-host", fmt.Sprintf("usb%d", i)) + s.Desc.IsolatedDevices[i].Usb = desc.NewUsbDevice("usb-host", dev.GetQemuId()) s.Desc.IsolatedDevices[i].Usb.Options = dev.GetPassthroughOptions() } else { - id := fmt.Sprintf("vfio-%d", i) + id := dev.GetQemuId() s.Desc.IsolatedDevices[i].VfioDevs = make([]*desc.VFIODevice, 0) vfioDev := &desc.VFIODevice{ PCIDevice: desc.NewPCIDevice(cont.CType, "vfio-pci", id), @@ -317,11 +317,11 @@ func (s *SKVMGuestInstance) initIsolatedDevices(pciRoot, pciBridge *desc.PCICont groupDevAddrs := dev.GetIOMMUGroupRestAddrs() for j := 0; j < len(groupDevAddrs); j++ { - gid := fmt.Sprintf("%s-%s", id, strings.ReplaceAll(groupDevAddrs[i], ":", "-")) + gid := fmt.Sprintf("%s-%d", id, j+1) vfioDev = &desc.VFIODevice{ PCIDevice: desc.NewPCIDevice(cont.CType, "vfio-pci", gid), } - vfioDev.HostAddr = groupDevAddrs[i] + vfioDev.HostAddr = groupDevAddrs[j] s.Desc.IsolatedDevices[i].VfioDevs = append( s.Desc.IsolatedDevices[i].VfioDevs, vfioDev, ) diff --git a/pkg/hostman/guestman/qemu/generate.go b/pkg/hostman/guestman/qemu/generate.go index a238587c82..5d60363164 100644 --- a/pkg/hostman/guestman/qemu/generate.go +++ b/pkg/hostman/guestman/qemu/generate.go @@ -31,30 +31,10 @@ type Monitor struct { Mode string } -func optionsToString(options map[string]string) string { - var cmd string - for key, value := range options { - if value != "" { - cmd += fmt.Sprintf(",%s=%s", key, value) - } else { - cmd += fmt.Sprintf(",%s", key) - } - } - return cmd -} - func generatePCIDeviceOption(dev *desc.PCIDevice) string { cmd := fmt.Sprintf( - "-device %s,id=%s", dev.DevType, dev.Id, + "-device %s,id=%s,%s", dev.DevType, dev.Id, dev.OptionsStr(), ) - if dev.PCIAddr != nil { - cmd += fmt.Sprintf(",bus=%s,addr=%s", dev.BusStr(), dev.SlotFunc()) - if dev.Multi != nil { - cmd += fmt.Sprintf(",%s", dev.MultiFunction()) - } - } - - cmd += optionsToString(dev.Options) return cmd } @@ -63,7 +43,7 @@ func chardevOption(c *desc.CharDev) string { if c.Name != "" { cmd += fmt.Sprintf(",name=%s", c.Name) } - cmd += optionsToString(c.Options) + cmd += desc.OptionsToString(c.Options) return cmd } @@ -72,7 +52,7 @@ func virtSerialPortOption(p *desc.VirtSerialPort, bus string) string { "-device virtserialport,bus=%s.0,chardev=%s,name=%s", bus, p.Chardev, p.Name, ) - cmd += optionsToString(p.Options) + cmd += desc.OptionsToString(p.Options) return cmd } @@ -106,7 +86,7 @@ func generateSpiceOptions(port uint, spice *desc.SSpiceDesc) []string { // spice spiceCmd := fmt.Sprintf("-spice port=%d", port) - spiceCmd += optionsToString(spice.Options) + spiceCmd += desc.OptionsToString(spice.Options) opts = append(opts, spiceCmd) // intel-hda and codec hda-duplex @@ -314,7 +294,7 @@ func generateCdromOptions(optDrv QemuOptions, cdroms []*desc.SGuestCdrom) []stri for _, cdrom := range cdroms { //cdromDriveId := cdrom driveOpt := fmt.Sprintf("id=%s", cdrom.Id) - driveOpt += optionsToString(cdrom.DriveOptions) + driveOpt += desc.OptionsToString(cdrom.DriveOptions) var cdromPath = cdrom.Path if len(cdromPath) > 0 { @@ -333,7 +313,7 @@ func generateCdromOptions(optDrv QemuOptions, cdroms []*desc.SGuestCdrom) []stri opts = append(opts, optDrv.Drive(driveOpt)) devOpt := fmt.Sprintf("%s,drive=%s", cdrom.Scsi.DevType, cdrom.Id) - devOpt += optionsToString(cdrom.Scsi.Options) + devOpt += desc.OptionsToString(cdrom.Scsi.Options) opts = append(opts, optDrv.Device(devOpt)) } @@ -347,7 +327,7 @@ func generateFloppyOptions(optDrv QemuOptions, floppys []*desc.SGuestFloppy) []s for _, floppy := range floppys { driveOpt := fmt.Sprintf("id=%s", floppy.Id) - driveOpt += optionsToString(floppy.DriveOptions) + driveOpt += desc.OptionsToString(floppy.DriveOptions) var floppyPath = floppy.Path if len(floppyPath) > 0 { @@ -459,7 +439,7 @@ func GetNicDeviceModel(name string) string { func generateUsbDeviceOption(usbControllerId string, usb *desc.UsbDevice) string { cmd := fmt.Sprintf("-device %s,bus=%s.0", usb.DevType, usbControllerId) - cmd += optionsToString(usb.Options) + cmd += desc.OptionsToString(usb.Options) return cmd } @@ -495,7 +475,7 @@ func generateIsolatedDeviceOptions(guestDesc *desc.SGuestDesc) []string { func generateObjectOption(o *desc.Object) string { cmd := fmt.Sprintf("-object %s,id=%s", o.ObjType, o.Id) - cmd += optionsToString(o.Options) + cmd += desc.OptionsToString(o.Options) return cmd } diff --git a/pkg/hostman/isolated_device/gpu.go b/pkg/hostman/isolated_device/gpu.go index 252e60e9d7..6479443b61 100644 --- a/pkg/hostman/isolated_device/gpu.go +++ b/pkg/hostman/isolated_device/gpu.go @@ -24,10 +24,12 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/sets" "yunion.io/x/pkg/utils" api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/hostman/guestman/desc" o "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/procutils" @@ -197,12 +199,55 @@ func (dev *sGPUBaseDevice) CustomProbe() error { return nil } -func (dev *sGPUBaseDevice) GetHotPlugOptions() ([]*HotPlugOption, error) { - return nil, fmt.Errorf("Not implemented") +func (dev *sGPUBaseDevice) GetQemuId() string { + return fmt.Sprintf("dev_%s", strings.ReplaceAll(dev.GetAddr(), ":", "_")) } -func (dev *sGPUBaseDevice) GetHotUnplugOptions() ([]*HotUnplugOption, error) { - return nil, fmt.Errorf("Not implemented") +func (dev *sGPUBaseDevice) GetHotPlugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotPlugOption, error) { + ret := make([]*HotPlugOption, 0) + + var masterDevOpt *HotPlugOption + for i := 0; i < len(isolatedDev.VfioDevs); i++ { + cmd := isolatedDev.VfioDevs[i].HostAddr + if optCmd := isolatedDev.VfioDevs[i].OptionsStr(); len(optCmd) > 0 { + cmd += fmt.Sprintf(",%s", optCmd) + } + opts := map[string]string{ + "host": cmd, + "id": isolatedDev.VfioDevs[i].Id, + } + if isolatedDev.VfioDevs[i].XVga { + opts["x-vga"] = "on" + } + devOpt := &HotPlugOption{ + Device: isolatedDev.VfioDevs[i].DevType, + Options: opts, + } + if isolatedDev.VfioDevs[i].Function == 0 { + masterDevOpt = devOpt + } else { + ret = append(ret, devOpt) + } + } + // if PCI slot function 0 already assigned, qemu will reject hotplug function + // so put function 0 at the enda + if masterDevOpt == nil { + return nil, errors.Errorf("GPU Device no function 0 found") + } + ret = append(ret, masterDevOpt) + return ret, nil +} + +func (dev *sGPUBaseDevice) GetHotUnplugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotUnplugOption, error) { + if len(isolatedDev.VfioDevs) == 0 { + return nil, errors.Errorf("device %s no pci ids", isolatedDev.Id) + } + + return []*HotUnplugOption{ + { + Id: isolatedDev.VfioDevs[0].Id, + }, + }, nil } type sGPUVGADevice struct { diff --git a/pkg/hostman/isolated_device/isolated_device.go b/pkg/hostman/isolated_device/isolated_device.go index 74c87a2532..67cfaa9073 100644 --- a/pkg/hostman/isolated_device/isolated_device.go +++ b/pkg/hostman/isolated_device/isolated_device.go @@ -24,6 +24,7 @@ import ( "yunion.io/x/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/hostman/guestman/desc" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" @@ -79,9 +80,10 @@ type IDevice interface { GetIOMMUGroupRestAddrs() []string GetVGACmd() string GetCPUCmd() string + GetQemuId() string - GetHotPlugOptions() ([]*HotPlugOption, error) - GetHotUnplugOptions() ([]*HotUnplugOption, error) + GetHotPlugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotPlugOption, error) + GetHotUnplugOptions(dev *desc.SGuestIsolatedDevice) ([]*HotUnplugOption, error) } type IsolatedDeviceManager interface { diff --git a/pkg/hostman/isolated_device/usb.go b/pkg/hostman/isolated_device/usb.go index 8686e3cef4..a2b9b51e04 100644 --- a/pkg/hostman/isolated_device/usb.go +++ b/pkg/hostman/isolated_device/usb.go @@ -24,6 +24,7 @@ import ( "yunion.io/x/pkg/utils" api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/hostman/guestman/desc" "yunion.io/x/onecloud/pkg/util/regutils2" ) @@ -97,12 +98,9 @@ func (dev *sUSBDevice) GetKernelDriver() (string, error) { return "", nil } -func (dev *sUSBDevice) GetQemuId() (string, error) { +func (dev *sUSBDevice) GetQemuId() string { addrParts := strings.Split(dev.dev.Addr, ":") - if len(addrParts) != 2 { - return "", errors.Errorf("Invalid addr %q", dev.dev.Addr) - } - return GetUSBDevId(dev.dev.VendorId, dev.dev.DeviceId, addrParts[0], addrParts[1]), nil + return GetUSBDevId(dev.dev.VendorId, dev.dev.DeviceId, addrParts[0], addrParts[1]) } func (dev *sUSBDevice) GetPassthroughOptions() map[string]string { @@ -120,7 +118,7 @@ func (dev *sUSBDevice) GetPassthroughCmd(index int) string { return opt } -func (dev *sUSBDevice) GetHotPlugOptions() ([]*HotPlugOption, error) { +func (dev *sUSBDevice) GetHotPlugOptions(*desc.SGuestIsolatedDevice) ([]*HotPlugOption, error) { opts, err := GetUSBDevQemuOptions(dev.dev.GetVendorDeviceId(), dev.dev.Addr) if err != nil { return nil, errors.Wrap(err, "GetUSBDevQemuOptions") @@ -133,13 +131,9 @@ func (dev *sUSBDevice) GetHotPlugOptions() ([]*HotPlugOption, error) { }, nil } -func (dev *sUSBDevice) GetHotUnplugOptions() ([]*HotUnplugOption, error) { - id, err := dev.GetQemuId() - if err != nil { - return nil, err - } +func (dev *sUSBDevice) GetHotUnplugOptions(*desc.SGuestIsolatedDevice) ([]*HotUnplugOption, error) { return []*HotUnplugOption{ - {Id: id}, + {Id: dev.GetQemuId()}, }, nil }