feat(gpus): add hotplug support of gpu devices

Signed-off-by: wanyaoqi <d3lx.yq@gmail.com>

pkg/compute/models/guest_actions.go

@@ -2136,7 +2136,7 @@ func (self *SGuest) startDetachIsolateDevice(ctx context.Context, userCred mccli
 		return httperrors.NewBadRequestError(msgFmt, device)
 	}
 	dev := iDev.(*SIsolatedDevice)
-	if dev.IsGPU() && self.GetStatus() != api.VM_READY {
+	if dev.IsGPU() && !utils.IsInStringArray(self.GetStatus(), []string{api.VM_READY, api.VM_RUNNING}) {
 		return httperrors.NewInvalidStatusError("Can't detach GPU when status is %q", self.GetStatus())
 	}
 	host, _ := self.GetHost()
@@ -2243,7 +2243,7 @@ func (self *SGuest) startAttachIsolatedDevice(ctx context.Context, userCred mccl
 		return httperrors.NewBadRequestError(msgFmt, device)
 	}
 	dev := iDev.(*SIsolatedDevice)
-	if dev.IsGPU() && self.GetStatus() != api.VM_READY {
+	if dev.IsGPU() && !utils.IsInStringArray(self.GetStatus(), []string{api.VM_READY, api.VM_RUNNING}) {
 		return httperrors.NewInvalidStatusError("Can't attach GPU when status is %q", self.GetStatus())
 	}
 	host, _ := self.GetHost()

pkg/hostman/guestman/desc/pci.go

@@ -96,6 +96,31 @@ func (d *PCIDevice) MultiFunction() string {
 	}
 }
 
+func OptionsToString(options map[string]string) string {
+	var cmd string
+	for key, value := range options {
+		if value != "" {
+			cmd += fmt.Sprintf(",%s=%s", key, value)
+		} else {
+			cmd += fmt.Sprintf(",%s", key)
+		}
+	}
+	return cmd
+}
+
+func (d *PCIDevice) OptionsStr() string {
+	cmd := ""
+	if d.PCIAddr != nil {
+		cmd += fmt.Sprintf("bus=%s,addr=%s", d.BusStr(), d.SlotFunc())
+		if d.Multi != nil {
+			cmd += fmt.Sprintf(",%s", d.MultiFunction())
+		}
+	}
+
+	cmd += OptionsToString(d.Options)
+	return cmd
+}
+
 // pvscsi or virtio-scsi-pci
 type SCSIAddr struct {
 	// The LUN identifies the specific logical unit

pkg/hostman/guestman/guesttasks.go

@@ -787,8 +787,13 @@ func (t *SGuestIsolatedDeviceSyncTask) removeDevice(dev *desc.SGuestIsolatedDevi
 			var i = 0
 			for ; i < len(t.guest.Desc.IsolatedDevices); i++ {
 				if t.guest.Desc.IsolatedDevices[i].Id == dev.Id {
-					if len(t.guest.Desc.IsolatedDevices[i].VfioDevs) > 0 {
-						//TODO: vfio dev hutplug
+					for j := 0; j < len(t.guest.Desc.IsolatedDevices[i].VfioDevs); j++ {
+						pciaddr := t.guest.Desc.IsolatedDevices[i].VfioDevs[j].PCIAddr
+						if pciaddr != nil {
+							if e := t.guest.pciAddrs.ReleasePCIAddress(pciaddr); e != nil {
+								log.Errorf("failed release vfio pci address %s", pciaddr)
+							}
+						}
 					}
 					break
 				}
@@ -807,7 +812,7 @@ func (t *SGuestIsolatedDeviceSyncTask) removeDevice(dev *desc.SGuestIsolatedDevi
 		return
 	}
 
-	opts, err := devObj.GetHotUnplugOptions()
+	opts, err := devObj.GetHotUnplugOptions(dev)
 	if err != nil {
 		cb(errors.Wrap(err, "GetHotPlugOptions").Error())
 		return
@@ -826,29 +831,86 @@ func (t *SGuestIsolatedDeviceSyncTask) addDevice(dev *desc.SGuestIsolatedDevice)
 		return
 	}
 
-	cb := func(res string) {
-		if len(res) > 0 {
-			t.errors = append(t.errors, fmt.Errorf("device add failed: %s", res))
-		} else {
-			if dev.DevType == api.USB_TYPE {
-				dev.Usb = desc.NewUsbDevice("usb-host",
-					fmt.Sprintf("usb%d", len(t.guest.Desc.IsolatedDevices)))
-				dev.Usb.Options = devObj.GetPassthroughOptions()
-			} else {
-				// TODO: vfio dev hotplug
+	onFail := func(err error) {
+		for i := 0; i < len(dev.VfioDevs); i++ {
+			if dev.VfioDevs[i].PCIAddr != nil {
+				if eRelease := t.guest.pciAddrs.ReleasePCIAddress(dev.VfioDevs[i].PCIAddr); eRelease != nil {
+					log.Errorf("failed release pci pci address %s: %s", dev.VfioDevs[i].PCIAddr, eRelease)
+				}
 			}
-			t.guest.Desc.IsolatedDevices = append(t.guest.Desc.IsolatedDevices, dev)
 		}
+		log.Errorln(err)
+		t.errors = append(t.errors, err)
 		t.syncDevice()
+		return
 	}
 
-	opts, err := devObj.GetHotPlugOptions()
+	if dev.DevType == api.USB_TYPE {
+		dev.Usb = desc.NewUsbDevice("usb-host", devObj.GetQemuId())
+		dev.Usb.Options = devObj.GetPassthroughOptions()
+	} else {
+		pciRoot := t.guest.getHotPlugPciController()
+		if pciRoot == nil {
+			log.Errorf("no hotplugable pci controller found")
+			t.errors = append(t.errors, errors.Errorf("no hotplugable pci controller found"))
+			t.syncDevice()
+			return
+		}
+		id := devObj.GetQemuId()
+		dev.VfioDevs = make([]*desc.VFIODevice, 0)
+		vfioDev := &desc.VFIODevice{
+			PCIDevice: desc.NewPCIDevice(pciRoot.CType, "vfio-pci", id),
+		}
+		dev.VfioDevs = append(dev.VfioDevs, vfioDev)
+		dev.VfioDevs[0].HostAddr = devObj.GetAddr()
+		if devObj.GetDeviceType() == api.GPU_VGA_TYPE {
+			dev.VfioDevs[0].XVga = true
+		}
+
+		groupDevAddrs := devObj.GetIOMMUGroupRestAddrs()
+		for j := 0; j < len(groupDevAddrs); j++ {
+			gid := fmt.Sprintf("%s-%d", id, j+1)
+			vfioDev = &desc.VFIODevice{
+				PCIDevice: desc.NewPCIDevice(pciRoot.CType, "vfio-pci", gid),
+			}
+			vfioDev.HostAddr = groupDevAddrs[j]
+			dev.VfioDevs = append(dev.VfioDevs, vfioDev)
+		}
+		multiFunc := true
+		err := t.guest.ensureDevicePciAddress(dev.VfioDevs[0].PCIDevice, 0, &multiFunc)
+		if err != nil {
+			err = errors.Wrapf(err, "ensure isolated device %s pci address", dev.VfioDevs[0].PCIAddr)
+		} else {
+			for j := 1; j < len(dev.VfioDevs); j++ {
+				dev.VfioDevs[j].PCIAddr = dev.VfioDevs[0].PCIAddr.Copy()
+				err = t.guest.ensureDevicePciAddress(dev.VfioDevs[j].PCIDevice, j, nil)
+				if err != nil {
+					err = errors.Wrapf(err, "ensure isolated device %s pci address", dev.VfioDevs[j].PCIAddr)
+					break
+				}
+			}
+		}
+		if err != nil {
+			onFail(err)
+			return
+		}
+	}
+
+	cb := func(res string) {
+		if len(res) > 0 {
+			onFail(fmt.Errorf("device add failed: %s", res))
+		} else {
+			t.guest.Desc.IsolatedDevices = append(t.guest.Desc.IsolatedDevices, dev)
+			t.syncDevice()
+		}
+	}
+
+	opts, err := devObj.GetHotPlugOptions(dev)
 	if err != nil {
 		cb(errors.Wrap(err, "GetHotPlugOptions").Error())
 		return
 	}
 
-	// TODO: support GPU
 	t.addDeviceCallBack(opts, 0, cb)
 }
 

pkg/hostman/guestman/pci.go

@@ -299,10 +299,10 @@ func (s *SKVMGuestInstance) initIsolatedDevices(pciRoot, pciBridge *desc.PCICont
 	for i := 0; i < len(s.Desc.IsolatedDevices); i++ {
 		dev := manager.GetDeviceByAddr(s.Desc.IsolatedDevices[i].Addr)
 		if s.Desc.IsolatedDevices[i].DevType == api.USB_TYPE {
-			s.Desc.IsolatedDevices[i].Usb = desc.NewUsbDevice("usb-host", fmt.Sprintf("usb%d", i))
+			s.Desc.IsolatedDevices[i].Usb = desc.NewUsbDevice("usb-host", dev.GetQemuId())
 			s.Desc.IsolatedDevices[i].Usb.Options = dev.GetPassthroughOptions()
 		} else {
-			id := fmt.Sprintf("vfio-%d", i)
+			id := dev.GetQemuId()
 			s.Desc.IsolatedDevices[i].VfioDevs = make([]*desc.VFIODevice, 0)
 			vfioDev := &desc.VFIODevice{
 				PCIDevice: desc.NewPCIDevice(cont.CType, "vfio-pci", id),
@@ -317,11 +317,11 @@ func (s *SKVMGuestInstance) initIsolatedDevices(pciRoot, pciBridge *desc.PCICont
 
 			groupDevAddrs := dev.GetIOMMUGroupRestAddrs()
 			for j := 0; j < len(groupDevAddrs); j++ {
-				gid := fmt.Sprintf("%s-%s", id, strings.ReplaceAll(groupDevAddrs[i], ":", "-"))
+				gid := fmt.Sprintf("%s-%d", id, j+1)
 				vfioDev = &desc.VFIODevice{
 					PCIDevice: desc.NewPCIDevice(cont.CType, "vfio-pci", gid),
 				}
-				vfioDev.HostAddr = groupDevAddrs[i]
+				vfioDev.HostAddr = groupDevAddrs[j]
 				s.Desc.IsolatedDevices[i].VfioDevs = append(
 					s.Desc.IsolatedDevices[i].VfioDevs, vfioDev,
 				)

pkg/hostman/guestman/qemu/generate.go

@@ -31,30 +31,10 @@ type Monitor struct {
 	Mode string
 }
 
-func optionsToString(options map[string]string) string {
-	var cmd string
-	for key, value := range options {
-		if value != "" {
-			cmd += fmt.Sprintf(",%s=%s", key, value)
-		} else {
-			cmd += fmt.Sprintf(",%s", key)
-		}
-	}
-	return cmd
-}
-
 func generatePCIDeviceOption(dev *desc.PCIDevice) string {
 	cmd := fmt.Sprintf(
-		"-device %s,id=%s", dev.DevType, dev.Id,
+		"-device %s,id=%s,%s", dev.DevType, dev.Id, dev.OptionsStr(),
 	)
-	if dev.PCIAddr != nil {
-		cmd += fmt.Sprintf(",bus=%s,addr=%s", dev.BusStr(), dev.SlotFunc())
-		if dev.Multi != nil {
-			cmd += fmt.Sprintf(",%s", dev.MultiFunction())
-		}
-	}
-
-	cmd += optionsToString(dev.Options)
 	return cmd
 }
 
@@ -63,7 +43,7 @@ func chardevOption(c *desc.CharDev) string {
 	if c.Name != "" {
 		cmd += fmt.Sprintf(",name=%s", c.Name)
 	}
-	cmd += optionsToString(c.Options)
+	cmd += desc.OptionsToString(c.Options)
 	return cmd
 }
 
@@ -72,7 +52,7 @@ func virtSerialPortOption(p *desc.VirtSerialPort, bus string) string {
 		"-device virtserialport,bus=%s.0,chardev=%s,name=%s",
 		bus, p.Chardev, p.Name,
 	)
-	cmd += optionsToString(p.Options)
+	cmd += desc.OptionsToString(p.Options)
 	return cmd
 }
 
@@ -106,7 +86,7 @@ func generateSpiceOptions(port uint, spice *desc.SSpiceDesc) []string {
 
 	// spice
 	spiceCmd := fmt.Sprintf("-spice port=%d", port)
-	spiceCmd += optionsToString(spice.Options)
+	spiceCmd += desc.OptionsToString(spice.Options)
 	opts = append(opts, spiceCmd)
 
 	// intel-hda and codec hda-duplex
@@ -314,7 +294,7 @@ func generateCdromOptions(optDrv QemuOptions, cdroms []*desc.SGuestCdrom) []stri
 	for _, cdrom := range cdroms {
 		//cdromDriveId := cdrom
 		driveOpt := fmt.Sprintf("id=%s", cdrom.Id)
-		driveOpt += optionsToString(cdrom.DriveOptions)
+		driveOpt += desc.OptionsToString(cdrom.DriveOptions)
 
 		var cdromPath = cdrom.Path
 		if len(cdromPath) > 0 {
@@ -333,7 +313,7 @@ func generateCdromOptions(optDrv QemuOptions, cdroms []*desc.SGuestCdrom) []stri
 				opts = append(opts, optDrv.Drive(driveOpt))
 
 				devOpt := fmt.Sprintf("%s,drive=%s", cdrom.Scsi.DevType, cdrom.Id)
-				devOpt += optionsToString(cdrom.Scsi.Options)
+				devOpt += desc.OptionsToString(cdrom.Scsi.Options)
 
 				opts = append(opts, optDrv.Device(devOpt))
 			}
@@ -347,7 +327,7 @@ func generateFloppyOptions(optDrv QemuOptions, floppys []*desc.SGuestFloppy) []s
 
 	for _, floppy := range floppys {
 		driveOpt := fmt.Sprintf("id=%s", floppy.Id)
-		driveOpt += optionsToString(floppy.DriveOptions)
+		driveOpt += desc.OptionsToString(floppy.DriveOptions)
 
 		var floppyPath = floppy.Path
 		if len(floppyPath) > 0 {
@@ -459,7 +439,7 @@ func GetNicDeviceModel(name string) string {
 
 func generateUsbDeviceOption(usbControllerId string, usb *desc.UsbDevice) string {
 	cmd := fmt.Sprintf("-device %s,bus=%s.0", usb.DevType, usbControllerId)
-	cmd += optionsToString(usb.Options)
+	cmd += desc.OptionsToString(usb.Options)
 	return cmd
 }
 
@@ -495,7 +475,7 @@ func generateIsolatedDeviceOptions(guestDesc *desc.SGuestDesc) []string {
 
 func generateObjectOption(o *desc.Object) string {
 	cmd := fmt.Sprintf("-object %s,id=%s", o.ObjType, o.Id)
-	cmd += optionsToString(o.Options)
+	cmd += desc.OptionsToString(o.Options)
 	return cmd
 }
 

pkg/hostman/isolated_device/gpu.go

@@ -24,10 +24,12 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/sets"
 	"yunion.io/x/pkg/utils"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
 	o "yunion.io/x/onecloud/pkg/hostman/options"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/procutils"
@@ -197,12 +199,55 @@ func (dev *sGPUBaseDevice) CustomProbe() error {
 	return nil
 }
 
-func (dev *sGPUBaseDevice) GetHotPlugOptions() ([]*HotPlugOption, error) {
-	return nil, fmt.Errorf("Not implemented")
+func (dev *sGPUBaseDevice) GetQemuId() string {
+	return fmt.Sprintf("dev_%s", strings.ReplaceAll(dev.GetAddr(), ":", "_"))
 }
 
-func (dev *sGPUBaseDevice) GetHotUnplugOptions() ([]*HotUnplugOption, error) {
-	return nil, fmt.Errorf("Not implemented")
+func (dev *sGPUBaseDevice) GetHotPlugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotPlugOption, error) {
+	ret := make([]*HotPlugOption, 0)
+
+	var masterDevOpt *HotPlugOption
+	for i := 0; i < len(isolatedDev.VfioDevs); i++ {
+		cmd := isolatedDev.VfioDevs[i].HostAddr
+		if optCmd := isolatedDev.VfioDevs[i].OptionsStr(); len(optCmd) > 0 {
+			cmd += fmt.Sprintf(",%s", optCmd)
+		}
+		opts := map[string]string{
+			"host": cmd,
+			"id":   isolatedDev.VfioDevs[i].Id,
+		}
+		if isolatedDev.VfioDevs[i].XVga {
+			opts["x-vga"] = "on"
+		}
+		devOpt := &HotPlugOption{
+			Device:  isolatedDev.VfioDevs[i].DevType,
+			Options: opts,
+		}
+		if isolatedDev.VfioDevs[i].Function == 0 {
+			masterDevOpt = devOpt
+		} else {
+			ret = append(ret, devOpt)
+		}
+	}
+	// if PCI slot function 0 already assigned, qemu will reject hotplug function
+	// so put function 0 at the enda
+	if masterDevOpt == nil {
+		return nil, errors.Errorf("GPU Device no function 0 found")
+	}
+	ret = append(ret, masterDevOpt)
+	return ret, nil
+}
+
+func (dev *sGPUBaseDevice) GetHotUnplugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotUnplugOption, error) {
+	if len(isolatedDev.VfioDevs) == 0 {
+		return nil, errors.Errorf("device %s no pci ids", isolatedDev.Id)
+	}
+
+	return []*HotUnplugOption{
+		{
+			Id: isolatedDev.VfioDevs[0].Id,
+		},
+	}, nil
 }
 
 type sGPUVGADevice struct {

pkg/hostman/isolated_device/isolated_device.go

@@ -24,6 +24,7 @@ import (
 	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
@@ -79,9 +80,10 @@ type IDevice interface {
 	GetIOMMUGroupRestAddrs() []string
 	GetVGACmd() string
 	GetCPUCmd() string
+	GetQemuId() string
 
-	GetHotPlugOptions() ([]*HotPlugOption, error)
-	GetHotUnplugOptions() ([]*HotUnplugOption, error)
+	GetHotPlugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotPlugOption, error)
+	GetHotUnplugOptions(dev *desc.SGuestIsolatedDevice) ([]*HotUnplugOption, error)
 }
 
 type IsolatedDeviceManager interface {

pkg/hostman/isolated_device/usb.go

@@ -24,6 +24,7 @@ import (
 	"yunion.io/x/pkg/utils"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
 	"yunion.io/x/onecloud/pkg/util/regutils2"
 )
 
@@ -97,12 +98,9 @@ func (dev *sUSBDevice) GetKernelDriver() (string, error) {
 	return "", nil
 }
 
-func (dev *sUSBDevice) GetQemuId() (string, error) {
+func (dev *sUSBDevice) GetQemuId() string {
 	addrParts := strings.Split(dev.dev.Addr, ":")
-	if len(addrParts) != 2 {
-		return "", errors.Errorf("Invalid addr %q", dev.dev.Addr)
-	}
-	return GetUSBDevId(dev.dev.VendorId, dev.dev.DeviceId, addrParts[0], addrParts[1]), nil
+	return GetUSBDevId(dev.dev.VendorId, dev.dev.DeviceId, addrParts[0], addrParts[1])
 }
 
 func (dev *sUSBDevice) GetPassthroughOptions() map[string]string {
@@ -120,7 +118,7 @@ func (dev *sUSBDevice) GetPassthroughCmd(index int) string {
 	return opt
 }
 
-func (dev *sUSBDevice) GetHotPlugOptions() ([]*HotPlugOption, error) {
+func (dev *sUSBDevice) GetHotPlugOptions(*desc.SGuestIsolatedDevice) ([]*HotPlugOption, error) {
 	opts, err := GetUSBDevQemuOptions(dev.dev.GetVendorDeviceId(), dev.dev.Addr)
 	if err != nil {
 		return nil, errors.Wrap(err, "GetUSBDevQemuOptions")
@@ -133,13 +131,9 @@ func (dev *sUSBDevice) GetHotPlugOptions() ([]*HotPlugOption, error) {
 	}, nil
 }
 
-func (dev *sUSBDevice) GetHotUnplugOptions() ([]*HotUnplugOption, error) {
-	id, err := dev.GetQemuId()
-	if err != nil {
-		return nil, err
-	}
+func (dev *sUSBDevice) GetHotUnplugOptions(*desc.SGuestIsolatedDevice) ([]*HotUnplugOption, error) {
 	return []*HotUnplugOption{
-		{Id: id},
+		{Id: dev.GetQemuId()},
 	}, nil
 }