Merge pull request #18020 from wanyaoqi/fix/vgpu-hotplug

fix(host): nvidia vgpu hotplug

pkg/hostman/isolated_device/gpu.go

@@ -29,7 +29,6 @@ import (
 	"yunion.io/x/pkg/utils"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
-	"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
 	o "yunion.io/x/onecloud/pkg/hostman/options"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/procutils"
@@ -216,53 +215,6 @@ func (dev *sGPUBaseDevice) GetQemuId() string {
 	return fmt.Sprintf("dev_%s", strings.ReplaceAll(dev.GetAddr(), ":", "_"))
 }
 
-func (dev *sGPUBaseDevice) GetHotPlugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotPlugOption, error) {
-	ret := make([]*HotPlugOption, 0)
-
-	var masterDevOpt *HotPlugOption
-	for i := 0; i < len(isolatedDev.VfioDevs); i++ {
-		cmd := isolatedDev.VfioDevs[i].HostAddr
-		if optCmd := isolatedDev.VfioDevs[i].OptionsStr(); len(optCmd) > 0 {
-			cmd += fmt.Sprintf(",%s", optCmd)
-		}
-		opts := map[string]string{
-			"host": cmd,
-			"id":   isolatedDev.VfioDevs[i].Id,
-		}
-		if isolatedDev.VfioDevs[i].XVga {
-			opts["x-vga"] = "on"
-		}
-		devOpt := &HotPlugOption{
-			Device:  isolatedDev.VfioDevs[i].DevType,
-			Options: opts,
-		}
-		if isolatedDev.VfioDevs[i].Function == 0 {
-			masterDevOpt = devOpt
-		} else {
-			ret = append(ret, devOpt)
-		}
-	}
-	// if PCI slot function 0 already assigned, qemu will reject hotplug function
-	// so put function 0 at the enda
-	if masterDevOpt == nil {
-		return nil, errors.Errorf("GPU Device no function 0 found")
-	}
-	ret = append(ret, masterDevOpt)
-	return ret, nil
-}
-
-func (dev *sGPUBaseDevice) GetHotUnplugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotUnplugOption, error) {
-	if len(isolatedDev.VfioDevs) == 0 {
-		return nil, errors.Errorf("device %s no pci ids", isolatedDev.Id)
-	}
-
-	return []*HotUnplugOption{
-		{
-			Id: isolatedDev.VfioDevs[0].Id,
-		},
-	}, nil
-}
-
 func getGuestAddr(index int) string {
 	vAddr := fmt.Sprintf("0x%x", 21+index) // from 0x15 above
 	return vAddr

pkg/hostman/isolated_device/isolated_device.go

@@ -637,6 +637,58 @@ func (dev *sBaseDevice) CustomProbe(idx int) error {
 	return nil
 }
 
+func (dev *sBaseDevice) GetHotPlugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotPlugOption, error) {
+	ret := make([]*HotPlugOption, 0)
+
+	var masterDevOpt *HotPlugOption
+	for i := 0; i < len(isolatedDev.VfioDevs); i++ {
+		opts := map[string]string{
+			"host": isolatedDev.VfioDevs[i].HostAddr,
+			"bus":  isolatedDev.VfioDevs[i].BusStr(),
+			"addr": isolatedDev.VfioDevs[i].SlotFunc(),
+			"id":   isolatedDev.VfioDevs[i].Id,
+		}
+		if isolatedDev.VfioDevs[i].Multi != nil {
+			if *isolatedDev.VfioDevs[i].Multi {
+				opts["multifunction"] = "on"
+			} else {
+				opts["multifunction"] = "off"
+			}
+		}
+		if isolatedDev.VfioDevs[i].XVga {
+			opts["x-vga"] = "on"
+		}
+		devOpt := &HotPlugOption{
+			Device:  isolatedDev.VfioDevs[i].DevType,
+			Options: opts,
+		}
+		if isolatedDev.VfioDevs[i].Function == 0 {
+			masterDevOpt = devOpt
+		} else {
+			ret = append(ret, devOpt)
+		}
+	}
+	// if PCI slot function 0 already assigned, qemu will reject hotplug function
+	// so put function 0 at the enda
+	if masterDevOpt == nil {
+		return nil, errors.Errorf("GPU Device no function 0 found")
+	}
+	ret = append(ret, masterDevOpt)
+	return ret, nil
+}
+
+func (dev *sBaseDevice) GetHotUnplugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotUnplugOption, error) {
+	if len(isolatedDev.VfioDevs) == 0 {
+		return nil, errors.Errorf("device %s no pci ids", isolatedDev.Id)
+	}
+
+	return []*HotUnplugOption{
+		{
+			Id: isolatedDev.VfioDevs[0].Id,
+		},
+	}, nil
+}
+
 func ParseOutput(output []byte, doTrim bool) []string {
 	lines := make([]string, 0)
 	for _, line := range strings.Split(string(output), "\n") {

pkg/hostman/isolated_device/nvidia_vgpu.go

@@ -151,14 +151,21 @@ func (dev *sNVIDIAVgpuDevice) GetHotPlugOptions(isolatedDev *desc.SGuestIsolated
 
 	var masterDevOpt *HotPlugOption
 	for i := 0; i < len(isolatedDev.VfioDevs); i++ {
-		cmd := isolatedDev.VfioDevs[i].HostAddr
-		if optCmd := isolatedDev.VfioDevs[i].OptionsStr(); len(optCmd) > 0 {
-			cmd += fmt.Sprintf(",%s", optCmd)
-		}
+		sysfsdev := path.Join("/sys/bus/mdev/devices", isolatedDev.MdevId)
 		opts := map[string]string{
-			"sysfsdev": path.Join("/sys/bus/mdev/devices", isolatedDev.MdevId),
+			"sysfsdev": sysfsdev,
+			"bus":      isolatedDev.VfioDevs[i].BusStr(),
+			"addr":     isolatedDev.VfioDevs[i].SlotFunc(),
 			"id":       isolatedDev.VfioDevs[i].Id,
 		}
+		if isolatedDev.VfioDevs[i].Multi != nil {
+			if *isolatedDev.VfioDevs[i].Multi {
+				opts["multifunction"] = "on"
+			} else {
+				opts["multifunction"] = "off"
+			}
+		}
+
 		devOpt := &HotPlugOption{
 			Device:  isolatedDev.VfioDevs[i].DevType,
 			Options: opts,

pkg/hostman/isolated_device/nvme.go

@@ -22,7 +22,6 @@ import (
 	"yunion.io/x/pkg/util/fileutils"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
-	"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
 )
 
 type sNVMEDevice struct {
@@ -43,50 +42,6 @@ func (dev *sNVMEDevice) GetQemuId() string {
 	return fmt.Sprintf("dev_%s", strings.ReplaceAll(dev.GetAddr(), ":", "_"))
 }
 
-func (dev *sNVMEDevice) GetHotPlugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotPlugOption, error) {
-	ret := make([]*HotPlugOption, 0)
-
-	var masterDevOpt *HotPlugOption
-	for i := 0; i < len(isolatedDev.VfioDevs); i++ {
-		cmd := isolatedDev.VfioDevs[i].HostAddr
-		if optCmd := isolatedDev.VfioDevs[i].OptionsStr(); len(optCmd) > 0 {
-			cmd += fmt.Sprintf(",%s", optCmd)
-		}
-		opts := map[string]string{
-			"host": cmd,
-			"id":   isolatedDev.VfioDevs[i].Id,
-		}
-		devOpt := &HotPlugOption{
-			Device:  isolatedDev.VfioDevs[i].DevType,
-			Options: opts,
-		}
-		if isolatedDev.VfioDevs[i].Function == 0 {
-			masterDevOpt = devOpt
-		} else {
-			ret = append(ret, devOpt)
-		}
-	}
-	// if PCI slot function 0 already assigned, qemu will reject hotplug function
-	// so put function 0 at the enda
-	if masterDevOpt == nil {
-		return nil, errors.Errorf("Device no function 0 found")
-	}
-	ret = append(ret, masterDevOpt)
-	return ret, nil
-}
-
-func (dev *sNVMEDevice) GetHotUnplugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotUnplugOption, error) {
-	if len(isolatedDev.VfioDevs) == 0 {
-		return nil, errors.Errorf("device %s no pci ids", isolatedDev.Id)
-	}
-
-	return []*HotUnplugOption{
-		{
-			Id: isolatedDev.VfioDevs[0].Id,
-		},
-	}, nil
-}
-
 func (dev *sNVMEDevice) GetNVMESizeMB() int {
 	return dev.sizeMB
 }

pkg/hostman/isolated_device/pci_device.go

@@ -17,10 +17,6 @@ package isolated_device
 import (
 	"fmt"
 	"strings"
-
-	"yunion.io/x/pkg/errors"
-
-	"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
 )
 
 type sGeneralPCIDevice struct {
@@ -39,50 +35,6 @@ func (dev *sGeneralPCIDevice) GetQemuId() string {
 	return fmt.Sprintf("dev_%s", strings.ReplaceAll(dev.GetAddr(), ":", "_"))
 }
 
-func (dev *sGeneralPCIDevice) GetHotPlugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotPlugOption, error) {
-	ret := make([]*HotPlugOption, 0)
-
-	var masterDevOpt *HotPlugOption
-	for i := 0; i < len(isolatedDev.VfioDevs); i++ {
-		cmd := isolatedDev.VfioDevs[i].HostAddr
-		if optCmd := isolatedDev.VfioDevs[i].OptionsStr(); len(optCmd) > 0 {
-			cmd += fmt.Sprintf(",%s", optCmd)
-		}
-		opts := map[string]string{
-			"host": cmd,
-			"id":   isolatedDev.VfioDevs[i].Id,
-		}
-		devOpt := &HotPlugOption{
-			Device:  isolatedDev.VfioDevs[i].DevType,
-			Options: opts,
-		}
-		if isolatedDev.VfioDevs[i].Function == 0 {
-			masterDevOpt = devOpt
-		} else {
-			ret = append(ret, devOpt)
-		}
-	}
-	// if PCI slot function 0 already assigned, qemu will reject hotplug function
-	// so put function 0 at the enda
-	if masterDevOpt == nil {
-		return nil, errors.Errorf("Device no function 0 found")
-	}
-	ret = append(ret, masterDevOpt)
-	return ret, nil
-}
-
-func (dev *sGeneralPCIDevice) GetHotUnplugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotUnplugOption, error) {
-	if len(isolatedDev.VfioDevs) == 0 {
-		return nil, errors.Errorf("device %s no pci ids", isolatedDev.Id)
-	}
-
-	return []*HotUnplugOption{
-		{
-			Id: isolatedDev.VfioDevs[0].Id,
-		},
-	}, nil
-}
-
 func newGeneralPCIDevice(dev *PCIDevice, devType string) *sGeneralPCIDevice {
 	return &sGeneralPCIDevice{
 		sBaseDevice: newBaseDevice(dev, devType),

pkg/hostman/isolated_device/sriov_base.go

@@ -6,9 +6,7 @@ import (
 	"strings"
 
 	"yunion.io/x/log"
-	"yunion.io/x/pkg/errors"
 
-	"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/procutils"
 )
@@ -80,50 +78,6 @@ func (dev *sSRIOVBaseDevice) GetQemuId() string {
 	return fmt.Sprintf("dev_%s", strings.ReplaceAll(dev.GetAddr(), ":", "_"))
 }
 
-func (dev *sSRIOVBaseDevice) GetHotPlugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotPlugOption, error) {
-	ret := make([]*HotPlugOption, 0)
-
-	var masterDevOpt *HotPlugOption
-	for i := 0; i < len(isolatedDev.VfioDevs); i++ {
-		cmd := isolatedDev.VfioDevs[i].HostAddr
-		if optCmd := isolatedDev.VfioDevs[i].OptionsStr(); len(optCmd) > 0 {
-			cmd += fmt.Sprintf(",%s", optCmd)
-		}
-		opts := map[string]string{
-			"host": cmd,
-			"id":   isolatedDev.VfioDevs[i].Id,
-		}
-		devOpt := &HotPlugOption{
-			Device:  isolatedDev.VfioDevs[i].DevType,
-			Options: opts,
-		}
-		if isolatedDev.VfioDevs[i].Function == 0 {
-			masterDevOpt = devOpt
-		} else {
-			ret = append(ret, devOpt)
-		}
-	}
-	// if PCI slot function 0 already assigned, qemu will reject hotplug function
-	// so put function 0 at the enda
-	if masterDevOpt == nil {
-		return nil, errors.Errorf("Device no function 0 found")
-	}
-	ret = append(ret, masterDevOpt)
-	return ret, nil
-}
-
-func (dev *sSRIOVBaseDevice) GetHotUnplugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotUnplugOption, error) {
-	if len(isolatedDev.VfioDevs) == 0 {
-		return nil, errors.Errorf("device %s no pci ids", isolatedDev.Id)
-	}
-
-	return []*HotUnplugOption{
-		{
-			Id: isolatedDev.VfioDevs[0].Id,
-		},
-	}, nil
-}
-
 func (dev *sSRIOVBaseDevice) CustomProbe(idx int) error {
 	// check environments on first probe
 	if idx == 0 {

pkg/hostman/monitor/qmp.go

@@ -459,91 +459,32 @@ func (m *QmpMonitor) GetBlocks(callback func([]QemuBlock)) {
 
 func (m *QmpMonitor) ChangeCdrom(dev string, path string, callback StringCallback) {
 	m.HumanMonitorCommand(fmt.Sprintf("change %s %s", dev, path), callback)
-	// var (
-	// 	args = map[string]interface{}{
-	// 		"arguments": map[string]interface{}{
-	// 			"device": dev,
-	// 			"target": path,
-	// 		},
-	// 	}
-	// 	cmd = &Command{
-	// 		Execute: "change",
-	// 		Args:    args,
-	// 	}
-
-	// 	cb = func(res *Response) {
-	// 		callback(m.actionResult(res))
-	// 	}
-	// )
-
-	// m.Query(cmd, cb)
 }
 
 func (m *QmpMonitor) EjectCdrom(dev string, callback StringCallback) {
 	m.HumanMonitorCommand(fmt.Sprintf("eject -f %s", dev), callback)
-	// XXX: 同下
-	// var (
-	// 	args = map[string]interface{}{
-	// 		"arguments": map[string]interface{}{
-	// 			"device": dev,
-	// 			"force":  true,
-	// 		},
-	// 	}
-	// 	cmd = &Command{
-	// 		Execute: "eject",
-	// 		Args:    args,
-	// 	}
-
-	// 	cb = func(res *Response) {
-	// 		callback(m.actionResult(res))
-	// 	}
-	// )
-
-	// m.Query(cmd, cb)
 }
 
 func (m *QmpMonitor) DriveDel(idstr string, callback StringCallback) {
 	m.HumanMonitorCommand(fmt.Sprintf("drive_del %s", idstr), callback)
-	// XXX: 同下
-	// var (
-	// 	args = map[string]interface{}{
-	// 		"arguments": map[string]interface{}{
-	// 			"device": idstr,
-	// 		},
-	// 	}
-	// 	cmd = &Command{
-	// 		Execute: "drive_del",
-	// 		Args:    args,
-	// 	}
-
-	// 	cb = func(res *Response) {
-	// 		callback(m.actionResult(res))
-	// 	}
-	// )
-
-	// m.Query(cmd, cb)
 }
 
 func (m *QmpMonitor) DeviceDel(idstr string, callback StringCallback) {
-	m.HumanMonitorCommand(fmt.Sprintf("device_del %s", idstr), callback)
-	// XXX: 同下
-	// var (
-	// 	args = map[string]interface{}{
-	// 		"arguments": map[string]interface{}{
-	// 			"device": idstr,
-	// 		},
-	// 	}
-	// 	cmd = &Command{
-	// 		Execute: "device_del",
-	// 		Args:    args,
-	// 	}
+	//m.HumanMonitorCommand(fmt.Sprintf("device_del %s", idstr), callback)
+	var (
+		args = map[string]interface{}{
+			"id": idstr,
+		}
+		cmd = &Command{
+			Execute: "device_del",
+			Args:    args,
+		}
 
-	// 	cb = func(res *Response) {
-	// 		callback(m.actionResult(res))
-	// 	}
-	// )
-
-	// m.Query(cmd, cb)
+		cb = func(res *Response) {
+			callback(m.actionResult(res))
+		}
+	)
+	m.Query(cmd, cb)
 }
 
 func (m *QmpMonitor) ObjectDel(idstr string, callback StringCallback) {
@@ -582,54 +523,27 @@ func (m *QmpMonitor) DriveAdd(bus, node string, params map[string]string, callba
 
 	cmd = fmt.Sprintf("%s %s %s", cmd, bus, strings.Join(paramsKvs, ","))
 	m.HumanMonitorCommand(cmd, callback)
-	// XXX: 同下
-	// var (
-	// 	args = map[string]interface{}{
-	// 		"arguments": map[string]interface{}{
-	// 			"bus":    bus,
-	// 			"params": params,
-	// 		},
-	// 	}
-	// 	cmd = &Command{
-	// 		Execute: "drive_add",
-	// 		Args:    args,
-	// 	}
-
-	// 	cb = func(res *Response) {
-	// 		callback(m.actionResult(res))
-	// 	}
-	// )
-
-	// m.Query(cmd, cb)
 }
 
 func (m *QmpMonitor) DeviceAdd(dev string, params map[string]string, callback StringCallback) {
-	var paramsKvs = []string{}
-	for k, v := range params {
-		paramsKvs = append(paramsKvs, fmt.Sprintf("%s=%v", k, v))
+	args := map[string]interface{}{
+		"driver": dev,
 	}
-	cmd := fmt.Sprintf("device_add %s,%s", dev, strings.Join(paramsKvs, ","))
-	m.HumanMonitorCommand(cmd, callback)
 
-	// XXX: 参数不对，之后再调，先用着hmp的参数
-	// var (
-	// 	args = map[string]interface{}{
-	// 		"arguments": map[string]interface{}{
-	// 			"driver": dev,
-	// 			"params": params,
-	// 		},
-	// 	}
-	// 	cmd = &Command{
-	// 		Execute: "device_add",
-	// 		Args:    args,
-	// 	}
+	for k, v := range params {
+		args[k] = v
+	}
 
-	// 	cb = func(res *Response) {
-	// 		callback(m.actionResult(res))
-	// 	}
-	// )
+	cmd := &Command{
+		Execute: "device_add",
+		Args:    args,
+	}
 
-	// m.Query(cmd, cb)
+	cb := func(res *Response) {
+		callback(m.actionResult(res))
+	}
+
+	m.Query(cmd, cb)
 }
 
 func (m *QmpMonitor) MigrateSetDowntime(dtSec float64, callback StringCallback) {
@@ -679,21 +593,6 @@ func (m *QmpMonitor) MigrateSetParameter(key string, val interface{}, callback S
 }
 
 func (m *QmpMonitor) MigrateIncoming(address string, callback StringCallback) {
-	/*
-			 * var (
-			 * 	cb = func(res *Response) {
-			 * 		callback(m.actionResult(res))
-			 * 	}
-			 * 	cmd = &Command{
-			 * 		Execute: "migrate-incoming",
-			 * 		Args: map[string]interface{}{
-			 * 			"uri": address,
-			 * 		},
-			 * 	}
-			 * )
-		     *
-			 * m.Query(cmd, cb)
-	*/
 	cmd := fmt.Sprintf("migrate_incoming %s", address)
 	m.HumanMonitorCommand(cmd, callback)
 }