From 70755a9a1bfce5ad0a4c738e53351744e10e01c5 Mon Sep 17 00:00:00 2001 From: wanyaoqi <18528551+wanyaoqi@users.noreply.github.com> Date: Sun, 13 Nov 2022 18:17:30 +0800 Subject: [PATCH] feat(region,host): sriov nic support (#15342) Support sriov nic passthrough to kvm guest. Signed-off-by: wanyaoqi Signed-off-by: wanyaoqi --- pkg/apis/compute/api.go | 15 +- pkg/apis/compute/isolated_device.go | 1 + pkg/apis/compute/network_const.go | 7 + pkg/cloudcommon/cmdline/parser.go | 8 + pkg/compute/guestdrivers/virtualization.go | 23 ++ pkg/compute/models/guest_actions.go | 80 +++++-- pkg/compute/models/guestnetworks.go | 17 +- pkg/compute/models/guests.go | 75 ++++++- pkg/compute/models/isolated_devices.go | 63 +++++- pkg/compute/tasks/guest_batch_create_task.go | 3 +- pkg/hostman/guestman/desc/desc.go | 2 + pkg/hostman/guestman/desc/pci.go | 8 + pkg/hostman/guestman/guestman.go | 4 +- pkg/hostman/guestman/guesttasks.go | 43 +++- pkg/hostman/guestman/pci.go | 72 +++--- pkg/hostman/guestman/qemu-kvm.go | 103 ++++++++- pkg/hostman/guestman/qemu-kvmhelper.go | 12 + pkg/hostman/guestman/qemu/generate.go | 4 + pkg/hostman/hostinfo/hostinfo.go | 20 +- pkg/hostman/isolated_device/gpu.go | 76 +++---- .../isolated_device/isolated_device.go | 50 ++++- pkg/hostman/isolated_device/nic.go | 208 ++++++++++++++++++ pkg/hostman/isolated_device/usb.go | 2 +- pkg/hostman/options/options.go | 5 +- .../predicates/isolated_device_predicate.go | 10 + .../algorithm/predicates/network_predicate.go | 28 +++ pkg/scheduler/cache/candidate/base.go | 16 ++ pkg/scheduler/core/types.go | 2 + pkg/scheduler/test/mock/core.go | 14 ++ 29 files changed, 814 insertions(+), 157 deletions(-) create mode 100644 pkg/hostman/isolated_device/nic.go diff --git a/pkg/apis/compute/api.go b/pkg/apis/compute/api.go index 16cfe5da77..d9eb0c45c8 100644 --- a/pkg/apis/compute/api.go +++ b/pkg/apis/compute/api.go @@ -69,6 +69,9 @@ type NetworkConfig struct { NetType string `json:"net_type"` NumQueues int `json:"num_queues"` + // sriov nic + SriovDevice *IsolatedDeviceConfig `json:"srive_device"` + RequireDesignatedIP bool `json:"require_designated_ip"` RequireTeaming bool `json:"require_teaming"` @@ -219,11 +222,13 @@ type DiskConfig struct { } type IsolatedDeviceConfig struct { - Index int `json:"index"` - Id string `json:"id"` - DevType string `json:"dev_type"` - Model string `json:"model"` - Vendor string `json:"vendor"` + Index int `json:"index"` + Id string `json:"id"` + DevType string `json:"dev_type"` + Model string `json:"model"` + Vendor string `json:"vendor"` + NetworkIndex *int8 `json:"network_index"` + WireId string `json:"wire_id"` } type BaremetalDiskConfig struct { diff --git a/pkg/apis/compute/isolated_device.go b/pkg/apis/compute/isolated_device.go index cb92f58240..be704e3358 100644 --- a/pkg/apis/compute/isolated_device.go +++ b/pkg/apis/compute/isolated_device.go @@ -107,4 +107,5 @@ type IsolatedDeviceJsonDesc struct { Addr string `json:"addr"` VendorDeviceId string `json:"vendor_device_id"` Vendor string `json:"vendor"` + NetworkIndex int8 `json:"network_index"` } diff --git a/pkg/apis/compute/network_const.go b/pkg/apis/compute/network_const.go index 78f06a4816..11d714b64d 100644 --- a/pkg/apis/compute/network_const.go +++ b/pkg/apis/compute/network_const.go @@ -53,6 +53,13 @@ const ( NETWORK_STATUS_DELETE_FAILED = "delete_failed" ) +const ( + NETWORK_DRIVER_VIRTIO = "virtio" + NETWORK_DRIVER_E1000 = "e1000" + NETWORK_DRIVER_VMXNET3 = "vmxnet3" + NETWORK_DRIVER_VFIO = "vfio-pci" +) + var ( ALL_NETWORK_TYPES = []string{ NETWORK_TYPE_GUEST, diff --git a/pkg/cloudcommon/cmdline/parser.go b/pkg/cloudcommon/cmdline/parser.go index ddbd8bebca..e6875b673e 100644 --- a/pkg/cloudcommon/cmdline/parser.go +++ b/pkg/cloudcommon/cmdline/parser.go @@ -261,6 +261,14 @@ func ParseNetworkConfig(desc string, idx int) (*compute.NetworkConfig, error) { netConfig.BwLimit = bw } else if p == "[vip]" { netConfig.Vip = true + } else if strings.HasPrefix(p, "sriov-nic-id=") { + netConfig.SriovDevice = &compute.IsolatedDeviceConfig{ + Id: p[len("sriov-nic-id="):], + } + } else if strings.HasPrefix(p, "sriov-nic-model=") { + netConfig.SriovDevice = &compute.IsolatedDeviceConfig{ + Model: p[len("sriov-nic-model="):], + } } else if utils.IsInStringArray(p, compute.ALL_NETWORK_TYPES) { netConfig.NetType = p } else { diff --git a/pkg/compute/guestdrivers/virtualization.go b/pkg/compute/guestdrivers/virtualization.go index 266ff55bce..c0ab50952d 100644 --- a/pkg/compute/guestdrivers/virtualization.go +++ b/pkg/compute/guestdrivers/virtualization.go @@ -109,6 +109,25 @@ func (self *SVirtualizedGuestDriver) Attach2RandomNetwork(guest *models.SGuest, if len(netConfig.NetType) > 0 { netTypes = []string{netConfig.NetType} } + + var sriovWires []string + if netConfig.SriovDevice != nil { + if netConfig.SriovDevice.Id != "" { + idev, err := models.IsolatedDeviceManager.FetchById(netConfig.SriovDevice.Id) + if err != nil { + return nil, errors.Wrap(err, "fetch isolated device") + } + dev, _ := idev.(*models.SIsolatedDevice) + sriovWires = []string{dev.WireId} + } else { + wires, err := models.IsolatedDeviceManager.FindUnusedNicWiresByModel(netConfig.SriovDevice.Model) + if err != nil { + return nil, errors.Wrap(err, "FindUnusedNicWiresByModel") + } + sriovWires = wires + } + } + for i := 0; i < len(hostwires); i += 1 { hostwire := hostwires[i] wire := hostwire.GetWire() @@ -123,6 +142,10 @@ func (self *SVirtualizedGuestDriver) Attach2RandomNetwork(guest *models.SGuest, continue } + if netConfig.SriovDevice != nil && !utils.IsInStringArray(wire.Id, sriovWires) { + continue + } + // !! if wirePattern != nil && !wirePattern.MatchString(wire.Id) && !wirePattern.MatchString(wire.Name) { continue diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index ec2c089783..7808674a0a 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -1827,7 +1827,7 @@ func (self *SGuest) PerformDetachIsolatedDevice(ctx context.Context, userCred mc logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) return nil, httperrors.NewBadRequestError(msg) } - err = self.startDetachIsolateDevice(ctx, userCred, device) + err = self.startDetachIsolateDeviceWithoutNic(ctx, userCred, device) if err != nil { return nil, err } @@ -1837,6 +1837,9 @@ func (self *SGuest) PerformDetachIsolatedDevice(ctx context.Context, userCred mc lockman.LockObject(ctx, host) defer lockman.ReleaseObject(ctx, host) for i := 0; i < len(devs); i++ { + if devs[i].DevType == api.NIC_TYPE { + continue + } err := self.detachIsolateDevice(ctx, userCred, &devs[i]) if err != nil { return nil, err @@ -1846,7 +1849,7 @@ func (self *SGuest) PerformDetachIsolatedDevice(ctx context.Context, userCred mc return nil, self.startIsolatedDevicesSyncTask(ctx, userCred, jsonutils.QueryBoolean(data, "auto_start", false), "") } -func (self *SGuest) startDetachIsolateDevice(ctx context.Context, userCred mcclient.TokenCredential, device string) error { +func (self *SGuest) startDetachIsolateDeviceWithoutNic(ctx context.Context, userCred mcclient.TokenCredential, device string) error { iDev, err := IsolatedDeviceManager.FetchByIdOrName(userCred, device) if err != nil { msgFmt := "Isolated device %s not found" @@ -1855,6 +1858,9 @@ func (self *SGuest) startDetachIsolateDevice(ctx context.Context, userCred mccli return httperrors.NewBadRequestError(msgFmt, device) } dev := iDev.(*SIsolatedDevice) + if dev.DevType == api.NIC_TYPE { + return httperrors.NewBadRequestError("Can't separately detach dev type %s", api.NIC_TYPE) + } if dev.IsGPU() && !utils.IsInStringArray(self.GetStatus(), []string{api.VM_READY, api.VM_RUNNING}) { return httperrors.NewInvalidStatusError("Can't detach GPU when status is %q", self.GetStatus()) } @@ -1873,6 +1879,7 @@ func (self *SGuest) detachIsolateDevice(ctx context.Context, userCred mcclient.T } _, err := db.Update(dev, func() error { dev.GuestId = "" + dev.NetworkIndex = -1 return nil }) if err != nil { @@ -1895,7 +1902,7 @@ func (self *SGuest) PerformAttachIsolatedDevice(ctx context.Context, userCred mc autoStart := jsonutils.QueryBoolean(data, "auto_start", false) if data.Contains("device") { device, _ := data.GetString("device") - err = self.StartAttachIsolatedDevice(ctx, userCred, device, autoStart) + err = self.StartAttachIsolatedDeviceWithoutNic(ctx, userCred, device, autoStart) } else if data.Contains("model") { vmodel, _ := data.GetString("model") var count int64 = 1 @@ -1937,7 +1944,7 @@ func (self *SGuest) startAttachIsolatedDevices(ctx context.Context, userCred mcc } defer func() { go host.ClearSchedDescCache() }() for i := 0; i < len(devs); i++ { - err = self.attachIsolatedDevice(ctx, userCred, &devs[i]) + err = self.attachIsolatedDevice(ctx, userCred, &devs[i], nil) if err != nil { return err } @@ -1945,15 +1952,15 @@ func (self *SGuest) startAttachIsolatedDevices(ctx context.Context, userCred mcc return nil } -func (self *SGuest) StartAttachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, device string, autoStart bool) error { - if err := self.startAttachIsolatedDevice(ctx, userCred, device); err != nil { +func (self *SGuest) StartAttachIsolatedDeviceWithoutNic(ctx context.Context, userCred mcclient.TokenCredential, device string, autoStart bool) error { + if err := self.startAttachIsolatedDeviceWithoutNic(ctx, userCred, device); err != nil { return err } // perform post attach task return self.startIsolatedDevicesSyncTask(ctx, userCred, autoStart, "") } -func (self *SGuest) startAttachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, device string) error { +func (self *SGuest) startAttachIsolatedDeviceWithoutNic(ctx context.Context, userCred mcclient.TokenCredential, device string) error { iDev, err := IsolatedDeviceManager.FetchByIdOrName(userCred, device) if err != nil { msgFmt := "Isolated device %s not found" @@ -1962,13 +1969,16 @@ func (self *SGuest) startAttachIsolatedDevice(ctx context.Context, userCred mccl return httperrors.NewBadRequestError(msgFmt, device) } dev := iDev.(*SIsolatedDevice) + if dev.DevType == api.NIC_TYPE { + return httperrors.NewBadRequestError("Can't separately attach dev type %s", api.NIC_TYPE) + } if dev.IsGPU() && !utils.IsInStringArray(self.GetStatus(), []string{api.VM_READY, api.VM_RUNNING}) { return httperrors.NewInvalidStatusError("Can't attach GPU when status is %q", self.GetStatus()) } host, _ := self.GetHost() lockman.LockObject(ctx, host) defer lockman.ReleaseObject(ctx, host) - err = self.attachIsolatedDevice(ctx, userCred, dev) + err = self.attachIsolatedDevice(ctx, userCred, dev, nil) var msg string if err != nil { msg = err.Error() @@ -1979,7 +1989,7 @@ func (self *SGuest) startAttachIsolatedDevice(ctx context.Context, userCred mccl return err } -func (self *SGuest) attachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, dev *SIsolatedDevice) error { +func (self *SGuest) attachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, dev *SIsolatedDevice, networkIndex *int8) error { if len(dev.GuestId) > 0 { return fmt.Errorf("Isolated device already attached to another guest: %s", dev.GuestId) } @@ -1989,6 +1999,11 @@ func (self *SGuest) attachIsolatedDevice(ctx context.Context, userCred mcclient. } _, err := db.Update(dev, func() error { dev.GuestId = self.Id + if networkIndex != nil { + dev.NetworkIndex = *networkIndex + } else { + dev.NetworkIndex = -1 + } return nil }) if err != nil { @@ -2033,13 +2048,13 @@ func (self *SGuest) PerformSetIsolatedDevice(ctx context.Context, userCred mccli // detach first for i := 0; i < len(delDevs); i++ { - err := self.startDetachIsolateDevice(ctx, userCred, delDevs[i]) + err := self.startDetachIsolateDeviceWithoutNic(ctx, userCred, delDevs[i]) if err != nil { return nil, err } } for i := 0; i < len(addDevs); i++ { - err := self.startAttachIsolatedDevice(ctx, userCred, addDevs[i]) + err := self.startAttachIsolatedDeviceWithoutNic(ctx, userCred, addDevs[i]) if err != nil { return nil, err } @@ -2292,7 +2307,7 @@ func (self *SGuest) PerformAttachnetwork(ctx context.Context, userCred mcclient. if count == 0 { return nil, httperrors.NewMissingParameterError("nets") } - var inicCnt, enicCnt int + var inicCnt, enicCnt, isolatedDevCount int for i := 0; i < count; i++ { err := isValidNetworkInfo(ctx, userCred, input.Nets[i], "") if err != nil { @@ -2305,6 +2320,22 @@ func (self *SGuest) PerformAttachnetwork(ctx context.Context, userCred mcclient. inicCnt = count // ibw = input.BwLimit } + if input.Nets[i].SriovDevice != nil { + if self.BackupHostId != "" { + return nil, httperrors.NewBadRequestError("Cannot create backup with isolated device") + } + devConfig, err := IsolatedDeviceManager.parseDeviceInfo(userCred, input.Nets[i].SriovDevice) + if err != nil { + return nil, httperrors.NewInputParameterError("parse isolated device description error %s", err) + } + err = IsolatedDeviceManager.isValidNicDeviceinfo(devConfig) + if err != nil { + return nil, err + } + input.Nets[i].SriovDevice = devConfig + input.Nets[i].Driver = api.NETWORK_DRIVER_VFIO + isolatedDevCount += 1 + } } pendingUsage := &SRegionQuota{ @@ -2313,24 +2344,44 @@ func (self *SGuest) PerformAttachnetwork(ctx context.Context, userCred mcclient. //Bw: ibw, //Ebw: ebw, } + pendingUsageHost := &SQuota{IsolatedDevice: isolatedDevCount} keys, err := self.GetRegionalQuotaKeys() if err != nil { return nil, err } + quotakeys, err := self.GetQuotaKeys() + if err != nil { + return nil, err + } + pendingUsageHost.SetKeys(quotakeys) pendingUsage.SetKeys(keys) err = quotas.CheckSetPendingQuota(ctx, userCred, pendingUsage) if err != nil { return nil, httperrors.NewOutOfQuotaError("%v", err) } + err = quotas.CheckSetPendingQuota(ctx, userCred, pendingUsageHost) + if err != nil { + return nil, httperrors.NewOutOfQuotaError("%v", err) + } host, _ := self.GetHost() defer host.ClearSchedDescCache() for i := 0; i < count; i++ { - _, err = self.attach2NetworkDesc(ctx, userCred, host, input.Nets[i], pendingUsage, nil) + gns, err := self.attach2NetworkDesc(ctx, userCred, host, input.Nets[i], pendingUsage, nil) logclient.AddSimpleActionLog(self, logclient.ACT_ATTACH_NETWORK, input.Nets[i], userCred, err == nil) if err != nil { quotas.CancelPendingUsage(ctx, userCred, pendingUsage, pendingUsage, false) return nil, httperrors.NewBadRequestError("%v", err) } + net := gns[0].GetNetwork() + if input.Nets[i].SriovDevice != nil { + input.Nets[i].SriovDevice.NetworkIndex = &gns[0].Index + input.Nets[i].SriovDevice.WireId = net.WireId + err = self.createIsolatedDeviceOnHost(ctx, userCred, host, input.Nets[i].SriovDevice, pendingUsageHost) + if err != nil { + quotas.CancelPendingUsage(ctx, userCred, pendingUsageHost, pendingUsageHost, false) + return nil, errors.Wrap(err, "self.createIsolatedDeviceOnHost") + } + } } if self.Status == api.VM_READY { @@ -2340,6 +2391,7 @@ func (self *SGuest) PerformAttachnetwork(ctx context.Context, userCred mcclient. } if err != nil { quotas.CancelPendingUsage(ctx, userCred, pendingUsage, pendingUsage, false) + quotas.CancelPendingUsage(ctx, userCred, pendingUsageHost, pendingUsageHost, false) } return nil, err } @@ -2767,7 +2819,7 @@ func (self *SGuest) isNotRunningStatus(status string) bool { } func (self *SGuest) SetStatus(userCred mcclient.TokenCredential, status, reason string) error { - if utils.IsInStringArray(status, api.VM_RUNNING_STATUS) { + if status == api.VM_RUNNING { if err := self.SetPowerStates(api.VM_POWER_STATES_ON); err != nil { return err } diff --git a/pkg/compute/models/guestnetworks.go b/pkg/compute/models/guestnetworks.go index 22767e620e..3f46079d40 100644 --- a/pkg/compute/models/guestnetworks.go +++ b/pkg/compute/models/guestnetworks.go @@ -270,7 +270,7 @@ func (manager *SGuestnetworkManager) newGuestNetwork( gn.Index = index gn.Virtual = virtual if len(driver) == 0 { - driver = "virtio" + driver = api.NETWORK_DRIVER_VIRTIO } gn.Driver = driver gn.NumQueues = numQueues @@ -639,6 +639,10 @@ func (manager *SGuestnetworkManager) DeleteGuestNics(ctx context.Context, userCr return errors.Wrapf(httperrors.ErrInvalidStatus, "eip associate with %s", gn.IpAddr) } guest := gn.GetGuest() + dev, err := guest.GetIsolatedDeviceByNetworkIndex(gn.Index) + if err != nil { + return errors.Wrap(err, "GetIsolatedDeviceByNetworkIndex") + } net := gn.GetNetwork() if regutils.MatchIP4Addr(gn.IpAddr) || regutils.MatchIP6Addr(gn.Ip6Addr) { net.updateDnsRecord(&gn, false) @@ -647,13 +651,18 @@ func (manager *SGuestnetworkManager) DeleteGuestNics(ctx context.Context, userCr // netman.get_manager().netmap_remove_node(gn.ip_addr) } } - // ?? - // gn.Delete(ctx, userCred) - err := gn.Delete(ctx, userCred) + err = gn.Delete(ctx, userCred) if err != nil { log.Errorf("%s", err) } gn.LogDetachEvent(ctx, userCred, guest, net) + if dev != nil { + err = guest.detachIsolateDevice(ctx, userCred, dev) + if err != nil { + return err + } + } + if reserve && regutils.MatchIP4Addr(gn.IpAddr) { ReservedipManager.ReserveIP(userCred, net, gn.IpAddr, "Delete to reserve") } diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index e4026bcd39..922c8cce4f 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -1625,6 +1625,22 @@ func (manager *SGuestManager) validateCreateData( if len(netConfig.Driver) == 0 { netConfig.Driver = osProf.NetDriver } + if netConfig.SriovDevice != nil { + if input.Backup { + return nil, httperrors.NewBadRequestError("Cannot create backup with isolated device") + } + devConfig, err := IsolatedDeviceManager.parseDeviceInfo(userCred, netConfig.SriovDevice) + if err != nil { + return nil, httperrors.NewInputParameterError("parse isolated device description error %s", err) + } + err = IsolatedDeviceManager.isValidNicDeviceinfo(devConfig) + if err != nil { + return nil, err + } + netConfig.SriovDevice = devConfig + netConfig.Driver = api.NETWORK_DRIVER_VFIO + } + netConfig.Project = ownerId.GetProjectId() netConfig.Domain = ownerId.GetProjectDomainId() input.Networks[idx] = netConfig @@ -1991,6 +2007,9 @@ func getGuestResourceRequirements( iNicCnt += 1 iBw += netConfig.BwLimit } + if netConfig.SriovDevice != nil { + devCount += 1 + } } if hasBackup { vcpuCount = vcpuCount * 2 @@ -3809,7 +3828,7 @@ func (self *SGuest) CreateNetworksOnHost( userCred mcclient.TokenCredential, host *SHost, netArray []*api.NetworkConfig, - pendingUsage quotas.IQuota, + pendingUsage, pendingUsageZone quotas.IQuota, candidateNets []*schedapi.CandidateNet, ) error { if len(netArray) == 0 { @@ -3837,10 +3856,20 @@ func (self *SGuest) CreateNetworksOnHost( } netConfig.NumQueues = numQueues } - _, err = self.attach2NetworkDesc(ctx, userCred, host, netConfig, pendingUsage, networkIds) + gns, err := self.attach2NetworkDesc(ctx, userCred, host, netConfig, pendingUsage, networkIds) if err != nil { return errors.Wrap(err, "self.attach2NetworkDesc") } + net := gns[0].GetNetwork() + if netConfig.SriovDevice != nil { + netConfig.SriovDevice.NetworkIndex = &gns[0].Index + netConfig.SriovDevice.WireId = net.WireId + err = self.createIsolatedDeviceOnHost(ctx, userCred, host, netConfig.SriovDevice, pendingUsageZone) + if err != nil { + return errors.Wrap(err, "self.createIsolatedDeviceOnHost") + } + } + } return nil } @@ -3894,8 +3923,29 @@ func (self *SGuest) attach2NamedNetworkDesc(ctx context.Context, userCred mcclie } if net != nil { if len(nicConfs) == 0 { - return nil, fmt.Errorf("no avaialble network interface?") + return nil, fmt.Errorf("no available network interface?") } + var sriovWires []string + if netConfig.SriovDevice != nil { + if netConfig.SriovDevice.Id != "" { + idev, err := IsolatedDeviceManager.FetchById(netConfig.SriovDevice.Id) + if err != nil { + return nil, errors.Wrap(err, "fetch isolated device") + } + dev, _ := idev.(*SIsolatedDevice) + sriovWires = []string{dev.WireId} + } else { + wires, err := IsolatedDeviceManager.FindUnusedNicWiresByModel(netConfig.SriovDevice.Model) + if err != nil { + return nil, errors.Wrap(err, "FindUnusedNicWiresByModel") + } + sriovWires = wires + } + if !utils.IsInStringArray(net.WireId, sriovWires) { + return nil, fmt.Errorf("no available sriov nic for wire %s", net.WireId) + } + } + gn, err := self.Attach2Network(ctx, userCred, Attach2NetworkArgs{ Network: net, PendingUsage: pendingUsage, @@ -4086,6 +4136,9 @@ func (self *SGuest) createDiskOnHost( func (self *SGuest) CreateIsolatedDeviceOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, devs []*api.IsolatedDeviceConfig, pendingUsage quotas.IQuota) error { for _, devConfig := range devs { + if devConfig.DevType == api.NIC_TYPE { + continue + } err := self.createIsolatedDeviceOnHost(ctx, userCred, host, devConfig, pendingUsage) if err != nil { return err @@ -4536,6 +4589,22 @@ func (self *SGuest) GetIsolatedDevices() ([]SIsolatedDevice, error) { return devs, nil } +func (self *SGuest) GetIsolatedDeviceByNetworkIndex(index int8) (*SIsolatedDevice, error) { + dev := SIsolatedDevice{} + q := IsolatedDeviceManager.Query().Equals("guest_id", self.Id).Equals("network_index", index) + if cnt, err := q.CountWithError(); err != nil { + return nil, err + } else if cnt == 0 { + return nil, nil + } + err := q.First(&dev) + if err != nil { + return nil, err + } + dev.SetModelManager(IsolatedDeviceManager, &dev) + return &dev, nil +} + func (self *SGuest) GetJsonDescAtHypervisor(ctx context.Context, host *SHost) *api.GuestJsonDesc { desc := &api.GuestJsonDesc{ Name: self.Name, diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index 15ab8c48ab..bbd16c5324 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -89,6 +89,10 @@ type SIsolatedDevice struct { // 云主机Id GuestId string `width:"36" charset:"ascii" nullable:"true" index:"true" list:"domain"` + // guest network index + NetworkIndex int8 `nullable:"true" default:"-1" list:"user" update:"user"` + // Nic wire id + WireId string `width:"36" charset:"ascii" nullable:"true" index:"true" list:"domain" update:"domain" create:"domain_optional"` // # pci address of `Bus:Device.Function` format, or usb bus address of `bus.addr` Addr string `width:"16" charset:"ascii" nullable:"true" list:"domain" update:"domain" create:"domain_optional"` @@ -410,6 +414,7 @@ func (manager *SIsolatedDeviceManager) parseDeviceInfo(userCred mcclient.TokenCr devConfig.Model = dev.Model devConfig.DevType = dev.DevType devConfig.Vendor = dev.getVendor() + devConfig.WireId = dev.WireId if dev.IsGPU() && len(devType) > 0 { if !utils.IsInStringArray(devType, VALID_GPU_TYPES) { return nil, fmt.Errorf("%s not valid for GPU device", devType) @@ -436,6 +441,23 @@ func (manager *SIsolatedDeviceManager) isValidDeviceinfo(config *api.IsolatedDev return nil } +func (manager *SIsolatedDeviceManager) isValidNicDeviceinfo(config *api.IsolatedDeviceConfig) error { + if len(config.Id) > 0 { + devObj, err := manager.FetchById(config.Id) + if err != nil { + return httperrors.NewResourceNotFoundError("IsolatedDevice %s not found", config.Id) + } + dev := devObj.(*SIsolatedDevice) + if len(dev.GuestId) > 0 { + return httperrors.NewConflictError("Isolated device already attached to another guest: %s", dev.GuestId) + } + if dev.DevType != api.NIC_TYPE { + return httperrors.NewBadRequestError("IsolatedDevice is not device type %s", api.NIC_TYPE) + } + } + return nil +} + func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDesc(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error { if len(devConfig.Id) > 0 { return manager.attachSpecificDeviceToGuest(ctx, guest, devConfig, userCred) @@ -453,19 +475,20 @@ func (manager *SIsolatedDeviceManager) attachSpecificDeviceToGuest(ctx context.C if len(devConfig.DevType) > 0 && devConfig.DevType != dev.DevType { dev.DevType = devConfig.DevType } - return guest.attachIsolatedDevice(ctx, userCred, dev) + return guest.attachIsolatedDevice(ctx, userCred, dev, devConfig.NetworkIndex) } func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByModel(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error { if len(devConfig.Model) == 0 { return fmt.Errorf("Not found model from info: %#v", devConfig) } - devs, err := manager.findHostUnusedByModel(devConfig.Model, host.Id) + // if dev type is not nic, wire is empty string + devs, err := manager.findHostUnusedByModelAndWire(devConfig.Model, host.Id, devConfig.WireId) if err != nil || len(devs) == 0 { - return fmt.Errorf("Can't found %s model on host", host.Id) + return fmt.Errorf("Can't found model %s on host %s", devConfig.Model, host.Id) } selectedDev := devs[0] - return guest.attachIsolatedDevice(ctx, userCred, &selectedDev) + return guest.attachIsolatedDevice(ctx, userCred, &selectedDev, devConfig.NetworkIndex) } func (manager *SIsolatedDeviceManager) findUnusedQuery() *sqlchemy.SQuery { @@ -494,6 +517,24 @@ func (manager *SIsolatedDeviceManager) FindUnusedByModels(models []string) ([]SI return devs, nil } +func (manager *SIsolatedDeviceManager) FindUnusedNicWiresByModel(modelName string) ([]string, error) { + q := manager.Query().IsNullOrEmpty("guest_id").Equals("dev_type", api.NIC_TYPE) + if len(modelName) > 0 { + q = q.Equals("model", modelName) + } + q = q.GroupBy("wire_id") + devs := make([]SIsolatedDevice, 0) + err := q.All(&devs) + if err != nil { + return nil, err + } + wires := make([]string, len(devs)) + for i := 0; i < len(devs); i++ { + wires[i] = devs[i].WireId + } + return wires, err +} + func (manager *SIsolatedDeviceManager) FindUnusedGpusOnHost(hostId string) ([]SIsolatedDevice, error) { devs := make([]SIsolatedDevice, 0) q := manager.UnusedGpuQuery() @@ -505,10 +546,10 @@ func (manager *SIsolatedDeviceManager) FindUnusedGpusOnHost(hostId string) ([]SI return devs, nil } -func (manager *SIsolatedDeviceManager) findHostUnusedByModel(model string, hostId string) ([]SIsolatedDevice, error) { +func (manager *SIsolatedDeviceManager) findHostUnusedByModelAndWire(model, hostId, wireId string) ([]SIsolatedDevice, error) { devs := make([]SIsolatedDevice, 0) q := manager.findUnusedQuery() - q = q.Equals("model", model).Equals("host_id", hostId) + q = q.Equals("model", model).Equals("host_id", hostId).Equals("wire_id", wireId) err := db.FetchModelObjects(manager, q, &devs) if err != nil { return nil, err @@ -524,6 +565,7 @@ func (manager *SIsolatedDeviceManager) ReleaseDevicesOfGuest(ctx context.Context for _, dev := range devs { _, err := db.Update(&dev, func() error { dev.GuestId = "" + dev.NetworkIndex = -1 return nil }) if err != nil { @@ -627,6 +669,7 @@ func (self *SIsolatedDevice) getDesc() *api.IsolatedDeviceJsonDesc { Addr: self.Addr, VendorDeviceId: self.VendorDeviceId, Vendor: self.getVendor(), + NetworkIndex: self.NetworkIndex, } } @@ -849,3 +892,11 @@ func (model *SIsolatedDevice) GetOwnerId() mcclient.IIdentityProvider { } return nil } + +func (model *SIsolatedDevice) SetNetworkIndex(idx int8) error { + _, err := db.Update(model, func() error { + model.NetworkIndex = idx + return nil + }) + return err +} diff --git a/pkg/compute/tasks/guest_batch_create_task.go b/pkg/compute/tasks/guest_batch_create_task.go index bb89b1229d..abb5778fe1 100644 --- a/pkg/compute/tasks/guest_batch_create_task.go +++ b/pkg/compute/tasks/guest_batch_create_task.go @@ -174,7 +174,8 @@ func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest pendingRegionUsage := models.SRegionQuota{} self.GetPendingUsage(&pendingRegionUsage, 1) // allocate networks - err = guest.CreateNetworksOnHost(ctx, self.UserCred, host, input.Networks, &pendingRegionUsage, candidate.Nets) + err = guest.CreateNetworksOnHost(ctx, self.UserCred, host, input.Networks, &pendingRegionUsage, &pendingUsage, candidate.Nets) + self.SetPendingUsage(&pendingUsage, 0) self.SetPendingUsage(&pendingRegionUsage, 1) if err != nil { log.Errorf("Network failed: %s", err) diff --git a/pkg/hostman/guestman/desc/desc.go b/pkg/hostman/guestman/desc/desc.go index 3c69fb3f87..f294c26f51 100644 --- a/pkg/hostman/guestman/desc/desc.go +++ b/pkg/hostman/guestman/desc/desc.go @@ -323,6 +323,8 @@ type SGuestControlDesc struct { ScalingGroupId string SecurityRules string AdminSecurityRules string + SrcIpCheck bool + SrcMacCheck bool EncryptKeyId string } diff --git a/pkg/hostman/guestman/desc/pci.go b/pkg/hostman/guestman/desc/pci.go index 4d93ea077e..cc2404ed74 100644 --- a/pkg/hostman/guestman/desc/pci.go +++ b/pkg/hostman/guestman/desc/pci.go @@ -323,6 +323,14 @@ func NewPCIDevice(controller PCI_CONTROLLER_TYPE, deviceType, id string) *PCIDev } } +func NewVfioDevice(controller PCI_CONTROLLER_TYPE, deviceType, id, hostAddr string, hasXVga bool) *VFIODevice { + return &VFIODevice{ + PCIDevice: NewPCIDevice(controller, deviceType, id), + HostAddr: hostAddr, + XVga: hasXVga, + } +} + func NewScsiDevice(controller, deviceType, id string) *SCSIDevice { return &SCSIDevice{ ControllerId: controller, diff --git a/pkg/hostman/guestman/guestman.go b/pkg/hostman/guestman/guestman.go index b212fbf409..af2c00d663 100644 --- a/pkg/hostman/guestman/guestman.go +++ b/pkg/hostman/guestman/guestman.go @@ -766,7 +766,9 @@ func (m *SGuestManager) StatusWithBlockJobsCount(ctx context.Context, params int return nil, nil } body := jsonutils.NewDict() - body.Set("power_status", jsonutils.NewString(guest.GetPowerStates())) + if guest != nil { + body.Set("power_status", jsonutils.NewString(guest.GetPowerStates())) + } body.Set("status", jsonutils.NewString(status)) hostutils.TaskComplete(ctx, body) return nil, nil diff --git a/pkg/hostman/guestman/guesttasks.go b/pkg/hostman/guestman/guesttasks.go index bb03b49634..e2d8a41ef2 100644 --- a/pkg/hostman/guestman/guesttasks.go +++ b/pkg/hostman/guestman/guesttasks.go @@ -642,6 +642,12 @@ func (n *SGuestNetworkSyncTask) delNicDevice(nic *desc.SGuestNetwork) { } func (n *SGuestNetworkSyncTask) addNic(nic *desc.SGuestNetwork) { + if nic.Driver == "vfio-pci" { + // vfio device will add on isolated devices sync task + n.onDeviceAdd(nic) + return + } + if err := n.guest.generateNicScripts(nic); err != nil { log.Errorln(err) n.errors = append(n.errors, err) @@ -822,15 +828,28 @@ func (t *SGuestIsolatedDeviceSyncTask) removeDevice(dev *desc.SGuestIsolatedDevi } func (t *SGuestIsolatedDeviceSyncTask) addDevice(dev *desc.SGuestIsolatedDevice) { + var err error devObj := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByIdent(dev.VendorDeviceId, dev.Addr) if devObj == nil { - err := errors.Errorf("Not found host isolated_device by %s %s", dev.VendorDeviceId, dev.Addr) + err = errors.Errorf("Not found host isolated_device by %s %s", dev.VendorDeviceId, dev.Addr) log.Errorln(err) t.errors = append(t.errors, err) t.syncDevice() return } + var setupScript string + if dev.DevType == api.NIC_TYPE { + setupScript, err = t.guest.sriovNicAttachInitScript(dev.NetworkIndex, devObj) + if err != nil { + err = errors.Errorf("sriovNicAttachInitScript %s", err) + log.Errorln(err) + t.errors = append(t.errors, err) + t.syncDevice() + return + } + } + onFail := func(err error) { for i := 0; i < len(dev.VfioDevs); i++ { if dev.VfioDevs[i].PCIAddr != nil { @@ -858,22 +877,15 @@ func (t *SGuestIsolatedDeviceSyncTask) addDevice(dev *desc.SGuestIsolatedDevice) } id := devObj.GetQemuId() dev.VfioDevs = make([]*desc.VFIODevice, 0) - vfioDev := &desc.VFIODevice{ - PCIDevice: desc.NewPCIDevice(pciRoot.CType, "vfio-pci", id), - } + vfioDev := desc.NewVfioDevice( + pciRoot.CType, "vfio-pci", id, devObj.GetAddr(), devObj.GetDeviceType() == api.GPU_VGA_TYPE, + ) dev.VfioDevs = append(dev.VfioDevs, vfioDev) - dev.VfioDevs[0].HostAddr = devObj.GetAddr() - if devObj.GetDeviceType() == api.GPU_VGA_TYPE { - dev.VfioDevs[0].XVga = true - } groupDevAddrs := devObj.GetIOMMUGroupRestAddrs() for j := 0; j < len(groupDevAddrs); j++ { gid := fmt.Sprintf("%s-%d", id, j+1) - vfioDev = &desc.VFIODevice{ - PCIDevice: desc.NewPCIDevice(pciRoot.CType, "vfio-pci", gid), - } - vfioDev.HostAddr = groupDevAddrs[j] + vfioDev = desc.NewVfioDevice(pciRoot.CType, "vfio-pci", gid, groupDevAddrs[j], false) dev.VfioDevs = append(dev.VfioDevs, vfioDev) } multiFunc := true @@ -905,6 +917,13 @@ func (t *SGuestIsolatedDeviceSyncTask) addDevice(dev *desc.SGuestIsolatedDevice) } } + if len(setupScript) > 0 { + output, err := procutils.NewRemoteCommandAsFarAsPossible("sh", "-c", setupScript).Output() + if err != nil { + log.Errorf("isolated device setup error %s, %s", output, err) + } + } + opts, err := devObj.GetHotPlugOptions(dev) if err != nil { cb(errors.Wrap(err, "GetHotPlugOptions").Error()) diff --git a/pkg/hostman/guestman/pci.go b/pkg/hostman/guestman/pci.go index 692d009886..6ac9e43680 100644 --- a/pkg/hostman/guestman/pci.go +++ b/pkg/hostman/guestman/pci.go @@ -257,34 +257,35 @@ func (s *SKVMGuestInstance) initGuestNetworks(pciRoot, pciBridge *desc.PCIContro } for i := 0; i < len(s.Desc.Nics); i++ { - switch s.GetOsName() { - case OS_NAME_MACOS: - vectors := 0 - s.Desc.Nics[i].Vectors = &vectors - s.Desc.Nics[i].Driver = "e1000" - case OS_NAME_VMWARE: - s.Desc.Nics[i].Driver = "vmxnet3" - } + if s.Desc.Nics[i].Driver != "vfio-pci" { + switch s.GetOsName() { + case OS_NAME_MACOS: + vectors := 0 + s.Desc.Nics[i].Vectors = &vectors + s.Desc.Nics[i].Driver = "e1000" + case OS_NAME_VMWARE: + s.Desc.Nics[i].Driver = "vmxnet3" + } + if s.Desc.Nics[i].NumQueues > 1 { + vectors := s.Desc.Nics[i].NumQueues * 2 + s.Desc.Nics[i].Vectors = &vectors + } - if s.Desc.Nics[i].NumQueues > 1 { - vectors := s.Desc.Nics[i].NumQueues * 2 - s.Desc.Nics[i].Vectors = &vectors - } + if err := s.generateNicScripts(s.Desc.Nics[i]); err != nil { + return errors.Wrapf(err, "generateNicScripts for nic: %v", s.Desc.Nics[i]) + } + s.Desc.Nics[i].UpscriptPath = s.getNicUpScriptPath(s.Desc.Nics[i]) + s.Desc.Nics[i].DownscriptPath = s.getNicDownScriptPath(s.Desc.Nics[i]) - if err := s.generateNicScripts(s.Desc.Nics[i]); err != nil { - return errors.Wrapf(err, "generateNicScripts for nic: %v", s.Desc.Nics[i]) - } - s.Desc.Nics[i].UpscriptPath = s.getNicUpScriptPath(s.Desc.Nics[i]) - s.Desc.Nics[i].DownscriptPath = s.getNicDownScriptPath(s.Desc.Nics[i]) - - id := fmt.Sprintf("netdev-%s", s.Desc.Nics[i].Ifname) - switch s.Desc.Nics[i].Driver { - case "virtio": - s.Desc.Nics[i].Pci = desc.NewPCIDevice(cont.CType, "virtio-net-pci", id) - case "e1000": - s.Desc.Nics[i].Pci = desc.NewPCIDevice(cont.CType, "e1000-82545em", id) - case "vmxnet3": - s.Desc.Nics[i].Pci = desc.NewPCIDevice(cont.CType, "vmxnet3", id) + id := fmt.Sprintf("netdev-%s", s.Desc.Nics[i].Ifname) + switch s.Desc.Nics[i].Driver { + case "virtio": + s.Desc.Nics[i].Pci = desc.NewPCIDevice(cont.CType, "virtio-net-pci", id) + case "e1000": + s.Desc.Nics[i].Pci = desc.NewPCIDevice(cont.CType, "e1000-82545em", id) + case "vmxnet3": + s.Desc.Nics[i].Pci = desc.NewPCIDevice(cont.CType, "vmxnet3", id) + } } } return nil @@ -304,24 +305,15 @@ func (s *SKVMGuestInstance) initIsolatedDevices(pciRoot, pciBridge *desc.PCICont } else { id := dev.GetQemuId() s.Desc.IsolatedDevices[i].VfioDevs = make([]*desc.VFIODevice, 0) - vfioDev := &desc.VFIODevice{ - PCIDevice: desc.NewPCIDevice(cont.CType, "vfio-pci", id), - } - s.Desc.IsolatedDevices[i].VfioDevs = append( - s.Desc.IsolatedDevices[i].VfioDevs, vfioDev, + vfioDev := desc.NewVfioDevice( + cont.CType, "vfio-pci", id, dev.GetAddr(), dev.GetDeviceType() == api.GPU_VGA_TYPE, ) - s.Desc.IsolatedDevices[i].VfioDevs[0].HostAddr = dev.GetAddr() - if dev.GetDeviceType() == api.GPU_VGA_TYPE { - s.Desc.IsolatedDevices[i].VfioDevs[0].XVga = true - } + s.Desc.IsolatedDevices[i].VfioDevs = append(s.Desc.IsolatedDevices[i].VfioDevs, vfioDev) groupDevAddrs := dev.GetIOMMUGroupRestAddrs() for j := 0; j < len(groupDevAddrs); j++ { gid := fmt.Sprintf("%s-%d", id, j+1) - vfioDev = &desc.VFIODevice{ - PCIDevice: desc.NewPCIDevice(cont.CType, "vfio-pci", gid), - } - vfioDev.HostAddr = groupDevAddrs[j] + vfioDev = desc.NewVfioDevice(cont.CType, "vfio-pci", gid, groupDevAddrs[j], false) s.Desc.IsolatedDevices[i].VfioDevs = append( s.Desc.IsolatedDevices[i].VfioDevs, vfioDev, ) @@ -641,7 +633,7 @@ func (s *SKVMGuestInstance) ensurePciAddresses() error { for i := 0; i < len(s.Desc.IsolatedDevices); i++ { if len(s.Desc.IsolatedDevices[i].VfioDevs) > 0 { - multiFunc := true + multiFunc := len(s.Desc.IsolatedDevices[i].VfioDevs) > 1 err = s.ensureDevicePciAddress(s.Desc.IsolatedDevices[i].VfioDevs[0].PCIDevice, 0, &multiFunc) if err != nil { return errors.Wrapf(err, "ensure isolated device %s pci address", s.Desc.IsolatedDevices[i].VfioDevs[0].PCIAddr) diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index 8ec941a572..7e46d2817c 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -49,6 +49,7 @@ import ( "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostbridge" "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostconsts" "yunion.io/x/onecloud/pkg/hostman/hostutils" + "yunion.io/x/onecloud/pkg/hostman/isolated_device" "yunion.io/x/onecloud/pkg/hostman/monitor" "yunion.io/x/onecloud/pkg/hostman/monitor/qga" "yunion.io/x/onecloud/pkg/hostman/options" @@ -1753,6 +1754,9 @@ func (s *SKVMGuestInstance) compareDescFloppys(newDesc *desc.SGuestDesc) []*desc func (s *SKVMGuestInstance) compareDescNetworks(newDesc *desc.SGuestDesc, ) ([]*desc.SGuestNetwork, []*desc.SGuestNetwork, [][2]*desc.SGuestNetwork) { var isValid = func(net *desc.SGuestNetwork) bool { + return net.Driver == "virtio" || net.Driver == "vfio-pci" + } + var isChangeNetworkValid = func(net *desc.SGuestNetwork) bool { return net.Driver == "virtio" } @@ -1779,11 +1783,14 @@ func (s *SKVMGuestInstance) compareDescNetworks(newDesc *desc.SGuestDesc, if isValid(n) { idx := findNet(addNics, n) if idx >= 0 { - // check if bridge changed - changedNics = append(changedNics, [2]*desc.SGuestNetwork{ - n, // old - addNics[idx], // new - }) + if isChangeNetworkValid(n) { + // check if bridge changed + changedNics = append(changedNics, [2]*desc.SGuestNetwork{ + n, // old + addNics[idx], // new + }) + } + // remove existing nic from new addNics = append(addNics[:idx], addNics[idx+1:]...) } else { @@ -1916,18 +1923,19 @@ func (s *SKVMGuestInstance) SyncConfig( tasks = append(tasks, task) } - if len(delNetworks)+len(addNetworks) > 0 { - task := NewGuestNetworkSyncTask(s, delNetworks, addNetworks) - runTaskNames = append(runTaskNames, jsonutils.NewString("networksync")) - tasks = append(tasks, task) - } - if len(delDevs)+len(addDevs) > 0 { task := NewGuestIsolatedDeviceSyncTask(s, delDevs, addDevs) runTaskNames = append(runTaskNames, jsonutils.NewString("isolated_device_sync")) tasks = append(tasks, task) } + // make sure network sync before isolated device + if len(delNetworks)+len(addNetworks) > 0 { + task := NewGuestNetworkSyncTask(s, delNetworks, addNetworks) + runTaskNames = append(runTaskNames, jsonutils.NewString("networksync")) + tasks = append(tasks, task) + } + lenTasks := len(tasks) var callBack = func(errs []error) { s.SaveLiveDesc(s.Desc) @@ -2519,6 +2527,79 @@ func (s *SKVMGuestInstance) generateDiskSetupScripts(disks []*desc.SGuestDisk) ( return cmd, nil } +func (s *SKVMGuestInstance) getSriovDeviceByNetworkIndex(networkIndex int8) (isolated_device.IDevice, error) { + manager := s.manager.GetHost().GetIsolatedDeviceManager() + for i := 0; i < len(s.Desc.IsolatedDevices); i++ { + if s.Desc.IsolatedDevices[i].DevType == api.NIC_TYPE && + s.Desc.IsolatedDevices[i].NetworkIndex == networkIndex { + if dev := manager.GetDeviceByAddr(s.Desc.IsolatedDevices[i].Addr); dev == nil { + return nil, errors.Errorf("get device by addr failed") + } else { + return dev, nil + } + } + } + return nil, errors.Errorf("network index %d no sriov device found", networkIndex) +} + +var sriovInitFunc = ` +function sriov_vf_init() { + ip link set dev $1 vf $2 mac $3 vlan $4 + ip link set dev $1 vf $2 spoofchk $5 + ip link set dev $1 vf $2 max_tx_rate $6 +}` + +func srcMacCheckFunc(srcMacCheck bool) string { + if srcMacCheck { + return "on" + } + return "off" +} + +func getVfVlan(vlan int) int { + if vlan == 1 { + return 0 + } + return vlan +} + +func (s *SKVMGuestInstance) sriovNicAttachInitScript(networkIndex int8, dev isolated_device.IDevice) (string, error) { + for i := range s.Desc.Nics { + if s.Desc.Nics[i].Driver == "vfio-pci" && s.Desc.Nics[i].Index == networkIndex { + cmd := fmt.Sprintf( + "sriov_vf_init %s %d %s %d %s %d\n", + dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac, + getVfVlan(s.Desc.Nics[i].Vlan), srcMacCheckFunc(s.Desc.SrcMacCheck), s.Desc.Nics[i].Bw, + ) + return sriovInitFunc + " && " + cmd, nil + } + } + return "", errors.Errorf("no nic found for index %d", networkIndex) +} + +func (s *SKVMGuestInstance) generateSRIOVInitScripts() (string, error) { + var cmd = "" + + for i := range s.Desc.Nics { + if s.Desc.Nics[i].Driver == "vfio-pci" { + dev, err := s.getSriovDeviceByNetworkIndex(s.Desc.Nics[i].Index) + if err != nil { + return "", err + } + cmd += fmt.Sprintf( + "sriov_vf_init %s %d %s %d %s %d\n", + dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac, + getVfVlan(s.Desc.Nics[i].Vlan), srcMacCheckFunc(s.Desc.SrcMacCheck), s.Desc.Nics[i].Bw, + ) + } + } + if len(cmd) > 0 { + cmd = sriovInitFunc + "\n" + cmd + } + + return cmd, nil +} + func (s *SKVMGuestInstance) getQemuCmdline() (string, error) { content, err := fileutils2.FileGetContents(s.GetStartScriptPath()) if err != nil { diff --git a/pkg/hostman/guestman/qemu-kvmhelper.go b/pkg/hostman/guestman/qemu-kvmhelper.go index dd29fb1450..f52e1e1379 100644 --- a/pkg/hostman/guestman/qemu-kvmhelper.go +++ b/pkg/hostman/guestman/qemu-kvmhelper.go @@ -342,6 +342,9 @@ func (s *SKVMGuestInstance) generateStartScript(data *jsonutils.JSONDict) (strin } for _, nic := range s.Desc.Nics { + if nic.Driver == api.NETWORK_DRIVER_VFIO { + continue + } downscript := s.getNicDownScriptPath(nic) cmd += fmt.Sprintf("%s %s\n", downscript, nic.Ifname) } @@ -361,6 +364,12 @@ func (s *SKVMGuestInstance) generateStartScript(data *jsonutils.JSONDict) (strin } cmd += diskScripts + sriovInitScripts, err := s.generateSRIOVInitScripts() + if err != nil { + return "", errors.Wrap(err, "generateSRIOVInitScripts") + } + cmd += sriovInitScripts + cmd += fmt.Sprintf("STATE_FILE=`ls -d %s* | head -n 1`\n", s.getStateFilePathRootPrefix()) cmd += fmt.Sprintf("PID_FILE=%s\n", input.PidFilePath) @@ -616,6 +625,9 @@ func (s *SKVMGuestInstance) generateStopScript(data *jsonutils.JSONDict) string cmd += fmt.Sprintf("done\n") for _, nic := range nics { + if nic.Driver == api.NETWORK_DRIVER_VFIO { + continue + } downscript := s.getNicDownScriptPath(nic) cmd += fmt.Sprintf("%s %s\n", downscript, nic.Ifname) } diff --git a/pkg/hostman/guestman/qemu/generate.go b/pkg/hostman/guestman/qemu/generate.go index fcb190d374..e7c43f74cd 100644 --- a/pkg/hostman/guestman/qemu/generate.go +++ b/pkg/hostman/guestman/qemu/generate.go @@ -373,6 +373,10 @@ func generateNicOptions(drvOpt QemuOptions, input *GenerateStartOptionsInput) ([ nics := input.GuestDesc.Nics for idx := range nics { + if nics[idx].Driver == api.NETWORK_DRIVER_VFIO { + continue + } + netDevOpt, err := getNicNetdevOption(drvOpt, nics[idx], input.IsKVMSupport) if err != nil { return nil, errors.Wrapf(err, "getNicNetdevOption %v", nics[idx]) diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index 284a3fe864..f6db093e88 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -1510,6 +1510,10 @@ func (h *SHostInfo) getNetworkInfo() { h.uploadNetworkInfo() } +func (h *SHostInfo) isVirtualFunction(nic string) bool { + return fileutils2.Exists(path.Join("/sys/class/net", nic, "device", "physfn")) +} + func (h *SHostInfo) uploadNetworkInfo() { phyNics, err := sysutils.Nics() if err != nil { @@ -1517,6 +1521,9 @@ func (h *SHostInfo) uploadNetworkInfo() { return } for _, pnic := range phyNics { + if h.isVirtualFunction(pnic.Dev) { + continue + } err := h.doSendPhysicalNicInfo(pnic) if err != nil { h.onFail(errors.Wrapf(err, "doSendPhysicalNicInfo %s", pnic.Dev)) @@ -1828,8 +1835,19 @@ func (h *SHostInfo) probeSyncIsolatedDevicesStep() { h.deployAdminAuthorizedKeys() } +func (h *SHostInfo) getNicsInterfaces() [][2]string { + res := [][2]string{} + for i := 0; i < len(h.Nics); i++ { + res = append(res, [2]string{h.Nics[i].Inter, h.Nics[i].WireId}) + } + return res +} + func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) { - if err := h.IsolatedDeviceMan.ProbePCIDevices(options.HostOptions.DisableGPU, options.HostOptions.DisableUSB); err != nil { + if err := h.IsolatedDeviceMan.ProbePCIDevices( + options.HostOptions.DisableGPU, options.HostOptions.DisableUSB, + options.HostOptions.DisableSRIOVNic, h.getNicsInterfaces(), + ); err != nil { return nil, errors.Wrap(err, "ProbePCIDevices") } diff --git a/pkg/hostman/isolated_device/gpu.go b/pkg/hostman/isolated_device/gpu.go index 6479443b61..92659339d3 100644 --- a/pkg/hostman/isolated_device/gpu.go +++ b/pkg/hostman/isolated_device/gpu.go @@ -158,31 +158,34 @@ func (dev *sGPUBaseDevice) DetectByAddr() error { return err } -func (dev *sGPUBaseDevice) CustomProbe() error { - // vfio kernel driver check - for _, driver := range []string{"vfio", "vfio_iommu_type1", "vfio-pci"} { - if err := procutils.NewRemoteCommandAsFarAsPossible("modprobe", driver).Run(); err != nil { - return fmt.Errorf("modprobe %s: %v", driver, err) +func (dev *sGPUBaseDevice) CustomProbe(idx int) error { + // check environments on first probe + if idx == 0 { + // vfio kernel driver check + for _, driver := range []string{"vfio", "vfio_iommu_type1", "vfio-pci"} { + if err := procutils.NewRemoteCommandAsFarAsPossible("modprobe", driver).Run(); err != nil { + return fmt.Errorf("modprobe %s: %v", driver, err) + } + } + // grub check + grubCmdline, err := fileutils2.FileGetContents("/proc/cmdline") + if err != nil { + return err + } + grubCmdline = strings.TrimSpace(grubCmdline) + params := sets.NewString(strings.Split(grubCmdline, " ")...) + if !params.IsSuperset(sets.NewString("intel_iommu=on", + "vfio_iommu_type1.allow_unsafe_interrupts=1")) { + return fmt.Errorf("Some GRUB_CMDLINE iommu parameters are missing") + } + isNouveauBlacklisted := false + if params.IsSuperset(sets.NewString("rdblacklist=nouveau", "nouveau.modeset=0")) || + params.IsSuperset(sets.NewString("rd.driver.blacklist=nouveau", "nouveau.modeset=0")) { + isNouveauBlacklisted = true + } + if !isNouveauBlacklisted { + return fmt.Errorf("Some GRUB_CMDLINE nouveau_blacklisted parameters are missing") } - } - // grub check - grubCmdline, err := fileutils2.FileGetContents("/proc/cmdline") - if err != nil { - return err - } - grubCmdline = strings.TrimSpace(grubCmdline) - params := sets.NewString(strings.Split(grubCmdline, " ")...) - if !params.IsSuperset(sets.NewString("intel_iommu=on", - "vfio_iommu_type1.allow_unsafe_interrupts=1")) { - return fmt.Errorf("Some GRUB_CMDLINE iommu parameters are missing") - } - isNouveauBlacklisted := false - if params.IsSuperset(sets.NewString("rdblacklist=nouveau", "nouveau.modeset=0")) || - params.IsSuperset(sets.NewString("rd.driver.blacklist=nouveau", "nouveau.modeset=0")) { - isNouveauBlacklisted = true - } - if !isNouveauBlacklisted { - return fmt.Errorf("Some GRUB_CMDLINE nouveau_blacklisted parameters are missing") } driver, err := dev.GetKernelDriver() if err != nil { @@ -250,36 +253,11 @@ func (dev *sGPUBaseDevice) GetHotUnplugOptions(isolatedDev *desc.SGuestIsolatedD }, nil } -type sGPUVGADevice struct { - *sGPUBaseDevice -} - -func (gpu *sGPUVGADevice) GetDeviceType() string { - return api.GPU_VGA_TYPE -} - -func (gpu *sGPUVGADevice) GetVGACmd() string { - return " -vga none" -} - func getGuestAddr(index int) string { vAddr := fmt.Sprintf("0x%x", 21+index) // from 0x15 above return vAddr } -func (gpu *sGPUVGADevice) GetPassthroughCmd(index int) string { - // vAddr := getGuestAddr(index) - return fmt.Sprintf(" -device vfio-pci,host=%s,multifunction=on,x-vga=on", gpu.GetAddr()) -} - -func (gpu *sGPUVGADevice) CustomProbe() error { - _, err := bashOutput(`cat /boot/cfg-$(uname -r) | grep -E "^CONFIG_VFIO_PCI_VGA=y"`) - if err != nil { - return fmt.Errorf("CONFIG_VFIO_PCI_VGA=y needs to be set in kernel compiling parameters") - } - return nil -} - type sGPUHPCDevice struct { *sGPUBaseDevice } diff --git a/pkg/hostman/isolated_device/isolated_device.go b/pkg/hostman/isolated_device/isolated_device.go index 67cfaa9073..d0808d6a1b 100644 --- a/pkg/hostman/isolated_device/isolated_device.go +++ b/pkg/hostman/isolated_device/isolated_device.go @@ -65,11 +65,12 @@ type IDevice interface { GetHostId() string SetHostId(hId string) GetGuestId() string + GetWireId() string GetVendorDeviceId() string GetAddr() string GetDeviceType() string GetModelName() string - CustomProbe() error + CustomProbe(idx int) error SetDeviceInfo(info CloudDeviceInfo) SetDetectedOnHost(isDetected bool) DetectByAddr() error @@ -82,15 +83,19 @@ type IDevice interface { GetCPUCmd() string GetQemuId() string + // sriov nic + GetPfName() string + GetVirtfn() int + GetHotPlugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotPlugOption, error) - GetHotUnplugOptions(dev *desc.SGuestIsolatedDevice) ([]*HotUnplugOption, error) + GetHotUnplugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotUnplugOption, error) } type IsolatedDeviceManager interface { GetDevices() []IDevice GetDeviceByIdent(vendorDevId string, addr string) IDevice GetDeviceByAddr(addr string) IDevice - ProbePCIDevices(skipGPUs, skipUSBs bool) error + ProbePCIDevices(skipGPUs, skipUSBs, skipSRIOVNics bool, nics [][2]string) error StartDetachTask() BatchCustomProbe() error AppendDetachedDevice(dev *CloudDeviceInfo) @@ -117,7 +122,7 @@ func (man *isolatedDeviceManager) GetDevices() []IDevice { return man.devices } -func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs bool) error { +func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs, skipSRIOVNics bool, nics [][2]string) error { man.devices = make([]IDevice, 0) if !skipGPUs { gpus, err := getPassthroughGPUS() @@ -144,6 +149,18 @@ func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs bool) error } } + if !skipSRIOVNics { + nics, err := getSRIOVNics(nics) + if err != nil { + log.Errorf("getSRIOVNics: %v", err) + return nil + } + for idx, nic := range nics { + man.devices = append(man.devices, nic) + log.Infof("Add sriov nic: %d => %#v", idx, nic) + } + } + return nil } @@ -179,8 +196,8 @@ func (man *isolatedDeviceManager) GetDeviceByAddr(addr string) IDevice { } func (man *isolatedDeviceManager) BatchCustomProbe() error { - for _, dev := range man.devices { - if err := dev.CustomProbe(); err != nil { + for i, dev := range man.devices { + if err := dev.CustomProbe(i); err != nil { return err } } @@ -250,6 +267,10 @@ func (dev *sBaseDevice) String() string { return dev.dev.String() } +func (dev *sBaseDevice) GetWireId() string { + return "" +} + func (dev *sBaseDevice) SetDeviceInfo(info CloudDeviceInfo) { if len(info.Id) != 0 { dev.cloudId = info.Id @@ -298,8 +319,20 @@ func (dev *sBaseDevice) GetDeviceType() string { return dev.devType } +func (dev *sBaseDevice) GetPfName() string { + return "" +} + +func (dev *sBaseDevice) GetVirtfn() int { + return -1 +} + func (dev *sBaseDevice) GetModelName() string { - return dev.dev.ModelName + if dev.dev.ModelName != "" { + return dev.dev.ModelName + } else { + return dev.dev.DeviceName + } } func (dev *sBaseDevice) GetGuestId() string { @@ -327,6 +360,9 @@ func GetApiResourceData(dev IDevice) *jsonutils.JSONDict { if len(dev.GetGuestId()) != 0 { data["guest_id"] = dev.GetGuestId() } + if len(dev.GetWireId()) != 0 { + data["wire_id"] = dev.GetWireId() + } return jsonutils.Marshal(data).(*jsonutils.JSONDict) } diff --git a/pkg/hostman/isolated_device/nic.go b/pkg/hostman/isolated_device/nic.go new file mode 100644 index 0000000000..0bdfe701e7 --- /dev/null +++ b/pkg/hostman/isolated_device/nic.go @@ -0,0 +1,208 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package isolated_device + +import ( + "fmt" + "io/ioutil" + "path" + "path/filepath" + "strconv" + "strings" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/hostman/guestman/desc" + "yunion.io/x/onecloud/pkg/util/procutils" +) + +func getSRIOVNics(hostNics [][2]string) ([]*sSRIOVNicDevice, error) { + sysfsNetDir := "/sys/class/net" + files, err := ioutil.ReadDir(sysfsNetDir) + if err != nil { + return nil, err + } + nics := [][2]string{} + for i := 0; i < len(files); i++ { + nicPath, err := filepath.EvalSymlinks(path.Join(sysfsNetDir, files[i].Name())) + if err != nil { + return nil, err + } + if !strings.HasPrefix(nicPath, "/sys/devices/pci0000:") { + continue + } + var idx int + for idx = range hostNics { + if hostNics[idx][0] == files[i].Name() { + nics = append(nics, hostNics[idx]) + } + } + } + log.Infof("host nics %s detected support sriov nics %v", hostNics, nics) + sriovNics := make([]*sSRIOVNicDevice, 0) + for i := 0; i < len(nics); i++ { + nicDir := path.Join(sysfsNetDir, nics[i][0], "device") + vfs, err := ioutil.ReadDir(nicDir) + if err != nil { + return nil, err + } + for j := 0; j < len(vfs); j++ { + if strings.HasPrefix(vfs[j].Name(), "virtfn") { + virtfn, err := strconv.Atoi(vfs[j].Name()[len("virtfn"):]) + if err != nil { + return nil, err + } + vfPath, err := filepath.EvalSymlinks(path.Join(nicDir, vfs[j].Name())) + if err != nil { + return nil, err + } + vfBDF := path.Base(vfPath) + vfDev, err := detectSRIOVNicDevice(vfBDF) + if err != nil { + return nil, err + } + sriovNics = append(sriovNics, NewSRIOVNicDevice(vfDev, api.NIC_TYPE, nics[i][1], nics[i][0], virtfn)) + } + } + } + return sriovNics, nil +} + +func detectSRIOVNicDevice(vfBDF string) (*PCIDevice, error) { + dev, err := detectPCIDevByAddrWithoutIOMMUGroup(vfBDF) + if err != nil { + return nil, err + } + driver, err := dev.getKernelDriver() + if err != nil { + return nil, err + } + if driver == VFIO_PCI_KERNEL_DRIVER { + return dev, nil + } + if driver != "" { + if err = dev.unbindDriver(); err != nil { + return nil, err + } + } + if err = dev.bindDriver(); err != nil { + return nil, err + } + return dev, nil +} + +func NewSRIOVNicDevice(dev *PCIDevice, devType, wireId, pfName string, virtfn int) *sSRIOVNicDevice { + return &sSRIOVNicDevice{ + sBaseDevice: newBaseDevice(dev, devType), + WireId: wireId, + pfName: pfName, + virtfn: virtfn, + } +} + +type sSRIOVNicDevice struct { + *sBaseDevice + + WireId string + pfName string + virtfn int +} + +func (dev *sSRIOVNicDevice) GetPfName() string { + return dev.pfName +} + +func (dev *sSRIOVNicDevice) GetVirtfn() int { + return dev.virtfn +} + +func (dev *sSRIOVNicDevice) GetVGACmd() string { + return "" +} + +func (dev *sSRIOVNicDevice) GetCPUCmd() string { + return "" +} + +func (dev *sSRIOVNicDevice) GetQemuId() string { + return fmt.Sprintf("dev_%s", strings.ReplaceAll(dev.GetAddr(), ":", "_")) +} + +func (dev *sSRIOVNicDevice) GetWireId() string { + return dev.WireId +} + +func (dev *sSRIOVNicDevice) GetHotPlugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotPlugOption, error) { + ret := make([]*HotPlugOption, 0) + + var masterDevOpt *HotPlugOption + for i := 0; i < len(isolatedDev.VfioDevs); i++ { + cmd := isolatedDev.VfioDevs[i].HostAddr + if optCmd := isolatedDev.VfioDevs[i].OptionsStr(); len(optCmd) > 0 { + cmd += fmt.Sprintf(",%s", optCmd) + } + opts := map[string]string{ + "host": cmd, + "id": isolatedDev.VfioDevs[i].Id, + } + devOpt := &HotPlugOption{ + Device: isolatedDev.VfioDevs[i].DevType, + Options: opts, + } + if isolatedDev.VfioDevs[i].Function == 0 { + masterDevOpt = devOpt + } else { + ret = append(ret, devOpt) + } + } + // if PCI slot function 0 already assigned, qemu will reject hotplug function + // so put function 0 at the enda + if masterDevOpt == nil { + return nil, errors.Errorf("Device no function 0 found") + } + ret = append(ret, masterDevOpt) + return ret, nil +} + +func (dev *sSRIOVNicDevice) GetHotUnplugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotUnplugOption, error) { + if len(isolatedDev.VfioDevs) == 0 { + return nil, errors.Errorf("device %s no pci ids", isolatedDev.Id) + } + + return []*HotUnplugOption{ + { + Id: isolatedDev.VfioDevs[0].Id, + }, + }, nil +} + +func (dev *sSRIOVNicDevice) CustomProbe(idx int) error { + // check environments on first probe + if idx == 0 { + for _, driver := range []string{"vfio", "vfio_iommu_type1", "vfio-pci"} { + if err := procutils.NewRemoteCommandAsFarAsPossible("modprobe", driver).Run(); err != nil { + return fmt.Errorf("modprobe %s: %v", driver, err) + } + } + } + + driver, err := dev.GetKernelDriver() + if err != nil { + return fmt.Errorf("Nic %s is occupied by another driver: %s", dev.GetAddr(), driver) + } + return nil +} diff --git a/pkg/hostman/isolated_device/usb.go b/pkg/hostman/isolated_device/usb.go index a2b9b51e04..3002792565 100644 --- a/pkg/hostman/isolated_device/usb.go +++ b/pkg/hostman/isolated_device/usb.go @@ -47,7 +47,7 @@ func (dev *sUSBDevice) GetVGACmd() string { return "" } -func (dev *sUSBDevice) CustomProbe() error { +func (dev *sUSBDevice) CustomProbe(int) error { // do nothing return nil } diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index 66b8b1002f..1671a24c27 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -170,8 +170,9 @@ type SHostOptions struct { DisableKVM bool `help:"force disable KVM" default:"false" json:"disable_kvm"` - DisableGPU bool `help:"force disable GPU detect" default:"false" json:"disable_gpu"` - DisableUSB bool `help:"force disable USB detect" default:"true" json:"disable_usb"` + DisableGPU bool `help:"force disable GPU detect" default:"false" json:"disable_gpu"` + DisableUSB bool `help:"force disable USB detect" default:"true" json:"disable_usb"` + DisableSRIOVNic bool `help:"force disable USB detect" default:"true" json:"disable_sriov_nic"` EthtoolEnableGso bool `help:"use ethtool to turn on or off GSO(generic segment offloading)" default:"false" json:"ethtool_enable_gso"` diff --git a/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go b/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go index cb247108d7..6a7cd9bcf2 100644 --- a/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go +++ b/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go @@ -18,6 +18,7 @@ import ( "context" "fmt" + "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/scheduler/core" ) @@ -46,6 +47,15 @@ func (f *IsolatedDevicePredicate) PreExecute(ctx context.Context, u *core.Unit, func (f *IsolatedDevicePredicate) Execute(ctx context.Context, u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { h := NewPredicateHelper(f, u, c) reqIsoDevs := u.SchedData().IsolatedDevices + if reqIsoDevs == nil { + reqIsoDevs = []*compute.IsolatedDeviceConfig{} + } + networks := u.SchedData().Networks + for i := 0; i < len(networks); i++ { + if networks[i].SriovDevice != nil { + reqIsoDevs = append(reqIsoDevs, networks[i].SriovDevice) + } + } getter := c.Getter() minCapacity := int64(0xFFFFFFFF) diff --git a/pkg/scheduler/algorithm/predicates/network_predicate.go b/pkg/scheduler/algorithm/predicates/network_predicate.go index 50cbd382b3..2c93ad406c 100644 --- a/pkg/scheduler/algorithm/predicates/network_predicate.go +++ b/pkg/scheduler/algorithm/predicates/network_predicate.go @@ -151,6 +151,24 @@ func IsNetworksAvailable(ctx context.Context, c core.Candidater, data *api.Sched return freeCnt, nil } +func checkSriovNic( + c core.Candidater, netWireId string, dev *computeapi.IsolatedDeviceConfig, +) error { + if dev.WireId != "" { + if dev.WireId != netWireId { + return fmt.Errorf("Network wire not matched sriov nic wire %s != %s", netWireId, dev.WireId) + } else { + return nil + } + } + getter := c.Getter() + devs := getter.UnusedIsolatedDevicesByModelAndWire(dev.Model, netWireId) + if len(devs) == 0 { + return fmt.Errorf("Network wire no sriov nic available") + } + return nil +} + func IsNetworkAvailable( ctx context.Context, c core.Candidater, data *api.SchedInfo, @@ -197,6 +215,16 @@ func IsNetworkAvailable( } } + if req.SriovDevice != nil { + err := checkSriovNic(c, n.WireId, req.SriovDevice) + if err != nil { + return FailReason{ + Reason: err.Error(), + Type: NetworkWire, + } + } + } + if n.IsPublic && n.PublicScope == string(rbacutils.ScopeSystem) { // system-wide share } else if n.IsPublic && n.PublicScope == string(rbacutils.ScopeDomain) && (n.DomainId == data.Domain || utils.IsInStringArray(data.Domain, n.GetSharedDomains())) { diff --git a/pkg/scheduler/cache/candidate/base.go b/pkg/scheduler/cache/candidate/base.go index dd1d22233a..803a1948c4 100644 --- a/pkg/scheduler/cache/candidate/base.go +++ b/pkg/scheduler/cache/candidate/base.go @@ -271,6 +271,10 @@ func (b baseHostGetter) UnusedIsolatedDevicesByModel(model string) []*core.Isola return b.h.UnusedIsolatedDevicesByModel(model) } +func (b baseHostGetter) UnusedIsolatedDevicesByModelAndWire(model, wire string) []*core.IsolatedDeviceDesc { + return b.h.UnusedIsolatedDevicesByModelAndWire(model, wire) +} + func (b baseHostGetter) GetIsolatedDevice(devID string) *core.IsolatedDeviceDesc { return b.h.GetIsolatedDevice(devID) } @@ -443,6 +447,17 @@ func (h *BaseHostDesc) UnusedIsolatedDevicesByModel(model string) []*core.Isolat return ret } +func (h *BaseHostDesc) UnusedIsolatedDevicesByModelAndWire(model, wire string) []*core.IsolatedDeviceDesc { + ret := make([]*core.IsolatedDeviceDesc, 0) + for _, dev := range h.UnusedIsolatedDevices() { + log.Errorf("dev wire is %s, dev model is %s, request model is %s, request wire is %s", dev.Model, dev.WireId, model, wire) + if strings.Contains(dev.Model, model) && dev.WireId == wire { + ret = append(ret, dev) + } + } + return ret +} + func (h *BaseHostDesc) GetIsolatedDevice(devID string) *core.IsolatedDeviceDesc { for _, dev := range h.IsolatedDevices { if dev.ID == devID { @@ -482,6 +497,7 @@ func (h *BaseHostDesc) fillIsolatedDevices(b *baseBuilder, host *computemodels.S Model: devModel.Model, Addr: devModel.Addr, VendorDeviceID: devModel.VendorDeviceId, + WireId: devModel.WireId, } devs[index] = dev } diff --git a/pkg/scheduler/core/types.go b/pkg/scheduler/core/types.go index 3a94070769..f292507a33 100644 --- a/pkg/scheduler/core/types.go +++ b/pkg/scheduler/core/types.go @@ -122,6 +122,7 @@ type CandidatePropertyGetter interface { UnusedIsolatedDevicesByType(devType string) []*IsolatedDeviceDesc UnusedIsolatedDevicesByVendorModel(vendorModel string) []*IsolatedDeviceDesc UnusedIsolatedDevicesByModel(model string) []*IsolatedDeviceDesc + UnusedIsolatedDevicesByModelAndWire(model, wire string) []*IsolatedDeviceDesc GetIsolatedDevice(devID string) *IsolatedDeviceDesc UnusedGpuDevices() []*IsolatedDeviceDesc GetIsolatedDevices() []*IsolatedDeviceDesc @@ -242,6 +243,7 @@ type IsolatedDeviceDesc struct { Model string Addr string VendorDeviceID string + WireId string } func (i *IsolatedDeviceDesc) VendorID() string { diff --git a/pkg/scheduler/test/mock/core.go b/pkg/scheduler/test/mock/core.go index e67b2b714e..ec72489caf 100644 --- a/pkg/scheduler/test/mock/core.go +++ b/pkg/scheduler/test/mock/core.go @@ -706,6 +706,20 @@ func (mr *MockCandidatePropertyGetterMockRecorder) UnusedIsolatedDevicesByModel( return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnusedIsolatedDevicesByModel", reflect.TypeOf((*MockCandidatePropertyGetter)(nil).UnusedIsolatedDevicesByModel), arg0) } +// UnusedIsolatedDevicesByModelAndWire mocks base method +func (m *MockCandidatePropertyGetter) UnusedIsolatedDevicesByModelAndWire(arg0, arg1 string) []*core.IsolatedDeviceDesc { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UnusedIsolatedDevicesByModelAndWire", arg0, arg1) + ret0, _ := ret[0].([]*core.IsolatedDeviceDesc) + return ret0 +} + +// UnusedIsolatedDevicesByModelAndWire indicates an expected call of UnusedIsolatedDevicesByModelAndWire +func (mr *MockCandidatePropertyGetterMockRecorder) UnusedIsolatedDevicesByModelAndWire(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnusedIsolatedDevicesByModelAndWire", reflect.TypeOf((*MockCandidatePropertyGetter)(nil).UnusedIsolatedDevicesByModel), arg0, arg1) +} + // UnusedIsolatedDevicesByType mocks base method func (m *MockCandidatePropertyGetter) UnusedIsolatedDevicesByType(arg0 string) []*core.IsolatedDeviceDesc { m.ctrl.T.Helper()