From 693257697e3e058085123d0611fcd4f215d7dbe4 Mon Sep 17 00:00:00 2001 From: wanyaoqi <18528551+wanyaoqi@users.noreply.github.com> Date: Sat, 18 Feb 2023 08:48:07 +0800 Subject: [PATCH] feat(region,host): support ovs offload. (#15915) Cloudpods Ovs offload implemention base on SR-IOV support. IsolatedDevices add field `OvsOffloadInterface`. Host config add `ovs_offload_nics` to specify which nic enable ovs offload, and removed flag `disable_sriov_nics` add flag `sriov_nics` to configure sriov nics. Tested Hardware: Mellanox ConnectX-5 NICs --- pkg/apis/compute/isolated_device.go | 15 +- pkg/compute/models/guestnetworks.go | 33 ++- pkg/compute/models/isolated_devices.go | 17 +- pkg/hostman/guestman/pci.go | 12 +- pkg/hostman/guestman/qemu-kvm.go | 35 ++- pkg/hostman/guestman/qemu-kvmhelper.go | 5 +- pkg/hostman/hostinfo/hostbridge/ovs.go | 11 +- pkg/hostman/hostinfo/hostinfo.go | 44 +++- .../isolated_device/isolated_device.go | 33 ++- pkg/hostman/isolated_device/nic.go | 201 +++++++++++++++++- pkg/hostman/options/options.go | 7 +- 11 files changed, 359 insertions(+), 54 deletions(-) diff --git a/pkg/apis/compute/isolated_device.go b/pkg/apis/compute/isolated_device.go index be704e3358..58e223359a 100644 --- a/pkg/apis/compute/isolated_device.go +++ b/pkg/apis/compute/isolated_device.go @@ -101,11 +101,12 @@ type IsolatedDeviceUpdateInput struct { } type IsolatedDeviceJsonDesc struct { - Id string `json:"id"` - DevType string `json:"dev_type"` - Model string `json:"model"` - Addr string `json:"addr"` - VendorDeviceId string `json:"vendor_device_id"` - Vendor string `json:"vendor"` - NetworkIndex int8 `json:"network_index"` + Id string `json:"id"` + DevType string `json:"dev_type"` + Model string `json:"model"` + Addr string `json:"addr"` + VendorDeviceId string `json:"vendor_device_id"` + Vendor string `json:"vendor"` + NetworkIndex int8 `json:"network_index"` + OvsOffloadInterface string `json:"ovs_offload_interface"` } diff --git a/pkg/compute/models/guestnetworks.go b/pkg/compute/models/guestnetworks.go index 08ad95fc34..7ed5b17573 100644 --- a/pkg/compute/models/guestnetworks.go +++ b/pkg/compute/models/guestnetworks.go @@ -579,7 +579,7 @@ func (self *SGuestnetwork) getJsonDesc() *api.GuestnetworkJsonDesc { desc.TeamWith = self.TeamWith guest := self.getGuest() - if guest.GetHypervisor() != api.HYPERVISOR_KVM || self.Driver == api.NETWORK_DRIVER_VFIO { + if guest.GetHypervisor() != api.HYPERVISOR_KVM || self.IsSriovWithoutOffload() { manual := true desc.Manual = &manual } @@ -587,6 +587,16 @@ func (self *SGuestnetwork) getJsonDesc() *api.GuestnetworkJsonDesc { return desc } +func (self *SGuestnetwork) IsSriovWithoutOffload() bool { + if self.Driver != api.NETWORK_DRIVER_VFIO { + return false + } + if dev, _ := self.GetIsolatedDevice(); dev != nil && dev.OvsOffloadInterface != "" { + return false + } + return true +} + func (manager *SGuestnetworkManager) GetGuestByAddress(address string) *SGuest { networks := manager.TableSpec().Instance() guests := GuestManager.Query() @@ -867,9 +877,30 @@ func (self *SGuestnetwork) GetVirtualIPs() []string { } func (self *SGuestnetwork) GetIfname() string { + if self.Driver == api.NETWORK_DRIVER_VFIO { + if dev, _ := self.GetIsolatedDevice(); dev != nil && dev.OvsOffloadInterface != "" { + return dev.OvsOffloadInterface + } + } return self.Ifname } +func (self *SGuestnetwork) GetIsolatedDevice() (*SIsolatedDevice, error) { + dev := SIsolatedDevice{} + q := IsolatedDeviceManager.Query().Equals("guest_id", self.GuestId).Equals("network_index", self.Index) + if cnt, err := q.CountWithError(); err != nil { + return nil, err + } else if cnt == 0 { + return nil, nil + } + err := q.First(&dev) + if err != nil { + return nil, err + } + dev.SetModelManager(IsolatedDeviceManager, &dev) + return &dev, nil +} + func (manager *SGuestnetworkManager) getRecentlyReleasedIPAddresses(networkId string, recentDuration time.Duration) map[string]bool { if recentDuration == 0 { return nil diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index fe2360904e..2989f9a4d9 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -94,6 +94,8 @@ type SIsolatedDevice struct { NetworkIndex int8 `nullable:"true" default:"-1" list:"user" update:"user"` // Nic wire id WireId string `width:"36" charset:"ascii" nullable:"true" index:"true" list:"domain" update:"domain" create:"domain_optional"` + // Offload interface name + OvsOffloadInterface string `width:"16" charset:"ascii" nullable:"true" list:"domain" update:"domain" create:"domain_optional"` // # pci address of `Bus:Device.Function` format, or usb bus address of `bus.addr` Addr string `width:"16" charset:"ascii" nullable:"true" list:"domain" update:"domain" create:"domain_optional"` @@ -664,13 +666,14 @@ func (manager *SIsolatedDeviceManager) TotalCount( func (self *SIsolatedDevice) getDesc() *api.IsolatedDeviceJsonDesc { return &api.IsolatedDeviceJsonDesc{ - Id: self.Id, - DevType: self.DevType, - Model: self.Model, - Addr: self.Addr, - VendorDeviceId: self.VendorDeviceId, - Vendor: self.getVendor(), - NetworkIndex: self.NetworkIndex, + Id: self.Id, + DevType: self.DevType, + Model: self.Model, + Addr: self.Addr, + VendorDeviceId: self.VendorDeviceId, + Vendor: self.getVendor(), + NetworkIndex: self.NetworkIndex, + OvsOffloadInterface: self.OvsOffloadInterface, } } diff --git a/pkg/hostman/guestman/pci.go b/pkg/hostman/guestman/pci.go index 459fc75254..1f0e66ec68 100644 --- a/pkg/hostman/guestman/pci.go +++ b/pkg/hostman/guestman/pci.go @@ -281,6 +281,12 @@ func (s *SKVMGuestInstance) initGuestNetworks(pciRoot, pciBridge *desc.PCIContro } for i := 0; i < len(s.Desc.Nics); i++ { + if err := s.generateNicScripts(s.Desc.Nics[i]); err != nil { + return errors.Wrapf(err, "generateNicScripts for nic: %v", s.Desc.Nics[i]) + } + s.Desc.Nics[i].UpscriptPath = s.getNicUpScriptPath(s.Desc.Nics[i]) + s.Desc.Nics[i].DownscriptPath = s.getNicDownScriptPath(s.Desc.Nics[i]) + if s.Desc.Nics[i].Driver != "vfio-pci" { switch s.GetOsName() { case OS_NAME_MACOS: @@ -295,12 +301,6 @@ func (s *SKVMGuestInstance) initGuestNetworks(pciRoot, pciBridge *desc.PCIContro s.Desc.Nics[i].Vectors = &vectors } - if err := s.generateNicScripts(s.Desc.Nics[i]); err != nil { - return errors.Wrapf(err, "generateNicScripts for nic: %v", s.Desc.Nics[i]) - } - s.Desc.Nics[i].UpscriptPath = s.getNicUpScriptPath(s.Desc.Nics[i]) - s.Desc.Nics[i].DownscriptPath = s.getNicDownScriptPath(s.Desc.Nics[i]) - id := fmt.Sprintf("netdev-%s", s.Desc.Nics[i].Ifname) switch s.Desc.Nics[i].Driver { case "virtio": diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index aaf459a8f7..89a92cc19c 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -2741,12 +2741,19 @@ func getVfVlan(vlan int) int { func (s *SKVMGuestInstance) sriovNicAttachInitScript(networkIndex int8, dev isolated_device.IDevice) (string, error) { for i := range s.Desc.Nics { if s.Desc.Nics[i].Driver == "vfio-pci" && s.Desc.Nics[i].Index == networkIndex { - cmd := fmt.Sprintf( - "sriov_vf_init %s %d %s %d %s %d\n", - dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac, - getVfVlan(s.Desc.Nics[i].Vlan), srcMacCheckFunc(s.Desc.SrcMacCheck), s.Desc.Nics[i].Bw, - ) - return sriovInitFunc + " && " + cmd, nil + if dev.GetOvsOffloadInterfaceName() != "" { + cmd := fmt.Sprintf("ip link set dev %s vf %d mac %s\n", + dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac) + cmd += s.getNicUpScriptPath(s.Desc.Nics[i]) + "\n" + return cmd, nil + } else { + cmd := fmt.Sprintf( + "sriov_vf_init %s %d %s %d %s %d\n", + dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac, + getVfVlan(s.Desc.Nics[i].Vlan), srcMacCheckFunc(s.Desc.SrcMacCheck), s.Desc.Nics[i].Bw, + ) + return sriovInitFunc + " && " + cmd, nil + } } } return "", errors.Errorf("no nic found for index %d", networkIndex) @@ -2761,11 +2768,17 @@ func (s *SKVMGuestInstance) generateSRIOVInitScripts() (string, error) { if err != nil { return "", err } - cmd += fmt.Sprintf( - "sriov_vf_init %s %d %s %d %s %d\n", - dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac, - getVfVlan(s.Desc.Nics[i].Vlan), srcMacCheckFunc(s.Desc.SrcMacCheck), s.Desc.Nics[i].Bw, - ) + if dev.GetOvsOffloadInterfaceName() != "" { + cmd += fmt.Sprintf("ip link set dev %s vf %d mac %s\n", + dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac) + cmd += s.getNicUpScriptPath(s.Desc.Nics[i]) + "\n" + } else { + cmd += fmt.Sprintf( + "sriov_vf_init %s %d %s %d %s %d\n", + dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac, + getVfVlan(s.Desc.Nics[i].Vlan), srcMacCheckFunc(s.Desc.SrcMacCheck), s.Desc.Nics[i].Bw, + ) + } } } if len(cmd) > 0 { diff --git a/pkg/hostman/guestman/qemu-kvmhelper.go b/pkg/hostman/guestman/qemu-kvmhelper.go index 995b3aa3db..b576a1ec94 100644 --- a/pkg/hostman/guestman/qemu-kvmhelper.go +++ b/pkg/hostman/guestman/qemu-kvmhelper.go @@ -662,7 +662,10 @@ func (s *SKVMGuestInstance) generateStopScript(data *jsonutils.JSONDict) string for _, nic := range nics { if nic.Driver == api.NETWORK_DRIVER_VFIO { - continue + dev, _ := s.GetSriovDeviceByNetworkIndex(nic.Index) + if dev != nil && dev.GetOvsOffloadInterfaceName() == "" { + continue + } } downscript := s.getNicDownScriptPath(nic) cmd += fmt.Sprintf("%s %s\n", downscript, nic.Ifname) diff --git a/pkg/hostman/hostinfo/hostbridge/ovs.go b/pkg/hostman/hostinfo/hostbridge/ovs.go index b8c8554890..f0e6d921a2 100644 --- a/pkg/hostman/hostinfo/hostbridge/ovs.go +++ b/pkg/hostman/hostinfo/hostbridge/ovs.go @@ -191,10 +191,13 @@ func (o *SOVSBridgeDriver) getUpScripts(nic *desc.SGuestNetwork, isVolatileHost s += "PORT=$(ovs-ofctl show $SWITCH | grep -w $IF)\n" s += "PORT=$(echo $PORT | awk 'BEGIN{FS=\"(\"}{print $1}')\n" s += "OFCTL=$(ovs-vsctl get-controller $SWITCH)\n" - s += "if [ -z \"$OFCTL\" ]; then\n" - s += " ovs-vsctl set Interface $IF ingress_policing_rate=$LIMIT\n" - s += " ovs-vsctl set Interface $IF ingress_policing_burst=$BURST\n" - s += "fi\n" + if nic.Driver != compute.NETWORK_DRIVER_VFIO { + s += "if [ -z \"$OFCTL\" ]; then\n" + s += " ovs-vsctl set Interface $IF ingress_policing_rate=$LIMIT\n" + s += " ovs-vsctl set Interface $IF ingress_policing_burst=$BURST\n" + s += "fi\n" + } + s += "if [ $LIMIT_DOWNLOAD != \"0mbit\" ]; then\n" s += " tc qdisc del dev $IF root 2>/dev/null\n" s += " tc qdisc add dev $IF root handle 1: htb default 10\n" diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index addfcc9a36..40b7c1b76d 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -1842,18 +1842,52 @@ func (h *SHostInfo) probeSyncIsolatedDevicesStep() { h.deployAdminAuthorizedKeys() } -func (h *SHostInfo) getNicsInterfaces() [][2]string { - res := [][2]string{} +func (h *SHostInfo) getNicsInterfaces(nics []string) []isolated_device.HostNic { + if len(nics) == 0 { + return nil + } + res := []isolated_device.HostNic{} for i := 0; i < len(h.Nics); i++ { - res = append(res, [2]string{h.Nics[i].Inter, h.Nics[i].WireId}) + if utils.IsInStringArray(h.Nics[i].Inter, nics) { + res = append(res, isolated_device.HostNic{h.Nics[i].Bridge, h.Nics[i].Inter, h.Nics[i].WireId}) + } } return res } +func (h *SHostInfo) getNicsOvsOffloadInterfaces(nics []string) ([]isolated_device.HostNic, error) { + if len(nics) == 0 { + return nil, nil + } + + res := []isolated_device.HostNic{} + for i := 0; i < len(h.Nics); i++ { + if utils.IsInStringArray(h.Nics[i].Inter, nics) { + if fileutils2.Exists(fmt.Sprintf("/sys/class/net/%s/bonding/slaves", h.Nics[i].Inter)) { + interStr, err := fileutils2.FileGetContents(fmt.Sprintf("/sys/class/net/%s/bonding/slaves", h.Nics[i].Inter)) + if err != nil { + return nil, err + } + inters := strings.Split(strings.TrimSpace(interStr), " ") + for _, inter := range inters { + res = append(res, isolated_device.HostNic{h.Nics[i].Bridge, inter, h.Nics[i].WireId}) + } + } else { + res = append(res, isolated_device.HostNic{h.Nics[i].Bridge, h.Nics[i].Inter, h.Nics[i].WireId}) + } + } + } + return res, nil +} + func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) { + offloadNics, err := h.getNicsOvsOffloadInterfaces(options.HostOptions.OvsOffloadNics) + if err != nil { + return nil, err + } + sriovNics := h.getNicsInterfaces(options.HostOptions.SRIOVNics) if err := h.IsolatedDeviceMan.ProbePCIDevices( - options.HostOptions.DisableGPU, options.HostOptions.DisableUSB, - options.HostOptions.DisableSRIOVNic, h.getNicsInterfaces(), + options.HostOptions.DisableGPU, options.HostOptions.DisableUSB, sriovNics, offloadNics, ); err != nil { return nil, errors.Wrap(err, "ProbePCIDevices") } diff --git a/pkg/hostman/isolated_device/isolated_device.go b/pkg/hostman/isolated_device/isolated_device.go index adaadc8adb..8d8dcdde68 100644 --- a/pkg/hostman/isolated_device/isolated_device.go +++ b/pkg/hostman/isolated_device/isolated_device.go @@ -66,6 +66,7 @@ type IDevice interface { SetHostId(hId string) GetGuestId() string GetWireId() string + GetOvsOffloadInterfaceName() string GetVendorDeviceId() string GetAddr() string GetDeviceType() string @@ -95,7 +96,7 @@ type IsolatedDeviceManager interface { GetDevices() []IDevice GetDeviceByIdent(vendorDevId string, addr string) IDevice GetDeviceByAddr(addr string) IDevice - ProbePCIDevices(skipGPUs, skipUSBs, skipSRIOVNics bool, nics [][2]string) error + ProbePCIDevices(skipGPUs, skipUSBs bool, sriovNics, ovsOffloadNics []HostNic) error StartDetachTask() BatchCustomProbe() error AppendDetachedDevice(dev *CloudDeviceInfo) @@ -122,7 +123,13 @@ func (man *isolatedDeviceManager) GetDevices() []IDevice { return man.devices } -func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs, skipSRIOVNics bool, nics [][2]string) error { +type HostNic struct { + Bridge string + Interface string + Wire string +} + +func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs bool, sriovNics, ovsOffloadNics []HostNic) error { man.devices = make([]IDevice, 0) if !skipGPUs { gpus, err := getPassthroughGPUS() @@ -149,8 +156,8 @@ func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs, skipSRIOVN } } - if !skipSRIOVNics { - nics, err := getSRIOVNics(nics) + if len(sriovNics) > 0 { + nics, err := getSRIOVNics(sriovNics) if err != nil { log.Errorf("getSRIOVNics: %v", err) return nil @@ -160,6 +167,17 @@ func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs, skipSRIOVN log.Infof("Add sriov nic: %d => %#v", idx, nic) } } + if len(ovsOffloadNics) > 0 { + nics, err := getOvsOffloadNics(ovsOffloadNics) + if err != nil { + log.Errorf("getOvsOffloadNics: %v", err) + return nil + } + for idx, nic := range nics { + man.devices = append(man.devices, nic) + log.Infof("Add sriov nic: %d => %#v", idx, nic) + } + } return nil } @@ -327,6 +345,10 @@ func (dev *sBaseDevice) GetVirtfn() int { return -1 } +func (dev *sBaseDevice) GetOvsOffloadInterfaceName() string { + return "" +} + func (dev *sBaseDevice) GetModelName() string { if dev.dev.ModelName != "" { return dev.dev.ModelName @@ -363,6 +385,9 @@ func GetApiResourceData(dev IDevice) *jsonutils.JSONDict { if len(dev.GetWireId()) != 0 { data["wire_id"] = dev.GetWireId() } + if len(dev.GetOvsOffloadInterfaceName()) != 0 { + data["ovs_offload_interface"] = dev.GetOvsOffloadInterfaceName() + } return jsonutils.Marshal(data).(*jsonutils.JSONDict) } diff --git a/pkg/hostman/isolated_device/nic.go b/pkg/hostman/isolated_device/nic.go index 0bdfe701e7..6ce86dbdaf 100644 --- a/pkg/hostman/isolated_device/nic.go +++ b/pkg/hostman/isolated_device/nic.go @@ -27,16 +27,17 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/hostman/guestman/desc" + "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/procutils" ) -func getSRIOVNics(hostNics [][2]string) ([]*sSRIOVNicDevice, error) { +func getSRIOVNics(hostNics []HostNic) ([]*sSRIOVNicDevice, error) { sysfsNetDir := "/sys/class/net" files, err := ioutil.ReadDir(sysfsNetDir) if err != nil { return nil, err } - nics := [][2]string{} + nics := []HostNic{} for i := 0; i < len(files); i++ { nicPath, err := filepath.EvalSymlinks(path.Join(sysfsNetDir, files[i].Name())) if err != nil { @@ -47,15 +48,20 @@ func getSRIOVNics(hostNics [][2]string) ([]*sSRIOVNicDevice, error) { } var idx int for idx = range hostNics { - if hostNics[idx][0] == files[i].Name() { + if hostNics[idx].Interface == files[i].Name() { nics = append(nics, hostNics[idx]) } } } + log.Infof("host nics %s detected support sriov nics %v", hostNics, nics) sriovNics := make([]*sSRIOVNicDevice, 0) for i := 0; i < len(nics); i++ { - nicDir := path.Join(sysfsNetDir, nics[i][0], "device") + nicDir := path.Join(sysfsNetDir, nics[i].Interface, "device") + err = ensureNumvfsEqualTotalvfs(nicDir) + if err != nil { + return nil, err + } vfs, err := ioutil.ReadDir(nicDir) if err != nil { return nil, err @@ -75,13 +81,31 @@ func getSRIOVNics(hostNics [][2]string) ([]*sSRIOVNicDevice, error) { if err != nil { return nil, err } - sriovNics = append(sriovNics, NewSRIOVNicDevice(vfDev, api.NIC_TYPE, nics[i][1], nics[i][0], virtfn)) + sriovNics = append(sriovNics, NewSRIOVNicDevice(vfDev, api.NIC_TYPE, nics[i].Wire, nics[i].Interface, virtfn)) } } } return sriovNics, nil } +func ensureNumvfsEqualTotalvfs(nicDir string) error { + sriovNumvfs := path.Join(nicDir, "sriov_numvfs") + sriovTotalvfs := path.Join(nicDir, "sriov_totalvfs") + numvfs, err := fileutils2.FileGetContents(sriovNumvfs) + if err != nil { + return err + } + totalvfs, err := fileutils2.FileGetContents(sriovTotalvfs) + if err != nil { + return err + } + log.Errorf("numvfs %s total vfs %s", numvfs, totalvfs) + if numvfs != totalvfs { + return fileutils2.FilePutContents(sriovNumvfs, fmt.Sprintf("%s", totalvfs), false) + } + return nil +} + func detectSRIOVNicDevice(vfBDF string) (*PCIDevice, error) { dev, err := detectPCIDevByAddrWithoutIOMMUGroup(vfBDF) if err != nil { @@ -206,3 +230,170 @@ func (dev *sSRIOVNicDevice) CustomProbe(idx int) error { } return nil } + +func getOvsOffloadNics(hostNics []HostNic) ([]*sOvsOffloadNicDevice, error) { + sysfsNetDir := "/sys/class/net" + files, err := ioutil.ReadDir(sysfsNetDir) + if err != nil { + return nil, errors.Wrap(err, "ioutil.ReadDir(sysfsNetDir)") + } + nics := []HostNic{} + for i := 0; i < len(files); i++ { + nicPath, err := filepath.EvalSymlinks(path.Join(sysfsNetDir, files[i].Name())) + if err != nil { + return nil, errors.Wrap(err, "filepath.EvalSymlinks nicPath") + } + if !strings.HasPrefix(nicPath, "/sys/devices/pci0000:") { + continue + } + var idx int + for idx = range hostNics { + if hostNics[idx].Interface == files[i].Name() { + nics = append(nics, hostNics[idx]) + } + } + } + + sriovNics := make([]*sOvsOffloadNicDevice, 0) + for i := range nics { + nicDir := path.Join(sysfsNetDir, nics[i].Interface, "device") + err = ensureNumvfsEqualTotalvfs(nicDir) + if err != nil { + return nil, errors.Wrap(err, "ensureNumvfsEqualTotalvfs") + } + + vfs, err := ioutil.ReadDir(nicDir) + if err != nil { + return nil, errors.Wrap(err, "ioutil.ReadDir") + } + + pfPath, err := filepath.EvalSymlinks(nicDir) + if err != nil { + return nil, errors.Wrap(err, "filepath.EvalSymlinks pfPath") + } + pfBDF := path.Base(pfPath) + + // /sys/class/net/ens1f0/compat/devlink/mode + devlinkPath := path.Join(sysfsNetDir, nics[i].Interface, "compat/devlink/mode") + linkMode, err := fileutils2.FileGetContents(devlinkPath) + if err != nil { + return nil, errors.Wrap(err, "fileutils2.FileGetContents(devlinkPath)") + } + log.Infof("nic %s link mode %s", nics[i].Interface, linkMode) + if strings.TrimSpace(linkMode) != "switchdev" { + err = fileutils2.FilePutContents(devlinkPath, "switchdev\n", false) + if err != nil { + return nil, errors.Wrap(err, "fileutils2.FilePutContents linkMode") + } + for j := 0; j < len(vfs); j++ { + if strings.HasPrefix(vfs[j].Name(), "virtfn") { + vfPath, err := filepath.EvalSymlinks(path.Join(nicDir, vfs[j].Name())) + if err != nil { + return nil, errors.Wrap(err, "filepath.EvalSymlinks") + } + vfBDF := path.Base(vfPath) + dev, err := detectPCIDevByAddrWithoutIOMMUGroup(vfBDF) + if err != nil { + return nil, errors.Wrap(err, "detectPCIDevByAddrWithoutIOMMUGroup") + } + err = dev.unbindDriver() + if err != nil { + return nil, errors.Wrap(err, "unbindDriver") + } + } + } + } + + // get interfaces + // grep -e 'PCI_SLOT_NAME=.*04:00.1' /sys/class/net/*/device/uevent + outs, err := procutils.NewRemoteCommandAsFarAsPossible("sh", "-c", + fmt.Sprintf("grep -e PCI_SLOT_NAME=.*%s /sys/class/net/*/device/uevent", pfBDF)).Output() + if err != nil { + return nil, errors.Wrap(err, "procutils.NewRemoteCommandAsFarAsPossible grep PCI_SLOT_NAME") + } + vfNames := map[int]string{} + for _, line := range strings.Split(string(outs), "\n") { + line = strings.TrimSpace(line) + segs := strings.Split(line, "/") + if len(segs) < 5 { + continue + } + vfName := segs[4] + if vfName == nics[i].Interface { + continue + } + // eg: pf0vf1 + portname, err := fileutils2.FileGetContents(fmt.Sprintf("/sys/class/net/%s/phys_port_name", vfName)) + if err != nil { + return nil, errors.Wrap(err, "fileutils2.FileGetContents portname") + } + sp := strings.Split(portname, "vf") + if len(sp) != 2 { + return nil, errors.Errorf("%s bad portname %s", vfName, portname) + } + virtfn, err := strconv.Atoi(strings.TrimSpace(sp[1])) + if err != nil { + return nil, errors.Wrapf(err, "failed parse %s portname %s", vfName, portname) + } + vfNames[virtfn] = vfName + } + log.Infof("vfnames: %v", vfNames) + + for j := 0; j < len(vfs); j++ { + if strings.HasPrefix(vfs[j].Name(), "virtfn") { + virtfn, err := strconv.Atoi(vfs[j].Name()[len("virtfn"):]) + if err != nil { + return nil, errors.Wrap(err, "strconv.Atoi virtfn") + } + vfPath, err := filepath.EvalSymlinks(path.Join(nicDir, vfs[j].Name())) + if err != nil { + return nil, errors.Wrap(err, "filepath.EvalSymlinks vfpath") + } + vfBDF := path.Base(vfPath) + vfDev, err := detectSRIOVNicDevice(vfBDF) + if err != nil { + return nil, errors.Wrap(err, "detectSRIOVNicDevice") + } + // out, err := procutils.NewRemoteCommandAsFarAsPossible( + // "ovs-vsctl", "--may-exist", "add-port", nics[i].Bridge, vfNames[virtfn]).Output() + // if err != nil { + // return nil, errors.Wrapf(err, " ovs-vsctl add-port %s %s failed: %s", nics[i].Bridge, vfNames[virtfn], out) + // } + + sriovNics = append(sriovNics, NewSRIOVOffloadNicDevice( + vfDev, api.NIC_TYPE, nics[i].Wire, nics[i].Interface, virtfn, vfNames[virtfn]), + ) + } + } + } + + // ovs-vsctl set Open_vSwitch . other_config:hw-offload=true + out, err := procutils.NewRemoteCommandAsFarAsPossible( + "ovs-vsctl", "set", "Open_vSwitch", ".", "other_config:hw-offload=true").Output() + if err != nil { + return nil, errors.Wrapf(err, "ovs enable hw offload failed: %s", out) + } + + return sriovNics, nil +} + +type sOvsOffloadNicDevice struct { + *sSRIOVNicDevice + + interfaceName string +} + +func NewSRIOVOffloadNicDevice(dev *PCIDevice, devType, wireId, pfName string, virtfn int, ifname string) *sOvsOffloadNicDevice { + return &sOvsOffloadNicDevice{ + sSRIOVNicDevice: NewSRIOVNicDevice(dev, devType, wireId, pfName, virtfn), + interfaceName: ifname, + } +} + +func (dev *sOvsOffloadNicDevice) setInterfaceName(ifname string) { + dev.interfaceName = ifname +} + +func (dev *sOvsOffloadNicDevice) GetOvsOffloadInterfaceName() string { + return dev.interfaceName +} diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index f74421976e..ff16908a5b 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -170,9 +170,10 @@ type SHostOptions struct { DisableKVM bool `help:"force disable KVM" default:"false" json:"disable_kvm"` - DisableGPU bool `help:"force disable GPU detect" default:"false" json:"disable_gpu"` - DisableUSB bool `help:"force disable USB detect" default:"true" json:"disable_usb"` - DisableSRIOVNic bool `help:"force disable USB detect" default:"true" json:"disable_sriov_nic"` + DisableGPU bool `help:"force disable GPU detect" default:"false" json:"disable_gpu"` + DisableUSB bool `help:"force disable USB detect" default:"true" json:"disable_usb"` + SRIOVNics []string `help:"nics enable sriov" json:"sriov_nics"` + OvsOffloadNics []string `help:"nics enable ovs offload" json:"ovs_offload_nics"` EthtoolEnableGso bool `help:"use ethtool to turn on or off GSO(generic segment offloading)" default:"false" json:"ethtool_enable_gso"`