feat(region,host): support ovs offload. (#15915)

Cloudpods Ovs offload implemention base on SR-IOV support.
IsolatedDevices add field `OvsOffloadInterface`.

Host config add `ovs_offload_nics` to specify which nic enable ovs
offload, and removed flag `disable_sriov_nics` add flag `sriov_nics` to
configure sriov nics.

Tested Hardware: Mellanox ConnectX-5 NICs

pkg/apis/compute/isolated_device.go

@@ -101,11 +101,12 @@ type IsolatedDeviceUpdateInput struct {
 }
 
 type IsolatedDeviceJsonDesc struct {
-	Id             string `json:"id"`
-	DevType        string `json:"dev_type"`
-	Model          string `json:"model"`
-	Addr           string `json:"addr"`
-	VendorDeviceId string `json:"vendor_device_id"`
-	Vendor         string `json:"vendor"`
-	NetworkIndex   int8   `json:"network_index"`
+	Id                  string `json:"id"`
+	DevType             string `json:"dev_type"`
+	Model               string `json:"model"`
+	Addr                string `json:"addr"`
+	VendorDeviceId      string `json:"vendor_device_id"`
+	Vendor              string `json:"vendor"`
+	NetworkIndex        int8   `json:"network_index"`
+	OvsOffloadInterface string `json:"ovs_offload_interface"`
 }

pkg/compute/models/guestnetworks.go

@@ -579,7 +579,7 @@ func (self *SGuestnetwork) getJsonDesc() *api.GuestnetworkJsonDesc {
 	desc.TeamWith = self.TeamWith
 
 	guest := self.getGuest()
-	if guest.GetHypervisor() != api.HYPERVISOR_KVM || self.Driver == api.NETWORK_DRIVER_VFIO {
+	if guest.GetHypervisor() != api.HYPERVISOR_KVM || self.IsSriovWithoutOffload() {
 		manual := true
 		desc.Manual = &manual
 	}
@@ -587,6 +587,16 @@ func (self *SGuestnetwork) getJsonDesc() *api.GuestnetworkJsonDesc {
 	return desc
 }
 
+func (self *SGuestnetwork) IsSriovWithoutOffload() bool {
+	if self.Driver != api.NETWORK_DRIVER_VFIO {
+		return false
+	}
+	if dev, _ := self.GetIsolatedDevice(); dev != nil && dev.OvsOffloadInterface != "" {
+		return false
+	}
+	return true
+}
+
 func (manager *SGuestnetworkManager) GetGuestByAddress(address string) *SGuest {
 	networks := manager.TableSpec().Instance()
 	guests := GuestManager.Query()
@@ -867,9 +877,30 @@ func (self *SGuestnetwork) GetVirtualIPs() []string {
 }
 
 func (self *SGuestnetwork) GetIfname() string {
+	if self.Driver == api.NETWORK_DRIVER_VFIO {
+		if dev, _ := self.GetIsolatedDevice(); dev != nil && dev.OvsOffloadInterface != "" {
+			return dev.OvsOffloadInterface
+		}
+	}
 	return self.Ifname
 }
 
+func (self *SGuestnetwork) GetIsolatedDevice() (*SIsolatedDevice, error) {
+	dev := SIsolatedDevice{}
+	q := IsolatedDeviceManager.Query().Equals("guest_id", self.GuestId).Equals("network_index", self.Index)
+	if cnt, err := q.CountWithError(); err != nil {
+		return nil, err
+	} else if cnt == 0 {
+		return nil, nil
+	}
+	err := q.First(&dev)
+	if err != nil {
+		return nil, err
+	}
+	dev.SetModelManager(IsolatedDeviceManager, &dev)
+	return &dev, nil
+}
+
 func (manager *SGuestnetworkManager) getRecentlyReleasedIPAddresses(networkId string, recentDuration time.Duration) map[string]bool {
 	if recentDuration == 0 {
 		return nil

pkg/compute/models/isolated_devices.go

@@ -94,6 +94,8 @@ type SIsolatedDevice struct {
 	NetworkIndex int8 `nullable:"true" default:"-1" list:"user" update:"user"`
 	// Nic wire id
 	WireId string `width:"36" charset:"ascii" nullable:"true" index:"true" list:"domain" update:"domain" create:"domain_optional"`
+	// Offload interface name
+	OvsOffloadInterface string `width:"16" charset:"ascii" nullable:"true" list:"domain" update:"domain" create:"domain_optional"`
 
 	// # pci address of `Bus:Device.Function` format, or usb bus address of `bus.addr`
 	Addr string `width:"16" charset:"ascii" nullable:"true" list:"domain" update:"domain" create:"domain_optional"`
@@ -664,13 +666,14 @@ func (manager *SIsolatedDeviceManager) TotalCount(
 
 func (self *SIsolatedDevice) getDesc() *api.IsolatedDeviceJsonDesc {
 	return &api.IsolatedDeviceJsonDesc{
-		Id:             self.Id,
-		DevType:        self.DevType,
-		Model:          self.Model,
-		Addr:           self.Addr,
-		VendorDeviceId: self.VendorDeviceId,
-		Vendor:         self.getVendor(),
-		NetworkIndex:   self.NetworkIndex,
+		Id:                  self.Id,
+		DevType:             self.DevType,
+		Model:               self.Model,
+		Addr:                self.Addr,
+		VendorDeviceId:      self.VendorDeviceId,
+		Vendor:              self.getVendor(),
+		NetworkIndex:        self.NetworkIndex,
+		OvsOffloadInterface: self.OvsOffloadInterface,
 	}
 }
 

pkg/hostman/guestman/pci.go

@@ -281,6 +281,12 @@ func (s *SKVMGuestInstance) initGuestNetworks(pciRoot, pciBridge *desc.PCIContro
 	}
 
 	for i := 0; i < len(s.Desc.Nics); i++ {
+		if err := s.generateNicScripts(s.Desc.Nics[i]); err != nil {
+			return errors.Wrapf(err, "generateNicScripts for nic: %v", s.Desc.Nics[i])
+		}
+		s.Desc.Nics[i].UpscriptPath = s.getNicUpScriptPath(s.Desc.Nics[i])
+		s.Desc.Nics[i].DownscriptPath = s.getNicDownScriptPath(s.Desc.Nics[i])
+
 		if s.Desc.Nics[i].Driver != "vfio-pci" {
 			switch s.GetOsName() {
 			case OS_NAME_MACOS:
@@ -295,12 +301,6 @@ func (s *SKVMGuestInstance) initGuestNetworks(pciRoot, pciBridge *desc.PCIContro
 				s.Desc.Nics[i].Vectors = &vectors
 			}
 
-			if err := s.generateNicScripts(s.Desc.Nics[i]); err != nil {
-				return errors.Wrapf(err, "generateNicScripts for nic: %v", s.Desc.Nics[i])
-			}
-			s.Desc.Nics[i].UpscriptPath = s.getNicUpScriptPath(s.Desc.Nics[i])
-			s.Desc.Nics[i].DownscriptPath = s.getNicDownScriptPath(s.Desc.Nics[i])
-
 			id := fmt.Sprintf("netdev-%s", s.Desc.Nics[i].Ifname)
 			switch s.Desc.Nics[i].Driver {
 			case "virtio":

pkg/hostman/guestman/qemu-kvm.go

@@ -2741,12 +2741,19 @@ func getVfVlan(vlan int) int {
 func (s *SKVMGuestInstance) sriovNicAttachInitScript(networkIndex int8, dev isolated_device.IDevice) (string, error) {
 	for i := range s.Desc.Nics {
 		if s.Desc.Nics[i].Driver == "vfio-pci" && s.Desc.Nics[i].Index == networkIndex {
-			cmd := fmt.Sprintf(
-				"sriov_vf_init %s %d %s %d %s %d\n",
-				dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac,
-				getVfVlan(s.Desc.Nics[i].Vlan), srcMacCheckFunc(s.Desc.SrcMacCheck), s.Desc.Nics[i].Bw,
-			)
-			return sriovInitFunc + " && " + cmd, nil
+			if dev.GetOvsOffloadInterfaceName() != "" {
+				cmd := fmt.Sprintf("ip link set dev %s vf %d mac %s\n",
+					dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac)
+				cmd += s.getNicUpScriptPath(s.Desc.Nics[i]) + "\n"
+				return cmd, nil
+			} else {
+				cmd := fmt.Sprintf(
+					"sriov_vf_init %s %d %s %d %s %d\n",
+					dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac,
+					getVfVlan(s.Desc.Nics[i].Vlan), srcMacCheckFunc(s.Desc.SrcMacCheck), s.Desc.Nics[i].Bw,
+				)
+				return sriovInitFunc + " && " + cmd, nil
+			}
 		}
 	}
 	return "", errors.Errorf("no nic found for index %d", networkIndex)
@@ -2761,11 +2768,17 @@ func (s *SKVMGuestInstance) generateSRIOVInitScripts() (string, error) {
 			if err != nil {
 				return "", err
 			}
-			cmd += fmt.Sprintf(
-				"sriov_vf_init %s %d %s %d %s %d\n",
-				dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac,
-				getVfVlan(s.Desc.Nics[i].Vlan), srcMacCheckFunc(s.Desc.SrcMacCheck), s.Desc.Nics[i].Bw,
-			)
+			if dev.GetOvsOffloadInterfaceName() != "" {
+				cmd += fmt.Sprintf("ip link set dev %s vf %d mac %s\n",
+					dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac)
+				cmd += s.getNicUpScriptPath(s.Desc.Nics[i]) + "\n"
+			} else {
+				cmd += fmt.Sprintf(
+					"sriov_vf_init %s %d %s %d %s %d\n",
+					dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac,
+					getVfVlan(s.Desc.Nics[i].Vlan), srcMacCheckFunc(s.Desc.SrcMacCheck), s.Desc.Nics[i].Bw,
+				)
+			}
 		}
 	}
 	if len(cmd) > 0 {

pkg/hostman/guestman/qemu-kvmhelper.go

@@ -662,7 +662,10 @@ func (s *SKVMGuestInstance) generateStopScript(data *jsonutils.JSONDict) string
 
 	for _, nic := range nics {
 		if nic.Driver == api.NETWORK_DRIVER_VFIO {
-			continue
+			dev, _ := s.GetSriovDeviceByNetworkIndex(nic.Index)
+			if dev != nil && dev.GetOvsOffloadInterfaceName() == "" {
+				continue
+			}
 		}
 		downscript := s.getNicDownScriptPath(nic)
 		cmd += fmt.Sprintf("%s %s\n", downscript, nic.Ifname)

pkg/hostman/hostinfo/hostbridge/ovs.go

@@ -191,10 +191,13 @@ func (o *SOVSBridgeDriver) getUpScripts(nic *desc.SGuestNetwork, isVolatileHost
 	s += "PORT=$(ovs-ofctl show $SWITCH | grep -w $IF)\n"
 	s += "PORT=$(echo $PORT | awk 'BEGIN{FS=\"(\"}{print $1}')\n"
 	s += "OFCTL=$(ovs-vsctl get-controller $SWITCH)\n"
-	s += "if [ -z \"$OFCTL\" ]; then\n"
-	s += "    ovs-vsctl set Interface $IF ingress_policing_rate=$LIMIT\n"
-	s += "    ovs-vsctl set Interface $IF ingress_policing_burst=$BURST\n"
-	s += "fi\n"
+	if nic.Driver != compute.NETWORK_DRIVER_VFIO {
+		s += "if [ -z \"$OFCTL\" ]; then\n"
+		s += "    ovs-vsctl set Interface $IF ingress_policing_rate=$LIMIT\n"
+		s += "    ovs-vsctl set Interface $IF ingress_policing_burst=$BURST\n"
+		s += "fi\n"
+	}
+
 	s += "if [ $LIMIT_DOWNLOAD != \"0mbit\" ]; then\n"
 	s += "    tc qdisc del dev $IF root 2>/dev/null\n"
 	s += "    tc qdisc add dev $IF root handle 1: htb default 10\n"

pkg/hostman/hostinfo/hostinfo.go

@@ -1842,18 +1842,52 @@ func (h *SHostInfo) probeSyncIsolatedDevicesStep() {
 	h.deployAdminAuthorizedKeys()
 }
 
-func (h *SHostInfo) getNicsInterfaces() [][2]string {
-	res := [][2]string{}
+func (h *SHostInfo) getNicsInterfaces(nics []string) []isolated_device.HostNic {
+	if len(nics) == 0 {
+		return nil
+	}
+	res := []isolated_device.HostNic{}
 	for i := 0; i < len(h.Nics); i++ {
-		res = append(res, [2]string{h.Nics[i].Inter, h.Nics[i].WireId})
+		if utils.IsInStringArray(h.Nics[i].Inter, nics) {
+			res = append(res, isolated_device.HostNic{h.Nics[i].Bridge, h.Nics[i].Inter, h.Nics[i].WireId})
+		}
 	}
 	return res
 }
 
+func (h *SHostInfo) getNicsOvsOffloadInterfaces(nics []string) ([]isolated_device.HostNic, error) {
+	if len(nics) == 0 {
+		return nil, nil
+	}
+
+	res := []isolated_device.HostNic{}
+	for i := 0; i < len(h.Nics); i++ {
+		if utils.IsInStringArray(h.Nics[i].Inter, nics) {
+			if fileutils2.Exists(fmt.Sprintf("/sys/class/net/%s/bonding/slaves", h.Nics[i].Inter)) {
+				interStr, err := fileutils2.FileGetContents(fmt.Sprintf("/sys/class/net/%s/bonding/slaves", h.Nics[i].Inter))
+				if err != nil {
+					return nil, err
+				}
+				inters := strings.Split(strings.TrimSpace(interStr), " ")
+				for _, inter := range inters {
+					res = append(res, isolated_device.HostNic{h.Nics[i].Bridge, inter, h.Nics[i].WireId})
+				}
+			} else {
+				res = append(res, isolated_device.HostNic{h.Nics[i].Bridge, h.Nics[i].Inter, h.Nics[i].WireId})
+			}
+		}
+	}
+	return res, nil
+}
+
 func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) {
+	offloadNics, err := h.getNicsOvsOffloadInterfaces(options.HostOptions.OvsOffloadNics)
+	if err != nil {
+		return nil, err
+	}
+	sriovNics := h.getNicsInterfaces(options.HostOptions.SRIOVNics)
 	if err := h.IsolatedDeviceMan.ProbePCIDevices(
-		options.HostOptions.DisableGPU, options.HostOptions.DisableUSB,
-		options.HostOptions.DisableSRIOVNic, h.getNicsInterfaces(),
+		options.HostOptions.DisableGPU, options.HostOptions.DisableUSB, sriovNics, offloadNics,
 	); err != nil {
 		return nil, errors.Wrap(err, "ProbePCIDevices")
 	}

pkg/hostman/isolated_device/isolated_device.go

@@ -66,6 +66,7 @@ type IDevice interface {
 	SetHostId(hId string)
 	GetGuestId() string
 	GetWireId() string
+	GetOvsOffloadInterfaceName() string
 	GetVendorDeviceId() string
 	GetAddr() string
 	GetDeviceType() string
@@ -95,7 +96,7 @@ type IsolatedDeviceManager interface {
 	GetDevices() []IDevice
 	GetDeviceByIdent(vendorDevId string, addr string) IDevice
 	GetDeviceByAddr(addr string) IDevice
-	ProbePCIDevices(skipGPUs, skipUSBs, skipSRIOVNics bool, nics [][2]string) error
+	ProbePCIDevices(skipGPUs, skipUSBs bool, sriovNics, ovsOffloadNics []HostNic) error
 	StartDetachTask()
 	BatchCustomProbe() error
 	AppendDetachedDevice(dev *CloudDeviceInfo)
@@ -122,7 +123,13 @@ func (man *isolatedDeviceManager) GetDevices() []IDevice {
 	return man.devices
 }
 
-func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs, skipSRIOVNics bool, nics [][2]string) error {
+type HostNic struct {
+	Bridge    string
+	Interface string
+	Wire      string
+}
+
+func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs bool, sriovNics, ovsOffloadNics []HostNic) error {
 	man.devices = make([]IDevice, 0)
 	if !skipGPUs {
 		gpus, err := getPassthroughGPUS()
@@ -149,8 +156,8 @@ func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs, skipSRIOVN
 		}
 	}
 
-	if !skipSRIOVNics {
-		nics, err := getSRIOVNics(nics)
+	if len(sriovNics) > 0 {
+		nics, err := getSRIOVNics(sriovNics)
 		if err != nil {
 			log.Errorf("getSRIOVNics: %v", err)
 			return nil
@@ -160,6 +167,17 @@ func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs, skipSRIOVN
 			log.Infof("Add sriov nic: %d => %#v", idx, nic)
 		}
 	}
+	if len(ovsOffloadNics) > 0 {
+		nics, err := getOvsOffloadNics(ovsOffloadNics)
+		if err != nil {
+			log.Errorf("getOvsOffloadNics: %v", err)
+			return nil
+		}
+		for idx, nic := range nics {
+			man.devices = append(man.devices, nic)
+			log.Infof("Add sriov nic: %d => %#v", idx, nic)
+		}
+	}
 
 	return nil
 }
@@ -327,6 +345,10 @@ func (dev *sBaseDevice) GetVirtfn() int {
 	return -1
 }
 
+func (dev *sBaseDevice) GetOvsOffloadInterfaceName() string {
+	return ""
+}
+
 func (dev *sBaseDevice) GetModelName() string {
 	if dev.dev.ModelName != "" {
 		return dev.dev.ModelName
@@ -363,6 +385,9 @@ func GetApiResourceData(dev IDevice) *jsonutils.JSONDict {
 	if len(dev.GetWireId()) != 0 {
 		data["wire_id"] = dev.GetWireId()
 	}
+	if len(dev.GetOvsOffloadInterfaceName()) != 0 {
+		data["ovs_offload_interface"] = dev.GetOvsOffloadInterfaceName()
+	}
 	return jsonutils.Marshal(data).(*jsonutils.JSONDict)
 }
 

pkg/hostman/isolated_device/nic.go

@@ -27,16 +27,17 @@ import (
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
-func getSRIOVNics(hostNics [][2]string) ([]*sSRIOVNicDevice, error) {
+func getSRIOVNics(hostNics []HostNic) ([]*sSRIOVNicDevice, error) {
 	sysfsNetDir := "/sys/class/net"
 	files, err := ioutil.ReadDir(sysfsNetDir)
 	if err != nil {
 		return nil, err
 	}
-	nics := [][2]string{}
+	nics := []HostNic{}
 	for i := 0; i < len(files); i++ {
 		nicPath, err := filepath.EvalSymlinks(path.Join(sysfsNetDir, files[i].Name()))
 		if err != nil {
@@ -47,15 +48,20 @@ func getSRIOVNics(hostNics [][2]string) ([]*sSRIOVNicDevice, error) {
 		}
 		var idx int
 		for idx = range hostNics {
-			if hostNics[idx][0] == files[i].Name() {
+			if hostNics[idx].Interface == files[i].Name() {
 				nics = append(nics, hostNics[idx])
 			}
 		}
 	}
+
 	log.Infof("host nics %s detected support sriov nics %v", hostNics, nics)
 	sriovNics := make([]*sSRIOVNicDevice, 0)
 	for i := 0; i < len(nics); i++ {
-		nicDir := path.Join(sysfsNetDir, nics[i][0], "device")
+		nicDir := path.Join(sysfsNetDir, nics[i].Interface, "device")
+		err = ensureNumvfsEqualTotalvfs(nicDir)
+		if err != nil {
+			return nil, err
+		}
 		vfs, err := ioutil.ReadDir(nicDir)
 		if err != nil {
 			return nil, err
@@ -75,13 +81,31 @@ func getSRIOVNics(hostNics [][2]string) ([]*sSRIOVNicDevice, error) {
 				if err != nil {
 					return nil, err
 				}
-				sriovNics = append(sriovNics, NewSRIOVNicDevice(vfDev, api.NIC_TYPE, nics[i][1], nics[i][0], virtfn))
+				sriovNics = append(sriovNics, NewSRIOVNicDevice(vfDev, api.NIC_TYPE, nics[i].Wire, nics[i].Interface, virtfn))
 			}
 		}
 	}
 	return sriovNics, nil
 }
 
+func ensureNumvfsEqualTotalvfs(nicDir string) error {
+	sriovNumvfs := path.Join(nicDir, "sriov_numvfs")
+	sriovTotalvfs := path.Join(nicDir, "sriov_totalvfs")
+	numvfs, err := fileutils2.FileGetContents(sriovNumvfs)
+	if err != nil {
+		return err
+	}
+	totalvfs, err := fileutils2.FileGetContents(sriovTotalvfs)
+	if err != nil {
+		return err
+	}
+	log.Errorf("numvfs %s total vfs %s", numvfs, totalvfs)
+	if numvfs != totalvfs {
+		return fileutils2.FilePutContents(sriovNumvfs, fmt.Sprintf("%s", totalvfs), false)
+	}
+	return nil
+}
+
 func detectSRIOVNicDevice(vfBDF string) (*PCIDevice, error) {
 	dev, err := detectPCIDevByAddrWithoutIOMMUGroup(vfBDF)
 	if err != nil {
@@ -206,3 +230,170 @@ func (dev *sSRIOVNicDevice) CustomProbe(idx int) error {
 	}
 	return nil
 }
+
+func getOvsOffloadNics(hostNics []HostNic) ([]*sOvsOffloadNicDevice, error) {
+	sysfsNetDir := "/sys/class/net"
+	files, err := ioutil.ReadDir(sysfsNetDir)
+	if err != nil {
+		return nil, errors.Wrap(err, "ioutil.ReadDir(sysfsNetDir)")
+	}
+	nics := []HostNic{}
+	for i := 0; i < len(files); i++ {
+		nicPath, err := filepath.EvalSymlinks(path.Join(sysfsNetDir, files[i].Name()))
+		if err != nil {
+			return nil, errors.Wrap(err, "filepath.EvalSymlinks nicPath")
+		}
+		if !strings.HasPrefix(nicPath, "/sys/devices/pci0000:") {
+			continue
+		}
+		var idx int
+		for idx = range hostNics {
+			if hostNics[idx].Interface == files[i].Name() {
+				nics = append(nics, hostNics[idx])
+			}
+		}
+	}
+
+	sriovNics := make([]*sOvsOffloadNicDevice, 0)
+	for i := range nics {
+		nicDir := path.Join(sysfsNetDir, nics[i].Interface, "device")
+		err = ensureNumvfsEqualTotalvfs(nicDir)
+		if err != nil {
+			return nil, errors.Wrap(err, "ensureNumvfsEqualTotalvfs")
+		}
+
+		vfs, err := ioutil.ReadDir(nicDir)
+		if err != nil {
+			return nil, errors.Wrap(err, "ioutil.ReadDir")
+		}
+
+		pfPath, err := filepath.EvalSymlinks(nicDir)
+		if err != nil {
+			return nil, errors.Wrap(err, "filepath.EvalSymlinks pfPath")
+		}
+		pfBDF := path.Base(pfPath)
+
+		// /sys/class/net/ens1f0/compat/devlink/mode
+		devlinkPath := path.Join(sysfsNetDir, nics[i].Interface, "compat/devlink/mode")
+		linkMode, err := fileutils2.FileGetContents(devlinkPath)
+		if err != nil {
+			return nil, errors.Wrap(err, "fileutils2.FileGetContents(devlinkPath)")
+		}
+		log.Infof("nic %s link mode %s", nics[i].Interface, linkMode)
+		if strings.TrimSpace(linkMode) != "switchdev" {
+			err = fileutils2.FilePutContents(devlinkPath, "switchdev\n", false)
+			if err != nil {
+				return nil, errors.Wrap(err, "fileutils2.FilePutContents linkMode")
+			}
+			for j := 0; j < len(vfs); j++ {
+				if strings.HasPrefix(vfs[j].Name(), "virtfn") {
+					vfPath, err := filepath.EvalSymlinks(path.Join(nicDir, vfs[j].Name()))
+					if err != nil {
+						return nil, errors.Wrap(err, "filepath.EvalSymlinks")
+					}
+					vfBDF := path.Base(vfPath)
+					dev, err := detectPCIDevByAddrWithoutIOMMUGroup(vfBDF)
+					if err != nil {
+						return nil, errors.Wrap(err, "detectPCIDevByAddrWithoutIOMMUGroup")
+					}
+					err = dev.unbindDriver()
+					if err != nil {
+						return nil, errors.Wrap(err, "unbindDriver")
+					}
+				}
+			}
+		}
+
+		// get interfaces
+		// grep -e 'PCI_SLOT_NAME=.*04:00.1'  /sys/class/net/*/device/uevent
+		outs, err := procutils.NewRemoteCommandAsFarAsPossible("sh", "-c",
+			fmt.Sprintf("grep -e PCI_SLOT_NAME=.*%s /sys/class/net/*/device/uevent", pfBDF)).Output()
+		if err != nil {
+			return nil, errors.Wrap(err, "procutils.NewRemoteCommandAsFarAsPossible grep PCI_SLOT_NAME")
+		}
+		vfNames := map[int]string{}
+		for _, line := range strings.Split(string(outs), "\n") {
+			line = strings.TrimSpace(line)
+			segs := strings.Split(line, "/")
+			if len(segs) < 5 {
+				continue
+			}
+			vfName := segs[4]
+			if vfName == nics[i].Interface {
+				continue
+			}
+			// eg: pf0vf1
+			portname, err := fileutils2.FileGetContents(fmt.Sprintf("/sys/class/net/%s/phys_port_name", vfName))
+			if err != nil {
+				return nil, errors.Wrap(err, "fileutils2.FileGetContents portname")
+			}
+			sp := strings.Split(portname, "vf")
+			if len(sp) != 2 {
+				return nil, errors.Errorf("%s bad portname %s", vfName, portname)
+			}
+			virtfn, err := strconv.Atoi(strings.TrimSpace(sp[1]))
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed parse %s portname %s", vfName, portname)
+			}
+			vfNames[virtfn] = vfName
+		}
+		log.Infof("vfnames: %v", vfNames)
+
+		for j := 0; j < len(vfs); j++ {
+			if strings.HasPrefix(vfs[j].Name(), "virtfn") {
+				virtfn, err := strconv.Atoi(vfs[j].Name()[len("virtfn"):])
+				if err != nil {
+					return nil, errors.Wrap(err, "strconv.Atoi virtfn")
+				}
+				vfPath, err := filepath.EvalSymlinks(path.Join(nicDir, vfs[j].Name()))
+				if err != nil {
+					return nil, errors.Wrap(err, "filepath.EvalSymlinks vfpath")
+				}
+				vfBDF := path.Base(vfPath)
+				vfDev, err := detectSRIOVNicDevice(vfBDF)
+				if err != nil {
+					return nil, errors.Wrap(err, "detectSRIOVNicDevice")
+				}
+				// out, err := procutils.NewRemoteCommandAsFarAsPossible(
+				// 	"ovs-vsctl", "--may-exist", "add-port", nics[i].Bridge, vfNames[virtfn]).Output()
+				// if err != nil {
+				// 	return nil, errors.Wrapf(err, " ovs-vsctl add-port %s %s failed: %s", nics[i].Bridge, vfNames[virtfn], out)
+				// }
+
+				sriovNics = append(sriovNics, NewSRIOVOffloadNicDevice(
+					vfDev, api.NIC_TYPE, nics[i].Wire, nics[i].Interface, virtfn, vfNames[virtfn]),
+				)
+			}
+		}
+	}
+
+	// ovs-vsctl set Open_vSwitch . other_config:hw-offload=true
+	out, err := procutils.NewRemoteCommandAsFarAsPossible(
+		"ovs-vsctl", "set", "Open_vSwitch", ".", "other_config:hw-offload=true").Output()
+	if err != nil {
+		return nil, errors.Wrapf(err, "ovs enable hw offload failed: %s", out)
+	}
+
+	return sriovNics, nil
+}
+
+type sOvsOffloadNicDevice struct {
+	*sSRIOVNicDevice
+
+	interfaceName string
+}
+
+func NewSRIOVOffloadNicDevice(dev *PCIDevice, devType, wireId, pfName string, virtfn int, ifname string) *sOvsOffloadNicDevice {
+	return &sOvsOffloadNicDevice{
+		sSRIOVNicDevice: NewSRIOVNicDevice(dev, devType, wireId, pfName, virtfn),
+		interfaceName:   ifname,
+	}
+}
+
+func (dev *sOvsOffloadNicDevice) setInterfaceName(ifname string) {
+	dev.interfaceName = ifname
+}
+
+func (dev *sOvsOffloadNicDevice) GetOvsOffloadInterfaceName() string {
+	return dev.interfaceName
+}

pkg/hostman/options/options.go

@@ -170,9 +170,10 @@ type SHostOptions struct {
 
 	DisableKVM bool `help:"force disable KVM" default:"false" json:"disable_kvm"`
 
-	DisableGPU      bool `help:"force disable GPU detect" default:"false" json:"disable_gpu"`
-	DisableUSB      bool `help:"force disable USB detect" default:"true" json:"disable_usb"`
-	DisableSRIOVNic bool `help:"force disable USB detect" default:"true" json:"disable_sriov_nic"`
+	DisableGPU     bool     `help:"force disable GPU detect" default:"false" json:"disable_gpu"`
+	DisableUSB     bool     `help:"force disable USB detect" default:"true" json:"disable_usb"`
+	SRIOVNics      []string `help:"nics enable sriov" json:"sriov_nics"`
+	OvsOffloadNics []string `help:"nics enable ovs offload" json:"ovs_offload_nics"`
 
 	EthtoolEnableGso bool `help:"use ethtool to turn on or off GSO(generic segment offloading)" default:"false" json:"ethtool_enable_gso"`