From edeffed5fe795b3c56da8071c30edc998d795dc3 Mon Sep 17 00:00:00 2001 From: wanyaoqi <18528551+wanyaoqi@users.noreply.github.com> Date: Tue, 14 Jun 2022 02:38:09 +0800 Subject: [PATCH] feat(region,host): support multiqueues on vhost network (#14455) Guest network add attribute num_queues, one is default. And we can assign queues on guest create, eg: server-create --net num-queues=4 Signed-off-by: wanyaoqi Co-authored-by: wanyaoqi --- pkg/apis/compute/api.go | 11 ++++++----- pkg/apis/compute/guestnetwork.go | 1 + pkg/cloudcommon/cmdline/parser.go | 2 ++ pkg/compute/models/guestnetworks.go | 6 ++++++ pkg/compute/models/guests.go | 13 +++++++++++++ pkg/hostman/guestman/qemu-kvmhelper.go | 7 +++++++ pkg/hostman/guestman/qemu/generate.go | 9 +++++++++ 7 files changed, 44 insertions(+), 5 deletions(-) diff --git a/pkg/apis/compute/api.go b/pkg/apis/compute/api.go index f2794a4020..cfce12f734 100644 --- a/pkg/apis/compute/api.go +++ b/pkg/apis/compute/api.go @@ -62,11 +62,12 @@ type NetworkConfig struct { // 驱动方式 // 若指定镜像的网络驱动方式，此参数会被覆盖 - Driver string `json:"driver"` - BwLimit int `json:"bw_limit"` - Vip bool `json:"vip"` - Reserved bool `json:"reserved"` - NetType string `json:"net_type"` + Driver string `json:"driver"` + BwLimit int `json:"bw_limit"` + Vip bool `json:"vip"` + Reserved bool `json:"reserved"` + NetType string `json:"net_type"` + NumQueues int `json:"num_queues"` RequireDesignatedIP bool `json:"require_designated_ip"` diff --git a/pkg/apis/compute/guestnetwork.go b/pkg/apis/compute/guestnetwork.go index 3878c5befd..a9eba886e2 100644 --- a/pkg/apis/compute/guestnetwork.go +++ b/pkg/apis/compute/guestnetwork.go @@ -90,6 +90,7 @@ type GuestnetworkJsonDesc struct { Ifname string `json:"ifname"` Masklen int8 `json:"masklen"` Driver string `json:"driver"` + NumQueues int `json:"num_queues"` Vlan int `json:"vlan"` Bw int `json:"bw"` Mtu int `json:"mtu"` diff --git a/pkg/cloudcommon/cmdline/parser.go b/pkg/cloudcommon/cmdline/parser.go index c847bb2462..5b0927d328 100644 --- a/pkg/cloudcommon/cmdline/parser.go +++ b/pkg/cloudcommon/cmdline/parser.go @@ -242,6 +242,8 @@ func ParseNetworkConfig(desc string, idx int) (*compute.NetworkConfig, error) { netConfig.StandbyAddrCount, _ = strconv.Atoi(p[len("standby-addr="):]) } else if utils.IsInStringArray(p, []string{"virtio", "e1000", "vmxnet3"}) { netConfig.Driver = p + } else if strings.HasPrefix(p, "num-queues=") { + netConfig.NumQueues, _ = strconv.Atoi(p[len("num-queues="):]) } else if regutils.MatchSize(p) { bw, err := fileutils.GetSizeMb(p, 'M', 1000) if err != nil { diff --git a/pkg/compute/models/guestnetworks.go b/pkg/compute/models/guestnetworks.go index 0802ff1725..bf987b4327 100644 --- a/pkg/compute/models/guestnetworks.go +++ b/pkg/compute/models/guestnetworks.go @@ -85,6 +85,8 @@ type SGuestnetwork struct { Ip6Addr string `width:"64" charset:"ascii" nullable:"true" list:"user"` // 虚拟网卡驱动 Driver string `width:"16" charset:"ascii" nullable:"true" list:"user" update:"user"` + // 网卡队列数 + NumQueues int `nullable:"true" default:"1" list:"user" update:"user"` // 带宽限制，单位mbps BwLimit int `nullable:"false" default:"0" list:"user"` // 网卡序号 @@ -256,6 +258,7 @@ type newGuestNetworkArgs struct { macAddr string bwLimit int nicDriver string + numQueues int teamWithMac string virtual bool @@ -276,6 +279,7 @@ func (manager *SGuestnetworkManager) newGuestNetwork( address = args.ipAddr mac = args.macAddr driver = args.nicDriver + numQueues = args.numQueues bwLimit = args.bwLimit virtual = args.virtual reserved = args.tryReserved @@ -294,6 +298,7 @@ func (manager *SGuestnetworkManager) newGuestNetwork( driver = "virtio" } gn.Driver = driver + gn.NumQueues = numQueues if bwLimit >= 0 { gn.BwLimit = bwLimit } @@ -584,6 +589,7 @@ func (self *SGuestnetwork) getJsonDesc() *api.GuestnetworkJsonDesc { desc.Ifname = self.GetIfname() desc.Masklen = net.GuestIpMask desc.Driver = self.Driver + desc.NumQueues = self.NumQueues desc.Vlan = net.VlanId desc.Bw = self.getBandwidth() desc.Mtu = self.getMtu(net) diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 82994af4eb..abd42b5d45 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -2951,6 +2951,7 @@ type Attach2NetworkArgs struct { BwLimit int NicDriver string + NumQueues int NicConfs []SNicConfig Virtual bool @@ -2973,6 +2974,7 @@ func (args *Attach2NetworkArgs) onceArgs(i int) attach2NetworkOnceArgs { bwLimit: args.BwLimit, nicDriver: args.NicDriver, + numQueues: args.NumQueues, nicConf: args.NicConfs[i], virtual: args.Virtual, @@ -2988,6 +2990,7 @@ func (args *Attach2NetworkArgs) onceArgs(i int) attach2NetworkOnceArgs { r.useDesignatedIP = false r.nicConf = args.NicConfs[i] r.nicDriver = "" + r.numQueues = 1 } return r } @@ -3003,6 +3006,7 @@ type attach2NetworkOnceArgs struct { bwLimit int nicDriver string + numQueues int nicConf SNicConfig teamWithMac string @@ -3073,6 +3077,7 @@ func (self *SGuest) attach2NetworkOnce( macAddr: args.nicConf.Mac, bwLimit: args.bwLimit, nicDriver: nicDriver, + numQueues: args.numQueues, teamWithMac: args.teamWithMac, virtual: args.virtual, @@ -3647,6 +3652,13 @@ func (self *SGuest) CreateNetworksOnHost( if candidateNet != nil { networkIds = candidateNet.NetworkIds } + if idx == 0 && netConfig.NumQueues == 0 { + numQueues := self.VcpuCount / 2 + if numQueues > 16 { + numQueues = 16 + } + netConfig.NumQueues = numQueues + } _, err = self.attach2NetworkDesc(ctx, userCred, host, netConfig, pendingUsage, networkIds) if err != nil { return errors.Wrap(err, "self.attach2NetworkDesc") @@ -3711,6 +3723,7 @@ func (self *SGuest) attach2NamedNetworkDesc(ctx context.Context, userCred mcclie PendingUsage: pendingUsage, IpAddr: netConfig.Address, NicDriver: netConfig.Driver, + NumQueues: netConfig.NumQueues, BwLimit: netConfig.BwLimit, Virtual: netConfig.Vip, TryReserved: netConfig.Reserved, diff --git a/pkg/hostman/guestman/qemu-kvmhelper.go b/pkg/hostman/guestman/qemu-kvmhelper.go index 3c63469fd1..7bac496a80 100644 --- a/pkg/hostman/guestman/qemu-kvmhelper.go +++ b/pkg/hostman/guestman/qemu-kvmhelper.go @@ -434,6 +434,13 @@ function nic_mtu() { } // inject nic and disks + for i := 0; i < len(input.Nics); i++ { + nic := nics[i].(*jsonutils.JSONDict) + if numQueues, _ := nic.Int("num_queues"); numQueues > 1 { + nic.Set("vectors", jsonutils.NewInt(2*numQueues+1)) + } + } + if input.OsName == OS_NAME_MACOS { for i := 0; i < len(input.Disks); i++ { disks[i].Driver = DISK_DRIVER_SATA diff --git a/pkg/hostman/guestman/qemu/generate.go b/pkg/hostman/guestman/qemu/generate.go index 96d85b1379..aa3ffb0ee2 100644 --- a/pkg/hostman/guestman/qemu/generate.go +++ b/pkg/hostman/guestman/qemu/generate.go @@ -433,11 +433,16 @@ func getNicNetdevOption(drvOpt QemuOptions, nic jsonutils.JSONObject, isKVMSuppo if downscript == "" { return "", errors.Error("downscript_path is empty") } + numQueues, _ := nic.Int("num_queues") + opt := "-netdev type=tap" opt += fmt.Sprintf(",id=%s", ifname) opt += fmt.Sprintf(",ifname=%s", ifname) if driver == "virtio" && isKVMSupport { opt += ",vhost=on,vhostforce=off" + if numQueues > 1 { + opt += fmt.Sprintf(",queues=%d", numQueues) + } } opt += fmt.Sprintf(",script=%s", upscript) opt += fmt.Sprintf(",downscript=%s", downscript) @@ -452,6 +457,7 @@ func getNicDeviceOption(drvOpt QemuOptions, nic jsonutils.JSONObject, input *Gen index, _ := nic.Int("index") vectors, _ := nic.Int("vectors") bw, _ := nic.Int("bw") + numQueues, _ := nic.Int("num_queues") cmd := fmt.Sprintf("-device %s", GetNicDeviceModel(driver)) cmd += fmt.Sprintf(",id=netdev-%s", ifname) @@ -467,6 +473,9 @@ func getNicDeviceOption(drvOpt QemuOptions, nic jsonutils.JSONObject, input *Gen cmd += fmt.Sprintf(",addr=0x%x", GetNicAddr(int(index), disksLen, isoDevsLen, input.IsVdiSpice)) } if driver == "virtio" { + if numQueues > 1 { + cmd += fmt.Sprintf(",mq=on") + } if nic.Contains("vectors") { cmd += fmt.Sprintf(",vectors=%d", vectors) }