fix: compatible with new io scheduler(bfq/mq-deadline) of new Linux kernel

1. support bfq/mq-deadline multiqueue io scheduler
2. add delay for link setup

pkg/hostman/hostinfo/hostbridge/hostbridge.go

@@ -21,6 +21,7 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
+	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -233,7 +234,12 @@ func (d *SBaseBridgeDriver) SetupRoutes(routespecs []iproute2.RouteSpec) error {
 	br := d.bridge.String()
 	r := iproute2.NewRoute(br)
 	for _, routespec := range routespecs {
-		r.AddByRouteSpec(routespec)
+		rt := iproute2.RouteSpec{
+			Dst: routespec.Dst,
+			Src: routespec.Src,
+			Gw:  routespec.Gw,
+		}
+		r.AddByRouteSpec(rt)
 	}
 	if err := r.Err(); err != nil {
 		return errors.Wrapf(err, "set routes on %s", br)
@@ -272,14 +278,39 @@ func (d *SBaseBridgeDriver) Setup(o IBridgeDriver) error {
 			if err := o.SetupAddresses(d.inter.Mask); err != nil {
 				return err
 			}
+			time.Sleep(1 * time.Second)
 			if len(slaveAddrs) > 0 {
-				if err := o.SetupSlaveAddresses(slaveAddrs); err != nil {
-					return err
+				tried := 0
+				const MAX_TRIES = 4
+				for tried < MAX_TRIES {
+					if err := o.SetupSlaveAddresses(slaveAddrs); err != nil {
+						log.Errorf("SetupSlaveAddresses fail: %s", err)
+						tried += 1
+						if tried >= MAX_TRIES {
+							return err
+						} else {
+							time.Sleep(time.Duration(tried) * time.Second)
+						}
+					} else {
+						break
+					}
 				}
 			}
 			if len(routes) > 0 {
-				if err := o.SetupRoutes(routes); err != nil {
-					return err
+				tried := 0
+				const MAX_TRIES = 4
+				for {
+					if err := o.SetupRoutes(routes); err != nil {
+						log.Errorf("SetupRoutes fail: %s", err)
+						tried += 1
+						if tried >= MAX_TRIES {
+							return err
+						} else {
+							time.Sleep(time.Duration(tried) * time.Second)
+						}
+					} else {
+						break
+					}
 				}
 			}
 		} else {

pkg/hostman/hostinfo/hostinfo.go

@@ -96,6 +96,8 @@ type SHostInfo struct {
 	FullName   string
 	SysError   map[string]string
 	SysWarning map[string]string
+
+	IoScheduler string
 }
 
 func (h *SHostInfo) GetIsolatedDeviceManager() *isolated_device.IsolatedDeviceManager {
@@ -336,11 +338,39 @@ func (h *SHostInfo) prepareEnv() error {
 		return errors.Wrap(err, "Execute 'ethtool -h'")
 	}
 
+	supportedSchedulers, _ := fileutils2.GetAllBlkdevsIoSchedulers()
+	// IoScheduler default to none scheduler
 	ioParams := make(map[string]string, 0)
-	if options.HostOptions.BlockIoScheduler == "deadline" {
-		ioParams["queue/scheduler"] = "deadline"
-	} else {
-		ioParams["queue/scheduler"] = "cfq"
+	switch options.HostOptions.BlockIoScheduler {
+	case "deadline":
+		if utils.IsInStringArray("mq-deadline", supportedSchedulers) {
+			h.IoScheduler = "mq-deadline"
+		} else if utils.IsInStringArray("deadline", supportedSchedulers) {
+			h.IoScheduler = "deadline"
+		} else {
+			h.IoScheduler = "none"
+		}
+	case "cfq":
+		if utils.IsInStringArray("bfq", supportedSchedulers) {
+			h.IoScheduler = "bfq"
+		} else if utils.IsInStringArray("cfq", supportedSchedulers) {
+			h.IoScheduler = "cfq"
+		} else {
+			h.IoScheduler = "none"
+		}
+	default:
+		if utils.IsInStringArray(options.HostOptions.BlockIoScheduler, supportedSchedulers) {
+			h.IoScheduler = options.HostOptions.BlockIoScheduler
+		} else {
+			h.IoScheduler = "none"
+		}
+	}
+
+	log.Infof("I/O Scheduler switch to %s", h.IoScheduler)
+
+	ioParams["queue/scheduler"] = h.IoScheduler
+	switch h.IoScheduler {
+	case "cfq":
 		ioParams["queue/iosched/group_isolation"] = "1"
 		ioParams["queue/iosched/slice_idle"] = "0"
 		ioParams["queue/iosched/group_idle"] = "0"
@@ -356,7 +386,7 @@ func (h *SHostInfo) prepareEnv() error {
 		log.Warningf("modprobe vhost_net error: %s", output)
 	}
 	if !options.HostOptions.DisableSetCgroup {
-		if !cgrouputils.Init() {
+		if !cgrouputils.Init(h.IoScheduler) {
 			return fmt.Errorf("Cannot initialize control group subsystem")
 		}
 	}

pkg/util/cgrouputils/cgrouputils.go

@@ -406,11 +406,23 @@ type CGroupIOTask struct {
 	*CGroupTask
 }
 
+var (
+	/*
+	 * A global IoScheduler variable, should be set before initialize CGROUP
+	 */
+	IoScheduler string
+)
+
 const (
 	IoWeightBase    = 100
 	IoWeightMax     = 1000
 	IoWeightMin     = 100
 	BLOCK_IO_WEIGHT = "blkio.weight"
+
+	BLOCK_IO_BFQ_WEIGHT = "blkio.bfq.weight"
+
+	IOSCHED_CFQ = "cfq"
+	IOSCHED_BFQ = "bfq"
 )
 
 func (c *CGroupIOTask) Module() string {
@@ -424,11 +436,23 @@ func (c *CGroupIOTask) GetConfig() map[string]string {
 	} else if wt < IoWeightMin {
 		wt = IoWeightMin
 	}
-	return map[string]string{BLOCK_IO_WEIGHT: fmt.Sprintf("%d", wt)}
+	switch IoScheduler {
+	case IOSCHED_CFQ:
+		return map[string]string{BLOCK_IO_WEIGHT: fmt.Sprintf("%d", wt)}
+	case IOSCHED_BFQ:
+		return map[string]string{BLOCK_IO_BFQ_WEIGHT: fmt.Sprintf("%d", wt)}
+	default:
+		return nil
+	}
 }
 
 func (c *CGroupIOTask) init() bool {
-	return SetRootParam(c.Module(), BLOCK_IO_WEIGHT, fmt.Sprintf("%d", IoWeightMax), "")
+	switch IoScheduler {
+	case IOSCHED_CFQ:
+		return SetRootParam(c.Module(), BLOCK_IO_WEIGHT, fmt.Sprintf("%d", IoWeightMax), "")
+	default:
+		return true
+	}
 }
 
 func NewCGroupIOTask(pid string, coreNum int) *CGroupIOTask {
@@ -536,7 +560,8 @@ func NewCGroupCPUSetTask(pid string, coreNum int, cpuset string) CGroupCPUSetTas
 	return task
 }
 
-func Init() bool {
+func Init(ioScheduler string) bool {
+	IoScheduler = ioScheduler
 	for _, hand := range []ICGroupTask{&CGroupTask{}, &CGroupCPUTask{}, &CGroupIOTask{}} {
 		if !hand.init() {
 			return false

pkg/util/fileutils2/fileutils.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/util/procutils"
 )
@@ -139,6 +140,40 @@ func IsBlockDeviceUsed(dev string) bool {
 	return false
 }
 
+func GetAllBlkdevsIoSchedulers() ([]string, error) {
+	if _, err := os.Stat("/sys/block"); !os.IsNotExist(err) {
+		blockDevs, err := ioutil.ReadDir("/sys/block")
+		if err != nil {
+			log.Errorf("ReadDir /sys/block error: %s", err)
+			return nil, errors.Wrap(err, "ioutil.ReadDir(/sys/block)")
+		}
+		for _, b := range blockDevs {
+			if IsBlockDevMounted(b.Name()) {
+				conf, err := GetBlkdevConfig(b.Name(), "queue/scheduler")
+				if err != nil {
+					log.Errorf("Get %s queue/scheduler fail %s", b.Name(), err)
+				} else {
+					algs := make([]string, 0)
+					for _, alg := range strings.Split(strings.TrimSpace(conf), " ") {
+						if len(alg) > 0 {
+							if alg[0] == '[' {
+								alg = alg[1 : len(alg)-1]
+							}
+							algs = append(algs, alg)
+						}
+					}
+					return algs, nil
+				}
+			}
+		}
+		log.Errorf("no block device avaiable")
+		return nil, nil
+	} else {
+		log.Errorf("stat /sys/block fail %s", err)
+		return nil, errors.Wrap(err, "stat /sys/block fail")
+	}
+}
+
 func ChangeAllBlkdevsParams(params map[string]string) {
 	if _, err := os.Stat("/sys/block"); !os.IsNotExist(err) {
 		blockDevs, err := ioutil.ReadDir("/sys/block")
@@ -167,6 +202,15 @@ func ChangeBlkdevParameter(dev, key, value string) {
 	}
 }
 
+func GetBlkdevConfig(dev, key string) (string, error) {
+	p := path.Join("/sys/block", dev, key)
+	if _, err := os.Stat(p); !os.IsNotExist(err) {
+		return FileGetContents(p)
+	} else {
+		return "", err
+	}
+}
+
 func FileGetContents(file string) (string, error) {
 	content, err := ioutil.ReadFile(file)
 	if err != nil {

pkg/util/fileutils2/fileutils_test.go

@@ -14,12 +14,12 @@
 
 package fileutils2
 
-// TODO: rewrite this test
-/*
 import (
 	"testing"
 )
 
+// TODO: rewrite this test
+/*
 func TestIsBlockDeviceUsed(t *testing.T) {
 	type args struct {
 		dev string
@@ -68,3 +68,8 @@ func TestGetDevId(t *testing.T) {
 		})
 	}
 }*/
+
+func TestGetAllBlkdevsIoScheduler(t *testing.T) {
+	scheds, _ := GetAllBlkdevsIoSchedulers()
+	t.Logf("scheduler: %#v", scheds)
+}

pkg/util/iproute2/route_test.go

@@ -15,6 +15,7 @@
 package iproute2
 
 import (
+	"net"
 	"testing"
 
 	"github.com/vishvananda/netlink"
@@ -60,4 +61,42 @@ func TestRoute(t *testing.T) {
 			t.Errorf("expecting ESRCH, got %v", err)
 		}
 	})
+
+	t.Run("route add", func(t *testing.T) {
+		ifname1 := genDummyName(t)
+		dum := addDummy(t, ifname1)
+		defer delDummy(t, dum)
+
+		ifname := "lo"
+
+		for _, cidr := range [][]string{
+			[]string{
+				"127.1.0.1/24",
+				"127.0.0.2",
+			},
+			[]string{
+				"192.168.120.243/24",
+				"127.0.0.2",
+			},
+		} {
+			_, ipnet, err := net.ParseCIDR(cidr[0])
+			if err != nil {
+				t.Errorf("ParseCIDR error %s", err)
+				continue
+			}
+			gw := net.ParseIP(cidr[1])
+
+			r := NewRoute(ifname)
+			r.AddByIPNet(ipnet, gw)
+			if err := r.Err(); err != nil {
+				t.Errorf("got error %v", err)
+			}
+
+			r.DelByIPNet(ipnet)
+
+			if err := r.Err(); err != nil {
+				t.Errorf("del error %v", err)
+			}
+		}
+	})
 }