diff --git a/pkg/hostman/hostinfo/hostbridge/hostbridge.go b/pkg/hostman/hostinfo/hostbridge/hostbridge.go index 9919decb80..48bf3a120c 100644 --- a/pkg/hostman/hostinfo/hostbridge/hostbridge.go +++ b/pkg/hostman/hostinfo/hostbridge/hostbridge.go @@ -21,6 +21,7 @@ import ( "strconv" "strings" "syscall" + "time" "yunion.io/x/jsonutils" "yunion.io/x/log" @@ -233,7 +234,12 @@ func (d *SBaseBridgeDriver) SetupRoutes(routespecs []iproute2.RouteSpec) error { br := d.bridge.String() r := iproute2.NewRoute(br) for _, routespec := range routespecs { - r.AddByRouteSpec(routespec) + rt := iproute2.RouteSpec{ + Dst: routespec.Dst, + Src: routespec.Src, + Gw: routespec.Gw, + } + r.AddByRouteSpec(rt) } if err := r.Err(); err != nil { return errors.Wrapf(err, "set routes on %s", br) @@ -272,14 +278,39 @@ func (d *SBaseBridgeDriver) Setup(o IBridgeDriver) error { if err := o.SetupAddresses(d.inter.Mask); err != nil { return err } + time.Sleep(1 * time.Second) if len(slaveAddrs) > 0 { - if err := o.SetupSlaveAddresses(slaveAddrs); err != nil { - return err + tried := 0 + const MAX_TRIES = 4 + for tried < MAX_TRIES { + if err := o.SetupSlaveAddresses(slaveAddrs); err != nil { + log.Errorf("SetupSlaveAddresses fail: %s", err) + tried += 1 + if tried >= MAX_TRIES { + return err + } else { + time.Sleep(time.Duration(tried) * time.Second) + } + } else { + break + } } } if len(routes) > 0 { - if err := o.SetupRoutes(routes); err != nil { - return err + tried := 0 + const MAX_TRIES = 4 + for { + if err := o.SetupRoutes(routes); err != nil { + log.Errorf("SetupRoutes fail: %s", err) + tried += 1 + if tried >= MAX_TRIES { + return err + } else { + time.Sleep(time.Duration(tried) * time.Second) + } + } else { + break + } } } } else { diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index 1199461833..eb9028dab1 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -96,6 +96,8 @@ type SHostInfo struct { FullName string SysError map[string]string SysWarning map[string]string + + IoScheduler string } func (h *SHostInfo) GetIsolatedDeviceManager() *isolated_device.IsolatedDeviceManager { @@ -336,11 +338,39 @@ func (h *SHostInfo) prepareEnv() error { return errors.Wrap(err, "Execute 'ethtool -h'") } + supportedSchedulers, _ := fileutils2.GetAllBlkdevsIoSchedulers() + // IoScheduler default to none scheduler ioParams := make(map[string]string, 0) - if options.HostOptions.BlockIoScheduler == "deadline" { - ioParams["queue/scheduler"] = "deadline" - } else { - ioParams["queue/scheduler"] = "cfq" + switch options.HostOptions.BlockIoScheduler { + case "deadline": + if utils.IsInStringArray("mq-deadline", supportedSchedulers) { + h.IoScheduler = "mq-deadline" + } else if utils.IsInStringArray("deadline", supportedSchedulers) { + h.IoScheduler = "deadline" + } else { + h.IoScheduler = "none" + } + case "cfq": + if utils.IsInStringArray("bfq", supportedSchedulers) { + h.IoScheduler = "bfq" + } else if utils.IsInStringArray("cfq", supportedSchedulers) { + h.IoScheduler = "cfq" + } else { + h.IoScheduler = "none" + } + default: + if utils.IsInStringArray(options.HostOptions.BlockIoScheduler, supportedSchedulers) { + h.IoScheduler = options.HostOptions.BlockIoScheduler + } else { + h.IoScheduler = "none" + } + } + + log.Infof("I/O Scheduler switch to %s", h.IoScheduler) + + ioParams["queue/scheduler"] = h.IoScheduler + switch h.IoScheduler { + case "cfq": ioParams["queue/iosched/group_isolation"] = "1" ioParams["queue/iosched/slice_idle"] = "0" ioParams["queue/iosched/group_idle"] = "0" @@ -356,7 +386,7 @@ func (h *SHostInfo) prepareEnv() error { log.Warningf("modprobe vhost_net error: %s", output) } if !options.HostOptions.DisableSetCgroup { - if !cgrouputils.Init() { + if !cgrouputils.Init(h.IoScheduler) { return fmt.Errorf("Cannot initialize control group subsystem") } } diff --git a/pkg/util/cgrouputils/cgrouputils.go b/pkg/util/cgrouputils/cgrouputils.go index b338cdc51c..048eb93ec5 100644 --- a/pkg/util/cgrouputils/cgrouputils.go +++ b/pkg/util/cgrouputils/cgrouputils.go @@ -406,11 +406,23 @@ type CGroupIOTask struct { *CGroupTask } +var ( + /* + * A global IoScheduler variable, should be set before initialize CGROUP + */ + IoScheduler string +) + const ( IoWeightBase = 100 IoWeightMax = 1000 IoWeightMin = 100 BLOCK_IO_WEIGHT = "blkio.weight" + + BLOCK_IO_BFQ_WEIGHT = "blkio.bfq.weight" + + IOSCHED_CFQ = "cfq" + IOSCHED_BFQ = "bfq" ) func (c *CGroupIOTask) Module() string { @@ -424,11 +436,23 @@ func (c *CGroupIOTask) GetConfig() map[string]string { } else if wt < IoWeightMin { wt = IoWeightMin } - return map[string]string{BLOCK_IO_WEIGHT: fmt.Sprintf("%d", wt)} + switch IoScheduler { + case IOSCHED_CFQ: + return map[string]string{BLOCK_IO_WEIGHT: fmt.Sprintf("%d", wt)} + case IOSCHED_BFQ: + return map[string]string{BLOCK_IO_BFQ_WEIGHT: fmt.Sprintf("%d", wt)} + default: + return nil + } } func (c *CGroupIOTask) init() bool { - return SetRootParam(c.Module(), BLOCK_IO_WEIGHT, fmt.Sprintf("%d", IoWeightMax), "") + switch IoScheduler { + case IOSCHED_CFQ: + return SetRootParam(c.Module(), BLOCK_IO_WEIGHT, fmt.Sprintf("%d", IoWeightMax), "") + default: + return true + } } func NewCGroupIOTask(pid string, coreNum int) *CGroupIOTask { @@ -536,7 +560,8 @@ func NewCGroupCPUSetTask(pid string, coreNum int, cpuset string) CGroupCPUSetTas return task } -func Init() bool { +func Init(ioScheduler string) bool { + IoScheduler = ioScheduler for _, hand := range []ICGroupTask{&CGroupTask{}, &CGroupCPUTask{}, &CGroupIOTask{}} { if !hand.init() { return false diff --git a/pkg/util/fileutils2/fileutils.go b/pkg/util/fileutils2/fileutils.go index 6fedbe1944..79c8db3b1b 100644 --- a/pkg/util/fileutils2/fileutils.go +++ b/pkg/util/fileutils2/fileutils.go @@ -25,6 +25,7 @@ import ( "strings" "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/onecloud/pkg/util/procutils" ) @@ -139,6 +140,40 @@ func IsBlockDeviceUsed(dev string) bool { return false } +func GetAllBlkdevsIoSchedulers() ([]string, error) { + if _, err := os.Stat("/sys/block"); !os.IsNotExist(err) { + blockDevs, err := ioutil.ReadDir("/sys/block") + if err != nil { + log.Errorf("ReadDir /sys/block error: %s", err) + return nil, errors.Wrap(err, "ioutil.ReadDir(/sys/block)") + } + for _, b := range blockDevs { + if IsBlockDevMounted(b.Name()) { + conf, err := GetBlkdevConfig(b.Name(), "queue/scheduler") + if err != nil { + log.Errorf("Get %s queue/scheduler fail %s", b.Name(), err) + } else { + algs := make([]string, 0) + for _, alg := range strings.Split(strings.TrimSpace(conf), " ") { + if len(alg) > 0 { + if alg[0] == '[' { + alg = alg[1 : len(alg)-1] + } + algs = append(algs, alg) + } + } + return algs, nil + } + } + } + log.Errorf("no block device avaiable") + return nil, nil + } else { + log.Errorf("stat /sys/block fail %s", err) + return nil, errors.Wrap(err, "stat /sys/block fail") + } +} + func ChangeAllBlkdevsParams(params map[string]string) { if _, err := os.Stat("/sys/block"); !os.IsNotExist(err) { blockDevs, err := ioutil.ReadDir("/sys/block") @@ -167,6 +202,15 @@ func ChangeBlkdevParameter(dev, key, value string) { } } +func GetBlkdevConfig(dev, key string) (string, error) { + p := path.Join("/sys/block", dev, key) + if _, err := os.Stat(p); !os.IsNotExist(err) { + return FileGetContents(p) + } else { + return "", err + } +} + func FileGetContents(file string) (string, error) { content, err := ioutil.ReadFile(file) if err != nil { diff --git a/pkg/util/fileutils2/fileutils_test.go b/pkg/util/fileutils2/fileutils_test.go index 2f76445e1f..b8f8b9b553 100644 --- a/pkg/util/fileutils2/fileutils_test.go +++ b/pkg/util/fileutils2/fileutils_test.go @@ -14,12 +14,12 @@ package fileutils2 -// TODO: rewrite this test -/* import ( "testing" ) +// TODO: rewrite this test +/* func TestIsBlockDeviceUsed(t *testing.T) { type args struct { dev string @@ -68,3 +68,8 @@ func TestGetDevId(t *testing.T) { }) } }*/ + +func TestGetAllBlkdevsIoScheduler(t *testing.T) { + scheds, _ := GetAllBlkdevsIoSchedulers() + t.Logf("scheduler: %#v", scheds) +} diff --git a/pkg/util/iproute2/route_test.go b/pkg/util/iproute2/route_test.go index 89e1894469..e1c85c3fe4 100644 --- a/pkg/util/iproute2/route_test.go +++ b/pkg/util/iproute2/route_test.go @@ -15,6 +15,7 @@ package iproute2 import ( + "net" "testing" "github.com/vishvananda/netlink" @@ -60,4 +61,42 @@ func TestRoute(t *testing.T) { t.Errorf("expecting ESRCH, got %v", err) } }) + + t.Run("route add", func(t *testing.T) { + ifname1 := genDummyName(t) + dum := addDummy(t, ifname1) + defer delDummy(t, dum) + + ifname := "lo" + + for _, cidr := range [][]string{ + []string{ + "127.1.0.1/24", + "127.0.0.2", + }, + []string{ + "192.168.120.243/24", + "127.0.0.2", + }, + } { + _, ipnet, err := net.ParseCIDR(cidr[0]) + if err != nil { + t.Errorf("ParseCIDR error %s", err) + continue + } + gw := net.ParseIP(cidr[1]) + + r := NewRoute(ifname) + r.AddByIPNet(ipnet, gw) + if err := r.Err(); err != nil { + t.Errorf("got error %v", err) + } + + r.DelByIPNet(ipnet) + + if err := r.Err(); err != nil { + t.Errorf("del error %v", err) + } + } + }) }