From 75df132f992a89b15491d9a3c62e30a354d01cda Mon Sep 17 00:00:00 2001 From: Jian Qiu Date: Sun, 1 Sep 2024 20:16:36 +0800 Subject: [PATCH] fix: migrate ipv6 routes from physical interface to bridge (#21131) Co-authored-by: Qiu Jian --- cmd/climc/shell/compute/networks.go | 18 +++ pkg/hostman/hostinfo/hostbridge/hostbridge.go | 132 +++++++++++++++--- pkg/util/netutils2/netutils_linux.go | 22 ++- 3 files changed, 152 insertions(+), 20 deletions(-) diff --git a/cmd/climc/shell/compute/networks.go b/cmd/climc/shell/compute/networks.go index 6c92fc59f6..f5b8826eee 100644 --- a/cmd/climc/shell/compute/networks.go +++ b/cmd/climc/shell/compute/networks.go @@ -26,6 +26,7 @@ import ( modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" "yunion.io/x/onecloud/pkg/mcclient/options" compute_options "yunion.io/x/onecloud/pkg/mcclient/options/compute" + "yunion.io/x/onecloud/pkg/util/netutils2" ) func init() { @@ -240,4 +241,21 @@ func init() { printObject(net) return nil }) + type NetworkInterfaceInfoOptions struct { + DEV string `help:"name of device, e.g. eth0"` + } + R(&NetworkInterfaceInfoOptions{}, "network-interface-info", "Show addr and routes of interface", func(s *mcclient.ClientSession, args *NetworkInterfaceInfoOptions) error { + netIf := netutils2.NewNetInterface(args.DEV) + + fmt.Println("[Slave Addresses]") + for _, addr := range netIf.GetSlaveAddresses() { + fmt.Printf("%s/%s\n", addr[0], addr[1]) + } + fmt.Println("[Routes]") + for _, r := range netIf.GetRouteSpecs() { + fmt.Printf("%s via %s dev %d\n", r.Dst.String(), r.Gw.String(), r.LinkIndex) + } + + return nil + }) } diff --git a/pkg/hostman/hostinfo/hostbridge/hostbridge.go b/pkg/hostman/hostinfo/hostbridge/hostbridge.go index ef907ba1ea..5fb94b598d 100644 --- a/pkg/hostman/hostinfo/hostbridge/hostbridge.go +++ b/pkg/hostman/hostinfo/hostbridge/hostbridge.go @@ -26,6 +26,8 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/netutils" + "yunion.io/x/pkg/util/regutils" "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/hostman/guestman/desc" @@ -45,7 +47,7 @@ type IBridgeDriver interface { Setup(IBridgeDriver) error SetupAddresses(net.IPMask) error SetupSlaveAddresses([][]string) error - SetupRoutes(routes []iproute2.RouteSpec) error + SetupRoutes(routes []iproute2.RouteSpec, add bool) error BringupInterface() error Exists() (bool, error) @@ -157,7 +159,7 @@ func (d *SBaseBridgeDriver) BringupInterface() error { return nil } -func trySetupSlaveAddressesRoutes(o IBridgeDriver, migrateAddrs [][]string, migrateRoutes []iproute2.RouteSpec) error { +func trySetupSlaveAddressesRoutes(o IBridgeDriver, migrateAddrs [][]string, delRoutes []iproute2.RouteSpec, migrateRoutes []iproute2.RouteSpec) error { if len(migrateAddrs) > 0 { tried := 0 const MAX_TRIES = 4 @@ -177,12 +179,31 @@ func trySetupSlaveAddressesRoutes(o IBridgeDriver, migrateAddrs [][]string, migr } } } + if len(delRoutes) > 0 { + tried := 0 + const MAX_TRIES = 4 + errs := make([]error, 0) + for { + if err := o.SetupRoutes(delRoutes, false); err != nil { + errs = append(errs, err) + log.Errorf("delRoutes fail: %s", err) + tried += 1 + if tried >= MAX_TRIES { + return errors.Wrap(errors.NewAggregate(errs), "DeleteRoutes") + } else { + time.Sleep(time.Duration(tried) * time.Second) + } + } else { + break + } + } + } if len(migrateRoutes) > 0 { tried := 0 const MAX_TRIES = 4 errs := make([]error, 0) for { - if err := o.SetupRoutes(migrateRoutes); err != nil { + if err := o.SetupRoutes(migrateRoutes, true); err != nil { errs = append(errs, err) log.Errorf("SetupRoutes fail: %s", err) tried += 1 @@ -203,6 +224,7 @@ func (d *SBaseBridgeDriver) MigrateSlaveConfigs(o IBridgeDriver) error { if d.inter != nil { migrateAddrs := make([][]string, 0) migrateRoutes := make([]iproute2.RouteSpec, 0) + delRoutes := make([]iproute2.RouteSpec, 0) { currentRoutes := d.bridge.GetRouteSpecs() currentSlaves := d.bridge.GetSlaveAddresses() @@ -233,20 +255,22 @@ func (d *SBaseBridgeDriver) MigrateSlaveConfigs(o IBridgeDriver) error { } for i := range routes { - if strings.HasPrefix(routes[i].Dst.String(), "fe80:") || strings.HasPrefix(routes[i].Dst.String(), "169.254.") { - // skip link local routes - continue - } find := false for j := range currentRoutes { - if routes[i].Dst.String() == currentRoutes[j].Dst.String() { + log.Infof("new %s(%d,%s) current %s(%d,%s)", routes[i].Dst.String(), routes[i].Table, routes[i].Gw.String(), currentRoutes[j].Dst.String(), currentRoutes[j].Table, currentRoutes[j].Gw.String()) + if routes[i].Dst.String() == currentRoutes[j].Dst.String() && routes[i].Table == currentRoutes[j].Table { + if routes[i].Gw.String() != currentRoutes[j].Gw.String() { + // need to replace + delRoutes = append(delRoutes, currentRoutes[j]) + migrateRoutes = append(migrateRoutes, routes[i]) + } find = true break } } if !find { for j := range slaveAddrs { - if routes[i].Dst.String() == fmt.Sprintf("%s/%s", slaveAddrs[j][0], slaveAddrs[j][1]) { + if routes[i].Dst.String() == addr2Prefix(slaveAddrs[j][0], slaveAddrs[j][1]) { find = true break } @@ -258,9 +282,9 @@ func (d *SBaseBridgeDriver) MigrateSlaveConfigs(o IBridgeDriver) error { } } } - log.Infof("to migrate routes: %s slaveAddress: %s", jsonutils.Marshal(migrateRoutes), jsonutils.Marshal(migrateAddrs)) + log.Infof("to migrate routes: %s slaveAddress: %s delRoutes: %s", jsonutils.Marshal(migrateRoutes), jsonutils.Marshal(migrateAddrs), jsonutils.Marshal(delRoutes)) { - err := trySetupSlaveAddressesRoutes(o, migrateAddrs, migrateRoutes) + err := trySetupSlaveAddressesRoutes(o, migrateAddrs, delRoutes, migrateRoutes) if err != nil { return errors.Wrap(err, "trySetupSlaveAddressesRoutes") } @@ -271,6 +295,19 @@ func (d *SBaseBridgeDriver) MigrateSlaveConfigs(o IBridgeDriver) error { return errors.Wrap(err, "ClearAddrs") } } + { + routes := d.inter.GetRouteSpecs() + if len(routes) > 0 { + rt := iproute2.NewRoute(d.inter.String()) + for i := range routes { + rt = rt.DelByCidr(routes[i].Dst.String()) + } + err := rt.Err() + if err != nil { + return errors.Wrap(err, "Clear rotues") + } + } + } } return nil } @@ -388,7 +425,7 @@ func (d *SBaseBridgeDriver) SetupSlaveAddresses(slaveAddrs [][]string) error { return nil } -func (d *SBaseBridgeDriver) SetupRoutes(routespecs []iproute2.RouteSpec) error { +func (d *SBaseBridgeDriver) SetupRoutes(routespecs []iproute2.RouteSpec, add bool) error { bridgeIP := d.inter.Addr bridgeMask := d.inter.Mask br := d.bridge.String() @@ -400,8 +437,14 @@ func (d *SBaseBridgeDriver) SetupRoutes(routespecs []iproute2.RouteSpec) error { continue } cmd := []string{ - "route", "add", routespec.Dst.String(), + "route", } + if add { + cmd = append(cmd, "add") + } else { + cmd = append(cmd, "del") + } + cmd = append(cmd, routespec.Dst.String()) if routespec.Gw != nil { cmd = append(cmd, "via", routespec.Gw.String()) } @@ -410,9 +453,13 @@ func (d *SBaseBridgeDriver) SetupRoutes(routespecs []iproute2.RouteSpec) error { output, err := procutils.NewRemoteCommandAsFarAsPossible("ip", cmd...).Output() if err != nil { errs = append(errs, errors.Wrapf(err, "run cmd: ip %s, output: %s", strings.Join(cmd, " "), output)) - cmd = append(cmd, "onlink") - if output, err := procutils.NewRemoteCommandAsFarAsPossible("ip", cmd...).Output(); err != nil { - errs = append(errs, errors.Wrapf(err, "run cmd: ip %s, output: %s", strings.Join(cmd, " "), output)) + if add { + cmd = append(cmd, "onlink") + if output, err := procutils.NewRemoteCommandAsFarAsPossible("ip", cmd...).Output(); err != nil { + errs = append(errs, errors.Wrapf(err, "run cmd: ip %s, output: %s", strings.Join(cmd, " "), output)) + return errors.Wrapf(errors.NewAggregate(errs), "setup route %s", routespec.String()) + } + } else { return errors.Wrapf(errors.NewAggregate(errs), "setup route %s", routespec.String()) } } @@ -420,6 +467,34 @@ func (d *SBaseBridgeDriver) SetupRoutes(routespecs []iproute2.RouteSpec) error { return nil } +func addr2Prefix(addrStr string, maskLenStr string) string { + if regutils.MatchIP6Addr(addrStr) { + v6Addr, _ := netutils.NewIPV6Addr(addrStr) + maskLen, _ := strconv.ParseInt(maskLenStr, 10, 64) + netAddr := v6Addr.NetAddr(uint8(maskLen)) + return fmt.Sprintf("%s/%d", netAddr.String(), maskLen) + } else { + v4Addr, _ := netutils.NewIPV4Addr(addrStr) + maskLen, _ := strconv.ParseInt(maskLenStr, 10, 64) + netAddr := v4Addr.NetAddr(int8(maskLen)) + return fmt.Sprintf("%s/%d", netAddr.String(), maskLen) + } +} + +func addr2Prefix2(addrStr string, mask net.IPMask) string { + if regutils.MatchIP6Addr(addrStr) { + v6Addr, _ := netutils.NewIPV6Addr(addrStr) + maskLen, _ := mask.Size() + netAddr := v6Addr.NetAddr(uint8(maskLen)) + return fmt.Sprintf("%s/%d", netAddr.String(), maskLen) + } else { + v4Addr, _ := netutils.NewIPV4Addr(addrStr) + maskLen, _ := mask.Size() + netAddr := v4Addr.NetAddr(int8(maskLen)) + return fmt.Sprintf("%s/%d", netAddr.String(), maskLen) + } +} + func (d *SBaseBridgeDriver) Setup(o IBridgeDriver) error { var routes []iproute2.RouteSpec var slaveAddrs [][]string @@ -453,7 +528,30 @@ func (d *SBaseBridgeDriver) Setup(o IBridgeDriver) error { return errors.Wrap(err, "SetupAddresses") } time.Sleep(1 * time.Second) - if err := trySetupSlaveAddressesRoutes(o, slaveAddrs, routes); err != nil { + setupRoutes := make([]iproute2.RouteSpec, 0) + { + for i := range routes { + find := false + if !find { + if routes[i].Dst.String() == addr2Prefix2(d.ip, d.inter.Mask) { + find = true + } + } + if !find { + for j := range slaveAddrs { + if routes[i].Dst.String() == addr2Prefix(slaveAddrs[j][0], slaveAddrs[j][1]) { + find = true + break + } + } + } + if !find { + // need to migrate route + setupRoutes = append(setupRoutes, routes[i]) + } + } + } + if err := trySetupSlaveAddressesRoutes(o, slaveAddrs, nil, setupRoutes); err != nil { return errors.Wrap(err, "trySetupSlaveAddressesRoutes") } } else { diff --git a/pkg/util/netutils2/netutils_linux.go b/pkg/util/netutils2/netutils_linux.go index a3ee6f7a13..5126262708 100644 --- a/pkg/util/netutils2/netutils_linux.go +++ b/pkg/util/netutils2/netutils_linux.go @@ -60,10 +60,26 @@ func (n *SNetInterface) GetRouteSpecs() []iproute2.RouteSpec { routeList := iproute2.NewRoute(n.name) routes4 := getRouteSpecs(routeList.List4) routes6 := getRouteSpecs(routeList.List6) - if len(routes6) > 0 { - routes4 = append(routes4, routes6...) + rets := make([]iproute2.RouteSpec, 0) + for i := range routes4 { + if routes4[i].Gw == nil { + continue + } + if strings.HasPrefix(routes4[i].Dst.String(), "169.") { + continue + } + rets = append(rets, routes4[i]) } - return routes4 + for i := range routes6 { + if routes6[i].Gw == nil { + continue + } + if strings.HasPrefix(routes6[i].Dst.String(), "fe80:") { + continue + } + rets = append(rets, routes6[i]) + } + return rets } func (n *SNetInterface) ClearAddrs() error {