fix: ipv6 ra should be sent periodically (#23258)

Co-authored-by: Qiu Jian <qiujian@yunionyun.com>

pkg/hostman/guestman/guestman.go

@@ -398,6 +398,8 @@ func (m *SGuestManager) Bootstrap() (chan struct{}, error) {
 		go m.cpufreqSimulateManager.StartSetCpuFreqSimulate()
 	}
 
+	m.host.OnGuestLoadingComplete()
+
 	return m.dirtyServersChan, nil
 }
 
@@ -570,8 +572,12 @@ func (m *SGuestManager) CPUSetRemove(ctx context.Context, sid string) error {
 	return guest.CPUSetRemove(ctx)
 }
 
-func (m *SGuestManager) IsGuestDir(f os.FileInfo) bool {
-	return hostutils.IsGuestDir(f, m.ServersPath)
+func (m *SGuestManager) IsGuestDir(f os.DirEntry) bool {
+	fi, err := f.Info()
+	if err != nil {
+		return false
+	}
+	return hostutils.IsGuestDir(fi, m.ServersPath)
 }
 
 func (m *SGuestManager) IsGuestExist(sid string) bool {
@@ -583,7 +589,7 @@ func (m *SGuestManager) IsGuestExist(sid string) bool {
 }
 
 func (m *SGuestManager) LoadExistingGuests() {
-	files, err := ioutil.ReadDir(m.ServersPath)
+	files, err := os.ReadDir(m.ServersPath)
 	if err != nil {
 		log.Errorf("List servers path %s error %s", m.ServersPath, err)
 	}
@@ -601,7 +607,7 @@ func (m *SGuestManager) GetServerDescFilePath(sid string) string {
 
 func (m *SGuestManager) GetServerDesc(sid string) (*desc.SGuestDesc, error) {
 	descPath := m.GetServerDescFilePath(sid)
-	descStr, err := ioutil.ReadFile(descPath)
+	descStr, err := os.ReadFile(descPath)
 	if err != nil {
 		return nil, errors.Wrapf(err, "read file %s", descPath)
 	}
@@ -702,6 +708,19 @@ func (m *SGuestManager) GetGuestNicDesc(
 	return guestDesc, nic
 }
 
+func (m *SGuestManager) GetAllGuestIPv6Macs(bridge string) []string {
+	macs := []string{}
+	m.Servers.Range(func(k, v interface{}) bool {
+		guest := v.(GuestRuntimeInstance)
+		if guest.IsLoaded() {
+			nicMacs := guest.GetIpv6NicMacs(bridge)
+			macs = append(macs, nicMacs...)
+		}
+		return true
+	})
+	return macs
+}
+
 func (m *SGuestManager) getGuestNicDescInCandidate(
 	mac, ip, port, bridge string,
 ) (*desc.SGuestDesc, *desc.SGuestNetwork) {

pkg/hostman/guestman/runtime.go

@@ -56,6 +56,7 @@ type GuestRuntimeInstance interface {
 	IsSuspend() bool
 	IsLoaded() bool
 	GetNicDescMatch(mac, ip, port, bridge string) *desc.SGuestNetwork
+	GetIpv6NicMacs(bridge string) []string
 	CleanGuest(ctx context.Context, params interface{}) (jsonutils.JSONObject, error)
 	CleanDirtyGuest(ctx context.Context) error
 	ImportServer(pendingDelete bool)
@@ -172,6 +173,16 @@ func (s *sBaseGuestInstance) GetNicDescMatch(mac, ip, port, bridge string) *desc
 	return nil
 }
 
+func (s *sBaseGuestInstance) GetIpv6NicMacs(bridge string) []string {
+	macs := []string{}
+	for _, nic := range s.Desc.Nics {
+		if nic.Bridge == bridge && len(nic.Ip6) > 0 {
+			macs = append(macs, nic.Mac)
+		}
+	}
+	return macs
+}
+
 func LoadGuestCpuset(m *SGuestManager, s GuestRuntimeInstance) error {
 	guestDesc := s.GetDesc()
 	if s.IsRunning() {

pkg/hostman/guestman/types/interface.go

@@ -29,6 +29,7 @@ var HealthCheckReactor IHealthCheckReactor
 
 type IGuestDescGetter interface {
 	GetGuestNicDesc(mac, ip, port, bridge string, isCandidate bool) (*desc.SGuestDesc, *desc.SGuestNetwork)
+	GetAllGuestIPv6Macs(bridge string) []string
 }
 
 var GuestDescGetter IGuestDescGetter

pkg/hostman/hostinfo/hostdhcp/dhcpserver6.go

@@ -44,8 +44,8 @@ type SGuestDHCP6Server struct {
 	ifaceDev *netutils2.SNetInterface
 
 	raExitCh   chan struct{}
-	raReqCh    chan *sRARequest
-	raReqQueue []*sRARequest
+	raReqCh    chan net.HardwareAddr
+	raReqQueue map[string]*sRARequest
 
 	gwMacCache *hashcache.Cache
 }
@@ -76,8 +76,8 @@ func NewGuestDHCP6Server(iface string, port int, relay *SDHCPRelayUpstream) (*SG
 	}
 
 	guestdhcp.raExitCh = make(chan struct{})
-	guestdhcp.raReqCh = make(chan *sRARequest, 100)
-	guestdhcp.raReqQueue = make([]*sRARequest, 0)
+	guestdhcp.raReqCh = make(chan net.HardwareAddr, 1024)
+	guestdhcp.raReqQueue = make(map[string]*sRARequest)
 	guestdhcp.gwMacCache = hashcache.NewCache(1024, 5*time.Minute)
 
 	return guestdhcp, nil
@@ -169,3 +169,15 @@ func (s *SGuestDHCP6Server) serveDHCPInternal(pkt dhcp.Packet, cliMac net.Hardwa
 	}
 	return nil, nil
 }
+
+func (s *SGuestDHCP6Server) InitRAQueue() {
+	macs := guestman.GuestDescGetter.GetAllGuestIPv6Macs(s.ifaceDev.String())
+	for _, mac := range macs {
+		hw, err := net.ParseMAC(mac)
+		if err != nil {
+			log.Errorf("ParseMAC %s: %v", mac, err)
+			continue
+		}
+		s.requestRA(hw)
+	}
+}

pkg/hostman/hostinfo/hostdhcp/icmp6handlers.go

@@ -29,28 +29,19 @@ import (
 )
 
 type sRARequest struct {
-	solicitation *icmp6.SRouterSolicitation
+	vmMac        net.HardwareAddr
 	attempts     int
 	succAttempts int
+	nextAttempt  time.Time
+}
+
+func (s *SGuestDHCP6Server) requestRA(vmMac net.HardwareAddr) {
+	s.raReqCh <- vmMac
 }
 
 func (s *SGuestDHCP6Server) handleRouterSolicitation(msg *icmp6.SRouterSolicitation) error {
 	// solicitation request from guest
-	var conf = s.getConfig(msg.SrcMac)
-	if conf != nil && conf.ClientIP6 != nil && conf.Gateway6 != nil {
-		req := &sRARequest{
-			solicitation: msg,
-			attempts:     1,
-		}
-		succ, err := s.sendRouterAdvertisement(msg)
-		if err != nil {
-			return errors.Wrapf(err, "sendRouterAdvertisement")
-		}
-		if succ {
-			req.succAttempts++
-		}
-		s.raReqCh <- req
-	}
+	s.requestRA(msg.SrcMac)
 	return nil
 }
 
@@ -100,8 +91,11 @@ func (s *SGuestDHCP6Server) requestGatewayMac(gwIP net.IP, vlanId uint16) {
 	}
 }
 
-func (s *SGuestDHCP6Server) sendRouterAdvertisement(solicitation *icmp6.SRouterSolicitation) (bool, error) {
-	var conf = s.getConfig(solicitation.SrcMac)
+func (s *SGuestDHCP6Server) sendRouterAdvertisement(vmMac net.HardwareAddr) (bool, error) {
+	var conf = s.getConfig(vmMac)
+	if conf == nil {
+		return false, errors.Wrapf(errors.ErrNotFound, "getConfig %s", vmMac.String())
+	}
 	if conf != nil && conf.ClientIP6 != nil && conf.Gateway6 != nil {
 		gwMacObj := s.gwMacCache.AtomicGet(conf.Gateway6.String())
 		if gwMacObj == nil {
@@ -130,7 +124,7 @@ func (s *SGuestDHCP6Server) sendRouterAdvertisement(solicitation *icmp6.SRouterS
 				// https://www.rfc-editor.org/rfc/rfc4861.html#page-18
 				// !!! MUST be the link-local address assigned to the interface from which this message is sent.
 				SrcIP:  gwLinkLocalIP,
-				DstMac: solicitation.SrcMac,
+				DstMac: vmMac,
 				DstIP:  net.ParseIP("ff02::1"),
 			},
 			CurHopLimit:    64,
@@ -138,7 +132,7 @@ func (s *SGuestDHCP6Server) sendRouterAdvertisement(solicitation *icmp6.SRouterS
 			IsOther:        true,
 			IsHomeAgent:    false,
 			Preference:     pref,
-			RouterLifetime: 9000,
+			RouterLifetime: uint16(options.HostOptions.Dhcp6RouterLifetimeSeconds),
 			ReachableTime:  0,
 			RetransTimer:   0,
 			MTU:            uint32(conf.MTU),
@@ -148,8 +142,8 @@ func (s *SGuestDHCP6Server) sendRouterAdvertisement(solicitation *icmp6.SRouterS
 					IsAutoconf:        false,
 					Prefix:            ipnet.IP,
 					PrefixLen:         conf.PrefixLen6,
-					ValidLifetime:     4500,
-					PreferredLifetime: 2250,
+					ValidLifetime:     uint32(options.HostOptions.Dhcp6RouterLifetimeSeconds / 2),
+					PreferredLifetime: uint32(options.HostOptions.Dhcp6RouterLifetimeSeconds / 4),
 				},
 			},
 		}
@@ -162,13 +156,13 @@ func (s *SGuestDHCP6Server) sendRouterAdvertisement(solicitation *icmp6.SRouterS
 					IsAutoconf:        false,
 					Prefix:            route.Prefix,
 					PrefixLen:         route.PrefixLen,
-					ValidLifetime:     4500,
-					PreferredLifetime: 2250,
+					ValidLifetime:     uint32(options.HostOptions.Dhcp6RouterLifetimeSeconds / 2),
+					PreferredLifetime: uint32(options.HostOptions.Dhcp6RouterLifetimeSeconds / 4),
 				})
 			} else if route.Gateway.String() != conf.Gateway6.String() {
 				// routes forwarded by router
 				ra.RouteInfo = append(ra.RouteInfo, icmp6.SRouteInfoOption{
-					RouteLifetime: 9000,
+					RouteLifetime: uint32(options.HostOptions.Dhcp6RouterLifetimeSeconds),
 					Prefix:        route.Prefix,
 					PrefixLen:     route.PrefixLen,
 					Preference:    pref,
@@ -182,7 +176,7 @@ func (s *SGuestDHCP6Server) sendRouterAdvertisement(solicitation *icmp6.SRouterS
 			return false, errors.Wrapf(err, "EncodePacket")
 		}
 
-		err = s.conn.SendRaw(bytes, solicitation.SrcMac)
+		err = s.conn.SendRaw(bytes, vmMac)
 		if err != nil {
 			log.Errorf("Send RouterAdvertisement error: %v", err)
 		}
@@ -195,6 +189,46 @@ func (s *SGuestDHCP6Server) stopRAServer() {
 	close(s.raReqCh)
 }
 
+func (s *SGuestDHCP6Server) handleRARequest(vmMac net.HardwareAddr) {
+	vmMacStr := vmMac.String()
+	raReq, ok := s.raReqQueue[vmMacStr]
+	if !ok {
+		raReq = &sRARequest{
+			vmMac: vmMac,
+		}
+		s.raReqQueue[vmMacStr] = raReq
+	} else {
+		raReq.nextAttempt = time.Time{}
+	}
+	// send RA immediately
+	raReq.attempts++
+	succ, err := s.sendRouterAdvertisement(vmMac)
+	if err != nil {
+		if errors.Cause(err) == errors.ErrNotFound {
+			// no such vm, giveup
+			delete(s.raReqQueue, vmMacStr)
+			return
+		}
+		log.Errorf("sendRouterAdvertisement error: %v", err)
+	}
+	if succ {
+		raReq.succAttempts++
+	}
+	if raReq.succAttempts < options.HostOptions.Dhcp6RouterAdvertisementAttempts {
+		if raReq.attempts > 2*options.HostOptions.Dhcp6RouterAdvertisementAttempts {
+			// giveup
+			delete(s.raReqQueue, vmMacStr)
+		} else {
+			// retry next round
+		}
+	} else {
+		// schedule RA when prefix router lifetime expires
+		raReq.attempts = 0
+		raReq.succAttempts = 0
+		raReq.nextAttempt = time.Now().Add(time.Second * time.Duration(options.HostOptions.Dhcp6RouterLifetimeSeconds/4))
+	}
+}
+
 func (s *SGuestDHCP6Server) startRAServer() {
 	// a tiny RA server
 	stop := false
@@ -204,29 +238,14 @@ func (s *SGuestDHCP6Server) startRAServer() {
 			stop = true
 		case raReq := <-s.raReqCh:
 			// handle RA request
-			s.raReqQueue = append(s.raReqQueue, raReq)
+			s.handleRARequest(raReq)
 		case <-time.After(time.Second * time.Duration(options.HostOptions.Dhcp6RouterAdvertisementIntervalSecs)):
 			// send RA
-			// log.Infof("timeout, to announce RA %d requests", len(s.raReqQueue))
-			if len(s.raReqQueue) > 0 {
-				raReqQueue := s.raReqQueue
-				s.raReqQueue = make([]*sRARequest, 0)
-
-				for i := range raReqQueue {
-					raReq := raReqQueue[i]
-					raReq.attempts++
-					succ, err := s.sendRouterAdvertisement(raReq.solicitation)
-					if err != nil {
-						log.Errorf("sendRouterAdvertisement error: %v", err)
-						continue
-					}
-					if succ {
-						raReq.succAttempts++
-					}
-					if raReq.succAttempts < options.HostOptions.Dhcp6RouterAdvertisementAttempts && raReq.attempts < 2*options.HostOptions.Dhcp6RouterAdvertisementAttempts {
-						s.raReqCh <- raReq
-					}
+			for _, raReq := range s.raReqQueue {
+				if !raReq.nextAttempt.IsZero() && raReq.nextAttempt.After(time.Now()) {
+					continue
 				}
+				s.handleRARequest(raReq.vmMac)
 			}
 		}
 	}

pkg/hostman/hostinfo/hostinfo.go

@@ -2801,6 +2801,14 @@ func (h *SHostInfo) startBindReservedCpus(processesPrefix []string) {
 	}
 }
 
+func (h *SHostInfo) OnGuestLoadingComplete() {
+	for _, nic := range h.Nics {
+		if nic.dhcpServer6 != nil {
+			nic.dhcpServer6.InitRAQueue()
+		}
+	}
+}
+
 func NewHostInfo() (*SHostInfo, error) {
 	var res = new(SHostInfo)
 	res.sysinfo = &SSysInfo{}

pkg/hostman/hostutils/hostutils.go

@@ -109,6 +109,8 @@ type IHost interface {
 
 	SetIGuestManager(guestman IGuestManager)
 	GetIGuestManager() IGuestManager
+
+	OnGuestLoadingComplete()
 }
 
 func GetComputeSession(ctx context.Context) *mcclient.ClientSession {

pkg/hostman/options/options.go

@@ -50,6 +50,7 @@ type SHostBaseOptions struct {
 
 	Dhcp6RouterAdvertisementIntervalSecs int `default:"3" help:"DHCPv6 router advertisement interval in seconds, default 3 seconds"`
 	Dhcp6RouterAdvertisementAttempts     int `default:"3" help:"DHCPv6 router advertisement attempts, default 3 attempts"`
+	Dhcp6RouterLifetimeSeconds           int `default:"9000" help:"DHCPv6 router lifetime in seconds, default 9000 seconds"`
 
 	Ext4LargefileSizeGb int `default:"4096" help:"Use largefile options when the ext4 fs greater than this size"`
 	Ext4HugefileSizeGb  int `default:"512" help:"Use huge options when the ext4 fs greater than this size"`