From fe96a6de057bd124260f96f0331c96c367bd240b Mon Sep 17 00:00:00 2001 From: Jian Qiu Date: Wed, 10 Sep 2025 18:59:00 +0800 Subject: [PATCH] fix: ipv6 ra should be sent periodically (#23258) Co-authored-by: Qiu Jian --- pkg/hostman/guestman/guestman.go | 27 ++++- pkg/hostman/guestman/runtime.go | 11 ++ pkg/hostman/guestman/types/interface.go | 1 + pkg/hostman/hostinfo/hostdhcp/dhcpserver6.go | 20 +++- .../hostinfo/hostdhcp/icmp6handlers.go | 111 ++++++++++-------- pkg/hostman/hostinfo/hostinfo.go | 8 ++ pkg/hostman/hostutils/hostutils.go | 2 + pkg/hostman/options/options.go | 1 + 8 files changed, 127 insertions(+), 54 deletions(-) diff --git a/pkg/hostman/guestman/guestman.go b/pkg/hostman/guestman/guestman.go index e878ce7d5a..2b7a4b7c7e 100644 --- a/pkg/hostman/guestman/guestman.go +++ b/pkg/hostman/guestman/guestman.go @@ -398,6 +398,8 @@ func (m *SGuestManager) Bootstrap() (chan struct{}, error) { go m.cpufreqSimulateManager.StartSetCpuFreqSimulate() } + m.host.OnGuestLoadingComplete() + return m.dirtyServersChan, nil } @@ -570,8 +572,12 @@ func (m *SGuestManager) CPUSetRemove(ctx context.Context, sid string) error { return guest.CPUSetRemove(ctx) } -func (m *SGuestManager) IsGuestDir(f os.FileInfo) bool { - return hostutils.IsGuestDir(f, m.ServersPath) +func (m *SGuestManager) IsGuestDir(f os.DirEntry) bool { + fi, err := f.Info() + if err != nil { + return false + } + return hostutils.IsGuestDir(fi, m.ServersPath) } func (m *SGuestManager) IsGuestExist(sid string) bool { @@ -583,7 +589,7 @@ func (m *SGuestManager) IsGuestExist(sid string) bool { } func (m *SGuestManager) LoadExistingGuests() { - files, err := ioutil.ReadDir(m.ServersPath) + files, err := os.ReadDir(m.ServersPath) if err != nil { log.Errorf("List servers path %s error %s", m.ServersPath, err) } @@ -601,7 +607,7 @@ func (m *SGuestManager) GetServerDescFilePath(sid string) string { func (m *SGuestManager) GetServerDesc(sid string) (*desc.SGuestDesc, error) { descPath := m.GetServerDescFilePath(sid) - descStr, err := ioutil.ReadFile(descPath) + descStr, err := os.ReadFile(descPath) if err != nil { return nil, errors.Wrapf(err, "read file %s", descPath) } @@ -702,6 +708,19 @@ func (m *SGuestManager) GetGuestNicDesc( return guestDesc, nic } +func (m *SGuestManager) GetAllGuestIPv6Macs(bridge string) []string { + macs := []string{} + m.Servers.Range(func(k, v interface{}) bool { + guest := v.(GuestRuntimeInstance) + if guest.IsLoaded() { + nicMacs := guest.GetIpv6NicMacs(bridge) + macs = append(macs, nicMacs...) + } + return true + }) + return macs +} + func (m *SGuestManager) getGuestNicDescInCandidate( mac, ip, port, bridge string, ) (*desc.SGuestDesc, *desc.SGuestNetwork) { diff --git a/pkg/hostman/guestman/runtime.go b/pkg/hostman/guestman/runtime.go index 846698f84f..53bb3ad370 100644 --- a/pkg/hostman/guestman/runtime.go +++ b/pkg/hostman/guestman/runtime.go @@ -56,6 +56,7 @@ type GuestRuntimeInstance interface { IsSuspend() bool IsLoaded() bool GetNicDescMatch(mac, ip, port, bridge string) *desc.SGuestNetwork + GetIpv6NicMacs(bridge string) []string CleanGuest(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) CleanDirtyGuest(ctx context.Context) error ImportServer(pendingDelete bool) @@ -172,6 +173,16 @@ func (s *sBaseGuestInstance) GetNicDescMatch(mac, ip, port, bridge string) *desc return nil } +func (s *sBaseGuestInstance) GetIpv6NicMacs(bridge string) []string { + macs := []string{} + for _, nic := range s.Desc.Nics { + if nic.Bridge == bridge && len(nic.Ip6) > 0 { + macs = append(macs, nic.Mac) + } + } + return macs +} + func LoadGuestCpuset(m *SGuestManager, s GuestRuntimeInstance) error { guestDesc := s.GetDesc() if s.IsRunning() { diff --git a/pkg/hostman/guestman/types/interface.go b/pkg/hostman/guestman/types/interface.go index 886aea431b..555b0ee5c5 100644 --- a/pkg/hostman/guestman/types/interface.go +++ b/pkg/hostman/guestman/types/interface.go @@ -29,6 +29,7 @@ var HealthCheckReactor IHealthCheckReactor type IGuestDescGetter interface { GetGuestNicDesc(mac, ip, port, bridge string, isCandidate bool) (*desc.SGuestDesc, *desc.SGuestNetwork) + GetAllGuestIPv6Macs(bridge string) []string } var GuestDescGetter IGuestDescGetter diff --git a/pkg/hostman/hostinfo/hostdhcp/dhcpserver6.go b/pkg/hostman/hostinfo/hostdhcp/dhcpserver6.go index 1d7055f370..b961b629de 100644 --- a/pkg/hostman/hostinfo/hostdhcp/dhcpserver6.go +++ b/pkg/hostman/hostinfo/hostdhcp/dhcpserver6.go @@ -44,8 +44,8 @@ type SGuestDHCP6Server struct { ifaceDev *netutils2.SNetInterface raExitCh chan struct{} - raReqCh chan *sRARequest - raReqQueue []*sRARequest + raReqCh chan net.HardwareAddr + raReqQueue map[string]*sRARequest gwMacCache *hashcache.Cache } @@ -76,8 +76,8 @@ func NewGuestDHCP6Server(iface string, port int, relay *SDHCPRelayUpstream) (*SG } guestdhcp.raExitCh = make(chan struct{}) - guestdhcp.raReqCh = make(chan *sRARequest, 100) - guestdhcp.raReqQueue = make([]*sRARequest, 0) + guestdhcp.raReqCh = make(chan net.HardwareAddr, 1024) + guestdhcp.raReqQueue = make(map[string]*sRARequest) guestdhcp.gwMacCache = hashcache.NewCache(1024, 5*time.Minute) return guestdhcp, nil @@ -169,3 +169,15 @@ func (s *SGuestDHCP6Server) serveDHCPInternal(pkt dhcp.Packet, cliMac net.Hardwa } return nil, nil } + +func (s *SGuestDHCP6Server) InitRAQueue() { + macs := guestman.GuestDescGetter.GetAllGuestIPv6Macs(s.ifaceDev.String()) + for _, mac := range macs { + hw, err := net.ParseMAC(mac) + if err != nil { + log.Errorf("ParseMAC %s: %v", mac, err) + continue + } + s.requestRA(hw) + } +} diff --git a/pkg/hostman/hostinfo/hostdhcp/icmp6handlers.go b/pkg/hostman/hostinfo/hostdhcp/icmp6handlers.go index 8f4c6ec1f6..c0f6fd8f88 100644 --- a/pkg/hostman/hostinfo/hostdhcp/icmp6handlers.go +++ b/pkg/hostman/hostinfo/hostdhcp/icmp6handlers.go @@ -29,28 +29,19 @@ import ( ) type sRARequest struct { - solicitation *icmp6.SRouterSolicitation + vmMac net.HardwareAddr attempts int succAttempts int + nextAttempt time.Time +} + +func (s *SGuestDHCP6Server) requestRA(vmMac net.HardwareAddr) { + s.raReqCh <- vmMac } func (s *SGuestDHCP6Server) handleRouterSolicitation(msg *icmp6.SRouterSolicitation) error { // solicitation request from guest - var conf = s.getConfig(msg.SrcMac) - if conf != nil && conf.ClientIP6 != nil && conf.Gateway6 != nil { - req := &sRARequest{ - solicitation: msg, - attempts: 1, - } - succ, err := s.sendRouterAdvertisement(msg) - if err != nil { - return errors.Wrapf(err, "sendRouterAdvertisement") - } - if succ { - req.succAttempts++ - } - s.raReqCh <- req - } + s.requestRA(msg.SrcMac) return nil } @@ -100,8 +91,11 @@ func (s *SGuestDHCP6Server) requestGatewayMac(gwIP net.IP, vlanId uint16) { } } -func (s *SGuestDHCP6Server) sendRouterAdvertisement(solicitation *icmp6.SRouterSolicitation) (bool, error) { - var conf = s.getConfig(solicitation.SrcMac) +func (s *SGuestDHCP6Server) sendRouterAdvertisement(vmMac net.HardwareAddr) (bool, error) { + var conf = s.getConfig(vmMac) + if conf == nil { + return false, errors.Wrapf(errors.ErrNotFound, "getConfig %s", vmMac.String()) + } if conf != nil && conf.ClientIP6 != nil && conf.Gateway6 != nil { gwMacObj := s.gwMacCache.AtomicGet(conf.Gateway6.String()) if gwMacObj == nil { @@ -130,7 +124,7 @@ func (s *SGuestDHCP6Server) sendRouterAdvertisement(solicitation *icmp6.SRouterS // https://www.rfc-editor.org/rfc/rfc4861.html#page-18 // !!! MUST be the link-local address assigned to the interface from which this message is sent. SrcIP: gwLinkLocalIP, - DstMac: solicitation.SrcMac, + DstMac: vmMac, DstIP: net.ParseIP("ff02::1"), }, CurHopLimit: 64, @@ -138,7 +132,7 @@ func (s *SGuestDHCP6Server) sendRouterAdvertisement(solicitation *icmp6.SRouterS IsOther: true, IsHomeAgent: false, Preference: pref, - RouterLifetime: 9000, + RouterLifetime: uint16(options.HostOptions.Dhcp6RouterLifetimeSeconds), ReachableTime: 0, RetransTimer: 0, MTU: uint32(conf.MTU), @@ -148,8 +142,8 @@ func (s *SGuestDHCP6Server) sendRouterAdvertisement(solicitation *icmp6.SRouterS IsAutoconf: false, Prefix: ipnet.IP, PrefixLen: conf.PrefixLen6, - ValidLifetime: 4500, - PreferredLifetime: 2250, + ValidLifetime: uint32(options.HostOptions.Dhcp6RouterLifetimeSeconds / 2), + PreferredLifetime: uint32(options.HostOptions.Dhcp6RouterLifetimeSeconds / 4), }, }, } @@ -162,13 +156,13 @@ func (s *SGuestDHCP6Server) sendRouterAdvertisement(solicitation *icmp6.SRouterS IsAutoconf: false, Prefix: route.Prefix, PrefixLen: route.PrefixLen, - ValidLifetime: 4500, - PreferredLifetime: 2250, + ValidLifetime: uint32(options.HostOptions.Dhcp6RouterLifetimeSeconds / 2), + PreferredLifetime: uint32(options.HostOptions.Dhcp6RouterLifetimeSeconds / 4), }) } else if route.Gateway.String() != conf.Gateway6.String() { // routes forwarded by router ra.RouteInfo = append(ra.RouteInfo, icmp6.SRouteInfoOption{ - RouteLifetime: 9000, + RouteLifetime: uint32(options.HostOptions.Dhcp6RouterLifetimeSeconds), Prefix: route.Prefix, PrefixLen: route.PrefixLen, Preference: pref, @@ -182,7 +176,7 @@ func (s *SGuestDHCP6Server) sendRouterAdvertisement(solicitation *icmp6.SRouterS return false, errors.Wrapf(err, "EncodePacket") } - err = s.conn.SendRaw(bytes, solicitation.SrcMac) + err = s.conn.SendRaw(bytes, vmMac) if err != nil { log.Errorf("Send RouterAdvertisement error: %v", err) } @@ -195,6 +189,46 @@ func (s *SGuestDHCP6Server) stopRAServer() { close(s.raReqCh) } +func (s *SGuestDHCP6Server) handleRARequest(vmMac net.HardwareAddr) { + vmMacStr := vmMac.String() + raReq, ok := s.raReqQueue[vmMacStr] + if !ok { + raReq = &sRARequest{ + vmMac: vmMac, + } + s.raReqQueue[vmMacStr] = raReq + } else { + raReq.nextAttempt = time.Time{} + } + // send RA immediately + raReq.attempts++ + succ, err := s.sendRouterAdvertisement(vmMac) + if err != nil { + if errors.Cause(err) == errors.ErrNotFound { + // no such vm, giveup + delete(s.raReqQueue, vmMacStr) + return + } + log.Errorf("sendRouterAdvertisement error: %v", err) + } + if succ { + raReq.succAttempts++ + } + if raReq.succAttempts < options.HostOptions.Dhcp6RouterAdvertisementAttempts { + if raReq.attempts > 2*options.HostOptions.Dhcp6RouterAdvertisementAttempts { + // giveup + delete(s.raReqQueue, vmMacStr) + } else { + // retry next round + } + } else { + // schedule RA when prefix router lifetime expires + raReq.attempts = 0 + raReq.succAttempts = 0 + raReq.nextAttempt = time.Now().Add(time.Second * time.Duration(options.HostOptions.Dhcp6RouterLifetimeSeconds/4)) + } +} + func (s *SGuestDHCP6Server) startRAServer() { // a tiny RA server stop := false @@ -204,29 +238,14 @@ func (s *SGuestDHCP6Server) startRAServer() { stop = true case raReq := <-s.raReqCh: // handle RA request - s.raReqQueue = append(s.raReqQueue, raReq) + s.handleRARequest(raReq) case <-time.After(time.Second * time.Duration(options.HostOptions.Dhcp6RouterAdvertisementIntervalSecs)): // send RA - // log.Infof("timeout, to announce RA %d requests", len(s.raReqQueue)) - if len(s.raReqQueue) > 0 { - raReqQueue := s.raReqQueue - s.raReqQueue = make([]*sRARequest, 0) - - for i := range raReqQueue { - raReq := raReqQueue[i] - raReq.attempts++ - succ, err := s.sendRouterAdvertisement(raReq.solicitation) - if err != nil { - log.Errorf("sendRouterAdvertisement error: %v", err) - continue - } - if succ { - raReq.succAttempts++ - } - if raReq.succAttempts < options.HostOptions.Dhcp6RouterAdvertisementAttempts && raReq.attempts < 2*options.HostOptions.Dhcp6RouterAdvertisementAttempts { - s.raReqCh <- raReq - } + for _, raReq := range s.raReqQueue { + if !raReq.nextAttempt.IsZero() && raReq.nextAttempt.After(time.Now()) { + continue } + s.handleRARequest(raReq.vmMac) } } } diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index 02c37e8169..1e8c82074b 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -2801,6 +2801,14 @@ func (h *SHostInfo) startBindReservedCpus(processesPrefix []string) { } } +func (h *SHostInfo) OnGuestLoadingComplete() { + for _, nic := range h.Nics { + if nic.dhcpServer6 != nil { + nic.dhcpServer6.InitRAQueue() + } + } +} + func NewHostInfo() (*SHostInfo, error) { var res = new(SHostInfo) res.sysinfo = &SSysInfo{} diff --git a/pkg/hostman/hostutils/hostutils.go b/pkg/hostman/hostutils/hostutils.go index 0caab62a35..d437b6066e 100644 --- a/pkg/hostman/hostutils/hostutils.go +++ b/pkg/hostman/hostutils/hostutils.go @@ -109,6 +109,8 @@ type IHost interface { SetIGuestManager(guestman IGuestManager) GetIGuestManager() IGuestManager + + OnGuestLoadingComplete() } func GetComputeSession(ctx context.Context) *mcclient.ClientSession { diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index 61262dd5d8..9db75f0ba0 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -50,6 +50,7 @@ type SHostBaseOptions struct { Dhcp6RouterAdvertisementIntervalSecs int `default:"3" help:"DHCPv6 router advertisement interval in seconds, default 3 seconds"` Dhcp6RouterAdvertisementAttempts int `default:"3" help:"DHCPv6 router advertisement attempts, default 3 attempts"` + Dhcp6RouterLifetimeSeconds int `default:"9000" help:"DHCPv6 router lifetime in seconds, default 9000 seconds"` Ext4LargefileSizeGb int `default:"4096" help:"Use largefile options when the ext4 fs greater than this size"` Ext4HugefileSizeGb int `default:"512" help:"Use huge options when the ext4 fs greater than this size"`