diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 8540151330..f690d12a1f 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -6274,6 +6274,7 @@ func (self *SGuest) ToSchedDesc() *schedapi.ScheduleInput { config.Hypervisor = self.GetHypervisor() desc.ServerConfig = *config desc.OsArch = self.OsArch + desc.ExtraCpuCount = self.ExtraCpuCount return desc } diff --git a/pkg/hostman/guestman/guesthelper.go b/pkg/hostman/guestman/guesthelper.go index 13114da85e..6a72eeabe5 100644 --- a/pkg/hostman/guestman/guesthelper.go +++ b/pkg/hostman/guestman/guesthelper.go @@ -20,6 +20,8 @@ import ( "sort" "sync" + "github.com/jaypipes/ghw/pkg/topology" + "yunion.io/x/cloudmux/pkg/multicloud/esxi/vcenter" "yunion.io/x/jsonutils" "yunion.io/x/log" @@ -29,6 +31,7 @@ import ( "yunion.io/x/onecloud/pkg/apis/compute" hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/hostman/guestman/desc" + "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/hostman/storageman" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/cgrouputils/cpuset" @@ -215,16 +218,29 @@ type SQgaGuestSetNetwork struct { type CpuSetCounter struct { Nodes []*NumaNode NumaEnabled bool + CPUCmtbound float32 Lock sync.Mutex } -func NewGuestCpuSetCounter(info *hostapi.HostTopology, reservedCpus *cpuset.CPUSet, numaAllocate bool, hugepageSizeKB, cpuCmtbound int) (*CpuSetCounter, error) { +func NewGuestCpuSetCounter( + info *hostapi.HostTopology, reservedCpus *cpuset.CPUSet, numaAllocate, isContainerHost bool, + hugepageSizeKB int, cpuCmtbound, memCmtBound float32, reservedMemMb int, +) (*CpuSetCounter, error) { cpuSetCounter := new(CpuSetCounter) cpuSetCounter.Nodes = make([]*NumaNode, len(info.Nodes)) cpuSetCounter.NumaEnabled = numaAllocate + cpuSetCounter.CPUCmtbound = cpuCmtbound hasL3Cache := false + nodeReserveMem := reservedMemMb / len(info.Nodes) * 1024 for i := 0; i < len(info.Nodes); i++ { - node, err := NewNumaNode(info.Nodes[i].ID, cpuSetCounter.NumaEnabled, hugepageSizeKB) + node, err := NewNumaNode( + info.Nodes[i], + cpuSetCounter.NumaEnabled, + isContainerHost, + hugepageSizeKB, + memCmtBound, + nodeReserveMem, + ) if err != nil { return nil, err } @@ -280,7 +296,7 @@ func NewGuestCpuSetCounter(info *hostapi.HostTopology, reservedCpus *cpuset.CPUS func (pq *CpuSetCounter) AllocCpusetWithNodeCount(vcpuCount int, memSizeKB int64, nodeCount int) (map[int]SAllocNumaCpus, error) { if !pq.NumaEnabled { - return pq.AllocCpuset(vcpuCount, memSizeKB, -1) + return pq.AllocCpuset(vcpuCount, memSizeKB, nil) } if len(pq.Nodes) < nodeCount { return nil, nil @@ -290,7 +306,7 @@ func (pq *CpuSetCounter) AllocCpusetWithNodeCount(vcpuCount int, memSizeKB int64 defer pq.Lock.Unlock() var res = map[int]SAllocNumaCpus{} var nodeAllocSize = memSizeKB / int64(nodeCount) - if nodeAllocSize/1024%1024 == 0 && pq.nodesFreeMemSizeEnough(nodeCount, memSizeKB) { + if pq.nodesEnough(nodeCount, vcpuCount, int(memSizeKB)) { var pcpuCount = vcpuCount / nodeCount var remPcpuCount = vcpuCount % nodeCount @@ -301,11 +317,11 @@ func (pq *CpuSetCounter) AllocCpusetWithNodeCount(vcpuCount int, memSizeKB int64 remPcpuCount -= 1 } res[pq.Nodes[i].NodeId] = SAllocNumaCpus{ - Cpuset: pq.Nodes[i].AllocCpuset1(npcpuCount), + Cpuset: pq.Nodes[i].AllocCpuset(npcpuCount), MemSizeKB: nodeAllocSize, Unregular: false, } - pq.Nodes[i].NumaHugeFreeMemSizeKB -= nodeAllocSize + pq.Nodes[i].NumaNodeFreeMemSizeKB -= nodeAllocSize pq.Nodes[i].VcpuCount += npcpuCount } } @@ -323,9 +339,7 @@ func (pq *CpuSetCounter) IsNumaEnabled() bool { return pq.NumaEnabled } -func (pq *CpuSetCounter) AllocCpuset(vcpuCount int, memSizeKB int64, perferNumaNode int8) (map[int]SAllocNumaCpus, error) { - res := map[int]SAllocNumaCpus{} - sourceVcpuCount := vcpuCount +func (pq *CpuSetCounter) AllocCpuset(vcpuCount int, memSizeKB int64, preferNumaNodes []int8) (map[int]SAllocNumaCpus, error) { pq.Lock.Lock() defer pq.Lock.Unlock() @@ -333,32 +347,29 @@ func (pq *CpuSetCounter) AllocCpuset(vcpuCount int, memSizeKB int64, perferNumaN return nil, nil } - if pq.NumaEnabled { - err := pq.AllocNumaNodes(vcpuCount, memSizeKB, perferNumaNode, res) - return res, err - } else { - if perferNumaNode > 0 { - for i := range pq.Nodes { - if pq.Nodes[i].NodeId != int(perferNumaNode) { - continue + if pq.NumaEnabled && len(preferNumaNodes) > 0 { + sortedNumaDistance := pq.getDistancesSeqByPreferNodes(preferNumaNodes) + for nodeCount := 1; nodeCount <= len(pq.Nodes); nodeCount *= 2 { + ret := pq.allocCpuNumaNodesByPreferNodes(vcpuCount, int(memSizeKB), nodeCount, sortedNumaDistance) + if ret != nil { + for i := range pq.Nodes { + if cpupin, ok := ret[pq.Nodes[i].NodeId]; ok { + pq.Nodes[i].VcpuCount += vcpuCount + pq.Nodes[i].NumaNodeFreeMemSizeKB -= cpupin.MemSizeKB + } } - if pq.Nodes[i].VcpuCount >= (pq.Nodes[0].VcpuCount + pq.Nodes[0].CpuCount) { - break - } - - allocCount := vcpuCount - if vcpuCount > pq.Nodes[0].CpuCount { - allocCount = vcpuCount/2 + vcpuCount%2 - } - res[pq.Nodes[i].NodeId] = SAllocNumaCpus{ - Cpuset: pq.Nodes[i].AllocCpuset(allocCount), - } - pq.Nodes[i].VcpuCount += sourceVcpuCount - vcpuCount -= allocCount sort.Sort(pq) - break + return ret, nil } } + } + + res := map[int]SAllocNumaCpus{} + sourceVcpuCount := vcpuCount + if pq.NumaEnabled { + err := pq.AllocNumaNodes(vcpuCount, memSizeKB, res) + return res, err + } else { for vcpuCount > 0 { count := vcpuCount if vcpuCount > pq.Nodes[0].CpuCount { @@ -375,37 +386,85 @@ func (pq *CpuSetCounter) AllocCpuset(vcpuCount int, memSizeKB int64, perferNumaN } } -func (pq *CpuSetCounter) AllocNumaNodes(vcpuCount int, memSizeKB int64, perferNumaNode int8, res map[int]SAllocNumaCpus) error { - var allocated = false +func (pq *CpuSetCounter) allocCpuNumaNodesByPreferNodes( + vcpuCount, memSizeKB, nodeCount int, sortedNumaDistance []SSortedNumaDistance, +) map[int]SAllocNumaCpus { + res := map[int]SAllocNumaCpus{} + var nodeAllocSize = memSizeKB / nodeCount + var pcpuCount = vcpuCount / nodeCount + var remPcpuCount = vcpuCount % nodeCount - // check preferred numa node is memory enough - if perferNumaNode >= 0 { - for i := 0; i < len(pq.Nodes); i++ { - if pq.Nodes[i].NodeId != int(perferNumaNode) { - continue - } - if pq.Nodes[i].NumaHugeFreeMemSizeKB >= memSizeKB { - res[pq.Nodes[i].NodeId] = SAllocNumaCpus{ - Cpuset: pq.Nodes[i].AllocCpuset1(vcpuCount), - MemSizeKB: memSizeKB, - Unregular: false, - } - pq.Nodes[i].NumaHugeFreeMemSizeKB -= memSizeKB - pq.Nodes[i].VcpuCount += vcpuCount - - allocated = true - } + allocatedNode := 0 + for i := range sortedNumaDistance { + if allocatedNode >= nodeCount { break } + + var npcpuCount = pcpuCount + if remPcpuCount > 0 { + npcpuCount += 1 + remPcpuCount -= 1 + } + nodeIdx := sortedNumaDistance[i].NodeIndex + if pq.Nodes[nodeIdx].nodeEnough(vcpuCount, memSizeKB, pq.CPUCmtbound, pq.NumaEnabled) { + cpuNumaPin := SAllocNumaCpus{ + Cpuset: pq.Nodes[nodeIdx].AllocCpuset(npcpuCount), + } + cpuNumaPin.MemSizeKB = int64(nodeAllocSize) + res[pq.Nodes[nodeIdx].NodeId] = cpuNumaPin + allocatedNode += 1 + } else { + log.Infof("node %v not enough", pq.Nodes[i]) + } + log.Infof("node %d, free mems %d", pq.Nodes[nodeIdx].NodeId, pq.Nodes[nodeIdx].NumaNodeFreeMemSizeKB) } + if allocatedNode < nodeCount { + return nil + } + return res +} + +type SSortedNumaDistance struct { + NodeIndex int + Distance int + FreeMemSize int +} + +func (pq *CpuSetCounter) getDistancesSeqByPreferNodes(preferNumaNodes []int8) []SSortedNumaDistance { + sortedNumaDistance := make([]SSortedNumaDistance, len(pq.Nodes)) + for i := range pq.Nodes { + distance := 0 + for j := range preferNumaNodes { + distance += pq.Nodes[i].Distances[preferNumaNodes[j]] + } + sortedNumaDistance[i] = SSortedNumaDistance{ + NodeIndex: i, + Distance: distance, + FreeMemSize: int(pq.Nodes[i].NumaNodeFreeMemSizeKB), + } + } + sort.Slice(sortedNumaDistance, func(i, j int) bool { + // 7 is tolerant max distances + if (sortedNumaDistance[i].Distance + 7) < sortedNumaDistance[j].Distance { + return true + } else { + return sortedNumaDistance[i].FreeMemSize > sortedNumaDistance[j].FreeMemSize + } + }) + return sortedNumaDistance +} + +func (pq *CpuSetCounter) AllocNumaNodes(vcpuCount int, memSizeKB int64, res map[int]SAllocNumaCpus) error { + var allocated = false + // alloc numa nodes in order 1, 2, 4, ... if !allocated { for nodeCount := 1; nodeCount <= len(pq.Nodes); nodeCount *= 2 { if nodeCount > vcpuCount { break } - if ok := pq.nodesFreeMemSizeEnough(nodeCount, memSizeKB); !ok { + if ok := pq.nodesEnough(nodeCount, vcpuCount, int(memSizeKB)); !ok { log.Infof("node count %d not enough", nodeCount) continue } @@ -423,11 +482,11 @@ func (pq *CpuSetCounter) AllocNumaNodes(vcpuCount int, memSizeKB int64, perferNu remPcpuCount -= 1 } res[pq.Nodes[i].NodeId] = SAllocNumaCpus{ - Cpuset: pq.Nodes[i].AllocCpuset1(npcpuCount), + Cpuset: pq.Nodes[i].AllocCpuset(npcpuCount), MemSizeKB: nodeAllocSize, Unregular: false, } - pq.Nodes[i].NumaHugeFreeMemSizeKB -= nodeAllocSize + pq.Nodes[i].NumaNodeFreeMemSizeKB -= nodeAllocSize pq.Nodes[i].VcpuCount += npcpuCount } allocated = true @@ -445,16 +504,41 @@ func (pq *CpuSetCounter) AllocNumaNodes(vcpuCount int, memSizeKB int64, perferNu return nil } +func (pq *CpuSetCounter) nodesEnough(nodeCount, vcpuCount int, memSizeKB int) bool { + var leastFree = memSizeKB / nodeCount + var leastCpuCount = vcpuCount / nodeCount + var remPcpuCount = vcpuCount % nodeCount + + for i := 0; i < nodeCount; i++ { + if pq.NumaEnabled { + if int(pq.Nodes[i].NumaNodeFreeMemSizeKB) < leastFree { + return false + } + } + + requireCpuCount := leastCpuCount + if remPcpuCount > 0 { + requireCpuCount += 1 + remPcpuCount -= 1 + } + if (pq.Nodes[i].VcpuCount + requireCpuCount) > int(float32(pq.Nodes[i].CpuCount)*pq.CPUCmtbound) { + return false + } + + } + return true +} + func (pq *CpuSetCounter) nodesFreeMemSizeEnough(nodeCount int, memSizeKB int64) bool { var freeMem int64 = 0 var leastFree = memSizeKB / int64(nodeCount) log.Debugf("request memsize %d, least free %d", memSizeKB, leastFree) for i := 0; i < nodeCount; i++ { - log.Debugf("index %d node %d free size %d", i, pq.Nodes[i].NodeId, pq.Nodes[i].NumaHugeFreeMemSizeKB) - if pq.Nodes[i].NumaHugeFreeMemSizeKB < leastFree { + log.Debugf("index %d node %d free size %d", i, pq.Nodes[i].NodeId, pq.Nodes[i].NumaNodeFreeMemSizeKB) + if pq.Nodes[i].NumaNodeFreeMemSizeKB < leastFree { return false } - freeMem += pq.Nodes[i].NumaHugeFreeMemSizeKB + freeMem += pq.Nodes[i].NumaNodeFreeMemSizeKB } return freeMem >= memSizeKB } @@ -466,12 +550,12 @@ func (pq *CpuSetCounter) setNumaNodes(numaMaps map[int]int, vcpuCount int64) map allocMem := int64(size) * 1024 //npcpuCount := int(vcpuCount*allocMem/memSizeKB + (vcpuCount*allocMem)%memSizeKB) res[pq.Nodes[i].NodeId] = SAllocNumaCpus{ - Cpuset: pq.Nodes[i].AllocCpuset1(int(vcpuCount)), + Cpuset: pq.Nodes[i].AllocCpuset(int(vcpuCount)), MemSizeKB: allocMem, Unregular: true, } - pq.Nodes[i].NumaHugeFreeMemSizeKB -= allocMem + pq.Nodes[i].NumaNodeFreeMemSizeKB -= allocMem pq.Nodes[i].VcpuCount += int(vcpuCount) } } @@ -509,7 +593,7 @@ func (pq *CpuSetCounter) ReleaseNumaCpus(memSizeMb int64, hostNode int, cpus []i } pq.Nodes[i].CpuDies.ReleaseCpus(cpus, vcpuCount) pq.Nodes[i].VcpuCount -= vcpuCount - pq.Nodes[i].NumaHugeFreeMemSizeKB += memSizeMb * 1024 + pq.Nodes[i].NumaNodeFreeMemSizeKB += memSizeMb * 1024 } sort.Sort(pq) } @@ -521,7 +605,7 @@ func (pq *CpuSetCounter) LoadNumaCpus(memSizeMb int64, hostNode int, cpus []int, } pq.Nodes[i].CpuDies.LoadCpus(cpus, vcpuCount) pq.Nodes[i].VcpuCount += vcpuCount - pq.Nodes[i].NumaHugeFreeMemSizeKB -= memSizeMb * 1024 + pq.Nodes[i].NumaNodeFreeMemSizeKB -= memSizeMb * 1024 } sort.Sort(pq) } @@ -552,13 +636,15 @@ func (pq *CpuSetCounter) LoadCpus(cpus []int, vcpuCpunt int) { func (pq CpuSetCounter) Len() int { return len(pq.Nodes) } func (pq CpuSetCounter) Less(i, j int) bool { + freeCpuI := int(float32(pq.Nodes[i].CpuCount)*pq.CPUCmtbound) - pq.Nodes[i].VcpuCount + freeCpuJ := int(float32(pq.Nodes[i].CpuCount)*pq.CPUCmtbound) - pq.Nodes[j].VcpuCount if pq.NumaEnabled { - if pq.Nodes[i].NumaHugeFreeMemSizeKB == pq.Nodes[j].NumaHugeFreeMemSizeKB { - return pq.Nodes[i].VcpuCount < pq.Nodes[j].VcpuCount + if pq.Nodes[i].NumaNodeFreeMemSizeKB == pq.Nodes[j].NumaNodeFreeMemSizeKB { + return freeCpuI > freeCpuJ } - return pq.Nodes[i].NumaHugeFreeMemSizeKB > pq.Nodes[j].NumaHugeFreeMemSizeKB + return pq.Nodes[i].NumaNodeFreeMemSizeKB > pq.Nodes[j].NumaNodeFreeMemSizeKB } else { - return pq.Nodes[i].VcpuCount < pq.Nodes[j].VcpuCount + return freeCpuI > freeCpuJ } } @@ -586,39 +672,67 @@ type NumaNode struct { CpuCount int NodeId int - NumaHugeMemSizeKB int64 - NumaHugeFreeMemSizeKB int64 + Distances []int + NumaNodeMemSizeKB int64 + NumaNodeFreeMemSizeKB int64 } -func NewNumaNode(nodeId int, numaAllocate bool, hugepageSizeKB int) (*NumaNode, error) { +func NewNumaNode( + nodeInfo *topology.Node, + numaAllocate, isContainerHost bool, + hugepageSizeKB int, memCmtBound float32, + reservedMemSizeKB int, +) (*NumaNode, error) { n := new(NumaNode) n.LogicalProcessors = cpuset.NewCPUSet() - n.NodeId = nodeId + n.NodeId = nodeInfo.ID + n.Distances = nodeInfo.Distances - if numaAllocate { - nodeHugepagePath := fmt.Sprintf("/sys/devices/system/node/node%d/hugepages/hugepages-%dkB", nodeId, hugepageSizeKB) + if !numaAllocate { + return n, nil + } + if isContainerHost { + if nodeInfo.Memory == nil { + return nil, errors.Errorf("node %d no memory info: %#v", nodeInfo.ID, nodeInfo) + } + n.NumaNodeMemSizeKB = int64(float32(nodeInfo.Memory.TotalUsableBytes/1024-int64(reservedMemSizeKB)) * memCmtBound) + } else { + nodeHugepagePath := fmt.Sprintf("/sys/devices/system/node/node%d/hugepages/hugepages-%dkB", n.NodeId, hugepageSizeKB) if !fileutils2.Exists(nodeHugepagePath) { return n, nil } - nrHugepage, err := fileutils2.FileGetIntContent(path.Join(nodeHugepagePath, "nr_hugepages")) if err != nil { - log.Errorf("failed get node %d nr hugepage %s", nodeId, err) + log.Errorf("failed get node %d nr hugepage %s", n.NodeId, err) return nil, errors.Wrap(err, "get numa node nr hugepage") } - n.NumaHugeMemSizeKB = int64(nrHugepage) * int64(hugepageSizeKB) - - //freeHugepage, err := fileutils2.FileGetIntContent(path.Join(nodeHugepagePath, "free_hugepages")) - //if err != nil { - // log.Errorf("failed get node %d free hugepage %s", nodeId, err) - // return nil, errors.Wrap(err, "get numa node free hugepage") - //} - n.NumaHugeFreeMemSizeKB = n.NumaHugeMemSizeKB + n.NumaNodeMemSizeKB = int64(nrHugepage) * int64(hugepageSizeKB) } + n.NumaNodeFreeMemSizeKB = n.NumaNodeMemSizeKB + return n, nil } -func (n *NumaNode) AllocCpuset1(vcpuCount int) []int { +func (n *NumaNode) nodeEnough(vcpuCount, memSizeKB int, cmtBound float32, enableNumaAlloc bool) bool { + if int(float32(n.CpuCount)*cmtBound)-n.VcpuCount < vcpuCount { + return false + } + if enableNumaAlloc { + if int(n.NumaNodeFreeMemSizeKB) < memSizeKB { + return false + } + } + return true +} + +func (n *NumaNode) AllocCpuset(vcpuCount int) []int { + if options.HostOptions.EnableStrictCpuBind { + return n.allocCpusetStrict(vcpuCount) + } + return n.allocCpusetOnNode(vcpuCount) +} + +func (n *NumaNode) allocCpusetStrict(vcpuCount int) []int { var allocCount = vcpuCount var dieCnt = 0 @@ -654,14 +768,15 @@ func (n *NumaNode) AllocCpuset1(vcpuCount int) []int { } } - var pcpus = make([]int, 0) + defer sort.Sort(n.CpuDies) + var ret = make([]int, 0) for i := 0; i < len(allocCpuCountMap); i++ { var allocCpuCount = allocCpuCountMap[i] for allocCpuCount > 0 { pcpus := n.CpuDies[i].LogicalProcessors.ToSliceNoSort() for j := 0; j < len(pcpus); j++ { if n.CpuDies[i].CpuFree[pcpus[j]] > 0 { - pcpus = append(pcpus, n.CpuDies[i].CpuFree[pcpus[j]]) + ret = append(ret, n.CpuDies[i].CpuFree[pcpus[j]]) n.CpuDies[i].CpuFree[pcpus[j]] -= 1 } allocCpuCount -= 1 @@ -671,10 +786,10 @@ func (n *NumaNode) AllocCpuset1(vcpuCount int) []int { } } } - return pcpus + return ret } -func (n *NumaNode) AllocCpuset(vcpuCount int) []int { +func (n *NumaNode) allocCpusetOnNode(vcpuCount int) []int { cpus := make([]int, 0) var allocCount = vcpuCount @@ -698,10 +813,10 @@ type CPUDie struct { VcpuCount int } -func (d *CPUDie) initCpuFree(cpuCmtbound int) { +func (d *CPUDie) initCpuFree(cpuCmtbound float32) { cpuFree := map[int]int{} for _, cpuId := range d.LogicalProcessors.ToSliceNoSort() { - cpuFree[cpuId] = cpuCmtbound + cpuFree[cpuId] = int(cpuCmtbound) } d.CpuFree = cpuFree } diff --git a/pkg/hostman/guestman/guestman.go b/pkg/hostman/guestman/guestman.go index 5c68e2b755..3f0d1cee8e 100644 --- a/pkg/hostman/guestman/guestman.go +++ b/pkg/hostman/guestman/guestman.go @@ -295,11 +295,14 @@ func (m *SGuestManager) Bootstrap() (chan struct{}, error) { hostTypo := m.host.GetHostTopology() if options.HostOptions.EnableHostAgentNumaAllocate { - m.numaAllocate = m.host.IsNumaAllocateEnabled() && m.host.IsHugepagesEnabled() && (len(hostTypo.Nodes) > 1) + enableMemAlloc := m.host.IsContainerHost() || m.host.IsHugepagesEnabled() + m.numaAllocate = !m.host.IsNumaAllocateEnabled() && enableMemAlloc && (len(hostTypo.Nodes) > 1) } cpuSet, err := NewGuestCpuSetCounter( - hostTypo, m.host.GetReservedCpusInfo(), m.numaAllocate, m.host.HugepageSizeKb(), m.host.CpuCmtBound()) + hostTypo, m.host.GetReservedCpusInfo(), m.numaAllocate, m.host.IsContainerHost(), + m.host.HugepageSizeKb(), m.host.CpuCmtBound(), m.host.MemCmtBound(), m.host.GetReservedMemMb(), + ) if err != nil { return nil, err } diff --git a/pkg/hostman/guestman/guesttasks.go b/pkg/hostman/guestman/guesttasks.go index fc5bc2e774..26f434aee3 100644 --- a/pkg/hostman/guestman/guesttasks.go +++ b/pkg/hostman/guestman/guesttasks.go @@ -2488,7 +2488,7 @@ func (task *SGuestHotplugCpuMemTask) startAddCpusWithFreeVcpuSet(vcpuSet []int) } } } else { - cpus, _ := task.manager.cpuSet.AllocCpuset(1, 0, -1) + cpus, _ := task.manager.cpuSet.AllocCpuset(1, 0, nil) for _, cpus := range cpus { //pcpus := cpuset.NewCPUSet(cpus.Cpuset...).String() //vcpus := fmt.Sprintf("%d-%d", vcpuId, vcpuId) diff --git a/pkg/hostman/guestman/pod.go b/pkg/hostman/guestman/pod.go index 8967c4a38a..981319aea3 100644 --- a/pkg/hostman/guestman/pod.go +++ b/pkg/hostman/guestman/pod.go @@ -760,11 +760,24 @@ func (s *sPodGuestInstance) namespacesFroPod(input *computeapi.PodCreateInput) * } } +func (s *sPodGuestInstance) updateGuestDesc() error { + s.Desc = new(desc.SGuestDesc) + err := jsonutils.Marshal(s.SourceDesc).Unmarshal(s.Desc) + if err != nil { + return errors.Wrap(err, "unmarshal source desc") + } + + return s.allocateCpuNumaPin() +} + func (s *sPodGuestInstance) _startPod(ctx context.Context, userCred mcclient.TokenCredential) (*computeapi.PodStartResponse, error) { podInput, err := s.getPodCreateParams() if err != nil { return nil, errors.Wrap(err, "getPodCreateParams") } + if err := s.updateGuestDesc(); err != nil { + return nil, errors.Wrap(err, "updateGuestDesc") + } if err := s.mountPodVolumes(); err != nil { return nil, errors.Wrap(err, "mountPodVolumes") } @@ -919,7 +932,6 @@ func (s *sPodGuestInstance) ensurePodRemoved(ctx context.Context, timeout int64) } func (s *sPodGuestInstance) stopPod(ctx context.Context, timeout int64) error { - ReleaseGuestCpuset(s.manager, s) if err := s.umountPodVolumes(); err != nil { return errors.Wrapf(err, "umount pod volumes") } @@ -927,7 +939,11 @@ func (s *sPodGuestInstance) stopPod(ctx context.Context, timeout int64) error { timeout = 15 } - return s.ensurePodRemoved(ctx, timeout) + if err := s.ensurePodRemoved(ctx, timeout); err != nil { + return err + } + ReleaseGuestCpuset(s.manager, s) + return nil } func (s *sPodGuestInstance) LoadDesc() error { @@ -1086,15 +1102,15 @@ func (s *sPodGuestInstance) allocateCpuNumaPin() error { } var cpus = make([]int, 0) - var perferNumaNode int8 = -1 + var preferNumaNodes = make([]int8, 0) for i := range s.Desc.IsolatedDevices { if s.Desc.IsolatedDevices[i].NumaNode >= 0 { - perferNumaNode = s.Desc.IsolatedDevices[i].NumaNode + preferNumaNodes = append(preferNumaNodes, s.Desc.IsolatedDevices[i].NumaNode) break } } - nodeNumaCpus, err := s.manager.cpuSet.AllocCpuset(int(s.Desc.Cpu), s.Desc.Mem*1024, perferNumaNode) + nodeNumaCpus, err := s.manager.cpuSet.AllocCpuset(int(s.Desc.Cpu), s.Desc.Mem*1024, preferNumaNodes) if err != nil { return err } @@ -1109,6 +1125,26 @@ func (s *sPodGuestInstance) allocateCpuNumaPin() error { Pcpus: cpuset.NewCPUSet(cpus...).String(), }, } + } else { + var cpuNumaPin = make([]*desc.SCpuNumaPin, 0) + for nodeId, numaCpus := range nodeNumaCpus { + if s.manager.numaAllocate { + unodeId := uint16(nodeId) + vcpuPin := make([]desc.SVCpuPin, len(numaCpus.Cpuset)) + for i := range numaCpus.Cpuset { + vcpuPin[i].Pcpu = numaCpus.Cpuset[i] + } + + memPin := &desc.SCpuNumaPin{ + SizeMB: numaCpus.MemSizeKB / 1024, // MB + NodeId: &unodeId, + VcpuPin: vcpuPin, + Unregular: numaCpus.Unregular, + } + cpuNumaPin = append(cpuNumaPin, memPin) + } + } + s.Desc.CpuNumaPin = cpuNumaPin } return SaveLiveDesc(s, s.Desc) @@ -1458,10 +1494,6 @@ func (s *sPodGuestInstance) createContainer(ctx context.Context, userCred mcclie }) } - if err := s.allocateCpuNumaPin(); err != nil { - return "", errors.Wrap(err, "allocateCpuNumaPin") - } - var cpuSetCpus string var cpuSetMems string var extraCpuCount int diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index 090ac8837f..19f3e6c63b 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -2692,15 +2692,15 @@ func (s *SKVMGuestInstance) setCgroupCPUSet() error { func (s *SKVMGuestInstance) allocGuestNumaCpuset() error { var cpus = make([]int, 0) var cpuNumaPin = make([]*desc.SCpuNumaPin, 0) - var perferNumaNode int8 = -1 + var preferNumaNodes = make([]int8, 0) for i := range s.Desc.IsolatedDevices { if s.Desc.IsolatedDevices[i].NumaNode >= 0 { - perferNumaNode = s.Desc.IsolatedDevices[i].NumaNode + preferNumaNodes = append(preferNumaNodes, s.Desc.IsolatedDevices[i].NumaNode) break } } - nodeNumaCpus, err := s.manager.cpuSet.AllocCpuset(int(s.Desc.Cpu), s.Desc.Mem*1024, perferNumaNode) + nodeNumaCpus, err := s.manager.cpuSet.AllocCpuset(int(s.Desc.Cpu), s.Desc.Mem*1024, preferNumaNodes) if err != nil { return err } diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index af711b2257..4138993766 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -99,7 +99,8 @@ type SHostInfo struct { onHostDown string reservedCpusInfo *api.HostReserveCpusInput enableNumaAllocate bool - cpuCmtBound int + cpuCmtBound float32 + memCmtBound float32 IsolatedDeviceMan isolated_device.IsolatedDeviceManager @@ -1257,7 +1258,8 @@ func (h *SHostInfo) initHostRecord() (*api.HostDetails, error) { } h.HostId = hostInfo.Id - h.cpuCmtBound = int(hostInfo.CpuCmtbound) + h.cpuCmtBound = hostInfo.CpuCmtbound + h.memCmtBound = hostInfo.MemCommitBound hostInfo, err = h.updateHostMetadata(hostInfo.Name) if err != nil { return nil, errors.Wrap(err, "updateHostMetadata") @@ -1284,8 +1286,8 @@ func (h *SHostInfo) initHostRecord() (*api.HostDetails, error) { } // set host reserved memory - if h.IsHugepagesEnabled() && h.getReservedMemMb() != hostInfo.MemReserved { - if err = h.updateHostReservedMem(h.getReservedMemMb()); err != nil { + if h.IsHugepagesEnabled() && h.GetReservedMemMb() != hostInfo.MemReserved { + if err = h.updateHostReservedMem(h.GetReservedMemMb()); err != nil { return nil, errors.Wrap(err, "updateHostReservedMem") } } @@ -1561,7 +1563,7 @@ func (h *SHostInfo) updateOrCreateHost(hostId string) (*api.HostDetails, error) input.MemSize = fmt.Sprintf("%d", h.GetMemory()) if len(hostId) == 0 { // first time create - input.MemReserved = fmt.Sprintf("%d", h.getReservedMemMb()) + input.MemReserved = fmt.Sprintf("%d", h.GetReservedMemMb()) } if h.IsHugepagesEnabled() { pageSizeKb := options.HostOptions.HugepageSizeMb * 1024 @@ -1713,7 +1715,7 @@ func (h *SHostInfo) getOSReservedMemMb() int { return reserved } -func (h *SHostInfo) getReservedMemMb() int { +func (h *SHostInfo) GetReservedMemMb() int { if h.IsHugepagesEnabled() { hp, _ := h.Mem.GetHugepages() return h.GetMemory() - int(hp.BytesMb()) @@ -2604,10 +2606,14 @@ func (h *SHostInfo) GetContainerRuntimeEndpoint() string { return options.HostOptions.ContainerRuntimeEndpoint } -func (h *SHostInfo) CpuCmtBound() int { +func (h *SHostInfo) CpuCmtBound() float32 { return h.cpuCmtBound } +func (h *SHostInfo) MemCmtBound() float32 { + return h.memCmtBound +} + func NewHostInfo() (*SHostInfo, error) { var res = new(SHostInfo) res.sysinfo = &SSysInfo{} diff --git a/pkg/hostman/hostutils/hostutils.go b/pkg/hostman/hostutils/hostutils.go index fdb557cd20..b9da93d8cc 100644 --- a/pkg/hostman/hostutils/hostutils.go +++ b/pkg/hostman/hostutils/hostutils.go @@ -71,11 +71,13 @@ type IHost interface { IsX8664() bool GetHostTopology() *hostapi.HostTopology GetReservedCpusInfo() *cpuset.CPUSet + GetReservedMemMb() int IsHugepagesEnabled() bool HugepageSizeKb() int IsNumaAllocateEnabled() bool - CpuCmtBound() int + CpuCmtBound() float32 + MemCmtBound() float32 IsKvmSupport() bool IsNestedVirtualization() bool diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index 149ba542d1..5a5b38f3d5 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -140,6 +140,7 @@ type SHostOptions struct { SetVncPassword bool `default:"true" help:"Auto set vnc password after monitor connected"` UseBootVga bool `default:"false" help:"Use boot VGA GPU for guest"` + EnableStrictCpuBind bool `default:"false" help:"Enable strict cpu bind, one vcpu bind one pcpu"` EnableHostAgentNumaAllocate bool `default:"false" help:"Enable host agent numa allocate"` EnableCpuBinding bool `default:"true" help:"Enable cpu binding and rebalance"` EnableOpenflowController bool `default:"false"` diff --git a/pkg/scheduler/cache/candidate/hosts.go b/pkg/scheduler/cache/candidate/hosts.go index c9a8d81839..a8fe31ced5 100644 --- a/pkg/scheduler/cache/candidate/hosts.go +++ b/pkg/scheduler/cache/candidate/hosts.go @@ -273,8 +273,8 @@ type NumaNode struct { NumaNodeFreeMemSizeKB int } -func (n *NumaNode) nodeEnough(vcpuCount, memSizeKB, cmtBound int, enableNumaAlloc bool) bool { - if n.CpuCount*cmtBound-n.VcpuCount < vcpuCount { +func (n *NumaNode) nodeEnough(vcpuCount, memSizeKB int, cmtBound float32, enableNumaAlloc bool) bool { + if int(float32(n.CpuCount)*cmtBound)-n.VcpuCount < vcpuCount { return false } if enableNumaAlloc { @@ -409,7 +409,7 @@ func (n *NumaNode) AllocCpuset(vcpuCount int) []int { return ret } -func NewNumaNode(nodeId int, nodeDistances []int, hugepageSizeKb int, nodeHugepages []hostapi.HostNodeHugepageNr, memSizeKB, memCmtBound int) *NumaNode { +func NewNumaNode(nodeId int, nodeDistances []int, hugepageSizeKb int, nodeHugepages []hostapi.HostNodeHugepageNr, memSizeKB int, memCmtBound float32) *NumaNode { n := new(NumaNode) n.LogicalProcessors = cpuset.NewCPUSet() n.NodeId = nodeId @@ -422,7 +422,7 @@ func NewNumaNode(nodeId int, nodeDistances []int, hugepageSizeKb int, nodeHugepa } } } else { - n.NumaNodeMemSizeKB = memSizeKB * memCmtBound + n.NumaNodeMemSizeKB = int(float32(memSizeKB) * memCmtBound) } n.NumaNodeFreeMemSizeKB = n.NumaNodeMemSizeKB @@ -432,7 +432,7 @@ func NewNumaNode(nodeId int, nodeDistances []int, hugepageSizeKb int, nodeHugepa type SHostTopo struct { Nodes []*NumaNode NumaEnabled bool - CPUCmtbound int + CPUCmtbound float32 HostName string } @@ -540,7 +540,7 @@ func (h *SHostTopo) nodesEnough(nodeCount, vcpuCount int, memSizeKB int) bool { requireCpuCount += 1 remPcpuCount -= 1 } - if (h.Nodes[i].VcpuCount + requireCpuCount) > h.Nodes[i].CpuCount*h.CPUCmtbound { + if (h.Nodes[i].VcpuCount + requireCpuCount) > int(float32(h.Nodes[i].CpuCount)*h.CPUCmtbound) { return false } @@ -549,7 +549,7 @@ func (h *SHostTopo) nodesEnough(nodeCount, vcpuCount int, memSizeKB int) bool { } func (h *SHostTopo) allocCpuNumaNodesByPreferNodes( - vcpuCount, memSizeKB, nodeCount int, preferNumaNodes []int, sortedNumaDistance []SSortedNumaDistance, + vcpuCount, memSizeKB, nodeCount int, sortedNumaDistance []SSortedNumaDistance, ) []scheduler.SCpuNumaPin { res := make([]scheduler.SCpuNumaPin, 0) var nodeAllocSize = memSizeKB / nodeCount @@ -591,8 +591,9 @@ func (h *SHostTopo) allocCpuNumaNodesByPreferNodes( } type SSortedNumaDistance struct { - NodeIndex int - Distance int + NodeIndex int + Distance int + FreeMemSize int } func (h *SHostTopo) getDistancesSeqByPreferNodes(preferNumaNodes []int) []SSortedNumaDistance { @@ -604,12 +605,18 @@ func (h *SHostTopo) getDistancesSeqByPreferNodes(preferNumaNodes []int) []SSorte distance += h.Nodes[i].Distances[preferNumaNodes[j]] } sortedNumaDistance[i] = SSortedNumaDistance{ - NodeIndex: i, - Distance: distance, + NodeIndex: i, + Distance: distance, + FreeMemSize: h.Nodes[i].NumaNodeFreeMemSizeKB, } } sort.Slice(sortedNumaDistance, func(i, j int) bool { - return sortedNumaDistance[i].Distance < sortedNumaDistance[j].Distance + // 7 is tolerant max distances + if (sortedNumaDistance[i].Distance + 7) < sortedNumaDistance[j].Distance { + return true + } else { + return sortedNumaDistance[i].FreeMemSize > sortedNumaDistance[j].FreeMemSize + } }) return sortedNumaDistance } @@ -619,7 +626,7 @@ func (h *SHostTopo) AllocCpuNumaNodes(vcpuCount, memSizeKB int, ignoreMemSingula log.Infof("preferNumaNodes %v", preferNumaNodes) sortedNumaDistance := h.getDistancesSeqByPreferNodes(preferNumaNodes) for nodeCount := 1; nodeCount <= len(h.Nodes); nodeCount *= 2 { - ret := h.allocCpuNumaNodesByPreferNodes(vcpuCount, memSizeKB, nodeCount, preferNumaNodes, sortedNumaDistance) + ret := h.allocCpuNumaNodesByPreferNodes(vcpuCount, memSizeKB, nodeCount, sortedNumaDistance) if ret != nil { return ret } @@ -711,7 +718,7 @@ func (b *HostBuilder) buildHostTopo( } } - node := NewNumaNode(info.Nodes[i].ID, info.Nodes[i].Distances, hugepageSizeKb, nodeHugepages, nodoMemSizeKB, int(desc.MemCmtbound)) + node := NewNumaNode(info.Nodes[i].ID, info.Nodes[i].Distances, hugepageSizeKb, nodeHugepages, nodoMemSizeKB, desc.MemCmtbound) cpuDies := make([]*CPUDie, 0) for j := 0; j < len(info.Nodes[i].Caches); j++ { @@ -768,7 +775,7 @@ func (b *HostBuilder) buildHostTopo( node.CpuDies = cpuDies hostTopo.Nodes[i] = node } - hostTopo.CPUCmtbound = int(desc.CPUCmtbound) + hostTopo.CPUCmtbound = desc.CPUCmtbound hostTopo.NumaEnabled = numaEnabled hostTopo.HostName = desc.Name @@ -1002,7 +1009,7 @@ func (h *HostDesc) GetFreeCpuNuma() scheduler.SortedFreeNumaCpuMam { nodeFree.CpuCount = h.HostTopo.Nodes[i].CpuCount nodeFree.MemSize = h.HostTopo.Nodes[i].NumaNodeFreeMemSizeKB / 1024 nodeFree.EnableNumaAllocate = h.HostTopo.NumaEnabled - nodeFree.FreeCpuCount = h.HostTopo.Nodes[i].CpuCount*int(h.CPUCmtbound) - h.HostTopo.Nodes[i].VcpuCount + nodeFree.FreeCpuCount = int(float32(h.HostTopo.Nodes[i].CpuCount)*h.CPUCmtbound) - h.HostTopo.Nodes[i].VcpuCount for cpuId, pending := range cpuPin { if h.HostTopo.Nodes[i].LogicalProcessors.Contains(cpuId) { nodeFree.FreeCpuCount -= pending @@ -1505,6 +1512,7 @@ func (b *HostBuilder) fillGuestsResourceInfo(desc *HostDesc, host *computemodels guestsOnHost = append(guestsOnHost, backupGuestsOnHost...) } + //pendingUsage := desc.GetPendingUsage() desc.Tenants = make(map[string]int64) for _, gst := range guestsOnHost { guest := gst.(computemodels.SGuest) @@ -1595,7 +1603,7 @@ func (b *HostBuilder) fillGuestsResourceInfo(desc *HostDesc, host *computemodels if host.EnableNumaAllocate && len(guestsCpuNumaPin) > 0 { desc.HostTopo.LoadCpuNumaPin(guestsCpuNumaPin) } - //log.Infof("host %s topo %s", jsonutils.Marshal(desc.HostTopo)) + //log.Infof("host %s topo %s", desc.Name, jsonutils.Marshal(desc.HostTopo)) desc.GuestCount = guestCount desc.CreatingGuestCount = creatingGuestCount