fix(scheduler): numa memory more balance on with prefer nodes (#21580)

* fix(scheduler): numa memory more balance on with prefer nodes

* fix(host): host agent allocate numa node

pkg/compute/models/guests.go

@@ -6274,6 +6274,7 @@ func (self *SGuest) ToSchedDesc() *schedapi.ScheduleInput {
 	config.Hypervisor = self.GetHypervisor()
 	desc.ServerConfig = *config
 	desc.OsArch = self.OsArch
+	desc.ExtraCpuCount = self.ExtraCpuCount
 	return desc
 }
 

pkg/hostman/guestman/guesthelper.go

@@ -20,6 +20,8 @@ import (
 	"sort"
 	"sync"
 
+	"github.com/jaypipes/ghw/pkg/topology"
+
 	"yunion.io/x/cloudmux/pkg/multicloud/esxi/vcenter"
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -29,6 +31,7 @@ import (
 	"yunion.io/x/onecloud/pkg/apis/compute"
 	hostapi "yunion.io/x/onecloud/pkg/apis/host"
 	"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
+	"yunion.io/x/onecloud/pkg/hostman/options"
 	"yunion.io/x/onecloud/pkg/hostman/storageman"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/cgrouputils/cpuset"
@@ -215,16 +218,29 @@ type SQgaGuestSetNetwork struct {
 type CpuSetCounter struct {
 	Nodes       []*NumaNode
 	NumaEnabled bool
+	CPUCmtbound float32
 	Lock        sync.Mutex
 }
 
-func NewGuestCpuSetCounter(info *hostapi.HostTopology, reservedCpus *cpuset.CPUSet, numaAllocate bool, hugepageSizeKB, cpuCmtbound int) (*CpuSetCounter, error) {
+func NewGuestCpuSetCounter(
+	info *hostapi.HostTopology, reservedCpus *cpuset.CPUSet, numaAllocate, isContainerHost bool,
+	hugepageSizeKB int, cpuCmtbound, memCmtBound float32, reservedMemMb int,
+) (*CpuSetCounter, error) {
 	cpuSetCounter := new(CpuSetCounter)
 	cpuSetCounter.Nodes = make([]*NumaNode, len(info.Nodes))
 	cpuSetCounter.NumaEnabled = numaAllocate
+	cpuSetCounter.CPUCmtbound = cpuCmtbound
 	hasL3Cache := false
+	nodeReserveMem := reservedMemMb / len(info.Nodes) * 1024
 	for i := 0; i < len(info.Nodes); i++ {
-		node, err := NewNumaNode(info.Nodes[i].ID, cpuSetCounter.NumaEnabled, hugepageSizeKB)
+		node, err := NewNumaNode(
+			info.Nodes[i],
+			cpuSetCounter.NumaEnabled,
+			isContainerHost,
+			hugepageSizeKB,
+			memCmtBound,
+			nodeReserveMem,
+		)
 		if err != nil {
 			return nil, err
 		}
@@ -280,7 +296,7 @@ func NewGuestCpuSetCounter(info *hostapi.HostTopology, reservedCpus *cpuset.CPUS
 
 func (pq *CpuSetCounter) AllocCpusetWithNodeCount(vcpuCount int, memSizeKB int64, nodeCount int) (map[int]SAllocNumaCpus, error) {
 	if !pq.NumaEnabled {
-		return pq.AllocCpuset(vcpuCount, memSizeKB, -1)
+		return pq.AllocCpuset(vcpuCount, memSizeKB, nil)
 	}
 	if len(pq.Nodes) < nodeCount {
 		return nil, nil
@@ -290,7 +306,7 @@ func (pq *CpuSetCounter) AllocCpusetWithNodeCount(vcpuCount int, memSizeKB int64
 	defer pq.Lock.Unlock()
 	var res = map[int]SAllocNumaCpus{}
 	var nodeAllocSize = memSizeKB / int64(nodeCount)
-	if nodeAllocSize/1024%1024 == 0 && pq.nodesFreeMemSizeEnough(nodeCount, memSizeKB) {
+	if pq.nodesEnough(nodeCount, vcpuCount, int(memSizeKB)) {
 		var pcpuCount = vcpuCount / nodeCount
 		var remPcpuCount = vcpuCount % nodeCount
 
@@ -301,11 +317,11 @@ func (pq *CpuSetCounter) AllocCpusetWithNodeCount(vcpuCount int, memSizeKB int64
 				remPcpuCount -= 1
 			}
 			res[pq.Nodes[i].NodeId] = SAllocNumaCpus{
-				Cpuset:    pq.Nodes[i].AllocCpuset1(npcpuCount),
+				Cpuset:    pq.Nodes[i].AllocCpuset(npcpuCount),
 				MemSizeKB: nodeAllocSize,
 				Unregular: false,
 			}
-			pq.Nodes[i].NumaHugeFreeMemSizeKB -= nodeAllocSize
+			pq.Nodes[i].NumaNodeFreeMemSizeKB -= nodeAllocSize
 			pq.Nodes[i].VcpuCount += npcpuCount
 		}
 	}
@@ -323,9 +339,7 @@ func (pq *CpuSetCounter) IsNumaEnabled() bool {
 	return pq.NumaEnabled
 }
 
-func (pq *CpuSetCounter) AllocCpuset(vcpuCount int, memSizeKB int64, perferNumaNode int8) (map[int]SAllocNumaCpus, error) {
-	res := map[int]SAllocNumaCpus{}
-	sourceVcpuCount := vcpuCount
+func (pq *CpuSetCounter) AllocCpuset(vcpuCount int, memSizeKB int64, preferNumaNodes []int8) (map[int]SAllocNumaCpus, error) {
 	pq.Lock.Lock()
 	defer pq.Lock.Unlock()
 
@@ -333,32 +347,29 @@ func (pq *CpuSetCounter) AllocCpuset(vcpuCount int, memSizeKB int64, perferNumaN
 		return nil, nil
 	}
 
-	if pq.NumaEnabled {
-		err := pq.AllocNumaNodes(vcpuCount, memSizeKB, perferNumaNode, res)
-		return res, err
-	} else {
-		if perferNumaNode > 0 {
-			for i := range pq.Nodes {
-				if pq.Nodes[i].NodeId != int(perferNumaNode) {
-					continue
+	if pq.NumaEnabled && len(preferNumaNodes) > 0 {
+		sortedNumaDistance := pq.getDistancesSeqByPreferNodes(preferNumaNodes)
+		for nodeCount := 1; nodeCount <= len(pq.Nodes); nodeCount *= 2 {
+			ret := pq.allocCpuNumaNodesByPreferNodes(vcpuCount, int(memSizeKB), nodeCount, sortedNumaDistance)
+			if ret != nil {
+				for i := range pq.Nodes {
+					if cpupin, ok := ret[pq.Nodes[i].NodeId]; ok {
+						pq.Nodes[i].VcpuCount += vcpuCount
+						pq.Nodes[i].NumaNodeFreeMemSizeKB -= cpupin.MemSizeKB
+					}
 				}
-				if pq.Nodes[i].VcpuCount >= (pq.Nodes[0].VcpuCount + pq.Nodes[0].CpuCount) {
-					break
-				}
-
-				allocCount := vcpuCount
-				if vcpuCount > pq.Nodes[0].CpuCount {
-					allocCount = vcpuCount/2 + vcpuCount%2
-				}
-				res[pq.Nodes[i].NodeId] = SAllocNumaCpus{
-					Cpuset: pq.Nodes[i].AllocCpuset(allocCount),
-				}
-				pq.Nodes[i].VcpuCount += sourceVcpuCount
-				vcpuCount -= allocCount
 				sort.Sort(pq)
-				break
+				return ret, nil
 			}
 		}
+	}
+
+	res := map[int]SAllocNumaCpus{}
+	sourceVcpuCount := vcpuCount
+	if pq.NumaEnabled {
+		err := pq.AllocNumaNodes(vcpuCount, memSizeKB, res)
+		return res, err
+	} else {
 		for vcpuCount > 0 {
 			count := vcpuCount
 			if vcpuCount > pq.Nodes[0].CpuCount {
@@ -375,37 +386,85 @@ func (pq *CpuSetCounter) AllocCpuset(vcpuCount int, memSizeKB int64, perferNumaN
 	}
 }
 
-func (pq *CpuSetCounter) AllocNumaNodes(vcpuCount int, memSizeKB int64, perferNumaNode int8, res map[int]SAllocNumaCpus) error {
-	var allocated = false
+func (pq *CpuSetCounter) allocCpuNumaNodesByPreferNodes(
+	vcpuCount, memSizeKB, nodeCount int, sortedNumaDistance []SSortedNumaDistance,
+) map[int]SAllocNumaCpus {
+	res := map[int]SAllocNumaCpus{}
+	var nodeAllocSize = memSizeKB / nodeCount
+	var pcpuCount = vcpuCount / nodeCount
+	var remPcpuCount = vcpuCount % nodeCount
 
-	// check preferred numa node is memory enough
-	if perferNumaNode >= 0 {
-		for i := 0; i < len(pq.Nodes); i++ {
-			if pq.Nodes[i].NodeId != int(perferNumaNode) {
-				continue
-			}
-			if pq.Nodes[i].NumaHugeFreeMemSizeKB >= memSizeKB {
-				res[pq.Nodes[i].NodeId] = SAllocNumaCpus{
-					Cpuset:    pq.Nodes[i].AllocCpuset1(vcpuCount),
-					MemSizeKB: memSizeKB,
-					Unregular: false,
-				}
-				pq.Nodes[i].NumaHugeFreeMemSizeKB -= memSizeKB
-				pq.Nodes[i].VcpuCount += vcpuCount
-
-				allocated = true
-			}
+	allocatedNode := 0
+	for i := range sortedNumaDistance {
+		if allocatedNode >= nodeCount {
 			break
 		}
+
+		var npcpuCount = pcpuCount
+		if remPcpuCount > 0 {
+			npcpuCount += 1
+			remPcpuCount -= 1
+		}
+		nodeIdx := sortedNumaDistance[i].NodeIndex
+		if pq.Nodes[nodeIdx].nodeEnough(vcpuCount, memSizeKB, pq.CPUCmtbound, pq.NumaEnabled) {
+			cpuNumaPin := SAllocNumaCpus{
+				Cpuset: pq.Nodes[nodeIdx].AllocCpuset(npcpuCount),
+			}
+			cpuNumaPin.MemSizeKB = int64(nodeAllocSize)
+			res[pq.Nodes[nodeIdx].NodeId] = cpuNumaPin
+			allocatedNode += 1
+		} else {
+			log.Infof("node %v not enough", pq.Nodes[i])
+		}
+		log.Infof("node %d, free mems %d", pq.Nodes[nodeIdx].NodeId, pq.Nodes[nodeIdx].NumaNodeFreeMemSizeKB)
 	}
 
+	if allocatedNode < nodeCount {
+		return nil
+	}
+	return res
+}
+
+type SSortedNumaDistance struct {
+	NodeIndex   int
+	Distance    int
+	FreeMemSize int
+}
+
+func (pq *CpuSetCounter) getDistancesSeqByPreferNodes(preferNumaNodes []int8) []SSortedNumaDistance {
+	sortedNumaDistance := make([]SSortedNumaDistance, len(pq.Nodes))
+	for i := range pq.Nodes {
+		distance := 0
+		for j := range preferNumaNodes {
+			distance += pq.Nodes[i].Distances[preferNumaNodes[j]]
+		}
+		sortedNumaDistance[i] = SSortedNumaDistance{
+			NodeIndex:   i,
+			Distance:    distance,
+			FreeMemSize: int(pq.Nodes[i].NumaNodeFreeMemSizeKB),
+		}
+	}
+	sort.Slice(sortedNumaDistance, func(i, j int) bool {
+		// 7 is tolerant max distances
+		if (sortedNumaDistance[i].Distance + 7) < sortedNumaDistance[j].Distance {
+			return true
+		} else {
+			return sortedNumaDistance[i].FreeMemSize > sortedNumaDistance[j].FreeMemSize
+		}
+	})
+	return sortedNumaDistance
+}
+
+func (pq *CpuSetCounter) AllocNumaNodes(vcpuCount int, memSizeKB int64, res map[int]SAllocNumaCpus) error {
+	var allocated = false
+
 	// alloc numa nodes in order 1, 2, 4, ...
 	if !allocated {
 		for nodeCount := 1; nodeCount <= len(pq.Nodes); nodeCount *= 2 {
 			if nodeCount > vcpuCount {
 				break
 			}
-			if ok := pq.nodesFreeMemSizeEnough(nodeCount, memSizeKB); !ok {
+			if ok := pq.nodesEnough(nodeCount, vcpuCount, int(memSizeKB)); !ok {
 				log.Infof("node count %d not enough", nodeCount)
 				continue
 			}
@@ -423,11 +482,11 @@ func (pq *CpuSetCounter) AllocNumaNodes(vcpuCount int, memSizeKB int64, perferNu
 					remPcpuCount -= 1
 				}
 				res[pq.Nodes[i].NodeId] = SAllocNumaCpus{
-					Cpuset:    pq.Nodes[i].AllocCpuset1(npcpuCount),
+					Cpuset:    pq.Nodes[i].AllocCpuset(npcpuCount),
 					MemSizeKB: nodeAllocSize,
 					Unregular: false,
 				}
-				pq.Nodes[i].NumaHugeFreeMemSizeKB -= nodeAllocSize
+				pq.Nodes[i].NumaNodeFreeMemSizeKB -= nodeAllocSize
 				pq.Nodes[i].VcpuCount += npcpuCount
 			}
 			allocated = true
@@ -445,16 +504,41 @@ func (pq *CpuSetCounter) AllocNumaNodes(vcpuCount int, memSizeKB int64, perferNu
 	return nil
 }
 
+func (pq *CpuSetCounter) nodesEnough(nodeCount, vcpuCount int, memSizeKB int) bool {
+	var leastFree = memSizeKB / nodeCount
+	var leastCpuCount = vcpuCount / nodeCount
+	var remPcpuCount = vcpuCount % nodeCount
+
+	for i := 0; i < nodeCount; i++ {
+		if pq.NumaEnabled {
+			if int(pq.Nodes[i].NumaNodeFreeMemSizeKB) < leastFree {
+				return false
+			}
+		}
+
+		requireCpuCount := leastCpuCount
+		if remPcpuCount > 0 {
+			requireCpuCount += 1
+			remPcpuCount -= 1
+		}
+		if (pq.Nodes[i].VcpuCount + requireCpuCount) > int(float32(pq.Nodes[i].CpuCount)*pq.CPUCmtbound) {
+			return false
+		}
+
+	}
+	return true
+}
+
 func (pq *CpuSetCounter) nodesFreeMemSizeEnough(nodeCount int, memSizeKB int64) bool {
 	var freeMem int64 = 0
 	var leastFree = memSizeKB / int64(nodeCount)
 	log.Debugf("request memsize %d, least free %d", memSizeKB, leastFree)
 	for i := 0; i < nodeCount; i++ {
-		log.Debugf("index %d node %d free size %d", i, pq.Nodes[i].NodeId, pq.Nodes[i].NumaHugeFreeMemSizeKB)
-		if pq.Nodes[i].NumaHugeFreeMemSizeKB < leastFree {
+		log.Debugf("index %d node %d free size %d", i, pq.Nodes[i].NodeId, pq.Nodes[i].NumaNodeFreeMemSizeKB)
+		if pq.Nodes[i].NumaNodeFreeMemSizeKB < leastFree {
 			return false
 		}
-		freeMem += pq.Nodes[i].NumaHugeFreeMemSizeKB
+		freeMem += pq.Nodes[i].NumaNodeFreeMemSizeKB
 	}
 	return freeMem >= memSizeKB
 }
@@ -466,12 +550,12 @@ func (pq *CpuSetCounter) setNumaNodes(numaMaps map[int]int, vcpuCount int64) map
 			allocMem := int64(size) * 1024
 			//npcpuCount := int(vcpuCount*allocMem/memSizeKB + (vcpuCount*allocMem)%memSizeKB)
 			res[pq.Nodes[i].NodeId] = SAllocNumaCpus{
-				Cpuset:    pq.Nodes[i].AllocCpuset1(int(vcpuCount)),
+				Cpuset:    pq.Nodes[i].AllocCpuset(int(vcpuCount)),
 				MemSizeKB: allocMem,
 				Unregular: true,
 			}
 
-			pq.Nodes[i].NumaHugeFreeMemSizeKB -= allocMem
+			pq.Nodes[i].NumaNodeFreeMemSizeKB -= allocMem
 			pq.Nodes[i].VcpuCount += int(vcpuCount)
 		}
 	}
@@ -509,7 +593,7 @@ func (pq *CpuSetCounter) ReleaseNumaCpus(memSizeMb int64, hostNode int, cpus []i
 		}
 		pq.Nodes[i].CpuDies.ReleaseCpus(cpus, vcpuCount)
 		pq.Nodes[i].VcpuCount -= vcpuCount
-		pq.Nodes[i].NumaHugeFreeMemSizeKB += memSizeMb * 1024
+		pq.Nodes[i].NumaNodeFreeMemSizeKB += memSizeMb * 1024
 	}
 	sort.Sort(pq)
 }
@@ -521,7 +605,7 @@ func (pq *CpuSetCounter) LoadNumaCpus(memSizeMb int64, hostNode int, cpus []int,
 		}
 		pq.Nodes[i].CpuDies.LoadCpus(cpus, vcpuCount)
 		pq.Nodes[i].VcpuCount += vcpuCount
-		pq.Nodes[i].NumaHugeFreeMemSizeKB -= memSizeMb * 1024
+		pq.Nodes[i].NumaNodeFreeMemSizeKB -= memSizeMb * 1024
 	}
 	sort.Sort(pq)
 }
@@ -552,13 +636,15 @@ func (pq *CpuSetCounter) LoadCpus(cpus []int, vcpuCpunt int) {
 func (pq CpuSetCounter) Len() int { return len(pq.Nodes) }
 
 func (pq CpuSetCounter) Less(i, j int) bool {
+	freeCpuI := int(float32(pq.Nodes[i].CpuCount)*pq.CPUCmtbound) - pq.Nodes[i].VcpuCount
+	freeCpuJ := int(float32(pq.Nodes[i].CpuCount)*pq.CPUCmtbound) - pq.Nodes[j].VcpuCount
 	if pq.NumaEnabled {
-		if pq.Nodes[i].NumaHugeFreeMemSizeKB == pq.Nodes[j].NumaHugeFreeMemSizeKB {
-			return pq.Nodes[i].VcpuCount < pq.Nodes[j].VcpuCount
+		if pq.Nodes[i].NumaNodeFreeMemSizeKB == pq.Nodes[j].NumaNodeFreeMemSizeKB {
+			return freeCpuI > freeCpuJ
 		}
-		return pq.Nodes[i].NumaHugeFreeMemSizeKB > pq.Nodes[j].NumaHugeFreeMemSizeKB
+		return pq.Nodes[i].NumaNodeFreeMemSizeKB > pq.Nodes[j].NumaNodeFreeMemSizeKB
 	} else {
-		return pq.Nodes[i].VcpuCount < pq.Nodes[j].VcpuCount
+		return freeCpuI > freeCpuJ
 	}
 }
 
@@ -586,39 +672,67 @@ type NumaNode struct {
 	CpuCount          int
 
 	NodeId                int
-	NumaHugeMemSizeKB     int64
-	NumaHugeFreeMemSizeKB int64
+	Distances             []int
+	NumaNodeMemSizeKB     int64
+	NumaNodeFreeMemSizeKB int64
 }
 
-func NewNumaNode(nodeId int, numaAllocate bool, hugepageSizeKB int) (*NumaNode, error) {
+func NewNumaNode(
+	nodeInfo *topology.Node,
+	numaAllocate, isContainerHost bool,
+	hugepageSizeKB int, memCmtBound float32,
+	reservedMemSizeKB int,
+) (*NumaNode, error) {
 	n := new(NumaNode)
 	n.LogicalProcessors = cpuset.NewCPUSet()
-	n.NodeId = nodeId
+	n.NodeId = nodeInfo.ID
+	n.Distances = nodeInfo.Distances
 
-	if numaAllocate {
-		nodeHugepagePath := fmt.Sprintf("/sys/devices/system/node/node%d/hugepages/hugepages-%dkB", nodeId, hugepageSizeKB)
+	if !numaAllocate {
+		return n, nil
+	}
+	if isContainerHost {
+		if nodeInfo.Memory == nil {
+			return nil, errors.Errorf("node %d no memory info: %#v", nodeInfo.ID, nodeInfo)
+		}
+		n.NumaNodeMemSizeKB = int64(float32(nodeInfo.Memory.TotalUsableBytes/1024-int64(reservedMemSizeKB)) * memCmtBound)
+	} else {
+		nodeHugepagePath := fmt.Sprintf("/sys/devices/system/node/node%d/hugepages/hugepages-%dkB", n.NodeId, hugepageSizeKB)
 		if !fileutils2.Exists(nodeHugepagePath) {
 			return n, nil
 		}
-
 		nrHugepage, err := fileutils2.FileGetIntContent(path.Join(nodeHugepagePath, "nr_hugepages"))
 		if err != nil {
-			log.Errorf("failed get node %d nr hugepage %s", nodeId, err)
+			log.Errorf("failed get node %d nr hugepage %s", n.NodeId, err)
 			return nil, errors.Wrap(err, "get numa node nr hugepage")
 		}
-		n.NumaHugeMemSizeKB = int64(nrHugepage) * int64(hugepageSizeKB)
-
-		//freeHugepage, err := fileutils2.FileGetIntContent(path.Join(nodeHugepagePath, "free_hugepages"))
-		//if err != nil {
-		//	log.Errorf("failed get node %d free hugepage %s", nodeId, err)
-		//	return nil, errors.Wrap(err, "get numa node free hugepage")
-		//}
-		n.NumaHugeFreeMemSizeKB = n.NumaHugeMemSizeKB
+		n.NumaNodeMemSizeKB = int64(nrHugepage) * int64(hugepageSizeKB)
 	}
+	n.NumaNodeFreeMemSizeKB = n.NumaNodeMemSizeKB
+
 	return n, nil
 }
 
-func (n *NumaNode) AllocCpuset1(vcpuCount int) []int {
+func (n *NumaNode) nodeEnough(vcpuCount, memSizeKB int, cmtBound float32, enableNumaAlloc bool) bool {
+	if int(float32(n.CpuCount)*cmtBound)-n.VcpuCount < vcpuCount {
+		return false
+	}
+	if enableNumaAlloc {
+		if int(n.NumaNodeFreeMemSizeKB) < memSizeKB {
+			return false
+		}
+	}
+	return true
+}
+
+func (n *NumaNode) AllocCpuset(vcpuCount int) []int {
+	if options.HostOptions.EnableStrictCpuBind {
+		return n.allocCpusetStrict(vcpuCount)
+	}
+	return n.allocCpusetOnNode(vcpuCount)
+}
+
+func (n *NumaNode) allocCpusetStrict(vcpuCount int) []int {
 	var allocCount = vcpuCount
 	var dieCnt = 0
 
@@ -654,14 +768,15 @@ func (n *NumaNode) AllocCpuset1(vcpuCount int) []int {
 		}
 	}
 
-	var pcpus = make([]int, 0)
+	defer sort.Sort(n.CpuDies)
+	var ret = make([]int, 0)
 	for i := 0; i < len(allocCpuCountMap); i++ {
 		var allocCpuCount = allocCpuCountMap[i]
 		for allocCpuCount > 0 {
 			pcpus := n.CpuDies[i].LogicalProcessors.ToSliceNoSort()
 			for j := 0; j < len(pcpus); j++ {
 				if n.CpuDies[i].CpuFree[pcpus[j]] > 0 {
-					pcpus = append(pcpus, n.CpuDies[i].CpuFree[pcpus[j]])
+					ret = append(ret, n.CpuDies[i].CpuFree[pcpus[j]])
 					n.CpuDies[i].CpuFree[pcpus[j]] -= 1
 				}
 				allocCpuCount -= 1
@@ -671,10 +786,10 @@ func (n *NumaNode) AllocCpuset1(vcpuCount int) []int {
 			}
 		}
 	}
-	return pcpus
+	return ret
 }
 
-func (n *NumaNode) AllocCpuset(vcpuCount int) []int {
+func (n *NumaNode) allocCpusetOnNode(vcpuCount int) []int {
 	cpus := make([]int, 0)
 
 	var allocCount = vcpuCount
@@ -698,10 +813,10 @@ type CPUDie struct {
 	VcpuCount         int
 }
 
-func (d *CPUDie) initCpuFree(cpuCmtbound int) {
+func (d *CPUDie) initCpuFree(cpuCmtbound float32) {
 	cpuFree := map[int]int{}
 	for _, cpuId := range d.LogicalProcessors.ToSliceNoSort() {
-		cpuFree[cpuId] = cpuCmtbound
+		cpuFree[cpuId] = int(cpuCmtbound)
 	}
 	d.CpuFree = cpuFree
 }

pkg/hostman/guestman/guestman.go

@@ -295,11 +295,14 @@ func (m *SGuestManager) Bootstrap() (chan struct{}, error) {
 	hostTypo := m.host.GetHostTopology()
 
 	if options.HostOptions.EnableHostAgentNumaAllocate {
-		m.numaAllocate = m.host.IsNumaAllocateEnabled() && m.host.IsHugepagesEnabled() && (len(hostTypo.Nodes) > 1)
+		enableMemAlloc := m.host.IsContainerHost() || m.host.IsHugepagesEnabled()
+		m.numaAllocate = !m.host.IsNumaAllocateEnabled() && enableMemAlloc && (len(hostTypo.Nodes) > 1)
 	}
 
 	cpuSet, err := NewGuestCpuSetCounter(
-		hostTypo, m.host.GetReservedCpusInfo(), m.numaAllocate, m.host.HugepageSizeKb(), m.host.CpuCmtBound())
+		hostTypo, m.host.GetReservedCpusInfo(), m.numaAllocate, m.host.IsContainerHost(),
+		m.host.HugepageSizeKb(), m.host.CpuCmtBound(), m.host.MemCmtBound(), m.host.GetReservedMemMb(),
+	)
 	if err != nil {
 		return nil, err
 	}

pkg/hostman/guestman/guesttasks.go

@@ -2488,7 +2488,7 @@ func (task *SGuestHotplugCpuMemTask) startAddCpusWithFreeVcpuSet(vcpuSet []int)
 				}
 			}
 		} else {
-			cpus, _ := task.manager.cpuSet.AllocCpuset(1, 0, -1)
+			cpus, _ := task.manager.cpuSet.AllocCpuset(1, 0, nil)
 			for _, cpus := range cpus {
 				//pcpus := cpuset.NewCPUSet(cpus.Cpuset...).String()
 				//vcpus := fmt.Sprintf("%d-%d", vcpuId, vcpuId)

pkg/hostman/guestman/pod.go

@@ -760,11 +760,24 @@ func (s *sPodGuestInstance) namespacesFroPod(input *computeapi.PodCreateInput) *
 	}
 }
 
+func (s *sPodGuestInstance) updateGuestDesc() error {
+	s.Desc = new(desc.SGuestDesc)
+	err := jsonutils.Marshal(s.SourceDesc).Unmarshal(s.Desc)
+	if err != nil {
+		return errors.Wrap(err, "unmarshal source desc")
+	}
+
+	return s.allocateCpuNumaPin()
+}
+
 func (s *sPodGuestInstance) _startPod(ctx context.Context, userCred mcclient.TokenCredential) (*computeapi.PodStartResponse, error) {
 	podInput, err := s.getPodCreateParams()
 	if err != nil {
 		return nil, errors.Wrap(err, "getPodCreateParams")
 	}
+	if err := s.updateGuestDesc(); err != nil {
+		return nil, errors.Wrap(err, "updateGuestDesc")
+	}
 	if err := s.mountPodVolumes(); err != nil {
 		return nil, errors.Wrap(err, "mountPodVolumes")
 	}
@@ -919,7 +932,6 @@ func (s *sPodGuestInstance) ensurePodRemoved(ctx context.Context, timeout int64)
 }
 
 func (s *sPodGuestInstance) stopPod(ctx context.Context, timeout int64) error {
-	ReleaseGuestCpuset(s.manager, s)
 	if err := s.umountPodVolumes(); err != nil {
 		return errors.Wrapf(err, "umount pod volumes")
 	}
@@ -927,7 +939,11 @@ func (s *sPodGuestInstance) stopPod(ctx context.Context, timeout int64) error {
 		timeout = 15
 	}
 
-	return s.ensurePodRemoved(ctx, timeout)
+	if err := s.ensurePodRemoved(ctx, timeout); err != nil {
+		return err
+	}
+	ReleaseGuestCpuset(s.manager, s)
+	return nil
 }
 
 func (s *sPodGuestInstance) LoadDesc() error {
@@ -1086,15 +1102,15 @@ func (s *sPodGuestInstance) allocateCpuNumaPin() error {
 	}
 
 	var cpus = make([]int, 0)
-	var perferNumaNode int8 = -1
+	var preferNumaNodes = make([]int8, 0)
 	for i := range s.Desc.IsolatedDevices {
 		if s.Desc.IsolatedDevices[i].NumaNode >= 0 {
-			perferNumaNode = s.Desc.IsolatedDevices[i].NumaNode
+			preferNumaNodes = append(preferNumaNodes, s.Desc.IsolatedDevices[i].NumaNode)
 			break
 		}
 	}
 
-	nodeNumaCpus, err := s.manager.cpuSet.AllocCpuset(int(s.Desc.Cpu), s.Desc.Mem*1024, perferNumaNode)
+	nodeNumaCpus, err := s.manager.cpuSet.AllocCpuset(int(s.Desc.Cpu), s.Desc.Mem*1024, preferNumaNodes)
 	if err != nil {
 		return err
 	}
@@ -1109,6 +1125,26 @@ func (s *sPodGuestInstance) allocateCpuNumaPin() error {
 				Pcpus: cpuset.NewCPUSet(cpus...).String(),
 			},
 		}
+	} else {
+		var cpuNumaPin = make([]*desc.SCpuNumaPin, 0)
+		for nodeId, numaCpus := range nodeNumaCpus {
+			if s.manager.numaAllocate {
+				unodeId := uint16(nodeId)
+				vcpuPin := make([]desc.SVCpuPin, len(numaCpus.Cpuset))
+				for i := range numaCpus.Cpuset {
+					vcpuPin[i].Pcpu = numaCpus.Cpuset[i]
+				}
+
+				memPin := &desc.SCpuNumaPin{
+					SizeMB:    numaCpus.MemSizeKB / 1024, // MB
+					NodeId:    &unodeId,
+					VcpuPin:   vcpuPin,
+					Unregular: numaCpus.Unregular,
+				}
+				cpuNumaPin = append(cpuNumaPin, memPin)
+			}
+		}
+		s.Desc.CpuNumaPin = cpuNumaPin
 	}
 
 	return SaveLiveDesc(s, s.Desc)
@@ -1458,10 +1494,6 @@ func (s *sPodGuestInstance) createContainer(ctx context.Context, userCred mcclie
 		})
 	}
 
-	if err := s.allocateCpuNumaPin(); err != nil {
-		return "", errors.Wrap(err, "allocateCpuNumaPin")
-	}
-
 	var cpuSetCpus string
 	var cpuSetMems string
 	var extraCpuCount int

pkg/hostman/guestman/qemu-kvm.go

@@ -2692,15 +2692,15 @@ func (s *SKVMGuestInstance) setCgroupCPUSet() error {
 func (s *SKVMGuestInstance) allocGuestNumaCpuset() error {
 	var cpus = make([]int, 0)
 	var cpuNumaPin = make([]*desc.SCpuNumaPin, 0)
-	var perferNumaNode int8 = -1
+	var preferNumaNodes = make([]int8, 0)
 	for i := range s.Desc.IsolatedDevices {
 		if s.Desc.IsolatedDevices[i].NumaNode >= 0 {
-			perferNumaNode = s.Desc.IsolatedDevices[i].NumaNode
+			preferNumaNodes = append(preferNumaNodes, s.Desc.IsolatedDevices[i].NumaNode)
 			break
 		}
 	}
 
-	nodeNumaCpus, err := s.manager.cpuSet.AllocCpuset(int(s.Desc.Cpu), s.Desc.Mem*1024, perferNumaNode)
+	nodeNumaCpus, err := s.manager.cpuSet.AllocCpuset(int(s.Desc.Cpu), s.Desc.Mem*1024, preferNumaNodes)
 	if err != nil {
 		return err
 	}

pkg/hostman/hostinfo/hostinfo.go

@@ -99,7 +99,8 @@ type SHostInfo struct {
 	onHostDown         string
 	reservedCpusInfo   *api.HostReserveCpusInput
 	enableNumaAllocate bool
-	cpuCmtBound        int
+	cpuCmtBound        float32
+	memCmtBound        float32
 
 	IsolatedDeviceMan isolated_device.IsolatedDeviceManager
 
@@ -1257,7 +1258,8 @@ func (h *SHostInfo) initHostRecord() (*api.HostDetails, error) {
 	}
 
 	h.HostId = hostInfo.Id
-	h.cpuCmtBound = int(hostInfo.CpuCmtbound)
+	h.cpuCmtBound = hostInfo.CpuCmtbound
+	h.memCmtBound = hostInfo.MemCommitBound
 	hostInfo, err = h.updateHostMetadata(hostInfo.Name)
 	if err != nil {
 		return nil, errors.Wrap(err, "updateHostMetadata")
@@ -1284,8 +1286,8 @@ func (h *SHostInfo) initHostRecord() (*api.HostDetails, error) {
 	}
 
 	// set host reserved memory
-	if h.IsHugepagesEnabled() && h.getReservedMemMb() != hostInfo.MemReserved {
-		if err = h.updateHostReservedMem(h.getReservedMemMb()); err != nil {
+	if h.IsHugepagesEnabled() && h.GetReservedMemMb() != hostInfo.MemReserved {
+		if err = h.updateHostReservedMem(h.GetReservedMemMb()); err != nil {
 			return nil, errors.Wrap(err, "updateHostReservedMem")
 		}
 	}
@@ -1561,7 +1563,7 @@ func (h *SHostInfo) updateOrCreateHost(hostId string) (*api.HostDetails, error)
 	input.MemSize = fmt.Sprintf("%d", h.GetMemory())
 	if len(hostId) == 0 {
 		// first time create
-		input.MemReserved = fmt.Sprintf("%d", h.getReservedMemMb())
+		input.MemReserved = fmt.Sprintf("%d", h.GetReservedMemMb())
 	}
 	if h.IsHugepagesEnabled() {
 		pageSizeKb := options.HostOptions.HugepageSizeMb * 1024
@@ -1713,7 +1715,7 @@ func (h *SHostInfo) getOSReservedMemMb() int {
 	return reserved
 }
 
-func (h *SHostInfo) getReservedMemMb() int {
+func (h *SHostInfo) GetReservedMemMb() int {
 	if h.IsHugepagesEnabled() {
 		hp, _ := h.Mem.GetHugepages()
 		return h.GetMemory() - int(hp.BytesMb())
@@ -2604,10 +2606,14 @@ func (h *SHostInfo) GetContainerRuntimeEndpoint() string {
 	return options.HostOptions.ContainerRuntimeEndpoint
 }
 
-func (h *SHostInfo) CpuCmtBound() int {
+func (h *SHostInfo) CpuCmtBound() float32 {
 	return h.cpuCmtBound
 }
 
+func (h *SHostInfo) MemCmtBound() float32 {
+	return h.memCmtBound
+}
+
 func NewHostInfo() (*SHostInfo, error) {
 	var res = new(SHostInfo)
 	res.sysinfo = &SSysInfo{}

pkg/hostman/hostutils/hostutils.go

@@ -71,11 +71,13 @@ type IHost interface {
 	IsX8664() bool
 	GetHostTopology() *hostapi.HostTopology
 	GetReservedCpusInfo() *cpuset.CPUSet
+	GetReservedMemMb() int
 
 	IsHugepagesEnabled() bool
 	HugepageSizeKb() int
 	IsNumaAllocateEnabled() bool
-	CpuCmtBound() int
+	CpuCmtBound() float32
+	MemCmtBound() float32
 
 	IsKvmSupport() bool
 	IsNestedVirtualization() bool

pkg/hostman/options/options.go

@@ -140,6 +140,7 @@ type SHostOptions struct {
 	SetVncPassword         bool `default:"true" help:"Auto set vnc password after monitor connected"`
 	UseBootVga             bool `default:"false" help:"Use boot VGA GPU for guest"`
 
+	EnableStrictCpuBind         bool   `default:"false" help:"Enable strict cpu bind, one vcpu bind one pcpu"`
 	EnableHostAgentNumaAllocate bool   `default:"false" help:"Enable host agent numa allocate"`
 	EnableCpuBinding            bool   `default:"true" help:"Enable cpu binding and rebalance"`
 	EnableOpenflowController    bool   `default:"false"`

pkg/scheduler/cache/candidate/hosts.go

@@ -273,8 +273,8 @@ type NumaNode struct {
 	NumaNodeFreeMemSizeKB int
 }
 
-func (n *NumaNode) nodeEnough(vcpuCount, memSizeKB, cmtBound int, enableNumaAlloc bool) bool {
-	if n.CpuCount*cmtBound-n.VcpuCount < vcpuCount {
+func (n *NumaNode) nodeEnough(vcpuCount, memSizeKB int, cmtBound float32, enableNumaAlloc bool) bool {
+	if int(float32(n.CpuCount)*cmtBound)-n.VcpuCount < vcpuCount {
 		return false
 	}
 	if enableNumaAlloc {
@@ -409,7 +409,7 @@ func (n *NumaNode) AllocCpuset(vcpuCount int) []int {
 	return ret
 }
 
-func NewNumaNode(nodeId int, nodeDistances []int, hugepageSizeKb int, nodeHugepages []hostapi.HostNodeHugepageNr, memSizeKB, memCmtBound int) *NumaNode {
+func NewNumaNode(nodeId int, nodeDistances []int, hugepageSizeKb int, nodeHugepages []hostapi.HostNodeHugepageNr, memSizeKB int, memCmtBound float32) *NumaNode {
 	n := new(NumaNode)
 	n.LogicalProcessors = cpuset.NewCPUSet()
 	n.NodeId = nodeId
@@ -422,7 +422,7 @@ func NewNumaNode(nodeId int, nodeDistances []int, hugepageSizeKb int, nodeHugepa
 			}
 		}
 	} else {
-		n.NumaNodeMemSizeKB = memSizeKB * memCmtBound
+		n.NumaNodeMemSizeKB = int(float32(memSizeKB) * memCmtBound)
 	}
 
 	n.NumaNodeFreeMemSizeKB = n.NumaNodeMemSizeKB
@@ -432,7 +432,7 @@ func NewNumaNode(nodeId int, nodeDistances []int, hugepageSizeKb int, nodeHugepa
 type SHostTopo struct {
 	Nodes       []*NumaNode
 	NumaEnabled bool
-	CPUCmtbound int
+	CPUCmtbound float32
 	HostName    string
 }
 
@@ -540,7 +540,7 @@ func (h *SHostTopo) nodesEnough(nodeCount, vcpuCount int, memSizeKB int) bool {
 			requireCpuCount += 1
 			remPcpuCount -= 1
 		}
-		if (h.Nodes[i].VcpuCount + requireCpuCount) > h.Nodes[i].CpuCount*h.CPUCmtbound {
+		if (h.Nodes[i].VcpuCount + requireCpuCount) > int(float32(h.Nodes[i].CpuCount)*h.CPUCmtbound) {
 			return false
 		}
 
@@ -549,7 +549,7 @@ func (h *SHostTopo) nodesEnough(nodeCount, vcpuCount int, memSizeKB int) bool {
 }
 
 func (h *SHostTopo) allocCpuNumaNodesByPreferNodes(
-	vcpuCount, memSizeKB, nodeCount int, preferNumaNodes []int, sortedNumaDistance []SSortedNumaDistance,
+	vcpuCount, memSizeKB, nodeCount int, sortedNumaDistance []SSortedNumaDistance,
 ) []scheduler.SCpuNumaPin {
 	res := make([]scheduler.SCpuNumaPin, 0)
 	var nodeAllocSize = memSizeKB / nodeCount
@@ -591,8 +591,9 @@ func (h *SHostTopo) allocCpuNumaNodesByPreferNodes(
 }
 
 type SSortedNumaDistance struct {
-	NodeIndex int
-	Distance  int
+	NodeIndex   int
+	Distance    int
+	FreeMemSize int
 }
 
 func (h *SHostTopo) getDistancesSeqByPreferNodes(preferNumaNodes []int) []SSortedNumaDistance {
@@ -604,12 +605,18 @@ func (h *SHostTopo) getDistancesSeqByPreferNodes(preferNumaNodes []int) []SSorte
 			distance += h.Nodes[i].Distances[preferNumaNodes[j]]
 		}
 		sortedNumaDistance[i] = SSortedNumaDistance{
-			NodeIndex: i,
-			Distance:  distance,
+			NodeIndex:   i,
+			Distance:    distance,
+			FreeMemSize: h.Nodes[i].NumaNodeFreeMemSizeKB,
 		}
 	}
 	sort.Slice(sortedNumaDistance, func(i, j int) bool {
-		return sortedNumaDistance[i].Distance < sortedNumaDistance[j].Distance
+		// 7 is tolerant max distances
+		if (sortedNumaDistance[i].Distance + 7) < sortedNumaDistance[j].Distance {
+			return true
+		} else {
+			return sortedNumaDistance[i].FreeMemSize > sortedNumaDistance[j].FreeMemSize
+		}
 	})
 	return sortedNumaDistance
 }
@@ -619,7 +626,7 @@ func (h *SHostTopo) AllocCpuNumaNodes(vcpuCount, memSizeKB int, ignoreMemSingula
 		log.Infof("preferNumaNodes %v", preferNumaNodes)
 		sortedNumaDistance := h.getDistancesSeqByPreferNodes(preferNumaNodes)
 		for nodeCount := 1; nodeCount <= len(h.Nodes); nodeCount *= 2 {
-			ret := h.allocCpuNumaNodesByPreferNodes(vcpuCount, memSizeKB, nodeCount, preferNumaNodes, sortedNumaDistance)
+			ret := h.allocCpuNumaNodesByPreferNodes(vcpuCount, memSizeKB, nodeCount, sortedNumaDistance)
 			if ret != nil {
 				return ret
 			}
@@ -711,7 +718,7 @@ func (b *HostBuilder) buildHostTopo(
 			}
 		}
 
-		node := NewNumaNode(info.Nodes[i].ID, info.Nodes[i].Distances, hugepageSizeKb, nodeHugepages, nodoMemSizeKB, int(desc.MemCmtbound))
+		node := NewNumaNode(info.Nodes[i].ID, info.Nodes[i].Distances, hugepageSizeKb, nodeHugepages, nodoMemSizeKB, desc.MemCmtbound)
 
 		cpuDies := make([]*CPUDie, 0)
 		for j := 0; j < len(info.Nodes[i].Caches); j++ {
@@ -768,7 +775,7 @@ func (b *HostBuilder) buildHostTopo(
 		node.CpuDies = cpuDies
 		hostTopo.Nodes[i] = node
 	}
-	hostTopo.CPUCmtbound = int(desc.CPUCmtbound)
+	hostTopo.CPUCmtbound = desc.CPUCmtbound
 	hostTopo.NumaEnabled = numaEnabled
 	hostTopo.HostName = desc.Name
 
@@ -1002,7 +1009,7 @@ func (h *HostDesc) GetFreeCpuNuma() scheduler.SortedFreeNumaCpuMam {
 		nodeFree.CpuCount = h.HostTopo.Nodes[i].CpuCount
 		nodeFree.MemSize = h.HostTopo.Nodes[i].NumaNodeFreeMemSizeKB / 1024
 		nodeFree.EnableNumaAllocate = h.HostTopo.NumaEnabled
-		nodeFree.FreeCpuCount = h.HostTopo.Nodes[i].CpuCount*int(h.CPUCmtbound) - h.HostTopo.Nodes[i].VcpuCount
+		nodeFree.FreeCpuCount = int(float32(h.HostTopo.Nodes[i].CpuCount)*h.CPUCmtbound) - h.HostTopo.Nodes[i].VcpuCount
 		for cpuId, pending := range cpuPin {
 			if h.HostTopo.Nodes[i].LogicalProcessors.Contains(cpuId) {
 				nodeFree.FreeCpuCount -= pending
@@ -1505,6 +1512,7 @@ func (b *HostBuilder) fillGuestsResourceInfo(desc *HostDesc, host *computemodels
 		guestsOnHost = append(guestsOnHost, backupGuestsOnHost...)
 	}
 
+	//pendingUsage := desc.GetPendingUsage()
 	desc.Tenants = make(map[string]int64)
 	for _, gst := range guestsOnHost {
 		guest := gst.(computemodels.SGuest)
@@ -1595,7 +1603,7 @@ func (b *HostBuilder) fillGuestsResourceInfo(desc *HostDesc, host *computemodels
 	if host.EnableNumaAllocate && len(guestsCpuNumaPin) > 0 {
 		desc.HostTopo.LoadCpuNumaPin(guestsCpuNumaPin)
 	}
-	//log.Infof("host %s topo %s", jsonutils.Marshal(desc.HostTopo))
+	//log.Infof("host %s topo %s", desc.Name, jsonutils.Marshal(desc.HostTopo))
 
 	desc.GuestCount = guestCount
 	desc.CreatingGuestCount = creatingGuestCount