feat(region,host): guest numa allocate (#19498)

cmd/climc/shell/compute/hosts.go

@@ -182,8 +182,9 @@ func init() {
 		CpuReserved       int64   `help:"CPU reserved"`
 		HostType          string  `help:"Change host type, CAUTION!!!!" choices:"hypervisor|kubelet|esxi|baremetal"`
 		// AccessIp          string  `help:"Change access ip, CAUTION!!!!"`
-		AccessMac string `help:"Change baremetal access MAC, CAUTION!!!!"`
-		Uuid      string `help:"Change baremetal UUID,  CAUTION!!!!"`
+		AccessMac          string `help:"Change baremetal access MAC, CAUTION!!!!"`
+		Uuid               string `help:"Change baremetal UUID,  CAUTION!!!!"`
+		EnableNumaAllocate string `help:"Host enable numa allocate" choices:"True|False"`
 
 		IpmiUsername string `help:"IPMI user"`
 		IpmiPassword string `help:"IPMI password"`
@@ -234,6 +235,13 @@ func init() {
 		if len(args.Sn) > 0 {
 			params.Add(jsonutils.NewString(args.Sn), "sn")
 		}
+		if len(args.EnableNumaAllocate) > 0 {
+			enableNumaAllocate := false
+			if args.EnableNumaAllocate == "True" {
+				enableNumaAllocate = true
+			}
+			params.Add(jsonutils.NewBool(enableNumaAllocate), "enable_numa_allocate")
+		}
 		if params.Size() == 0 {
 			return fmt.Errorf("Not data to update")
 		}

pkg/apis/compute/host.go

@@ -506,6 +506,8 @@ type HostUpdateInput struct {
 
 	// 主机启动模式, 可能值位PXE和ISO
 	BootMode string `json:"boot_mode"`
+
+	EnableNumaAllocate *bool `json:"enable_numa_allocate"`
 }
 
 type HostOfflineInput struct {

pkg/apis/compute/isolated_device.go

@@ -127,6 +127,7 @@ type IsolatedDeviceJsonDesc struct {
 	DiskIndex           int8   `json:"disk_index"`
 	NvmeSizeMB          int    `json:"nvme_size_mb"`
 	MdevId              string `json:"mdev_id"`
+	NumaNode            int8   `json:"numa_node"`
 }
 
 type IsolatedDeviceModelCreateInput struct {

pkg/apis/compute/zz_generated.model.go

@@ -1272,7 +1272,8 @@ type SHost struct {
 	// 内存超分比
 	MemCmtbound float32 `json:"mem_cmtbound"`
 	// 页大小
-	PageSizeKB int `json:"page_size_kb"`
+	PageSizeKB         int  `json:"page_size_kb"`
+	EnableNumaAllocate bool `json:"enable_numa_allocate"`
 	// 存储大小,单位Mb
 	StorageSize int64 `json:"storage_size"`
 	// 存储类型
@@ -1547,6 +1548,8 @@ type SIsolatedDevice struct {
 	ReservedStorage int `json:"reserved_storage"`
 	// PciInfo stores extra PCIE information
 	PcieInfo *IsolatedDevicePCIEInfo `json:"pcie_info"`
+	// device numa node
+	NumaNode byte `json:"numa_node"`
 }
 
 // SIsolatedDeviceModel is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SIsolatedDeviceModel.

pkg/compute/models/hosts.go

@@ -147,7 +147,8 @@ type SHost struct {
 	// 内存超分比
 	MemCmtbound float32 `nullable:"true" default:"1" list:"domain" update:"domain" create:"domain_optional"`
 	// 页大小
-	PageSizeKB int `nullable:"false" default:"4" list:"domain" update:"domain" create:"domain_optional"`
+	PageSizeKB         int  `nullable:"false" default:"4" list:"domain" update:"domain" create:"domain_optional"`
+	EnableNumaAllocate bool `nullable:"true" default:"false" list:"domain" update:"domain" create:"domain_optional"`
 
 	// 存储大小,单位Mb
 	StorageSize int64 `nullable:"true" list:"domain" update:"domain" create:"domain_optional"`
@@ -3851,6 +3852,13 @@ func (hh *SHost) ValidateUpdateData(ctx context.Context, userCred mcclient.Token
 	if len(input.Name) > 0 {
 		hh.UpdateDnsRecords(false)
 	}
+	if input.EnableNumaAllocate != nil {
+		if cnt, err := hh.GetRunningGuestCount(); err != nil {
+			return input, errors.Wrap(err, "GetRunningGuestCount")
+		} else if cnt > 0 {
+			return input, errors.Errorf("Host has running guest, can't enable/disable numa allocate")
+		}
+	}
 	return input, nil
 }
 

pkg/compute/models/isolated_devices.go

@@ -144,6 +144,8 @@ type SIsolatedDevice struct {
 
 	// PciInfo stores extra PCIE information
 	PcieInfo *api.IsolatedDevicePCIEInfo `nullable:"true" create:"optional" list:"user" get:"user" update:"domain"`
+	// device numa node
+	NumaNode int8 `nullable:"true" default:"-1" list:"domain" update:"domain" create:"domain_optional"`
 }
 
 func (manager *SIsolatedDeviceManager) ExtraSearchConditions(ctx context.Context, q *sqlchemy.SQuery, like string) []sqlchemy.ICondition {
@@ -805,6 +807,7 @@ func (self *SIsolatedDevice) getDesc() *api.IsolatedDeviceJsonDesc {
 		DiskIndex:           self.DiskIndex,
 		NvmeSizeMB:          self.NvmeSizeMB,
 		MdevId:              self.MdevId,
+		NumaNode:            self.NumaNode,
 	}
 }
 

pkg/hostman/guestman/desc/desc.go

@@ -37,16 +37,11 @@ type SGuestCpu struct {
 	// CpuCacheMode string
 }
 
-type CpuPin struct {
+type SCpuPin struct {
 	Vcpus string
 	Pcpus string
 }
 
-type SMemObject struct {
-	*Object
-	SizeMB int64
-}
-
 type SMemDevice struct {
 	Type string
 	Id   string
@@ -55,28 +50,53 @@ type SMemDevice struct {
 type SMemSlot struct {
 	SizeMB int64
 
-	MemObj *Object
+	MemObj *SMemDesc
 	MemDev *SMemDevice
 }
 
+type SCpuNumaPin struct {
+	SizeMB    int64
+	Regular   bool
+	HostNodes *uint16 `json:",omitempty"`
+	Vcpus     *string `json:",omitempty"`
+	Pcpus     *string `json:",omitempty"`
+}
+
+type SMemDesc struct {
+	*Object
+
+	NodeId *uint16 `json:",omitempty"`
+	// vcpus
+	Cpus *string `json:",omitempty"`
+}
+
+type SMemsDesc struct {
+	SMemDesc
+
+	Mems []SMemDesc `json:",omitempty"`
+}
+
 type SGuestMem struct {
 	Slots  uint
 	MaxMem uint
 
 	SizeMB int64
-	Mem    *Object `json:",omitempty"`
 
+	Mem *SMemsDesc `json:",omitempty"`
+
+	// hotplug mem devices
 	MemSlots []*SMemSlot `json:",omitempty"`
 }
 
 type SGuestHardwareDesc struct {
 	Cpu     int64
 	CpuDesc *SGuestCpu `json:",omitempty"`
-	VcpuPin []CpuPin   `json:",omitempty"`
+	VcpuPin []SCpuPin  `json:",omitempty"`
 	// Clock   *SGuestClock `json:",omitempty"`
 
-	Mem     int64
-	MemDesc *SGuestMem `json:",omitempty"`
+	Mem        int64
+	MemDesc    *SGuestMem     `json:",omitempty"`
+	CpuNumaPin []*SCpuNumaPin `json:",omitempty"`
 
 	Bios      string
 	BootOrder string

pkg/hostman/guestman/desc/pci.go

@@ -396,3 +396,30 @@ func NewUsbController(masterbus string, port int) *UsbController {
 	}
 	return uc
 }
+
+func NewMemDesc(objType, id string, nodeId *uint16, cpus *string) *SMemDesc {
+	md := &SMemDesc{
+		Object: NewObject(objType, id),
+		NodeId: nodeId,
+		Cpus:   cpus,
+	}
+
+	return md
+}
+
+func (m *SMemDesc) SetHostNodes(hostNode int) {
+	if hostNode >= 0 {
+		m.Options["host-nodes"] = fmt.Sprintf("%d", hostNode)
+		m.Options["policy"] = "bind"
+	} else {
+		delete(m.Options, "host-nodes")
+		delete(m.Options, "policy")
+	}
+}
+
+func NewMemsDesc(defaultDesc SMemDesc, appendDesc []SMemDesc) *SMemsDesc {
+	return &SMemsDesc{
+		SMemDesc: defaultDesc,
+		Mems:     appendDesc,
+	}
+}

pkg/hostman/guestman/guesthelper.go

@@ -15,12 +15,15 @@
 package guestman
 
 import (
-	"container/heap"
+	"fmt"
+	"path"
+	"sort"
 	"sync"
 
 	"yunion.io/x/cloudmux/pkg/multicloud/esxi/vcenter"
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/apis/compute"
 	hostapi "yunion.io/x/onecloud/pkg/apis/host"
@@ -28,6 +31,7 @@ import (
 	"yunion.io/x/onecloud/pkg/hostman/storageman"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/cgrouputils/cpuset"
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
 )
 
 type SBaseParams struct {
@@ -190,19 +194,21 @@ type SQgaGuestSetPassword struct {
 }
 
 type CpuSetCounter struct {
-	Nodes []*NumaNode
-	Lock  sync.Mutex
+	Nodes       []*NumaNode
+	NumaEnabled bool
+	Lock        sync.Mutex
 }
 
-func NewGuestCpuSetCounter(info *hostapi.HostTopology, reservedCpus *cpuset.CPUSet) *CpuSetCounter {
-	log.Infof("NewGuestCpuSetCounter from topo: %s", jsonutils.Marshal(info))
+func NewGuestCpuSetCounter(info *hostapi.HostTopology, reservedCpus *cpuset.CPUSet, numaAllocate bool, hugepageSizeKB int) (*CpuSetCounter, error) {
 	cpuSetCounter := new(CpuSetCounter)
 	cpuSetCounter.Nodes = make([]*NumaNode, len(info.Nodes))
+	cpuSetCounter.NumaEnabled = numaAllocate
 	hasL3Cache := false
 	for i := 0; i < len(info.Nodes); i++ {
-		node := new(NumaNode)
-		node.LogicalProcessors = cpuset.NewCPUSet()
-		node.NodeId = info.Nodes[i].ID
+		node, err := NewNumaNode(info.Nodes[i].ID, cpuSetCounter.NumaEnabled, hugepageSizeKB)
+		if err != nil {
+			return nil, err
+		}
 		cpuDies := make([]*CPUDie, 0)
 		for j := 0; j < len(info.Nodes[i].Caches); j++ {
 			if info.Nodes[i].Caches[j].Level != 3 {
@@ -227,10 +233,10 @@ func NewGuestCpuSetCounter(info *hostapi.HostTopology, reservedCpus *cpuset.CPUS
 			dieBuilder := cpuset.NewBuilder()
 			for j := 0; j < len(info.Nodes[i].Cores); j++ {
 				for k := 0; k < len(info.Nodes[i].Cores[j].LogicalProcessors); k++ {
-					if reservedCpus != nil && reservedCpus.Contains(int(info.Nodes[i].Cores[j].LogicalProcessors[k])) {
+					if reservedCpus != nil && reservedCpus.Contains(info.Nodes[i].Cores[j].LogicalProcessors[k]) {
 						continue
 					}
-					dieBuilder.Add(int(info.Nodes[i].Cores[j].LogicalProcessors[k]))
+					dieBuilder.Add(info.Nodes[i].Cores[j].LogicalProcessors[k])
 				}
 			}
 			cpuDie.LogicalProcessors = dieBuilder.Result()
@@ -243,31 +249,187 @@ func NewGuestCpuSetCounter(info *hostapi.HostTopology, reservedCpus *cpuset.CPUS
 		node.CpuDies = cpuDies
 		cpuSetCounter.Nodes[i] = node
 	}
-	heap.Init(cpuSetCounter)
-	return cpuSetCounter
+	sort.Sort(cpuSetCounter)
+	log.Infof("cpusetcounter %s", jsonutils.Marshal(cpuSetCounter))
+	return cpuSetCounter, nil
 }
 
-func (pq *CpuSetCounter) AllocCpuset(vcpuCount int) map[int][]int {
-	res := map[int][]int{}
+func (pq *CpuSetCounter) AllocCpusetWithNodeCount(vcpuCount int, memSizeKB int64, nodeCount int) (map[int]SAllocNumaCpus, error) {
+	if !pq.NumaEnabled {
+		return pq.AllocCpuset(vcpuCount, memSizeKB, -1)
+	}
+	if len(pq.Nodes) < nodeCount {
+		return nil, nil
+	}
+
+	pq.Lock.Lock()
+	defer pq.Lock.Unlock()
+	var res = map[int]SAllocNumaCpus{}
+	var nodeAllocSize = memSizeKB / int64(nodeCount)
+	if nodeAllocSize/1024%1024 == 0 && pq.nodesFreeMemSizeEnough(nodeCount, memSizeKB) {
+		var pcpuCount = vcpuCount / nodeCount
+		var remPcpuCount = vcpuCount % nodeCount
+
+		for i := 0; i < nodeCount; i++ {
+			var npcpuCount = pcpuCount
+			if remPcpuCount > 0 {
+				npcpuCount += 1
+				remPcpuCount -= 1
+			}
+			res[pq.Nodes[i].NodeId] = SAllocNumaCpus{
+				Cpuset:    pq.Nodes[i].AllocCpuset(npcpuCount),
+				MemSizeKB: nodeAllocSize,
+				Regular:   true,
+			}
+			pq.Nodes[i].NumaHugeFreeMemSizeKB -= nodeAllocSize
+			pq.Nodes[i].VcpuCount += npcpuCount
+		}
+	}
+	return res, nil
+}
+
+type SAllocNumaCpus struct {
+	Cpuset    []int
+	MemSizeKB int64
+
+	Regular bool
+}
+
+func (pq *CpuSetCounter) IsNumaEnabled() bool {
+	return pq.NumaEnabled
+}
+
+func (pq *CpuSetCounter) AllocCpuset(vcpuCount int, memSizeKB int64, perferNumaNode int8) (map[int]SAllocNumaCpus, error) {
+	res := map[int]SAllocNumaCpus{}
 	sourceVcpuCount := vcpuCount
 	pq.Lock.Lock()
 	defer pq.Lock.Unlock()
-	for vcpuCount > 0 {
-		count := vcpuCount
-		if vcpuCount > pq.Nodes[0].CpuCount {
-			count = vcpuCount/2 + vcpuCount%2
+
+	if pq.NumaEnabled {
+		err := pq.AllocNumaNodes(vcpuCount, memSizeKB, perferNumaNode, res)
+		return res, err
+	} else {
+		for vcpuCount > 0 {
+			count := vcpuCount
+			if vcpuCount > pq.Nodes[0].CpuCount {
+				count = vcpuCount/2 + vcpuCount%2
+			}
+			res[pq.Nodes[0].NodeId] = SAllocNumaCpus{
+				Cpuset: pq.Nodes[0].AllocCpuset(count),
+			}
+			pq.Nodes[0].VcpuCount += sourceVcpuCount
+			sort.Sort(pq)
+			vcpuCount -= count
 		}
-		res[pq.Nodes[0].NodeId] = pq.Nodes[0].AllocCpuset(count)
-		pq.Nodes[0].VcpuCount += sourceVcpuCount
-		heap.Fix(pq, 0)
-		vcpuCount -= count
+		return res, nil
 	}
+}
+
+func (pq *CpuSetCounter) AllocNumaNodes(vcpuCount int, memSizeKB int64, perferNumaNode int8, res map[int]SAllocNumaCpus) error {
+	var allocated = false
+
+	// check preferred numa node is memory enough
+	if perferNumaNode >= 0 {
+		for i := 0; i < len(pq.Nodes); i++ {
+			if pq.Nodes[i].NodeId != int(perferNumaNode) {
+				continue
+			}
+			if pq.Nodes[i].NumaHugeFreeMemSizeKB >= memSizeKB {
+				res[pq.Nodes[i].NodeId] = SAllocNumaCpus{
+					Cpuset:    pq.Nodes[i].AllocCpuset(vcpuCount),
+					MemSizeKB: memSizeKB,
+					Regular:   true,
+				}
+				pq.Nodes[i].NumaHugeFreeMemSizeKB -= memSizeKB
+				pq.Nodes[i].VcpuCount += vcpuCount
+
+				allocated = true
+			}
+			break
+		}
+	}
+
+	// alloc numa nodes in order 1, 2, 4, ...
+	if !allocated {
+		for nodeCount := 1; nodeCount <= len(pq.Nodes); nodeCount *= 2 {
+			if nodeCount > vcpuCount {
+				break
+			}
+			if ok := pq.nodesFreeMemSizeEnough(nodeCount, memSizeKB); !ok {
+				log.Infof("node count %d not enough", nodeCount)
+				continue
+			}
+			var nodeAllocSize = memSizeKB / int64(nodeCount)
+			if nodeAllocSize/1024%1024 > 0 {
+				continue
+			}
+
+			var pcpuCount = vcpuCount / nodeCount
+			var remPcpuCount = vcpuCount % nodeCount
+			for i := 0; i < nodeCount; i++ {
+				var npcpuCount = pcpuCount
+				if remPcpuCount > 0 {
+					npcpuCount += 1
+					remPcpuCount -= 1
+				}
+				res[pq.Nodes[i].NodeId] = SAllocNumaCpus{
+					Cpuset:    pq.Nodes[i].AllocCpuset(npcpuCount),
+					MemSizeKB: nodeAllocSize,
+					Regular:   true,
+				}
+				pq.Nodes[i].NumaHugeFreeMemSizeKB -= nodeAllocSize
+				pq.Nodes[i].VcpuCount += npcpuCount
+			}
+			allocated = true
+			break
+		}
+	}
+
+	// alloc numa nodes in order free numa node size
+	//if !allocated {
+	//	if ok := pq.nodesFreeMemSizeEnough(len(pq.Nodes), memSizeKB); !ok {
+	//		return errors.Errorf("free hugepage is not enough")
+	//	}
+	//}
+	sort.Sort(pq)
+	return nil
+}
+
+func (pq *CpuSetCounter) nodesFreeMemSizeEnough(nodeCount int, memSizeKB int64) bool {
+	var freeMem int64 = 0
+	var leastFree = memSizeKB / int64(nodeCount)
+	log.Debugf("request memsize %d, least free %d", memSizeKB, leastFree)
+	for i := 0; i < nodeCount; i++ {
+		log.Debugf("index %d node %d free size %d", i, pq.Nodes[i].NodeId, pq.Nodes[i].NumaHugeFreeMemSizeKB)
+		if pq.Nodes[i].NumaHugeFreeMemSizeKB < leastFree {
+			return false
+		}
+		freeMem += pq.Nodes[i].NumaHugeFreeMemSizeKB
+	}
+	return freeMem >= memSizeKB
+}
+
+func (pq *CpuSetCounter) setNumaNodes(numaMaps map[int]int, vcpuCount int64) map[int]SAllocNumaCpus {
+	res := map[int]SAllocNumaCpus{}
+	for i := range pq.Nodes {
+		if size, ok := numaMaps[pq.Nodes[i].NodeId]; ok {
+			allocMem := int64(size) * 1024
+			//npcpuCount := int(vcpuCount*allocMem/memSizeKB + (vcpuCount*allocMem)%memSizeKB)
+			res[pq.Nodes[i].NodeId] = SAllocNumaCpus{
+				Cpuset:    pq.Nodes[i].AllocCpuset(int(vcpuCount)),
+				MemSizeKB: allocMem,
+				Regular:   false,
+			}
+
+			pq.Nodes[i].NumaHugeFreeMemSizeKB -= allocMem
+			pq.Nodes[i].VcpuCount += int(vcpuCount)
+		}
+	}
+	sort.Sort(pq)
 	return res
 }
 
 func (pq *CpuSetCounter) ReleaseCpus(cpus []int, vcpuCount int) {
-	pq.Lock.Lock()
-	defer pq.Lock.Unlock()
 	var numaCpuCount = map[int][]int{}
 	for i := 0; i < len(cpus); i++ {
 		for j := 0; j < len(pq.Nodes); j++ {
@@ -285,14 +447,36 @@ func (pq *CpuSetCounter) ReleaseCpus(cpus []int, vcpuCount int) {
 		if numaCpus, ok := numaCpuCount[pq.Nodes[i].NodeId]; ok {
 			pq.Nodes[i].CpuDies.ReleaseCpus(numaCpus, vcpuCount)
 			pq.Nodes[i].VcpuCount -= vcpuCount
-			heap.Fix(pq, i)
 		}
 	}
+	sort.Sort(pq)
+}
+
+func (pq *CpuSetCounter) ReleaseNumaCpus(memSizeMb int64, hostNode int, cpus []int, vcpuCount int) {
+	for i := 0; i < len(pq.Nodes); i++ {
+		if pq.Nodes[i].NodeId != hostNode {
+			continue
+		}
+		pq.Nodes[i].CpuDies.ReleaseCpus(cpus, vcpuCount)
+		pq.Nodes[i].VcpuCount -= vcpuCount
+		pq.Nodes[i].NumaHugeFreeMemSizeKB += memSizeMb * 1024
+	}
+	sort.Sort(pq)
+}
+
+func (pq *CpuSetCounter) LoadNumaCpus(memSizeMb int64, hostNode int, cpus []int, vcpuCount int) {
+	for i := 0; i < len(pq.Nodes); i++ {
+		if pq.Nodes[i].NodeId != hostNode {
+			continue
+		}
+		pq.Nodes[i].CpuDies.LoadCpus(cpus, vcpuCount)
+		pq.Nodes[i].VcpuCount += vcpuCount
+		pq.Nodes[i].NumaHugeFreeMemSizeKB -= memSizeMb * 1024
+	}
+	sort.Sort(pq)
 }
 
 func (pq *CpuSetCounter) LoadCpus(cpus []int, vcpuCpunt int) {
-	pq.Lock.Lock()
-	defer pq.Lock.Unlock()
 	var numaCpuCount = map[int][]int{}
 	for i := 0; i < len(cpus); i++ {
 		for j := 0; j < len(pq.Nodes); j++ {
@@ -310,15 +494,22 @@ func (pq *CpuSetCounter) LoadCpus(cpus []int, vcpuCpunt int) {
 		if numaCpus, ok := numaCpuCount[pq.Nodes[i].NodeId]; ok {
 			pq.Nodes[i].CpuDies.LoadCpus(numaCpus, vcpuCpunt)
 			pq.Nodes[i].VcpuCount += vcpuCpunt
-			heap.Fix(pq, i)
 		}
 	}
+	sort.Sort(pq)
 }
 
 func (pq CpuSetCounter) Len() int { return len(pq.Nodes) }
 
 func (pq CpuSetCounter) Less(i, j int) bool {
-	return pq.Nodes[i].VcpuCount < pq.Nodes[j].VcpuCount
+	if pq.NumaEnabled {
+		if pq.Nodes[i].NumaHugeFreeMemSizeKB == pq.Nodes[j].NumaHugeFreeMemSizeKB {
+			return pq.Nodes[i].VcpuCount < pq.Nodes[j].VcpuCount
+		}
+		return pq.Nodes[i].NumaHugeFreeMemSizeKB > pq.Nodes[j].NumaHugeFreeMemSizeKB
+	} else {
+		return pq.Nodes[i].VcpuCount < pq.Nodes[j].VcpuCount
+	}
 }
 
 func (pq CpuSetCounter) Swap(i, j int) {
@@ -343,22 +534,55 @@ type NumaNode struct {
 	LogicalProcessors cpuset.CPUSet
 	VcpuCount         int
 	CpuCount          int
-	NodeId            int
+
+	NodeId                int
+	NumaHugeMemSizeKB     int64
+	NumaHugeFreeMemSizeKB int64
+}
+
+func NewNumaNode(nodeId int, numaAllocate bool, hugepageSizeKB int) (*NumaNode, error) {
+	n := new(NumaNode)
+	n.LogicalProcessors = cpuset.NewCPUSet()
+	n.NodeId = nodeId
+
+	if numaAllocate {
+		nodeHugepagePath := fmt.Sprintf("/sys/devices/system/node/node%d/hugepages/hugepages-%dkB", nodeId, hugepageSizeKB)
+		if !fileutils2.Exists(nodeHugepagePath) {
+			return n, nil
+		}
+
+		nrHugepage, err := fileutils2.FileGetIntContent(path.Join(nodeHugepagePath, "nr_hugepages"))
+		if err != nil {
+			log.Errorf("failed get node %d nr hugepage %s", nodeId, err)
+			return nil, errors.Wrap(err, "get numa node nr hugepage")
+		}
+		n.NumaHugeMemSizeKB = int64(nrHugepage) * int64(hugepageSizeKB)
+
+		//freeHugepage, err := fileutils2.FileGetIntContent(path.Join(nodeHugepagePath, "free_hugepages"))
+		//if err != nil {
+		//	log.Errorf("failed get node %d free hugepage %s", nodeId, err)
+		//	return nil, errors.Wrap(err, "get numa node free hugepage")
+		//}
+		n.NumaHugeFreeMemSizeKB = n.NumaHugeMemSizeKB
+	}
+	return n, nil
 }
 
 func (n *NumaNode) AllocCpuset(vcpuCount int) []int {
 	cpus := make([]int, 0)
-	for vcpuCount > 0 {
-		dies := n.CpuDies
-		count := vcpuCount
-		if vcpuCount > dies[0].LogicalProcessors.Size() {
-			count = dies[0].LogicalProcessors.Size()
+
+	var allocCount = vcpuCount
+	for i := range n.CpuDies {
+		n.CpuDies[i].VcpuCount += vcpuCount
+		cpus = append(cpus, n.CpuDies[i].LogicalProcessors.ToSliceNoSort()...)
+		if allocCount > n.CpuDies[i].LogicalProcessors.Size() {
+			allocCount -= n.CpuDies[i].LogicalProcessors.Size()
+		} else {
+			break
 		}
-		dies[0].VcpuCount += count
-		heap.Fix(&n.CpuDies, 0)
-		vcpuCount -= count
-		cpus = append(cpus, dies[0].LogicalProcessors.ToSliceNoSort()...)
 	}
+
+	sort.Sort(n.CpuDies)
 	return cpus
 }
 
@@ -410,9 +634,9 @@ func (pq *SorttedCPUDie) ReleaseCpus(cpus []int, vcpuCount int) {
 	for i := 0; i < len(*pq); i++ {
 		if _, ok := cpuDies[i]; ok {
 			(*pq)[i].VcpuCount -= vcpuCount
-			heap.Fix(pq, i)
 		}
 	}
+	sort.Sort(pq)
 }
 
 func (pq *SorttedCPUDie) LoadCpus(cpus []int, vcpuCount int) {
@@ -433,7 +657,7 @@ func (pq *SorttedCPUDie) LoadCpus(cpus []int, vcpuCount int) {
 	for i := 0; i < len(*pq); i++ {
 		if _, ok := cpuDies[i]; ok {
 			(*pq)[i].VcpuCount += vcpuCount
-			heap.Fix(pq, i)
 		}
 	}
+	sort.Sort(pq)
 }

pkg/hostman/guestman/guestman.go

@@ -45,6 +45,7 @@ import (
 	fwdpb "yunion.io/x/onecloud/pkg/hostman/guestman/forwarder/api"
 	"yunion.io/x/onecloud/pkg/hostman/guestman/types"
 	deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
+	"yunion.io/x/onecloud/pkg/hostman/hostinfo/hostconsts"
 	"yunion.io/x/onecloud/pkg/hostman/hostutils"
 	"yunion.io/x/onecloud/pkg/hostman/monitor"
 	"yunion.io/x/onecloud/pkg/hostman/options"
@@ -101,11 +102,12 @@ type SGuestManager struct {
 	qemuMachineCpuMax map[string]uint
 	qemuMaxMem        int
 
-	cpuSet     *CpuSetCounter
-	pythonPath string
+	numaAllocate bool
+	cpuSet       *CpuSetCounter
+	pythonPath   string
 }
 
-func NewGuestManager(host hostutils.IHost, serversPath string) *SGuestManager {
+func NewGuestManager(host hostutils.IHost, serversPath string) (*SGuestManager, error) {
 	manager := &SGuestManager{}
 	manager.host = host
 	manager.ServersPath = serversPath
@@ -116,15 +118,16 @@ func NewGuestManager(host hostutils.IHost, serversPath string) *SGuestManager {
 	manager.ServersLock = &sync.Mutex{}
 	manager.TrafficLock = &sync.Mutex{}
 	manager.GuestStartWorker = appsrv.NewWorkerManager("GuestStart", 1, appsrv.DEFAULT_BACKLOG, false)
-	manager.cpuSet = NewGuestCpuSetCounter(host.GetHostTopology(), host.GetReservedCpusInfo())
+
 	// manager.StartCpusetBalancer()
-	manager.LoadExistingGuests()
-	manager.host.StartDHCPServer()
 	manager.dirtyServersChan = make(chan struct{})
 	manager.dirtyServers = make([]*SKVMGuestInstance, 0)
 	manager.qemuMachineCpuMax = make(map[string]uint, 0)
-	procutils.NewCommand("mkdir", "-p", manager.QemuLogDir()).Run()
-	return manager
+	err := procutils.NewCommand("mkdir", "-p", manager.QemuLogDir()).Run()
+	if err != nil {
+		return nil, errors.Wrap(err, "mkdir qemu log dir")
+	}
+	return manager, nil
 }
 
 func (m *SGuestManager) InitQemuMaxCpus(machineCaps []monitor.MachineInfo, kvmMaxCpus uint) {
@@ -232,7 +235,18 @@ func (m *SGuestManager) CleanServer(sid string) {
 	m.Servers.Delete(sid)
 }
 
-func (m *SGuestManager) Bootstrap() chan struct{} {
+func (m *SGuestManager) Bootstrap() (chan struct{}, error) {
+	hostTypo := m.host.GetHostTopology()
+	m.numaAllocate = m.host.IsNumaAllocateEnabled() && m.host.IsHugepagesEnabled() && (len(hostTypo.Nodes) > 1)
+	cpuSet, err := NewGuestCpuSetCounter(
+		hostTypo, m.host.GetReservedCpusInfo(), m.numaAllocate, m.host.HugepageSizeKb())
+	if err != nil {
+		return nil, err
+	}
+	m.cpuSet = cpuSet
+	m.LoadExistingGuests()
+	m.host.StartDHCPServer()
+
 	if m.isLoaded || len(m.ServersPath) == 0 {
 		log.Errorln("Guestman bootstrap has been called!!!!!")
 	} else {
@@ -244,7 +258,7 @@ func (m *SGuestManager) Bootstrap() chan struct{} {
 			m.OnLoadExistingGuestsComplete()
 		}
 	}
-	return m.dirtyServersChan
+	return m.dirtyServersChan, nil
 }
 
 func (m *SGuestManager) VerifyExistingGuests(pendingDelete bool) {
@@ -424,6 +438,8 @@ func (m *SGuestManager) LoadServer(sid string) {
 
 func (m *SGuestManager) loadGuestCpuset(guest *SKVMGuestInstance) {
 	if guest.GetPid() > 0 {
+		m.cpuSet.Lock.Lock()
+		defer m.cpuSet.Lock.Unlock()
 		for _, vcpuPin := range guest.Desc.VcpuPin {
 			pcpuSet, err := cpuset.Parse(vcpuPin.Pcpus)
 			if err != nil {
@@ -437,6 +453,28 @@ func (m *SGuestManager) loadGuestCpuset(guest *SKVMGuestInstance) {
 			}
 			m.cpuSet.LoadCpus(pcpuSet.ToSlice(), vcpuSet.Size())
 		}
+		for _, numaCpuset := range guest.Desc.CpuNumaPin {
+			pcpuSet, err := cpuset.Parse(*numaCpuset.Pcpus)
+			if err != nil {
+				log.Errorf("failed parse %s pcpus: %s", guest.GetName(), *numaCpuset.Pcpus)
+				continue
+			}
+			vcpuCount := int(guest.Desc.Cpu)
+			if numaCpuset.Vcpus != nil {
+				vcpuSet, err := cpuset.Parse(*numaCpuset.Vcpus)
+				if err != nil {
+					log.Errorf("failed parse %s vcpus: %s", guest.GetName(), *numaCpuset.Vcpus)
+					continue
+				}
+				vcpuCount = vcpuSet.Size()
+			}
+			hostNodes := -1
+			if numaCpuset.HostNodes != nil {
+				hostNodes = int(*numaCpuset.HostNodes)
+			}
+
+			m.cpuSet.LoadNumaCpus(numaCpuset.SizeMB, hostNodes, pcpuSet.ToSlice(), vcpuCount)
+		}
 	}
 }
 
@@ -1421,7 +1459,7 @@ func (m *SGuestManager) ExitGuestCleanup() {
 		return true
 	})
 	if !options.HostOptions.DisableSetCgroup {
-		cgrouputils.CgroupCleanAll()
+		cgrouputils.CgroupCleanAll(hostconsts.HOST_CGROUP)
 	}
 }
 
@@ -1646,12 +1684,17 @@ func Stop() {
 	guestManager.ExitGuestCleanup()
 }
 
-func Init(host hostutils.IHost, serversPath string) {
+func Init(host hostutils.IHost, serversPath string) error {
 	if guestManager == nil {
-		guestManager = NewGuestManager(host, serversPath)
+		manager, err := NewGuestManager(host, serversPath)
+		if err != nil {
+			return err
+		}
+		guestManager = manager
 		types.HealthCheckReactor = guestManager
 		types.GuestDescGetter = guestManager
 	}
+	return nil
 }
 
 func GetGuestManager() *SGuestManager {

pkg/hostman/guestman/guesttasks.go

@@ -42,6 +42,7 @@ import (
 	"yunion.io/x/onecloud/pkg/hostman/monitor"
 	"yunion.io/x/onecloud/pkg/hostman/options"
 	"yunion.io/x/onecloud/pkg/hostman/storageman"
+	"yunion.io/x/onecloud/pkg/util/cgrouputils/cpuset"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/util/qemuimg"
@@ -2386,6 +2387,7 @@ type SGuestHotplugCpuMemTask struct {
 
 	originalCpuCount int
 	addedCpuCount    int
+	addedVcpuIds     []int
 
 	addedMemSize    int
 	memSlotNewIndex *int
@@ -2415,7 +2417,69 @@ func (task *SGuestHotplugCpuMemTask) Start() {
 }
 
 func (task *SGuestHotplugCpuMemTask) startAddCpu() {
-	task.Monitor.GetCpuCount(task.onGetCpuCount)
+	if task.Desc.MemDesc.Mem.Cpus != nil && len(task.Desc.CpuNumaPin) > 0 {
+		task.buildVcpusMap()
+	} else {
+		task.Monitor.GetCpuCount(task.onGetCpuCount)
+	}
+}
+
+func (task *SGuestHotplugCpuMemTask) buildVcpusMap() {
+	vcpuSet, _ := cpuset.Parse(*task.Desc.MemDesc.Mem.Cpus)
+
+	for i := range task.Desc.MemDesc.Mem.Mems {
+		if task.Desc.MemDesc.Mem.Mems[i].Cpus != nil {
+			memVcpuSet, _ := cpuset.Parse(*task.Desc.MemDesc.Mem.Mems[i].Cpus)
+			vcpuSet = vcpuSet.Union(memVcpuSet)
+		}
+	}
+
+	for i := range task.Desc.CpuNumaPin {
+		if task.Desc.CpuNumaPin[i].Vcpus != nil {
+			allocedVcpus, _ := cpuset.Parse(*task.Desc.CpuNumaPin[i].Vcpus)
+			vcpuSet = vcpuSet.Difference(allocedVcpus)
+		}
+	}
+
+	task.startAddCpusWithFreeVcpuSet(vcpuSet.ToSlice())
+}
+
+func (task *SGuestHotplugCpuMemTask) startAddCpusWithFreeVcpuSet(vcpuSet []int) {
+	if task.addedCpuCount >= task.addCpuCount {
+		task.startAddMem()
+		return
+	}
+
+	vcpuId := vcpuSet[0]
+	cb := func(reason string) {
+		if len(reason) > 0 {
+			log.Errorln(reason)
+			task.onFail(reason)
+			return
+		}
+
+		cpus, _ := task.manager.cpuSet.AllocCpuset(1, 0, -1)
+		for _, cpus := range cpus {
+			pcpus := cpuset.NewCPUSet(cpus.Cpuset...).String()
+			vcpus := fmt.Sprintf("%d-%d", vcpuId, vcpuId)
+			cpuPin := &desc.SCpuNumaPin{
+				SizeMB:  0,
+				Pcpus:   &pcpus,
+				Vcpus:   &vcpus,
+				Regular: true,
+			}
+			task.Desc.CpuNumaPin = append(task.Desc.CpuNumaPin, cpuPin)
+		}
+		if task.addedVcpuIds == nil {
+			task.addedVcpuIds = []int{vcpuId}
+		} else {
+			task.addedVcpuIds = append(task.addedVcpuIds, vcpuId)
+		}
+
+		task.addedCpuCount += 1
+		task.startAddCpusWithFreeVcpuSet(vcpuSet[1:])
+	}
+	task.Monitor.AddCpu(vcpuId, cb)
 }
 
 func (task *SGuestHotplugCpuMemTask) onGetCpuCount(count int) {
@@ -2450,12 +2514,12 @@ func (task *SGuestHotplugCpuMemTask) startAddMem() {
 }
 
 func (task *SGuestHotplugCpuMemTask) onGetSlotIndex(index int) {
-	var newIndex = index
+	var newIndex = index + len(task.Desc.MemDesc.Mem.Mems)
 	task.memSlotNewIndex = &newIndex
 
 	var objType string
 	var id = fmt.Sprintf("mem%d", *task.memSlotNewIndex)
-	var options map[string]string
+	var opts map[string]string
 
 	if task.manager.host.IsHugepagesEnabled() {
 		memPath := fmt.Sprintf("/dev/hugepages/%s-%d", task.GetId(), index)
@@ -2480,7 +2544,7 @@ func (task *SGuestHotplugCpuMemTask) onGetSlotIndex(index int) {
 		}
 
 		objType = "memory-backend-file"
-		options = map[string]string{
+		opts = map[string]string{
 			"size":     fmt.Sprintf("%dM", task.addMemSize),
 			"mem-path": memPath,
 			"share":    "on",
@@ -2488,22 +2552,22 @@ func (task *SGuestHotplugCpuMemTask) onGetSlotIndex(index int) {
 		}
 	} else {
 		objType = "memory-backend-ram"
-		options = map[string]string{
+		opts = map[string]string{
 			"size": fmt.Sprintf("%dM", task.addMemSize),
 		}
 	}
-	// options["id"] = id
+	opts["id"] = id
 	cb := func(reason string) {
 		if reason == "" {
-			memObj := desc.NewObject(objType, id)
-			memObj.Options = options
+			memObj := desc.NewMemDesc(objType, id, nil, nil)
+			memObj.Options = opts
 			task.memSlot = new(desc.SMemSlot)
 			task.memSlot.MemObj = memObj
 			task.memSlot.SizeMB = int64(task.addMemSize)
 		}
 		task.onAddMemObject(reason)
 	}
-	task.Monitor.ObjectAdd(objType, options, cb)
+	task.Monitor.ObjectAdd(objType, opts, cb)
 }
 
 func (task *SGuestHotplugCpuMemTask) onAddMemFailed(reason string) {
@@ -2553,6 +2617,15 @@ func (task *SGuestHotplugCpuMemTask) updateGuestDesc() {
 			task.Desc.MemDesc.MemSlots = make([]*desc.SMemSlot, 0)
 		}
 		task.Desc.MemDesc.MemSlots = append(task.Desc.MemDesc.MemSlots, task.memSlot)
+
+		if task.manager.numaAllocate {
+			hugepageId := fmt.Sprintf("%s-%d", task.getOriginId(), *task.memSlotNewIndex)
+			task.validateNumaAllocated(hugepageId, false, true, nil)
+		}
+	}
+
+	if len(task.addedVcpuIds) > 0 {
+		task.setCgroupCPUSet()
 	}
 	if task.addedCpuCount > 0 && len(task.Desc.VcpuPin) == 1 {
 		task.Desc.VcpuPin[0].Vcpus = fmt.Sprintf("0-%d", task.Desc.Cpu-1)

pkg/hostman/guestman/pci.go

@@ -54,7 +54,10 @@ func (s *SKVMGuestInstance) initGuestDesc() error {
 	if err != nil {
 		return err
 	}
-	s.initMemDesc(s.Desc.Mem)
+	err = s.initMemDesc(s.Desc.Mem)
+	if err != nil {
+		return errors.Wrap(err, "initMemDesc")
+	}
 	s.initMachineDesc()
 
 	pciRoot, pciBridge := s.initGuestPciControllers(s.manager.host.IsKvmSupport())

pkg/hostman/guestman/pci_test.go

@@ -151,7 +151,8 @@ func TestSKVMGuestInstance_initGuestDesc(t *testing.T) {
 	// s.initMemDesc()
 	s.Desc.MemDesc = new(desc.SGuestMem)
 	s.Desc.MemDesc.SizeMB = s.Desc.Mem
-	s.Desc.MemDesc.Mem = desc.NewObject("memory-backend-memfd", "mem")
+	memDesc := desc.NewMemDesc("memory-backend-memfd", "mem", nil, nil)
+	s.Desc.MemDesc.Mem = desc.NewMemsDesc(*memDesc, nil)
 	s.Desc.MemDesc.Mem.Options = map[string]string{
 		"size":  fmt.Sprintf("%dM", s.Desc.Mem),
 		"share": "on", "prealloc": "on",

pkg/hostman/guestman/qemu-kvm.go

@@ -159,6 +159,187 @@ func (s *SKVMGuestInstance) updateGuestDesc() error {
 	return s.SaveLiveDesc(s.Desc)
 }
 
+func (s *SKVMGuestInstance) releaseCpuNumaPin(cpuNumaPin []*desc.SCpuNumaPin) {
+	for _, numaCpus := range cpuNumaPin {
+		pcpuSet, err := cpuset.Parse(*numaCpus.Pcpus)
+		if err != nil {
+			log.Errorf("failed parse %s pcpus: %s", s.GetName(), *numaCpus.Pcpus)
+			continue
+		}
+		vcpuCount := int(s.Desc.Cpu)
+		if numaCpus.Vcpus != nil {
+			vcpuSet, err := cpuset.Parse(*numaCpus.Vcpus)
+			if err != nil {
+				log.Errorf("failed parse %s vcpus: %s", s.GetName(), *numaCpus.Vcpus)
+				continue
+			}
+			vcpuCount = vcpuSet.Size()
+		}
+		hostNodes := -1
+		if numaCpus.HostNodes != nil {
+			hostNodes = int(*numaCpus.HostNodes)
+		}
+		s.manager.cpuSet.ReleaseNumaCpus(numaCpus.SizeMB, hostNodes, pcpuSet.ToSlice(), vcpuCount)
+	}
+}
+
+// release allocated numa mems and realloc numa mems
+func (s *SKVMGuestInstance) reallocateNumaNodes(isMigrate bool) error {
+	s.manager.cpuSet.Lock.Lock()
+	defer s.manager.cpuSet.Lock.Unlock()
+
+	s.releaseCpuNumaPin(s.Desc.CpuNumaPin)
+	s.Desc.CpuNumaPin = nil
+
+	if isMigrate {
+		if err := s.reallocateMigrateNumaNodes(); err != nil {
+			return errors.Wrap(err, "reallocateMigrateNumaNodes")
+		}
+	} else {
+		if err := s.allocGuestNumaCpuset(); err != nil {
+			return errors.Wrap(err, "allocGuestNumaCpuset")
+		}
+		if err := s.initMemDesc(s.Desc.Mem); err != nil {
+			return errors.Wrap(err, "fixNumaAllocate")
+		}
+	}
+
+	return s.SaveLiveDesc(s.Desc)
+}
+
+func (s *SKVMGuestInstance) reallocateMigrateNumaNodes() error {
+	nodeNumaCpus, err := s.manager.cpuSet.AllocCpusetWithNodeCount(int(s.Desc.Cpu), s.Desc.Mem*1024, len(s.Desc.MemDesc.Mem.Mems)+1)
+	if err != nil {
+		return errors.Wrap(err, "AllocCpusetWithNodeCount")
+	}
+	var cpuNumaPin = make([]*desc.SCpuNumaPin, 0)
+	if len(nodeNumaCpus) > 0 {
+		for nodeId, numaCpus := range nodeNumaCpus {
+			unodeId := uint16(nodeId)
+			pcpus := cpuset.NewCPUSet(numaCpus.Cpuset...).String()
+			memPin := &desc.SCpuNumaPin{
+				SizeMB:    numaCpus.MemSizeKB / 1024, // MB
+				HostNodes: &unodeId,
+				Pcpus:     &pcpus,
+				Regular:   numaCpus.Regular,
+			}
+			cpuNumaPin = append(cpuNumaPin, memPin)
+		}
+	}
+
+	if len(cpuNumaPin) > 0 {
+		s.Desc.CpuNumaPin = cpuNumaPin
+		s.Desc.MemDesc.Mem.SMemDesc.SetHostNodes(int(*cpuNumaPin[0].HostNodes))
+		for i := range s.Desc.MemDesc.Mem.Mems {
+			s.Desc.MemDesc.Mem.Mems[i].SetHostNodes(int(*cpuNumaPin[i+1].HostNodes))
+		}
+	} else {
+		s.Desc.MemDesc.Mem.SMemDesc.SetHostNodes(-1)
+		for i := range s.Desc.MemDesc.Mem.Mems {
+			s.Desc.MemDesc.Mem.Mems[i].SetHostNodes(-1)
+		}
+	}
+
+	return nil
+}
+
+func (s *SKVMGuestInstance) validateNumaAllocated(keywords string, isMigrate, isHotPlug bool, vcpuOrder []string) error {
+	if len(s.Desc.CpuNumaPin) > 0 {
+		if isMigrate {
+			for i := range s.Desc.CpuNumaPin {
+				s.Desc.CpuNumaPin[i].Vcpus = &vcpuOrder[i]
+			}
+			return s.SaveLiveDesc(s.Desc)
+		}
+		if !isHotPlug {
+			return s.SaveLiveDesc(s.Desc)
+		}
+	}
+
+	guestPid := s.GetPid()
+	if guestPid <= 0 {
+		return errors.Errorf("guest not running? pid %d", guestPid)
+	}
+	numaMapPath := fmt.Sprintf("/proc/%d/numa_maps", guestPid)
+	for {
+		if !fileutils2.Exists(numaMapPath) {
+			return errors.Errorf("guest not running? pid %d", guestPid)
+		}
+		// wait hugepage mem allocate
+		if s.Monitor != nil && s.Monitor.IsConnected() {
+			break
+		}
+		time.Sleep(time.Millisecond * 300)
+	}
+
+	content, err := fileutils2.FileGetContents(numaMapPath)
+	if err != nil {
+		return errors.Wrap(err, "read numa_maps")
+	}
+
+	readNodeAllocateMap := map[int]int{}
+	numaNodeRegex := regexp.MustCompile(`^N[0-9]+=[0-9]+$`)
+	for _, line := range strings.Split(content, "\n") {
+		if idx := strings.Index(line, "hugepages"); idx < 0 {
+			continue
+		}
+		if idx := strings.Index(line, keywords); idx < 0 {
+			continue
+		}
+
+		// ... huge dirty=15 N0=3 N1=5 N2=5 N3=2 kernelpagesize_kB=1048576
+		segs := strings.Split(line, " ")
+		for _, seg := range segs {
+			if !numaNodeRegex.MatchString(seg) {
+				continue
+			}
+			log.Infof("hugepages segs %v", seg)
+			nodeAllocate := strings.Split(seg[1:], "=")
+			if len(nodeAllocate) != 2 {
+				continue
+			}
+			node, _ := strconv.Atoi(nodeAllocate[0])
+			size, _ := strconv.Atoi(nodeAllocate[1])
+			if _, ok := readNodeAllocateMap[node]; !ok {
+				readNodeAllocateMap[node] = 0
+			}
+			readNodeAllocateMap[node] += size * 1024
+		}
+	}
+	log.Infof("read node allocate map %v", readNodeAllocateMap)
+
+	s.manager.cpuSet.Lock.Lock()
+	defer s.manager.cpuSet.Lock.Unlock()
+	nodeNumaCpus := s.manager.cpuSet.setNumaNodes(readNodeAllocateMap, s.Desc.Cpu)
+
+	var cpuNumaPin = make([]*desc.SCpuNumaPin, 0)
+	for nodeId, numaCpus := range nodeNumaCpus {
+		unodeId := uint16(nodeId)
+		pcpus := cpuset.NewCPUSet(numaCpus.Cpuset...).String()
+		memPin := &desc.SCpuNumaPin{
+			SizeMB:    numaCpus.MemSizeKB / 1024, // MB
+			HostNodes: &unodeId,
+			Pcpus:     &pcpus,
+			Regular:   numaCpus.Regular,
+		}
+		cpuNumaPin = append(cpuNumaPin, memPin)
+	}
+
+	if len(s.Desc.CpuNumaPin) > 0 { // hotplug mems
+		s.Desc.CpuNumaPin = append(s.Desc.CpuNumaPin, cpuNumaPin...)
+		return s.SaveLiveDesc(s.Desc)
+	}
+
+	if len(vcpuOrder) > 0 {
+		for i := range cpuNumaPin {
+			cpuNumaPin[i].Vcpus = &vcpuOrder[i]
+		}
+	}
+
+	s.Desc.CpuNumaPin = cpuNumaPin
+	return s.SaveLiveDesc(s.Desc)
+}
+
 func (s *SKVMGuestInstance) initLiveDescFromSourceGuest(srcDesc *desc.SGuestDesc) error {
 	srcDesc.SGuestProjectDesc = s.SourceDesc.SGuestProjectDesc
 	srcDesc.SGuestRegionDesc = s.SourceDesc.SGuestRegionDesc
@@ -204,8 +385,58 @@ func (s *SKVMGuestInstance) initLiveDescFromSourceGuest(srcDesc *desc.SGuestDesc
 		srcDesc.Nics[i].DownscriptPath = s.getNicDownScriptPath(srcDesc.Nics[i])
 	}
 
+	nodeNumaCpus, err := s.manager.cpuSet.AllocCpusetWithNodeCount(int(srcDesc.Cpu), srcDesc.Mem*1024, len(srcDesc.MemDesc.Mem.Mems)+1)
+	if err != nil {
+		return errors.Wrap(err, "AllocCpusetWithNodeCount")
+	}
+
+	var cpus = make([]int, 0)
+	var cpuNumaPin = make([]*desc.SCpuNumaPin, 0)
+	for nodeId, numaCpus := range nodeNumaCpus {
+		if s.manager.numaAllocate {
+			unodeId := uint16(nodeId)
+			pcpus := cpuset.NewCPUSet(numaCpus.Cpuset...).String()
+			memPin := &desc.SCpuNumaPin{
+				SizeMB:    numaCpus.MemSizeKB / 1024, // MB
+				HostNodes: &unodeId,
+				Pcpus:     &pcpus,
+				Regular:   numaCpus.Regular,
+			}
+			cpuNumaPin = append(cpuNumaPin, memPin)
+		}
+		cpus = append(cpus, numaCpus.Cpuset...)
+	}
+
+	if s.manager.numaAllocate {
+		srcDesc.VcpuPin = nil
+		srcDesc.CpuNumaPin = nil
+	} else {
+		srcDesc.VcpuPin = []desc.SCpuPin{
+			{
+				Vcpus: fmt.Sprintf("0-%d", srcDesc.Cpu-1),
+				Pcpus: cpuset.NewCPUSet(cpus...).String(),
+			},
+		}
+		for i := range srcDesc.CpuNumaPin {
+			srcDesc.CpuNumaPin[i].Regular = false
+		}
+	}
+
+	if len(cpuNumaPin) > 0 {
+		srcDesc.MemDesc.Mem.SMemDesc.SetHostNodes(int(*cpuNumaPin[0].HostNodes))
+		for i := range srcDesc.MemDesc.Mem.Mems {
+			srcDesc.MemDesc.Mem.Mems[i].SetHostNodes(int(*cpuNumaPin[i+1].HostNodes))
+		}
+		srcDesc.CpuNumaPin = cpuNumaPin
+	} else {
+		srcDesc.MemDesc.Mem.SMemDesc.SetHostNodes(-1)
+		for i := range srcDesc.MemDesc.Mem.Mems {
+			srcDesc.MemDesc.Mem.Mems[i].SetHostNodes(-1)
+		}
+	}
+
 	s.Desc = srcDesc
-	err := s.loadGuestPciAddresses()
+	err = s.loadGuestPciAddresses()
 	if err != nil {
 		return errors.Wrap(err, "initLiveDescFromSourceGuest")
 	}
@@ -530,24 +761,32 @@ func (s *SKVMGuestInstance) asyncScriptStart(ctx context.Context, params interfa
 		return nil, errors.Wrap(err, "fuse mount")
 	}
 
-	if jsonutils.QueryBoolean(data, "need_migrate", false) {
+	var vcpuOrder = make([]string, 0)
+	isMigrate := jsonutils.QueryBoolean(data, "need_migrate", false)
+	if isMigrate {
 		var sourceDesc = new(desc.SGuestDesc)
 		err = data.Unmarshal(sourceDesc, "src_desc")
 		if err != nil {
 			return nil, errors.Wrap(err, "unmarshal src desc")
 		}
+		for i := range sourceDesc.CpuNumaPin {
+			if sourceDesc.CpuNumaPin[i].Vcpus != nil {
+				vcpus := *sourceDesc.CpuNumaPin[i].Vcpus
+				vcpuOrder = append(vcpuOrder, vcpus)
+			}
+		}
 		err = s.initLiveDescFromSourceGuest(sourceDesc)
 	} else {
 		err = s.updateGuestDesc()
 	}
 
 	// init live migrate listen port
-	if jsonutils.QueryBoolean(data, "need_migrate", false) || s.Desc.IsSlave {
-		log.Infof("backup guest alloc dest port %v", s.LiveMigrateDestPort)
+	if isMigrate || s.Desc.IsSlave {
 		migratePort := s.manager.GetLiveMigrateFreePort()
 		defer s.manager.unsetPort(migratePort)
 		migratePortInt64 := int64(migratePort)
 		s.LiveMigrateDestPort = &migratePortInt64
+		log.Infof("backup guest alloc dest port %v", s.LiveMigrateDestPort)
 	}
 
 	if err != nil {
@@ -573,6 +812,13 @@ func (s *SKVMGuestInstance) asyncScriptStart(ctx context.Context, params interfa
 			data.Set("vnc_port", jsonutils.NewInt(int64(vncPort)))
 		}
 
+		if tried > 1 && s.manager.numaAllocate {
+			if err = s.reallocateNumaNodes(isMigrate); err != nil {
+				log.Errorf("failed fix numa allocated mems %s", err)
+				goto finally
+			}
+		}
+
 		err = s.saveScripts(data)
 		if err != nil {
 			goto finally
@@ -591,6 +837,14 @@ func (s *SKVMGuestInstance) asyncScriptStart(ctx context.Context, params interfa
 			log.Errorf("Start VM failed %s: %s", s.GetName(), err)
 			time.Sleep(time.Duration(1<<uint(tried-1)) * time.Second)
 		} else {
+			if s.manager.numaAllocate {
+				if err = s.validateNumaAllocated(s.Desc.Uuid, isMigrate, false, vcpuOrder); err != nil {
+					log.Errorf("VM %s validateNumaAllocated: %s", s.GetName(), err)
+					isStarted = false
+					s.scriptStop()
+					continue
+				}
+			}
 			log.Infof("VM started %s ...", s.GetName())
 		}
 	}
@@ -601,6 +855,8 @@ func (s *SKVMGuestInstance) asyncScriptStart(ctx context.Context, params interfa
 		s.syncMeta = s.CleanImportMetadata()
 		return nil, nil
 	}
+	// release guest acquired cpu mems on guest start failed
+	s.releaseGuestCpuset()
 	log.Errorf("Async start server %s failed: %s!!!", s.GetName(), err)
 	if ctx != nil && len(appctx.AppContextTaskId(ctx)) >= 0 {
 		hostutils.TaskFailed(ctx, fmt.Sprintf("Async start server failed: %s", err))
@@ -1060,7 +1316,7 @@ func (s *SKVMGuestInstance) migrateStartNbdServer(nbdServerPort int) error {
 	var err = make(chan error)
 	onNbdServerStarted := func(res string) {
 		if len(res) > 0 {
-			err <- errors.Errorf("failed enable multifd %s", res)
+			err <- errors.Errorf("failed start nbd server %s", res)
 		} else {
 			err <- nil
 		}
@@ -1322,6 +1578,12 @@ func (s *SKVMGuestInstance) guestRun(ctx context.Context) {
 			return
 		}
 
+		err = s.setupGuest()
+		if err != nil {
+			hostutils.TaskFailed(ctx, err.Error())
+			return
+		}
+
 		if s.hasVirtioBlkDriver() {
 			// virtio driver bind iothread, need migrate use driver mirror
 			nbdServerPort := s.manager.GetNBDServerFreePort()
@@ -1435,6 +1697,10 @@ func (s *SKVMGuestInstance) SlaveDisksBlockStream() error {
 }
 
 func (s *SKVMGuestInstance) releaseGuestCpuset() {
+	s.manager.cpuSet.Lock.Lock()
+	defer s.manager.cpuSet.Lock.Unlock()
+	s.releaseCpuNumaPin(s.Desc.CpuNumaPin)
+
 	for _, vcpuPin := range s.Desc.VcpuPin {
 		pcpuSet, err := cpuset.Parse(vcpuPin.Pcpus)
 		if err != nil {
@@ -1449,6 +1715,7 @@ func (s *SKVMGuestInstance) releaseGuestCpuset() {
 		s.manager.cpuSet.ReleaseCpus(pcpuSet.ToSlice(), vcpuSet.Size())
 	}
 	s.Desc.VcpuPin = nil
+	s.Desc.CpuNumaPin = nil
 	s.SaveLiveDesc(s.Desc)
 }
 
@@ -1460,6 +1727,7 @@ func (s *SKVMGuestInstance) clearCgroup(pid int) {
 	cgrupName := s.GetCgroupName()
 	log.Infof("cgroup destroy %d %s", pid, cgrupName)
 	if pid > 0 && !options.HostOptions.DisableSetCgroup {
+		s.CleanupCpuset()
 		cgrouputils.CgroupDestroy(strconv.Itoa(pid), cgrupName)
 	}
 }
@@ -1843,6 +2111,23 @@ func (s *SKVMGuestInstance) ExitCleanup(clear bool) {
 }
 
 func (s *SKVMGuestInstance) CleanupCpuset() {
+	cgPath := path.Join(cgrouputils.RootTaskPath("cpuset"), s.GetCgroupName())
+	cgName := s.GetCgroupName()
+	cgFiles, err := ioutil.ReadDir(cgPath)
+	if err != nil {
+		log.Warningf("failed read dir %s: %s", cgPath, err)
+	}
+	for _, fi := range cgFiles {
+		if !fi.IsDir() {
+			continue
+		}
+		subCgName := path.Join(cgName, fi.Name())
+		task := cgrouputils.NewCGroupCPUSetTask(strconv.Itoa(s.GetPid()), subCgName, 0, "")
+		if !task.RemoveTask() {
+			log.Warningf("remove cpuset cgroup error: %s, pid: %d", s.Id, s.GetPid())
+		}
+	}
+
 	task := cgrouputils.NewCGroupCPUSetTask(strconv.Itoa(s.GetPid()), s.GetCgroupName(), 0, "")
 	if !task.RemoveTask() {
 		log.Warningf("remove cpuset cgroup error: %s, pid: %d", s.Id, s.GetPid())
@@ -1969,6 +2254,8 @@ func (s *SKVMGuestInstance) scriptStart(ctx context.Context) error {
 		}
 		if err = s.StartMonitor(ctx, nil); err == nil {
 			return nil
+		} else {
+			log.Warningf("Guest %s failed start monitor %s", s.GetName(), err)
 		}
 		time.Sleep(time.Millisecond * 10)
 	}
@@ -2401,18 +2688,16 @@ func (s *SKVMGuestInstance) GetCgroupName() string {
 		return ""
 	}
 
-	if val, _ := s.Desc.Metadata["__enable_cgroup_cpuset"]; val == "true" {
-		return fmt.Sprintf("%s/server_%s_%d", hostconsts.HOST_CGROUP, s.Id, s.cgroupPid)
-	}
-
-	return ""
+	return fmt.Sprintf("%s/server_%s_%d", hostconsts.HOST_CGROUP, s.Id, s.cgroupPid)
 }
 
 func (s *SKVMGuestInstance) GuestPrelaunchSetCgroup() {
 	s.cgroupPid = s.GetPid()
 	s.setCgroupIo()
 	s.setCgroupCpu()
-	s.setCgroupCPUSet()
+	if err := s.setCgroupCPUSet(); err != nil {
+		log.Errorf("Guest %s failed set cgroup cpuset: %s", s.GetName(), err)
+	}
 }
 
 func (s *SKVMGuestInstance) setCgroupPid() {
@@ -2447,44 +2732,120 @@ func (s *SKVMGuestInstance) setCgroupCpu() {
 	cgrouputils.CgroupSet(strconv.Itoa(s.cgroupPid), s.GetCgroupName(), int(cpu)*cpuWeight)
 }
 
-func (s *SKVMGuestInstance) setCgroupCPUSet() {
-	var cpus []int
-	if cpuset, ok := s.Desc.Metadata[api.VM_METADATA_CGROUP_CPUSET]; ok {
-		cpusetJson, err := jsonutils.ParseString(cpuset)
-		if err != nil {
-			log.Errorf("failed parse server %s cpuset %s: %s", s.Id, cpuset, err)
-			return
+func (s *SKVMGuestInstance) setCgroupCPUSet() error {
+	if !s.IsRunning() {
+		return nil
+	}
+	cgName := s.GetCgroupName()
+	if cgName == "" {
+		return errors.Errorf("failed get cgroup name")
+	}
+
+	var cpusetStr string
+	if len(s.Desc.CpuNumaPin) == 0 {
+		cpus := []string{}
+		for _, vcpuPin := range s.Desc.VcpuPin {
+			cpus = append(cpus, vcpuPin.Pcpus)
 		}
-		input := new(api.ServerCPUSetInput)
-		err = cpusetJson.Unmarshal(input)
-		if err != nil {
-			log.Errorf("failed unmarshal server %s cpuset %s", s.Id, err)
-			return
-		}
-		cpus = input.CPUS
+		cpusetStr = strings.Join(cpus, ",")
 	} else {
-		cpus = s.allocGuestCpuset()
+		for i := range s.Desc.CpuNumaPin {
+			cpusetStr += fmt.Sprintf(",%s", *s.Desc.CpuNumaPin[i].Pcpus)
+		}
 	}
-	if _, err := s.CPUSet(context.Background(), cpus); err != nil {
-		log.Errorf("Do CPUSet error: %v", err)
-		return
+
+	guestPid := strconv.Itoa(s.GetPid())
+	// guest root cpuset group
+	task := cgrouputils.NewCGroupCPUSetTask(guestPid, cgName, 0, cpusetStr)
+	if !task.SetTask() {
+		return errors.Errorf("Cgroup cpuset task failed")
 	}
-	s.Desc.VcpuPin = []desc.CpuPin{
-		{
-			Vcpus: fmt.Sprintf("0-%d", s.Desc.Cpu-1),
-			Pcpus: cpuset.NewCPUSet(cpus...).String(),
-		},
+
+	vcpuThreads, err := s.getVcpuThreadIdMap(s.GetPid())
+	if err != nil {
+		return err
 	}
-	s.SaveLiveDesc(s.Desc)
+
+	for i := range s.Desc.CpuNumaPin {
+		if !s.Desc.CpuNumaPin[i].Regular || s.Desc.CpuNumaPin[i].Vcpus == nil {
+			continue
+		}
+
+		vcpuSet, _ := cpuset.Parse(*s.Desc.CpuNumaPin[i].Vcpus)
+		pcpuSet := *s.Desc.CpuNumaPin[i].Pcpus
+		for _, vcpuId := range vcpuSet.ToSlice() {
+			vcpuThreadId, ok := vcpuThreads[vcpuId]
+			if !ok {
+				return errors.Errorf("failed get vcpu %d thread id from %v", vcpuId, vcpuThreads)
+			}
+
+			vcpuCgname := path.Join(cgName, vcpuThreadId)
+			taskVcpu := cgrouputils.NewCGroupSubCPUSetTask(guestPid, vcpuCgname, 0, pcpuSet, []string{vcpuThreadId})
+			if !taskVcpu.SetTask() {
+				return errors.Errorf("Vcpu set cgroup cpuset task failed")
+			}
+		}
+	}
+	return nil
 }
 
-func (s *SKVMGuestInstance) allocGuestCpuset() []int {
-	var cpuset = []int{}
-	numaCpus := s.manager.cpuSet.AllocCpuset(int(s.Desc.Cpu))
-	for _, cpus := range numaCpus {
-		cpuset = append(cpuset, cpus...)
+func (s *SKVMGuestInstance) allocGuestNumaCpuset() error {
+	var cpus = make([]int, 0)
+	var cpuNumaPin = make([]*desc.SCpuNumaPin, 0)
+	var perferNumaNode int8 = -1
+	for i := range s.Desc.IsolatedDevices {
+		if s.Desc.IsolatedDevices[i].NumaNode >= 0 {
+			perferNumaNode = s.Desc.IsolatedDevices[i].NumaNode
+			break
+		}
 	}
-	return cpuset
+
+	nodeNumaCpus, err := s.manager.cpuSet.AllocCpuset(int(s.Desc.Cpu), s.Desc.Mem*1024, perferNumaNode)
+	if err != nil {
+		return err
+	}
+	log.Infof("alloc numa cpus %v", nodeNumaCpus)
+	for nodeId, numaCpus := range nodeNumaCpus {
+		if s.manager.numaAllocate {
+			unodeId := uint16(nodeId)
+			pcpus := cpuset.NewCPUSet(numaCpus.Cpuset...).String()
+			memPin := &desc.SCpuNumaPin{
+				SizeMB:    numaCpus.MemSizeKB / 1024, // MB
+				HostNodes: &unodeId,
+				Pcpus:     &pcpus,
+				Regular:   numaCpus.Regular,
+			}
+			cpuNumaPin = append(cpuNumaPin, memPin)
+		}
+
+		cpus = append(cpus, numaCpus.Cpuset...)
+	}
+	if len(cpuNumaPin) > 0 {
+		s.Desc.CpuNumaPin = cpuNumaPin
+	} else if !s.manager.numaAllocate {
+		if scpuset, ok := s.Desc.Metadata[api.VM_METADATA_CGROUP_CPUSET]; ok {
+			cpusetJson, err := jsonutils.ParseString(scpuset)
+			if err != nil {
+				log.Errorf("failed parse server %s cpuset %s: %s", s.Id, scpuset, err)
+				return errors.Errorf("failed parse server %s cpuset %s: %s", s.Id, scpuset, err)
+			}
+			input := new(api.ServerCPUSetInput)
+			err = cpusetJson.Unmarshal(input)
+			if err != nil {
+				log.Errorf("failed unmarshal server %s cpuset %s", s.Id, err)
+				return errors.Errorf("failed unmarshal server %s cpuset %s", s.Id, err)
+			}
+			cpus = input.CPUS
+		}
+
+		s.Desc.VcpuPin = []desc.SCpuPin{
+			{
+				Vcpus: fmt.Sprintf("0-%d", s.Desc.Cpu-1),
+				Pcpus: cpuset.NewCPUSet(cpus...).String(),
+			},
+		}
+	}
+	return nil
 }
 
 func (s *SKVMGuestInstance) CreateFromDesc(desc *desc.SGuestDesc) error {
@@ -2639,8 +3000,50 @@ func (s *SKVMGuestInstance) doBlockIoThrottle() {
 	}
 }
 
+func (s *SKVMGuestInstance) startHotPlugVcpus(vcpuSet []int) error {
+	var c = make(chan error)
+
+	for i := range vcpuSet {
+		if vcpuSet[i] == 0 {
+			// skip vcpu 0, added on qemu cmdline -smp 1
+			continue
+		}
+
+		s.Monitor.AddCpu(vcpuSet[i], func(res string) {
+			var e error = nil
+			if len(res) > 0 {
+				e = errors.Errorf("failed add cpu %d: %s", vcpuSet[i], res)
+			}
+			c <- e
+		})
+		if err, _ := <-c; err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *SKVMGuestInstance) hotPlugCpus() error {
+	var vcpuSet = make([]int, 0)
+	if len(s.Desc.MemDesc.Mem.Mems) > 0 {
+		for i := range s.Desc.CpuNumaPin {
+			vcpus, err := cpuset.Parse(*s.Desc.CpuNumaPin[i].Vcpus)
+			if err != nil {
+				return errors.Wrap(err, "parse vcpus")
+			}
+			vcpuSet = append(vcpuSet, vcpus.ToSlice()...)
+		}
+	}
+	return s.startHotPlugVcpus(vcpuSet)
+}
+
 func (s *SKVMGuestInstance) onGuestPrelaunch() error {
-	s.LiveMigrateDestPort = nil
+	if s.LiveMigrateDestPort == nil && !s.Desc.IsSlave {
+		if err := s.setupGuest(); err != nil {
+			return err
+		}
+	}
+
 	if !s.Desc.IsSlave {
 		if options.HostOptions.SetVncPassword {
 			s.SetVncPassword()
@@ -2651,10 +3054,18 @@ func (s *SKVMGuestInstance) onGuestPrelaunch() error {
 			}
 		}
 		s.OnResumeSyncMetadataInfo()
-		s.GuestPrelaunchSetCgroup()
-		s.optimizeOom()
 		s.doBlockIoThrottle()
 	}
+	s.LiveMigrateDestPort = nil
+	return nil
+}
+
+func (s *SKVMGuestInstance) setupGuest() error {
+	if err := s.hotPlugCpus(); err != nil {
+		return err
+	}
+	s.GuestPrelaunchSetCgroup()
+	s.optimizeOom()
 	return nil
 }
 
@@ -3135,6 +3546,42 @@ func (s *SKVMGuestInstance) CPUSet(ctx context.Context, input []int) (*api.Serve
 	return new(api.ServerCPUSetResp), nil
 }
 
+func (s *SKVMGuestInstance) getVcpuThreadIdMap(guestPid int) (map[int]string, error) {
+	tasksDir := fmt.Sprintf("/proc/%d/task", guestPid)
+	taskFiles, err := ioutil.ReadDir(tasksDir)
+	if err != nil {
+		return nil, errors.Wrapf(err, "read dir %s", tasksDir)
+	}
+
+	var vcpuThreads = map[int]string{}
+	for i := range taskFiles {
+		if !taskFiles[i].IsDir() {
+			log.Warningf("task %s/%s is not dir?", tasksDir, taskFiles[i].Name())
+			continue
+		}
+		_, err := strconv.Atoi(taskFiles[i].Name())
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed parse thread id %s", taskFiles[i].Name())
+		}
+
+		// vcpu thread comm eg: CPU 0/KVM
+		taskComm := path.Join(tasksDir, taskFiles[i].Name(), "comm")
+		if cmd, err := fileutils2.FileGetContents(taskComm); err != nil {
+			return nil, errors.Wrapf(err, "failed get task %s command", taskComm)
+		} else {
+			cmd = strings.TrimSpace(cmd)
+			if strings.HasPrefix(cmd, "CPU ") && strings.HasSuffix(cmd, "/KVM") {
+				vcpuId, err := strconv.Atoi(cmd[4 : len(cmd)-4])
+				if err != nil {
+					return nil, errors.Wrapf(err, "failed parse %s to int", cmd)
+				}
+				vcpuThreads[vcpuId] = taskFiles[i].Name()
+			}
+		}
+	}
+	return vcpuThreads, nil
+}
+
 func (s *SKVMGuestInstance) CPUSetRemove(ctx context.Context) error {
 	delete(s.Desc.Metadata, api.VM_METADATA_CGROUP_CPUSET)
 	if err := s.SaveLiveDesc(s.Desc); err != nil {

pkg/hostman/guestman/qemu-kvmhelper.go

@@ -886,13 +886,20 @@ func (s *SKVMGuestInstance) initCpuDesc(cpuMax uint) error {
 		return err
 	}
 	s.Desc.CpuDesc = cpuDesc
+
+	err = s.allocGuestNumaCpuset()
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 
-func (s *SKVMGuestInstance) initMemDesc(memSizeMB int64) {
+func (s *SKVMGuestInstance) initMemDesc(memSizeMB int64) error {
 	s.Desc.MemDesc = s.archMan.GenerateMemDesc()
 	s.Desc.MemDesc.SizeMB = memSizeMB
-	s.initDefaultMemObject(memSizeMB)
+
+	return s.initGuestMemObjects(memSizeMB)
 }
 
 func (s *SKVMGuestInstance) memObjectType() string {
@@ -905,24 +912,88 @@ func (s *SKVMGuestInstance) memObjectType() string {
 	}
 }
 
-func (s *SKVMGuestInstance) initDefaultMemObject(memSizeMB int64) {
-	s.Desc.MemDesc.Mem = desc.NewObject(s.memObjectType(), "mem")
+func (s *SKVMGuestInstance) initGuestMemObjects(memSizeMB int64) error {
+	if len(s.Desc.CpuNumaPin) == 0 {
+		s.initDefaultMemObject(memSizeMB)
+		return nil
+	}
+
+	var numaMems int64
+	var numaCpus = int(s.Desc.CpuDesc.MaxCpus) / len(s.Desc.CpuNumaPin)
+	var leastCpus = int(s.Desc.CpuDesc.MaxCpus) % len(s.Desc.CpuNumaPin)
+	var numaVcpuCount = int(s.Desc.Cpu) / len(s.Desc.CpuNumaPin)
+	var mems = make([]desc.SMemDesc, 0)
+	for i := 0; i < len(s.Desc.CpuNumaPin); i++ {
+		numaMems += s.Desc.CpuNumaPin[i].SizeMB
+		memId := "mem"
+		nodeId := uint16(i)
+		if i > 0 {
+			memId += strconv.Itoa(i - 1)
+		}
+
+		vcpuCount := numaVcpuCount
+		if i == 0 {
+			vcpuCount += int(s.Desc.Cpu) % len(s.Desc.CpuNumaPin)
+		}
+
+		cpuStart := i * numaCpus
+		cpuEnd := (i+1)*numaCpus - 1
+		if i+1 == len(s.Desc.CpuNumaPin) {
+			cpuEnd += leastCpus
+		}
+		vcpus := fmt.Sprintf("%d-%d", cpuStart, cpuEnd)
+		vcpuAlloc := fmt.Sprintf("%d-%d", cpuStart, cpuStart+vcpuCount-1)
+		s.Desc.CpuNumaPin[i].Vcpus = &vcpuAlloc
+
+		if !s.Desc.CpuNumaPin[i].Regular {
+			continue
+		}
+		memDesc := desc.NewMemDesc(s.memObjectType(), memId, &nodeId, &vcpus)
+		memDesc.Options = s.getMemObjectOptions(s.Desc.CpuNumaPin[i].SizeMB, s.Desc.Uuid, s.Desc.CpuNumaPin[i].HostNodes)
+		mems = append(mems, *memDesc)
+	}
+	if len(mems) == 0 {
+		// numa mems not regular
+		s.initDefaultMemObject(memSizeMB)
+		return nil
+	}
+
+	if numaMems != memSizeMB {
+		return errors.Errorf("numa memory size not equal request mem size")
+	}
+	s.Desc.MemDesc.Mem = desc.NewMemsDesc(mems[0], mems[1:])
+	return nil
+}
+
+func (s *SKVMGuestInstance) getMemObjectOptions(memSizeMB int64, memPathSuffix string, hostNodes *uint16) map[string]string {
+	var opts map[string]string
 	if s.manager.host.IsHugepagesEnabled() {
-		s.Desc.MemDesc.Mem.Options = map[string]string{
-			"mem-path": fmt.Sprintf("/dev/hugepages/%s", s.Desc.Uuid),
+		opts = map[string]string{
+			"mem-path": fmt.Sprintf("/dev/hugepages/%s", memPathSuffix),
 			"size":     fmt.Sprintf("%dM", memSizeMB),
 			"share":    "on", "prealloc": "on",
 		}
+		if hostNodes != nil {
+			opts["host-nodes"] = fmt.Sprintf("%d", *hostNodes)
+			opts["policy"] = "bind"
+		}
 	} else if s.isMemcleanEnabled() {
-		s.Desc.MemDesc.Mem.Options = map[string]string{
+		opts = map[string]string{
 			"size":  fmt.Sprintf("%dM", memSizeMB),
 			"share": "on", "prealloc": "on",
 		}
 	} else {
-		s.Desc.MemDesc.Mem.Options = map[string]string{
+		opts = map[string]string{
 			"size": fmt.Sprintf("%dM", memSizeMB),
 		}
 	}
+	return opts
+}
+
+func (s *SKVMGuestInstance) initDefaultMemObject(memSizeMB int64) {
+	defaultDesc := desc.NewMemDesc(s.memObjectType(), "mem", nil, nil)
+	defaultDesc.Options = s.getMemObjectOptions(memSizeMB, s.Desc.Uuid, nil)
+	s.Desc.MemDesc.Mem = desc.NewMemsDesc(*defaultDesc, nil)
 }
 
 func (s *SKVMGuestInstance) defaultMemNodeHasObject(memDevs []monitor.Memdev) bool {
@@ -944,7 +1015,7 @@ func (s *SKVMGuestInstance) initMemDescFromMemoryInfo(
 			return errors.Errorf("unsupported memory device type %s", memoryDevicesInfoList[i].Type)
 		}
 		memSize -= (memoryDevicesInfoList[i].Data.Size / 1024 / 1024)
-		memObj := desc.NewObject(s.memObjectType(), path.Base(memoryDevicesInfoList[i].Data.Memdev))
+		memObj := desc.NewMemDesc(s.memObjectType(), path.Base(memoryDevicesInfoList[i].Data.Memdev), nil, nil)
 		memObj.Options = map[string]string{
 			"size": fmt.Sprintf("%dM", memoryDevicesInfoList[i].Data.Size/1024/1024),
 		}

pkg/hostman/guestman/qemu/generate.go

@@ -136,8 +136,25 @@ func generatePciControllerOptions(controllers []*desc.PCIController) []string {
 	return opts
 }
 
-func generateNumaOption(memId string) string {
-	return fmt.Sprintf("-numa node,memdev=%s", memId)
+func generateNumaOption(memId string, nodeId *uint16, cpus *string) string {
+	cmd := fmt.Sprintf("-numa node,memdev=%s", memId)
+	if nodeId != nil {
+		cmd += fmt.Sprintf(",nodeid=%d", *nodeId)
+	}
+	if cpus != nil {
+		cpuSegs := strings.Split(*cpus, ",")
+		for _, cpuSeg := range cpuSegs {
+			cmd += fmt.Sprintf(",cpus=%s", cpuSeg)
+		}
+	}
+	return cmd
+}
+
+func generateMemObjectWithNumaOptions(mem *desc.SMemDesc) string {
+	cmds := []string{}
+	cmds = append(cmds, generateObjectOption(mem.Object))
+	cmds = append(cmds, generateNumaOption(mem.Id, mem.NodeId, mem.Cpus))
+	return strings.Join(cmds, " ")
 }
 
 func generateMemoryOption(memDesc *desc.SGuestMem) string {
@@ -147,13 +164,15 @@ func generateMemoryOption(memDesc *desc.SGuestMem) string {
 		memDesc.SizeMB, memDesc.Slots, memDesc.MaxMem,
 	))
 	if memDesc.Mem != nil {
-		cmds = append(cmds, generateObjectOption(memDesc.Mem))
-		cmds = append(cmds, generateNumaOption(memDesc.Mem.Id))
+		cmds = append(cmds, generateMemObjectWithNumaOptions(&memDesc.Mem.SMemDesc))
+		for i := range memDesc.Mem.Mems {
+			cmds = append(cmds, generateMemObjectWithNumaOptions(&memDesc.Mem.Mems[i]))
+		}
 	}
 	for i := 0; i < len(memDesc.MemSlots); i++ {
 		memDev := memDesc.MemSlots[i].MemDev
 		memObj := memDesc.MemSlots[i].MemObj
-		cmds = append(cmds, generateObjectOption(memObj))
+		cmds = append(cmds, generateObjectOption(memObj.Object))
 		cmds = append(cmds, fmt.Sprintf("-device %s,id=%s,memdev=%s", memDev.Type, memDev.Id, memObj.Id))
 	}
 	return strings.Join(cmds, " ")
@@ -168,15 +187,20 @@ func generateMachineOption(machine string, machineDesc *desc.SGuestMachine) stri
 	return cmd
 }
 
-func generateSMPOption(cpu *desc.SGuestCpu) string {
+func generateSMPOption(guestDesc *desc.SGuestDesc) string {
+	cpu := guestDesc.CpuDesc
+	startCpus := cpu.Cpus
+	if len(guestDesc.MemDesc.Mem.Mems) > 0 {
+		startCpus = 1
+	}
 	if cpu.MaxCpus%2 > 0 {
 		return fmt.Sprintf(
-			"-smp cpus=%d,maxcpus=%d", cpu.Cpus, cpu.MaxCpus,
+			"-smp cpus=%d,maxcpus=%d", startCpus, cpu.MaxCpus,
 		)
 	} else {
 		return fmt.Sprintf(
 			"-smp cpus=%d,sockets=%d,cores=%d,maxcpus=%d",
-			cpu.Cpus, cpu.Sockets, cpu.Cores, cpu.MaxCpus,
+			startCpus, cpu.Sockets, cpu.Cores, cpu.MaxCpus,
 		)
 	}
 }
@@ -700,7 +724,7 @@ func GenerateStartOptions(
 		drvOpt.Global(),
 		generateMachineOption(input.GuestDesc.Machine, input.GuestDesc.MachineDesc),
 		drvOpt.KeyboardLayoutLanguage("en-us"),
-		generateSMPOption(input.GuestDesc.CpuDesc),
+		generateSMPOption(input.GuestDesc),
 		drvOpt.Name(input.GuestDesc.Name),
 		drvOpt.UUID(input.EnableUUID, input.GuestDesc.Uuid),
 		generateMemoryOption(input.GuestDesc.MemDesc),

pkg/hostman/host_services.go

@@ -92,7 +92,10 @@ func (host *SHostService) RunService() {
 	}
 
 	var guestChan chan struct{}
-	guestman.Init(hostInstance, options.HostOptions.ServersPath)
+
+	if err := guestman.Init(hostInstance, options.HostOptions.ServersPath); err != nil {
+		log.Fatalf("guest manager init error: %s", err)
+	}
 	guestman.GetGuestManager().InitQemuMaxCpus(
 		hostInstance.GetQemuMachineInfoList(), hostInstance.GetKVMMaxCpus(),
 	)
@@ -103,7 +106,11 @@ func (host *SHostService) RunService() {
 
 	hostInstance.StartRegister(2)
 	// <-hostinfo.Instance().IsRegistered // wait host and guest init
-	guestChan = guestman.GetGuestManager().Bootstrap()
+	var err error
+	guestChan, err = guestman.GetGuestManager().Bootstrap()
+	if err != nil {
+		log.Fatalf("Guest manager Bootstrap %s", err)
+	}
 	// hostmetrics after guestmanager bootstrap
 	hostmetrics.Init()
 	hostmetrics.Start()

pkg/hostman/hostinfo/hostinfo.go

@@ -91,9 +91,10 @@ type SHostInfo struct {
 
 	kubeletConfig kubelet.KubeletConfig
 
-	isInit           bool
-	onHostDown       string
-	reservedCpusInfo *api.HostReserveCpusInput
+	isInit             bool
+	onHostDown         string
+	reservedCpusInfo   *api.HostReserveCpusInput
+	enableNumaAllocate bool
 
 	IsolatedDeviceMan isolated_device.IsolatedDeviceManager
 
@@ -1157,6 +1158,12 @@ func (h *SHostInfo) initHostRecord() (*api.HostDetails, error) {
 		return nil, errors.Wrap(err, "parse reserved cpus info")
 	}
 
+	// enable numa allocate
+	if hostInfo.EnableNumaAllocate {
+		h.enableNumaAllocate = true
+		log.Infof("host enabled numa allocate")
+	}
+
 	// set host reserved memory
 	if h.IsHugepagesEnabled() && h.getReservedMemMb() != hostInfo.MemReserved {
 		if err = h.updateHostReservedMem(h.getReservedMemMb()); err != nil {
@@ -2416,6 +2423,10 @@ func (h *SHostInfo) GetReservedCpusInfo() *cpuset.CPUSet {
 	return &cpus
 }
 
+func (h *SHostInfo) IsNumaAllocateEnabled() bool {
+	return h.enableNumaAllocate
+}
+
 func NewHostInfo() (*SHostInfo, error) {
 	var res = new(SHostInfo)
 	res.sysinfo = &SSysInfo{}

pkg/hostman/hostutils/hostutils.go

@@ -56,6 +56,7 @@ type IHost interface {
 
 	IsHugepagesEnabled() bool
 	HugepageSizeKb() int
+	IsNumaAllocateEnabled() bool
 
 	IsKvmSupport() bool
 	IsNestedVirtualization() bool

pkg/hostman/isolated_device/isolated_device.go

@@ -29,6 +29,7 @@ import (
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
@@ -88,6 +89,7 @@ type IDevice interface {
 	GetVGACmd() string
 	GetCPUCmd() string
 	GetQemuId() string
+	GetNumaNode() (int, error)
 
 	// sriov nic
 	GetPfName() string
@@ -503,6 +505,15 @@ func (dev *sBaseDevice) GetVirtfn() int {
 	return -1
 }
 
+func (dev *sBaseDevice) GetNumaNode() (int, error) {
+	numaNodePath := fmt.Sprintf("/sys/bus/pci/devices/0000:%s/numa_node", dev.GetAddr())
+	numaNode, err := fileutils2.FileGetIntContent(numaNodePath)
+	if err != nil {
+		return -1, errors.Wrap(err, "get device numa node")
+	}
+	return numaNode, nil
+}
+
 func (dev *sBaseDevice) GetOvsOffloadInterfaceName() string {
 	return ""
 }
@@ -568,6 +579,11 @@ func GetApiResourceData(dev IDevice) *jsonutils.JSONDict {
 	if dev.GetNVMESizeMB() > 0 {
 		data["nvme_size_mb"] = dev.GetNVMESizeMB()
 	}
+	if numaNode, err := dev.GetNumaNode(); err == nil {
+		data["numa_node"] = numaNode
+	} else {
+		log.Errorf("failed get dev %s numa node %s", dev.GetAddr(), err)
+	}
 
 	if dev.GetMdevId() != "" {
 		data["mdev_id"] = dev.GetMdevId()

pkg/hostman/isolated_device/nvidia_vgpu.go

@@ -164,6 +164,15 @@ func (dev *sNVIDIAVgpuDevice) GetQemuId() string {
 	return "dev_" + dev.mdevId
 }
 
+func (dev *sNVIDIAVgpuDevice) GetNumaNode() (int, error) {
+	numaNodePath := fmt.Sprintf("/sys/bus/pci/devices/0000:%s/numa_node", dev.GetAddr())
+	numaNode, err := fileutils2.FileGetIntContent(numaNodePath)
+	if err != nil {
+		return -1, errors.Wrap(err, "get device numa node")
+	}
+	return numaNode, nil
+}
+
 func (dev *sNVIDIAVgpuDevice) GetHotPlugOptions(isolatedDev *desc.SGuestIsolatedDevice, guestDesc *desc.SGuestDesc) ([]*HotPlugOption, error) {
 	ret := make([]*HotPlugOption, 0)
 

pkg/util/cgrouputils/cgrouputils.go

@@ -61,18 +61,20 @@ type ICGroupTask interface {
 }
 
 type CGroupTask struct {
-	pid    string
-	name   string
-	weight float64
+	pid       string
+	threadIds []string
+	name      string
+	weight    float64
 
 	hand ICGroupTask
 }
 
-func NewCGroupTask(pid, name string, coreNum int) *CGroupTask {
+func NewCGroupTask(pid, name string, coreNum int, threadIds []string) *CGroupTask {
 	return &CGroupTask{
-		pid:    pid,
-		name:   name,
-		weight: float64(coreNum) / normalizeBase,
+		pid:       pid,
+		name:      name,
+		weight:    float64(coreNum) / normalizeBase,
+		threadIds: threadIds,
 	}
 }
 
@@ -124,6 +126,7 @@ func GetRootParam(module, name, pid string) string {
 	return strings.TrimSpace(param)
 }
 
+// cpuset, task, tid, cgname
 func SetRootParam(module, name, value, pid string) bool {
 	param := GetRootParam(module, name, pid)
 	if param != value {
@@ -140,8 +143,16 @@ func SetRootParam(module, name, value, pid string) bool {
 }
 
 // cleanup
-func CleanupNonexistPids(module string) {
+func CleanupNonexistPids(module string, subName string) {
 	var root = RootTaskPath(module)
+	cleanNonexitPidsWithRoot(root)
+	if subName != "" {
+		root = path.Join(RootTaskPath(module), subName)
+		cleanNonexitPidsWithRoot(root)
+	}
+}
+
+func cleanNonexitPidsWithRoot(root string) {
 	files, err := ioutil.ReadDir(root)
 	if err != nil {
 		log.Errorf("GetTaskIds failed: %s", err)
@@ -151,11 +162,35 @@ func CleanupNonexistPids(module string) {
 	for _, file := range files {
 		ids = append(ids, file.Name())
 	}
-	re := regexp.MustCompile(`^\d+$`)
+
+	re1 := regexp.MustCompile(`^\d+$`)
+	re2 := regexp.MustCompile(`^server_[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}_\d+$`)
 	for _, pid := range ids {
-		if re.MatchString(pid) && fileutils2.IsDir(path.Join(root, pid)) {
-			if !fileutils2.Exists(path.Join("/proc", pid)) {
+		spid := ""
+		if re1.MatchString(pid) {
+			spid = pid
+		} else if re2.MatchString(pid) {
+			segs := strings.Split(pid, "_")
+			spid = segs[len(segs)-1]
+		}
+
+		if fileutils2.IsDir(path.Join(root, pid)) {
+			if !fileutils2.Exists(path.Join("/proc", spid)) {
 				log.Infof("Cgroup clenup %s", pid)
+
+				subFiles, err := ioutil.ReadDir(path.Join(root, pid))
+				if err != nil {
+					log.Errorf("sub dir %s GetTaskIds failed: %s", path.Join(root, pid), err)
+				} else {
+					for _, fi := range subFiles {
+						if !fi.IsDir() {
+							continue
+						}
+						if err := os.Remove(path.Join(root, pid, fi.Name())); err != nil {
+							log.Errorf("CleanupNonexistPids pid=%s tid=%s error: %s", pid, fi.Name(), err)
+						}
+					}
+				}
 				if err := os.Remove(path.Join(root, pid)); err != nil {
 					log.Errorf("CleanupNonexistPids pid=%s error: %s", pid, err)
 				}
@@ -211,6 +246,10 @@ func (c *CGroupTask) GetTaskIds() []string {
 		return nil
 	}
 
+	if len(c.threadIds) > 0 {
+		return c.threadIds
+	}
+
 	files, err := ioutil.ReadDir(fmt.Sprintf("/proc/%s/task", c.pid))
 	if err != nil {
 		log.Errorf("GetTaskIds failed: %s", err)
@@ -258,7 +297,7 @@ func (c *CGroupTask) MoveTasksToRoot() {
 func (c *CGroupTask) RemoveTask() bool {
 	if c.taskIsExist() {
 		c.MoveTasksToRoot()
-		log.Infof("Remove task path %s", c.TaskPath())
+		log.Infof("Remove task path %s %s", c.TaskPath(), c.name)
 		if err := os.Remove(c.TaskPath()); err != nil {
 			log.Errorf("Remove task path failed %s", err)
 			return false
@@ -330,12 +369,12 @@ func (c *CGroupTask) SetTask() bool {
 	return false
 }
 
-func (c *CGroupTask) PushPid(pid string, isRoot bool) {
+func (c *CGroupTask) PushPid(tid string, isRoot bool) {
 	if c.pid == "" {
 		return
 	}
 
-	subdir := fmt.Sprintf("/proc/%s/task/%s", c.pid, pid)
+	subdir := fmt.Sprintf("/proc/%s/task/%s", c.pid, tid)
 	if fi, err := os.Stat(subdir); err != nil {
 		log.Errorf("Fail to put pid in task %s", err)
 		return
@@ -349,9 +388,9 @@ func (c *CGroupTask) PushPid(pid string, isRoot bool) {
 		data := re.Split(stat, -1)
 		if data[2] != "Z" {
 			if isRoot {
-				SetRootParam(c.hand.Module(), CGROUP_TASKS, pid, "")
+				SetRootParam(c.hand.Module(), CGROUP_TASKS, tid, "")
 			} else {
-				c.SetParam(CGROUP_TASKS, pid)
+				c.SetParam(CGROUP_TASKS, tid)
 			}
 		}
 	}
@@ -441,7 +480,7 @@ func (c *CGroupCPUTask) init() bool {
 }
 
 func NewCGroupCPUTask(pid, name string, coreNum int) CGroupCPUTask {
-	cgroup := CGroupCPUTask{NewCGroupTask(pid, name, coreNum)}
+	cgroup := CGroupCPUTask{NewCGroupTask(pid, name, coreNum, nil)}
 	cgroup.hand = &cgroup
 	return cgroup
 }
@@ -504,7 +543,7 @@ func (c *CGroupIOTask) init() bool {
 }
 
 func NewCGroupIOTask(pid, name string, coreNum int) *CGroupIOTask {
-	task := &CGroupIOTask{NewCGroupTask(pid, name, coreNum)}
+	task := &CGroupIOTask{NewCGroupTask(pid, name, coreNum, nil)}
 	task.SetHand(task)
 	return task
 }
@@ -566,7 +605,7 @@ func (c *CGroupMemoryTask) GetConfig() map[string]string {
 
 func NewCGroupMemoryTask(pid, name string, coreNum int) *CGroupMemoryTask {
 	task := &CGroupMemoryTask{
-		CGroupTask: NewCGroupTask(pid, name, coreNum),
+		CGroupTask: NewCGroupTask(pid, name, coreNum, nil),
 	}
 	task.SetHand(task)
 	return task
@@ -610,7 +649,16 @@ func (c *CGroupCPUSetTask) CustomConfig(key, value string) bool {
 
 func NewCGroupCPUSetTask(pid, name string, coreNum int, cpuset string) CGroupCPUSetTask {
 	task := CGroupCPUSetTask{
-		CGroupTask: NewCGroupTask(pid, name, coreNum),
+		CGroupTask: NewCGroupTask(pid, name, coreNum, nil),
+		cpuset:     cpuset,
+	}
+	task.SetHand(&task)
+	return task
+}
+
+func NewCGroupSubCPUSetTask(pid, name string, coreNum int, cpuset string, threadIds []string) CGroupCPUSetTask {
+	task := CGroupCPUSetTask{
+		CGroupTask: NewCGroupTask(pid, name, coreNum, threadIds),
 		cpuset:     cpuset,
 	}
 	task.SetHand(&task)
@@ -655,7 +703,7 @@ func CgroupDestroy(pid, name string) bool {
 		&CGroupCPUTask{&CGroupTask{}},
 		&CGroupIOTask{&CGroupTask{}},
 		&CGroupMemoryTask{&CGroupTask{}},
-		&CGroupCPUSetTask{&CGroupTask{}, ""},
+		//&CGroupCPUSetTask{&CGroupTask{}, ""},
 		&CGroupIOHardlimitTask{CGroupIOTask: &CGroupIOTask{&CGroupTask{}}},
 	}
 	for _, hand := range tasks {
@@ -667,7 +715,7 @@ func CgroupDestroy(pid, name string) bool {
 	return true
 }
 
-func CgroupCleanAll() {
+func CgroupCleanAll(subName string) {
 	tasks := []ICGroupTask{
 		&CGroupCPUTask{&CGroupTask{}},
 		&CGroupIOTask{&CGroupTask{}},
@@ -677,6 +725,6 @@ func CgroupCleanAll() {
 	}
 	for _, hand := range tasks {
 		hand.SetHand(hand)
-		CleanupNonexistPids(hand.Module())
+		CleanupNonexistPids(hand.Module(), subName)
 	}
 }

pkg/util/cgrouputils/cgrouputils_test.go

@@ -29,5 +29,5 @@ func TestCgroupSet(t *testing.T) {
 	pid = strings.TrimSpace(pid)
 	t.Logf("Start %s cgroup set", pid)
 	CgroupSet(pid, "", 1)
-	CgroupCleanAll()
+	CgroupCleanAll("")
 }

pkg/util/fileutils2/fileutils.go

@@ -23,6 +23,7 @@ import (
 	"path"
 	"path/filepath"
 	"regexp"
+	"strconv"
 	"strings"
 
 	"yunion.io/x/log"
@@ -220,6 +221,18 @@ func FileGetContents(file string) (string, error) {
 	return string(content), nil
 }
 
+func FileGetIntContent(file string) (int, error) {
+	content, err := FileGetContents(file)
+	if err != nil {
+		return -1, errors.Wrap(err, "FileGetContents")
+	}
+	val, err := strconv.Atoi(strings.TrimSpace(content))
+	if err != nil {
+		return -1, errors.Wrapf(err, "convert %s to int", content)
+	}
+	return val, nil
+}
+
 func GetFsFormat(diskPath string) string {
 	ret, err := procutils.NewCommand("blkid", "-o", "value", "-s", "TYPE", diskPath).Output()
 	if err != nil {