fix(region): virtual isolated device balance on guest start (#23049)

cmd/climc/shell/compute/hosts.go

@@ -72,6 +72,7 @@ func init() {
 	cmd.Get("ipmi", &options.BaseIdOptions{})
 	cmd.Get("vnc", &options.BaseIdOptions{})
 	cmd.Get("app-options", &options.BaseIdOptions{})
+	cmd.Get("isolated-device-numa-stats", &compute.HostIsolatedDeviceNumaStatsOptions{})
 	cmd.GetWithCustomShow("worker-stats", func(data jsonutils.JSONObject) {
 		stats, _ := data.GetArray("workers")
 		printList(&printutils.ListResult{Data: stats}, nil)

cmd/climc/shell/compute/servers.go

@@ -124,6 +124,7 @@ func init() {
 	cmd.Perform("set-nic-traffic-limit", &options.ServerNicTrafficLimitOptions{})
 	cmd.Perform("add-sub-ips", &options.ServerAddSubIpsOptions{})
 	cmd.Perform("update-sub-ips", &options.ServerUpdateSubIpsOptions{})
+	cmd.BatchPerform("restore-virtual-isolated-devices", &options.ServerIdsOptions{})
 	cmd.BatchPerform("set-os-info", &options.ServerSetOSInfoOptions{})
 	cmd.BatchPerform("start-rescue", &options.ServerStartOptions{})
 	cmd.BatchPerform("stop-rescue", &options.ServerStartOptions{})

pkg/apis/compute/guest_const.go

@@ -337,6 +337,8 @@ const (
 	VM_METADATA_RELEASED_DEVICES = "released_devices"
 
 	VM_METADATA_CPU_NUMA_PIN = "__cpu_numa_pin"
+
+	VM_METADATA_VIRTUAL_ISOLATED_DEVICE_CONFIG = "__virtual_isolated_device_config"
 )
 
 // windows allow a maximal length of 15

pkg/apis/compute/host.go

@@ -760,6 +760,10 @@ type HostUploadGuestsStatusInput struct {
 	Guests map[string]*HostUploadGuestStatusInput `json:"guests"`
 }
 
+type HostIsolatedDeviceNumaStatsInput struct {
+	DevType string
+}
+
 type GuestUploadContainerStatusResponse struct {
 	Error string `json:"error"`
 	OK    bool   `json:"ok"`

pkg/apis/compute/isolated_device_const.go

@@ -51,6 +51,7 @@ var VITRUAL_DEVICE_TYPES = []string{
 	CONTAINER_DEV_NVIDIA_MPS,
 	CONTAINER_DEV_ASCEND_NPU,
 	CONTAINER_DEV_NVIDIA_GPU_SHARE,
+	CONTAINER_DEV_VASTAITECH_GPU,
 }
 
 var ID_VENDOR_MAP = map[string]string{

pkg/cloudcommon/options/options.go

@@ -125,7 +125,8 @@ type BaseOptions struct {
 	GlobalHTTPProxy  string `help:"Global http proxy"`
 	GlobalHTTPSProxy string `help:"Global https proxy"`
 
-	IgnoreNonrunningGuests bool `default:"true" help:"Count memory for running guests only when do scheduling. Ignore memory allocation for non-running guests"`
+	IgnoreNonrunningGuests   bool `default:"true" help:"Count memory for running guests only when do scheduling. Ignore memory allocation for non-running guests"`
+	VirtualDeviceNumaBalance bool `default:"true" help:"fix virtual device numa node balance on guest start"`
 
 	PlatformName  string            `help:"identity name of this platform" default:"Cloudpods"`
 	PlatformNames map[string]string `help:"identity name of this platform by language"`

pkg/compute/guestdrivers/pod.go

@@ -602,10 +602,7 @@ func (p *SPodDriver) BeforeDetachIsolatedDevice(ctx context.Context, userCred mc
 		newDevs := make([]*api.ContainerDevice, 0)
 		releasedDevs := make(map[string]models.ContainerReleasedDevice)
 		for _, curDev := range devs {
-			if curDev.IsolatedDevice == nil {
-				continue
-			}
-			if curDev.IsolatedDevice.Id != dev.GetId() {
+			if curDev.IsolatedDevice == nil || curDev.IsolatedDevice.Id != dev.GetId() {
 				tmpDev := curDev
 				newDevs = append(newDevs, tmpDev)
 			} else {

pkg/compute/models/guest_actions.go

@@ -1128,6 +1128,61 @@ func (self *SGuest) StartResumeTask(ctx context.Context, userCred mcclient.Token
 	return driver.StartResumeTask(ctx, userCred, self, nil, parentTaskId)
 }
 
+func (self *SGuest) PerformRestoreVirtualIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	config := self.GetMetadataJson(ctx, api.VM_METADATA_VIRTUAL_ISOLATED_DEVICE_CONFIG, userCred)
+	if config == nil {
+		return nil, nil
+	}
+	devConfigs := make([]api.IsolatedDeviceConfig, 0)
+	err := config.Unmarshal(&devConfigs)
+	if err != nil {
+		return nil, errors.Wrap(err, "unmarshal virtual dev configs")
+	}
+	devs, err := self.GetIsolatedDevices()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetIsolatedDevices")
+	}
+	devCount := map[string]int{}
+	for i := range devConfigs {
+		key := devConfigs[i].DevType + "-" + devConfigs[i].Model
+		if cnt, ok := devCount[key]; ok {
+			devCount[key] = cnt + 1
+		} else {
+			devCount[key] = 1
+		}
+	}
+	for i := range devs {
+		key := devConfigs[i].DevType + "-" + devConfigs[i].Model
+		if cnt, ok := devCount[key]; ok {
+			devCount[key] = cnt - 1
+		}
+	}
+
+	host, err := self.GetHost()
+	if err != nil {
+		return nil, errors.Wrap(err, "get host")
+	}
+	lockman.LockObject(ctx, host)
+	defer lockman.ReleaseObject(ctx, host)
+
+	usedDeviceMap := map[string]*SIsolatedDevice{}
+	for key, cnt := range devCount {
+		if cnt <= 0 {
+			continue
+		}
+		segs := strings.SplitN(key, "-", 2)
+		devConfig := &api.IsolatedDeviceConfig{
+			Model:   segs[1],
+			DevType: segs[0],
+		}
+		err := IsolatedDeviceManager.attachHostDeviceToGuestByModel(ctx, self, host, devConfig, userCred, usedDeviceMap, nil)
+		if err != nil {
+			return nil, errors.Wrap(err, "attachHostDeviceToGuestByModel")
+		}
+	}
+	return nil, nil
+}
+
 // 开机
 func (self *SGuest) PerformStart(
 	ctx context.Context,
@@ -1560,6 +1615,68 @@ func (self *SGuest) StartInsertVfdTask(ctx context.Context, floppyOrdinal int64,
 	return nil
 }
 
+func (self *SGuest) RebalanceVirtualIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential) error {
+	devs, err := self.GetIsolatedDevices()
+	if err != nil {
+		return errors.Wrap(err, "guest get isolated devices")
+	}
+	if len(devs) == 0 {
+		return nil
+	}
+	host, err := self.GetHost()
+	if err != nil {
+		return errors.Wrap(err, "guest get host")
+	}
+	var numaNodeBalance = true
+	detachDevs := make([]SIsolatedDevice, 0)
+	originDevConfigs := make([]api.IsolatedDeviceConfig, 0)
+	for i := range devs {
+		if utils.IsInStringArray(devs[i].DevType, api.VITRUAL_DEVICE_TYPES) {
+			originDevConfigs = append(originDevConfigs, api.IsolatedDeviceConfig{
+				Model:   devs[i].Model,
+				DevType: devs[i].DevType,
+			})
+			detachDevs = append(detachDevs, devs[i])
+			isBalance, err := host.VirtualDeviceNumaBalance(devs[i].DevType, devs[i].NumaNode)
+			if err != nil {
+				return errors.Wrap(err, "VirtualDeviceNumaBalance")
+			}
+			if numaNodeBalance && !isBalance {
+				numaNodeBalance = false
+			}
+		}
+	}
+	log.Infof("Guest %s on host %s start virtual devices numa node balance %v", self.Id, host.Id, numaNodeBalance)
+	if !numaNodeBalance {
+		err := self.SetMetadata(ctx, api.VM_METADATA_VIRTUAL_ISOLATED_DEVICE_CONFIG, originDevConfigs, userCred)
+		if err != nil {
+			return errors.Wrap(err, "set metadata virtual isolated device config")
+		}
+		lockman.LockObject(ctx, host)
+		defer lockman.ReleaseObject(ctx, host)
+
+		for i := 0; i < len(detachDevs); i++ {
+			err := self.detachIsolateDevice(ctx, userCred, &detachDevs[i])
+			if err != nil {
+				return errors.Wrapf(err, "detach device %s", detachDevs[i].GetId())
+			}
+		}
+
+		usedDeviceMap := map[string]*SIsolatedDevice{}
+		for i := range detachDevs {
+			devConfig := &api.IsolatedDeviceConfig{
+				Model:   detachDevs[i].Model,
+				DevType: detachDevs[i].DevType,
+			}
+			err := IsolatedDeviceManager.attachHostDeviceToGuestByModel(ctx, self, host, devConfig, userCred, usedDeviceMap, nil)
+			if err != nil {
+				return errors.Wrap(err, "attachHostDeviceToGuestByModel")
+			}
+		}
+	}
+	return nil
+}
+
 func (self *SGuest) StartGueststartTask(
 	ctx context.Context, userCred mcclient.TokenCredential,
 	data *jsonutils.JSONDict, parentTaskId string,
@@ -1584,6 +1701,13 @@ func (self *SGuest) StartGueststartTask(
 		}
 	}
 
+	if options.Options.VirtualDeviceNumaBalance {
+		err := self.RebalanceVirtualIsolatedDevices(ctx, userCred)
+		if err != nil {
+			return errors.Wrap(err, "rebalance virtual isolated devices")
+		}
+	}
+
 	if schedStart {
 		return self.GuestSchedStartTask(ctx, userCred, data, parentTaskId)
 	} else {

pkg/compute/models/hosts.go

@@ -8021,6 +8021,17 @@ func (h *SHost) GetDetailsWorkerStats(ctx context.Context, userCred mcclient.Tok
 	return h.Request(ctx, userCred, httputils.GET, "/worker_stats", nil, nil)
 }
 
+func (hh *SHost) GetDetailsIsolatedDeviceNumaStats(ctx context.Context, userCred mcclient.TokenCredential, input *api.HostIsolatedDeviceNumaStatsInput) (jsonutils.JSONObject, error) {
+	if !utils.IsInStringArray(input.DevType, api.VALID_PASSTHROUGH_TYPES) {
+		return nil, httperrors.NewInputParameterError("dev_type %s is invalid", input.DevType)
+	}
+	stats, err := IsolatedDeviceManager.GetHostAllocatedIsolatedDeviceNumaStats(input.DevType, hh.Id)
+	if err != nil {
+		return nil, err
+	}
+	return jsonutils.Marshal(stats), nil
+}
+
 func (hh *SHost) IsAttach2Wire(wireId string) bool {
 	netifs := hh.getNetifsOnWire(wireId)
 	return len(netifs) > 0

pkg/compute/models/isolated_devices.go

@@ -1742,3 +1742,49 @@ func (model *SIsolatedDevice) PerformPrivate(ctx context.Context, userCred mccli
 	}
 	return nil, nil
 }
+
+type HostIsolatedDevicesNumaStat struct {
+	NumaNode         int8
+	NumaNodeDevCount int
+}
+
+func (manager *SIsolatedDeviceManager) GetHostAllocatedIsolatedDeviceNumaStats(devType, hostId string) ([]HostIsolatedDevicesNumaStat, error) {
+	q := manager.Query().Equals("host_id", hostId).Equals("dev_type", devType)
+	guestQ := GuestManager.Query().SubQuery()
+	q = q.Join(guestQ, sqlchemy.Equals(q.Field("guest_id"), guestQ.Field("id")))
+	q = q.Filter(sqlchemy.NotEquals(guestQ.Field("status"), api.VM_READY)).GroupBy(q.Field("numa_node"))
+	q = q.AppendField(sqlchemy.COUNT("numa_node_dev_count", q.Field("numa_node")))
+
+	subQ := q.SubQuery()
+	numaQ := subQ.Query(subQ.Field("numa_node"), subQ.Field("numa_node_dev_count"))
+	stats := make([]HostIsolatedDevicesNumaStat, 0)
+	err := numaQ.All(&stats)
+	if err != nil {
+		return nil, err
+	}
+	return stats, nil
+}
+
+func (host *SHost) VirtualDeviceNumaBalance(devType string, numaNode int8) (bool, error) {
+	if numaNode < 0 {
+		return true, nil
+	}
+	stats, err := IsolatedDeviceManager.GetHostAllocatedIsolatedDeviceNumaStats(devType, host.Id)
+	if err != nil {
+		return true, err
+	}
+	log.Debugf("VirtualDeviceNumaBalance numa stats %s", jsonutils.Marshal(stats))
+
+	if len(stats) <= 1 {
+		return true, nil
+	}
+
+	sort.Slice(stats, func(i, j int) bool {
+		return stats[i].NumaNodeDevCount > stats[j].NumaNodeDevCount
+	})
+
+	if stats[0].NumaNodeDevCount-stats[len(stats)-1].NumaNodeDevCount > 1 {
+		return false, nil
+	}
+	return true, nil
+}

pkg/compute/tasks/guest/pod_start_task.go

@@ -37,7 +37,10 @@ func init() {
 func (t *PodStartTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
 	t.SetStage("OnPodStarted", nil)
 	pod := obj.(*models.SGuest)
-	pod.StartGueststartTask(ctx, t.GetUserCred(), jsonutils.NewDict(), t.GetTaskId())
+	err := pod.StartGueststartTask(ctx, t.GetUserCred(), jsonutils.NewDict(), t.GetTaskId())
+	if err != nil {
+		t.OnPodStartedFailed(ctx, pod, jsonutils.NewString(err.Error()))
+	}
 }
 
 func (t *PodStartTask) OnPodStarted(ctx context.Context, pod *models.SGuest, _ jsonutils.JSONObject) {

pkg/hostman/options/options.go

@@ -149,7 +149,7 @@ type SHostOptions struct {
 	UseBootVga             bool `default:"false" help:"Use boot VGA GPU for guest"`
 
 	EnableStrictCpuBind         bool   `default:"false" help:"Enable strict cpu bind, one vcpu bind one pcpu"`
-	EnableHostAgentNumaAllocate bool   `default:"false" help:"Enable host agent numa allocate"`
+	EnableHostAgentNumaAllocate bool   `default:"true" help:"Enable host agent numa allocate"`
 	EnableCpuBinding            bool   `default:"true" help:"Enable cpu binding and rebalance"`
 	EnableOpenflowController    bool   `default:"false"`
 	BootVgaPciAddr              string `help:"Specific boot vga pci addr incase detect wrong device"`

pkg/mcclient/options/compute/host.go

@@ -178,6 +178,16 @@ func (h HostValidateIPMI) Params() (jsonutils.JSONObject, error) {
 	}), nil
 }
 
+type HostIsolatedDeviceNumaStatsOptions struct {
+	options.BaseIdOptions
+
+	DevType string `json:"dev_type"`
+}
+
+func (o *HostIsolatedDeviceNumaStatsOptions) Params() (jsonutils.JSONObject, error) {
+	return jsonutils.Marshal(o), nil
+}
+
 type HostUpdateOptions struct {
 	options.BaseIdOptions
 	Name        string  `help:"New name of the host"`