diff --git a/cmd/climc/shell/compute/hosts.go b/cmd/climc/shell/compute/hosts.go index d1ddc014d2..14c58bcb91 100644 --- a/cmd/climc/shell/compute/hosts.go +++ b/cmd/climc/shell/compute/hosts.go @@ -72,6 +72,7 @@ func init() { cmd.Get("ipmi", &options.BaseIdOptions{}) cmd.Get("vnc", &options.BaseIdOptions{}) cmd.Get("app-options", &options.BaseIdOptions{}) + cmd.Get("isolated-device-numa-stats", &compute.HostIsolatedDeviceNumaStatsOptions{}) cmd.GetWithCustomShow("worker-stats", func(data jsonutils.JSONObject) { stats, _ := data.GetArray("workers") printList(&printutils.ListResult{Data: stats}, nil) diff --git a/cmd/climc/shell/compute/servers.go b/cmd/climc/shell/compute/servers.go index 9eebc4daef..507aec1b27 100644 --- a/cmd/climc/shell/compute/servers.go +++ b/cmd/climc/shell/compute/servers.go @@ -124,6 +124,7 @@ func init() { cmd.Perform("set-nic-traffic-limit", &options.ServerNicTrafficLimitOptions{}) cmd.Perform("add-sub-ips", &options.ServerAddSubIpsOptions{}) cmd.Perform("update-sub-ips", &options.ServerUpdateSubIpsOptions{}) + cmd.BatchPerform("restore-virtual-isolated-devices", &options.ServerIdsOptions{}) cmd.BatchPerform("set-os-info", &options.ServerSetOSInfoOptions{}) cmd.BatchPerform("start-rescue", &options.ServerStartOptions{}) cmd.BatchPerform("stop-rescue", &options.ServerStartOptions{}) diff --git a/pkg/apis/compute/guest_const.go b/pkg/apis/compute/guest_const.go index 83017bc571..d077c22bf4 100644 --- a/pkg/apis/compute/guest_const.go +++ b/pkg/apis/compute/guest_const.go @@ -337,6 +337,8 @@ const ( VM_METADATA_RELEASED_DEVICES = "released_devices" VM_METADATA_CPU_NUMA_PIN = "__cpu_numa_pin" + + VM_METADATA_VIRTUAL_ISOLATED_DEVICE_CONFIG = "__virtual_isolated_device_config" ) // windows allow a maximal length of 15 diff --git a/pkg/apis/compute/host.go b/pkg/apis/compute/host.go index 48e019e27d..0112994f5e 100644 --- a/pkg/apis/compute/host.go +++ b/pkg/apis/compute/host.go @@ -760,6 +760,10 @@ type HostUploadGuestsStatusInput struct { Guests map[string]*HostUploadGuestStatusInput `json:"guests"` } +type HostIsolatedDeviceNumaStatsInput struct { + DevType string +} + type GuestUploadContainerStatusResponse struct { Error string `json:"error"` OK bool `json:"ok"` diff --git a/pkg/apis/compute/isolated_device_const.go b/pkg/apis/compute/isolated_device_const.go index 0655e9e3a5..05d5bff96b 100644 --- a/pkg/apis/compute/isolated_device_const.go +++ b/pkg/apis/compute/isolated_device_const.go @@ -51,6 +51,7 @@ var VITRUAL_DEVICE_TYPES = []string{ CONTAINER_DEV_NVIDIA_MPS, CONTAINER_DEV_ASCEND_NPU, CONTAINER_DEV_NVIDIA_GPU_SHARE, + CONTAINER_DEV_VASTAITECH_GPU, } var ID_VENDOR_MAP = map[string]string{ diff --git a/pkg/cloudcommon/options/options.go b/pkg/cloudcommon/options/options.go index ddb74843b2..bb6071000a 100644 --- a/pkg/cloudcommon/options/options.go +++ b/pkg/cloudcommon/options/options.go @@ -125,7 +125,8 @@ type BaseOptions struct { GlobalHTTPProxy string `help:"Global http proxy"` GlobalHTTPSProxy string `help:"Global https proxy"` - IgnoreNonrunningGuests bool `default:"true" help:"Count memory for running guests only when do scheduling. Ignore memory allocation for non-running guests"` + IgnoreNonrunningGuests bool `default:"true" help:"Count memory for running guests only when do scheduling. Ignore memory allocation for non-running guests"` + VirtualDeviceNumaBalance bool `default:"true" help:"fix virtual device numa node balance on guest start"` PlatformName string `help:"identity name of this platform" default:"Cloudpods"` PlatformNames map[string]string `help:"identity name of this platform by language"` diff --git a/pkg/compute/guestdrivers/pod.go b/pkg/compute/guestdrivers/pod.go index 2a12909ee0..885ff2cab4 100644 --- a/pkg/compute/guestdrivers/pod.go +++ b/pkg/compute/guestdrivers/pod.go @@ -602,10 +602,7 @@ func (p *SPodDriver) BeforeDetachIsolatedDevice(ctx context.Context, userCred mc newDevs := make([]*api.ContainerDevice, 0) releasedDevs := make(map[string]models.ContainerReleasedDevice) for _, curDev := range devs { - if curDev.IsolatedDevice == nil { - continue - } - if curDev.IsolatedDevice.Id != dev.GetId() { + if curDev.IsolatedDevice == nil || curDev.IsolatedDevice.Id != dev.GetId() { tmpDev := curDev newDevs = append(newDevs, tmpDev) } else { diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index 7e1fe7485a..d6ac7c64cc 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -1128,6 +1128,61 @@ func (self *SGuest) StartResumeTask(ctx context.Context, userCred mcclient.Token return driver.StartResumeTask(ctx, userCred, self, nil, parentTaskId) } +func (self *SGuest) PerformRestoreVirtualIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + config := self.GetMetadataJson(ctx, api.VM_METADATA_VIRTUAL_ISOLATED_DEVICE_CONFIG, userCred) + if config == nil { + return nil, nil + } + devConfigs := make([]api.IsolatedDeviceConfig, 0) + err := config.Unmarshal(&devConfigs) + if err != nil { + return nil, errors.Wrap(err, "unmarshal virtual dev configs") + } + devs, err := self.GetIsolatedDevices() + if err != nil { + return nil, errors.Wrap(err, "GetIsolatedDevices") + } + devCount := map[string]int{} + for i := range devConfigs { + key := devConfigs[i].DevType + "-" + devConfigs[i].Model + if cnt, ok := devCount[key]; ok { + devCount[key] = cnt + 1 + } else { + devCount[key] = 1 + } + } + for i := range devs { + key := devConfigs[i].DevType + "-" + devConfigs[i].Model + if cnt, ok := devCount[key]; ok { + devCount[key] = cnt - 1 + } + } + + host, err := self.GetHost() + if err != nil { + return nil, errors.Wrap(err, "get host") + } + lockman.LockObject(ctx, host) + defer lockman.ReleaseObject(ctx, host) + + usedDeviceMap := map[string]*SIsolatedDevice{} + for key, cnt := range devCount { + if cnt <= 0 { + continue + } + segs := strings.SplitN(key, "-", 2) + devConfig := &api.IsolatedDeviceConfig{ + Model: segs[1], + DevType: segs[0], + } + err := IsolatedDeviceManager.attachHostDeviceToGuestByModel(ctx, self, host, devConfig, userCred, usedDeviceMap, nil) + if err != nil { + return nil, errors.Wrap(err, "attachHostDeviceToGuestByModel") + } + } + return nil, nil +} + // 开机 func (self *SGuest) PerformStart( ctx context.Context, @@ -1560,6 +1615,68 @@ func (self *SGuest) StartInsertVfdTask(ctx context.Context, floppyOrdinal int64, return nil } +func (self *SGuest) RebalanceVirtualIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential) error { + devs, err := self.GetIsolatedDevices() + if err != nil { + return errors.Wrap(err, "guest get isolated devices") + } + if len(devs) == 0 { + return nil + } + host, err := self.GetHost() + if err != nil { + return errors.Wrap(err, "guest get host") + } + var numaNodeBalance = true + detachDevs := make([]SIsolatedDevice, 0) + originDevConfigs := make([]api.IsolatedDeviceConfig, 0) + for i := range devs { + if utils.IsInStringArray(devs[i].DevType, api.VITRUAL_DEVICE_TYPES) { + originDevConfigs = append(originDevConfigs, api.IsolatedDeviceConfig{ + Model: devs[i].Model, + DevType: devs[i].DevType, + }) + detachDevs = append(detachDevs, devs[i]) + isBalance, err := host.VirtualDeviceNumaBalance(devs[i].DevType, devs[i].NumaNode) + if err != nil { + return errors.Wrap(err, "VirtualDeviceNumaBalance") + } + if numaNodeBalance && !isBalance { + numaNodeBalance = false + } + } + } + log.Infof("Guest %s on host %s start virtual devices numa node balance %v", self.Id, host.Id, numaNodeBalance) + if !numaNodeBalance { + err := self.SetMetadata(ctx, api.VM_METADATA_VIRTUAL_ISOLATED_DEVICE_CONFIG, originDevConfigs, userCred) + if err != nil { + return errors.Wrap(err, "set metadata virtual isolated device config") + } + lockman.LockObject(ctx, host) + defer lockman.ReleaseObject(ctx, host) + + for i := 0; i < len(detachDevs); i++ { + err := self.detachIsolateDevice(ctx, userCred, &detachDevs[i]) + if err != nil { + return errors.Wrapf(err, "detach device %s", detachDevs[i].GetId()) + } + } + + usedDeviceMap := map[string]*SIsolatedDevice{} + for i := range detachDevs { + devConfig := &api.IsolatedDeviceConfig{ + Model: detachDevs[i].Model, + DevType: detachDevs[i].DevType, + } + err := IsolatedDeviceManager.attachHostDeviceToGuestByModel(ctx, self, host, devConfig, userCred, usedDeviceMap, nil) + if err != nil { + return errors.Wrap(err, "attachHostDeviceToGuestByModel") + } + } + } + return nil +} + func (self *SGuest) StartGueststartTask( ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, parentTaskId string, @@ -1584,6 +1701,13 @@ func (self *SGuest) StartGueststartTask( } } + if options.Options.VirtualDeviceNumaBalance { + err := self.RebalanceVirtualIsolatedDevices(ctx, userCred) + if err != nil { + return errors.Wrap(err, "rebalance virtual isolated devices") + } + } + if schedStart { return self.GuestSchedStartTask(ctx, userCred, data, parentTaskId) } else { diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 0b481f4170..b99578dc47 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -8021,6 +8021,17 @@ func (h *SHost) GetDetailsWorkerStats(ctx context.Context, userCred mcclient.Tok return h.Request(ctx, userCred, httputils.GET, "/worker_stats", nil, nil) } +func (hh *SHost) GetDetailsIsolatedDeviceNumaStats(ctx context.Context, userCred mcclient.TokenCredential, input *api.HostIsolatedDeviceNumaStatsInput) (jsonutils.JSONObject, error) { + if !utils.IsInStringArray(input.DevType, api.VALID_PASSTHROUGH_TYPES) { + return nil, httperrors.NewInputParameterError("dev_type %s is invalid", input.DevType) + } + stats, err := IsolatedDeviceManager.GetHostAllocatedIsolatedDeviceNumaStats(input.DevType, hh.Id) + if err != nil { + return nil, err + } + return jsonutils.Marshal(stats), nil +} + func (hh *SHost) IsAttach2Wire(wireId string) bool { netifs := hh.getNetifsOnWire(wireId) return len(netifs) > 0 diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index 771884c001..2d92253e9f 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -1742,3 +1742,49 @@ func (model *SIsolatedDevice) PerformPrivate(ctx context.Context, userCred mccli } return nil, nil } + +type HostIsolatedDevicesNumaStat struct { + NumaNode int8 + NumaNodeDevCount int +} + +func (manager *SIsolatedDeviceManager) GetHostAllocatedIsolatedDeviceNumaStats(devType, hostId string) ([]HostIsolatedDevicesNumaStat, error) { + q := manager.Query().Equals("host_id", hostId).Equals("dev_type", devType) + guestQ := GuestManager.Query().SubQuery() + q = q.Join(guestQ, sqlchemy.Equals(q.Field("guest_id"), guestQ.Field("id"))) + q = q.Filter(sqlchemy.NotEquals(guestQ.Field("status"), api.VM_READY)).GroupBy(q.Field("numa_node")) + q = q.AppendField(sqlchemy.COUNT("numa_node_dev_count", q.Field("numa_node"))) + + subQ := q.SubQuery() + numaQ := subQ.Query(subQ.Field("numa_node"), subQ.Field("numa_node_dev_count")) + stats := make([]HostIsolatedDevicesNumaStat, 0) + err := numaQ.All(&stats) + if err != nil { + return nil, err + } + return stats, nil +} + +func (host *SHost) VirtualDeviceNumaBalance(devType string, numaNode int8) (bool, error) { + if numaNode < 0 { + return true, nil + } + stats, err := IsolatedDeviceManager.GetHostAllocatedIsolatedDeviceNumaStats(devType, host.Id) + if err != nil { + return true, err + } + log.Debugf("VirtualDeviceNumaBalance numa stats %s", jsonutils.Marshal(stats)) + + if len(stats) <= 1 { + return true, nil + } + + sort.Slice(stats, func(i, j int) bool { + return stats[i].NumaNodeDevCount > stats[j].NumaNodeDevCount + }) + + if stats[0].NumaNodeDevCount-stats[len(stats)-1].NumaNodeDevCount > 1 { + return false, nil + } + return true, nil +} diff --git a/pkg/compute/tasks/guest/pod_start_task.go b/pkg/compute/tasks/guest/pod_start_task.go index 0481173f88..725b51d76d 100644 --- a/pkg/compute/tasks/guest/pod_start_task.go +++ b/pkg/compute/tasks/guest/pod_start_task.go @@ -37,7 +37,10 @@ func init() { func (t *PodStartTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { t.SetStage("OnPodStarted", nil) pod := obj.(*models.SGuest) - pod.StartGueststartTask(ctx, t.GetUserCred(), jsonutils.NewDict(), t.GetTaskId()) + err := pod.StartGueststartTask(ctx, t.GetUserCred(), jsonutils.NewDict(), t.GetTaskId()) + if err != nil { + t.OnPodStartedFailed(ctx, pod, jsonutils.NewString(err.Error())) + } } func (t *PodStartTask) OnPodStarted(ctx context.Context, pod *models.SGuest, _ jsonutils.JSONObject) { diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index 9db75f0ba0..45ff9c7190 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -149,7 +149,7 @@ type SHostOptions struct { UseBootVga bool `default:"false" help:"Use boot VGA GPU for guest"` EnableStrictCpuBind bool `default:"false" help:"Enable strict cpu bind, one vcpu bind one pcpu"` - EnableHostAgentNumaAllocate bool `default:"false" help:"Enable host agent numa allocate"` + EnableHostAgentNumaAllocate bool `default:"true" help:"Enable host agent numa allocate"` EnableCpuBinding bool `default:"true" help:"Enable cpu binding and rebalance"` EnableOpenflowController bool `default:"false"` BootVgaPciAddr string `help:"Specific boot vga pci addr incase detect wrong device"` diff --git a/pkg/mcclient/options/compute/host.go b/pkg/mcclient/options/compute/host.go index 78e597fb17..4d8fe22325 100644 --- a/pkg/mcclient/options/compute/host.go +++ b/pkg/mcclient/options/compute/host.go @@ -178,6 +178,16 @@ func (h HostValidateIPMI) Params() (jsonutils.JSONObject, error) { }), nil } +type HostIsolatedDeviceNumaStatsOptions struct { + options.BaseIdOptions + + DevType string `json:"dev_type"` +} + +func (o *HostIsolatedDeviceNumaStatsOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(o), nil +} + type HostUpdateOptions struct { options.BaseIdOptions Name string `help:"New name of the host"`