From 7d7ca5bdc41d0b0d3cc5ed33b509f6ce22a0cec2 Mon Sep 17 00:00:00 2001 From: wanyaoqi <18528551+wanyaoqi@users.noreply.github.com> Date: Fri, 14 Jun 2024 19:39:45 +0800 Subject: [PATCH] fix(region,host): isolated device batch sync (#20536) --- pkg/cloudcommon/db/db_dispatcher.go | 7 +++ pkg/compute/models/guest_actions.go | 2 +- pkg/compute/models/hosts.go | 67 ++++++++++++++++++++++++++ pkg/compute/models/isolated_devices.go | 15 +++++- pkg/hostman/hostinfo/hostinfo.go | 43 ++++++++++------- 5 files changed, 115 insertions(+), 19 deletions(-) diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index de462684a2..411cd380c5 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -1767,6 +1767,13 @@ func reflectDispatcherInternal( } } +func DoUpdate(manager IModelManager, item IModel, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + lockman.LockObject(ctx, item) + defer lockman.ReleaseObject(ctx, item) + + return updateItem(manager, item, ctx, userCred, query, data) +} + func updateItem(manager IModelManager, item IModel, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { var err error diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index e351341f50..0a5c956a0e 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -2054,7 +2054,7 @@ func (self *SGuest) startAttachIsolatedDevices(ctx context.Context, userCred mcc host, _ := self.GetHost() lockman.LockObject(ctx, host) defer lockman.ReleaseObject(ctx, host) - devs, err := IsolatedDeviceManager.GetDevsOnHost(host.Id, devModel, count) + devs, err := IsolatedDeviceManager.GetUnusedDevsOnHost(host.Id, devModel, count) if err != nil { return httperrors.NewInternalServerError("fetch gpu failed %s", err) } diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index f44d1238de..f9beb4ae98 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -6623,6 +6623,73 @@ func (hh *SHost) PerformProbeIsolatedDevices(ctx context.Context, userCred mccli return hh.GetHostDriver().RequestProbeIsolatedDevices(ctx, userCred, hh, data) } +func (hh *SHost) PerformSyncIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + devs, err := IsolatedDeviceManager.GetAllDevsOnHost(hh.Id) + if err != nil { + return nil, err + } + reqDevs, err := data.GetArray("isolated_devices") + if err != nil { + return nil, httperrors.NewMissingParameterError("isolated_devices") + } + + retDevs := jsonutils.NewArray() + foundDevIndex := map[int]struct{}{} + for i := range devs { + dev, err := IsolatedDeviceManager.FetchById(devs[i].Id) + if err != nil { + return nil, err + } + + foundDev := false + for j := range reqDevs { + venderDeviceId, _ := reqDevs[j].GetString("vendor_device_id") + devAddr, _ := reqDevs[j].GetString("addr") + mdevId, _ := reqDevs[j].GetString("mdev_id") + if devs[i].VendorDeviceId == venderDeviceId && devs[i].Addr == devAddr && devs[i].MdevId == mdevId { + // update isolated device + devRet, err := db.DoUpdate(IsolatedDeviceManager, dev, ctx, userCred, jsonutils.NewDict(), reqDevs[j]) + if err != nil { + return nil, err + } + retDevs.Add(devRet) + foundDevIndex[j] = struct{}{} + foundDev = true + } + } + + if !foundDev { + // detach isolated device + isolatedDev := dev.(*SIsolatedDevice) + params := jsonutils.NewDict() + params.Set("purge", jsonutils.JSONTrue) + _, err := isolatedDev.PerformPurge(ctx, userCred, nil, params) + if err != nil { + return nil, err + } + } + } + + for i := range reqDevs { + if _, ok := foundDevIndex[i]; ok { + continue + } + // create isolated device + dev, err := db.DoCreate(IsolatedDeviceManager, ctx, userCred, nil, reqDevs[i], userCred) + if err != nil { + return nil, err + } + devRet, err := db.GetItemDetails(IsolatedDeviceManager, dev, ctx, userCred) + if err != nil { + return nil, err + } + retDevs.Add(devRet) + } + res := jsonutils.NewDict() + res.Set("isolated_devices", retDevs) + return res, nil +} + func (hh *SHost) GetPinnedCpusetCores(ctx context.Context, userCred mcclient.TokenCredential) (map[string][]int, error) { gsts, err := hh.GetGuests() if err != nil { diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index ca033988ec..30284e4ecd 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -982,7 +982,20 @@ func (manager *SIsolatedDeviceManager) DeleteDevicesByHost(ctx context.Context, } } -func (manager *SIsolatedDeviceManager) GetDevsOnHost(hostId string, model string, count int) ([]SIsolatedDevice, error) { +func (manager *SIsolatedDeviceManager) GetAllDevsOnHost(hostId string) ([]SIsolatedDevice, error) { + devs := make([]SIsolatedDevice, 0) + q := manager.Query().Equals("host_id", hostId) + err := db.FetchModelObjects(manager, q, &devs) + if err != nil { + return nil, err + } + if len(devs) == 0 { + return nil, nil + } + return devs, nil +} + +func (manager *SIsolatedDeviceManager) GetUnusedDevsOnHost(hostId string, model string, count int) ([]SIsolatedDevice, error) { devs := make([]SIsolatedDevice, 0) q := manager.Query().Equals("host_id", hostId).Equals("model", model).IsNullOrEmpty("guest_id").Limit(count) err := db.FetchModelObjects(manager, q, &devs) diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index 832e46e430..868f9bfde1 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -2087,10 +2087,31 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) { sriovNics, offloadNics, options.HostOptions.PTNVMEConfigs, options.HostOptions.AMDVgpuPFs, options.HostOptions.NVIDIAVgpuPFs, enableDevWhitelist) - objs, err := h.getRemoteIsolatedDevices() - if err != nil { - return nil, errors.Wrap(err, "getRemoteIsolatedDevices") + h.IsolatedDeviceMan.BatchCustomProbe() + // sync each isolated device found + updateDevs := jsonutils.NewArray() + for _, dev := range h.IsolatedDeviceMan.GetDevices() { + dev.SetHostId(h.HostId) + data := isolated_device.GetApiResourceData(dev) + updateDevs.Add(data) } + + params := jsonutils.NewDict() + params.Set("isolated_devices", updateDevs) + ret, err := modules.Hosts.PerformAction(h.GetSession(), h.HostId, "sync-isolated-devices", params) + if err != nil { + return nil, errors.Wrap(err, "sync isolated devices") + } + devRet, err := ret.Get("isolated_devices") + if err != nil { + return nil, errors.Wrap(err, "sync isolated devices faild get dev rets") + } + devRets, _ := devRet.(*jsonutils.JSONArray) + if devRets.Length() != len(h.IsolatedDeviceMan.GetDevices()) { + return nil, errors.Wrap(err, "sync devices not match") + } + + objs, _ := devRets.GetArray() for _, obj := range objs { info := isolated_device.CloudDeviceInfo{} if err := obj.Unmarshal(&info); err != nil { @@ -2100,23 +2121,11 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) { if dev != nil { dev.SetDeviceInfo(info) } else { - // detach device - h.IsolatedDeviceMan.AppendDetachedDevice(&info) + return nil, errors.Wrapf(err, "unknown device %s", obj) } } - h.IsolatedDeviceMan.StartDetachTask() - h.IsolatedDeviceMan.BatchCustomProbe() - // sync each isolated device found - updateDevs := jsonutils.NewArray() - for _, dev := range h.IsolatedDeviceMan.GetDevices() { - if obj, err := isolated_device.SyncDeviceInfo(h.GetSession(), h.HostId, dev); err != nil { - return nil, errors.Wrapf(err, "Sync device %s", dev) - } else { - updateDevs.Add(obj) - } - } - return updateDevs, nil + return devRets, nil } func (h *SHostInfo) deployAdminAuthorizedKeys() {