From 695f6c0e31ba5b57f1007974009c65826440c5f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Wed, 10 Jul 2024 16:12:26 +0800 Subject: [PATCH] fix(region): optimized guest list speed (#20759) --- pkg/apis/compute/guests.go | 20 ++++--- pkg/apis/compute/host.go | 5 ++ pkg/compute/models/guest_queries.go | 74 ++++++++++++++++++++++- pkg/compute/models/guests.go | 78 ------------------------- pkg/compute/models/hostresource.go | 2 + pkg/mcclient/options/compute/servers.go | 1 - 6 files changed, 91 insertions(+), 89 deletions(-) diff --git a/pkg/apis/compute/guests.go b/pkg/apis/compute/guests.go index a71c48ea82..e0125dbc4a 100644 --- a/pkg/apis/compute/guests.go +++ b/pkg/apis/compute/guests.go @@ -190,18 +190,11 @@ type ServerDetails struct { // 系统管理员可见的安全组规则 AdminSecurityRules string `json:"admin_security_rules"` - // list - AttachTime time.Time `json:"attach_time"` - // common IsPrepaidRecycle bool `json:"is_prepaid_recycle"` - // 备份主机所在宿主机名称 - BackupHostName string `json:"backup_host_name"` - // 备份主机所在宿主机状态 - BackupHostStatus string `json:"backup_host_status"` - // 主备机同步状态 - BackupGuestSyncStatus string `json:"backup_guest_sync_status"` + // 主备机信息 + BackupInfo // 是否可以回收 CanRecycle bool `json:"can_recycle"` @@ -275,6 +268,15 @@ type ServerDetails struct { MonitorUrl string `json:"monitor_url"` } +type BackupInfo struct { + // 备份主机所在宿主机名称 + BackupHostName string `json:"backup_host_name"` + // 备份主机所在宿主机状态 + BackupHostStatus string `json:"backup_host_status"` + // 主备机同步状态 + BackupGuestSyncStatus string `json:"backup_guest_sync_status"` +} + type Floppy struct { Ordinal int `json:"ordinal"` Detail string `json:"detail"` diff --git a/pkg/apis/compute/host.go b/pkg/apis/compute/host.go index 976973d5c0..d42823ca4a 100644 --- a/pkg/apis/compute/host.go +++ b/pkg/apis/compute/host.go @@ -291,6 +291,11 @@ type HostResourceInfo struct { // 宿主机状态 HostStatus string `json:"host_status"` + HostResourceType string `json:"host_resource_type"` + + // 宿主机计费类型 + HostBillingType string `json:"host_billing_type"` + // 宿主机服务状态` HostServiceStatus string `json:"host_service_status"` diff --git a/pkg/compute/models/guest_queries.go b/pkg/compute/models/guest_queries.go index 9c4511d945..143e677e24 100644 --- a/pkg/compute/models/guest_queries.go +++ b/pkg/compute/models/guest_queries.go @@ -18,6 +18,7 @@ import ( "context" "database/sql" "strings" + "time" "yunion.io/x/cloudmux/pkg/cloudprovider" "yunion.io/x/jsonutils" @@ -28,9 +29,11 @@ import ( "yunion.io/x/sqlchemy" "yunion.io/x/onecloud/pkg/apis" + billing_api "yunion.io/x/onecloud/pkg/apis/billing" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/stringutils2" ) @@ -50,6 +53,7 @@ func (manager *SGuestManager) FetchCustomizeColumns( encRows := manager.SEncryptedResourceManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) guestIds := make([]string, len(objs)) guests := make([]SGuest, len(objs)) + backupHostIds := make([]string, len(objs)) for i := range objs { rows[i] = api.ServerDetails{ VirtualResourceDetails: virtRows[i], @@ -59,6 +63,7 @@ func (manager *SGuestManager) FetchCustomizeColumns( } guest := objs[i].(*SGuest) guestIds[i] = guest.GetId() + backupHostIds[i] = guest.BackupHostId guests[i] = *guest } @@ -282,12 +287,66 @@ func (manager *SGuestManager) FetchCustomizeColumns( } } + if len(fields) == 0 || fields.Contains("backup_host_name") || fields.Contains("backup_host_status") && len(backupHostIds) > 0 { + backups, _ := fetchGuestBackupInfo(backupHostIds) + meta := []db.SMetadata{} + db.Metadata.Query().In("obj_id", guestIds).Equals("obj_type", manager.Keyword()).Equals("key", api.MIRROR_JOB).All(&meta) + syncStatus := map[string]string{} + for _, v := range meta { + syncStatus[v.ObjId] = v.Value + } + if len(backups) > 0 || len(syncStatus) > 0 { + for i := range rows { + rows[i].BackupInfo, _ = backups[backupHostIds[i]] + rows[i].BackupGuestSyncStatus, _ = syncStatus[guestIds[i]] + } + } + } + for i := range rows { - rows[i] = guests[i].moreExtraInfo(ctx, rows[i], userCred, query, fields, isList) + if len(fields) == 0 || fields.Contains("auto_delete_at") { + if guests[i].PendingDeleted { + pendingDeletedAt := guests[i].PendingDeletedAt.Add(time.Second * time.Duration(options.Options.PendingDeleteExpireSeconds)) + rows[i].AutoDeleteAt = pendingDeletedAt + } + } + if len(fields) == 0 || fields.Contains("can_recycle") { + if guests[i].BillingType == billing_api.BILLING_TYPE_PREPAID && !guests[i].ExpiredAt.Before(time.Now()) && len(rows[i].ManagerId) > 0 { + rows[i].CanRecycle = true + } + } + + rows[i].IsPrepaidRecycle = (rows[i].HostResourceType == api.HostResourceTypePrepaidRecycle && rows[i].HostBillingType == billing_api.BILLING_TYPE_PREPAID) + + drv, _ := GetDriver(guests[i].Hypervisor, rows[i].Provider) + if drv != nil { + rows[i].CdromSupport, _ = drv.IsSupportCdrom(&guests[i]) + rows[i].FloppySupport, _ = drv.IsSupportFloppy(&guests[i]) + rows[i].MonitorUrl = drv.FetchMonitorUrl(ctx, &guests[i]) + } + if len(guests[i].HostId) == 0 && guests[i].Status == api.VM_SCHEDULE_FAILED { rows[i].Brand = "Unknown" rows[i].Provider = "Unknown" } + + if !isList { + rows[i].Networks = guests[i].getNetworksDetails() + rows[i].VirtualIps = strings.Join(guests[i].getVirtualIPs(), ",") + rows[i].SecurityRules = guests[i].getSecurityGroupsRules() + + osName := guests[i].GetOS() + if len(osName) > 0 { + rows[i].OsName = osName + if len(guests[i].OsType) == 0 { + rows[i].OsType = osName + } + } + + if userCred.HasSystemAdminPrivilege() { + rows[i].AdminSecurityRules = guests[i].getAdminSecurityRules() + } + } } return rows @@ -696,6 +755,19 @@ func fetchGuestGpuInstanceTypes(guestIds []string) (map[string]*GpuSpec, error) return ret, nil } +func fetchGuestBackupInfo(hostIds []string) (map[string]api.BackupInfo, error) { + ret := map[string]api.BackupInfo{} + hosts := []SHost{} + err := HostManager.Query().In("id", hostIds).All(&hosts) + if err != nil { + return nil, err + } + for _, host := range hosts { + ret[host.Id] = api.BackupInfo{BackupHostName: host.Name, BackupHostStatus: host.HostStatus} + } + return ret, nil +} + func fetchGuestIsolatedDevices(guestIds []string) map[string][]api.SIsolatedDevice { q := IsolatedDeviceManager.Query().In("guest_id", guestIds) devs := make([]SIsolatedDevice, 0) diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index d4da072fb6..f50ac817d7 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -2534,84 +2534,6 @@ func (self *SGuest) getExtBandwidth() int { return self.getBandwidth(true) } -func (self *SGuest) moreExtraInfo( - ctx context.Context, - out api.ServerDetails, - userCred mcclient.TokenCredential, - query jsonutils.JSONObject, - fields stringutils2.SSortedStrings, - isList bool, -) api.ServerDetails { - // extra.Add(jsonutils.NewInt(int64(self.getExtBandwidth())), "ext_bw") - - if isList { - if query.Contains("group") { - groupId, _ := query.GetString("group") - q := GroupguestManager.Query().Equals("group_id", groupId).Equals("guest_id", self.Id) - var groupGuest SGroupguest - err := q.First(&groupGuest) - if err == nil { - out.AttachTime = groupGuest.CreatedAt - } - } - } else { - out.Networks = self.getNetworksDetails() - out.VirtualIps = strings.Join(self.getVirtualIPs(), ",") - out.SecurityRules = self.getSecurityGroupsRules() - - osName := self.GetOS() - if len(osName) > 0 { - out.OsName = osName - if len(self.OsType) == 0 { - out.OsType = osName - } - } - - if userCred.HasSystemAdminPrivilege() { - out.AdminSecurityRules = self.getAdminSecurityRules() - } - - } - - out.IsPrepaidRecycle = self.IsPrepaidRecycle() - - if len(self.BackupHostId) > 0 && (len(fields) == 0 || fields.Contains("backup_host_name") || fields.Contains("backup_host_status")) { - backupHost := HostManager.FetchHostById(self.BackupHostId) - if backupHost != nil { - if len(fields) == 0 || fields.Contains("backup_host_name") { - out.BackupHostName = backupHost.Name - } - if len(fields) == 0 || fields.Contains("backup_host_status") { - out.BackupHostStatus = backupHost.HostStatus - } - out.BackupGuestSyncStatus = self.GetGuestBackupMirrorJobStatus(ctx, userCred) - } - } - - if len(fields) == 0 || fields.Contains("can_recycle") { - err := self.CanPerformPrepaidRecycle() - if err == nil { - out.CanRecycle = true - } - } - - if len(fields) == 0 || fields.Contains("auto_delete_at") { - if self.PendingDeleted { - pendingDeletedAt := self.PendingDeletedAt.Add(time.Second * time.Duration(options.Options.PendingDeleteExpireSeconds)) - out.AutoDeleteAt = pendingDeletedAt - } - } - - drv, _ := self.GetDriver() - if drv != nil { - out.CdromSupport, _ = drv.IsSupportCdrom(self) - out.FloppySupport, _ = drv.IsSupportFloppy(self) - out.MonitorUrl = drv.FetchMonitorUrl(ctx, self) - } - - return out -} - func (self *SGuestManager) GetMetadataHiddenKeys() []string { return []string{ api.VM_METADATA_CREATE_PARAMS, diff --git a/pkg/compute/models/hostresource.go b/pkg/compute/models/hostresource.go index ba43d88f2f..ce02cc20bd 100644 --- a/pkg/compute/models/hostresource.go +++ b/pkg/compute/models/hostresource.go @@ -109,6 +109,8 @@ func (manager *SHostResourceBaseManager) FetchCustomizeColumns( rows[i].HostServiceStatus = host.HostStatus rows[i].HostType = host.HostType rows[i].ManagerId = host.ManagerId + rows[i].HostResourceType = host.ResourceType + rows[i].HostBillingType = host.BillingType rows[i].ZoneId = host.ZoneId } zoneList[i] = &SZoneResourceBase{rows[i].ZoneId} diff --git a/pkg/mcclient/options/compute/servers.go b/pkg/mcclient/options/compute/servers.go index 1d586442d0..918d07ceea 100644 --- a/pkg/mcclient/options/compute/servers.go +++ b/pkg/mcclient/options/compute/servers.go @@ -53,7 +53,6 @@ type ServerListOptions struct { UsableServerForEip string `help:"Eip id or name"` WithoutUserMeta *bool `help:"Show Servers without user metadata"` EipAssociable *bool `help:"Show Servers can associate with eip"` - Group string `help:"Instance Group ID or Name"` HostSn string `help:"Host SN"` IpAddr string `help:"Fileter by ip"` IpAddrs []string `help:"Fileter by ips"`