From f423eb90117e804dd40e84138855bf5d54360e21 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Thu, 2 Apr 2020 01:40:38 +0800 Subject: [PATCH] fix: disk/nics usage should exclude those pending_deleted --- pkg/compute/models/storages.go | 38 ++++++++++++++++------------ pkg/compute/models/wires.go | 13 ++++++++++ pkg/compute/usages/handler.go | 45 ++++++++++++++++++++++++---------- 3 files changed, 68 insertions(+), 28 deletions(-) diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 2020b4614d..bb73cdaf03 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -756,7 +756,7 @@ type StorageCapacityStat struct { TotalSizeVirtual float64 } -func filterDisksByScope(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) *sqlchemy.SSubQuery { +func filterDisksByScope(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, pendingDeleted bool) *sqlchemy.SSubQuery { q := DiskManager.Query() switch scope { case rbacutils.ScopeSystem: @@ -765,11 +765,16 @@ func filterDisksByScope(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityPr case rbacutils.ScopeProject: q = q.Filter(sqlchemy.Equals(q.Field("tenant_id"), ownerId.GetProjectId())) } + if pendingDeleted { + q = q.IsTrue("pending_deleted") + } else { + q = q.IsFalse("pending_deleted") + } return q.SubQuery() } -func (manager *SStorageManager) disksReadyQ(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) *sqlchemy.SSubQuery { - disks := filterDisksByScope(scope, ownerId) +func (manager *SStorageManager) disksReadyQ(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, pendingDeleted bool) *sqlchemy.SSubQuery { + disks := filterDisksByScope(scope, ownerId, pendingDeleted) q := disks.Query( disks.Field("storage_id"), sqlchemy.SUM("used_capacity", disks.Field("disk_size")), @@ -778,7 +783,7 @@ func (manager *SStorageManager) disksReadyQ(scope rbacutils.TRbacScope, ownerId return q.SubQuery() } -func (manager *SStorageManager) diskIsAttachedQ(isAttached bool, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) *sqlchemy.SSubQuery { +func (manager *SStorageManager) diskIsAttachedQ(isAttached bool, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, pendingDeleted bool) *sqlchemy.SSubQuery { sumKey := "attached_used_capacity" cond := sqlchemy.In if !isAttached { @@ -786,7 +791,7 @@ func (manager *SStorageManager) diskIsAttachedQ(isAttached bool, scope rbacutils cond = sqlchemy.NotIn } sq := GuestdiskManager.Query("disk_id").SubQuery() - disks := filterDisksByScope(scope, ownerId) + disks := filterDisksByScope(scope, ownerId, pendingDeleted) disks = disks.Query().Filter(cond(disks.Field("id"), sq)).SubQuery() q := disks.Query( disks.Field("storage_id"), @@ -795,16 +800,16 @@ func (manager *SStorageManager) diskIsAttachedQ(isAttached bool, scope rbacutils return q.SubQuery() } -func (manager *SStorageManager) diskAttachedQ(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) *sqlchemy.SSubQuery { - return manager.diskIsAttachedQ(true, scope, ownerId) +func (manager *SStorageManager) diskAttachedQ(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, pendingDeleted bool) *sqlchemy.SSubQuery { + return manager.diskIsAttachedQ(true, scope, ownerId, pendingDeleted) } -func (manager *SStorageManager) diskDetachedQ(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) *sqlchemy.SSubQuery { - return manager.diskIsAttachedQ(false, scope, ownerId) +func (manager *SStorageManager) diskDetachedQ(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, pendingDeleted bool) *sqlchemy.SSubQuery { + return manager.diskIsAttachedQ(false, scope, ownerId, pendingDeleted) } -func (manager *SStorageManager) disksFailedQ(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) *sqlchemy.SSubQuery { - disks := filterDisksByScope(scope, ownerId) +func (manager *SStorageManager) disksFailedQ(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, pendingDeleted bool) *sqlchemy.SSubQuery { + disks := filterDisksByScope(scope, ownerId, pendingDeleted) q := disks.Query( disks.Field("storage_id"), sqlchemy.SUM("failed_capacity", disks.Field("disk_size")), @@ -818,11 +823,12 @@ func (manager *SStorageManager) totalCapacityQ( resourceTypes []string, providers []string, brands []string, cloudEnv string, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, + pendingDeleted bool, ) *sqlchemy.SQuery { - stmt := manager.disksReadyQ(scope, ownerId) - stmt2 := manager.disksFailedQ(scope, ownerId) - attachedDisks := manager.diskAttachedQ(scope, ownerId) - detachedDisks := manager.diskDetachedQ(scope, ownerId) + stmt := manager.disksReadyQ(scope, ownerId, pendingDeleted) + stmt2 := manager.disksFailedQ(scope, ownerId, pendingDeleted) + attachedDisks := manager.diskAttachedQ(scope, ownerId, pendingDeleted) + detachedDisks := manager.diskDetachedQ(scope, ownerId, pendingDeleted) storages := manager.Query().SubQuery() q := storages.Query( storages.Field("capacity"), @@ -916,6 +922,7 @@ func (manager *SStorageManager) TotalCapacity( providers []string, brands []string, cloudEnv string, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, + pendingDeleted bool, ) StoragesCapacityStat { res1 := manager.calculateCapacity( manager.totalCapacityQ( @@ -924,6 +931,7 @@ func (manager *SStorageManager) TotalCapacity( resourceTypes, providers, brands, cloudEnv, scope, ownerId, + pendingDeleted, ), ) return res1 diff --git a/pkg/compute/models/wires.go b/pkg/compute/models/wires.go index 90e5aa9bb6..d77fd7d861 100644 --- a/pkg/compute/models/wires.go +++ b/pkg/compute/models/wires.go @@ -447,6 +447,7 @@ func (manager *SWireManager) totalCountQ( providers []string, brands []string, cloudEnv string, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, + pendingDeleted bool, ) *sqlchemy.SQuery { guests := filterByScopeOwnerId(GuestManager.Query(), scope, ownerId).SubQuery() hosts := HostManager.Query().SubQuery() @@ -461,6 +462,11 @@ func (manager *SWireManager) totalCountQ( gNicQ = gNicQ.Join(guests, sqlchemy.Equals(guests.Field("id"), gNics.Field("guest_id"))) gNicQ = gNicQ.Join(hosts, sqlchemy.Equals(guests.Field("host_id"), hosts.Field("id"))) gNicQ = gNicQ.Filter(sqlchemy.IsTrue(hosts.Field("enabled"))) + if pendingDeleted { + gNicQ = gNicQ.Filter(sqlchemy.IsTrue(guests.Field("pending_deleted"))) + } else { + gNicQ = gNicQ.Filter(sqlchemy.IsFalse(guests.Field("pending_deleted"))) + } hNics := HostnetworkManager.Query().SubQuery() hNicQ := hNics.Query( @@ -489,6 +495,11 @@ func (manager *SWireManager) totalCountQ( sqlchemy.COUNT("lbnic_count"), ) lbNicQ = lbNicQ.Join(lbs, sqlchemy.Equals(lbs.Field("id"), lbNics.Field("loadbalancer_id"))) + if pendingDeleted { + lbNicQ = lbNicQ.Filter(sqlchemy.IsTrue(lbs.Field("pending_deleted"))) + } else { + lbNicQ = lbNicQ.Filter(sqlchemy.IsFalse(lbs.Field("pending_deleted"))) + } gNicSQ := gNicQ.GroupBy(gNics.Field("network_id")).SubQuery() hNicSQ := hNicQ.GroupBy(hNics.Field("network_id")).SubQuery() @@ -566,6 +577,7 @@ func (manager *SWireManager) TotalCount( providers []string, brands []string, cloudEnv string, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, + pendingDeleted bool, ) WiresCountStat { stat := WiresCountStat{} err := manager.totalCountQ( @@ -573,6 +585,7 @@ func (manager *SWireManager) TotalCount( hostTypes, providers, brands, cloudEnv, scope, ownerId, + pendingDeleted, ).First(&stat) if err != nil { log.Errorf("Wire total count: %v", err) diff --git a/pkg/compute/usages/handler.go b/pkg/compute/usages/handler.go index d235190fe7..e96f1927e2 100644 --- a/pkg/compute/usages/handler.go +++ b/pkg/compute/usages/handler.go @@ -241,8 +241,6 @@ func getAdminGeneralUsage(userCred mcclient.IIdentityProvider, rangeObjs []db.IS count.Add("all.memory_commit_rate.running", runningMemCmtRate) count.Add("all.cpu_commit_rate.running", runningCpuCmtRate) - storageUsage := StorageUsage("", rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv) - count.Include( HostAllUsage("", userCred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv), HostAllUsage("prepaid_pool", userCred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), @@ -254,9 +252,10 @@ func getAdminGeneralUsage(userCred mcclient.IIdentityProvider, rangeObjs []db.IS BaremetalUsage(userCred, rangeObjs, hostTypes, providers, brands, cloudEnv), - storageUsage, - StorageUsage("prepaid_pool", rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), - StorageUsage("any_pool", rangeObjs, hostTypes, nil, providers, brands, cloudEnv), + StorageUsage("", rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, false), + StorageUsage("prepaid_pool", rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, false), + StorageUsage("any_pool", rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false), + StorageUsage("any_pool.pending_delete", rangeObjs, hostTypes, nil, providers, brands, cloudEnv, true), GuestNormalUsage("all.servers", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv), GuestNormalUsage("all.servers.prepaid_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), @@ -309,8 +308,6 @@ func getCommonGeneralUsage(scope rbacutils.TRbacScope, cred mcclient.IIdentityPr bucketUsage := BucketUsage(scope, cred, rangeObjs, providers, brands, cloudEnv) - disksUsage := DisksUsage(getKey(scope, "disks"), rangeObjs, nil, nil, providers, brands, cloudEnv, scope, cred) - nicsUsage := nicsUsage(rangeObjs, nil, providers, brands, cloudEnv, scope, cred) count = guestNormalUsage.Include( @@ -337,7 +334,8 @@ func getCommonGeneralUsage(scope rbacutils.TRbacScope, cred mcclient.IIdentityPr bucketUsage, - disksUsage, + DisksUsage(getKey(scope, "disks"), rangeObjs, nil, nil, providers, brands, cloudEnv, scope, cred, false), + DisksUsage(getKey(scope, "pending_delete_disks"), rangeObjs, nil, nil, providers, brands, cloudEnv, scope, cred, true), nicsUsage, @@ -431,6 +429,7 @@ func StorageUsage( rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string, + pendingDeleted bool, ) Usage { sPrefix := "storages" dPrefix := "all.disks" @@ -443,7 +442,7 @@ func StorageUsage( rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, - rbacutils.ScopeSystem, nil, + rbacutils.ScopeSystem, nil, pendingDeleted, ) count[sPrefix] = result.Capacity count[fmt.Sprintf("%s.virtual", sPrefix)] = result.CapacityVirtual @@ -461,9 +460,19 @@ func StorageUsage( return count } -func DisksUsage(dPrefix string, rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) Usage { +func DisksUsage( + dPrefix string, + rangeObjs []db.IStandaloneModel, + hostTypes []string, + resourceTypes []string, + providers []string, + brands []string, + cloudEnv string, + scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, + pendingDeleted bool, +) Usage { count := make(map[string]interface{}) - result := models.StorageManager.TotalCapacity(rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, scope, ownerId) + result := models.StorageManager.TotalCapacity(rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, scope, ownerId, pendingDeleted) count[dPrefix] = result.CapacityUsed count[fmt.Sprintf("%s.unready", dPrefix)] = result.CapacityUnready count[fmt.Sprintf("%s.attached", dPrefix)] = result.AttachedCapacity @@ -474,7 +483,7 @@ func DisksUsage(dPrefix string, rangeObjs []db.IStandaloneModel, hostTypes []str func WireUsage(rangeObjs []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) Usage { count := make(map[string]interface{}) - result := models.WireManager.TotalCount(rangeObjs, hostTypes, providers, brands, cloudEnv, rbacutils.ScopeSystem, nil) + result := models.WireManager.TotalCount(rangeObjs, hostTypes, providers, brands, cloudEnv, rbacutils.ScopeSystem, nil, false) count["wires"] = result.WiresCount count["networks"] = result.NetCount count["all.nics.guest"] = result.GuestNicCount @@ -483,16 +492,26 @@ func WireUsage(rangeObjs []db.IStandaloneModel, hostTypes []string, providers [] count["all.nics.group"] = result.GroupNicCount count["all.nics.lb"] = result.LbNicCount count["all.nics"] = result.NicCount() + + result = models.WireManager.TotalCount(rangeObjs, hostTypes, providers, brands, cloudEnv, rbacutils.ScopeSystem, nil, true) + count["all.nics.guest.pending_delete"] = result.GuestNicCount + count["all.nics.lb.pending_delete"] = result.LbNicCount + count["all.nics.pending_delete"] = result.GuestNicCount + result.LbNicCount + return count } func nicsUsage(rangeObjs []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) Usage { count := make(map[string]interface{}) - result := models.WireManager.TotalCount(rangeObjs, hostTypes, providers, brands, cloudEnv, scope, ownerId) + result := models.WireManager.TotalCount(rangeObjs, hostTypes, providers, brands, cloudEnv, scope, ownerId, false) count["nics.guest"] = result.GuestNicCount count["nics.group"] = result.GroupNicCount count["nics.lb"] = result.LbNicCount count["nics"] = result.GuestNicCount + result.GroupNicCount + result.LbNicCount + result = models.WireManager.TotalCount(rangeObjs, hostTypes, providers, brands, cloudEnv, scope, ownerId, true) + count["nics.guest.pending_delete"] = result.GuestNicCount + count["nics.lb.pending_delete"] = result.LbNicCount + count["nics.pending_delete"] = result.GuestNicCount + result.LbNicCount return count }