From 971a592b09b6844c08824b4fcacae64222c34d16 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Tue, 9 Jul 2019 23:35:08 +0800 Subject: [PATCH 1/2] fix: possible reduce timeout for quota list --- pkg/cloudcommon/db/quotas/handler.go | 9 +++++--- pkg/cloudcommon/db/tenantcache.go | 32 +++++++++++++++++++++------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/pkg/cloudcommon/db/quotas/handler.go b/pkg/cloudcommon/db/quotas/handler.go index ff144243a8..d250e7c129 100644 --- a/pkg/cloudcommon/db/quotas/handler.go +++ b/pkg/cloudcommon/db/quotas/handler.go @@ -466,7 +466,8 @@ func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, targetDomainId if len(projectId) > 0 { quota.Set("tenant_id", jsonutils.NewString(projectId)) quota.Set("domain_id", jsonutils.NewString(domainId)) - project, err := db.TenantCacheManager.FetchTenantById(ctx, projectId) + // fetch without cache expiration check + project, err := db.TenantCacheManager.FetchTenantByIdWithoutExpireCheck(ctx, projectId) if err != nil { return nil, err } @@ -474,7 +475,8 @@ func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, targetDomainId quota.Set("project_domain", jsonutils.NewString(project.Domain)) } else { quota.Set("domain_id", jsonutils.NewString(domainId)) - domain, err := db.TenantCacheManager.FetchDomainById(ctx, domainId) + // fetch without cache expiration check + domain, err := db.TenantCacheManager.FetchDomainByIdWithoutExpireCheck(ctx, domainId) if err != nil { return nil, err } @@ -497,7 +499,8 @@ func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, targetDomainId return nil, httperrors.NewInternalServerError("query domain initial quotas %s", err) } quota.Set("domain_id", jsonutils.NewString(targetDomainId)) - domain, err := db.TenantCacheManager.FetchDomainById(ctx, targetDomainId) + // fetch without cache expiration check + domain, err := db.TenantCacheManager.FetchDomainByIdWithoutExpireCheck(ctx, targetDomainId) if err != nil { return nil, err } diff --git a/pkg/cloudcommon/db/tenantcache.go b/pkg/cloudcommon/db/tenantcache.go index 8a0daf303e..07c7e48934 100644 --- a/pkg/cloudcommon/db/tenantcache.go +++ b/pkg/cloudcommon/db/tenantcache.go @@ -97,7 +97,7 @@ func (manager *STenantCacheManager) updateTenantCache(userCred mcclient.TokenCre userCred.GetProjectDomainId(), userCred.GetProjectDomain()) } -func (manager *STenantCacheManager) fetchTenant(ctx context.Context, idStr string, isDomain bool, filter func(q *sqlchemy.SQuery) *sqlchemy.SQuery) (*STenant, error) { +func (manager *STenantCacheManager) fetchTenant(ctx context.Context, idStr string, isDomain bool, noExpireCheck bool, filter func(q *sqlchemy.SQuery) *sqlchemy.SQuery) (*STenant, error) { q := manager.Query() if isDomain { q = q.Equals("domain_id", identityapi.KeystoneDomainRoot) @@ -114,7 +114,7 @@ func (manager *STenantCacheManager) fetchTenant(ctx context.Context, idStr strin return nil, errors.Wrap(err, "query") } else if tobj != nil { tenant := tobj.(*STenant) - if !tenant.IsExpired() { + if noExpireCheck || !tenant.IsExpired() { return tenant, nil } } @@ -137,7 +137,7 @@ func (t *STenant) IsExpired() bool { } func (manager *STenantCacheManager) FetchTenantByIdOrName(ctx context.Context, idStr string) (*STenant, error) { - return manager.fetchTenant(ctx, idStr, false, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { + return manager.fetchTenant(ctx, idStr, false, false, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { if stringutils2.IsUtf8(idStr) { return q.Equals("name", idStr) } else { @@ -150,13 +150,21 @@ func (manager *STenantCacheManager) FetchTenantByIdOrName(ctx context.Context, i } func (manager *STenantCacheManager) FetchTenantById(ctx context.Context, idStr string) (*STenant, error) { - return manager.fetchTenant(ctx, idStr, false, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { + return manager.fetchTenantById(ctx, idStr, false) +} + +func (manager *STenantCacheManager) FetchTenantByIdWithoutExpireCheck(ctx context.Context, idStr string) (*STenant, error) { + return manager.fetchTenantById(ctx, idStr, false) +} + +func (manager *STenantCacheManager) fetchTenantById(ctx context.Context, idStr string, noExpireCheck bool) (*STenant, error) { + return manager.fetchTenant(ctx, idStr, false, noExpireCheck, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { return q.Filter(sqlchemy.Equals(q.Field("id"), idStr)) }) } func (manager *STenantCacheManager) FetchTenantByName(ctx context.Context, idStr string) (*STenant, error) { - return manager.fetchTenant(ctx, idStr, false, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { + return manager.fetchTenant(ctx, idStr, false, false, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { return q.Filter(sqlchemy.Equals(q.Field("name"), idStr)) }) } @@ -185,7 +193,7 @@ func (manager *STenantCacheManager) fetchTenantFromKeystone(ctx context.Context, } func (manager *STenantCacheManager) FetchDomainByIdOrName(ctx context.Context, idStr string) (*STenant, error) { - return manager.fetchTenant(ctx, idStr, true, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { + return manager.fetchTenant(ctx, idStr, true, false, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { if stringutils2.IsUtf8(idStr) { return q.Equals("name", idStr) } else { @@ -198,13 +206,21 @@ func (manager *STenantCacheManager) FetchDomainByIdOrName(ctx context.Context, i } func (manager *STenantCacheManager) FetchDomainById(ctx context.Context, idStr string) (*STenant, error) { - return manager.fetchTenant(ctx, idStr, true, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { + return manager.fetchDomainById(ctx, idStr, false) +} + +func (manager *STenantCacheManager) FetchDomainByIdWithoutExpireCheck(ctx context.Context, idStr string) (*STenant, error) { + return manager.fetchDomainById(ctx, idStr, true) +} + +func (manager *STenantCacheManager) fetchDomainById(ctx context.Context, idStr string, noExpireCheck bool) (*STenant, error) { + return manager.fetchTenant(ctx, idStr, true, noExpireCheck, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { return q.Filter(sqlchemy.Equals(q.Field("id"), idStr)) }) } func (manager *STenantCacheManager) FetchDomainByName(ctx context.Context, idStr string) (*STenant, error) { - return manager.fetchTenant(ctx, idStr, true, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { + return manager.fetchTenant(ctx, idStr, true, false, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { return q.Filter(sqlchemy.Equals(q.Field("name"), idStr)) }) } From c56e6c32e7cb44d007f62a09cd131f2db6a0da29 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Wed, 10 Jul 2019 00:09:40 +0800 Subject: [PATCH 2/2] fix: disk quota usage should exclude pending_deleted disks --- pkg/compute/models/disks.go | 7 ++++++- pkg/compute/models/quotas.go | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 4bef4c4d5c..b0b8ef2921 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -1324,7 +1324,7 @@ func (manager *SDiskManager) newFromCloudDisk(ctx context.Context, userCred mccl return &disk, nil } -func totalDiskSize(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, active tristate.TriState, ready tristate.TriState, includeSystem bool) int { +func totalDiskSize(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, active tristate.TriState, ready tristate.TriState, includeSystem bool, pendingDelete bool) int { disks := DiskManager.Query().SubQuery() q := disks.Query(sqlchemy.SUM("total", disks.Field("disk_size"))) if !active.IsNone() { @@ -1358,6 +1358,11 @@ func totalDiskSize(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvide q = q.Filter(sqlchemy.OR(sqlchemy.IsNull(disks.Field("is_system")), sqlchemy.IsFalse(disks.Field("is_system")))) } + if pendingDelete { + q = q.Filter(sqlchemy.IsTrue(disks.Field("pending_deleted"))) + } else { + q = q.Filter(sqlchemy.OR(sqlchemy.IsNull(disks.Field("pending_deleted")), sqlchemy.IsFalse(disks.Field("pending_deleted")))) + } row := q.Row() size := sql.NullInt64{} diff --git a/pkg/compute/models/quotas.go b/pkg/compute/models/quotas.go index efcf6a708c..bfbdb3fb73 100644 --- a/pkg/compute/models/quotas.go +++ b/pkg/compute/models/quotas.go @@ -90,7 +90,7 @@ func (self *SQuota) FetchSystemQuota(scope rbacutils.TRbacScope, ownerId mcclien } func (self *SQuota) FetchUsage(ctx context.Context, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, name []string) error { - diskSize := totalDiskSize(scope, ownerId, tristate.None, tristate.None, false) + diskSize := totalDiskSize(scope, ownerId, tristate.None, tristate.None, false, false) net := totalGuestNicCount(scope, ownerId, nil, false) lbnic, _ := totalLBNicCount(scope, ownerId) // net := WireManager.TotalCount(nil, nil, nil, "", scope, ownerId)