From 1d7fd20df4737aa4ef3db6bab80712a2e38c3162 Mon Sep 17 00:00:00 2001 From: Jian Qiu Date: Wed, 19 Jun 2019 02:45:52 +0800 Subject: [PATCH] misc bugfix (#1296) --- pkg/cloudcommon/db/quotas/handler.go | 2 ++ pkg/cloudcommon/db/quotas/quotas.go | 12 ++++---- pkg/compute/models/cloudaccounts.go | 3 +- pkg/compute/models/guests.go | 6 +++- pkg/compute/models/managedresource.go | 4 +++ pkg/compute/models/networks.go | 4 +-- pkg/compute/models/quotas.go | 15 ++++++---- pkg/compute/models/storages.go | 5 ++-- pkg/compute/models/wires.go | 35 +++++++++++++++++++----- pkg/compute/usages/handler.go | 2 +- pkg/httperrors/errors.go | 7 ++++- pkg/httperrors/httperrors.go | 4 +++ pkg/keystone/models/identity_provider.go | 6 +++- pkg/keystone/models/sync.go | 4 +++ 14 files changed, 82 insertions(+), 27 deletions(-) diff --git a/pkg/cloudcommon/db/quotas/handler.go b/pkg/cloudcommon/db/quotas/handler.go index 8477ccea1e..45dd1959d1 100644 --- a/pkg/cloudcommon/db/quotas/handler.go +++ b/pkg/cloudcommon/db/quotas/handler.go @@ -340,6 +340,8 @@ func (manager *SQuotaBaseManager) getDomainTotalQuota(ctx context.Context, targe if len(excludes) > 0 { q = q.NotIn("tenant_id", excludes) } + // dsable platform + q = q.IsNullOrEmpty("platform") rows, err := q.Rows() if err != nil && err != sql.ErrNoRows { return nil, err diff --git a/pkg/cloudcommon/db/quotas/quotas.go b/pkg/cloudcommon/db/quotas/quotas.go index 19fd3b9529..7373d9f82c 100644 --- a/pkg/cloudcommon/db/quotas/quotas.go +++ b/pkg/cloudcommon/db/quotas/quotas.go @@ -66,7 +66,7 @@ func (manager *SQuotaBaseManager) CancelPendingUsage(ctx context.Context, userCr lockman.LockClass(ctx, manager, mcclient.OwnerIdString(ownerId, scope)) defer lockman.ReleaseClass(ctx, manager, mcclient.OwnerIdString(ownerId, scope)) - return manager._cancelPendingUsage(ctx, userCred, scope, ownerId, platform, localUsage, cancelUsage) + return manager._cancelPendingUsage(ctx, userCred, scope, ownerId, nil, localUsage, cancelUsage) } func (manager *SQuotaBaseManager) _cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string, localUsage IQuota, cancelUsage IQuota) error { @@ -92,11 +92,11 @@ func (manager *SQuotaBaseManager) _cancelPendingUsage(ctx context.Context, userC } func (manager *SQuotaBaseManager) GetPendingUsage(ctx context.Context, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string, quota IQuota) error { - return manager.pendingStore.GetQuota(ctx, scope, ownerId, platform, quota) + return manager.pendingStore.GetQuota(ctx, scope, ownerId, nil, quota) } func (manager *SQuotaBaseManager) GetQuota(ctx context.Context, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string, quota IQuota) error { - err := manager.getQuotaInternal(ctx, scope, ownerId, platform, quota) + err := manager.getQuotaInternal(ctx, scope, ownerId, nil, quota) if err != nil && err != sql.ErrNoRows { return err } @@ -110,7 +110,7 @@ func (manager *SQuotaBaseManager) SetQuota(ctx context.Context, userCred mcclien lockman.LockClass(ctx, manager, mcclient.OwnerIdString(ownerId, scope)) defer lockman.ReleaseClass(ctx, manager, mcclient.OwnerIdString(ownerId, scope)) - return manager._setQuota(ctx, userCred, scope, ownerId, platform, quota) + return manager._setQuota(ctx, userCred, scope, ownerId, nil, quota) } func (manager *SQuotaBaseManager) _setQuota(ctx context.Context, userCred mcclient.TokenCredential, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string, quota IQuota) error { @@ -122,7 +122,7 @@ func (manager *SQuotaBaseManager) CheckQuota(ctx context.Context, userCred mccli lockman.LockClass(ctx, manager, mcclient.OwnerIdString(ownerId, scope)) defer lockman.ReleaseClass(ctx, manager, mcclient.OwnerIdString(ownerId, scope)) - return manager._checkQuota(ctx, userCred, scope, ownerId, platform, request) + return manager._checkQuota(ctx, userCred, scope, ownerId, nil, request) } func (manager *SQuotaBaseManager) _checkQuota(ctx context.Context, userCred mcclient.TokenCredential, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string, request IQuota) (IQuota, error) { @@ -161,7 +161,7 @@ func (manager *SQuotaBaseManager) CheckSetPendingQuota(ctx context.Context, user lockman.LockClass(ctx, manager, mcclient.OwnerIdString(ownerId, scope)) defer lockman.ReleaseClass(ctx, manager, mcclient.OwnerIdString(ownerId, scope)) - return manager._checkSetPendingQuota(ctx, userCred, scope, ownerId, platform, quota) + return manager._checkSetPendingQuota(ctx, userCred, scope, ownerId, nil, quota) } func (manager *SQuotaBaseManager) _checkSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string, quota IQuota) error { diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go index 8b74b7a1db..ae78625369 100644 --- a/pkg/compute/models/cloudaccounts.go +++ b/pkg/compute/models/cloudaccounts.go @@ -600,7 +600,8 @@ func (self *SCloudaccount) importSubAccount(ctx context.Context, userCred mcclie // find default project of domain t, err := db.TenantCacheManager.FindFirstProjectOfDomain(ownerId.GetProjectDomainId()) if err != nil { - log.Fatalf("cannot find a valid porject for domain %s", ownerId.GetProjectDomainId()) + log.Errorf("cannot find a valid porject for domain %s", ownerId.GetProjectDomainId()) + return nil, err } ownerId = &db.SOwnerId{ DomainId: t.DomainId, diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index e83a5eb501..1e1836d5c3 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -2791,7 +2791,7 @@ func (self *SGuest) createDiskOnStorage(ctx context.Context, userCred mcclient.T if storage.IsLocal() || billingType == billing_api.BILLING_TYPE_PREPAID || isWithServerCreate { autoDelete = true } - disk, err := storage.createDisk(diskName, diskConfig, userCred, self.ProjectId, autoDelete, self.IsSystem, + disk, err := storage.createDisk(diskName, diskConfig, userCred, self.GetOwnerId(), autoDelete, self.IsSystem, billingType, billingCycle) if err != nil { @@ -3294,6 +3294,8 @@ func (self *SGuest) GetJsonDescAtHypervisor(ctx context.Context, host *SHost) *j tc, _ := self.GetTenantCache(ctx) if tc != nil { desc.Add(jsonutils.NewString(tc.GetName()), "tenant") + desc.Add(jsonutils.NewString(tc.DomainId), "domain_id") + desc.Add(jsonutils.NewString(tc.Domain), "project_domain") } desc.Add(jsonutils.NewString(self.ProjectId), "tenant_id") @@ -3429,6 +3431,8 @@ func (self *SGuest) GetJsonDescAtBaremetal(ctx context.Context, host *SHost) *js tc, _ := self.GetTenantCache(ctx) if tc != nil { desc.Add(jsonutils.NewString(tc.GetName()), "tenant") + desc.Add(jsonutils.NewString(tc.DomainId), "domain_id") + desc.Add(jsonutils.NewString(tc.Domain), "project_domain") } desc.Add(jsonutils.NewString(self.ProjectId), "tenant_id") diff --git a/pkg/compute/models/managedresource.go b/pkg/compute/models/managedresource.go index 7283934a3f..94f423a678 100644 --- a/pkg/compute/models/managedresource.go +++ b/pkg/compute/models/managedresource.go @@ -358,6 +358,8 @@ type SCloudProviderInfo struct { ManagerId string `json:",omitempty"` ManagerProject string `json:",omitempty"` ManagerProjectId string `json:",omitempty"` + ManagerDomain string `json:",omitempty"` + ManagerDomainId string `json:",omitempty"` Region string `json:",omitempty"` RegionId string `json:",omitempty"` RegionExtId string `json:",omitempty"` @@ -418,6 +420,8 @@ func MakeCloudProviderInfo(region *SCloudregion, zone *SZone, provider *SCloudpr tc, err := db.TenantCacheManager.FetchTenantById(appctx.Background, provider.ProjectId) if err == nil { info.ManagerProject = tc.GetName() + info.ManagerDomain = tc.Domain + info.ManagerDomainId = tc.DomainId } } diff --git a/pkg/compute/models/networks.go b/pkg/compute/models/networks.go index 5eadaf6627..cb51a811f9 100644 --- a/pkg/compute/models/networks.go +++ b/pkg/compute/models/networks.go @@ -571,8 +571,8 @@ func (self *SNetwork) SyncWithCloudNetwork(ctx context.Context, userCred mcclien self.GuestIpMask = extNet.GetIpMask() self.GuestGateway = extNet.GetGateway() self.ServerType = extNet.GetServerType() - self.IsPublic = extNet.GetIsPublic() - self.PublicScope = string(extNet.GetPublicScope()) + // self.IsPublic = extNet.GetIsPublic() + // self.PublicScope = string(extNet.GetPublicScope()) self.AllocTimoutSeconds = extNet.GetAllocTimeoutSeconds() diff --git a/pkg/compute/models/quotas.go b/pkg/compute/models/quotas.go index 5f9c322df0..74663213f8 100644 --- a/pkg/compute/models/quotas.go +++ b/pkg/compute/models/quotas.go @@ -88,7 +88,8 @@ func (self *SQuota) FetchSystemQuota(scope rbacutils.TRbacScope) { func (self *SQuota) FetchUsage(ctx context.Context, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, name []string) error { diskSize := totalDiskSize(scope, ownerId, tristate.None, tristate.None, false) - net := totalGuestNicCount(scope, ownerId, nil, false) + // net := totalGuestNicCount(scope, ownerId, nil, false) + net := WireManager.TotalCount(nil, nil, nil, "", scope, ownerId) hypervisors := sets.NewString(api.HYPERVISORS...) hypervisors.Delete(api.HYPERVISOR_CONTAINER) guest := totalGuestResourceCount(scope, ownerId, nil, nil, hypervisors.List(), false, false, nil, nil, nil, "") @@ -101,11 +102,15 @@ func (self *SQuota) FetchUsage(ctx context.Context, scope rbacutils.TRbacScope, self.Cpu = guest.TotalCpuCount self.Memory = guest.TotalMemSize self.Storage = diskSize - self.Port = net.InternalNicCount + net.InternalVirtualNicCount self.Eip = eipUsage.Total() - self.Eport = net.ExternalNicCount + net.ExternalVirtualNicCount - self.Bw = net.InternalBandwidth - self.Ebw = net.ExternalBandwidth + // self.Port = net.InternalNicCount + net.InternalVirtualNicCount + // self.Eport = net.ExternalNicCount + net.ExternalVirtualNicCount + // self.Bw = net.InternalBandwidth + // self.Ebw = net.ExternalBandwidth + self.Port = net.GuestNicCount + net.GroupNicCount + net.LbNicCount + if scope == rbacutils.ScopeSystem { + self.Port += net.HostNicCount + net.ReservedCount + } self.Group = 0 self.Secgroup, _ = totalSecurityGroupCount(scope, ownerId) self.IsolatedDevice = guest.TotalIsolatedCount diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 559880b798..faf9f4221e 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -931,7 +931,7 @@ func (manager *SStorageManager) TotalCapacity(rangeObj db.IStandaloneModel, host } func (self *SStorage) createDisk(name string, diskConfig *api.DiskConfig, userCred mcclient.TokenCredential, - ownerProjId string, autoDelete bool, isSystem bool, + ownerId mcclient.IIdentityProvider, autoDelete bool, isSystem bool, billingType string, billingCycle string, ) (*SDisk, error) { disk := SDisk{} @@ -942,7 +942,8 @@ func (self *SStorage) createDisk(name string, diskConfig *api.DiskConfig, userCr disk.StorageId = self.Id disk.AutoDelete = autoDelete - disk.ProjectId = ownerProjId + disk.ProjectId = ownerId.GetProjectId() + disk.DomainId = ownerId.GetProjectDomainId() disk.IsSystem = isSystem disk.BillingType = billingType diff --git a/pkg/compute/models/wires.go b/pkg/compute/models/wires.go index 9c11594bc3..79848b7966 100644 --- a/pkg/compute/models/wires.go +++ b/pkg/compute/models/wires.go @@ -33,6 +33,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type SWireManager struct { @@ -334,9 +335,22 @@ func (manager *SWireManager) newFromCloudWire(ctx context.Context, userCred mccl return &wire, nil } -func (manager *SWireManager) totalCountQ(rangeObj db.IStandaloneModel, hostTypes []string, providers []string, cloudEnv string) *sqlchemy.SQuery { +func filterByScopeOwnerId(q *sqlchemy.SQuery, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) *sqlchemy.SQuery { + switch scope { + case rbacutils.ScopeSystem: + case rbacutils.ScopeDomain: + q = q.Equals("domain_id", ownerId.GetProjectDomainId()) + case rbacutils.ScopeProject: + q = q.Equals("tenant_id", ownerId.GetProjectId()) + } + return q +} + +func (manager *SWireManager) totalCountQ(rangeObj db.IStandaloneModel, hostTypes []string, providers []string, cloudEnv string, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) *sqlchemy.SQuery { guests := GuestManager.Query().SubQuery() hosts := HostManager.Query().SubQuery() + groups := GroupManager.Query().SubQuery() + lbs := LoadbalancerManager.Query().SubQuery() gNics := GuestnetworkManager.Query().SubQuery() gNicQ := gNics.Query( @@ -346,6 +360,7 @@ func (manager *SWireManager) totalCountQ(rangeObj db.IStandaloneModel, hostTypes gNicQ = gNicQ.Join(guests, sqlchemy.Equals(guests.Field("id"), gNics.Field("guest_id"))) gNicQ = gNicQ.Join(hosts, sqlchemy.Equals(guests.Field("host_id"), hosts.Field("id"))) gNicQ = gNicQ.Filter(sqlchemy.IsTrue(hosts.Field("enabled"))) + gNicQ = filterByScopeOwnerId(gNicQ, scope, ownerId) hNics := HostnetworkManager.Query().SubQuery() hNicQ := hNics.Query( @@ -362,20 +377,26 @@ func (manager *SWireManager) totalCountQ(rangeObj db.IStandaloneModel, hostTypes ) groupNics := GroupnetworkManager.Query().SubQuery() - grpNicSQ := groupNics.Query( + grpNicQ := groupNics.Query( groupNics.Field("network_id"), sqlchemy.COUNT("grpnic_count"), - ).GroupBy(groupNics.Field("network_id")).SubQuery() + ) + grpNicQ = grpNicQ.Join(groups, sqlchemy.Equals(groups.Field("id"), groupNics.Field("group_id"))) + grpNicQ = filterByScopeOwnerId(grpNicQ, scope, ownerId) lbNics := LoadbalancernetworkManager.Query().SubQuery() - lbNicSQ := lbNics.Query( + lbNicQ := lbNics.Query( lbNics.Field("network_id"), sqlchemy.COUNT("lbnic_count"), - ).GroupBy(lbNics.Field("network_id")).SubQuery() + ) + lbNicQ = lbNicQ.Join(lbs, sqlchemy.Equals(lbs.Field("id"), lbNics.Field("loadbalancer_id"))) + lbNicQ = filterByScopeOwnerId(lbNicQ, scope, ownerId) gNicSQ := gNicQ.GroupBy(gNics.Field("network_id")).SubQuery() hNicSQ := hNicQ.GroupBy(hNics.Field("network_id")).SubQuery() revSQ := revQ.GroupBy(revIps.Field("network_id")).SubQuery() + grpNicSQ := grpNicQ.GroupBy(groupNics.Field("network_id")).SubQuery() + lbNicSQ := lbNicQ.GroupBy(lbNics.Field("network_id")).SubQuery() networks := NetworkManager.Query().SubQuery() netQ := networks.Query( @@ -461,9 +482,9 @@ func (wstat WiresCountStat) NicCount() int { return wstat.GuestNicCount + wstat.HostNicCount + wstat.ReservedCount + wstat.GroupNicCount + wstat.LbNicCount } -func (manager *SWireManager) TotalCount(rangeObj db.IStandaloneModel, hostTypes []string, providers []string, cloudEnv string) WiresCountStat { +func (manager *SWireManager) TotalCount(rangeObj db.IStandaloneModel, hostTypes []string, providers []string, cloudEnv string, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) WiresCountStat { stat := WiresCountStat{} - err := manager.totalCountQ(rangeObj, hostTypes, providers, cloudEnv).First(&stat) + err := manager.totalCountQ(rangeObj, hostTypes, providers, cloudEnv, scope, ownerId).First(&stat) if err != nil { log.Errorf("Wire total count: %v", err) } diff --git a/pkg/compute/usages/handler.go b/pkg/compute/usages/handler.go index 949ec1e669..f9f559d0db 100644 --- a/pkg/compute/usages/handler.go +++ b/pkg/compute/usages/handler.go @@ -455,7 +455,7 @@ func StorageUsage(prefix string, rangeObj db.IStandaloneModel, hostTypes []strin func WireUsage(rangeObj db.IStandaloneModel, hostTypes []string, providers []string, cloudEnv string) Usage { count := make(map[string]interface{}) - result := models.WireManager.TotalCount(rangeObj, hostTypes, providers, cloudEnv) + result := models.WireManager.TotalCount(rangeObj, hostTypes, providers, cloudEnv, rbacutils.ScopeSystem, nil) count["wires"] = result.WiresCount count["networks"] = result.NetCount count["all.nics.guest"] = result.GuestNicCount diff --git a/pkg/httperrors/errors.go b/pkg/httperrors/errors.go index 6f18710217..9b8abb292c 100644 --- a/pkg/httperrors/errors.go +++ b/pkg/httperrors/errors.go @@ -267,5 +267,10 @@ func NewGeneralError(err error) *httputils.JSONClientError { func NewProtectedResourceError(msg string, params ...interface{}) *httputils.JSONClientError { msg, err := errorMessage(msg, params...) - return NewJsonClientError(403, "ProtectedResourceError(", msg, err) + return NewJsonClientError(403, "ProtectedResourceError", msg, err) +} + +func NewNoProjectError(msg string, params ...interface{}) *httputils.JSONClientError { + msg, err := errorMessage(msg, params...) + return NewJsonClientError(403, "NoProjectError", msg, err) } diff --git a/pkg/httperrors/httperrors.go b/pkg/httperrors/httperrors.go index 20ddd26dba..adc4f38e4c 100644 --- a/pkg/httperrors/httperrors.go +++ b/pkg/httperrors/httperrors.go @@ -144,3 +144,7 @@ func TimeoutError(w http.ResponseWriter, msg string, params ...interface{}) { func ProtectedResourceError(w http.ResponseWriter, msg string, params ...interface{}) { JsonClientError(w, NewProtectedResourceError(msg, params...)) } + +func NoProjectError(w http.ResponseWriter, msg string, params ...interface{}) { + JsonClientError(w, NewNoProjectError(msg, params...)) +} diff --git a/pkg/keystone/models/identity_provider.go b/pkg/keystone/models/identity_provider.go index c790ca1c60..fa589f0186 100644 --- a/pkg/keystone/models/identity_provider.go +++ b/pkg/keystone/models/identity_provider.go @@ -361,7 +361,11 @@ func (self *SIdentityProvider) getSyncIntervalSeconds() int { } func (self *SIdentityProvider) NeedSync() bool { - if self.Driver != api.IdentityProviderSyncFull { + drvCls := driver.GetDriverClass(self.Driver) + if drvCls == nil { + return false + } + if drvCls.SyncMethod() != api.IdentityProviderSyncFull { return false } if !self.LastSync.IsZero() && time.Now().Sub(self.LastSync) < time.Duration(self.getSyncIntervalSeconds())*time.Second { diff --git a/pkg/keystone/models/sync.go b/pkg/keystone/models/sync.go index 69c1fcd4e8..eacc49aa4b 100644 --- a/pkg/keystone/models/sync.go +++ b/pkg/keystone/models/sync.go @@ -45,20 +45,24 @@ func AutoSyncIdentityProviderTask(ctx context.Context, userCred mcclient.TokenCr func syncIdentityProvider(ctx context.Context, userCred mcclient.TokenCredential, idp *SIdentityProvider) error { if idp.SyncStatus != api.IdentitySyncStatusIdle { + log.Debugf("IDP %s cannot sync in non-idle status", idp.Name) return nil } if !idp.CanSync() { + log.Debugf("IDP %s cannot sync", idp.Name) return nil } if !idp.NeedSync() { + log.Debugf("IDP %s no need to sync", idp.Name) return nil } drvCls := driver.GetDriverClass(idp.Driver) if drvCls.SyncMethod() == api.IdentityProviderSyncLocal { // skip, no need to sync + log.Debugf("IDP %s is local, no need to sync", idp.Name) return nil } submitIdpSyncTask(ctx, userCred, idp)