From ce032ecf1776bce10f90aa3337daf040d2578139 Mon Sep 17 00:00:00 2001 From: Qu Xuan Date: Thu, 3 Sep 2020 20:41:15 +0800 Subject: [PATCH] fix: optimzed sync --- pkg/apis/cloudid/clouduser.go | 1 - pkg/cloudcommon/db/db_dispatcher.go | 51 ++++++++++--------- .../db/statusinfrasuserresource.go | 2 +- pkg/cloudid/models/cloudaccount.go | 34 ------------- pkg/cloudid/models/clouduser.go | 20 ++------ 5 files changed, 32 insertions(+), 76 deletions(-) diff --git a/pkg/apis/cloudid/clouduser.go b/pkg/apis/cloudid/clouduser.go index 477370fd12..1c8ed32fae 100644 --- a/pkg/apis/cloudid/clouduser.go +++ b/pkg/apis/cloudid/clouduser.go @@ -37,7 +37,6 @@ const ( type ClouduserCreateInput struct { apis.StatusDomainLevelUserResourceCreateInput apis.StatusBaseResourceCreateInput - apis.ProjectizedResourceInput // 云订阅ID, 若此参数为空, 则cloudpolicy_ids的权限会绑定到此账号的所有订阅, 若不为空则cloudpolicy_ids仅绑定的指定的订阅 // 此参数仅对Google,Azure生效 diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index ebc8f37ee4..747d0e58ec 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -1124,29 +1124,31 @@ func _doCreateItem( log.Errorf("doCreateItem: fail to decode json data %s", data) return nil, fmt.Errorf("fail to decode json data %s", data) } - var err error - var generateName string - if manager.EnableGenerateName() { - generateName, _ = dataDict.GetString("generate_name") - if len(generateName) > 0 { - dataDict.Remove("generate_name") - newName, err := GenerateName2(manager, ownerId, generateName, nil, baseIndex) - if err != nil { - return nil, err - } - dataDict.Add(jsonutils.NewString(newName), "name") - } /*else { - name, _ := data.GetString("name") - if len(name) > 0 { - err = NewNameValidator(manager, ownerId, name) - if err != nil { - return nil, err - } - } - }*/ + var err error + + info := struct { + GenerateName string + Name string + }{} + dataDict.Unmarshal(&info) + if !manager.EnableGenerateName() { + if len(info.GenerateName) > 0 && len(info.Name) == 0 { + info.Name = info.GenerateName + } + info.GenerateName = "" } + if len(info.GenerateName) > 0 { + info.Name, err = GenerateName2(manager, ownerId, info.GenerateName, nil, baseIndex) + if err != nil { + return nil, err + } + } + + dataDict.Set("name", jsonutils.NewString(info.Name)) + dataDict.Remove("generate_name") + if batchCreate { dataDict, err = manager.BatchCreateValidateCreateData(ctx, userCred, ownerId, query, dataDict) } else { @@ -1158,9 +1160,8 @@ func _doCreateItem( } // run name validation after validate create data uniqValues := manager.FetchUniqValues(ctx, dataDict) - name, _ := dataDict.GetString("name") - if len(name) > 0 { - err = NewNameValidator(manager, ownerId, name, uniqValues) + if len(info.Name) > 0 { + err = NewNameValidator(manager, ownerId, info.Name, uniqValues) if err != nil { return nil, err } @@ -1188,9 +1189,9 @@ func _doCreateItem( return nil, httperrors.NewGeneralError(err) } // save generateName - if len(generateName) > 0 { + if len(info.GenerateName) > 0 { if standaloneMode, ok := model.(IStandaloneModel); ok { - standaloneMode.SetMetadata(ctx, "generate_name", generateName, userCred) + standaloneMode.SetMetadata(ctx, "generate_name", info.GenerateName, userCred) } } // HACK: set data same as dataDict diff --git a/pkg/cloudcommon/db/statusinfrasuserresource.go b/pkg/cloudcommon/db/statusinfrasuserresource.go index 524980aebc..23362f1ec2 100644 --- a/pkg/cloudcommon/db/statusinfrasuserresource.go +++ b/pkg/cloudcommon/db/statusinfrasuserresource.go @@ -34,7 +34,7 @@ type SStatusDomainLevelUserResourceBase struct { SStatusDomainLevelResourceBase // 本地用户Id - OwnerId string `width:"128" charset:"ascii" index:"true" list:"user" nullable:"false" create:"required"` + OwnerId string `width:"128" charset:"ascii" index:"true" list:"user" nullable:"false" create:"optional"` } type SStatusDomainLevelUserResourceBaseManager struct { diff --git a/pkg/cloudid/models/cloudaccount.go b/pkg/cloudid/models/cloudaccount.go index 2e3cda40eb..9539516307 100644 --- a/pkg/cloudid/models/cloudaccount.go +++ b/pkg/cloudid/models/cloudaccount.go @@ -19,7 +19,6 @@ import ( "database/sql" "net/http" "net/url" - "time" "golang.org/x/net/http/httpproxy" @@ -51,8 +50,6 @@ type SCloudaccountManager struct { } var CloudaccountManager *SCloudaccountManager -var isCloudacountSynced bool -var providersForSystemPolicySynced []string func init() { CloudaccountManager = &SCloudaccountManager{ @@ -64,8 +61,6 @@ func init() { ), } CloudaccountManager.SetVirtualObject(CloudaccountManager) - isCloudacountSynced = false - providersForSystemPolicySynced = []string{} } type SCloudaccount struct { @@ -280,18 +275,6 @@ func (manager *SCloudaccountManager) SyncCloudaccounts(ctx context.Context, user result = account.syncCloudprovider(ctx, userCred) log.Debugf("sync cloudprovider for cloudaccount %s(%s) result: %s", account.Name, account.Id, result.Result()) } - isCloudacountSynced = true -} - -func waitForSync() { - now := time.Now() - for !isCloudacountSynced { - log.Infof("cloudaccount not sync, wait for 10 seconds") - time.Sleep(time.Second * 10) - if time.Now().Sub(now) > time.Minute*3 { - break - } - } } func (self SCloudaccount) GetGlobalId() string { @@ -779,10 +762,6 @@ func (self *SCloudaccount) syncSystemCloudpoliciesFromCloud(ctx context.Context, result.Add() } - if !utils.IsInStringArray(self.Provider, providersForSystemPolicySynced) { - providersForSystemPolicySynced = append(providersForSystemPolicySynced, self.Provider) - } - log.Infof("Sync %s system policies result: %s", self.Provider, result.Result()) return nil } @@ -952,14 +931,7 @@ func (manager *SCloudaccountManager) SyncCloudidSystemPolicies(ctx context.Conte log.Errorf("GetSupportCloudIdAccounts error: %v", err) return } - providers := []string{} for i := range accounts { - if isStart && utils.IsInStringArray(accounts[i].Provider, providersForSystemPolicySynced) { - continue - } - if utils.IsInStringArray(accounts[i].Provider, providers) { - continue - } _, err := accounts[i].GetProvider() // 检查账号是否可以正常连接 if err != nil { log.Errorf("GetProvider for account %s(%s) error: %v", accounts[i].Name, accounts[i].Provider, err) @@ -970,7 +942,6 @@ func (manager *SCloudaccountManager) SyncCloudidSystemPolicies(ctx context.Conte log.Errorf("StartSystemCloudpolicySyncTask for account %s(%s) error: %v", accounts[i].Name, accounts[i].Provider, err) continue } - providers = append(providers, accounts[i].Provider) } } @@ -985,17 +956,12 @@ func (self *SCloudaccount) StartSystemCloudpolicySyncTask(ctx context.Context, u } func (manager *SCloudaccountManager) SyncCloudidResources(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) { - waitForSync() accounts, err := manager.GetSupportCloudIdAccounts() if err != nil { log.Errorf("GetSupportCloudIdAccounts error: %v", err) return } for i := range accounts { - if !utils.IsInStringArray(accounts[i].Provider, providersForSystemPolicySynced) { - log.Warningf("%s system policies not syncing, sync for next loop", accounts[i].Provider) - continue - } err = accounts[i].StartSyncCloudIdResourcesTask(ctx, userCred, "") if err != nil { log.Errorf("StartSyncCloudIdResourcesTask for account %s(%s) error: %v", accounts[i].Name, accounts[i].Provider, err) diff --git a/pkg/cloudid/models/clouduser.go b/pkg/cloudid/models/clouduser.go index 7e82b553c9..a5779841d3 100644 --- a/pkg/cloudid/models/clouduser.go +++ b/pkg/cloudid/models/clouduser.go @@ -73,9 +73,6 @@ type SClouduser struct { MobilePhone string `width:"36" charset:"ascii" nullable:"true" list:"user" create:"domain_optional"` // 邮箱地址 Email string `width:"36" charset:"ascii" nullable:"true" list:"user" create:"domain_optional"` - - // 项目Id - ProjectId string `name:"tenant_id" width:"128" charset:"ascii" nullable:"false" index:"true" list:"user" json:"tenant_id"` } func (manager *SClouduserManager) EnableGenerateName() bool { @@ -93,12 +90,7 @@ func (manager *SClouduserManager) GetResourceCount() ([]db.SScopeResourceCount, if err != nil { return nil, errors.Wrap(err, "CalculateResourceCount.owner_id") } - q = manager.Query() - projectCnt, err := db.CalculateResourceCount(q, "project_id") - if err != nil { - return nil, errors.Wrapf(err, "CalculateResourceCount.project_id") - } - return append(domainCnt, append(userCnt, projectCnt...)...), nil + return append(domainCnt, userCnt...), nil } func (manager *SClouduserManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { @@ -371,17 +363,15 @@ func (manager *SClouduserManager) ValidateCreateData(ctx context.Context, userCr return input, err } - if len(input.Name) == 0 && len(input.OwnerId) == 0 { - return input, httperrors.NewMissingParameterError("missing name or owner_id") - } - - if len(input.Name) == 0 { + if len(input.OwnerId) > 0 { user, err := db.UserCacheManager.FetchUserById(ctx, input.OwnerId) if err != nil { return input, errors.Wrap(err, "FetchUserById") } input.OwnerId = user.Id - input.Name = user.Name + if len(input.Name) == 0 { + input.Name = user.Name + } } policyExternalIds := []string{}