opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-08-30 17:13:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

add more optimization

Browse Source
This commit is contained in:
Qiu Jian
2019-03-28 22:58:16 +08:00
committed by Yousong Zhou
parent d86068aa7b
commit 8ef19141a2
10 changed files with 37 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/cloudcommon/agent/doc.go
+1
View File
@@ -0,0 +1 @@
package agent // import "yunion.io/x/onecloud/pkg/cloudcommon/agent"
pkg/cloudcommon/db/taskman/tasks.go
+1
View File
@@ -51,6 +51,7 @@ var TaskManager *STaskManager
func init() {
TaskManager = &STaskManager{SResourceBaseManager: db.NewResourceBaseManager(STask{}, "tasks_tbl", "task", "tasks")}
TaskManager.TableSpec().AddIndex(true, "deleted", "obj_name", "obj_id", "created_at", "stage")
}
type STask struct {
pkg/compute/models/cloudaccounts.go
+1 -1
View File
@@ -1173,7 +1173,7 @@ func (account *SCloudaccount) SubmitSyncAccountTask(ctx context.Context, userCre
syncRange := SSyncRange{FullSync: true}
providers := account.GetEnabledCloudproviders()
for i := range providers {
providers[i].syncCloudproviderRegions(ctx, userCred, &syncRange, nil, autoSync)
providers[i].syncCloudproviderRegions(ctx, userCred, syncRange, nil, autoSync)
syncCnt += 1
}
}
pkg/compute/models/cloudproviderregions.go
+11 -8
View File
@@ -204,7 +204,8 @@ func (self *SCloudproviderregion) markSyncing(userCred mcclient.TokenCredential)
return nil
}
func (self *SCloudproviderregion) markEndSync(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, deepSync bool) error {
func (self *SCloudproviderregion) markEndSync(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, deepSync *bool) error {
log.Debugf("markEndSync deepSync %v", *deepSync)
err := self.markEndSyncInternal(userCred, syncResults, deepSync)
if err != nil {
return err
@@ -216,12 +217,12 @@ func (self *SCloudproviderregion) markEndSync(ctx context.Context, userCred mccl
return nil
}
func (self *SCloudproviderregion) markEndSyncInternal(userCred mcclient.TokenCredential, syncResults SSyncResultSet, deepSync bool) error {
func (self *SCloudproviderregion) markEndSyncInternal(userCred mcclient.TokenCredential, syncResults SSyncResultSet, deepSync *bool) error {
_, err := db.Update(self, func() error {
self.SyncStatus = CLOUD_PROVIDER_SYNC_STATUS_IDLE
self.LastSyncEndAt = timeutils.UtcNow()
self.SyncResults = jsonutils.Marshal(syncResults)
if deepSync {
if *deepSync {
self.LastDeepSyncAt = timeutils.UtcNow()
}
return nil
@@ -249,11 +250,11 @@ func (set SSyncResultSet) Add(manager db.IModelManager, result compare.SyncResul
res.DelErrCnt += result.DelErrCnt
}
func (self *SCloudproviderregion) DoSync(ctx context.Context, userCred mcclient.TokenCredential, syncRange *SSyncRange) error {
func (self *SCloudproviderregion) DoSync(ctx context.Context, userCred mcclient.TokenCredential, syncRange SSyncRange) error {
syncResults := SSyncResultSet{}
self.markSyncing(userCred)
defer self.markEndSync(ctx, userCred, syncResults, syncRange.DeepSync)
defer self.markEndSync(ctx, userCred, syncResults, &syncRange.DeepSync)
localRegion := self.GetRegion()
provider := self.GetProvider()
@@ -263,20 +264,22 @@ func (self *SCloudproviderregion) DoSync(ctx context.Context, userCred mcclient.
return err
}
log.Debugf("need to do deep sync ... %v", syncRange.DeepSync)
if !syncRange.DeepSync {
intval := self.getSyncIntervalSeconds(nil)
if self.LastDeepSyncAt.IsZero() || (time.Now().Sub(self.LastDeepSyncAt) > time.Duration(intval)*time.Second*8 && rand.Float32() < 0.5) {
syncRange.DeepSync = true
}
}
log.Debugf("no need to do deep sync ... %v", syncRange.DeepSync)
if localRegion.isManaged() {
remoteRegion, err := driver.GetIRegionById(localRegion.ExternalId)
if err == nil {
err = syncPublicCloudProviderInfo(ctx, userCred, syncResults, provider, driver, localRegion, remoteRegion, syncRange)
err = syncPublicCloudProviderInfo(ctx, userCred, syncResults, provider, driver, localRegion, remoteRegion, &syncRange)
}
} else {
err = syncOnPremiseCloudProviderInfo(ctx, userCred, syncResults, provider, driver, syncRange)
err = syncOnPremiseCloudProviderInfo(ctx, userCred, syncResults, provider, driver, &syncRange)
}
if err != nil {
@@ -297,7 +300,7 @@ func (self *SCloudproviderregion) getSyncTaskKey() string {
}
}
func (self *SCloudproviderregion) submitSyncTask(userCred mcclient.TokenCredential, syncRange *SSyncRange, waitChan chan bool) {
func (self *SCloudproviderregion) submitSyncTask(userCred mcclient.TokenCredential, syncRange SSyncRange, waitChan chan bool) {
self.markStartSync(userCred)
RunSyncCloudproviderRegionTask(self.getSyncTaskKey(), func() {
self.DoSync(context.Background(), userCred, syncRange)
pkg/compute/models/cloudproviders.go
+2 -2
View File
@@ -900,7 +900,7 @@ func (provider *SCloudprovider) GetCloudproviderRegions() []SCloudproviderregion
return CloudproviderRegionManager.fetchRecordsByQuery(q)
}
func (provider *SCloudprovider) syncCloudproviderRegions(ctx context.Context, userCred mcclient.TokenCredential, syncRange *SSyncRange, wg *sync.WaitGroup, autoSync bool) {
func (provider *SCloudprovider) syncCloudproviderRegions(ctx context.Context, userCred mcclient.TokenCredential, syncRange SSyncRange, wg *sync.WaitGroup, autoSync bool) {
provider.markSyncing(userCred)
cprs := provider.GetCloudproviderRegions()
syncCnt := 0
@@ -924,7 +924,7 @@ func (provider *SCloudprovider) syncCloudproviderRegions(ctx context.Context, us
}
}
func (provider *SCloudprovider) SyncCallSyncCloudproviderRegions(ctx context.Context, userCred mcclient.TokenCredential, syncRange *SSyncRange) {
func (provider *SCloudprovider) SyncCallSyncCloudproviderRegions(ctx context.Context, userCred mcclient.TokenCredential, syncRange SSyncRange) {
var wg sync.WaitGroup
provider.syncCloudproviderRegions(ctx, userCred, syncRange, &wg, false)
wg.Wait()
pkg/compute/models/cloudsync.go
+9 -7
View File
@@ -37,6 +37,7 @@ func (self *SSyncableBaseResource) CanSync() bool {
type sStoragecacheSyncPair struct {
local *SStoragecache
remote cloudprovider.ICloudStoragecache
isNew bool
}
func (pair *sStoragecacheSyncPair) syncCloudImages(ctx context.Context, userCred mcclient.TokenCredential) compare.SyncResult {
@@ -328,7 +329,7 @@ func syncZoneStorages(ctx context.Context, userCred mcclient.TokenCredential, sy
func syncStorageCaches(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, localStorage *SStorage, remoteStorage cloudprovider.ICloudStorage) (cachePair sStoragecacheSyncPair) {
remoteCache := remoteStorage.GetIStoragecache()
localCache, err := StoragecacheManager.SyncWithCloudStoragecache(ctx, userCred, remoteCache)
localCache, isNew, err := StoragecacheManager.SyncWithCloudStoragecache(ctx, userCred, remoteCache)
if err != nil {
msg := fmt.Sprintf("SyncWithCloudStoragecache for storage %s failed %s", remoteStorage.GetName(), err)
log.Errorf(msg)
@@ -341,6 +342,7 @@ func syncStorageCaches(ctx context.Context, userCred mcclient.TokenCredential, p
}
cachePair.local = localCache
cachePair.remote = remoteCache
cachePair.isNew = isNew
return
}
@@ -810,9 +812,9 @@ func syncPublicCloudProviderInfo(
syncRegionLoadbalancerCertificates(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
syncRegionLoadbalancers(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
if syncRange.DeepSync {
log.Debugf("storageCachePairs count %d", len(storageCachePairs))
for i := range storageCachePairs {
log.Debugf("storageCachePairs count %d", len(storageCachePairs))
for i := range storageCachePairs {
if storageCachePairs[i].isNew || syncRange.DeepSync {
result := storageCachePairs[i].syncCloudImages(ctx, userCred)
syncResults.Add(StoragecachedimageManager, result)
@@ -876,9 +878,9 @@ func syncOnPremiseCloudProviderInfo(
syncHostVMs(ctx, userCred, syncResults, provider, driver, &localHosts[i], remoteHosts[i], syncRange)
}
if syncRange.DeepSync {
log.Debugf("storageCachePairs count %d", len(storageCachePairs))
for i := range storageCachePairs {
log.Debugf("storageCachePairs count %d", len(storageCachePairs))
for i := range storageCachePairs {
if storageCachePairs[i].isNew || syncRange.DeepSync {
result := storageCachePairs[i].syncCloudImages(ctx, userCred)
syncResults.Add(StoragecachedimageManager, result)
msg := result.Result()
pkg/compute/models/guests.go
+1 -1
View File
@@ -197,7 +197,7 @@ type SGuest struct {
KeypairId string `width:"36" charset:"ascii" nullable:"true" list:"user" create:"optional"` // Column(VARCHAR(36, charset='ascii'), nullable=True)
HostId string `width:"36" charset:"ascii" nullable:"true" list:"admin" get:"admin"` // Column(VARCHAR(36, charset='ascii'), nullable=True)
HostId string `width:"36" charset:"ascii" nullable:"true" list:"admin" get:"admin" index:"true"` // Column(VARCHAR(36, charset='ascii'), nullable=True)
BackupHostId string `width:"36" charset:"ascii" nullable:"true" list:"admin" get:"admin"`
Vga string `width:"36" charset:"ascii" nullable:"true" list:"user" update:"user" create:"optional"` // Column(VARCHAR(36, charset='ascii'), nullable=True)
pkg/compute/models/storagecaches.go
+9 -4
View File
@@ -137,22 +137,27 @@ func (self *SStoragecache) getHostId() (string, error) {
return ret, nil
}
func (manager *SStoragecacheManager) SyncWithCloudStoragecache(ctx context.Context, userCred mcclient.TokenCredential, cloudCache cloudprovider.ICloudStoragecache) (*SStoragecache, error) {
func (manager *SStoragecacheManager) SyncWithCloudStoragecache(ctx context.Context, userCred mcclient.TokenCredential, cloudCache cloudprovider.ICloudStoragecache) (*SStoragecache, bool, error) {
lockman.LockClass(ctx, manager, manager.GetOwnerId(userCred))
defer lockman.ReleaseClass(ctx, manager, manager.GetOwnerId(userCred))
localCacheObj, err := manager.FetchByExternalId(cloudCache.GetGlobalId())
if err != nil {
if err == sql.ErrNoRows {
return manager.newFromCloudStoragecache(ctx, userCred, cloudCache)
localCache, err := manager.newFromCloudStoragecache(ctx, userCred, cloudCache)
if err != nil {
return nil, false, err
} else {
return localCache, true, nil
}
} else {
log.Errorf("%s", err)
return nil, err
return nil, false, err
}
} else {
localCache := localCacheObj.(*SStoragecache)
localCache.syncWithCloudStoragecache(ctx, userCred, cloudCache)
return localCache, nil
return localCache, false, nil
}
}
pkg/compute/tasks/cloud_provider_sync_info_task.go
+1 -1
View File
@@ -60,7 +60,7 @@ func (self *CloudProviderSyncInfoTask) OnInit(ctx context.Context, obj db.IStand
self.SetStage("OnSyncCloudProviderInfoComplete", nil)
taskman.LocalTaskRun(self, func() (jsonutils.JSONObject, error) {
provider.SyncCallSyncCloudproviderRegions(ctx, self.UserCred, &syncRange)
provider.SyncCallSyncCloudproviderRegions(ctx, self.UserCred, syncRange)
return nil, nil
})
}
pkg/scheduler/data_manager/sku/doc.go
+1
View File
@@ -0,0 +1 @@
package sku // import "yunion.io/x/onecloud/pkg/scheduler/data_manager/sku"