opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-24 16:03:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(region): Optimization for snapshotpolicy sync

Browse Source 
If the number of added cloudsnapshotpolicy more than 5, we do
clustering for all snapshotpolicies and find suitable one from these.
Otherwise, we find suitable one from database everytimes.
This commit is contained in:
郑雨
2019-11-20 15:08:58 +08:00
parent 2507a86ca9
commit 2c448579e6
1 changed files with 73 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/compute/models/snapshotpolicy.go
+73 -39
View File
@@ -365,27 +365,6 @@ func (manager *SSnapshotPolicyManager) SyncSnapshotPolicies(ctx context.Context,
return syncResult
}
// fetch all snapshotpolicy
q := SnapshotPolicyManager.Query()
allSnapshotPolicies := make([]SSnapshotPolicy, 0, 10)
err = q.All(&allSnapshotPolicies)
if err != nil {
syncResult.Error(err)
return syncResult
}
// cluster snapshotpolicy
snapshotpolicyCluster := make(map[uint64][]*SSnapshotPolicy)
for i := range allSnapshotPolicies {
key := allSnapshotPolicies[i].Key()
list, ok := snapshotpolicyCluster[key]
if !ok {
list = make([]*SSnapshotPolicy, 0, 1)
}
list = append(list, &allSnapshotPolicies[i])
// sliceHeader change
snapshotpolicyCluster[key] = list
}
// structure two sets (externalID, snapshotpolicyCache), (snapshotPolicyID, snapshotPolicy)
spSet, spCacheSet := make(map[string]*SSnapshotPolicy), make(map[string]*SSnapshotPolicyCache)
for i := range snapshotPolicies {
@@ -434,16 +413,7 @@ func (manager *SSnapshotPolicyManager) SyncSnapshotPolicies(ctx context.Context,
}
}
for i := range added {
locol, err := manager.newFromCloudSnapshotPolicy(ctx, userCred, snapshotpolicyCluster, added[i], region,
syncOwnerId, provider)
if err != nil {
syncResult.AddError(err)
} else {
syncMetadata(ctx, userCred, locol, added[i])
syncResult.Add()
}
}
syncResult = manager.allNewFromCloudSnapshotPolicy(ctx, userCred, added, region, syncOwnerId, provider, syncResult)
for i := range commondb {
_, err = db.Update(commondb[i], func() error {
@@ -464,6 +434,50 @@ func (manager *SSnapshotPolicyManager) SyncSnapshotPolicies(ctx context.Context,
return syncResult
}
func (manager *SSnapshotPolicyManager) allNewFromCloudSnapshotPolicy(
ctx context.Context, userCred mcclient.TokenCredential, added []cloudprovider.ICloudSnapshotPolicy,
region *SCloudregion, syncOwnerId mcclient.IIdentityProvider, provider *SCloudprovider,
syncResult compare.SyncResult) compare.SyncResult {
var snapshotpolicyCluster map[uint64][]*SSnapshotPolicy
if len(added) > 5 {
// the number of added is large
// fetch all snapshotpolicy
q := SnapshotPolicyManager.Query()
allSnapshotPolicies := make([]SSnapshotPolicy, 0, 10)
err := q.All(&allSnapshotPolicies)
if err != nil {
syncResult.Error(err)
return syncResult
}
// cluster snapshotpolicy
snapshotpolicyCluster := make(map[uint64][]*SSnapshotPolicy)
for i := range allSnapshotPolicies {
key := allSnapshotPolicies[i].Key()
list, ok := snapshotpolicyCluster[key]
if !ok {
list = make([]*SSnapshotPolicy, 0, 1)
}
list = append(list, &allSnapshotPolicies[i])
// sliceHeader change
snapshotpolicyCluster[key] = list
}
}
for i := range added {
local, err := manager.newFromCloudSnapshotPolicy(ctx, userCred, snapshotpolicyCluster, added[i], region,
syncOwnerId, provider)
if err != nil {
syncResult.AddError(err)
} else {
syncMetadata(ctx, userCred, local, added[i])
syncResult.Add()
}
}
return syncResult
}
func (manager *SSnapshotPolicyManager) newFromCloudSnapshotPolicy(
ctx context.Context, userCred mcclient.TokenCredential, snapshotpolicyCluster map[uint64][]*SSnapshotPolicy,
ext cloudprovider.ICloudSnapshotPolicy, region *SCloudregion, syncOwnerId mcclient.IIdentityProvider, provider *SCloudprovider,
@@ -483,21 +497,41 @@ func (manager *SSnapshotPolicyManager) newFromCloudSnapshotPolicy(
snapshotPolicyTmp.TimePoints = SnapshotPolicyManager.TimePointsParseIntArray(atp)
snapshotPolicyTmp.IsActivated = tristate.NewFromBool(ext.IsActivated())
extkey := snapshotPolicyTmp.Key()
extProjectId := SnapshotPolicyManager.FetchProjectId(ctx, userCred, syncOwnerId, ext, provider.GetId())
var snapshotPolicy *SSnapshotPolicy
if list, ok := snapshotpolicyCluster[extkey]; ok {
// find first snapshotpolicy enough to rebase
for _, sp := range list {
if sp.ProjectId == extProjectId {
snapshotPolicy = sp
break
// find suitable snapshotpolicy
if snapshotpolicyCluster == nil {
q := manager.Query().Equals("repeat_weekdays", snapshotPolicyTmp.RepeatWeekdays).Equals("time_points",
snapshotPolicyTmp.TimePoints).Equals("retention_days", snapshotPolicyTmp.RetentionDays).Equals(
"is_activated", snapshotPolicyTmp.IsActivated.Bool()).Equals("tenant_id", extProjectId)
count, err := q.CountWithError()
if err != nil {
return nil, err
}
if count > 0 {
snapshotPolicy = &SSnapshotPolicy{}
err = q.First(snapshotPolicy)
if err != nil {
return nil, err
}
snapshotPolicy.SetModelManager(manager, snapshotPolicy)
}
} else {
extkey := snapshotPolicyTmp.Key()
if list, ok := snapshotpolicyCluster[extkey]; ok {
// find first snapshotpolicy enough to rebase
for _, sp := range list {
if sp.ProjectId == extProjectId {
snapshotPolicy = sp
break
}
}
}
}
// no such suitable snapshotpolicy in list
// no such suitable snapshotpolicy
if snapshotPolicy == nil {
snapshotPolicyTmp.SetModelManager(manager, &snapshotPolicyTmp)
newName, err := db.GenerateName(manager, syncOwnerId, ext.GetName())