From 2c448579e6a9cc827769926e5f97143c32cd68d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=83=91=E9=9B=A8?= <zhengyu@yunion.cn>
Date: Tue, 19 Nov 2019 11:18:11 +0800
Subject: [PATCH] refactor(region): Optimization for snapshotpolicy sync

If the number of added cloudsnapshotpolicy more than 5, we do
clustering for all snapshotpolicies and find suitable one from these.
Otherwise, we find suitable one from database everytimes.
---
 pkg/compute/models/snapshotpolicy.go | 112 +++++++++++++++++----------
 1 file changed, 73 insertions(+), 39 deletions(-)

diff --git a/pkg/compute/models/snapshotpolicy.go b/pkg/compute/models/snapshotpolicy.go
index 87249f9663..04ed89dcf9 100644
--- a/pkg/compute/models/snapshotpolicy.go
+++ b/pkg/compute/models/snapshotpolicy.go
@@ -365,27 +365,6 @@ func (manager *SSnapshotPolicyManager) SyncSnapshotPolicies(ctx context.Context,
 		return syncResult
 	}
 
-	// fetch all snapshotpolicy
-	q := SnapshotPolicyManager.Query()
-	allSnapshotPolicies := make([]SSnapshotPolicy, 0, 10)
-	err = q.All(&allSnapshotPolicies)
-	if err != nil {
-		syncResult.Error(err)
-		return syncResult
-	}
-	// cluster snapshotpolicy
-	snapshotpolicyCluster := make(map[uint64][]*SSnapshotPolicy)
-	for i := range allSnapshotPolicies {
-		key := allSnapshotPolicies[i].Key()
-		list, ok := snapshotpolicyCluster[key]
-		if !ok {
-			list = make([]*SSnapshotPolicy, 0, 1)
-		}
-		list = append(list, &allSnapshotPolicies[i])
-		// sliceHeader change
-		snapshotpolicyCluster[key] = list
-	}
-
 	// structure two sets (externalID, snapshotpolicyCache), (snapshotPolicyID, snapshotPolicy)
 	spSet, spCacheSet := make(map[string]*SSnapshotPolicy), make(map[string]*SSnapshotPolicyCache)
 	for i := range snapshotPolicies {
@@ -434,16 +413,7 @@ func (manager *SSnapshotPolicyManager) SyncSnapshotPolicies(ctx context.Context,
 		}
 	}
 
-	for i := range added {
-		locol, err := manager.newFromCloudSnapshotPolicy(ctx, userCred, snapshotpolicyCluster, added[i], region,
-			syncOwnerId, provider)
-		if err != nil {
-			syncResult.AddError(err)
-		} else {
-			syncMetadata(ctx, userCred, locol, added[i])
-			syncResult.Add()
-		}
-	}
+	syncResult = manager.allNewFromCloudSnapshotPolicy(ctx, userCred, added, region, syncOwnerId, provider, syncResult)
 
 	for i := range commondb {
 		_, err = db.Update(commondb[i], func() error {
@@ -464,6 +434,50 @@ func (manager *SSnapshotPolicyManager) SyncSnapshotPolicies(ctx context.Context,
 	return syncResult
 }
 
+func (manager *SSnapshotPolicyManager) allNewFromCloudSnapshotPolicy(
+	ctx context.Context, userCred mcclient.TokenCredential, added []cloudprovider.ICloudSnapshotPolicy,
+	region *SCloudregion, syncOwnerId mcclient.IIdentityProvider, provider *SCloudprovider,
+	syncResult compare.SyncResult) compare.SyncResult {
+
+	var snapshotpolicyCluster map[uint64][]*SSnapshotPolicy
+
+	if len(added) > 5 {
+		// the number of added is large
+		// fetch all snapshotpolicy
+		q := SnapshotPolicyManager.Query()
+		allSnapshotPolicies := make([]SSnapshotPolicy, 0, 10)
+		err := q.All(&allSnapshotPolicies)
+		if err != nil {
+			syncResult.Error(err)
+			return syncResult
+		}
+		// cluster snapshotpolicy
+		snapshotpolicyCluster := make(map[uint64][]*SSnapshotPolicy)
+		for i := range allSnapshotPolicies {
+			key := allSnapshotPolicies[i].Key()
+			list, ok := snapshotpolicyCluster[key]
+			if !ok {
+				list = make([]*SSnapshotPolicy, 0, 1)
+			}
+			list = append(list, &allSnapshotPolicies[i])
+			// sliceHeader change
+			snapshotpolicyCluster[key] = list
+		}
+	}
+
+	for i := range added {
+		local, err := manager.newFromCloudSnapshotPolicy(ctx, userCred, snapshotpolicyCluster, added[i], region,
+			syncOwnerId, provider)
+		if err != nil {
+			syncResult.AddError(err)
+		} else {
+			syncMetadata(ctx, userCred, local, added[i])
+			syncResult.Add()
+		}
+	}
+	return syncResult
+}
+
 func (manager *SSnapshotPolicyManager) newFromCloudSnapshotPolicy(
 	ctx context.Context, userCred mcclient.TokenCredential, snapshotpolicyCluster map[uint64][]*SSnapshotPolicy,
 	ext cloudprovider.ICloudSnapshotPolicy, region *SCloudregion, syncOwnerId mcclient.IIdentityProvider, provider *SCloudprovider,
@@ -483,21 +497,41 @@ func (manager *SSnapshotPolicyManager) newFromCloudSnapshotPolicy(
 	snapshotPolicyTmp.TimePoints = SnapshotPolicyManager.TimePointsParseIntArray(atp)
 	snapshotPolicyTmp.IsActivated = tristate.NewFromBool(ext.IsActivated())
 
-	extkey := snapshotPolicyTmp.Key()
 	extProjectId := SnapshotPolicyManager.FetchProjectId(ctx, userCred, syncOwnerId, ext, provider.GetId())
+
 	var snapshotPolicy *SSnapshotPolicy
 
-	if list, ok := snapshotpolicyCluster[extkey]; ok {
-		// find first snapshotpolicy enough to rebase
-		for _, sp := range list {
-			if sp.ProjectId == extProjectId {
-				snapshotPolicy = sp
-				break
+	// find suitable snapshotpolicy
+	if snapshotpolicyCluster == nil {
+		q := manager.Query().Equals("repeat_weekdays", snapshotPolicyTmp.RepeatWeekdays).Equals("time_points",
+			snapshotPolicyTmp.TimePoints).Equals("retention_days", snapshotPolicyTmp.RetentionDays).Equals(
+			"is_activated", snapshotPolicyTmp.IsActivated.Bool()).Equals("tenant_id", extProjectId)
+		count, err := q.CountWithError()
+		if err != nil {
+			return nil, err
+		}
+		if count > 0 {
+			snapshotPolicy = &SSnapshotPolicy{}
+			err = q.First(snapshotPolicy)
+			if err != nil {
+				return nil, err
+			}
+			snapshotPolicy.SetModelManager(manager, snapshotPolicy)
+		}
+	} else {
+		extkey := snapshotPolicyTmp.Key()
+		if list, ok := snapshotpolicyCluster[extkey]; ok {
+			// find first snapshotpolicy enough to rebase
+			for _, sp := range list {
+				if sp.ProjectId == extProjectId {
+					snapshotPolicy = sp
+					break
+				}
 			}
 		}
 	}
 
-	// no such suitable snapshotpolicy in list
+	// no such suitable snapshotpolicy
 	if snapshotPolicy == nil {
 		snapshotPolicyTmp.SetModelManager(manager, &snapshotPolicyTmp)
 		newName, err := db.GenerateName(manager, syncOwnerId, ext.GetName())