fix(region): Cluster automatically when syncing snapshotpolicy

Fix two bug about listing snapshotpolicycache:
1. no accont info 2. filter error
The func 'Key' of snapshotpolicy generate a Identification for
snapshotpolicy. If the key of cloud snapshotpolicy is same with local
snaphotpolicy, the cloud sp became a cache of this local sp.

pkg/compute/models/snapshotpolicy.go

@@ -365,6 +365,27 @@ func (manager *SSnapshotPolicyManager) SyncSnapshotPolicies(ctx context.Context,
 		return syncResult
 	}
 
+	// fetch all snapshotpolicy
+	q := SnapshotPolicyManager.Query()
+	allSnapshotPolicies := make([]SSnapshotPolicy, 0, 10)
+	err = q.All(&allSnapshotPolicies)
+	if err != nil {
+		syncResult.Error(err)
+		return syncResult
+	}
+	// cluster snapshotpolicy
+	snapshotpolicyCluster := make(map[uint64][]*SSnapshotPolicy)
+	for i := range allSnapshotPolicies {
+		key := allSnapshotPolicies[i].Key()
+		list, ok := snapshotpolicyCluster[key]
+		if !ok {
+			list = make([]*SSnapshotPolicy, 0, 1)
+		}
+		list = append(list, &allSnapshotPolicies[i])
+		// sliceHeader change
+		snapshotpolicyCluster[key] = list
+	}
+
 	// structure two sets (externalID, snapshotpolicyCache), (snapshotPolicyID, snapshotPolicy)
 	spSet, spCacheSet := make(map[string]*SSnapshotPolicy), make(map[string]*SSnapshotPolicyCache)
 	for i := range snapshotPolicies {
@@ -414,7 +435,8 @@ func (manager *SSnapshotPolicyManager) SyncSnapshotPolicies(ctx context.Context,
 	}
 
 	for i := range added {
-		locol, err := manager.newFromCloudSnapshotPolicy(ctx, userCred, added[i], region, syncOwnerId, provider)
+		locol, err := manager.newFromCloudSnapshotPolicy(ctx, userCred, snapshotpolicyCluster, added[i], region,
+			syncOwnerId, provider)
 		if err != nil {
 			syncResult.AddError(err)
 		} else {
@@ -443,38 +465,64 @@ func (manager *SSnapshotPolicyManager) SyncSnapshotPolicies(ctx context.Context,
 }
 
 func (manager *SSnapshotPolicyManager) newFromCloudSnapshotPolicy(
-	ctx context.Context, userCred mcclient.TokenCredential,
-	ext cloudprovider.ICloudSnapshotPolicy, region *SCloudregion,
-	syncOwnerId mcclient.IIdentityProvider, provider *SCloudprovider,
+	ctx context.Context, userCred mcclient.TokenCredential, snapshotpolicyCluster map[uint64][]*SSnapshotPolicy,
+	ext cloudprovider.ICloudSnapshotPolicy, region *SCloudregion, syncOwnerId mcclient.IIdentityProvider, provider *SCloudprovider,
 ) (*SSnapshotPolicy, error) {
 
-	snapshotPolicy := SSnapshotPolicy{}
-	snapshotPolicy.SetModelManager(manager, &snapshotPolicy)
-
-	newName, err := db.GenerateName(manager, syncOwnerId, ext.GetName())
-	if err != nil {
-		return nil, err
-	}
-	snapshotPolicy.Name = newName
-	snapshotPolicy.Status = ext.GetStatus()
-	snapshotPolicy.RetentionDays = ext.GetRetentionDays()
+	snapshotPolicyTmp := SSnapshotPolicy{}
+	snapshotPolicyTmp.RetentionDays = ext.GetRetentionDays()
 	arw, err := ext.GetRepeatWeekdays()
 	if err != nil {
 		return nil, err
 	}
-	snapshotPolicy.RepeatWeekdays = SnapshotPolicyManager.RepeatWeekdaysParseIntArray(arw)
+	snapshotPolicyTmp.RepeatWeekdays = SnapshotPolicyManager.RepeatWeekdaysParseIntArray(arw)
 	atp, err := ext.GetTimePoints()
 	if err != nil {
 		return nil, err
 	}
-	snapshotPolicy.TimePoints = SnapshotPolicyManager.TimePointsParseIntArray(atp)
+	snapshotPolicyTmp.TimePoints = SnapshotPolicyManager.TimePointsParseIntArray(atp)
+	snapshotPolicyTmp.IsActivated = tristate.NewFromBool(ext.IsActivated())
 
-	snapshotPolicy.IsActivated = tristate.NewFromBool(ext.IsActivated())
+	extkey := snapshotPolicyTmp.Key()
+	extProjectId := SnapshotPolicyManager.FetchProjectId(ctx, userCred, syncOwnerId, ext, provider.GetId())
+	var snapshotPolicy *SSnapshotPolicy
 
-	err = manager.TableSpec().Insert(&snapshotPolicy)
-	if err != nil {
-		log.Errorf("newFromCloudEip fail %s", err)
-		return nil, err
+	if list, ok := snapshotpolicyCluster[extkey]; ok {
+		// find first snapshotpolicy enough to rebase
+		for _, sp := range list {
+			if sp.ProjectId == extProjectId {
+				snapshotPolicy = sp
+				break
+			}
+		}
+	}
+
+	// no such suitable snapshotpolicy in list
+	if snapshotPolicy == nil {
+		snapshotPolicyTmp.SetModelManager(manager, &snapshotPolicyTmp)
+		newName, err := db.GenerateName(manager, syncOwnerId, ext.GetName())
+		if err != nil {
+			return nil, err
+		}
+		snapshotPolicyTmp.Name = newName
+		snapshotPolicyTmp.Status = ext.GetStatus()
+
+		err = manager.TableSpec().Insert(&snapshotPolicyTmp)
+		if err != nil {
+			log.Errorf("newFromCloudEip fail %s", err)
+			return nil, err
+		}
+		// sync project
+		SyncCloudProject(userCred, &snapshotPolicyTmp, syncOwnerId, ext, provider.GetId())
+		// update snapshotpolicyCluster
+		key := snapshotPolicyTmp.Key()
+		list, ok := snapshotpolicyCluster[key]
+		if !ok {
+			list = make([]*SSnapshotPolicy, 0)
+		}
+		list = append(list, &snapshotPolicyTmp)
+		snapshotpolicyCluster[key] = list
+		snapshotPolicy = &snapshotPolicyTmp
 	}
 
 	// add cache
@@ -488,10 +536,40 @@ func (manager *SSnapshotPolicyManager) newFromCloudSnapshotPolicy(
 			snapshotPolicy.GetId())
 	}
 
-	SyncCloudProject(userCred, &snapshotPolicy, syncOwnerId, ext, provider.GetId())
+	db.OpsLog.LogEvent(snapshotPolicy, db.ACT_CREATE, snapshotPolicy.GetShortDesc(ctx), userCred)
+	return snapshotPolicy, nil
+}
 
-	db.OpsLog.LogEvent(&snapshotPolicy, db.ACT_CREATE, snapshotPolicy.GetShortDesc(ctx), userCred)
-	return &snapshotPolicy, nil
+func (spm *SSnapshotPolicyManager) FetchProjectId(ctx context.Context, userCred mcclient.TokenCredential,
+	syncOwnerId mcclient.IIdentityProvider, cloudSP cloudprovider.ICloudSnapshotPolicy, managerId string) string {
+	var newOwnerId mcclient.IIdentityProvider
+	if extProjectId := cloudSP.GetProjectId(); len(extProjectId) > 0 {
+		extProject, err := ExternalProjectManager.GetProject(extProjectId, managerId)
+		if err != nil {
+			log.Errorln(err)
+		} else {
+			newOwnerId = extProject.GetOwnerId()
+		}
+	}
+	if newOwnerId == nil && syncOwnerId != nil && len(syncOwnerId.GetProjectId()) > 0 {
+		newOwnerId = syncOwnerId
+	}
+	if newOwnerId == nil {
+		newOwnerId = userCred
+	}
+	return newOwnerId.GetProjectId()
+}
+
+func (sp *SSnapshotPolicy) Key() uint64 {
+	var key uint64
+	key |= (uint64(sp.RepeatWeekdays) << 56) | (uint64(sp.TimePoints) << 24)
+	// that sp.RetentionDays is -1 means permanent retention, sp.RetentionDays+1 must be less than 2^23
+	r := sp.RetentionDays + 1&(1<<24-1)
+	key |= uint64(r) << 1
+	if sp.IsActivated.IsTrue() {
+		key |= 1
+	}
+	return key
 }
 
 func (sp *SSnapshotPolicy) Equals(cloudSP cloudprovider.ICloudSnapshotPolicy) bool {

pkg/compute/models/snapshotpolicy_test.go

@@ -0,0 +1,72 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"testing"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/tristate"
+)
+
+func TestSSnapshotPolicy_Key(t *testing.T) {
+	cases := []struct {
+		in   *SSnapshotPolicy
+		want uint64
+	}{
+		{
+			&SSnapshotPolicy{
+				RepeatWeekdays: 0,
+				TimePoints:     0,
+				RetentionDays:  7,
+				IsActivated:    tristate.True,
+			},
+			15 + 2,
+		},
+		{
+			&SSnapshotPolicy{
+				RepeatWeekdays: 11,
+			},
+			11<<56 + 2,
+		},
+		{
+			&SSnapshotPolicy{
+				TimePoints: 234,
+			},
+			234<<24 + 2,
+		},
+		{
+			&SSnapshotPolicy{
+				RetentionDays: -1,
+			},
+			0,
+		},
+		{
+			&SSnapshotPolicy{
+				RepeatWeekdays: 13,
+				TimePoints:     123,
+				RetentionDays:  7,
+			},
+			936748724556660752,
+		},
+	}
+
+	for i, c := range cases {
+		g := c.in.Key()
+		if c.want != g {
+			log.Errorf("the %d case, want %d, get %d", i, c.want, g)
+		}
+	}
+}

pkg/compute/models/snapshotpolicycache.go

@@ -76,7 +76,7 @@ func (spcm *SSnapshotPolicyCacheManager) ListItemFilter(ctx context.Context, q *
 	if err != nil {
 		return nil, err
 	}
-	if snapshotpolicyIden, _ := query.GetString("snapshotpolicyIden"); len(snapshotpolicyIden) > 0 {
+	if snapshotpolicyIden, _ := query.GetString("snapshotpolicy"); len(snapshotpolicyIden) > 0 {
 		snapshotpolicy, err := SnapshotPolicyManager.FetchByIdOrName(userCred, snapshotpolicyIden)
 		if err != nil {
 			if errors.Cause(err) == sql.ErrNoRows {
@@ -117,6 +117,10 @@ func (spc *SSnapshotPolicyCache) GetCustomizeColumns(ctx context.Context, userCr
 	if regionInfo != nil {
 		extra.Update(regionInfo)
 	}
+	accountInfo := spc.SManagedResourceBase.GetCustomizeColumns(ctx, userCred, query)
+	if accountInfo != nil {
+		extra.Update(accountInfo)
+	}
 	return extra
 }