From 72a5e80a8d8182a60e8b85cd1ff521ff02d12a0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= <qu_xuan@icloud.com>
Date: Sat, 29 Jan 2022 00:56:31 +0800
Subject: [PATCH] fix(region): optimized public image sync (#13355)

---
 pkg/compute/models/cloudimages.go | 41 +++++++++++++++++++++++++++++++
 pkg/compute/models/skus_tools.go  | 15 +++++++++++
 pkg/compute/options/options.go    |  3 +--
 pkg/compute/service/service.go    |  2 +-
 4 files changed, 58 insertions(+), 3 deletions(-)

diff --git a/pkg/compute/models/cloudimages.go b/pkg/compute/models/cloudimages.go
index 37dca04c0b..ae964433f1 100644
--- a/pkg/compute/models/cloudimages.go
+++ b/pkg/compute/models/cloudimages.go
@@ -52,13 +52,53 @@ type SCloudimage struct {
 }
 
 func SyncPublicCloudImages(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
+	if isStart {
+		cnt, err := CloudimageManager.Query().CountWithError()
+		if err != nil {
+			return
+		}
+		if cnt > 0 {
+			log.Infof("Public cloud image has already synced, skip syncing")
+			return
+		}
+	}
 	regions := []SCloudregion{}
 	q := CloudregionManager.Query().In("provider", CloudproviderManager.GetPublicProviderProvidersQuery())
 	err := db.FetchModelObjects(CloudregionManager, q, &regions)
 	if err != nil {
 		return
 	}
+
+	meta, err := FetchSkuResourcesMeta()
+	if err != nil {
+		log.Errorf("SyncServerSkus.FetchSkuResourcesMeta %s", err)
+		return
+	}
+
+	index, err := meta.getServerSkuIndex()
+	if err != nil {
+		log.Errorf("getServerSkuIndex error: %v", err)
+		return
+	}
+
 	for i := range regions {
+		region := &regions[i]
+		oldMd5, _ := imageIndex[region.ExternalId]
+		newMd5, ok := index[region.ExternalId]
+		if ok {
+			imageIndex[region.ExternalId] = newMd5
+		}
+
+		if newMd5 == EMPTY_MD5 {
+			log.Infof("%s images is empty skip syncing", region.Name)
+			continue
+		}
+
+		if len(oldMd5) > 0 && newMd5 == oldMd5 {
+			log.Infof("%s cloud images not changed skip syncing", region.Name)
+			continue
+		}
+
 		err = regions[i].SyncCloudImages(ctx, userCred, !isStart)
 		if err != nil {
 			log.Errorf("SyncCloudImages for region %s(%s) error: %v", regions[i].Name, regions[i].Id, err)
@@ -75,6 +115,7 @@ func SyncPublicCloudImages(ctx context.Context, userCred mcclient.TokenCredentia
 			}
 		}
 	}
+	return
 }
 
 func (self *SCloudimage) syncRemove(ctx context.Context, userCred mcclient.TokenCredential) error {
diff --git a/pkg/compute/models/skus_tools.go b/pkg/compute/models/skus_tools.go
index e876018239..213957a9fa 100644
--- a/pkg/compute/models/skus_tools.go
+++ b/pkg/compute/models/skus_tools.go
@@ -57,6 +57,7 @@ type SSkuResourcesMeta struct {
 }
 
 var skuIndex = map[string]string{}
+var imageIndex = map[string]string{}
 
 func (self *SSkuResourcesMeta) getZoneIdBySuffix(zoneMaps map[string]string, suffix string) string {
 	for externalId, id := range zoneMaps {
@@ -400,6 +401,19 @@ func (self *SSkuResourcesMeta) getServerSkuIndex() (map[string]string, error) {
 	return ret, nil
 }
 
+func (self *SSkuResourcesMeta) getCloudimageIndex() (map[string]string, error) {
+	resp, err := self.request(fmt.Sprintf("%s/index.json", self.ImageBase))
+	if err != nil {
+		return map[string]string{}, errors.Wrapf(err, "request")
+	}
+	ret := map[string]string{}
+	err = resp.Unmarshal(ret)
+	if err != nil {
+		return map[string]string{}, errors.Wrapf(err, "resp.Unmarshal")
+	}
+	return ret, nil
+}
+
 func (self *SSkuResourcesMeta) getWafIndex() (map[string]string, error) {
 	resp, err := self.request(fmt.Sprintf("%s/index.json", self.WafBase))
 	if err != nil {
@@ -516,6 +530,7 @@ func SyncServerSkus(ctx context.Context, userCred mcclient.TokenCredential, isSt
 	index, err := meta.getServerSkuIndex()
 	if err != nil {
 		log.Errorf("getServerSkuIndex error: %v", err)
+		return
 	}
 
 	cloudregions := fetchSkuSyncCloudregions()
diff --git a/pkg/compute/options/options.go b/pkg/compute/options/options.go
index d32d640085..cb6ec82155 100644
--- a/pkg/compute/options/options.go
+++ b/pkg/compute/options/options.go
@@ -114,8 +114,7 @@ type ComputeOptions struct {
 	ConvertKubeletDockerVolumeSize   string `default:"256g" help:"Docker volume size"`
 
 	// cloud image sync
-	SyncCloudImagesDay  int `default:"1" help:"Days auto sync public cloud images data, default 1 day"`
-	SyncCloudImagesHour int `default:"3" help:"What hour start sync public cloud images, default 03:00"`
+	CloudImagesSyncIntervalHours int `default:"3" help:"Interval to sync public cloud image, defualt is 3 hour"`
 
 	EnablePreAllocateIpAddr bool `help:"Enable private and public cloud private ip pre allocate, default false" default:"false"`
 
diff --git a/pkg/compute/service/service.go b/pkg/compute/service/service.go
index 05390cd289..378ba3b2a2 100644
--- a/pkg/compute/service/service.go
+++ b/pkg/compute/service/service.go
@@ -174,7 +174,7 @@ func StartService() {
 		cron.AddJobEveryFewDays("SyncElasticCacheSkus", opts.SyncSkusDay, opts.SyncSkusHour, 0, 0, models.SyncElasticCacheSkus, true)
 		cron.AddJobEveryFewDays("StorageSnapshotsRecycle", 1, 2, 0, 0, models.StorageManager.StorageSnapshotsRecycle, false)
 
-		cron.AddJobEveryFewDays("SyncCloudImages", opts.SyncCloudImagesDay, opts.SyncCloudImagesHour, 0, 0, models.SyncPublicCloudImages, true)
+		cron.AddJobAtIntervalsWithStartRun("SyncCloudImages", time.Duration(opts.CloudImagesSyncIntervalHours)*time.Hour, models.SyncPublicCloudImages, true)
 
 		cron.AddJobEveryFewHour("InspectAllTemplate", 1, 0, 0, models.GuestTemplateManager.InspectAllTemplate, true)