fix(region): optimized server sku sku

pkg/compute/models/skus_tools.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
-	"io/ioutil"
 	"net/http"
 	"strings"
 
@@ -37,6 +36,10 @@ import (
 	"yunion.io/x/onecloud/pkg/util/httputils"
 )
 
+const (
+	EMPTY_MD5 = "d751713988987e9331980363e24189ce"
+)
+
 /*
 资源套餐下载连接信息
 server: 虚拟机
@@ -51,6 +54,8 @@ type SSkuResourcesMeta struct {
 	NasBase          string `json:"nas_base"`
 }
 
+var skuIndex = map[string]string{}
+
 func (self *SSkuResourcesMeta) getZoneIdBySuffix(zoneMaps map[string]string, suffix string) string {
 	for externalId, id := range zoneMaps {
 		if strings.HasSuffix(externalId, suffix) {
@@ -235,7 +240,7 @@ func (self *SSkuResourcesMeta) GetServerSkusByRegionExternalId(regionExternalId
 	}
 	for _, obj := range objs {
 		sku := SServerSku{}
-		sku.SetModelManager(ElasticcacheSkuManager, &sku)
+		sku.SetModelManager(ServerSkuManager, &sku)
 		err = obj.Unmarshal(&sku)
 		if err != nil {
 			return nil, errors.Wrapf(err, "obj.Unmarshal")
@@ -324,41 +329,34 @@ func (self *SSkuResourcesMeta) getObjsByRegion(base string, region string) ([]js
 	return items, nil
 }
 
+func (self *SSkuResourcesMeta) request(url string) (jsonutils.JSONObject, error) {
+	client := httputils.GetAdaptiveTimeoutClient()
+
+	header := http.Header{}
+	header.Set("User-Agent", "vendor/yunion-OneCloud@"+v.Get().GitVersion)
+	_, resp, err := httputils.JSONRequest(client, context.TODO(), httputils.GET, url, header, nil, false)
+	return resp, err
+}
+
+func (self *SSkuResourcesMeta) getServerSkuIndex() (map[string]string, error) {
+	resp, err := self.request(fmt.Sprintf("%s/index.json", self.ServerBase))
+	if err != nil {
+		return map[string]string{}, errors.Wrapf(err, "request")
+	}
+	ret := map[string]string{}
+	err = resp.Unmarshal(ret)
+	if err != nil {
+		return map[string]string{}, errors.Wrapf(err, "resp.Unmarshal")
+	}
+	return ret, nil
+}
+
 func (self *SSkuResourcesMeta) _get(url string) ([]jsonutils.JSONObject, error) {
 	if !strings.HasPrefix(url, "http") {
 		return nil, fmt.Errorf("SkuResourcesMeta.get invalid url %s.expected has prefix 'http'", url)
 	}
 
-	req, err := http.NewRequest(http.MethodGet, url, nil)
-	if err != nil {
-		return nil, fmt.Errorf("SkuResourcesMeta.get.NewRequest %s", err)
-	}
-
-	userAgent := "vendor/yunion-OneCloud@" + v.Get().GitVersion
-	req.Header.Set("User-Agent", userAgent)
-
-	transport := httputils.GetTransport(true)
-	transport.Proxy = options.Options.HttpTransportProxyFunc()
-	client := &http.Client{
-		Transport: transport,
-	}
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("SkuResourcesMeta.get.Get %s", err)
-	}
-
-	defer resp.Body.Close()
-	content, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return nil, fmt.Errorf("SkuResourcesMeta.get.ReadAll %s", err)
-	}
-
-	jsonContent, err := jsonutils.Parse(content)
-	if err != nil {
-		return nil, fmt.Errorf("SkuResourcesMeta.get.Parse %s", err)
-	}
-
+	jsonContent, err := self.request(url)
 	var ret []jsonutils.JSONObject
 	err = jsonContent.Unmarshal(&ret)
 	if err != nil {
@@ -450,9 +448,30 @@ func SyncServerSkus(ctx context.Context, userCred mcclient.TokenCredential, isSt
 		return
 	}
 
+	index, err := meta.getServerSkuIndex()
+	if err != nil {
+		log.Errorf("getServerSkuIndex error: %v", err)
+	}
+
 	cloudregions := fetchSkuSyncCloudregions()
 	for i := range cloudregions {
 		region := &cloudregions[i]
+		oldMd5, _ := skuIndex[region.ExternalId]
+		newMd5, ok := index[region.ExternalId]
+		if ok {
+			skuIndex[region.ExternalId] = newMd5
+		}
+
+		if newMd5 == EMPTY_MD5 {
+			log.Infof("%s Server Skus is empty skip syncing", region.Name)
+			continue
+		}
+
+		if len(oldMd5) > 0 && newMd5 == oldMd5 {
+			log.Infof("%s Server Skus not Changed skip syncing", region.Name)
+			continue
+		}
+
 		result := ServerSkuManager.SyncServerSkus(ctx, userCred, region, meta)
 		notes := fmt.Sprintf("SyncServerSkusByRegion %s result: %s", region.Name, result.Result())
 		log.Infof(notes)

pkg/compute/options/options.go

@@ -102,6 +102,8 @@ type ComputeOptions struct {
 	TimePointsLimit     int `default:"1" help:"time point of every days, default 1 point"`
 	RepeatWeekdaysLimit int `default:"7" help:"day point of every weekday, default 7 points"`
 
+	ServerSkuSyncIntervalMinutes int `default:"60" help:"Interval to sync public cloud server skus, defualt is 1 hour"`
+
 	// sku sync
 	SyncSkusDay  int `default:"1" help:"Days auto sync skus data, default 1 day"`
 	SyncSkusHour int `default:"3" help:"What hour start sync skus, default 03:00"`

pkg/compute/service/service.go

@@ -150,7 +150,9 @@ func StartService() {
 
 		cron.AddJobEveryFewHour("AutoDiskSnapshot", 1, 5, 0, models.DiskManager.AutoDiskSnapshot, false)
 		cron.AddJobEveryFewHour("SnapshotsCleanup", 1, 35, 0, models.SnapshotManager.CleanupSnapshots, false)
-		cron.AddJobEveryFewDays("SyncSkus", opts.SyncSkusDay, opts.SyncSkusHour, 0, 0, models.SyncServerSkus, true)
+
+		cron.AddJobAtIntervalsWithStartRun("SyncSkus", time.Duration(opts.ServerSkuSyncIntervalMinutes)*time.Minute, models.SyncServerSkus, true)
+
 		cron.AddJobEveryFewDays("SyncDBInstanceSkus", opts.SyncSkusDay, opts.SyncSkusHour, 0, 0, models.SyncDBInstanceSkus, true)
 		cron.AddJobEveryFewDays("SyncNatSkus", opts.SyncSkusDay, opts.SyncSkusHour, 0, 0, models.SyncNatSkus, false)
 		cron.AddJobEveryFewDays("SyncNasSkus", opts.SyncSkusDay, opts.SyncSkusHour, 0, 0, models.SyncNasSkus, false)