fix(cloudmon): add Slb metrics

go.mod

@@ -83,14 +83,14 @@ require (
 	k8s.io/client-go v0.19.3
 	k8s.io/cluster-bootstrap v0.19.3
 	moul.io/http2curl/v2 v2.3.0
-	yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230323062012-21f8a2ca0add
+	yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230329090243-cc4cdbd90fd5
 	yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656
 	yunion.io/x/jsonutils v1.0.1-0.20220819091305-3bab322ab4fd
 	yunion.io/x/log v1.0.0
 	yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900
-	yunion.io/x/pkg v1.0.1-0.20230102060551-df05ccecb71c
+	yunion.io/x/pkg v1.0.1-0.20230327034731-79f520e7d373
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v1.1.2-0.20230223125758-76ea06389ac5
+	yunion.io/x/sqlchemy v1.1.2-0.20230326161357-912d44dd66d6
 	yunion.io/x/structarg v0.0.0-20220312084958-9c6c79c7d1c6
 )
 

go.sum

@@ -1164,8 +1164,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
-yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230323062012-21f8a2ca0add h1:w0kPDxDar0F0p9fWSBoqTCsBojhFrRzxsiJYiGO3bLY=
-yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230323062012-21f8a2ca0add/go.mod h1:sWqblYRhQCO63xeKdxdAT3wczCCY8Mc+TDa3WD/a5Kg=
+yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230329090243-cc4cdbd90fd5 h1:w9vF4QzyJtkkDBPY7TOH1kUvUzPi6kGDmuFYHZrbmG8=
+yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230329090243-cc4cdbd90fd5/go.mod h1:VBAtfyGMDW11x5JmF0vUEmn/PmNpZ8CKBAQCiyz+cDY=
 yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656 h1:0zlZD5uhZoIHgLVAWCz2aHaYk2ZrNsACCYD7R6EIBII=
 yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
@@ -1182,12 +1182,12 @@ yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZV
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v1.0.1-0.20220819080140-55b265a8658e/go.mod h1:nYOMflWG1wQC6301rGVglQva9bYbJxHThEJUxRAgMFc=
-yunion.io/x/pkg v1.0.1-0.20230102060551-df05ccecb71c h1:qkkyHkL8bFWumxWvtdbYhY4aDfKi+sBS21Im/i6Jeds=
-yunion.io/x/pkg v1.0.1-0.20230102060551-df05ccecb71c/go.mod h1:ksCJVQ+DwKrJ5QBEoU8pzrDFfDaZVAFH/iJ6yQCYxJk=
+yunion.io/x/pkg v1.0.1-0.20230327034731-79f520e7d373 h1:VdQgvUo6JoOZNsPzRdoFKbD+3tEt4saHRhI/kUf6DeM=
+yunion.io/x/pkg v1.0.1-0.20230327034731-79f520e7d373/go.mod h1:ksCJVQ+DwKrJ5QBEoU8pzrDFfDaZVAFH/iJ6yQCYxJk=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
 yunion.io/x/sqlchemy v1.0.1/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
-yunion.io/x/sqlchemy v1.1.2-0.20230223125758-76ea06389ac5 h1:7+m2F8B8eKJ4ELtIWpEEnEhL3WsIjpLocAaMx+83U5o=
-yunion.io/x/sqlchemy v1.1.2-0.20230223125758-76ea06389ac5/go.mod h1:xXVfJLIxxTRTnPEaVIIoRZXqWyO8+takIOYhc+zO5KE=
+yunion.io/x/sqlchemy v1.1.2-0.20230326161357-912d44dd66d6 h1:qs8BU3AH7MsKeWLzuaj2bCc2/VnwyYqzl55VJXS25X4=
+yunion.io/x/sqlchemy v1.1.2-0.20230326161357-912d44dd66d6/go.mod h1:xXVfJLIxxTRTnPEaVIIoRZXqWyO8+takIOYhc+zO5KE=
 yunion.io/x/structarg v0.0.0-20220312084958-9c6c79c7d1c6 h1:WuWXhY3DvhdRTzWCJ/kwt3Ss6KIq7+KqJwb+esvNGwU=
 yunion.io/x/structarg v0.0.0-20220312084958-9c6c79c7d1c6/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/apis/compute/loadbalancer.go

@@ -230,3 +230,20 @@ type LoadbalancerDissociateEipInput struct {
 	// 是否自动释放
 	AudoDelete *bool `json:"auto_delete"`
 }
+
+func (self LoadbalancerDetails) GetMetricTags() map[string]string {
+	ret := map[string]string{
+		"id":             self.Id,
+		"brand":          self.Brand,
+		"backend_group":  self.BackendGroup,
+		"cloudregion":    self.Cloudregion,
+		"domain_id":      self.DomainId,
+		"project_domain": self.ProjectDomain,
+		"region_ext_id":  self.RegionExtId,
+		"status":         self.Status,
+		"tenant":         self.Project,
+		"tenant_id":      self.ProjectId,
+		"external_id":    self.ExternalId,
+	}
+	return ret
+}

pkg/cloudmon/providerdriver/base.go

@@ -583,6 +583,71 @@ func (self *SCollectByResourceIdDriver) CollectK8sMetrics(ctx context.Context, m
 	return self.sendMetrics(ctx, manager, "k8s", len(res), metrics)
 }
 
+func (self *SCollectByResourceIdDriver) CollectLoadbalancerMetrics(ctx context.Context, manager api.CloudproviderDetails, provider cloudprovider.ICloudProvider, res map[string]api.LoadbalancerDetails, start, end time.Time) error {
+	metrics := []influxdb.SMetricData{}
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	for i := range res {
+		wg.Add(1)
+		go func(lb api.LoadbalancerDetails) {
+			defer func() {
+				wg.Done()
+			}()
+			opts := &cloudprovider.MetricListOptions{
+				ResourceType: cloudprovider.METRIC_RESOURCE_TYPE_LB,
+				StartTime:    start,
+				EndTime:      end,
+			}
+			opts.ResourceId = lb.ExternalId
+
+			tags := []influxdb.SKeyValue{}
+			for k, v := range lb.GetMetricTags() {
+				tags = append(tags, influxdb.SKeyValue{
+					Key:   k,
+					Value: v,
+				})
+			}
+
+			data, err := provider.GetMetrics(opts)
+			if err != nil {
+				if errors.Cause(err) != cloudprovider.ErrNotImplemented && errors.Cause(err) != cloudprovider.ErrNotSupported {
+					log.Errorf("get loadbalancers %s(%s) error: %v", lb.Name, lb.Id, err)
+					return
+				}
+				return
+			}
+
+			for _, values := range data {
+				for _, value := range values.Values {
+					metric := influxdb.SMetricData{
+						Name:      values.MetricType.Name(),
+						Timestamp: value.Timestamp,
+						Tags:      tags,
+						Metrics: []influxdb.SKeyValue{
+							{
+								Key:   values.MetricType.Key(),
+								Value: strconv.FormatFloat(value.Value, 'E', -1, 64),
+							},
+						},
+					}
+					for k, v := range value.Tags {
+						metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+							Key:   k,
+							Value: v,
+						})
+					}
+					mu.Lock()
+					metrics = append(metrics, metric)
+					mu.Unlock()
+				}
+			}
+		}(res[i])
+	}
+	wg.Wait()
+
+	return self.sendMetrics(ctx, manager, "slb", len(res), metrics)
+}
+
 type SCollectByMetricTypeDriver struct {
 	SBaseCollectDriver
 }

vendor/modules.txt

@@ -1442,7 +1442,7 @@ sigs.k8s.io/structured-merge-diff/v4/value
 # sigs.k8s.io/yaml v1.2.0
 ## explicit; go 1.12
 sigs.k8s.io/yaml
-# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230323062012-21f8a2ca0add
+# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230329090243-cc4cdbd90fd5
 ## explicit; go 1.18
 yunion.io/x/cloudmux/pkg/apis
 yunion.io/x/cloudmux/pkg/apis/billing
@@ -1537,7 +1537,7 @@ yunion.io/x/log/hooks
 yunion.io/x/ovsdb/cli_util
 yunion.io/x/ovsdb/schema/ovn_nb
 yunion.io/x/ovsdb/types
-# yunion.io/x/pkg v1.0.1-0.20230102060551-df05ccecb71c
+# yunion.io/x/pkg v1.0.1-0.20230327034731-79f520e7d373
 ## explicit; go 1.18
 yunion.io/x/pkg/appctx
 yunion.io/x/pkg/errors
@@ -1545,6 +1545,7 @@ yunion.io/x/pkg/gotypes
 yunion.io/x/pkg/object
 yunion.io/x/pkg/prettytable
 yunion.io/x/pkg/sortedmap
+yunion.io/x/pkg/sortedstring
 yunion.io/x/pkg/trace
 yunion.io/x/pkg/tristate
 yunion.io/x/pkg/util/billing
@@ -1591,7 +1592,7 @@ yunion.io/x/pkg/utils
 # yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
 ## explicit; go 1.12
 yunion.io/x/s3cli
-# yunion.io/x/sqlchemy v1.1.2-0.20230223125758-76ea06389ac5
+# yunion.io/x/sqlchemy v1.1.2-0.20230326161357-912d44dd66d6
 ## explicit; go 1.17
 yunion.io/x/sqlchemy
 yunion.io/x/sqlchemy/backends

vendor/yunion.io/x/cloudmux/pkg/cloudprovider/metrics.go

@@ -222,10 +222,13 @@ const (
 	// 支持平台: aliyun, apsara
 	LB_METRIC_TYPE_NET_PACKET_TX TMetricType = "haproxy.packet_tx"
 	// 非活跃连接数
-	// 支持平台: apsara, aliyun
+	// 支持平台: apsara, aliyun, huawei
 	LB_METRIC_TYPE_NET_INACTIVE_CONNECTION = "haproxy.inactive_connection"
+	// 活跃连接数
+	// 支持平台: apsara, aliyun, huawei
+	LB_METRIC_TYPE_NET_ACTIVE_CONNECTION = "haproxy.active_connection"
 	// 最大并发数
-	// 支持平台: apsara, aliyun
+	// 支持平台: apsara, aliyun, huawei
 	LB_METRIC_TYPE_MAX_CONNECTION = "haproxy.max_connection"
 	// 后端异常ECS实例个数
 	// 支持平台: apsara, aliyun

vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/hcso.go

@@ -17,10 +17,12 @@ package hcso
 import (
 	"context"
 	"fmt"
+	"net"
 	"net/http"
 	"net/url"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/huaweicloud/huaweicloud-sdk-go/auth/aksk"
 
@@ -98,16 +100,34 @@ func (self *SHuaweiClient) request(method httputils.THttpMethod, url string, que
 	if strings.Contains(url, "/OS-CREDENTIAL/credentials") && len(self.ownerId) > 0 {
 		header.Set("X-Domain-Id", self.ownerId)
 	}
-	_, resp, err := httputils.JSONRequest(client, context.Background(), method, url, header, body, self.debug)
-	if err != nil {
-		if e, ok := err.(*httputils.JSONClientError); ok && e.Code == 404 {
-			return nil, errors.Wrapf(cloudprovider.ErrNotFound, err.Error())
+	var resp jsonutils.JSONObject
+	var err error
+	for i := 0; i < 3; i++ {
+		_, resp, err = requestWithRetry(client, context.Background(), method, url, header, body, self.debug)
+		if method == httputils.GET && needRetry(err) {
+			time.Sleep(time.Second * 15)
+			continue
 		}
-		return nil, err
+		return resp, err
 	}
 	return resp, err
 }
 
+func requestWithRetry(client *akClient, ctx context.Context, method httputils.THttpMethod, urlStr string, header http.Header, body jsonutils.JSONObject, debug bool) (http.Header, jsonutils.JSONObject, error) {
+	var bodystr string
+	if !gotypes.IsNil(body) {
+		bodystr = body.String()
+	}
+	jbody := strings.NewReader(bodystr)
+	if header == nil {
+		header = http.Header{}
+	}
+	header.Set("Content-Length", strconv.FormatInt(int64(len(bodystr)), 10))
+	header.Set("Content-Type", "application/json")
+	resp, err := httputils.RequestWithRetry(client, ctx, method, urlStr, header, jbody, debug)
+	return httputils.ParseJSONResponse(bodystr, resp, err, debug)
+}
+
 func (self *SHuaweiClient) resetEndpoint(endpoint, serviceName string) string {
 	if len(endpoint) == 0 {
 		domain := self.HuaweiClientConfig.endpoints.EndpointDomain
@@ -258,3 +278,20 @@ func (self *SHuaweiClient) patchRequest(method httputils.THttpMethod, url string
 	}
 	return respValue, err
 }
+
+func needRetry(err error) bool {
+	if err == nil {
+		return false
+	}
+	switch e := err.(type) {
+	case *url.Error:
+		switch e.Err.(type) {
+		case *net.DNSError, *net.OpError, net.UnknownNetworkError:
+			return true
+		}
+		if strings.Contains(err.Error(), "The throttling threshold has been reached: policy ip over ratelimit") {
+			return true
+		}
+	}
+	return false
+}

vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/modelarts.go

@@ -373,6 +373,15 @@ func (self *SModelartsPool) SetAutoRenew(bc billing.SBillingCycle) error {
 }
 
 func (self *SModelartsPool) Refresh() error {
+	self.Status.Resource = SNodeStatus{}
+	pool, err := self.region.client.modelartsPoolById(self.GetId())
+	if err == nil {
+		pool.Unmarshal(self)
+		return nil
+	}
+	if err != errors.ErrNotFound {
+		return errors.Wrap(err, "modelartsPoolById")
+	}
 	pools := make([]SModelartsPool, 0)
 	resObj, err := self.region.client.modelartsPoolListWithStatus("pools", "failed", nil)
 	if err != nil {
@@ -385,13 +394,9 @@ func (self *SModelartsPool) Refresh() error {
 	for _, pool := range pools {
 		if pool.GetId() == self.GetId() {
 			self.Status.Phase = "CreationFailed"
+			break
 		}
 	}
-	self.Status.Resource = SNodeStatus{}
-	pool, err := self.region.client.modelartsPoolById(self.GetId())
-	if err != nil {
-		return errors.Wrapf(err, "GetModelartsPool(%s)", self.GetId())
-	}
 	return jsonutils.Update(self, pool)
 }
 

vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/monitor.go

@@ -353,6 +353,87 @@ func (self *SHuaweiClient) getBucketMetrics(opts *cloudprovider.MetricListOption
 	return result, nil
 }
 
+func (self *SHuaweiClient) getLoadbalancerMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) {
+	namespace, dimesionName, metricNames := "SYS.ELB", "lbaas_instance_id", []string{
+		"m1_cps",
+		"m2_act_conn",
+		"m7_in_Bps",
+		"m8_out_Bps",
+		"mc_l7_http_2xx",
+		"md_l7_http_3xx",
+		"me_l7_http_4xx",
+		"mf_l7_http_5xx",
+	}
+	result := []cloudprovider.MetricValues{}
+
+	for _, metricName := range metricNames {
+		temp := make(map[string]string)
+		temp["namespace"] = namespace
+		temp["metric_name"] = metricName
+		temp["from"] = strconv.Itoa(int(opts.StartTime.UnixMilli()))
+		temp["to"] = strconv.Itoa(int(opts.EndTime.UnixMilli()))
+		temp["period"] = "1"
+		temp["filter"] = "average"
+		temp["dim.0"] = fmt.Sprintf("%s,%s", dimesionName, opts.ResourceId)
+		resp, err := self.commonMonitor(temp)
+		if err != nil {
+			return nil, err
+		}
+		metricData := MetricData{}
+		err = resp.Unmarshal(&metricData)
+		if err != nil {
+			return nil, errors.Wrapf(err, "resp.Unmarshal")
+		}
+		ret := cloudprovider.MetricValues{
+			Id:     opts.ResourceId,
+			Unit:   metricData.Unit,
+			Values: []cloudprovider.MetricValue{},
+		}
+		tags := map[string]string{}
+		switch metricData.MetricName {
+		case "m1_cps":
+			ret.MetricType = cloudprovider.LB_METRIC_TYPE_MAX_CONNECTION
+		case "m2_act_conn":
+			ret.MetricType = cloudprovider.LB_METRIC_TYPE_NET_ACTIVE_CONNECTION
+		case "m7_in_Bps":
+			ret.MetricType = cloudprovider.LB_METRIC_TYPE_NET_BPS_RX
+		case "m8_out_Bps":
+			ret.MetricType = cloudprovider.LB_METRIC_TYPE_NET_BPS_TX
+		case "mc_l7_http_2xx":
+			ret.MetricType = cloudprovider.LB_METRIC_TYPE_HRSP_COUNT
+			tags = map[string]string{"request": "2xx"}
+		case "md_l7_http_3xx":
+			ret.MetricType = cloudprovider.LB_METRIC_TYPE_HRSP_COUNT
+			tags = map[string]string{"request": "3xx"}
+		case "md_l7_http_4xx":
+			ret.MetricType = cloudprovider.LB_METRIC_TYPE_HRSP_COUNT
+			tags = map[string]string{"request": "4xx"}
+		case "md_l7_http_5xx":
+			ret.MetricType = cloudprovider.LB_METRIC_TYPE_HRSP_COUNT
+			tags = map[string]string{"request": "5xx"}
+		case "me_l7_http_4xx":
+			ret.MetricType = cloudprovider.LB_METRIC_TYPE_HRSP_COUNT
+			tags = map[string]string{"request": "4xx"}
+		case "mf_l7_http_5xx":
+			ret.MetricType = cloudprovider.LB_METRIC_TYPE_HRSP_COUNT
+			tags = map[string]string{"request": "5xx"}
+		default:
+			log.Warningf("invalid metricName %s for %s %s", metricData.MetricName, opts.ResourceType, opts.ResourceId)
+			continue
+		}
+		for _, value := range metricData.Datapoints {
+			metricValue := cloudprovider.MetricValue{
+				Value:     value.Average,
+				Timestamp: time.UnixMilli(value.Timestamp),
+				Tags:      tags,
+			}
+			ret.Values = append(ret.Values, metricValue)
+		}
+		result = append(result, ret)
+	}
+	return result, nil
+}
+
 func (self *SHuaweiClient) GetMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) {
 	switch opts.ResourceType {
 	case cloudprovider.METRIC_RESOURCE_TYPE_SERVER:
@@ -363,6 +444,8 @@ func (self *SHuaweiClient) GetMetrics(opts *cloudprovider.MetricListOptions) ([]
 		return self.getBucketMetrics(opts)
 	case cloudprovider.METRIC_RESOURCE_TYPE_MODELARTS_POOL:
 		return self.getModelartsPoolMetrics(opts)
+	case cloudprovider.METRIC_RESOURCE_TYPE_LB:
+		return self.getLoadbalancerMetrics(opts)
 	default:
 		return nil, errors.Wrapf(cloudprovider.ErrNotSupported, "%s", opts.ResourceType)
 	}

vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/modelarts_pool.go

@@ -372,6 +372,15 @@ func (self *SModelartsPool) SetAutoRenew(bc billing.SBillingCycle) error {
 }
 
 func (self *SModelartsPool) Refresh() error {
+	self.Status.Resource = SNodeStatus{}
+	pool, err := self.region.client.modelartsPoolById(self.GetId())
+	if err == nil {
+		pool.Unmarshal(self)
+		return nil
+	}
+	if err != errors.ErrNotFound {
+		return errors.Wrap(err, "modelartsPoolById")
+	}
 	pools := make([]SModelartsPool, 0)
 	resObj, err := self.region.client.modelartsPoolListWithStatus("pools", "failed", nil)
 	if err != nil {
@@ -384,13 +393,9 @@ func (self *SModelartsPool) Refresh() error {
 	for _, pool := range pools {
 		if pool.GetId() == self.GetId() {
 			self.Status.Phase = "CreationFailed"
+			break
 		}
 	}
-	self.Status.Resource = SNodeStatus{}
-	pool, err := self.region.client.modelartsPoolById(self.GetId())
-	if err != nil {
-		return errors.Wrapf(err, "GetModelartsPool(%s)", self.GetId())
-	}
 	return jsonutils.Update(self, pool)
 }
 

vendor/yunion.io/x/pkg/sortedstring/sortedstrings.go

@@ -0,0 +1,201 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sortedstring
+
+import (
+	"sort"
+)
+
+type SSortedStrings []string
+
+func NewSortedStrings(strs []string) SSortedStrings {
+	if strs == nil {
+		return SSortedStrings{}
+	}
+	sort.Strings(strs)
+	return SSortedStrings(strs)
+}
+
+func Append(ss SSortedStrings, ele ...string) SSortedStrings {
+	ss = ss.Append(ele...)
+	return ss
+}
+
+func (ss SSortedStrings) Append(ele ...string) SSortedStrings {
+	if ss == nil {
+		ss = NewSortedStrings([]string{})
+	}
+	for _, e := range ele {
+		pos, find := ss.Index(e)
+		if find {
+			continue
+		}
+		ss = append(ss, e)
+		copy(ss[pos+1:], ss[pos:])
+		ss[pos] = e
+	}
+	return ss
+}
+
+func (ss SSortedStrings) Remove(ele ...string) SSortedStrings {
+	if ss == nil {
+		return ss
+	}
+	for _, e := range ele {
+		pos, find := ss.Index(e)
+		if !find {
+			continue
+		}
+		if pos < len(ss)-1 {
+			copy(ss[pos:], ss[pos+1:])
+		}
+		ss = ss[:len(ss)-1]
+	}
+	return ss
+}
+
+func (ss SSortedStrings) Index(needle string) (int, bool) {
+	i := 0
+	j := len(ss) - 1
+	for i <= j {
+		m := (i + j) / 2
+		if ss[m] < needle {
+			i = m + 1
+		} else if ss[m] > needle {
+			j = m - 1
+		} else {
+			return m, true
+		}
+	}
+	return j + 1, false
+}
+
+func (ss SSortedStrings) Contains(needle string) bool {
+	_, find := ss.Index(needle)
+	return find
+}
+
+func (ss SSortedStrings) ContainsAny(needles ...string) bool {
+	for i := range needles {
+		_, find := ss.Index(needles[i])
+		if find {
+			return true
+		}
+	}
+	return false
+}
+
+func (ss SSortedStrings) ContainsAll(needles ...string) bool {
+	for i := range needles {
+		_, find := ss.Index(needles[i])
+		if !find {
+			return false
+		}
+	}
+	return true
+}
+
+func Contains(a, b SSortedStrings) bool {
+	_, _, bNoA := Split(a, b)
+	if len(bNoA) == 0 {
+		return true
+	} else {
+		return false
+	}
+}
+
+func Equals(a, b SSortedStrings) bool {
+	aNoB, _, bNoA := Split(a, b)
+	if len(aNoB) == 0 && len(bNoA) == 0 {
+		return true
+	} else {
+		return false
+	}
+}
+
+func Split(a, b SSortedStrings) (aNoB SSortedStrings, aAndB SSortedStrings, bNoA SSortedStrings) {
+	a_b := make([]string, 0)
+	b_a := make([]string, 0)
+	anb := make([]string, 0)
+	i := 0
+	j := 0
+	for i < len(a) && j < len(b) {
+		if a[i] == b[j] {
+			anb = append(anb, a[i])
+			i += 1
+			j += 1
+		} else if a[i] < b[j] {
+			a_b = append(a_b, a[i])
+			i += 1
+		} else if a[i] > b[j] {
+			b_a = append(b_a, b[j])
+			j += 1
+		}
+	}
+	if i < len(a) {
+		a_b = append(a_b, a[i:]...)
+	}
+	if j < len(b) {
+		b_a = append(b_a, b[j:]...)
+	}
+	aNoB = SSortedStrings(a_b)
+	aAndB = SSortedStrings(anb)
+	bNoA = SSortedStrings(b_a)
+	return
+}
+
+func Merge(a, b SSortedStrings) SSortedStrings {
+	ret := make([]string, 0)
+	i := 0
+	j := 0
+	for i < len(a) && j < len(b) {
+		if a[i] == b[j] {
+			ret = append(ret, a[i])
+			i += 1
+			j += 1
+		} else if a[i] < b[j] {
+			ret = append(ret, a[i])
+			i += 1
+		} else if a[i] > b[j] {
+			ret = append(ret, b[j])
+			j += 1
+		}
+	}
+	if i < len(a) {
+		ret = append(ret, a[i:]...)
+	}
+	if j < len(b) {
+		ret = append(ret, b[j:]...)
+	}
+	return SSortedStrings(ret)
+}
+
+func Intersect(a, b SSortedStrings) SSortedStrings {
+	ret := make([]string, 0)
+	i := 0
+	j := 0
+	for i < len(a) && j < len(b) {
+		if a[i] == b[j] {
+			ret = append(ret, a[i])
+			i += 1
+			j += 1
+		} else if a[i] < b[j] {
+			i += 1
+		} else if a[i] > b[j] {
+			j += 1
+		}
+	}
+	return SSortedStrings(ret)
+}

vendor/yunion.io/x/pkg/util/httputils/httputils.go

@@ -487,6 +487,8 @@ func isHTTPReqErrorRetryable(err error) bool {
 		} else if strings.Contains(err.Error(), "net/http: timeout awaiting response headers") {
 			// Retry errors due to server not sending the response before timeout
 			return true
+		} else if strings.Contains(err.Error(), "dial tcp: lookup") {
+			return true
 		}
 	}
 	return false

vendor/yunion.io/x/sqlchemy/backends/clickhouse/clickhouse.go

@@ -173,7 +173,7 @@ func (click *SClickhouseBackend) FetchTableColumnSpecs(ts sqlchemy.ITableSpec) (
 	if err != nil {
 		return nil, errors.Wrap(err, "show create table")
 	}
-	primaries, orderbys, partition, ttl := parseCreateTable(defStr)
+	primaries, orderbys, partitions, ttl := parseCreateTable(defStr)
 	var ttlCfg sColumnTTL
 	if len(ttl) > 0 {
 		ttlCfg, err = parseTTLExpression(ttl)
@@ -189,8 +189,10 @@ func (click *SClickhouseBackend) FetchTableColumnSpecs(ts sqlchemy.ITableSpec) (
 			if utils.IsInStringArray(clickSpec.Name(), orderbys) {
 				clickSpec.SetOrderBy(true)
 			}
-			if strings.Contains(partition, clickSpec.Name()) {
-				clickSpec.SetPartitionBy(partition)
+			for _, part := range partitions {
+				if strings.Contains(part, clickSpec.Name()) {
+					clickSpec.SetPartitionBy(part)
+				}
 			}
 			if ttlCfg.ColName == clickSpec.Name() {
 				clickSpec.SetTTL(ttlCfg.Count, ttlCfg.Unit)

vendor/yunion.io/x/sqlchemy/backends/clickhouse/column.go

@@ -181,7 +181,7 @@ func (c *SBooleanColumn) IsZero(val interface{}) bool {
 		return bVal == nil
 	}
 	bVal := val.(bool)
-	return bVal == false
+	return !bVal
 }
 
 // NewBooleanColumn return an instance of SBooleanColumn
@@ -241,10 +241,10 @@ func (c *STristateColumn) IsZero(val interface{}) bool {
 
 // NewTristateColumn return an instance of STristateColumn
 func NewTristateColumn(name string, tagmap map[string]string, isPointer bool) STristateColumn {
-	if _, ok := tagmap[sqlchemy.TAG_NULLABLE]; ok {
-		// tristate always nullable
-		delete(tagmap, sqlchemy.TAG_NULLABLE)
-	}
+	//if _, ok := tagmap[sqlchemy.TAG_NULLABLE]; ok {
+	// tristate always nullable
+	delete(tagmap, sqlchemy.TAG_NULLABLE)
+	//}
 	bc := STristateColumn{SClickhouseBaseColumn: NewClickhouseBaseColumn(name, "UInt8", tagmap, isPointer)}
 	return bc
 }
@@ -298,7 +298,6 @@ func (c *SIntegerColumn) ConvertFromString(str string) interface{} {
 		default:
 			panic(fmt.Sprintf("unsupported type %s", ctype))
 		}
-		return val
 	} else {
 		val, _ := strconv.ParseInt(str, 10, 64)
 		switch ctype {
@@ -313,7 +312,6 @@ func (c *SIntegerColumn) ConvertFromString(str string) interface{} {
 		default:
 			panic(fmt.Sprintf("unsupported type %s", ctype))
 		}
-		return val
 	}
 }
 
@@ -359,16 +357,16 @@ func (c *SFloatColumn) IsZero(val interface{}) bool {
 	if c.IsPointer() {
 		switch val.(type) {
 		case *float32:
-			return val.(*float32) == nil
+			return val == nil
 		case *float64:
-			return val.(*float64) == nil
+			return val == nil
 		}
 	} else {
 		switch val.(type) {
 		case float32:
-			return val.(float32) == 0.0
+			return val == 0.0
 		case float64:
-			return val.(float64) == 0.0
+			return val == 0.0
 		}
 	}
 	return true
@@ -386,7 +384,6 @@ func (c *SFloatColumn) ConvertFromString(str string) interface{} {
 	default:
 		panic(fmt.Sprintf("unsupported type %s", ctype))
 	}
-	return val
 }
 
 // NewFloatColumn returns an instance of SFloatColumn
@@ -425,16 +422,16 @@ func (c *SDecimalColumn) IsZero(val interface{}) bool {
 	if c.IsPointer() {
 		switch val.(type) {
 		case *float32:
-			return val.(*float32) == nil
+			return val == nil
 		case *float64:
-			return val.(*float64) == nil
+			return val == nil
 		}
 	} else {
 		switch val.(type) {
 		case float32:
-			return val.(float32) == 0.0
+			return val == 0.0
 		case float64:
-			return val.(float64) == 0.0
+			return val == 0.0
 		}
 	}
 	return true

vendor/yunion.io/x/sqlchemy/backends/clickhouse/columninfo.go

@@ -16,6 +16,7 @@ package clickhouse
 
 import (
 	"regexp"
+	"sort"
 	"strings"
 
 	"yunion.io/x/log"
@@ -140,6 +141,7 @@ func parseKeys(keyStr string) []string {
 		key = strings.TrimSpace(key)
 		ret = append(ret, key)
 	}
+	sort.Strings(ret)
 	return ret
 }
 
@@ -163,7 +165,27 @@ func findSegment(sqlStr string, prefix string) string {
 	return ""
 }
 
-func parseCreateTable(sqlStr string) (primaries []string, orderbys []string, partition string, ttl string) {
+func trimPartition(partStr string) string {
+	for {
+		partStr = strings.TrimSpace(partStr)
+		if partStr[0] == '(' {
+			partStr = partStr[1 : len(partStr)-1]
+		} else {
+			break
+		}
+	}
+	partStr = strings.ReplaceAll(partStr, " ", "")
+	return partStr
+}
+
+func parsePartitions(partStr string) []string {
+	partStr = trimPartition(partStr)
+	parts := strings.Split(partStr, ",")
+	sort.Strings(parts)
+	return parts
+}
+
+func parseCreateTable(sqlStr string) (primaries []string, orderbys []string, partitions []string, ttl string) {
 	matches := primaryKeyRegexp.FindAllStringSubmatch(sqlStr, -1)
 	if len(matches) > 0 {
 		primaries = parseKeys(matches[0][1])
@@ -172,7 +194,8 @@ func parseCreateTable(sqlStr string) (primaries []string, orderbys []string, par
 	if len(matches) > 0 {
 		orderbys = parseKeys(matches[0][1])
 	}
-	partition = findSegment(sqlStr, partitionByPrefix)
+	partitionStr := findSegment(sqlStr, partitionByPrefix)
+	partitions = parsePartitions(partitionStr)
 	ttl = findSegment(sqlStr, ttlPrefix)
 	return
 }

vendor/yunion.io/x/sqlchemy/backends/clickhouse/sync.go

@@ -16,10 +16,14 @@ package clickhouse
 
 import (
 	"fmt"
+	"sort"
 	"strings"
+	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/sortedstring"
+	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 )
 
@@ -42,18 +46,32 @@ func findTtlColumn(cols []sqlchemy.IColumnSpec) sColumnTTL {
 	return ret
 }
 
+func findPartitions(cols []sqlchemy.IColumnSpec) []string {
+	parts := make([]string, 0)
+	for i := range cols {
+		if c, ok := cols[i].(IClickhouseColumnSpec); ok {
+			part := strings.ReplaceAll(c.PartitionBy(), " ", "")
+			if len(part) > 0 && !utils.IsInStringArray(part, parts) {
+				parts = append(parts, part)
+			}
+		}
+	}
+	sort.Strings(parts)
+	return parts
+}
+
 func (clickhouse *SClickhouseBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpec, changes sqlchemy.STableChanges) []string {
 	ret := make([]string, 0)
 
+	needCopyTable := false
+
 	alters := make([]string, 0)
 	// first check if primary key is modifed
 	changePrimary := false
-	oldHasPrimary := false
 
 	for _, col := range changes.RemoveColumns {
 		if col.IsPrimary() {
 			changePrimary = true
-			oldHasPrimary = true
 		}
 	}
 
@@ -61,19 +79,21 @@ func (clickhouse *SClickhouseBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpe
 		if cols.OldCol.IsPrimary() != cols.NewCol.IsPrimary() {
 			changePrimary = true
 		}
-		if cols.OldCol.IsPrimary() {
-			oldHasPrimary = true
-		}
 	}
 	for _, col := range changes.AddColumns {
 		if col.IsPrimary() {
 			changePrimary = true
 		}
 	}
-	if changePrimary && oldHasPrimary {
-		sql := fmt.Sprintf("DROP PRIMARY KEY")
-		alters = append(alters, sql)
+
+	if changePrimary {
+		log.Infof("primary key changed")
+		needCopyTable = true
 	}
+	// if changePrimary && oldHasPrimary {
+	// 	sql := fmt.Sprintf("DROP PRIMARY KEY")
+	// 	alters = append(alters, sql)
+	// }
 	/* IGNORE DROP STATEMENT */
 	for _, col := range changes.RemoveColumns {
 		sql := fmt.Sprintf("DROP COLUMN `%s`", col.Name())
@@ -120,6 +140,7 @@ func (clickhouse *SClickhouseBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpe
 		}
 	}*/
 
+	// check TTL
 	oldTtlSpec := findTtlColumn(changes.OldColumns)
 	newTtlSpec := findTtlColumn(ts.Columns())
 	log.Debugf("old: %s new: %s", jsonutils.Marshal(oldTtlSpec), jsonutils.Marshal(newTtlSpec))
@@ -140,5 +161,35 @@ func (clickhouse *SClickhouseBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpe
 		ret = append(ret, sql)
 	}
 
+	// check partitions
+	oldPartitions := findPartitions(changes.OldColumns)
+	newPartitions := findPartitions(ts.Columns())
+	if !sortedstring.Equals(oldPartitions, newPartitions) {
+		log.Infof("partition inconsistemt: old=%s new=%s", oldPartitions, newPartitions)
+		needCopyTable = true
+	}
+
+	// needCopyTable
+	if needCopyTable {
+		// create new table
+		alterTableName := fmt.Sprintf("%s_tmp_%d", ts.Name(), time.Now().Unix())
+		alterTable := ts.(*sqlchemy.STableSpec).Clone(alterTableName, 0)
+		createSqls := alterTable.CreateSQLs()
+		ret = append(ret, createSqls...)
+		colNames := make([]string, 0)
+		for _, c := range ts.Columns() {
+			colNames = append(colNames, fmt.Sprintf("`%s`", c.Name()))
+		}
+		colNamesStr := strings.Join(colNames, ",")
+		// copy data
+		sql := fmt.Sprintf("INSERT INTO `%s` (%s) SELECT %s FROM `%s`", alterTableName, colNamesStr, colNamesStr, ts.Name())
+		ret = append(ret, sql)
+		// rename tables
+		sql = fmt.Sprintf("RENAME TABLE `%s` TO `%s_backup`", ts.Name(), alterTableName)
+		ret = append(ret, sql)
+		sql = fmt.Sprintf("RENAME TABLE `%s` TO `%s`", alterTableName, ts.Name())
+		ret = append(ret, sql)
+	}
+
 	return ret
 }

vendor/yunion.io/x/sqlchemy/column.go

@@ -121,6 +121,11 @@ type IColumnSpec interface {
 	IsString() bool
 
 	IsDateTime() bool
+
+	// index of column, to preserve the column position
+	GetColIndex() int
+	// setter of column index
+	SetColIndex(idx int)
 }
 
 // SBaseColumn is the base structure represents a column
@@ -136,6 +141,7 @@ type SBaseColumn struct {
 	isIndex       bool
 	isAllowZero   bool
 	tags          map[string]string
+	colIndex      int
 }
 
 // IsPointer implementation of SBaseColumn for IColumnSpec
@@ -279,6 +285,14 @@ func (c *SBaseColumn) IsDateTime() bool {
 	return false
 }
 
+func (c *SBaseColumn) GetColIndex() int {
+	return c.colIndex
+}
+
+func (c *SBaseColumn) SetColIndex(idx int) {
+	c.colIndex = idx
+}
+
 // NewBaseColumn returns an instance of SBaseColumn
 func NewBaseColumn(name string, sqltype string, tagmap map[string]string, isPointer bool) SBaseColumn {
 	var val string
@@ -333,6 +347,7 @@ func NewBaseColumn(name string, sqltype string, tagmap map[string]string, isPoin
 		tags:          tagmap,
 		isPointer:     isPointer,
 		isAllowZero:   isAllowZero,
+		colIndex:      -1,
 	}
 }
 

vendor/yunion.io/x/sqlchemy/query.go

@@ -81,6 +81,7 @@ func (tq *SQuery) HasField(f IQueryField) bool {
 	}
 	for i := range tq.fields {
 		fi := tq.fields[i]
+		// log.Debugf("field at %d: %s", i, fi.Name())
 		if fi.Name() == f.Name() {
 			return true
 		}
@@ -90,6 +91,7 @@ func (tq *SQuery) HasField(f IQueryField) bool {
 
 // AppendField appends query field to a query
 func (tq *SQuery) AppendField(f ...IQueryField) *SQuery {
+	// log.Debugf("AppendField tq has fields %d", len(tq.fields))
 	for i := range f {
 		if !tq.HasField(f[i]) {
 			tq.fields = append(tq.fields, f[i])

vendor/yunion.io/x/sqlchemy/subquery.go

@@ -49,7 +49,7 @@ func (sqf *SSubQueryField) Reference() string {
 
 // Label implementation of SSubQueryField for IQueryField
 func (sqf *SSubQueryField) Label(label string) IQueryField {
-	if len(label) > 0 && label != sqf.field.Name() {
+	if len(label) > 0 {
 		sqf.alias = label
 	}
 	return sqf

vendor/yunion.io/x/sqlchemy/sync.go

@@ -32,6 +32,12 @@ func compareColumnSpec(c1, c2 IColumnSpec) int {
 	return strings.Compare(c1.Name(), c2.Name())
 }
 
+func compareColumnIndex(c1, c2 IColumnSpec) int {
+	i1 := c1.GetColIndex()
+	i2 := c2.GetColIndex()
+	return i1 - i2
+}
+
 type SUpdateColumnSpec struct {
 	OldCol IColumnSpec
 	NewCol IColumnSpec

vendor/yunion.io/x/sqlchemy/table.go

@@ -17,8 +17,10 @@ package sqlchemy
 import (
 	"fmt"
 	"reflect"
+	"sort"
 
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/utils"
 )
 
@@ -141,8 +143,61 @@ func (ts *STableSpec) Expression() string {
 	return fmt.Sprintf("`%s`", ts.name)
 }
 
+func (ts *STableSpec) SyncColumnIndexes() error {
+	if !ts.Exists() {
+		return errors.Wrap(errors.ErrNotFound, "table not exists")
+	}
+
+	cols, err := ts.Database().backend.FetchTableColumnSpecs(ts)
+	if err != nil {
+		log.Errorf("fetchColumnDefs fail: %s", err)
+		return errors.Wrap(err, "FetchTableColumnSpecs")
+	}
+	if len(cols) != len(ts._columns) {
+		return errors.Wrapf(errors.ErrInvalidStatus, "ts col %d != actual col %d", len(ts._columns), len(cols))
+	}
+	for i := range cols {
+		cols[i].SetColIndex(i)
+	}
+	// sort colums
+	sort.Slice(cols, func(i, j int) bool {
+		return compareColumnSpec(cols[i], cols[j]) < 0
+	})
+	sort.Slice(ts._columns, func(i, j int) bool {
+		return compareColumnSpec(ts._columns[i], ts._columns[j]) < 0
+	})
+	// compare columns and assign colindex
+	for i := range ts._columns {
+		comp := compareColumnSpec(cols[i], ts._columns[i])
+		if comp != 0 {
+			return errors.Wrapf(errors.ErrInvalidStatus, "colname %s != %s", cols[i].Name(), ts._columns[i].Name())
+		}
+		ts._columns[i].SetColIndex(cols[i].GetColIndex())
+	}
+
+	// sort columns according to colindex
+	sort.Slice(ts._columns, func(i, j int) bool {
+		return compareColumnIndex(ts._columns[i], ts._columns[j]) < 0
+	})
+
+	return nil
+}
+
 // Clone makes a clone of a table, so we may create a new table of the same schema
 func (ts *STableSpec) Clone(name string, autoIncOffset int64) *STableSpec {
+	nts, _ := ts.CloneWithSyncColumnOrder(name, autoIncOffset, false)
+	return nts
+}
+
+// Clone makes a clone of a table, so we may create a new table of the same schema
+func (ts *STableSpec) CloneWithSyncColumnOrder(name string, autoIncOffset int64, syncColOrder bool) (*STableSpec, error) {
+	if ts.Exists() && syncColOrder {
+		// if table exists, sync column index
+		err := ts.SyncColumnIndexes()
+		if err != nil {
+			return nil, errors.Wrap(err, "SyncColumnIndexes")
+		}
+	}
 	columns := ts.Columns()
 	newCols := make([]IColumnSpec, len(columns))
 	for i := range newCols {
@@ -170,7 +225,7 @@ func (ts *STableSpec) Clone(name string, autoIncOffset int64) *STableSpec {
 		newIndexes[i] = ts._indexes[i].clone(nts)
 	}
 	nts._indexes = newIndexes
-	return nts
+	return nts, nil
 }
 
 // Columns implementation of STableSpec for ITableSpec