From eb6322cd4f2ffde842755f51d4a0df94e048e091 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Thu, 1 Dec 2022 01:21:56 +0800 Subject: [PATCH] fix(cloudmon): hcs metric (#15472) --- cmd/climc/shell/compute/cloudaccounts.go | 2 +- cmd/cryptool/shell/decrypt.go | 14 + go.mod | 2 +- go.sum | 4 +- .../modules/compute/mod_cloudproviders.go | 2 + pkg/mcclient/options/cloudaccounts.go | 15 +- vendor/modules.txt | 2 +- .../x/cloudmux/pkg/multicloud/hcs/hcs.go | 16 + .../x/cloudmux/pkg/multicloud/hcs/monitor.go | 574 +++++------------- .../pkg/multicloud/hcs/provider/provider.go | 20 +- .../pkg/multicloud/hcso/client/auth/signer.go | 11 - 11 files changed, 214 insertions(+), 448 deletions(-) diff --git a/cmd/climc/shell/compute/cloudaccounts.go b/cmd/climc/shell/compute/cloudaccounts.go index 94fc47dad1..21755e36d5 100644 --- a/cmd/climc/shell/compute/cloudaccounts.go +++ b/cmd/climc/shell/compute/cloudaccounts.go @@ -64,7 +64,7 @@ func init() { cmd.UpdateWithKeyword("update-openstack", &options.SOpenStackCloudAccountUpdateOptions{}) cmd.UpdateWithKeyword("update-huawei", &options.SHuaweiCloudAccountUpdateOptions{}) cmd.UpdateWithKeyword("update-hcso", &options.SHCSOAccountUpdateOptions{}) - cmd.UpdateWithKeyword("update-hcs", &options.SHCSOAccountUpdateOptions{}) + cmd.UpdateWithKeyword("update-hcs", &options.SHCSAccountUpdateOptions{}) cmd.UpdateWithKeyword("update-ucloud", &options.SUcloudCloudAccountUpdateOptions{}) cmd.UpdateWithKeyword("update-zstack", &options.SZStackCloudAccountUpdateOptions{}) cmd.UpdateWithKeyword("update-s3", &options.SS3CloudAccountUpdateOptions{}) diff --git a/cmd/cryptool/shell/decrypt.go b/cmd/cryptool/shell/decrypt.go index 5e9763b09d..7da266e687 100644 --- a/cmd/cryptool/shell/decrypt.go +++ b/cmd/cryptool/shell/decrypt.go @@ -35,4 +35,18 @@ func init() { fmt.Println(text) return nil }) + + type EncryptSecretOptions struct { + KEY string `help:"secret key"` + SECRET string `help:"secret to enscrypt"` + } + shellutils.R(&EncryptSecretOptions{}, "encrypt", "Encrypt", func(args *EncryptSecretOptions) error { + text, err := utils.EncryptAESBase64(args.KEY, args.SECRET) + if err != nil { + return err + } + fmt.Println(text) + return nil + }) + } diff --git a/go.mod b/go.mod index 1b65a6a1ca..446e8d7abc 100644 --- a/go.mod +++ b/go.mod @@ -84,7 +84,7 @@ require ( k8s.io/client-go v0.19.3 k8s.io/cluster-bootstrap v0.19.3 moul.io/http2curl/v2 v2.3.0 - yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20221129103405-4636d4b6d0e6 + yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20221130104655-dfbb3d8e7a2e yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656 yunion.io/x/jsonutils v1.0.1-0.20220819091305-3bab322ab4fd yunion.io/x/log v1.0.0 diff --git a/go.sum b/go.sum index 79478f6500..8caa0cfa6e 100644 --- a/go.sum +++ b/go.sum @@ -1176,8 +1176,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20221129103405-4636d4b6d0e6 h1:sQw0C75yeObEbUJtrbu1PwWP1FSir5TLJZgQm7kXrtQ= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20221129103405-4636d4b6d0e6/go.mod h1:drc/WDD65r6wI1tSejk19xcl9QwmzZlgJGXN7a9o2KQ= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20221130104655-dfbb3d8e7a2e h1:m6vnTTEnR59HMwgQu4XfJqlD4F6uX+fnWJZFKW/wI7s= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20221130104655-dfbb3d8e7a2e/go.mod h1:drc/WDD65r6wI1tSejk19xcl9QwmzZlgJGXN7a9o2KQ= yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656 h1:0zlZD5uhZoIHgLVAWCz2aHaYk2ZrNsACCYD7R6EIBII= yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= diff --git a/pkg/mcclient/modules/compute/mod_cloudproviders.go b/pkg/mcclient/modules/compute/mod_cloudproviders.go index 99b170096d..ed9019588a 100644 --- a/pkg/mcclient/modules/compute/mod_cloudproviders.go +++ b/pkg/mcclient/modules/compute/mod_cloudproviders.go @@ -77,6 +77,8 @@ type SCloudDelegate struct { Options struct { cloudprovider.SHCSOEndpoints + Account string + Password string } ProxySetting proxyapi.SProxySetting } diff --git a/pkg/mcclient/options/cloudaccounts.go b/pkg/mcclient/options/cloudaccounts.go index 43ae7ba26f..dd50bd8f4e 100644 --- a/pkg/mcclient/options/cloudaccounts.go +++ b/pkg/mcclient/options/cloudaccounts.go @@ -877,10 +877,23 @@ func (opts *SHCSOAccountUpdateOptions) Params() (jsonutils.JSONObject, error) { type SHCSAccountUpdateOptions struct { SCloudAccountUpdateBaseOptions + Account string + Password string } func (opts *SHCSAccountUpdateOptions) Params() (jsonutils.JSONObject, error) { - return jsonutils.Marshal(opts), nil + params := jsonutils.Marshal(opts.SCloudAccountUpdateBaseOptions).(*jsonutils.JSONDict) + options := jsonutils.NewDict() + if len(opts.Account) > 0 { + options.Set("account", jsonutils.NewString(opts.Account)) + } + if len(opts.Password) > 0 { + options.Set("password", jsonutils.NewString(opts.Password)) + } + if options.Length() > 0 { + params.Set("options", options) + } + return params, nil } type SUcloudCloudAccountUpdateOptions struct { diff --git a/vendor/modules.txt b/vendor/modules.txt index 509d3c8bab..0570595ec0 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1439,7 +1439,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.2.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20221129103405-4636d4b6d0e6 +# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20221130104655-dfbb3d8e7a2e ## explicit; go 1.18 yunion.io/x/cloudmux/pkg/apis yunion.io/x/cloudmux/pkg/apis/billing diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcs/hcs.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcs/hcs.go index 421d451b7d..cd98baafca 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcs/hcs.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcs/hcs.go @@ -55,9 +55,18 @@ type HcsConfig struct { accessKey string accessSecret string + account string + password string + + token string + debug bool } +func (self *HcsConfig) isAccountValid() bool { + return len(self.account) > 0 && len(self.password) > 0 +} + func NewHcsConfig(accessKey, accessSecret, projectId, url string) *HcsConfig { cfg := &HcsConfig{ projectId: projectId, @@ -79,6 +88,12 @@ func (cfg *HcsConfig) Debug(debug bool) *HcsConfig { return cfg } +func (cfg *HcsConfig) WithAccount(account, password string) *HcsConfig { + cfg.account = account + cfg.password = password + return cfg +} + type SHcsClient struct { *HcsConfig @@ -96,6 +111,7 @@ type SHcsClient struct { defaultRegion string httpClient *http.Client + token string lock sThrottlingThreshold } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcs/monitor.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcs/monitor.go index 06131282e4..9d2bfdfc78 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcs/monitor.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcs/monitor.go @@ -15,479 +15,193 @@ package hcs import ( + "context" "fmt" + "net/http" + "net/url" + "strconv" + "strings" "time" "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/pkg/errors" - - api "yunion.io/x/cloudmux/pkg/apis/compute" - "yunion.io/x/cloudmux/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/util/httputils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/gotypes" + + "yunion.io/x/cloudmux/pkg/cloudprovider" ) -type MetricData struct { - Namespace string - MetricName string - Dimensions []struct { - Name string - Value string - } - Datapoints []struct { - Average float64 - Timestamp int64 - } - Unit string +type SCloudVM struct { + NativeId string + ResId string } -func (self *SHcsClient) monitorPost(resource string, params map[string]interface{}) (jsonutils.JSONObject, error) { - url := self._url("ces", "V1.0", self.defaultRegion, resource) - return self.request(httputils.POST, url, nil, params) +var vmMetric = map[string]cloudprovider.TMetricType{ + "562958543421441": cloudprovider.VM_METRIC_TYPE_CPU_USAGE, + "562958543486979": cloudprovider.VM_METRIC_TYPE_MEM_USAGE, + "562958543552537": cloudprovider.VM_METRIC_TYPE_NET_BPS_RX, //kb + "562958543552538": cloudprovider.VM_METRIC_TYPE_NET_BPS_TX, //kb + "562958543618052": cloudprovider.VM_METRIC_TYPE_DISK_USAGE, + "562958543618072": cloudprovider.VM_METRIC_TYPE_DISK_IO_READ_BPS, + "562958543618073": cloudprovider.VM_METRIC_TYPE_DISK_IO_WRITE_BPS, + "562958543618061": cloudprovider.VM_METRIC_TYPE_DISK_IO_WRITE_IOPS, + "562958543618062": cloudprovider.VM_METRIC_TYPE_DISK_IO_READ_IOPS, } func (self *SHcsClient) getServerMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { - params := map[string]interface{}{ - "from": fmt.Sprintf("%d", opts.StartTime.UnixMilli()), - "to": fmt.Sprintf("%d", opts.EndTime.UnixMilli()), - "period": "1", - "filter": "average", - } - metrics := []interface{}{} - namespace, dimesionName, metricNames := "SYS.ECS", "instance_id", []string{ - "cpu_util", - "network_incoming_bytes_aggregate_rate", - "network_outgoing_bytes_aggregate_rate", - "disk_read_bytes_rate", - "disk_write_bytes_rate", - "disk_read_requests_rate", - "disk_write_requests_rate", - } - for _, metricName := range metricNames { - metrics = append(metrics, map[string]interface{}{ - "namespace": namespace, - "metric_name": metricName, - "dimensions": []map[string]string{ - { - "name": dimesionName, - "value": opts.ResourceId, + params := url.Values{} + params.Set("pageNo", "1") + params.Set("pageSize", "1") + params.Set("condition", jsonutils.Marshal(map[string]interface{}{ + "constraint": []map[string]interface{}{ + { + "simple": map[string]interface{}{ + "name": "nativeId", + "operator": "equal", + "value": opts.ResourceId, }, }, - }) - } - params["metrics"] = metrics - resp, err := self.monitorPost("batch-query-metric-data", params) + }, + }).String()) + resp, err := self.ocRequest(httputils.GET, "rest/tenant-resource/v1/instances/CLOUD_VM", params, nil) if err != nil { - return nil, err - } - metricData := []MetricData{} - err = resp.Unmarshal(&metricData, "metrics") - if err != nil { - return nil, errors.Wrapf(err, "resp.Unmarshal") + return nil, errors.Wrapf(err, "ocRequest") } + vms := []SCloudVM{} + resp.Unmarshal(&vms, "objList") result := []cloudprovider.MetricValues{} - for i := range metricData { - ret := cloudprovider.MetricValues{ - Id: opts.ResourceId, - Unit: metricData[i].Unit, - Values: []cloudprovider.MetricValue{}, - } - tags := map[string]string{} - switch metricData[i].MetricName { - case "cpu_util": - ret.MetricType = cloudprovider.VM_METRIC_TYPE_CPU_USAGE - case "network_incoming_bytes_aggregate_rate": - ret.MetricType = cloudprovider.VM_METRIC_TYPE_NET_BPS_RX - tags = map[string]string{"net_type": "internet"} - case "network_outgoing_bytes_aggregate_rate": - ret.MetricType = cloudprovider.VM_METRIC_TYPE_NET_BPS_TX - tags = map[string]string{"net_type": "internet"} - case "disk_read_bytes_rate": - ret.MetricType = cloudprovider.VM_METRIC_TYPE_DISK_IO_READ_BPS - case "disk_write_bytes_rate": - ret.MetricType = cloudprovider.VM_METRIC_TYPE_DISK_IO_WRITE_BPS - case "disk_read_requests_rate": - ret.MetricType = cloudprovider.VM_METRIC_TYPE_DISK_IO_READ_IOPS - case "disk_write_requests_rate": - ret.MetricType = cloudprovider.VM_METRIC_TYPE_DISK_IO_WRITE_IOPS - default: - log.Warningf("invalid metricName %s for %s %s", metricData[i].MetricName, opts.ResourceType, opts.ResourceId) + for i := range vms { + if vms[i].NativeId != opts.ResourceId { continue } - for _, value := range metricData[i].Datapoints { - metricValue := cloudprovider.MetricValue{ - Value: value.Average, - Timestamp: time.UnixMilli(value.Timestamp), - Tags: tags, + metricIds := []string{} + for m := range vmMetric { + metricIds = append(metricIds, m) + } + body := map[string]interface{}{ + "obj_type_id": "562958543355904", + "indicator_ids": metricIds, + "obj_ids": []string{vms[i].ResId}, + "interval": "MINUTE", + "range": "BEGIN_END_TIME", + "begin_time": opts.StartTime.Unix() * 1000, + "end_time": opts.EndTime.Unix() * 1000, + } + resp, err := self.ocRequest(httputils.POST, "rest/performance/v1/data-svc/history-data/action/query", nil, body) + if err != nil { + return nil, errors.Wrapf(err, "ocRequest") + } + if gotypes.IsNil(resp) || !resp.Contains("data") { + break + } + for m, metricType := range vmMetric { + series := []map[string]string{} + resp.Unmarshal(&series, "data", vms[i].ResId, m, "series") + if len(series) == 0 { + continue } - ret.Values = append(ret.Values, metricValue) + ret := cloudprovider.MetricValues{ + Id: opts.ResourceId, + Values: []cloudprovider.MetricValue{}, + MetricType: metricType, + } + for i := range series { + for _date, _value := range series[i] { + date, _ := strconv.Atoi(_date) + value, _ := strconv.ParseFloat(_value, 32) + if metricType == cloudprovider.VM_METRIC_TYPE_NET_BPS_RX || metricType == cloudprovider.VM_METRIC_TYPE_NET_BPS_TX { + value *= 1024 + } + ret.Values = append(ret.Values, cloudprovider.MetricValue{ + Value: value, + Timestamp: time.UnixMilli(int64(date)), + }) + } + } + result = append(result, ret) } - result = append(result, ret) } return result, nil } -func (self *SHcsClient) getRedisMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { +func (self *SHcsClient) ocAuth() error { + url := fmt.Sprintf("https://oc.%s.%s/rest/plat/smapp/v1/oauth/token", self.defaultRegion, self.authUrl) + client := self.getDefaultClient() + cli := httputils.NewJsonClient(client) params := map[string]interface{}{ - "from": fmt.Sprintf("%d", opts.StartTime.UnixMilli()), - "to": fmt.Sprintf("%d", opts.EndTime.UnixMilli()), - "period": "1", - "filter": "average", + "grantType": "password", + "userName": self.account, + "value": self.password, } - metrics := []interface{}{} - namespace, dimesionName, metricNames := "SYS.DCS", "dcs_instance_id", []string{ - "cpu_usage", - "memory_usage", - "instantaneous_input_kbps", - "instantaneous_output_kbps", - "connected_clients", - "instantaneous_ops", - "keys", - "expires", - "used_memory_dataset", - } - for _, metricName := range metricNames { - metrics = append(metrics, map[string]interface{}{ - "namespace": namespace, - "metric_name": metricName, - "dimensions": []map[string]string{ - { - "name": dimesionName, - "value": opts.ResourceId, - }, - }, - }) - } - params["metrics"] = metrics - resp, err := self.monitorPost("batch-query-metric-data", params) + req := httputils.NewJsonRequest(httputils.PUT, url, jsonutils.Marshal(params)) + _, resp, err := cli.Send(context.Background(), req, &hcsError{Url: url}, self.debug) if err != nil { - return nil, err + return errors.Wrapf(err, "Send") } - metricData := []MetricData{} - err = resp.Unmarshal(&metricData, "metrics") - if err != nil { - return nil, errors.Wrapf(err, "resp.Unmarshal") - } - result := []cloudprovider.MetricValues{} - for i := range metricData { - ret := cloudprovider.MetricValues{ - Id: opts.ResourceId, - Unit: metricData[i].Unit, - Values: []cloudprovider.MetricValue{}, - } - tags := map[string]string{} - switch metricData[i].MetricName { - case "cpu_usage": - ret.MetricType = cloudprovider.REDIS_METRIC_TYPE_CPU_USAGE - case "memory_usage": - ret.MetricType = cloudprovider.REDIS_METRIC_TYPE_MEM_USAGE - case "instantaneous_input_kbps": - ret.MetricType = cloudprovider.REDIS_METRIC_TYPE_NET_BPS_RX - case "instantaneous_output_kbps": - ret.MetricType = cloudprovider.REDIS_METRIC_TYPE_NET_BPS_TX - case "connected_clients": - ret.MetricType = cloudprovider.REDIS_METRIC_TYPE_USED_CONN - case "instantaneous_ops": - ret.MetricType = cloudprovider.REDIS_METRIC_TYPE_OPT_SES - case "keys": - ret.MetricType = cloudprovider.REDIS_METRIC_TYPE_CACHE_KEYS - case "expires": - ret.MetricType = cloudprovider.REDIS_METRIC_TYPE_CACHE_EXP_KEYS - case "used_memory_dataset": - ret.MetricType = cloudprovider.REDIS_METRIC_TYPE_DATA_MEM_USAGE - default: - log.Warningf("invalid metricName %s for %s %s", metricData[i].MetricName, opts.ResourceType, opts.ResourceId) - continue - } - for _, value := range metricData[i].Datapoints { - metricValue := cloudprovider.MetricValue{ - Value: value.Average, - Timestamp: time.UnixMilli(value.Timestamp), - Tags: tags, - } - ret.Values = append(ret.Values, metricValue) - } - result = append(result, ret) - } - return result, nil + self.token, err = resp.GetString("accessSession") + return errors.Wrapf(err, "resp.accessSession") } -func (self *SHcsClient) getRdsMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { - params := map[string]interface{}{ - "from": fmt.Sprintf("%d", opts.StartTime.UnixMilli()), - "to": fmt.Sprintf("%d", opts.EndTime.UnixMilli()), - "period": "1", - "filter": "average", - } - metrics := []interface{}{} - namespace, dimesionName, metricNames := "SYS.RDS", "rds_cluster_id", []string{ - "rds001_cpu_util", - "rds002_mem_util", - "rds004_bytes_in", - "rds004_bytes_in", - "rds005_bytes_out", - "rds039_disk_util", - "rds049_disk_read_throughput", - "rds050_disk_write_throughput", - "rds006_conn_count", - "rds008_qps", - "rds009_tps", - "rds013_innodb_reads", - "rds014_innodb_writes", - } - switch opts.Engine { - case api.DBINSTANCE_TYPE_POSTGRESQL: - dimesionName = "postgresql_cluster_id" - case api.DBINSTANCE_TYPE_SQLSERVER: - dimesionName = "rds_cluster_sqlserver_id" - } - - for _, metricName := range metricNames { - metrics = append(metrics, map[string]interface{}{ - "namespace": namespace, - "metric_name": metricName, - "dimensions": []map[string]string{ - { - "name": dimesionName, - "value": opts.ResourceId, - }, - }, - }) - } - params["metrics"] = metrics - resp, err := self.monitorPost("batch-query-metric-data", params) - if err != nil { - return nil, err - } - metricData := []MetricData{} - err = resp.Unmarshal(&metricData, "metrics") - if err != nil { - return nil, errors.Wrapf(err, "resp.Unmarshal") - } - result := []cloudprovider.MetricValues{} - for i := range metricData { - ret := cloudprovider.MetricValues{ - Id: opts.ResourceId, - Unit: metricData[i].Unit, - Values: []cloudprovider.MetricValue{}, +func (self *SHcsClient) ocRequest(method httputils.THttpMethod, resource string, query url.Values, params map[string]interface{}) (jsonutils.JSONObject, error) { + self.lock.CheckingLock() + if len(self.token) == 0 { + err := self.ocAuth() + if err != nil { + return nil, errors.Wrapf(err, "ocAuth") } - tags := map[string]string{} - switch metricData[i].MetricName { - case "rds001_cpu_util": - ret.MetricType = cloudprovider.RDS_METRIC_TYPE_CPU_USAGE - case "rds002_mem_util": - ret.MetricType = cloudprovider.RDS_METRIC_TYPE_MEM_USAGE - case "rds004_bytes_in": - ret.MetricType = cloudprovider.RDS_METRIC_TYPE_NET_BPS_RX - case "rds005_bytes_out": - ret.MetricType = cloudprovider.RDS_METRIC_TYPE_NET_BPS_TX - case "rds039_disk_util": - ret.MetricType = cloudprovider.RDS_METRIC_TYPE_DISK_USAGE - case "rds049_disk_read_throughput": - ret.MetricType = cloudprovider.RDS_METRIC_TYPE_DISK_READ_BPS - case "rds050_disk_write_throughput": - ret.MetricType = cloudprovider.RDS_METRIC_TYPE_DISK_WRITE_BPS - case "rds006_conn_count": - ret.MetricType = cloudprovider.RDS_METRIC_TYPE_CONN_COUNT - case "rds008_qps": - ret.MetricType = cloudprovider.RDS_METRIC_TYPE_QPS - case "rds009_tps": - ret.MetricType = cloudprovider.RDS_METRIC_TYPE_TPS - case "rds013_innodb_reads": - ret.MetricType = cloudprovider.RDS_METRIC_TYPE_INNODB_READ_BPS - case "rds014_innodb_writes": - ret.MetricType = cloudprovider.RDS_METRIC_TYPE_INNODB_WRITE_BPS - default: - log.Warningf("invalid metricName %s for %s %s", metricData[i].MetricName, opts.ResourceType, opts.ResourceId) - continue + } + client := self.getDefaultClient() + url := fmt.Sprintf("oc.%s.%s/%s", self.defaultRegion, self.authUrl, resource) + if len(query) > 0 { + url = fmt.Sprintf("%s?%s", url, query.Encode()) + } + url = strings.TrimPrefix(url, "http://") + if !strings.HasPrefix(url, "https://") { + url = fmt.Sprintf("https://%s", url) + } + var body jsonutils.JSONObject = nil + if len(params) > 0 { + body = jsonutils.Marshal(params) + } + header := http.Header{} + header.Set("X-Auth-Token", self.token) + cli := httputils.NewJsonClient(client) + req := httputils.NewJsonRequest(method, url, body) + req.SetHeader(header) + var resp jsonutils.JSONObject + var err error + for i := 0; i < 4; i++ { + _, resp, err = cli.Send(context.Background(), req, &hcsError{Url: url, Params: params}, self.debug) + if err == nil { + break } - for _, value := range metricData[i].Datapoints { - metricValue := cloudprovider.MetricValue{ - Value: value.Average, - Timestamp: time.UnixMilli(value.Timestamp), - Tags: tags, + if err != nil { + e, ok := err.(*hcsError) + if ok { + if e.Code == 404 { + return nil, errors.Wrapf(cloudprovider.ErrNotFound, err.Error()) + } + if e.Code == 429 { + log.Errorf("request %s %v try later", url, err) + self.lock.Lock() + time.Sleep(time.Second * 15) + continue + } } - ret.Values = append(ret.Values, metricValue) + return nil, err } - result = append(result, ret) } - return result, nil -} - -func (self *SHcsClient) getBucketMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { - params := map[string]interface{}{ - "from": opts.StartTime.UnixMilli(), - "to": opts.EndTime.UnixMilli(), - "period": "1", - "filter": "average", - } - metrics := []interface{}{} - namespace, dimesionName, metricNames := "SYS.OBS", "bucket_name", []string{ - "download_bytes", - "upload_bytes", - "first_byte_latency", - "get_request_count", - "request_count_4xx", - "request_count_5xx", - } - - for _, metricName := range metricNames { - metrics = append(metrics, map[string]interface{}{ - "namespace": namespace, - "metric_name": metricName, - "dimensions": []map[string]string{ - { - "name": dimesionName, - "value": opts.ResourceId, - }, - }, - }) - } - params["metrics"] = metrics - resp, err := self.monitorPost("batch-query-metric-data", params) - if err != nil { - return nil, err - } - metricData := []MetricData{} - err = resp.Unmarshal(&metricData, "metrics") - if err != nil { - return nil, errors.Wrapf(err, "resp.Unmarshal") - } - result := []cloudprovider.MetricValues{} - for i := range metricData { - ret := cloudprovider.MetricValues{ - Id: opts.ResourceId, - Unit: metricData[i].Unit, - Values: []cloudprovider.MetricValue{}, - } - tags := map[string]string{} - switch metricData[i].MetricName { - case "download_bytes": - ret.MetricType = cloudprovider.BUCKET_METRIC_TYPE_NET_BPS_TX - case "upload_bytes": - ret.MetricType = cloudprovider.BUCKET_METRIC_TYPE_NET_BPS_RX - case "first_byte_latency": - ret.MetricType = cloudprovider.BUCKET_METRIC_TYPE_LATECY - tags = map[string]string{"request": "get"} - case "get_request_count": - ret.MetricType = cloudprovider.BUCKET_METRYC_TYPE_REQ_COUNT - tags = map[string]string{"request": "get"} - case "request_count_4xx": - ret.MetricType = cloudprovider.BUCKET_METRYC_TYPE_REQ_COUNT - tags = map[string]string{"request": "4xx"} - case "request_count_5xx": - ret.MetricType = cloudprovider.BUCKET_METRYC_TYPE_REQ_COUNT - tags = map[string]string{"request": "5xx"} - default: - log.Warningf("invalid metricName %s for %s %s", metricData[i].MetricName, opts.ResourceType, opts.ResourceId) - continue - } - for _, value := range metricData[i].Datapoints { - metricValue := cloudprovider.MetricValue{ - Value: value.Average, - Timestamp: time.UnixMilli(value.Timestamp), - Tags: tags, - } - ret.Values = append(ret.Values, metricValue) - } - result = append(result, ret) - } - return result, nil -} - -func (self *SHcsClient) getLoadbalancerMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { - params := map[string]interface{}{ - "from": fmt.Sprintf("%d", opts.StartTime.UnixMilli()), - "to": fmt.Sprintf("%d", opts.EndTime.UnixMilli()), - "period": "1", - "filter": "average", - } - metrics := []interface{}{} - namespace, dimesionName, metricNames := "SYS.ELB", "lb_instance_id", []string{ - "m7_in_Bps", - "m8_out_Bps", - "mc_l7_http_2xx", - "md_l7_http_3xx", - "me_l7_http_4xx", - "mf_l7_http_5xx", - } - - for _, metricName := range metricNames { - metrics = append(metrics, map[string]interface{}{ - "namespace": namespace, - "metric_name": metricName, - "dimensions": []map[string]string{ - { - "name": dimesionName, - "value": opts.ResourceId, - }, - }, - }) - } - params["metrics"] = metrics - resp, err := self.monitorPost("batch-query-metric-data", params) - if err != nil { - return nil, err - } - metricData := []MetricData{} - err = resp.Unmarshal(&metricData, "metrics") - if err != nil { - return nil, errors.Wrapf(err, "resp.Unmarshal") - } - result := []cloudprovider.MetricValues{} - for i := range metricData { - ret := cloudprovider.MetricValues{ - Id: opts.ResourceId, - Unit: metricData[i].Unit, - Values: []cloudprovider.MetricValue{}, - } - tags := map[string]string{} - switch metricData[i].MetricName { - case "m7_in_Bps": - ret.MetricType = cloudprovider.LB_METRIC_TYPE_NET_BPS_RX - case "m8_out_Bps": - ret.MetricType = cloudprovider.LB_METRIC_TYPE_NET_BPS_TX - case "mc_l7_http_2xx": - ret.MetricType = cloudprovider.LB_METRIC_TYPE_HRSP_COUNT - tags = map[string]string{"request": "2xx"} - case "md_l7_http_3xx": - ret.MetricType = cloudprovider.LB_METRIC_TYPE_HRSP_COUNT - tags = map[string]string{"request": "3xx"} - case "md_l7_http_4xx": - ret.MetricType = cloudprovider.LB_METRIC_TYPE_HRSP_COUNT - tags = map[string]string{"request": "4xx"} - case "md_l7_http_5xx": - ret.MetricType = cloudprovider.LB_METRIC_TYPE_HRSP_COUNT - tags = map[string]string{"request": "5xx"} - default: - log.Warningf("invalid metricName %s for %s %s", metricData[i].MetricName, opts.ResourceType, opts.ResourceId) - continue - } - for _, value := range metricData[i].Datapoints { - metricValue := cloudprovider.MetricValue{ - Value: value.Average, - Timestamp: time.UnixMilli(value.Timestamp), - Tags: tags, - } - ret.Values = append(ret.Values, metricValue) - } - result = append(result, ret) - } - return result, nil + return resp, err } func (self *SHcsClient) GetMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { + if !self.isAccountValid() { + return nil, errors.Wrapf(cloudprovider.ErrNotSupported, "missing account info") + } switch opts.ResourceType { case cloudprovider.METRIC_RESOURCE_TYPE_SERVER: return self.getServerMetrics(opts) - case cloudprovider.METRIC_RESOURCE_TYPE_REDIS: - return self.getRedisMetrics(opts) - case cloudprovider.METRIC_RESOURCE_TYPE_RDS: - return self.getRdsMetrics(opts) - case cloudprovider.METRIC_RESOURCE_TYPE_BUCKET: - return self.getBucketMetrics(opts) - case cloudprovider.METRIC_RESOURCE_TYPE_LB: - return self.getLoadbalancerMetrics(opts) default: return nil, errors.Wrapf(cloudprovider.ErrNotSupported, "%s", opts.ResourceType) } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcs/provider/provider.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcs/provider/provider.go index e287eda305..7be6e36177 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcs/provider/provider.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcs/provider/provider.go @@ -20,6 +20,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" api "yunion.io/x/cloudmux/pkg/apis/compute" "yunion.io/x/cloudmux/pkg/cloudprovider" @@ -103,11 +104,20 @@ func parseAccount(account string) (accessKey string, projectId string) { } func (self *SHcsProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { + account, password := "", "" accessKey, projectId := parseAccount(cfg.Account) + if cfg.Options != nil { + account, _ = cfg.Options.GetString("account") + _password, _ := cfg.Options.GetString("password") + if len(account) > 0 && len(_password) > 0 { + password, _ = utils.DescryptAESBase64(cfg.Secret, _password) + } + } client, err := hcs.NewHcsClient( hcs.NewHcsConfig( accessKey, cfg.Secret, projectId, cfg.URL, - ).CloudproviderConfig(cfg), + ).CloudproviderConfig(cfg). + WithAccount(account, password), ) if err != nil { return nil, err @@ -213,3 +223,11 @@ func (self *SHcsProvider) GetObjectCannedAcls(regionId string) []string { func (self *SHcsProvider) GetCapabilities() []string { return self.client.GetCapabilities() } + +func (self *SHcsProvider) GetMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { + metrics, err := self.client.GetMetrics(opts) + if err != nil { + return nil, errors.Wrapf(err, "GetMetrics") + } + return metrics, nil +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/client/auth/signer.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/client/auth/signer.go index 1e66b0c2f3..c4f49a8481 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/client/auth/signer.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/client/auth/signer.go @@ -15,7 +15,6 @@ package auth import ( - "crypto/md5" "crypto/sha256" "encoding/hex" "fmt" @@ -185,21 +184,11 @@ func canonicalHeaderNames(request requests.IRequest) string { return strings.ToLower(ret) } -var SigningKeyCache = map[string][]byte{} - func getSigningKey(secretKey, date, regionId, service string) string { - joinedKey := strings.Join([]string{secretKey, date, regionId, service}, "") - cacheKey := fmt.Sprintf("%x", md5.Sum([]byte(joinedKey))) - if v, ok := SigningKeyCache[cacheKey]; ok { - return string(v) - } - ret := []byte("SDK" + secretKey) for _, k := range []string{date, regionId, service, "sdk_request"} { ret = signers.HmacSha256(k, ret) } - - SigningKeyCache[cacheKey] = ret return string(ret) }