diff --git a/go.mod b/go.mod index 86606429a6..895cdf762d 100644 --- a/go.mod +++ b/go.mod @@ -83,7 +83,7 @@ require ( k8s.io/client-go v0.19.3 k8s.io/cluster-bootstrap v0.19.3 moul.io/http2curl/v2 v2.3.0 - yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230301101220-ebd68c96818a + yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230306104021-eebbddd5e493 yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656 yunion.io/x/jsonutils v1.0.1-0.20220819091305-3bab322ab4fd yunion.io/x/log v1.0.0 diff --git a/go.sum b/go.sum index 0ecdf8cfd1..abb6321433 100644 --- a/go.sum +++ b/go.sum @@ -1164,8 +1164,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230301101220-ebd68c96818a h1:gSfHCwI/E6VXEkofCj8Fw3J5Byc8Pf9Ubt6ijtNPONk= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230301101220-ebd68c96818a/go.mod h1:yEMFtIKPyIdIfPzsw7oTcH41JXc8G5StUAQw3ZoGj5I= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230306104021-eebbddd5e493 h1:5YjYZfwgBYBiwXKiitu4c2pFvFPZgm8f32IXnq1DJE8= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230306104021-eebbddd5e493/go.mod h1:sWqblYRhQCO63xeKdxdAT3wczCCY8Mc+TDa3WD/a5Kg= yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656 h1:0zlZD5uhZoIHgLVAWCz2aHaYk2ZrNsACCYD7R6EIBII= yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= diff --git a/pkg/cloudmon/providerdriver/huaweistack.go b/pkg/cloudmon/providerdriver/huaweistack.go index 917b1c0d02..5b8999b385 100644 --- a/pkg/cloudmon/providerdriver/huaweistack.go +++ b/pkg/cloudmon/providerdriver/huaweistack.go @@ -29,7 +29,7 @@ import ( ) type HuaweiStackCollect struct { - SBaseCollectDriver + SCollectByResourceIdDriver } func (self *HuaweiStackCollect) GetProvider() string { diff --git a/vendor/modules.txt b/vendor/modules.txt index eda3a484c0..a1e4431044 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1442,7 +1442,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.2.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230301101220-ebd68c96818a +# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230306104021-eebbddd5e493 ## explicit; go 1.18 yunion.io/x/cloudmux/pkg/apis yunion.io/x/cloudmux/pkg/apis/billing diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/hcso.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/hcso.go index a2f65f4ea2..4e5a313a81 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/hcso.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/hcso.go @@ -218,6 +218,16 @@ func (self *SHuaweiClient) modelartsResourceflavors(params map[string]interface{ return self.request(httputils.GET, uri, url.Values{}, params) } +func (self *SHuaweiClient) commonMonitor(params map[string]string) (jsonutils.JSONObject, error) { + endpoint := self.resetEndpoint(self.endpoints.Ces, "ces") + uri := fmt.Sprintf("https://%s/V1.0/%s/metric-data", endpoint, self.projectId) + url := url.Values{} + for k, v := range params { + url.Set(k, v) + } + return self.request(httputils.GET, uri, url, nil) +} + func (self *SHuaweiClient) patchRequest(method httputils.THttpMethod, url string, query url.Values, params map[string]interface{}) (jsonutils.JSONObject, error) { client := self.getAkClient() if len(query) > 0 { diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/monitor.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/monitor.go index a9d2212d79..4eff351f59 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/monitor.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/monitor.go @@ -15,21 +15,39 @@ package hcso import ( + "fmt" + "strconv" "time" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" + api "yunion.io/x/cloudmux/pkg/apis/compute" "yunion.io/x/cloudmux/pkg/cloudprovider" "yunion.io/x/cloudmux/pkg/multicloud/huawei/client/modules" ) +type MetricData struct { + Namespace string + MetricName string + Dimensions []struct { + Name string + Value string + } + Datapoints []struct { + Average float64 + Timestamp int64 + } + Unit string +} + func (r *SRegion) GetMetrics() ([]modules.SMetricMeta, error) { return r.ecsClient.CloudEye.ListMetrics() } -func (r *SRegion) GetMetricsData(metrics []modules.SMetricMeta, since time.Time, until time.Time) ([]modules.SMetricData, error) { - return r.ecsClient.CloudEye.GetMetricsData(metrics, since, until) +func (r *SRegion) GetMetricsData(metrics []modules.SMetricMeta, since time.Time, until time.Time) ([]cloudprovider.MetricValues, error) { + return r.client.getServerMetrics(&cloudprovider.MetricListOptions{ResourceType: cloudprovider.METRIC_RESOURCE_TYPE_SERVER, MetricType: cloudprovider.VM_METRIC_TYPE_CPU_USAGE}) } func (self *SHuaweiClient) getModelartsPoolMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { @@ -97,8 +115,246 @@ func (self *SHuaweiClient) getModelartsPoolMetrics(opts *cloudprovider.MetricLis return result, nil } +func (self *SHuaweiClient) getServerMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { + result := []cloudprovider.MetricValues{} + namespace, dimesionName, metricNames := "SYS.ECS", "instance_id", []string{ + "cpu_util", + "network_incoming_bytes_aggregate_rate", + "network_outgoing_bytes_aggregate_rate", + "disk_read_bytes_rate", + "disk_write_bytes_rate", + "disk_read_requests_rate", + "disk_write_requests_rate", + } + for _, metricName := range metricNames { + temp := make(map[string]string) + temp["namespace"] = namespace + temp["metric_name"] = metricName + temp["from"] = strconv.Itoa(int(opts.StartTime.UnixMilli())) + temp["to"] = strconv.Itoa(int(opts.EndTime.UnixMilli())) + temp["period"] = "1" + temp["filter"] = "average" + temp["dim.0"] = fmt.Sprintf("%s,%s", dimesionName, opts.ResourceId) + resp, err := self.commonMonitor(temp) + if err != nil { + log.Errorf("get monitor err:%s,input:%v", err.Error(), jsonutils.Marshal(temp)) + continue + } + metricData := MetricData{} + err = resp.Unmarshal(&metricData) + if err != nil { + return nil, errors.Wrapf(err, "resp.Unmarshal") + } + ret := cloudprovider.MetricValues{ + Id: opts.ResourceId, + Unit: metricData.Unit, + Values: []cloudprovider.MetricValue{}, + } + tags := map[string]string{} + switch metricData.MetricName { + case "cpu_util": + ret.MetricType = cloudprovider.VM_METRIC_TYPE_CPU_USAGE + case "network_incoming_bytes_aggregate_rate": + ret.MetricType = cloudprovider.VM_METRIC_TYPE_NET_BPS_RX + tags = map[string]string{"net_type": "internet"} + case "network_outgoing_bytes_aggregate_rate": + ret.MetricType = cloudprovider.VM_METRIC_TYPE_NET_BPS_TX + tags = map[string]string{"net_type": "internet"} + case "disk_read_bytes_rate": + ret.MetricType = cloudprovider.VM_METRIC_TYPE_DISK_IO_READ_BPS + case "disk_write_bytes_rate": + ret.MetricType = cloudprovider.VM_METRIC_TYPE_DISK_IO_WRITE_BPS + case "disk_read_requests_rate": + ret.MetricType = cloudprovider.VM_METRIC_TYPE_DISK_IO_READ_IOPS + case "disk_write_requests_rate": + ret.MetricType = cloudprovider.VM_METRIC_TYPE_DISK_IO_WRITE_IOPS + default: + log.Warningf("invalid metricName %s for %s %s", metricData.MetricName, opts.ResourceType, opts.ResourceId) + continue + } + for _, value := range metricData.Datapoints { + metricValue := cloudprovider.MetricValue{ + Value: value.Average, + Timestamp: time.UnixMilli(value.Timestamp), + Tags: tags, + } + ret.Values = append(ret.Values, metricValue) + } + result = append(result, ret) + } + return result, nil +} + +func (self *SHuaweiClient) getRdsMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { + result := []cloudprovider.MetricValues{} + namespace, dimesionName, metricNames := "SYS.RDS", "rds_cluster_id", []string{ + "rds001_cpu_util", + "rds002_mem_util", + "rds004_bytes_in", + "rds004_bytes_in", + "rds005_bytes_out", + "rds039_disk_util", + "rds049_disk_read_throughput", + "rds050_disk_write_throughput", + "rds006_conn_count", + "rds008_qps", + "rds009_tps", + "rds013_innodb_reads", + "rds014_innodb_writes", + } + switch opts.Engine { + case api.DBINSTANCE_TYPE_POSTGRESQL: + dimesionName = "postgresql_cluster_id" + case api.DBINSTANCE_TYPE_SQLSERVER: + dimesionName = "rds_cluster_sqlserver_id" + } + + for _, metricName := range metricNames { + temp := make(map[string]string) + temp["namespace"] = namespace + temp["metric_name"] = metricName + temp["from"] = strconv.Itoa(int(opts.StartTime.UnixMilli())) + temp["to"] = strconv.Itoa(int(opts.EndTime.UnixMilli())) + temp["period"] = "1" + temp["filter"] = "average" + temp["dim.0"] = fmt.Sprintf("%s,%s", dimesionName, opts.ResourceId) + resp, err := self.commonMonitor(temp) + if err != nil { + log.Errorf("get monitor err:%s,input:%v", err.Error(), jsonutils.Marshal(temp)) + continue + } + metricData := MetricData{} + err = resp.Unmarshal(&metricData, "metrics") + if err != nil { + return nil, errors.Wrapf(err, "resp.Unmarshal") + } + ret := cloudprovider.MetricValues{ + Id: opts.ResourceId, + Unit: metricData.Unit, + Values: []cloudprovider.MetricValue{}, + } + tags := map[string]string{} + switch metricData.MetricName { + case "rds001_cpu_util": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_CPU_USAGE + case "rds002_mem_util": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_MEM_USAGE + case "rds004_bytes_in": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_NET_BPS_RX + case "rds005_bytes_out": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_NET_BPS_TX + case "rds039_disk_util": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_DISK_USAGE + case "rds049_disk_read_throughput": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_DISK_READ_BPS + case "rds050_disk_write_throughput": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_DISK_WRITE_BPS + case "rds006_conn_count": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_CONN_COUNT + case "rds008_qps": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_QPS + case "rds009_tps": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_TPS + case "rds013_innodb_reads": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_INNODB_READ_BPS + case "rds014_innodb_writes": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_INNODB_WRITE_BPS + default: + log.Warningf("invalid metricName %s for %s %s", metricData.MetricName, opts.ResourceType, opts.ResourceId) + continue + } + for _, value := range metricData.Datapoints { + metricValue := cloudprovider.MetricValue{ + Value: value.Average, + Timestamp: time.UnixMilli(value.Timestamp), + Tags: tags, + } + ret.Values = append(ret.Values, metricValue) + } + result = append(result, ret) + } + return result, nil +} + +func (self *SHuaweiClient) getBucketMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { + result := []cloudprovider.MetricValues{} + namespace, dimesionName, metricNames := "SYS.OBS", "bucket_name", []string{ + "download_bytes", + "upload_bytes", + "first_byte_latency", + "get_request_count", + "request_count_4xx", + "request_count_5xx", + } + + for _, metricName := range metricNames { + temp := make(map[string]string) + temp["namespace"] = namespace + temp["metric_name"] = metricName + temp["from"] = strconv.Itoa(int(opts.StartTime.UnixMilli())) + temp["to"] = strconv.Itoa(int(opts.EndTime.UnixMilli())) + temp["period"] = "1" + temp["filter"] = "average" + temp["dim.0"] = fmt.Sprintf("%s,%s", dimesionName, opts.ResourceId) + resp, err := self.commonMonitor(temp) + if err != nil { + log.Errorf("get monitor err:%s,input:%v", err.Error(), jsonutils.Marshal(temp)) + continue + } + metricData := MetricData{} + err = resp.Unmarshal(&metricData, "metrics") + if err != nil { + return nil, errors.Wrapf(err, "resp.Unmarshal") + } + ret := cloudprovider.MetricValues{ + Id: opts.ResourceId, + Unit: metricData.Unit, + Values: []cloudprovider.MetricValue{}, + } + tags := map[string]string{} + switch metricData.MetricName { + case "download_bytes": + ret.MetricType = cloudprovider.BUCKET_METRIC_TYPE_NET_BPS_TX + case "upload_bytes": + ret.MetricType = cloudprovider.BUCKET_METRIC_TYPE_NET_BPS_RX + case "first_byte_latency": + ret.MetricType = cloudprovider.BUCKET_METRIC_TYPE_LATECY + tags = map[string]string{"request": "get"} + case "get_request_count": + ret.MetricType = cloudprovider.BUCKET_METRYC_TYPE_REQ_COUNT + tags = map[string]string{"request": "get"} + case "request_count_4xx": + ret.MetricType = cloudprovider.BUCKET_METRYC_TYPE_REQ_COUNT + tags = map[string]string{"request": "4xx"} + case "request_count_5xx": + ret.MetricType = cloudprovider.BUCKET_METRYC_TYPE_REQ_COUNT + tags = map[string]string{"request": "5xx"} + default: + log.Warningf("invalid metricName %s for %s %s", metricData.MetricName, opts.ResourceType, opts.ResourceId) + continue + } + for _, value := range metricData.Datapoints { + metricValue := cloudprovider.MetricValue{ + Value: value.Average, + Timestamp: time.UnixMilli(value.Timestamp), + Tags: tags, + } + ret.Values = append(ret.Values, metricValue) + } + result = append(result, ret) + + } + return result, nil +} + func (self *SHuaweiClient) GetMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { switch opts.ResourceType { + case cloudprovider.METRIC_RESOURCE_TYPE_SERVER: + return self.getServerMetrics(opts) + case cloudprovider.METRIC_RESOURCE_TYPE_RDS: + return self.getRdsMetrics(opts) + case cloudprovider.METRIC_RESOURCE_TYPE_BUCKET: + return self.getBucketMetrics(opts) case cloudprovider.METRIC_RESOURCE_TYPE_MODELARTS_POOL: return self.getModelartsPoolMetrics(opts) default: