opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-24 16:03:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #16144 from gouqi11/addHcsoMonitor

Browse Source 
fix(cloudmon): add HCSO base monitor
This commit is contained in:
Zexi Li
2023-03-07 10:22:29 +08:00
committed by GitHub
parent 2d2ffee10c 5754ac6055
commit 50822f2a2e
6 changed files with 273 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+1 -1
View File
@@ -83,7 +83,7 @@ require (
k8s.io/client-go v0.19.3
k8s.io/cluster-bootstrap v0.19.3
moul.io/http2curl/v2 v2.3.0
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230301101220-ebd68c96818a
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230306104021-eebbddd5e493
yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656
yunion.io/x/jsonutils v1.0.1-0.20220819091305-3bab322ab4fd
yunion.io/x/log v1.0.0
go.sum
+2 -2
View File
@@ -1164,8 +1164,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230301101220-ebd68c96818a h1:gSfHCwI/E6VXEkofCj8Fw3J5Byc8Pf9Ubt6ijtNPONk=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230301101220-ebd68c96818a/go.mod h1:yEMFtIKPyIdIfPzsw7oTcH41JXc8G5StUAQw3ZoGj5I=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230306104021-eebbddd5e493 h1:5YjYZfwgBYBiwXKiitu4c2pFvFPZgm8f32IXnq1DJE8=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230306104021-eebbddd5e493/go.mod h1:sWqblYRhQCO63xeKdxdAT3wczCCY8Mc+TDa3WD/a5Kg=
yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656 h1:0zlZD5uhZoIHgLVAWCz2aHaYk2ZrNsACCYD7R6EIBII=
yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
pkg/cloudmon/providerdriver/huaweistack.go
+1 -1
View File
@@ -29,7 +29,7 @@ import (
)
type HuaweiStackCollect struct {
SBaseCollectDriver
SCollectByResourceIdDriver
}
func (self *HuaweiStackCollect) GetProvider() string {
vendor/modules.txt
Vendored
+1 -1
View File
@@ -1442,7 +1442,7 @@ sigs.k8s.io/structured-merge-diff/v4/value
# sigs.k8s.io/yaml v1.2.0
## explicit; go 1.12
sigs.k8s.io/yaml
# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230301101220-ebd68c96818a
# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230306104021-eebbddd5e493
## explicit; go 1.18
yunion.io/x/cloudmux/pkg/apis
yunion.io/x/cloudmux/pkg/apis/billing
vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/hcso.go
Generated Vendored
+10
View File
@@ -218,6 +218,16 @@ func (self *SHuaweiClient) modelartsResourceflavors(params map[string]interface{
return self.request(httputils.GET, uri, url.Values{}, params)
}
func (self *SHuaweiClient) commonMonitor(params map[string]string) (jsonutils.JSONObject, error) {
endpoint := self.resetEndpoint(self.endpoints.Ces, "ces")
uri := fmt.Sprintf("https://%s/V1.0/%s/metric-data", endpoint, self.projectId)
url := url.Values{}
for k, v := range params {
url.Set(k, v)
}
return self.request(httputils.GET, uri, url, nil)
}
func (self *SHuaweiClient) patchRequest(method httputils.THttpMethod, url string, query url.Values, params map[string]interface{}) (jsonutils.JSONObject, error) {
client := self.getAkClient()
if len(query) > 0 {
vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/monitor.go
Generated Vendored
+258 -2
View File
@@ -15,21 +15,39 @@
package hcso
import (
"fmt"
"strconv"
"time"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
api "yunion.io/x/cloudmux/pkg/apis/compute"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/cloudmux/pkg/multicloud/huawei/client/modules"
)
type MetricData struct {
Namespace string
MetricName string
Dimensions []struct {
Name string
Value string
}
Datapoints []struct {
Average float64
Timestamp int64
}
Unit string
}
func (r *SRegion) GetMetrics() ([]modules.SMetricMeta, error) {
return r.ecsClient.CloudEye.ListMetrics()
}
func (r *SRegion) GetMetricsData(metrics []modules.SMetricMeta, since time.Time, until time.Time) ([]modules.SMetricData, error) {
return r.ecsClient.CloudEye.GetMetricsData(metrics, since, until)
func (r *SRegion) GetMetricsData(metrics []modules.SMetricMeta, since time.Time, until time.Time) ([]cloudprovider.MetricValues, error) {
return r.client.getServerMetrics(&cloudprovider.MetricListOptions{ResourceType: cloudprovider.METRIC_RESOURCE_TYPE_SERVER, MetricType: cloudprovider.VM_METRIC_TYPE_CPU_USAGE})
}
func (self *SHuaweiClient) getModelartsPoolMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) {
@@ -97,8 +115,246 @@ func (self *SHuaweiClient) getModelartsPoolMetrics(opts *cloudprovider.MetricLis
return result, nil
}
func (self *SHuaweiClient) getServerMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) {
result := []cloudprovider.MetricValues{}
namespace, dimesionName, metricNames := "SYS.ECS", "instance_id", []string{
"cpu_util",
"network_incoming_bytes_aggregate_rate",
"network_outgoing_bytes_aggregate_rate",
"disk_read_bytes_rate",
"disk_write_bytes_rate",
"disk_read_requests_rate",
"disk_write_requests_rate",
}
for _, metricName := range metricNames {
temp := make(map[string]string)
temp["namespace"] = namespace
temp["metric_name"] = metricName
temp["from"] = strconv.Itoa(int(opts.StartTime.UnixMilli()))
temp["to"] = strconv.Itoa(int(opts.EndTime.UnixMilli()))
temp["period"] = "1"
temp["filter"] = "average"
temp["dim.0"] = fmt.Sprintf("%s,%s", dimesionName, opts.ResourceId)
resp, err := self.commonMonitor(temp)
if err != nil {
log.Errorf("get monitor err:%s,input:%v", err.Error(), jsonutils.Marshal(temp))
continue
}
metricData := MetricData{}
err = resp.Unmarshal(&metricData)
if err != nil {
return nil, errors.Wrapf(err, "resp.Unmarshal")
}
ret := cloudprovider.MetricValues{
Id: opts.ResourceId,
Unit: metricData.Unit,
Values: []cloudprovider.MetricValue{},
}
tags := map[string]string{}
switch metricData.MetricName {
case "cpu_util":
ret.MetricType = cloudprovider.VM_METRIC_TYPE_CPU_USAGE
case "network_incoming_bytes_aggregate_rate":
ret.MetricType = cloudprovider.VM_METRIC_TYPE_NET_BPS_RX
tags = map[string]string{"net_type": "internet"}
case "network_outgoing_bytes_aggregate_rate":
ret.MetricType = cloudprovider.VM_METRIC_TYPE_NET_BPS_TX
tags = map[string]string{"net_type": "internet"}
case "disk_read_bytes_rate":
ret.MetricType = cloudprovider.VM_METRIC_TYPE_DISK_IO_READ_BPS
case "disk_write_bytes_rate":
ret.MetricType = cloudprovider.VM_METRIC_TYPE_DISK_IO_WRITE_BPS
case "disk_read_requests_rate":
ret.MetricType = cloudprovider.VM_METRIC_TYPE_DISK_IO_READ_IOPS
case "disk_write_requests_rate":
ret.MetricType = cloudprovider.VM_METRIC_TYPE_DISK_IO_WRITE_IOPS
default:
log.Warningf("invalid metricName %s for %s %s", metricData.MetricName, opts.ResourceType, opts.ResourceId)
continue
}
for _, value := range metricData.Datapoints {
metricValue := cloudprovider.MetricValue{
Value: value.Average,
Timestamp: time.UnixMilli(value.Timestamp),
Tags: tags,
}
ret.Values = append(ret.Values, metricValue)
}
result = append(result, ret)
}
return result, nil
}
func (self *SHuaweiClient) getRdsMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) {
result := []cloudprovider.MetricValues{}
namespace, dimesionName, metricNames := "SYS.RDS", "rds_cluster_id", []string{
"rds001_cpu_util",
"rds002_mem_util",
"rds004_bytes_in",
"rds004_bytes_in",
"rds005_bytes_out",
"rds039_disk_util",
"rds049_disk_read_throughput",
"rds050_disk_write_throughput",
"rds006_conn_count",
"rds008_qps",
"rds009_tps",
"rds013_innodb_reads",
"rds014_innodb_writes",
}
switch opts.Engine {
case api.DBINSTANCE_TYPE_POSTGRESQL:
dimesionName = "postgresql_cluster_id"
case api.DBINSTANCE_TYPE_SQLSERVER:
dimesionName = "rds_cluster_sqlserver_id"
}
for _, metricName := range metricNames {
temp := make(map[string]string)
temp["namespace"] = namespace
temp["metric_name"] = metricName
temp["from"] = strconv.Itoa(int(opts.StartTime.UnixMilli()))
temp["to"] = strconv.Itoa(int(opts.EndTime.UnixMilli()))
temp["period"] = "1"
temp["filter"] = "average"
temp["dim.0"] = fmt.Sprintf("%s,%s", dimesionName, opts.ResourceId)
resp, err := self.commonMonitor(temp)
if err != nil {
log.Errorf("get monitor err:%s,input:%v", err.Error(), jsonutils.Marshal(temp))
continue
}
metricData := MetricData{}
err = resp.Unmarshal(&metricData, "metrics")
if err != nil {
return nil, errors.Wrapf(err, "resp.Unmarshal")
}
ret := cloudprovider.MetricValues{
Id: opts.ResourceId,
Unit: metricData.Unit,
Values: []cloudprovider.MetricValue{},
}
tags := map[string]string{}
switch metricData.MetricName {
case "rds001_cpu_util":
ret.MetricType = cloudprovider.RDS_METRIC_TYPE_CPU_USAGE
case "rds002_mem_util":
ret.MetricType = cloudprovider.RDS_METRIC_TYPE_MEM_USAGE
case "rds004_bytes_in":
ret.MetricType = cloudprovider.RDS_METRIC_TYPE_NET_BPS_RX
case "rds005_bytes_out":
ret.MetricType = cloudprovider.RDS_METRIC_TYPE_NET_BPS_TX
case "rds039_disk_util":
ret.MetricType = cloudprovider.RDS_METRIC_TYPE_DISK_USAGE
case "rds049_disk_read_throughput":
ret.MetricType = cloudprovider.RDS_METRIC_TYPE_DISK_READ_BPS
case "rds050_disk_write_throughput":
ret.MetricType = cloudprovider.RDS_METRIC_TYPE_DISK_WRITE_BPS
case "rds006_conn_count":
ret.MetricType = cloudprovider.RDS_METRIC_TYPE_CONN_COUNT
case "rds008_qps":
ret.MetricType = cloudprovider.RDS_METRIC_TYPE_QPS
case "rds009_tps":
ret.MetricType = cloudprovider.RDS_METRIC_TYPE_TPS
case "rds013_innodb_reads":
ret.MetricType = cloudprovider.RDS_METRIC_TYPE_INNODB_READ_BPS
case "rds014_innodb_writes":
ret.MetricType = cloudprovider.RDS_METRIC_TYPE_INNODB_WRITE_BPS
default:
log.Warningf("invalid metricName %s for %s %s", metricData.MetricName, opts.ResourceType, opts.ResourceId)
continue
}
for _, value := range metricData.Datapoints {
metricValue := cloudprovider.MetricValue{
Value: value.Average,
Timestamp: time.UnixMilli(value.Timestamp),
Tags: tags,
}
ret.Values = append(ret.Values, metricValue)
}
result = append(result, ret)
}
return result, nil
}
func (self *SHuaweiClient) getBucketMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) {
result := []cloudprovider.MetricValues{}
namespace, dimesionName, metricNames := "SYS.OBS", "bucket_name", []string{
"download_bytes",
"upload_bytes",
"first_byte_latency",
"get_request_count",
"request_count_4xx",
"request_count_5xx",
}
for _, metricName := range metricNames {
temp := make(map[string]string)
temp["namespace"] = namespace
temp["metric_name"] = metricName
temp["from"] = strconv.Itoa(int(opts.StartTime.UnixMilli()))
temp["to"] = strconv.Itoa(int(opts.EndTime.UnixMilli()))
temp["period"] = "1"
temp["filter"] = "average"
temp["dim.0"] = fmt.Sprintf("%s,%s", dimesionName, opts.ResourceId)
resp, err := self.commonMonitor(temp)
if err != nil {
log.Errorf("get monitor err:%s,input:%v", err.Error(), jsonutils.Marshal(temp))
continue
}
metricData := MetricData{}
err = resp.Unmarshal(&metricData, "metrics")
if err != nil {
return nil, errors.Wrapf(err, "resp.Unmarshal")
}
ret := cloudprovider.MetricValues{
Id: opts.ResourceId,
Unit: metricData.Unit,
Values: []cloudprovider.MetricValue{},
}
tags := map[string]string{}
switch metricData.MetricName {
case "download_bytes":
ret.MetricType = cloudprovider.BUCKET_METRIC_TYPE_NET_BPS_TX
case "upload_bytes":
ret.MetricType = cloudprovider.BUCKET_METRIC_TYPE_NET_BPS_RX
case "first_byte_latency":
ret.MetricType = cloudprovider.BUCKET_METRIC_TYPE_LATECY
tags = map[string]string{"request": "get"}
case "get_request_count":
ret.MetricType = cloudprovider.BUCKET_METRYC_TYPE_REQ_COUNT
tags = map[string]string{"request": "get"}
case "request_count_4xx":
ret.MetricType = cloudprovider.BUCKET_METRYC_TYPE_REQ_COUNT
tags = map[string]string{"request": "4xx"}
case "request_count_5xx":
ret.MetricType = cloudprovider.BUCKET_METRYC_TYPE_REQ_COUNT
tags = map[string]string{"request": "5xx"}
default:
log.Warningf("invalid metricName %s for %s %s", metricData.MetricName, opts.ResourceType, opts.ResourceId)
continue
}
for _, value := range metricData.Datapoints {
metricValue := cloudprovider.MetricValue{
Value: value.Average,
Timestamp: time.UnixMilli(value.Timestamp),
Tags: tags,
}
ret.Values = append(ret.Values, metricValue)
}
result = append(result, ret)
}
return result, nil
}
func (self *SHuaweiClient) GetMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) {
switch opts.ResourceType {
case cloudprovider.METRIC_RESOURCE_TYPE_SERVER:
return self.getServerMetrics(opts)
case cloudprovider.METRIC_RESOURCE_TYPE_RDS:
return self.getRdsMetrics(opts)
case cloudprovider.METRIC_RESOURCE_TYPE_BUCKET:
return self.getBucketMetrics(opts)
case cloudprovider.METRIC_RESOURCE_TYPE_MODELARTS_POOL:
return self.getModelartsPoolMetrics(opts)
default: