mirror of
https://github.com/yunionio/cloudpods.git
synced 2026-09-24 16:03:43 +08:00
Merge pull request #13858 from zhaoxiangchun/feature/zxc-bingo-master
feat(cloudmon): support bingo cloud metric pull
This commit is contained in:
Zexi Li
@@ -0,0 +1,34 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package bingomon
|
||||
|
||||
import "yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
|
||||
|
||||
const (
|
||||
SERVER_METRIC_NAMESPACE = "AWS/EC2"
|
||||
HOST_METRIC_NAMESPACE = "AWS/HOST"
|
||||
)
|
||||
|
||||
//multiCloud查询指标列表组装
|
||||
var bingoMetricSpecs = map[string][]string{
|
||||
"CPUUtilization": {common.DEFAULT_STATISTICS, common.UNIT_PERCENT, common.INFLUXDB_FIELD_CPU_USAGE},
|
||||
"MemeryUsage": {common.DEFAULT_STATISTICS, common.UNIT_PERCENT, common.INFLUXDB_FIELD_MEM_USAGE},
|
||||
"NetworkIn": {common.DEFAULT_STATISTICS, common.UNIT_BYTEPS, common.INFLUXDB_FIELD_NET_BPS_RX},
|
||||
"NetworkOut": {common.DEFAULT_STATISTICS, common.UNIT_BYTEPS, common.INFLUXDB_FIELD_NET_BPS_TX},
|
||||
"DiskReadBytes": {common.DEFAULT_STATISTICS, common.UNIT_BYTEPS, common.INFLUXDB_FIELD_DISK_READ_BPS},
|
||||
"DiskWriteBytes": {common.DEFAULT_STATISTICS, common.UNIT_BYTEPS, common.INFLUXDB_FIELD_DISK_WRITE_BPS},
|
||||
"DiskReadOps": {common.DEFAULT_STATISTICS, common.UNIT_CPS, common.INFLUXDB_FIELD_DISK_READ_IOPS},
|
||||
"DiskWriteOps": {common.DEFAULT_STATISTICS, common.UNIT_CPS, common.INFLUXDB_FIELD_DISK_WRITE_IOPS},
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package bingomon
|
||||
|
||||
import (
|
||||
"yunion.io/x/jsonutils"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/apis/compute"
|
||||
"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
|
||||
"yunion.io/x/onecloud/pkg/cloudmon/options"
|
||||
"yunion.io/x/onecloud/pkg/mcclient"
|
||||
)
|
||||
|
||||
func init() {
|
||||
factory := SBingoCloudReportFactory{}
|
||||
common.RegisterFactory(&factory)
|
||||
}
|
||||
|
||||
type SBingoCloudReportFactory struct {
|
||||
common.CommonReportFactory
|
||||
}
|
||||
|
||||
func (self *SBingoCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
|
||||
args *options.ReportOptions, operatorType string) common.ICloudReport {
|
||||
return &SBingoCloudReport{
|
||||
common.CloudReportBase{
|
||||
SProvider: provider,
|
||||
Session: session,
|
||||
Args: args,
|
||||
Operator: operatorType,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (self *SBingoCloudReportFactory) GetId() string {
|
||||
return compute.CLOUD_PROVIDER_BINGO_CLOUD
|
||||
}
|
||||
|
||||
type SBingoCloudReport struct {
|
||||
common.CloudReportBase
|
||||
}
|
||||
|
||||
func (self *SBingoCloudReport) Report() error {
|
||||
var servers []jsonutils.JSONObject
|
||||
var err error
|
||||
|
||||
servers, err = self.GetResourceByOperator()
|
||||
|
||||
providerInstance, err := self.InitProviderInstance()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, region := range regionList {
|
||||
servers := regionServerMap[region.GetGlobalId()]
|
||||
switch common.MonType(self.Operator) {
|
||||
case common.SERVER:
|
||||
err = self.collectRegionMetricOfResource(region, servers)
|
||||
case common.HOST:
|
||||
err = self.collectRegionMetricOfResource(region, servers)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package bingomon
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
"yunion.io/x/log"
|
||||
"yunion.io/x/pkg/errors"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
|
||||
"yunion.io/x/onecloud/pkg/cloudprovider"
|
||||
"yunion.io/x/onecloud/pkg/multicloud/bingocloud"
|
||||
"yunion.io/x/onecloud/pkg/util/influxdb"
|
||||
)
|
||||
|
||||
func (self *SBingoCloudReport) collectRegionMetricOfResource(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
|
||||
dataList := make([]influxdb.SMetricData, 0)
|
||||
bingoReg := region.(*bingocloud.SRegion)
|
||||
since, until, err := common.TimeRangeFromArgs(self.Args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
namespace, metrics := self.getMetricSpecs(servers)
|
||||
dimensionId := self.getDimensionId()
|
||||
if dimensionId == nil {
|
||||
return errors.Errorf("can not get BingoCloudReport getDimensionId")
|
||||
}
|
||||
|
||||
for metricName, influxDbSpecs := range metrics {
|
||||
for _, server := range servers {
|
||||
external_id, _ := server.GetString(dimensionId.LocalId)
|
||||
if len(external_id) == 0 {
|
||||
continue
|
||||
}
|
||||
name, _ := server.GetString("name")
|
||||
dimension := bingocloud.Dimension{
|
||||
Name: dimensionId.ExtId,
|
||||
Value: external_id,
|
||||
}
|
||||
rtnMetric, err := bingoReg.DescribeMetricList(dimension, namespace, metricName, since,
|
||||
until, "")
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
continue
|
||||
}
|
||||
if rtnMetric != nil {
|
||||
serverMetric, err := self.collectMetricFromThisServer(server, *rtnMetric, influxDbSpecs)
|
||||
if err != nil {
|
||||
log.Errorf("provider: %s,metric: %s collectMetricFromThisServer: %s, err: %#v", self.SProvider.Name,
|
||||
metricName, name, err)
|
||||
continue
|
||||
}
|
||||
dataList = append(dataList, serverMetric...)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
|
||||
}
|
||||
|
||||
func (self *SBingoCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, rtnMetric bingocloud.GetMetricStatisticsOutput,
|
||||
influxDbSpecs []string) ([]influxdb.SMetricData, error) {
|
||||
datas := make([]influxdb.SMetricData, 0)
|
||||
for _, point := range rtnMetric.Datapoints.Member {
|
||||
metric, err := self.NewMetricFromJson(server)
|
||||
//metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
metric.Timestamp = *point.Timestamp
|
||||
//根据条件拼装metric的tag和metirc信息
|
||||
influxDbSpec := influxDbSpecs[2]
|
||||
var pairsKey string
|
||||
if strings.Contains(influxDbSpec, ",") {
|
||||
pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
|
||||
} else {
|
||||
pairsKey = common.SubstringAfter(influxDbSpec, ".")
|
||||
}
|
||||
fieldValue := *point.Average
|
||||
if influxDbSpecs[1] == common.UNIT_BYTEPS && strings.Contains(pairsKey, common.UNIT_BPS) {
|
||||
fieldValue = fieldValue * 8
|
||||
}
|
||||
|
||||
measurement := self.getMeasurement(influxDbSpec)
|
||||
tag := common.SubstringAfter(influxDbSpec, ",")
|
||||
if tag != "" && strings.Contains(influxDbSpec, "=") {
|
||||
metric.Tags = append(metric.Tags, influxdb.SKeyValue{
|
||||
Key: common.SubstringBefore(tag, "="),
|
||||
Value: common.SubstringAfter(tag, "="),
|
||||
})
|
||||
}
|
||||
metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
|
||||
Key: pairsKey,
|
||||
Value: strconv.FormatFloat(fieldValue, 'E', -1, 64),
|
||||
})
|
||||
metric.Name = measurement
|
||||
datas = append(datas, metric)
|
||||
}
|
||||
|
||||
return datas, nil
|
||||
}
|
||||
|
||||
func (self *SBingoCloudReport) getMetricSpecs(res []jsonutils.JSONObject) (string, map[string][]string) {
|
||||
switch common.MonType(self.Operator) {
|
||||
case common.SERVER:
|
||||
return SERVER_METRIC_NAMESPACE, bingoMetricSpecs
|
||||
case common.HOST:
|
||||
return HOST_METRIC_NAMESPACE, bingoMetricSpecs
|
||||
}
|
||||
return "", map[string][]string{}
|
||||
}
|
||||
|
||||
func (self *SBingoCloudReport) getMeasurement(influxDbSpec string) (measurement string) {
|
||||
switch common.MonType(self.Operator) {
|
||||
case common.HOST:
|
||||
measurement = common.SubstringBetween(influxDbSpec, "vm_", ".")
|
||||
if strings.Contains(influxDbSpec, "vm_netio") {
|
||||
measurement = "net"
|
||||
}
|
||||
default:
|
||||
measurement = common.SubstringBefore(influxDbSpec, ".")
|
||||
}
|
||||
return measurement
|
||||
}
|
||||
|
||||
func (self *SBingoCloudReport) getDataPointValue(valueKey string, influxDbSpecs []string, rtnMetric jsonutils.JSONObject) (float64, error) {
|
||||
key := common.UNIT_AVERAGE
|
||||
switch common.MonType(self.Operator) {
|
||||
case common.OSS:
|
||||
key = valueKey
|
||||
case common.K8S:
|
||||
key = "Value"
|
||||
}
|
||||
fieldValue, err := rtnMetric.Float(key)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return fieldValue, nil
|
||||
}
|
||||
|
||||
func (self *SBingoCloudReport) getDimensionId() *common.DimensionId {
|
||||
switch common.MonType(self.Operator) {
|
||||
case common.SERVER:
|
||||
return &common.DimensionId{
|
||||
LocalId: "external_id",
|
||||
ExtId: "InstanceId",
|
||||
}
|
||||
case common.HOST:
|
||||
return &common.DimensionId{
|
||||
LocalId: "access_ip",
|
||||
ExtId: "HostId",
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
package bingomon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/bingomon"
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/apsaramon"
|
||||
_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/awsmon"
|
||||
_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/azuremon"
|
||||
_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/bingomon"
|
||||
_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/cloudaccountmon"
|
||||
_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/gcpmon"
|
||||
_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/huaweimon"
|
||||
@@ -32,6 +33,7 @@ import (
|
||||
_ "yunion.io/x/onecloud/pkg/multicloud/apsara/provider"
|
||||
_ "yunion.io/x/onecloud/pkg/multicloud/aws/provider"
|
||||
_ "yunion.io/x/onecloud/pkg/multicloud/azure/provider"
|
||||
_ "yunion.io/x/onecloud/pkg/multicloud/bingocloud/provider"
|
||||
_ "yunion.io/x/onecloud/pkg/multicloud/esxi/provider"
|
||||
_ "yunion.io/x/onecloud/pkg/multicloud/google/provider"
|
||||
_ "yunion.io/x/onecloud/pkg/multicloud/hcso/provider"
|
||||
|
||||
@@ -78,7 +78,7 @@ var (
|
||||
SupportMetricBrands = []string{compute.CLOUD_PROVIDER_ALIYUN, compute.CLOUD_PROVIDER_VMWARE, compute.CLOUD_PROVIDER_APSARA,
|
||||
compute.CLOUD_PROVIDER_QCLOUD, compute.CLOUD_PROVIDER_AZURE, compute.CLOUD_PROVIDER_AWS,
|
||||
compute.CLOUD_PROVIDER_HUAWEI, compute.CLOUD_PROVIDER_HCSO, compute.CLOUD_PROVIDER_ZSTACK,
|
||||
compute.CLOUD_PROVIDER_GOOGLE, compute.CLOUD_PROVIDER_ECLOUD, compute.CLOUD_PROVIDER_JDCLOUD}
|
||||
compute.CLOUD_PROVIDER_GOOGLE, compute.CLOUD_PROVIDER_ECLOUD, compute.CLOUD_PROVIDER_JDCLOUD, compute.CLOUD_PROVIDER_BINGO_CLOUD}
|
||||
|
||||
ResMonTypeList = []string{string(SERVER), string(HOST), string(REDIS), string(RDS), string(OSS),
|
||||
string(ELB), string(K8S)}
|
||||
|
||||
@@ -50,12 +50,13 @@ type ReportOptions struct {
|
||||
CloudproviderSyncInterval int64 `help:"CloudproviderSyncInterval unit:minute" default:"30"`
|
||||
AlertRecordHistoryInterval int64 `help:"AlertRecordHistoryInterval unit:day" default:"1"`
|
||||
// 定时执行间隔，同时也会影响metric拉取间隔
|
||||
Interval string `help:"interval" default:"6" unit:"minute"`
|
||||
Timeout int64 `help:"command timeout unit:second" default:"10"`
|
||||
SinceTime string `help:"sinceTime"`
|
||||
EndTime string `help:"endTime"`
|
||||
Provider []string `help:"List objects from the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei|ZStack|Google|Apsara|JDcloud|Ecloud|HCSO" json:"provider,omitempty"`
|
||||
MetricInterval string `help:"metric interval eg:PT1M"`
|
||||
Interval string `help:"interval" default:"6" unit:"minute"`
|
||||
Timeout int64 `help:"command timeout unit:second" default:"10"`
|
||||
SinceTime string `help:"sinceTime"`
|
||||
EndTime string `help:"endTime"`
|
||||
Provider []string `help:"List objects from the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei
|
||||
|ZStack|Google|Apsara|JDcloud|Ecloud|HCSO|BingoCloud" json:"provider,omitempty"`
|
||||
MetricInterval string `help:"metric interval eg:PT1M"`
|
||||
PingProbeOptions
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package bingocloud
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/cloudwatch"
|
||||
|
||||
"yunion.io/x/pkg/errors"
|
||||
)
|
||||
|
||||
type Dimension struct {
|
||||
Name string
|
||||
Value string
|
||||
}
|
||||
|
||||
func (self *SRegion) DescribeMetricList(dimension Dimension, ns string, metricNm string, since time.Time,
|
||||
until time.Time,
|
||||
nextToken string,
|
||||
) (*GetMetricStatisticsOutput, error) {
|
||||
params := map[string]string{}
|
||||
if len(ns) > 0 {
|
||||
params["Namespace"] = ns
|
||||
}
|
||||
if len(metricNm) > 0 {
|
||||
params["MetricName"] = metricNm
|
||||
}
|
||||
idx := 1
|
||||
if len(dimension.Value) > 0 {
|
||||
params[fmt.Sprintf("Dimensions.member.%d.Name", idx)] = dimension.Name
|
||||
params[fmt.Sprintf("Dimensions.member.%d.Value", idx)] = dimension.Value
|
||||
idx++
|
||||
}
|
||||
if !since.IsZero() {
|
||||
params["StartTime"] = since.Format(time.RFC3339)
|
||||
}
|
||||
if !until.IsZero() {
|
||||
params["EndTime"] = until.Format(time.RFC3339)
|
||||
}
|
||||
params["Statistics.member.1"] = "Average"
|
||||
params["Period"] = "60"
|
||||
jsonObject, err := self.invoke("GetMetricStatistics", params)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "GetMetricStatistics err")
|
||||
}
|
||||
if jsonObject == nil {
|
||||
return nil, errors.Errorf("GetMetricStatistics return jsonObject is nil")
|
||||
}
|
||||
rtn, _ := jsonObject.Get("GetMetricStatisticsResult")
|
||||
output := new(GetMetricStatisticsOutput)
|
||||
err = rtn.Unmarshal(output)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Unmarshal GetMetricStatisticsOutput err")
|
||||
}
|
||||
return output, nil
|
||||
}
|
||||
|
||||
type GetMetricStatisticsOutput struct {
|
||||
Datapoints GetMetricStatisticsDatapoints
|
||||
ObjName string
|
||||
Period int64
|
||||
}
|
||||
type GetMetricStatisticsDatapoints struct {
|
||||
Member []cloudwatch.Datapoint
|
||||
}
|
||||
Reference in New Issue
Block a user