diff --git a/pkg/cloudmon/collectors/bingomon/bingoUtils.go b/pkg/cloudmon/collectors/bingomon/bingoUtils.go new file mode 100644 index 0000000000..c5beb95d7f --- /dev/null +++ b/pkg/cloudmon/collectors/bingomon/bingoUtils.go @@ -0,0 +1,34 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bingomon + +import "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + +const ( + SERVER_METRIC_NAMESPACE = "AWS/EC2" + HOST_METRIC_NAMESPACE = "AWS/HOST" +) + +//multiCloud查询指标列表组装 +var bingoMetricSpecs = map[string][]string{ + "CPUUtilization": {common.DEFAULT_STATISTICS, common.UNIT_PERCENT, common.INFLUXDB_FIELD_CPU_USAGE}, + "MemeryUsage": {common.DEFAULT_STATISTICS, common.UNIT_PERCENT, common.INFLUXDB_FIELD_MEM_USAGE}, + "NetworkIn": {common.DEFAULT_STATISTICS, common.UNIT_BYTEPS, common.INFLUXDB_FIELD_NET_BPS_RX}, + "NetworkOut": {common.DEFAULT_STATISTICS, common.UNIT_BYTEPS, common.INFLUXDB_FIELD_NET_BPS_TX}, + "DiskReadBytes": {common.DEFAULT_STATISTICS, common.UNIT_BYTEPS, common.INFLUXDB_FIELD_DISK_READ_BPS}, + "DiskWriteBytes": {common.DEFAULT_STATISTICS, common.UNIT_BYTEPS, common.INFLUXDB_FIELD_DISK_WRITE_BPS}, + "DiskReadOps": {common.DEFAULT_STATISTICS, common.UNIT_CPS, common.INFLUXDB_FIELD_DISK_READ_IOPS}, + "DiskWriteOps": {common.DEFAULT_STATISTICS, common.UNIT_CPS, common.INFLUXDB_FIELD_DISK_WRITE_IOPS}, +} diff --git a/pkg/cloudmon/collectors/bingomon/bingocloudreport.go b/pkg/cloudmon/collectors/bingomon/bingocloudreport.go new file mode 100644 index 0000000000..febdd6a208 --- /dev/null +++ b/pkg/cloudmon/collectors/bingomon/bingocloudreport.go @@ -0,0 +1,82 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bingomon + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/cloudmon/options" + "yunion.io/x/onecloud/pkg/mcclient" +) + +func init() { + factory := SBingoCloudReportFactory{} + common.RegisterFactory(&factory) +} + +type SBingoCloudReportFactory struct { + common.CommonReportFactory +} + +func (self *SBingoCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession, + args *options.ReportOptions, operatorType string) common.ICloudReport { + return &SBingoCloudReport{ + common.CloudReportBase{ + SProvider: provider, + Session: session, + Args: args, + Operator: operatorType, + }, + } +} + +func (self *SBingoCloudReportFactory) GetId() string { + return compute.CLOUD_PROVIDER_BINGO_CLOUD +} + +type SBingoCloudReport struct { + common.CloudReportBase +} + +func (self *SBingoCloudReport) Report() error { + var servers []jsonutils.JSONObject + var err error + + servers, err = self.GetResourceByOperator() + + providerInstance, err := self.InitProviderInstance() + if err != nil { + return err + } + regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance) + if err != nil { + return err + } + for _, region := range regionList { + servers := regionServerMap[region.GetGlobalId()] + switch common.MonType(self.Operator) { + case common.SERVER: + err = self.collectRegionMetricOfResource(region, servers) + case common.HOST: + err = self.collectRegionMetricOfResource(region, servers) + } + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/cloudmon/collectors/bingomon/bingoservice.go b/pkg/cloudmon/collectors/bingomon/bingoservice.go new file mode 100644 index 0000000000..8f39c86ed3 --- /dev/null +++ b/pkg/cloudmon/collectors/bingomon/bingoservice.go @@ -0,0 +1,171 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bingomon + +import ( + "strconv" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/bingocloud" + "yunion.io/x/onecloud/pkg/util/influxdb" +) + +func (self *SBingoCloudReport) collectRegionMetricOfResource(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + bingoReg := region.(*bingocloud.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + namespace, metrics := self.getMetricSpecs(servers) + dimensionId := self.getDimensionId() + if dimensionId == nil { + return errors.Errorf("can not get BingoCloudReport getDimensionId") + } + + for metricName, influxDbSpecs := range metrics { + for _, server := range servers { + external_id, _ := server.GetString(dimensionId.LocalId) + if len(external_id) == 0 { + continue + } + name, _ := server.GetString("name") + dimension := bingocloud.Dimension{ + Name: dimensionId.ExtId, + Value: external_id, + } + rtnMetric, err := bingoReg.DescribeMetricList(dimension, namespace, metricName, since, + until, "") + if err != nil { + log.Errorln(err) + continue + } + if rtnMetric != nil { + serverMetric, err := self.collectMetricFromThisServer(server, *rtnMetric, influxDbSpecs) + if err != nil { + log.Errorf("provider: %s,metric: %s collectMetricFromThisServer: %s, err: %#v", self.SProvider.Name, + metricName, name, err) + continue + } + dataList = append(dataList, serverMetric...) + } + } + + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SBingoCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, rtnMetric bingocloud.GetMetricStatisticsOutput, + influxDbSpecs []string) ([]influxdb.SMetricData, error) { + datas := make([]influxdb.SMetricData, 0) + for _, point := range rtnMetric.Datapoints.Member { + metric, err := self.NewMetricFromJson(server) + //metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0)) + if err != nil { + return nil, err + } + metric.Timestamp = *point.Timestamp + //根据条件拼装metric的tag和metirc信息 + influxDbSpec := influxDbSpecs[2] + var pairsKey string + if strings.Contains(influxDbSpec, ",") { + pairsKey = common.SubstringBetween(influxDbSpec, ".", ",") + } else { + pairsKey = common.SubstringAfter(influxDbSpec, ".") + } + fieldValue := *point.Average + if influxDbSpecs[1] == common.UNIT_BYTEPS && strings.Contains(pairsKey, common.UNIT_BPS) { + fieldValue = fieldValue * 8 + } + + measurement := self.getMeasurement(influxDbSpec) + tag := common.SubstringAfter(influxDbSpec, ",") + if tag != "" && strings.Contains(influxDbSpec, "=") { + metric.Tags = append(metric.Tags, influxdb.SKeyValue{ + Key: common.SubstringBefore(tag, "="), + Value: common.SubstringAfter(tag, "="), + }) + } + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: pairsKey, + Value: strconv.FormatFloat(fieldValue, 'E', -1, 64), + }) + metric.Name = measurement + datas = append(datas, metric) + } + + return datas, nil +} + +func (self *SBingoCloudReport) getMetricSpecs(res []jsonutils.JSONObject) (string, map[string][]string) { + switch common.MonType(self.Operator) { + case common.SERVER: + return SERVER_METRIC_NAMESPACE, bingoMetricSpecs + case common.HOST: + return HOST_METRIC_NAMESPACE, bingoMetricSpecs + } + return "", map[string][]string{} +} + +func (self *SBingoCloudReport) getMeasurement(influxDbSpec string) (measurement string) { + switch common.MonType(self.Operator) { + case common.HOST: + measurement = common.SubstringBetween(influxDbSpec, "vm_", ".") + if strings.Contains(influxDbSpec, "vm_netio") { + measurement = "net" + } + default: + measurement = common.SubstringBefore(influxDbSpec, ".") + } + return measurement +} + +func (self *SBingoCloudReport) getDataPointValue(valueKey string, influxDbSpecs []string, rtnMetric jsonutils.JSONObject) (float64, error) { + key := common.UNIT_AVERAGE + switch common.MonType(self.Operator) { + case common.OSS: + key = valueKey + case common.K8S: + key = "Value" + } + fieldValue, err := rtnMetric.Float(key) + if err != nil { + return 0, err + } + return fieldValue, nil +} + +func (self *SBingoCloudReport) getDimensionId() *common.DimensionId { + switch common.MonType(self.Operator) { + case common.SERVER: + return &common.DimensionId{ + LocalId: "external_id", + ExtId: "InstanceId", + } + case common.HOST: + return &common.DimensionId{ + LocalId: "access_ip", + ExtId: "HostId", + } + + } + return nil +} diff --git a/pkg/cloudmon/collectors/bingomon/doc.go b/pkg/cloudmon/collectors/bingomon/doc.go new file mode 100644 index 0000000000..8b008f1515 --- /dev/null +++ b/pkg/cloudmon/collectors/bingomon/doc.go @@ -0,0 +1 @@ +package bingomon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/bingomon" diff --git a/pkg/cloudmon/collectors/cloudmoninit.go b/pkg/cloudmon/collectors/cloudmoninit.go index 256538983d..1dafc995f1 100644 --- a/pkg/cloudmon/collectors/cloudmoninit.go +++ b/pkg/cloudmon/collectors/cloudmoninit.go @@ -20,6 +20,7 @@ import ( _ "yunion.io/x/onecloud/pkg/cloudmon/collectors/apsaramon" _ "yunion.io/x/onecloud/pkg/cloudmon/collectors/awsmon" _ "yunion.io/x/onecloud/pkg/cloudmon/collectors/azuremon" + _ "yunion.io/x/onecloud/pkg/cloudmon/collectors/bingomon" _ "yunion.io/x/onecloud/pkg/cloudmon/collectors/cloudaccountmon" _ "yunion.io/x/onecloud/pkg/cloudmon/collectors/gcpmon" _ "yunion.io/x/onecloud/pkg/cloudmon/collectors/huaweimon" @@ -32,6 +33,7 @@ import ( _ "yunion.io/x/onecloud/pkg/multicloud/apsara/provider" _ "yunion.io/x/onecloud/pkg/multicloud/aws/provider" _ "yunion.io/x/onecloud/pkg/multicloud/azure/provider" + _ "yunion.io/x/onecloud/pkg/multicloud/bingocloud/provider" _ "yunion.io/x/onecloud/pkg/multicloud/esxi/provider" _ "yunion.io/x/onecloud/pkg/multicloud/google/provider" _ "yunion.io/x/onecloud/pkg/multicloud/hcso/provider" diff --git a/pkg/cloudmon/collectors/common/multiCloudMonUtils.go b/pkg/cloudmon/collectors/common/multiCloudMonUtils.go index 87fb988e2e..8b9baae0c6 100644 --- a/pkg/cloudmon/collectors/common/multiCloudMonUtils.go +++ b/pkg/cloudmon/collectors/common/multiCloudMonUtils.go @@ -78,7 +78,7 @@ var ( SupportMetricBrands = []string{compute.CLOUD_PROVIDER_ALIYUN, compute.CLOUD_PROVIDER_VMWARE, compute.CLOUD_PROVIDER_APSARA, compute.CLOUD_PROVIDER_QCLOUD, compute.CLOUD_PROVIDER_AZURE, compute.CLOUD_PROVIDER_AWS, compute.CLOUD_PROVIDER_HUAWEI, compute.CLOUD_PROVIDER_HCSO, compute.CLOUD_PROVIDER_ZSTACK, - compute.CLOUD_PROVIDER_GOOGLE, compute.CLOUD_PROVIDER_ECLOUD, compute.CLOUD_PROVIDER_JDCLOUD} + compute.CLOUD_PROVIDER_GOOGLE, compute.CLOUD_PROVIDER_ECLOUD, compute.CLOUD_PROVIDER_JDCLOUD, compute.CLOUD_PROVIDER_BINGO_CLOUD} ResMonTypeList = []string{string(SERVER), string(HOST), string(REDIS), string(RDS), string(OSS), string(ELB), string(K8S)} diff --git a/pkg/cloudmon/options/options.go b/pkg/cloudmon/options/options.go index 4f95065591..545c2cd7d7 100644 --- a/pkg/cloudmon/options/options.go +++ b/pkg/cloudmon/options/options.go @@ -50,12 +50,13 @@ type ReportOptions struct { CloudproviderSyncInterval int64 `help:"CloudproviderSyncInterval unit:minute" default:"30"` AlertRecordHistoryInterval int64 `help:"AlertRecordHistoryInterval unit:day" default:"1"` // 定时执行间隔,同时也会影响metric拉取间隔 - Interval string `help:"interval" default:"6" unit:"minute"` - Timeout int64 `help:"command timeout unit:second" default:"10"` - SinceTime string `help:"sinceTime"` - EndTime string `help:"endTime"` - Provider []string `help:"List objects from the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei|ZStack|Google|Apsara|JDcloud|Ecloud|HCSO" json:"provider,omitempty"` - MetricInterval string `help:"metric interval eg:PT1M"` + Interval string `help:"interval" default:"6" unit:"minute"` + Timeout int64 `help:"command timeout unit:second" default:"10"` + SinceTime string `help:"sinceTime"` + EndTime string `help:"endTime"` + Provider []string `help:"List objects from the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei +|ZStack|Google|Apsara|JDcloud|Ecloud|HCSO|BingoCloud" json:"provider,omitempty"` + MetricInterval string `help:"metric interval eg:PT1M"` PingProbeOptions } diff --git a/pkg/multicloud/bingocloud/monitor.go b/pkg/multicloud/bingocloud/monitor.go new file mode 100644 index 0000000000..cdd44d4995 --- /dev/null +++ b/pkg/multicloud/bingocloud/monitor.go @@ -0,0 +1,65 @@ +package bingocloud + +import ( + "fmt" + "time" + + "github.com/aws/aws-sdk-go/service/cloudwatch" + + "yunion.io/x/pkg/errors" +) + +type Dimension struct { + Name string + Value string +} + +func (self *SRegion) DescribeMetricList(dimension Dimension, ns string, metricNm string, since time.Time, + until time.Time, + nextToken string, +) (*GetMetricStatisticsOutput, error) { + params := map[string]string{} + if len(ns) > 0 { + params["Namespace"] = ns + } + if len(metricNm) > 0 { + params["MetricName"] = metricNm + } + idx := 1 + if len(dimension.Value) > 0 { + params[fmt.Sprintf("Dimensions.member.%d.Name", idx)] = dimension.Name + params[fmt.Sprintf("Dimensions.member.%d.Value", idx)] = dimension.Value + idx++ + } + if !since.IsZero() { + params["StartTime"] = since.Format(time.RFC3339) + } + if !until.IsZero() { + params["EndTime"] = until.Format(time.RFC3339) + } + params["Statistics.member.1"] = "Average" + params["Period"] = "60" + jsonObject, err := self.invoke("GetMetricStatistics", params) + if err != nil { + return nil, errors.Wrap(err, "GetMetricStatistics err") + } + if jsonObject == nil { + return nil, errors.Errorf("GetMetricStatistics return jsonObject is nil") + } + rtn, _ := jsonObject.Get("GetMetricStatisticsResult") + output := new(GetMetricStatisticsOutput) + err = rtn.Unmarshal(output) + if err != nil { + return nil, errors.Wrap(err, "Unmarshal GetMetricStatisticsOutput err") + } + return output, nil +} + +type GetMetricStatisticsOutput struct { + Datapoints GetMetricStatisticsDatapoints + ObjName string + Period int64 +} +type GetMetricStatisticsDatapoints struct { + Member []cloudwatch.Datapoint +}