diff --git a/pkg/apis/monitor/metric.go b/pkg/apis/monitor/metric.go index 92730d58f0..01ca767768 100644 --- a/pkg/apis/monitor/metric.go +++ b/pkg/apis/monitor/metric.go @@ -29,6 +29,7 @@ const ( METRIC_RES_TYPE_TENANT = "tenant" METRIC_RES_TYPE_DOMAIN = "domain" METRIC_RES_TYPE_STORAGE = "storage" + METRIC_RES_TYPE_ELB = "elb" //ext is prefix! METRIC_RES_TYPE_JENKINS = "ext_jenkins" METRIC_RES_TYPE_EXT_MYSQL = "ext_mysql" diff --git a/pkg/cloudmon/collectors/azuremon/azurereport.go b/pkg/cloudmon/collectors/azuremon/azurereport.go index 4843af2c3f..67e04c781f 100644 --- a/pkg/cloudmon/collectors/azuremon/azurereport.go +++ b/pkg/cloudmon/collectors/azuremon/azurereport.go @@ -18,7 +18,6 @@ import ( "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/onecloud/pkg/mcclient/modules" ) func init() { @@ -50,7 +49,7 @@ type SAzureCloudReport struct { } func (self *SAzureCloudReport) Report() error { - servers, err := self.GetAllserverOfThisProvider(&modules.Servers) + servers, err := self.GetResourceByOperator() if err != nil { return err } @@ -64,19 +63,19 @@ func (self *SAzureCloudReport) Report() error { } for _, region := range regionList { servers := regionServerMap[region.GetGlobalId()] - switch self.Operator { - case "server": - err = common.CollectRegionMetricAsync(self.Args.Batch, region, servers, self) - //err = self.collectRegionMetricOfHost(region, servers) - //case "redis": - // err = self.collectRegionMetricOfRedis(region, servers) - //case "rds": - // err = self.collectRegionMetricOfRds(region, servers) - //case "oss": - // err = self.collectRegionMetricOfOss(region, servers) - //case "elb": - // err = self.collectRegionMetricOfElb(region, servers) - } + err = common.CollectRegionMetricAsync(self.Args.Batch, region, servers, self) + //switch self.Operator { + //case "server": + // //err = self.collectRegionMetricOfHost(region, servers) + // //case "redis": + // // err = self.collectRegionMetricOfRedis(region, servers) + // //case "rds": + // // err = self.collectRegionMetricOfRds(region, servers) + // //case "oss": + // // err = self.collectRegionMetricOfOss(region, servers) + // //case "elb": + // // err = self.collectRegionMetricOfElb(region, servers) + //} if err != nil { return err } diff --git a/pkg/cloudmon/collectors/azuremon/azureservice.go b/pkg/cloudmon/collectors/azuremon/azureservice.go index 3407a043d6..1309082cb2 100644 --- a/pkg/cloudmon/collectors/azuremon/azureservice.go +++ b/pkg/cloudmon/collectors/azuremon/azureservice.go @@ -15,6 +15,7 @@ package azuremon import ( + "fmt" "strconv" "strings" @@ -22,6 +23,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" + com_api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/multicloud/azure" @@ -50,8 +52,7 @@ func (self *SAzureCloudReport) collectRegionMetricOfHost(region cloudprovider.IC continue } classicKey := "microsoft.classiccompute/virtualmachines" - ns := "Microsoft.Compute/virtualMachines" - metricSpecs := azureMetricSpecs + ns, metricSpecs := self.getMetricSpecs(server) if strings.Contains(strings.ToLower(external_id), classicKey) { ns = classicKey metricSpecs = azureClassicMetricsSpec @@ -75,11 +76,13 @@ func (self *SAzureCloudReport) collectRegionMetricOfHost(region cloudprovider.IC for _, value := range *rtnMetrics.Value { if value.Name.LocalizedValue != nil { if metricName == *(value.Name.LocalizedValue) || (value.Name.Value != nil && *value.Name.Value == metricName) { - metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict)) - if err != nil { - return errors.Wrapf(err, "fill vm %q capacity", srvPrefix) + if self.Operator == string(common.SERVER) { + metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict)) + if err != nil { + return errors.Wrapf(err, "fill vm %q capacity", srvPrefix) + } + dataList = append(dataList, metric) } - dataList = append(dataList, metric) if value.Timeseries != nil { for _, timeserie := range *value.Timeseries { serverMetric, err := self.collectMetricFromThisServer(server, timeserie, influxDbSpecs) @@ -106,7 +109,7 @@ func (self *SAzureCloudReport) collectMetricFromThisServer(server jsonutils.JSON rtnMetric azure.TimeSeriesElement, influxDbSpecs []string) ([]influxdb.SMetricData, error) { datas := make([]influxdb.SMetricData, 0) for _, data := range *rtnMetric.Data { - metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0)) + metric, err := self.NewMetricFromJson(server) if err != nil { return nil, err } @@ -153,3 +156,39 @@ func (self *SAzureCloudReport) collectMetricFromThisServer(server jsonutils.JSON return datas, nil } + +func (self *SAzureCloudReport) getMetricSpecs(res jsonutils.JSONObject) (string, map[string][]string) { + switch common.MonType(self.Operator) { + case common.SERVER: + return SERVER_METRIC_NAMESPACE, azureMetricSpecs + case common.REDIS: + return REDIS_METRIC_NAMESPACE, azureRedisMetricsSpec + case common.RDS: + return self.getRdsMetricSpecsByEngine(res) + case common.ELB: + return ELB_METRIC_NAMESPACE, azureElbMetricSpecs + default: + return SERVER_METRIC_NAMESPACE, azureMetricSpecs + } +} + +func (self *SAzureCloudReport) getRdsMetricSpecsByEngine(res jsonutils.JSONObject) (string, map[string][]string) { + engine, _ := res.GetString("engine") + externalId, _ := res.GetString("external_id") + suffix := "servers" + if strings.Contains(externalId, "flexible") { + suffix = "flexibleServers" + } + switch engine { + case com_api.DBINSTANCE_TYPE_SQLSERVER: + return "Microsoft.Sql/servers/databases", azureRdsMetricsSpec + case com_api.DBINSTANCE_TYPE_MYSQL: + return fmt.Sprintf("Microsoft.DBforMySQL/%s", suffix), azureRdsMetricsSpec + case com_api.DBINSTANCE_TYPE_POSTGRESQL: + return fmt.Sprintf("Microsoft.DBforPostgreSQL/%s", suffix), azureRdsMetricsSpec + case com_api.DBINSTANCE_TYPE_MARIADB: + return "Microsoft.DBforMariaDB/servers", azureRdsMetricsSpec + default: + return fmt.Sprintf("Microsoft.DBforMySQL/%s", suffix), azureRdsMetricsSpec + } +} diff --git a/pkg/cloudmon/collectors/azuremon/azureutils.go b/pkg/cloudmon/collectors/azuremon/azureutils.go index 527b0aeb3c..5d1ef17159 100644 --- a/pkg/cloudmon/collectors/azuremon/azureutils.go +++ b/pkg/cloudmon/collectors/azuremon/azureutils.go @@ -27,6 +27,7 @@ const ( UNIT_MEM = "byte" UNIT_MSEC = "ms" UNIT_COUNT_SEC = "count/s" + UNIT_BYTES = "byte" //ESC监控指标 INFLUXDB_FIELD_CPU_USAGE = "vm_cpu.usage_active" @@ -58,8 +59,11 @@ const ( INFLUXDB_FIELD_RDS_DISK_USAGE = "rds_disk.used_percent" INFLUXDB_FIELD_RDS_DISK_READ_BPS = "rds_diskio.read_bps" INFLUXDB_FIELD_RDS_DISK_WRITE_BPS = "rds_diskio.write_bps" + INFLUXDB_FIELD_RDS_DISK_IO_PERSENT = "rds_diskio.used_percent" INFLUXDB_FIELD_RDS_CONN_COUNT = "rds_conn.used_count" INFLUXDB_FIELD_RDS_CONN_USAGE = "rds_conn.used_percent" + INFLUXDB_FIELD_RDS_CONN_ACTIVE = "rds_conn.active_count" + INFLUXDB_FIELD_RDS_CONN_FAILED = "rds_conn.failed_count" INFLUXDB_FIELD_RDS_QPS = "rds_qps.query_qps" INFLUXDB_FIELD_RDS_TPS = "rds_tps.trans_qps" @@ -76,6 +80,8 @@ const ( INFLUXDB_FIFLD_REDIS_CACHE_KEYS = "dcs_cachekeys.key_count" INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire" INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte" + INFLUXDB_FIFLD_REDIS_SERVER_LOAD = "dcs_cpu.server_load" + INFLUXDB_FIFLD_REDIS_CONN_ERRORS = "dcs_conn.errors" //对象存储OSS监控指标 INFLUXDB_FIELD_OSS_NET_BPS_RX = "oss_netio.bps_recv" @@ -94,20 +100,22 @@ const ( INFLUXDB_FIELD_OSS_REQ_COUNT_4XX = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx" //负载均衡监控指标 - INFLUXDB_FIELD_ELB_NET_BPS_RX = "haproxy.bin" - INFLUXDB_FIELD_ELB_NET_BPS_TX = "haproxy.bout" - INFLUXDB_FIELD_ELB_REQ_RATE = "haproxy.req_rate,request=http" - INFLUXDB_FIELD_ELB_CONN_RATE = "haproxy.conn_rate,request=tcp" - INFLUXDB_FIELD_ELB_DREQ_COUNT = "haproxy.dreq,request=http" - INFLUXDB_FIELD_ELB_DCONN_COUNT = "haproxy.dcon,request=tcp" - INFLUXDB_FIELD_ELB_HRSP_COUNT = "haproxy.hrsp_Nxx" - INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx" - INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx" - INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx" - INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx" - INFLUXDB_FIELD_ELB_CHC_STATUS = "haproxy.check_status" - INFLUXDB_FIELD_ELB_CHC_CODE = "haproxy.check_code" - INFLUXDB_FIELD_ELB_LAST_CHC = "haproxy.last_chk" + INFLUXDB_FIELD_ELB_NET_BPS_RX = "haproxy.bin" + INFLUXDB_FIELD_ELB_NET_BPS_TX = "haproxy.bout" + INFLUXDB_FIELD_ELB_REQ_RATE = "haproxy.req_rate,request=http" + INFLUXDB_FIELD_ELB_CONN_RATE = "haproxy.conn_rate,request=tcp" + INFLUXDB_FIELD_ELB_DREQ_COUNT = "haproxy.dreq,request=http" + INFLUXDB_FIELD_ELB_DCONN_COUNT = "haproxy.dcon,request=tcp" + INFLUXDB_FIELD_ELB_HRSP_COUNT = "haproxy.hrsp_Nxx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx" + INFLUXDB_FIELD_ELB_CHC_STATUS = "haproxy.check_status" + INFLUXDB_FIELD_ELB_CHC_CODE = "haproxy.check_code" + INFLUXDB_FIELD_ELB_LAST_CHC = "haproxy.last_chk" + INFLUXDB_FIELD_ELB_SNAT_PORT = "haproxy.used_snat_port" + INFLUXDB_FIELD_ELB_SNAT_CONN_COUNT = "haproxy.snat_conn_count" KEY_VMS = "vms" KEY_CPUS = "cpus" @@ -133,6 +141,10 @@ const ( MetricKeyClassicNetworkOut = "Network Out" MetricKeyClassicDiskReadBPS = "Disk Read Bytes/Sec" MetricKeyClassicDiskWriteBPS = "Disk Write Bytes/Sec" + + SERVER_METRIC_NAMESPACE = "Microsoft.Compute/virtualMachines" + REDIS_METRIC_NAMESPACE = "Microsoft.Cache/redis" + ELB_METRIC_NAMESPACE = "Microsoft.Network/loadBalancers" ) var azureMetricSpecs = map[string][]string{ @@ -154,3 +166,39 @@ var azureClassicMetricsSpec = map[string][]string{ MetricKeyDiskReadOPS: {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_DISK_READ_IOPS}, MetricKeyDiskWriteOPS: {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_DISK_WRITE_IOPS}, } + +var azureRedisMetricsSpec = map[string][]string{ + "percentProcessorTime": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_CPU_USAGE}, + "usedmemorypercentage": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_MEM_USAGE}, + "connectedclients": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CONN_USAGE}, + "operationsPerSecond": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_OPT_SES}, + "alltotalkeys": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_KEYS}, + "expiredkeys": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS}, + "usedmemory": {DEFAULT_STATISTICS, UNIT_BYTES, INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE}, + "serverLoad": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIFLD_REDIS_SERVER_LOAD}, + "errors": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIFLD_REDIS_CONN_ERRORS}, +} + +//mariadb,mysql,postgresql +var azureRdsMetricsSpec = map[string][]string{ + "cpu_percent": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CPU_USAGE}, + "memory_percent": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_MEM_USAGE}, + "storage_percent": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_DISK_USAGE}, + "network_bytes_ingress": {DEFAULT_STATISTICS, UNIT_BYTES, INFLUXDB_FIELD_RDS_NET_BPS_RX}, + "network_bytes_egress": {DEFAULT_STATISTICS, UNIT_BYTES, INFLUXDB_FIELD_RDS_NET_BPS_TX}, + "io_consumption_percent": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_DISK_IO_PERSENT}, + "active_connections": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_RDS_CONN_ACTIVE}, + "connections_failed": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_RDS_CONN_FAILED}, +} + +var azureRdsMetricsSpecSqlserver = map[string][]string{ + "cpu_percent": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CPU_USAGE}, + "memory_usage_percent": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_MEM_USAGE}, + "storage_percent": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_DISK_USAGE}, + "connections_failed": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_RDS_CONN_FAILED}, +} + +var azureElbMetricSpecs = map[string][]string{ + "SnatConnectionCount": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_ELB_SNAT_PORT}, + "UsedSnatPorts": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_ELB_SNAT_CONN_COUNT}, +} diff --git a/pkg/cloudmon/collectors/common/basemon.go b/pkg/cloudmon/collectors/common/basemon.go index b375e9ba1f..4737afbcfb 100644 --- a/pkg/cloudmon/collectors/common/basemon.go +++ b/pkg/cloudmon/collectors/common/basemon.go @@ -47,6 +47,24 @@ func (self *CloudReportBase) Report() error { return fmt.Errorf("No Implment the method") } +func (self *CloudReportBase) GetResourceByOperator() ([]jsonutils.JSONObject, error) { + var servers []jsonutils.JSONObject + var err error + switch MonType(self.Operator) { + case REDIS: + servers, err = self.GetAllserverOfThisProvider(&modules.ElasticCache) + case RDS: + servers, err = self.GetAllserverOfThisProvider(&modules.DBInstance) + case OSS: + servers, err = self.GetAllserverOfThisProvider(&modules.Buckets) + case ELB: + servers, err = self.GetAllserverOfThisProvider(&modules.Loadbalancers) + default: + servers, err = self.GetAllserverOfThisProvider(&modules.Servers) + } + return servers, err +} + func (self *CloudReportBase) GetAllserverOfThisProvider(manager modulebase.Manager) ([]jsonutils.JSONObject, error) { query := jsonutils.NewDict() diff --git a/pkg/monitor/dbinit/metric_dbinit.go b/pkg/monitor/dbinit/metric_dbinit.go index 1d0c31b552..110a00a4e7 100644 --- a/pkg/monitor/dbinit/metric_dbinit.go +++ b/pkg/monitor/dbinit/metric_dbinit.go @@ -229,10 +229,18 @@ func init() { newMetricFieldCreateInput("used_percent", "Percentage of used disks", monitor.METRIC_UNIT_PERCENT, 1), }) + // rds_conn + RegistryMetricCreateInput("rds_conn", "Rds connection", monitor.METRIC_RES_TYPE_RDS, + monitor.METRIC_DATABASE_TELE, 5, []monitor.MetricFieldCreateInput{ + newMetricFieldCreateInput("active_count", "active connection count", monitor.METRIC_UNIT_COUNT, 1), + newMetricFieldCreateInput("failed_count", "failed connection count", monitor.METRIC_UNIT_COUNT, 2), + }) + // dcs_cpu RegistryMetricCreateInput("dcs_cpu", "Redis CPU usage", monitor.METRIC_RES_TYPE_REDIS, monitor.METRIC_DATABASE_TELE, 1, []monitor.MetricFieldCreateInput{ newMetricFieldCreateInput("usage_percent", "CPU active state utilization rate", monitor.METRIC_UNIT_PERCENT, 1), + newMetricFieldCreateInput("server_load", "server load", monitor.METRIC_UNIT_PERCENT, 2), }) // dcs_mem @@ -253,6 +261,7 @@ func init() { RegistryMetricCreateInput("dcs_conn", "Redis connect", monitor.METRIC_RES_TYPE_REDIS, monitor.METRIC_DATABASE_TELE, 5, []monitor.MetricFieldCreateInput{ newMetricFieldCreateInput("used_percent", "Connection usage", monitor.METRIC_UNIT_PERCENT, 1), + newMetricFieldCreateInput("errors", "Connection errors", monitor.METRIC_UNIT_COUNT, 1), }) // dcs_instantopt @@ -280,6 +289,13 @@ func init() { newMetricFieldCreateInput("balance", "balance", monitor.METRIC_UNIT_RMB, 1), }) + // haproxy + RegistryMetricCreateInput("haproxy", "load balance", monitor.METRIC_RES_TYPE_ELB, + monitor.METRIC_DATABASE_TELE, 1, []monitor.MetricFieldCreateInput{ + newMetricFieldCreateInput("used_snat_port", "used SNAT port count", monitor.METRIC_UNIT_COUNT, 1), + newMetricFieldCreateInput("snat_conn_count", "SNAT connection count", monitor.METRIC_UNIT_COUNT, 1), + }) + // agent_cpu RegistryMetricCreateInput("agent_cpu", "CPU usage", monitor.METRIC_RES_TYPE_AGENT, monitor.METRIC_DATABASE_TELE, 1, []monitor.MetricFieldCreateInput{