From a024d94bcba386b8779bc8730fc060cbc83038b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Wed, 12 Mar 2025 16:40:12 +0800 Subject: [PATCH] fix(monitor): add service metric monitor (#22226) --- pkg/apis/monitor/metric.go | 9 ++- pkg/cloudmon/misc/system.go | 61 ++++++++++++++++--- pkg/monitor/dbinit/measurements/all.go | 5 ++ pkg/monitor/dbinit/measurements/db_state.go | 55 +++++++++++++++++ pkg/monitor/dbinit/measurements/service.go | 40 ++++++++++++ .../dbinit/measurements/service_http_code.go | 46 ++++++++++++++ .../measurements/service_process_stats.go | 40 ++++++++++++ 7 files changed, 244 insertions(+), 12 deletions(-) create mode 100644 pkg/monitor/dbinit/measurements/db_state.go create mode 100644 pkg/monitor/dbinit/measurements/service.go create mode 100644 pkg/monitor/dbinit/measurements/service_http_code.go create mode 100644 pkg/monitor/dbinit/measurements/service_process_stats.go diff --git a/pkg/apis/monitor/metric.go b/pkg/apis/monitor/metric.go index e0e761c548..d5ebdc61c5 100644 --- a/pkg/apis/monitor/metric.go +++ b/pkg/apis/monitor/metric.go @@ -32,6 +32,10 @@ const ( METRIC_RES_TYPE_ELB = "elb" METRIC_RES_TYPE_K8S = "k8s" METRIC_RES_TYPE_CONTAINER = "container" + METRIC_RES_TYPE_WORKER = "worker" + METRIC_RES_TYPE_HTTP_REQUEST = "http_request" + METRIC_RES_TYPE_DB_STATS = "db_stats" + METRIC_RES_TYPE_PROCESS = "process" //ext is prefix！ METRIC_RES_TYPE_JENKINS = "ext_jenkins" @@ -52,8 +56,9 @@ const ( METRIC_UNIT_MB = "Mb" METRIC_UNIT_NULL = "NULL" - METRIC_DATABASE_TELE = "telegraf" - METRIC_DATABASE_METER = "meter_db" + METRIC_DATABASE_TELE = "telegraf" + METRIC_DATABASE_METER = "meter_db" + METRIC_DATABASE_SYSTEM = "system" ) var ( diff --git a/pkg/cloudmon/misc/system.go b/pkg/cloudmon/misc/system.go index ca7380c598..b51aef96f5 100644 --- a/pkg/cloudmon/misc/system.go +++ b/pkg/cloudmon/misc/system.go @@ -26,14 +26,17 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/util/httputils" + "yunion.io/x/pkg/util/version" "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/apis" + compute_api "yunion.io/x/onecloud/pkg/apis/compute" api "yunion.io/x/onecloud/pkg/apis/identity" "yunion.io/x/onecloud/pkg/cloudcommon/tsdb" "yunion.io/x/onecloud/pkg/cloudmon/options" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules/compute" "yunion.io/x/onecloud/pkg/mcclient/modules/identity" "yunion.io/x/onecloud/pkg/util/influxdb" ) @@ -67,7 +70,13 @@ func CollectServiceMetrics(ctx context.Context, userCred mcclient.TokenCredentia return } s := auth.GetAdminSession(ctx, options.Options.CommonOptions.Region) - err := func() error { + urls, err := tsdb.GetDefaultServiceSourceURLs(s, options.Options.SessionEndpointType) + if err != nil { + return + } + + tk := auth.AdminCredential().GetTokenString() + err = func() error { endpoints, err := getEndpoints(ctx, s) if err != nil { return errors.Wrapf(err, "getEndpoints") @@ -83,7 +92,6 @@ func CollectServiceMetrics(ctx context.Context, userCred mcclient.TokenCredentia continue } url := httputils.JoinPath(ep.Url, "version") - tk := auth.AdminCredential().GetTokenString() hdr := http.Header{} hdr.Set("X-Auth-Token", tk) resp, err := httputils.Request( @@ -106,15 +114,48 @@ func CollectServiceMetrics(ctx context.Context, userCred mcclient.TokenCredentia } metrics = append(metrics, part...) } - urls, err := tsdb.GetDefaultServiceSourceURLs(s, options.Options.SessionEndpointType) - if err != nil { - return errors.Wrap(err, "GetServiceURLs") - } return influxdb.SendMetrics(urls, SYSTEM_METRIC_DATABASE, metrics, false) }() if err != nil { log.Errorf("collect service metric error: %v", err) } + params := compute_api.HostListInput{} + limit := 20 + params.Limit = &limit + params.Brand = []string{compute_api.CLOUD_PROVIDER_ONECLOUD} + params.Scope = "system" + params.Status = []string{compute_api.HOST_STATUS_RUNNING} + details := false + params.Details = &details + hosts := []compute_api.HostDetails{} + for { + offset := len(hosts) + params.Offset = &offset + resp, err := compute.Hosts.List(s, jsonutils.Marshal(params)) + if err != nil { + return + } + part := []compute_api.HostDetails{} + err = jsonutils.Update(&part, resp.Data) + if err != nil { + return + } + hosts = append(hosts, part...) + if len(hosts) >= resp.Total { + break + } + } + metrics := []influxdb.SMetricData{} + for _, host := range hosts { + service := fmt.Sprintf("host-%s", host.Name) + part, err := collectWorkerMetrics(ctx, host.ManagerUri, service, version.GetShortString(), tk) + if err != nil { + log.Errorf("collect host %s metric error: %v", service, err) + continue + } + metrics = append(metrics, part...) + } + influxdb.SendMetrics(urls, SYSTEM_METRIC_DATABASE, metrics, false) } func collectStatsMetrics(ctx context.Context, ep api.EndpointDetails, version, token string) ([]influxdb.SMetricData, error) { @@ -257,8 +298,8 @@ func collectStatsMetrics(ctx context.Context, ep api.EndpointDetails, version, t return result, nil } -func collectWorkerMetrics(ctx context.Context, ep api.EndpointDetails, version, token string) ([]influxdb.SMetricData, error) { - statsUrl := httputils.JoinPath(ep.Url, "worker_stats") +func collectWorkerMetrics(ctx context.Context, url, service, version, token string) ([]influxdb.SMetricData, error) { + statsUrl := httputils.JoinPath(url, "worker_stats") hdr := http.Header{} hdr.Set("X-Auth-Token", token) _, ret, err := httputils.JSONRequest( @@ -301,7 +342,7 @@ func collectWorkerMetrics(ctx context.Context, ep api.EndpointDetails, version, }, { Key: "service", - Value: ep.ServiceName, + Value: service, }, { Key: "worker_name", @@ -484,7 +525,7 @@ func collectServiceMetrics(ctx context.Context, ep api.EndpointDetails, version, errs = append(errs, err) } ret = append(ret, stats...) - worker, err := collectWorkerMetrics(ctx, ep, version, token) + worker, err := collectWorkerMetrics(ctx, ep.Url, ep.ServiceName, version, token) if err != nil { errs = append(errs, err) } diff --git a/pkg/monitor/dbinit/measurements/all.go b/pkg/monitor/dbinit/measurements/all.go index e4e7ceac4a..66d73a5c67 100644 --- a/pkg/monitor/dbinit/measurements/all.go +++ b/pkg/monitor/dbinit/measurements/all.go @@ -31,6 +31,11 @@ var All = []SMeasurement{ system, vasmi, + worker, + serviceHttpCode, + serviceProcessStats, + dbStats, + vmCpu, vmMem, vmDisk, diff --git a/pkg/monitor/dbinit/measurements/db_state.go b/pkg/monitor/dbinit/measurements/db_state.go new file mode 100644 index 0000000000..1e07f2c2ff --- /dev/null +++ b/pkg/monitor/dbinit/measurements/db_state.go @@ -0,0 +1,55 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package measurements + +import "yunion.io/x/onecloud/pkg/apis/monitor" + +var dbStats = SMeasurement{ + Context: []SMonitorContext{ + { + "db_stats", "Database Stats", + monitor.METRIC_RES_TYPE_DB_STATS, monitor.METRIC_DATABASE_SYSTEM, + }, + }, + Metrics: []SMetric{ + { + "idle", "Database Idle", monitor.METRIC_UNIT_NULL, + }, + { + "in_use", "Database InUse", monitor.METRIC_UNIT_NULL, + }, + { + "max_idle_closed", "Database max idle closed", monitor.METRIC_UNIT_NULL, + }, + { + "max_idle_time_closed", "Database max idle time closed", monitor.METRIC_UNIT_NULL, + }, + { + "max_lifetime_closed", "Database max lifetime closed", monitor.METRIC_UNIT_NULL, + }, + { + "max_open_connections", "Database max open connections", monitor.METRIC_UNIT_NULL, + }, + { + "open_connections", "Database open connections", monitor.METRIC_UNIT_NULL, + }, + { + "wait_count", "Database wait count", monitor.METRIC_UNIT_NULL, + }, + { + "wait_duration", "Database wait duration", monitor.METRIC_UNIT_NULL, + }, + }, +} diff --git a/pkg/monitor/dbinit/measurements/service.go b/pkg/monitor/dbinit/measurements/service.go new file mode 100644 index 0000000000..c92736d031 --- /dev/null +++ b/pkg/monitor/dbinit/measurements/service.go @@ -0,0 +1,40 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package measurements + +import "yunion.io/x/onecloud/pkg/apis/monitor" + +var worker = SMeasurement{ + Context: []SMonitorContext{ + { + "worker", "Worker queue", + monitor.METRIC_RES_TYPE_WORKER, monitor.METRIC_DATABASE_SYSTEM, + }, + }, + Metrics: []SMetric{ + { + "active_worker_cnt", "Active Worker Count", monitor.METRIC_UNIT_NULL, + }, + { + "max_worker_count", "Max Worker Count", monitor.METRIC_UNIT_NULL, + }, + { + "detach_worker_cnt", "Detach worker Count", monitor.METRIC_UNIT_NULL, + }, + { + "queue_cnt", "Worker Queue Count", monitor.METRIC_UNIT_NULL, + }, + }, +} diff --git a/pkg/monitor/dbinit/measurements/service_http_code.go b/pkg/monitor/dbinit/measurements/service_http_code.go new file mode 100644 index 0000000000..591d95d79c --- /dev/null +++ b/pkg/monitor/dbinit/measurements/service_http_code.go @@ -0,0 +1,46 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package measurements + +import "yunion.io/x/onecloud/pkg/apis/monitor" + +var serviceHttpCode = SMeasurement{ + Context: []SMonitorContext{ + { + "http_request", "HTTP Request hit", + monitor.METRIC_RES_TYPE_HTTP_REQUEST, monitor.METRIC_DATABASE_SYSTEM, + }, + }, + Metrics: []SMetric{ + { + "duration.2xx", "http code 2xxx duration", monitor.METRIC_UNIT_NULL, + }, + { + "duration.4xx", "http code 4xxx duration", monitor.METRIC_UNIT_NULL, + }, + { + "duration.5xx", "http code 5xxx duration", monitor.METRIC_UNIT_NULL, + }, + { + "hit.2xx", "http code 2xxx hit", monitor.METRIC_UNIT_NULL, + }, + { + "hit.4xx", "http code 4xxx hit", monitor.METRIC_UNIT_NULL, + }, + { + "hit.5xx", "http code 5xxx hit", monitor.METRIC_UNIT_NULL, + }, + }, +} diff --git a/pkg/monitor/dbinit/measurements/service_process_stats.go b/pkg/monitor/dbinit/measurements/service_process_stats.go new file mode 100644 index 0000000000..28a862c731 --- /dev/null +++ b/pkg/monitor/dbinit/measurements/service_process_stats.go @@ -0,0 +1,40 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package measurements + +import "yunion.io/x/onecloud/pkg/apis/monitor" + +var serviceProcessStats = SMeasurement{ + Context: []SMonitorContext{ + { + "process", "Service process stats", + monitor.METRIC_RES_TYPE_PROCESS, monitor.METRIC_DATABASE_SYSTEM, + }, + }, + Metrics: []SMetric{ + { + "cpu_percent", "CPU percent", monitor.METRIC_UNIT_NULL, + }, + { + "mem_percent", "Memory percent", monitor.METRIC_UNIT_NULL, + }, + { + "mem_size", "Memory size", monitor.METRIC_UNIT_NULL, + }, + { + "goroutine_num", "Goroutine num", monitor.METRIC_UNIT_NULL, + }, + }, +}