Add status label to teleport_upgrader_counts metrics (#60119)

* Add status label to upgrader metrics

* CR changes

lib/auth/auth.go

@@ -1028,6 +1028,7 @@ var (
 		[]string{
 			teleport.TagUpgrader,
 			teleport.TagVersion,
+			teleport.TagUpgraderStatus,
 		},
 	)
 
@@ -2167,8 +2168,9 @@ func (a *Server) updateAgentMetrics() {
 	upgraderCountsMetric.Reset()
 	for metadata, count := range imp.UpgraderCounts() {
 		upgraderCountsMetric.With(prometheus.Labels{
-			teleport.TagUpgrader: metadata.upgraderType,
-			teleport.TagVersion:  metadata.version,
+			teleport.TagUpgrader:       metadata.upgraderType,
+			teleport.TagVersion:        metadata.version,
+			teleport.TagUpgraderStatus: metadata.status,
 		}).Set(float64(count))
 	}
 }

lib/auth/periodic.go

@@ -121,6 +121,8 @@ type instanceMetadata struct {
 	upgraderType string
 	// upgraderVersion specifies the upgrader version
 	upgraderVersion string
+	// upgraderStatus is status from the upgrader.
+	upgraderStatus string
 }
 
 func newInstanceMetricsPeriodic() *instanceMetricsPeriodic {
@@ -138,12 +140,17 @@ func (i *instanceMetricsPeriodic) VisitInstance(instance *proto.UpstreamInventor
 		installMethod = strings.Join(installMethods, ",")
 	}
 
+	var upgraderStatus string
+	if instance.GetUpdaterInfo() != nil {
+		upgraderStatus = instance.GetUpdaterInfo().UpdaterStatus.String()
+	}
 	iMetadata := instanceMetadata{
 		os:              metadata.GetOS(),
 		version:         instance.GetVersion(),
 		installMethod:   installMethod,
 		upgraderType:    instance.GetExternalUpgrader(),
 		upgraderVersion: instance.GetExternalUpgraderVersion(),
+		upgraderStatus:  upgraderStatus,
 	}
 	i.metadata = append(i.metadata, iMetadata)
 }
@@ -185,6 +192,7 @@ func (i *instanceMetricsPeriodic) InstallMethodCounts() map[string]int {
 type upgrader struct {
 	upgraderType string
 	version      string
+	status       string
 }
 
 // UpgraderCounts returns the count for the different upgrader version and type combinations.
@@ -199,6 +207,7 @@ func (i *instanceMetricsPeriodic) UpgraderCounts() map[upgrader]int {
 		upgrader := upgrader{
 			upgraderType: metadata.upgraderType,
 			version:      metadata.upgraderVersion,
+			status:       metadata.upgraderStatus,
 		}
 		result[upgrader]++
 	}

lib/auth/periodic_test.go

@@ -107,18 +107,26 @@ func TestUpgraderCounts(t *testing.T) {
 		{
 			desc: "mixed",
 			instances: []*proto.UpstreamInventoryHello{
-				{ExternalUpgrader: "kube", ExternalUpgraderVersion: "13.0.0"},
-				{ExternalUpgrader: "kube", ExternalUpgraderVersion: "14.0.0"},
-				{ExternalUpgrader: "unit", ExternalUpgraderVersion: "13.0.0"},
+				{ExternalUpgrader: "kube", ExternalUpgraderVersion: "13.0.0", UpdaterInfo: &types.UpdaterV2Info{
+					UpdaterStatus: types.UpdaterStatus_UPDATER_STATUS_OK,
+				}},
+				{ExternalUpgrader: "kube", ExternalUpgraderVersion: "14.0.0", UpdaterInfo: &types.UpdaterV2Info{
+					UpdaterStatus: types.UpdaterStatus_UPDATER_STATUS_OK,
+				}},
+				{ExternalUpgrader: "unit", ExternalUpgraderVersion: "13.0.0", UpdaterInfo: &types.UpdaterV2Info{
+					UpdaterStatus: types.UpdaterStatus_UPDATER_STATUS_OK,
+				}},
 				{},
-				{ExternalUpgrader: "unit", ExternalUpgraderVersion: "14.0.0"},
+				{ExternalUpgrader: "unit", ExternalUpgraderVersion: "14.0.0", UpdaterInfo: &types.UpdaterV2Info{
+					UpdaterStatus: types.UpdaterStatus_UPDATER_STATUS_PINNED,
+				}},
 				{},
 			},
 			expected: map[upgrader]int{
-				{"kube", "13.0.0"}: 1,
-				{"kube", "14.0.0"}: 1,
-				{"unit", "13.0.0"}: 1,
-				{"unit", "14.0.0"}: 1,
+				{"kube", "13.0.0", "UPDATER_STATUS_OK"}:     1,
+				{"kube", "14.0.0", "UPDATER_STATUS_OK"}:     1,
+				{"unit", "13.0.0", "UPDATER_STATUS_OK"}:     1,
+				{"unit", "14.0.0", "UPDATER_STATUS_PINNED"}: 1,
 			},
 		},
 		{
@@ -138,8 +146,8 @@ func TestUpgraderCounts(t *testing.T) {
 				{ExternalUpgrader: "unit", ExternalUpgraderVersion: "13.0.0"},
 			},
 			expected: map[upgrader]int{
-				{"kube", "13.0.0"}: 2,
-				{"unit", "13.0.0"}: 2,
+				{"kube", "13.0.0", ""}: 2,
+				{"unit", "13.0.0", ""}: 2,
 			},
 		},
 		{
@@ -149,8 +157,8 @@ func TestUpgraderCounts(t *testing.T) {
 				{ExternalUpgrader: "unit"},
 			},
 			expected: map[upgrader]int{
-				{"kube", ""}: 1,
-				{"unit", ""}: 1,
+				{"kube", "", ""}: 1,
+				{"unit", "", ""}: 1,
 			},
 		},
 		{

metrics.go

@@ -108,6 +108,9 @@ const (
 	// TagUpgrader is a metric tag for upgraders.
 	TagUpgrader = "upgrader"
 
+	// TagUpgraderStatus is propagated status from the upgrader.
+	TagUpgraderStatus = "status"
+
 	// MetricsAccessRequestsCreated provides total number of created access requests.
 	MetricAccessRequestsCreated = "access_requests_created"
 	// TagRoles is a number of roles requested as a part of access request.