mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
fix: introduce dedicated queries for workspaces and workspace agents metrics (#19786)
aid in differentiation between sources of calls to `GetWorkspaces` but introducing new queries for metrics specific use cases --------- Signed-off-by: Callum Styan <callumstyan@gmail.com>
This commit is contained in:
Callum Styan
@@ -181,10 +181,8 @@ func Workspaces(ctx context.Context, logger slog.Logger, registerer prometheus.R
|
||||
done := make(chan struct{})
|
||||
|
||||
updateWorkspaceMetrics := func() {
|
||||
ws, err := db.GetWorkspaces(ctx, database.GetWorkspacesParams{
|
||||
Deleted: false,
|
||||
WithSummary: false,
|
||||
})
|
||||
// Don't count deleted workspaces as part of these metrics.
|
||||
ws, err := db.GetWorkspacesForWorkspaceMetrics(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
workspaceLatestBuildTotals.Reset()
|
||||
@@ -346,88 +344,67 @@ func Agents(ctx context.Context, logger slog.Logger, registerer prometheus.Regis
|
||||
timer := prometheus.NewTimer(metricsCollectorAgents)
|
||||
derpMap := derpMapFn()
|
||||
|
||||
workspaceRows, err := db.GetWorkspaces(ctx, database.GetWorkspacesParams{
|
||||
AgentInactiveDisconnectTimeoutSeconds: int64(agentInactiveDisconnectTimeout.Seconds()),
|
||||
})
|
||||
workspaceAgents, err := db.GetWorkspaceAgentsForMetrics(ctx)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "can't get workspace rows", slog.Error(err))
|
||||
logger.Error(ctx, "can't get workspace agents", slog.Error(err))
|
||||
goto done
|
||||
}
|
||||
|
||||
for _, workspace := range workspaceRows {
|
||||
templateName := workspace.TemplateName
|
||||
templateVersionName := workspace.TemplateVersionName.String
|
||||
if !workspace.TemplateVersionName.Valid {
|
||||
for _, agent := range workspaceAgents {
|
||||
// Collect information about agents
|
||||
templateVersionName := agent.TemplateVersionName.String
|
||||
if !agent.TemplateVersionName.Valid {
|
||||
templateVersionName = "unknown"
|
||||
}
|
||||
agentsGauge.WithLabelValues(VectorOperationAdd, 1, agent.OwnerUsername, agent.WorkspaceName, agent.TemplateName, templateVersionName)
|
||||
|
||||
// username :=
|
||||
connectionStatus := agent.WorkspaceAgent.Status(agentInactiveDisconnectTimeout)
|
||||
node := (*coordinator.Load()).Node(agent.WorkspaceAgent.ID)
|
||||
|
||||
agents, err := db.GetWorkspaceAgentsInLatestBuildByWorkspaceID(ctx, workspace.ID)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "can't get workspace agents", slog.F("workspace_id", workspace.ID), slog.Error(err))
|
||||
agentsGauge.WithLabelValues(VectorOperationAdd, 0, workspace.OwnerUsername, workspace.Name, templateName, templateVersionName)
|
||||
continue
|
||||
tailnetNode := "unknown"
|
||||
if node != nil {
|
||||
tailnetNode = node.ID.String()
|
||||
}
|
||||
|
||||
if len(agents) == 0 {
|
||||
logger.Debug(ctx, "workspace agents are unavailable", slog.F("workspace_id", workspace.ID))
|
||||
agentsGauge.WithLabelValues(VectorOperationAdd, 0, workspace.OwnerUsername, workspace.Name, templateName, templateVersionName)
|
||||
continue
|
||||
}
|
||||
agentsConnectionsGauge.WithLabelValues(VectorOperationSet, 1, agent.WorkspaceAgent.Name, agent.OwnerUsername, agent.WorkspaceName, string(connectionStatus.Status), string(agent.WorkspaceAgent.LifecycleState), tailnetNode)
|
||||
|
||||
for _, agent := range agents {
|
||||
// Collect information about agents
|
||||
agentsGauge.WithLabelValues(VectorOperationAdd, 1, workspace.OwnerUsername, workspace.Name, templateName, templateVersionName)
|
||||
|
||||
connectionStatus := agent.Status(agentInactiveDisconnectTimeout)
|
||||
node := (*coordinator.Load()).Node(agent.ID)
|
||||
|
||||
tailnetNode := "unknown"
|
||||
if node != nil {
|
||||
tailnetNode = node.ID.String()
|
||||
}
|
||||
|
||||
agentsConnectionsGauge.WithLabelValues(VectorOperationSet, 1, agent.Name, workspace.OwnerUsername, workspace.Name, string(connectionStatus.Status), string(agent.LifecycleState), tailnetNode)
|
||||
|
||||
if node == nil {
|
||||
logger.Debug(ctx, "can't read in-memory node for agent", slog.F("agent_id", agent.ID))
|
||||
} else {
|
||||
// Collect information about connection latencies
|
||||
for rawRegion, latency := range node.DERPLatency {
|
||||
regionParts := strings.SplitN(rawRegion, "-", 2)
|
||||
regionID, err := strconv.Atoi(regionParts[0])
|
||||
if err != nil {
|
||||
logger.Error(ctx, "can't convert DERP region", slog.F("agent_id", agent.ID), slog.F("raw_region", rawRegion), slog.Error(err))
|
||||
continue
|
||||
}
|
||||
|
||||
region, found := derpMap.Regions[regionID]
|
||||
if !found {
|
||||
// It's possible that a workspace agent is using an old DERPMap
|
||||
// and reports regions that do not exist. If that's the case,
|
||||
// report the region as unknown!
|
||||
region = &tailcfg.DERPRegion{
|
||||
RegionID: regionID,
|
||||
RegionName: fmt.Sprintf("Unnamed %d", regionID),
|
||||
}
|
||||
}
|
||||
|
||||
agentsConnectionLatenciesGauge.WithLabelValues(VectorOperationSet, latency, agent.Name, workspace.OwnerUsername, workspace.Name, region.RegionName, fmt.Sprintf("%v", node.PreferredDERP == regionID))
|
||||
if node == nil {
|
||||
logger.Debug(ctx, "can't read in-memory node for agent", slog.F("agent_id", agent.WorkspaceAgent.ID))
|
||||
} else {
|
||||
// Collect information about connection latencies
|
||||
for rawRegion, latency := range node.DERPLatency {
|
||||
regionParts := strings.SplitN(rawRegion, "-", 2)
|
||||
regionID, err := strconv.Atoi(regionParts[0])
|
||||
if err != nil {
|
||||
logger.Error(ctx, "can't convert DERP region", slog.F("agent_id", agent.WorkspaceAgent.ID), slog.F("raw_region", rawRegion), slog.Error(err))
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Collect information about registered applications
|
||||
apps, err := db.GetWorkspaceAppsByAgentID(ctx, agent.ID)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
logger.Error(ctx, "can't get workspace apps", slog.F("agent_id", agent.ID), slog.Error(err))
|
||||
continue
|
||||
}
|
||||
region, found := derpMap.Regions[regionID]
|
||||
if !found {
|
||||
// It's possible that a workspace agent is using an old DERPMap
|
||||
// and reports regions that do not exist. If that's the case,
|
||||
// report the region as unknown!
|
||||
region = &tailcfg.DERPRegion{
|
||||
RegionID: regionID,
|
||||
RegionName: fmt.Sprintf("Unnamed %d", regionID),
|
||||
}
|
||||
}
|
||||
|
||||
for _, app := range apps {
|
||||
agentsAppsGauge.WithLabelValues(VectorOperationAdd, 1, agent.Name, workspace.OwnerUsername, workspace.Name, app.DisplayName, string(app.Health))
|
||||
agentsConnectionLatenciesGauge.WithLabelValues(VectorOperationSet, latency, agent.WorkspaceAgent.Name, agent.OwnerUsername, agent.WorkspaceName, region.RegionName, fmt.Sprintf("%v", node.PreferredDERP == regionID))
|
||||
}
|
||||
}
|
||||
|
||||
// Collect information about registered applications
|
||||
apps, err := db.GetWorkspaceAppsByAgentID(ctx, agent.WorkspaceAgent.ID)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
logger.Error(ctx, "can't get workspace apps", slog.F("agent_id", agent.WorkspaceAgent.ID), slog.Error(err))
|
||||
continue
|
||||
}
|
||||
|
||||
for _, app := range apps {
|
||||
agentsAppsGauge.WithLabelValues(VectorOperationAdd, 1, agent.WorkspaceAgent.Name, agent.OwnerUsername, agent.WorkspaceName, app.DisplayName, string(app.Health))
|
||||
}
|
||||
}
|
||||
|
||||
agentsGauge.Commit()
|
||||
|
||||
@@ -291,6 +291,7 @@ func TestWorkspaceLatestBuildTotals(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, metric := range m.Metric {
|
||||
fmt.Printf("metric: %+v\n", metric)
|
||||
count, ok := tc.Status[codersdk.ProvisionerJobStatus(metric.Label[0].GetValue())]
|
||||
if metric.Gauge.GetValue() == 0 {
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user