Merge pull request #15881 from ioito/hotfix/qx-project-tags

fix(cloudmon): project tags for metric

pkg/apis/compute/guests.go

@@ -16,6 +16,7 @@ package compute
 
 import (
 	"fmt"
+	"strings"
 	"time"
 
 	"yunion.io/x/jsonutils"
@@ -23,6 +24,7 @@ import (
 
 	"yunion.io/x/onecloud/pkg/apis"
 	"yunion.io/x/onecloud/pkg/apis/billing"
+	"yunion.io/x/onecloud/pkg/apis/cloudcommon/db"
 	imageapi "yunion.io/x/onecloud/pkg/apis/image"
 	"yunion.io/x/onecloud/pkg/httperrors"
 )
@@ -294,6 +296,11 @@ func (self ServerDetails) GetMetricTags() map[string]string {
 		"account_id":          self.AccountId,
 		"external_id":         self.ExternalId,
 	}
+	for k, v := range self.Metadata {
+		if strings.HasPrefix(k, db.USER_TAG_PREFIX) {
+			ret[k] = v
+		}
+	}
 	return ret
 }
 

pkg/cloudmon/resources/resources.go

@@ -28,12 +28,14 @@ import (
 
 	"yunion.io/x/onecloud/pkg/apis"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudmon/options"
 	"yunion.io/x/onecloud/pkg/cloudmon/providerdriver"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
 	"yunion.io/x/onecloud/pkg/mcclient/modules/compute"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/identity"
 	"yunion.io/x/onecloud/pkg/util/influxdb"
 )
 
@@ -45,6 +47,33 @@ type sBaseInfo struct {
 	ImportedAt time.Time
 	DeletedAt  time.Time
 	UpdatedAt  time.Time
+	Metadata   map[string]string
+}
+
+type sProjectTag struct {
+	lock sync.Mutex
+	tags map[string]map[string]string
+}
+
+func (self *sProjectTag) SetTags(projectId string, tags map[string]string) {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	self.tags[projectId] = tags
+}
+
+func (self *sProjectTag) GetTags(projectId string) map[string]string {
+	tags, _ := self.tags[projectId]
+	return tags
+}
+
+func (self *sProjectTag) RemoveTags(projectId string) {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	delete(self.tags, projectId)
+}
+
+var projectTags = &sProjectTag{
+	tags: map[string]map[string]string{},
 }
 
 type SBaseResources struct {
@@ -98,18 +127,16 @@ func (self *SBaseResources) init(ctx context.Context) error {
 		}
 		offset += len(resp.Data)
 		for i := range resp.Data {
-			baseInfo := struct {
-				Id         string
-				ExternalId string
-				ManagerId  string
-				CreatedAt  time.Time
-				ImportedAt time.Time
-			}{}
+			baseInfo := sBaseInfo{}
 			resp.Data[i].Unmarshal(&baseInfo)
 			if len(baseInfo.ExternalId) == 0 && (self.manager.GetKeyword() != compute.Cloudproviders.GetKeyword() &&
-				self.manager.GetKeyword() != compute.Cloudaccounts.GetKeyword()) {
+				self.manager.GetKeyword() != compute.Cloudaccounts.GetKeyword() &&
+				self.manager.GetKeyword() != identity.Projects.GetKeyword()) {
 				continue
 			}
+			if self.manager.GetKeyword() == identity.Projects.GetKeyword() {
+				projectTags.SetTags(baseInfo.Id, baseInfo.Metadata)
+			}
 			key := baseInfo.ExternalId
 			if len(key) == 0 {
 				key = baseInfo.Id
@@ -177,9 +204,13 @@ func (self *SBaseResources) increment(ctx context.Context) error {
 		baseInfo := sBaseInfo{}
 		ret[i].Unmarshal(&baseInfo)
 		if len(baseInfo.ExternalId) == 0 && (self.manager.GetKeyword() != compute.Cloudproviders.GetKeyword() &&
+			self.manager.GetKeyword() != identity.Projects.GetKeyword() &&
 			self.manager.GetKeyword() != compute.Cloudaccounts.GetKeyword()) {
 			continue
 		}
+		if self.manager.GetKeyword() == identity.Projects.GetKeyword() {
+			projectTags.SetTags(baseInfo.Id, baseInfo.Metadata)
+		}
 		key := baseInfo.ExternalId
 		if len(key) == 0 {
 			key = baseInfo.Id
@@ -239,9 +270,13 @@ func (self *SBaseResources) decrement(ctx context.Context) error {
 	for i := range ret {
 		baseInfo := sBaseInfo{}
 		ret[i].Unmarshal(&baseInfo)
-		if len(baseInfo.ExternalId) == 0 && self.manager.GetKeyword() != compute.Cloudproviders.GetKeyword() {
+		if len(baseInfo.ExternalId) == 0 && self.manager.GetKeyword() != compute.Cloudproviders.GetKeyword() &&
+			self.manager.GetKeyword() != identity.Projects.GetKeyword() {
 			continue
 		}
+		if self.manager.GetKeyword() == identity.Projects.GetKeyword() {
+			projectTags.RemoveTags(baseInfo.Id)
+		}
 		key := baseInfo.ExternalId
 		if len(key) == 0 {
 			key = baseInfo.Id
@@ -296,6 +331,9 @@ func (self *SBaseResources) update(ctx context.Context) error {
 	for i := range ret {
 		baseInfo := sBaseInfo{}
 		ret[i].Unmarshal(&baseInfo)
+		if self.manager.GetKeyword() == identity.Projects.GetKeyword() {
+			projectTags.SetTags(baseInfo.Id, baseInfo.Metadata)
+		}
 		if len(baseInfo.ExternalId) == 0 {
 			continue
 		}
@@ -346,6 +384,7 @@ type SResources struct {
 	Storages       TResource
 	ModelartsPool  TResource
 	Wires          TResource
+	Projects       TResource
 }
 
 func NewResources() *SResources {
@@ -362,6 +401,7 @@ func NewResources() *SResources {
 		KubeClusters:   NewBaseResources(&compute.KubeClusters),
 		ModelartsPool:  NewBaseResources(&compute.ModelartsPools),
 		Wires:          NewBaseResources(&compute.Wires),
+		Projects:       NewBaseResources(&identity.Projects),
 	}
 }
 
@@ -373,6 +413,10 @@ func (self *SResources) Init(ctx context.Context, userCred mcclient.TokenCredent
 			if err != nil {
 				errs = append(errs, errors.Wrapf(err, "Cloudaccount.init"))
 			}
+			err = self.Projects.init(ctx)
+			if err != nil {
+				errs = append(errs, errors.Wrapf(err, "Projects.init"))
+			}
 			err = self.Cloudproviders.init(ctx)
 			if err != nil {
 				errs = append(errs, errors.Wrapf(err, "Cloudproviders.init"))
@@ -435,6 +479,10 @@ func (self *SResources) IncrementSync(ctx context.Context, userCred mcclient.Tok
 		if err != nil {
 			errs = append(errs, errors.Wrapf(err, "Cloudaccounts.increment"))
 		}
+		err = self.Projects.increment(ctx)
+		if err != nil {
+			errs = append(errs, errors.Wrapf(err, "Projects.increment"))
+		}
 		err = self.Cloudproviders.increment(ctx)
 		if err != nil {
 			errs = append(errs, errors.Wrapf(err, "Cloudproviders.increment"))
@@ -540,6 +588,10 @@ func (self *SResources) DecrementSync(ctx context.Context, userCred mcclient.Tok
 		if err != nil {
 			errs = append(errs, errors.Wrapf(err, "ModelartsPool.decrement"))
 		}
+		err = self.Projects.decrement(ctx)
+		if err != nil {
+			errs = append(errs, errors.Wrapf(err, "Projects.decrement"))
+		}
 		return errors.NewAggregate(errs)
 	}()
 	if err != nil {
@@ -557,6 +609,10 @@ func (self *SResources) UpdateSync(ctx context.Context, userCred mcclient.TokenC
 		if err != nil {
 			errs = append(errs, errors.Wrapf(err, "Cloudacconts.update"))
 		}
+		err = self.Projects.update(ctx)
+		if err != nil {
+			errs = append(errs, errors.Wrapf(err, "Projects.update"))
+		}
 		err = self.DBInstances.update(ctx)
 		if err != nil {
 			errs = append(errs, errors.Wrapf(err, "DBInstances.update"))
@@ -657,6 +713,14 @@ func (self *SResources) CollectMetrics(ctx context.Context, userCred mcclient.To
 			if err != nil {
 				log.Errorf("unmarsha server resources error: %v", err)
 			}
+			for t := range servers {
+				tags := projectTags.GetTags(servers[t].ProjectId)
+				for k, v := range tags {
+					if strings.HasPrefix(k, db.USER_TAG_PREFIX) {
+						servers[t].Metadata[k] = v
+					}
+				}
+			}
 			err = driver.CollectServerMetrics(ctx, manager, provider, servers, startTime, endTime)
 			if err != nil && errors.Cause(err) != cloudprovider.ErrNotImplemented && errors.Cause(err) != cloudprovider.ErrNotSupported {
 				log.Errorf("CollectServerMetrics for %s(%s) error: %v", manager.Name, manager.Provider, err)

pkg/util/influxdb/metrics.go

@@ -28,7 +28,9 @@ type SKeyValue struct {
 }
 
 func (kv SKeyValue) String() string {
-	return fmt.Sprintf("%s=%s", strings.ReplaceAll(strings.Trim(kv.Key, " "), ",", ""), strings.ReplaceAll(strings.ReplaceAll(kv.Value, " ", "+"), ",", "+"))
+	k := strings.ReplaceAll(strings.ReplaceAll(strings.Trim(kv.Key, " "), ",", ""), " ", "+")
+	v := strings.ReplaceAll(strings.ReplaceAll(strings.Trim(kv.Value, ""), " ", "+"), ",", "+")
+	return fmt.Sprintf("%s=%s", k, v)
 }
 
 type TKeyValuePairs []SKeyValue