fix(cloudevent): add distinct filed for cloudevent

pkg/apis/cloudevent/cloudevent.go

@@ -39,6 +39,12 @@ type CloudeventListInput struct {
 	// 操作类型
 	Action []string `json:"action"`
 
+	// 资源类别
+	ResourceType []string `json:"resource_type"`
+
+	// 执行状态
+	Success *bool `json:"success"`
+
 	// 操作日志起始时间
 	Since time.Time `json:"since"`
 	// 操作日志截止时间

pkg/apis/input.go

@@ -345,3 +345,8 @@ type GetMetadataInput struct {
 
 // 获取资源标签（元数据）输出
 type GetMetadataOutput map[string]string
+
+type DistinctFieldInput struct {
+	Field      []string
+	ExtraField []string
+}

pkg/cloudcommon/db/distinct_fields.go

@@ -0,0 +1,88 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package db
+
+import (
+	"context"
+	"fmt"
+
+	"yunion.io/x/pkg/errors"
+)
+
+type SDistinctFieldManager struct {
+	SModelBaseManager
+}
+
+var DistinctFieldManager *SDistinctFieldManager
+
+func init() {
+	DistinctFieldManager = &SDistinctFieldManager{
+		SModelBaseManager: NewModelBaseManager(
+			SDistinctField{},
+			"distinct_fields_tbl",
+			"distinct_field",
+			"distinct_fields",
+		),
+	}
+	DistinctFieldManager.SetVirtualObject(DistinctFieldManager)
+}
+
+const (
+	DISTINCT_FIELD_SEP = "::"
+)
+
+type SDistinctField struct {
+	SModelBase
+
+	// 资源类型
+	// example: network
+	ObjType string `width:"40" charset:"ascii" index:"true" list:"user" get:"user"`
+
+	// 资源组合ID
+	// example: obj_type::key::value
+	Id string `width:"128" charset:"utf8" primary:"true" list:"user" get:"user"`
+
+	// Distinct Field
+	// exmaple: 部门
+	Key string `width:"64" charset:"utf8" primary:"true" list:"user" get:"user"`
+
+	// Distinct Value
+	// example: 技术部
+	Value string `charset:"utf8" list:"user" get:"user"`
+}
+
+func (manager *SDistinctFieldManager) GetObjectDistinctFields(objType string) ([]SDistinctField, error) {
+	q := manager.Query().Equals("obj_type", objType)
+	fields := []SDistinctField{}
+	err := FetchModelObjects(manager, q, &fields)
+	if err != nil {
+		return nil, errors.Wrapf(err, "FetchModelObjects")
+	}
+	return fields, nil
+}
+
+func (manager *SDistinctFieldManager) InsertOrUpdate(ctx context.Context, modelManager IModelManager, key, value string) error {
+	if len(key) == 0 || len(value) == 0 {
+		return fmt.Errorf("empty key or value")
+	}
+	distinct := &SDistinctField{
+		ObjType: modelManager.Keyword(),
+		Key:     key,
+		Value:   value,
+		Id:      modelManager.Keyword() + DISTINCT_FIELD_SEP + key + DISTINCT_FIELD_SEP + value,
+	}
+	distinct.SetModelManager(manager, distinct)
+	return manager.TableSpec().InsertOrUpdate(ctx, distinct)
+}

pkg/cloudcommon/db/fetch.go

@@ -515,3 +515,26 @@ func FetchQueryObjectsByIds(q *sqlchemy.SQuery, fieldName string, ids []string,
 func FetchStandaloneObjectsByIds(modelManager IModelManager, ids []string, targets interface{}) error {
 	return FetchModelObjectsByIds(modelManager, "id", ids, targets)
 }
+
+func FetchDistinctField(modelManager IModelManager, field string) ([]string, error) {
+	q := modelManager.Query(field).Distinct()
+	rows, err := q.Rows()
+	if err != nil {
+		if errors.Cause(err) == sql.ErrNoRows {
+			return nil, nil
+		}
+		return nil, errors.Wrapf(err, "q.Rows")
+	}
+	defer rows.Close()
+
+	values := []string{}
+	for rows.Next() {
+		var value string
+		err := rows.Scan(&value)
+		if err != nil {
+			return values, errors.Wrap(err, "rows.Scan")
+		}
+		values = append(values, value)
+	}
+	return values, nil
+}

pkg/cloudevent/models/cloudevents.go

@@ -23,6 +23,7 @@ import (
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/sqlchemy"
 
+	"yunion.io/x/onecloud/pkg/apis"
 	api "yunion.io/x/onecloud/pkg/apis/cloudevent"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
@@ -136,6 +137,14 @@ func (manager *SCloudeventManager) ListItemFilter(
 		q = q.In("action", input.Action)
 	}
 
+	if len(input.ResourceType) > 0 {
+		q = q.In("resource_type", input.ResourceType)
+	}
+
+	if input.Success != nil {
+		q = q.Equals("success", *input.Success)
+	}
+
 	if !input.Since.IsZero() {
 		q = q.GT("created_at", input.Since)
 	}
@@ -237,6 +246,18 @@ func (manager *SCloudeventManager) SyncCloudevent(ctx context.Context, userCred
 			event.Brand = event.Provider
 		}
 
+		for k, v := range map[string]string{
+			"service":       event.Service,
+			"resoruce_type": event.ResourceType,
+			"action":        event.Action,
+			"account":       event.Account,
+			"manager":       event.Manager,
+			"provider":      event.Provider,
+			"brand":         event.Brand,
+		} {
+			db.DistinctFieldManager.InsertOrUpdate(ctx, manager, k, v)
+		}
+
 		event.CreatedAt = iEvent.GetCreatedAt()
 		event.SetModelManager(manager, event)
 		err := manager.TableSpec().Insert(ctx, event)
@@ -257,6 +278,51 @@ func (manager *SCloudeventManager) GetPagingConfig() *db.SPagingConfig {
 	}
 }
 
+func (manager *SCloudeventManager) GetPropertyDistinctField(ctx context.Context, userCred mcclient.TokenCredential, input apis.DistinctFieldInput) (jsonutils.JSONObject, error) {
+	fields, err := db.DistinctFieldManager.GetObjectDistinctFields(manager.Keyword())
+	if err != nil {
+		return nil, errors.Wrapf(err, "DistinctFieldManager.GetObjectDistinctFields")
+	}
+	fieldMaps := map[string][]string{}
+	for _, field := range fields {
+		_, ok := fieldMaps[field.Key]
+		if !ok {
+			fieldMaps[field.Key] = []string{}
+		}
+		fieldMaps[field.Key] = append(fieldMaps[field.Key], field.Value)
+	}
+	ret := map[string][]string{}
+	for _, key := range input.Field {
+		ret[key], _ = fieldMaps[key]
+	}
+	return jsonutils.Marshal(ret), nil
+}
+
+func (manager *SCloudeventManager) initDistinctField() error {
+	fileds, err := db.DistinctFieldManager.GetObjectDistinctFields(manager.Keyword())
+	if err != nil {
+		return errors.Wrapf(err, "GetObjectDistinctFields")
+	}
+	if len(fileds) > 0 {
+		return nil
+	}
+	for _, key := range []string{"service", "resource_type", "action", "account", "manager", "provider", "brand"} {
+		values, err := db.FetchDistinctField(manager, key)
+		if err != nil {
+			return errors.Wrapf(err, "db.FetchDistinctField")
+		}
+		for _, value := range values {
+			if len(value) > 0 {
+				err = db.DistinctFieldManager.InsertOrUpdate(nil, manager, key, value)
+				if err != nil {
+					return errors.Wrapf(err, "DistinctFieldManager.InsertOrUpdate(%s, %s)", key, value)
+				}
+			}
+		}
+	}
+	return nil
+}
+
 func (manager *SCloudeventManager) InitializeData() error {
 	events := []SCloudevent{}
 	q := manager.Query().IsNullOrEmpty("brand")
@@ -273,5 +339,5 @@ func (manager *SCloudeventManager) InitializeData() error {
 			return err
 		}
 	}
-	return nil
+	return manager.initDistinctField()
 }

pkg/cloudevent/service/handlers.go

@@ -48,6 +48,7 @@ func InitHandlers(app *appsrv.Application) {
 		taskman.TaskObjectManager,
 		db.UserCacheManager,
 		db.TenantCacheManager,
+		db.DistinctFieldManager,
 		models.CloudproviderManager,
 	} {
 		db.RegisterModelManager(manager)