From 53969f14e26b34faad347060f1742a5cae34632b Mon Sep 17 00:00:00 2001 From: Qu Xuan Date: Wed, 3 Feb 2021 11:10:50 +0800 Subject: [PATCH] fix(cloudevent): add distinct filed for cloudevent --- pkg/apis/cloudevent/cloudevent.go | 6 ++ pkg/apis/input.go | 5 ++ pkg/cloudcommon/db/distinct_fields.go | 88 +++++++++++++++++++++++++++ pkg/cloudcommon/db/fetch.go | 23 +++++++ pkg/cloudevent/models/cloudevents.go | 68 ++++++++++++++++++++- pkg/cloudevent/service/handlers.go | 1 + 6 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 pkg/cloudcommon/db/distinct_fields.go diff --git a/pkg/apis/cloudevent/cloudevent.go b/pkg/apis/cloudevent/cloudevent.go index d0b1bea237..9a3a7dd348 100644 --- a/pkg/apis/cloudevent/cloudevent.go +++ b/pkg/apis/cloudevent/cloudevent.go @@ -39,6 +39,12 @@ type CloudeventListInput struct { // 操作类型 Action []string `json:"action"` + // 资源类别 + ResourceType []string `json:"resource_type"` + + // 执行状态 + Success *bool `json:"success"` + // 操作日志起始时间 Since time.Time `json:"since"` // 操作日志截止时间 diff --git a/pkg/apis/input.go b/pkg/apis/input.go index 3004dbee15..2ebf240424 100644 --- a/pkg/apis/input.go +++ b/pkg/apis/input.go @@ -345,3 +345,8 @@ type GetMetadataInput struct { // 获取资源标签（元数据）输出 type GetMetadataOutput map[string]string + +type DistinctFieldInput struct { + Field []string + ExtraField []string +} diff --git a/pkg/cloudcommon/db/distinct_fields.go b/pkg/cloudcommon/db/distinct_fields.go new file mode 100644 index 0000000000..9596650949 --- /dev/null +++ b/pkg/cloudcommon/db/distinct_fields.go @@ -0,0 +1,88 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package db + +import ( + "context" + "fmt" + + "yunion.io/x/pkg/errors" +) + +type SDistinctFieldManager struct { + SModelBaseManager +} + +var DistinctFieldManager *SDistinctFieldManager + +func init() { + DistinctFieldManager = &SDistinctFieldManager{ + SModelBaseManager: NewModelBaseManager( + SDistinctField{}, + "distinct_fields_tbl", + "distinct_field", + "distinct_fields", + ), + } + DistinctFieldManager.SetVirtualObject(DistinctFieldManager) +} + +const ( + DISTINCT_FIELD_SEP = "::" +) + +type SDistinctField struct { + SModelBase + + // 资源类型 + // example: network + ObjType string `width:"40" charset:"ascii" index:"true" list:"user" get:"user"` + + // 资源组合ID + // example: obj_type::key::value + Id string `width:"128" charset:"utf8" primary:"true" list:"user" get:"user"` + + // Distinct Field + // exmaple: 部门 + Key string `width:"64" charset:"utf8" primary:"true" list:"user" get:"user"` + + // Distinct Value + // example: 技术部 + Value string `charset:"utf8" list:"user" get:"user"` +} + +func (manager *SDistinctFieldManager) GetObjectDistinctFields(objType string) ([]SDistinctField, error) { + q := manager.Query().Equals("obj_type", objType) + fields := []SDistinctField{} + err := FetchModelObjects(manager, q, &fields) + if err != nil { + return nil, errors.Wrapf(err, "FetchModelObjects") + } + return fields, nil +} + +func (manager *SDistinctFieldManager) InsertOrUpdate(ctx context.Context, modelManager IModelManager, key, value string) error { + if len(key) == 0 || len(value) == 0 { + return fmt.Errorf("empty key or value") + } + distinct := &SDistinctField{ + ObjType: modelManager.Keyword(), + Key: key, + Value: value, + Id: modelManager.Keyword() + DISTINCT_FIELD_SEP + key + DISTINCT_FIELD_SEP + value, + } + distinct.SetModelManager(manager, distinct) + return manager.TableSpec().InsertOrUpdate(ctx, distinct) +} diff --git a/pkg/cloudcommon/db/fetch.go b/pkg/cloudcommon/db/fetch.go index 2d9fbe72a4..beb4d2da01 100644 --- a/pkg/cloudcommon/db/fetch.go +++ b/pkg/cloudcommon/db/fetch.go @@ -515,3 +515,26 @@ func FetchQueryObjectsByIds(q *sqlchemy.SQuery, fieldName string, ids []string, func FetchStandaloneObjectsByIds(modelManager IModelManager, ids []string, targets interface{}) error { return FetchModelObjectsByIds(modelManager, "id", ids, targets) } + +func FetchDistinctField(modelManager IModelManager, field string) ([]string, error) { + q := modelManager.Query(field).Distinct() + rows, err := q.Rows() + if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return nil, nil + } + return nil, errors.Wrapf(err, "q.Rows") + } + defer rows.Close() + + values := []string{} + for rows.Next() { + var value string + err := rows.Scan(&value) + if err != nil { + return values, errors.Wrap(err, "rows.Scan") + } + values = append(values, value) + } + return values, nil +} diff --git a/pkg/cloudevent/models/cloudevents.go b/pkg/cloudevent/models/cloudevents.go index 8c0e9dcc1c..61db9b5897 100644 --- a/pkg/cloudevent/models/cloudevents.go +++ b/pkg/cloudevent/models/cloudevents.go @@ -23,6 +23,7 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/cloudevent" "yunion.io/x/onecloud/pkg/cloudcommon/consts" "yunion.io/x/onecloud/pkg/cloudcommon/db" @@ -136,6 +137,14 @@ func (manager *SCloudeventManager) ListItemFilter( q = q.In("action", input.Action) } + if len(input.ResourceType) > 0 { + q = q.In("resource_type", input.ResourceType) + } + + if input.Success != nil { + q = q.Equals("success", *input.Success) + } + if !input.Since.IsZero() { q = q.GT("created_at", input.Since) } @@ -237,6 +246,18 @@ func (manager *SCloudeventManager) SyncCloudevent(ctx context.Context, userCred event.Brand = event.Provider } + for k, v := range map[string]string{ + "service": event.Service, + "resoruce_type": event.ResourceType, + "action": event.Action, + "account": event.Account, + "manager": event.Manager, + "provider": event.Provider, + "brand": event.Brand, + } { + db.DistinctFieldManager.InsertOrUpdate(ctx, manager, k, v) + } + event.CreatedAt = iEvent.GetCreatedAt() event.SetModelManager(manager, event) err := manager.TableSpec().Insert(ctx, event) @@ -257,6 +278,51 @@ func (manager *SCloudeventManager) GetPagingConfig() *db.SPagingConfig { } } +func (manager *SCloudeventManager) GetPropertyDistinctField(ctx context.Context, userCred mcclient.TokenCredential, input apis.DistinctFieldInput) (jsonutils.JSONObject, error) { + fields, err := db.DistinctFieldManager.GetObjectDistinctFields(manager.Keyword()) + if err != nil { + return nil, errors.Wrapf(err, "DistinctFieldManager.GetObjectDistinctFields") + } + fieldMaps := map[string][]string{} + for _, field := range fields { + _, ok := fieldMaps[field.Key] + if !ok { + fieldMaps[field.Key] = []string{} + } + fieldMaps[field.Key] = append(fieldMaps[field.Key], field.Value) + } + ret := map[string][]string{} + for _, key := range input.Field { + ret[key], _ = fieldMaps[key] + } + return jsonutils.Marshal(ret), nil +} + +func (manager *SCloudeventManager) initDistinctField() error { + fileds, err := db.DistinctFieldManager.GetObjectDistinctFields(manager.Keyword()) + if err != nil { + return errors.Wrapf(err, "GetObjectDistinctFields") + } + if len(fileds) > 0 { + return nil + } + for _, key := range []string{"service", "resource_type", "action", "account", "manager", "provider", "brand"} { + values, err := db.FetchDistinctField(manager, key) + if err != nil { + return errors.Wrapf(err, "db.FetchDistinctField") + } + for _, value := range values { + if len(value) > 0 { + err = db.DistinctFieldManager.InsertOrUpdate(nil, manager, key, value) + if err != nil { + return errors.Wrapf(err, "DistinctFieldManager.InsertOrUpdate(%s, %s)", key, value) + } + } + } + } + return nil +} + func (manager *SCloudeventManager) InitializeData() error { events := []SCloudevent{} q := manager.Query().IsNullOrEmpty("brand") @@ -273,5 +339,5 @@ func (manager *SCloudeventManager) InitializeData() error { return err } } - return nil + return manager.initDistinctField() } diff --git a/pkg/cloudevent/service/handlers.go b/pkg/cloudevent/service/handlers.go index 1c5056a0fa..d128ab6e96 100644 --- a/pkg/cloudevent/service/handlers.go +++ b/pkg/cloudevent/service/handlers.go @@ -48,6 +48,7 @@ func InitHandlers(app *appsrv.Application) { taskman.TaskObjectManager, db.UserCacheManager, db.TenantCacheManager, + db.DistinctFieldManager, models.CloudproviderManager, } { db.RegisterModelManager(manager)