From e11ceb4c4b2339263838684b8128211920c0f26e Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Sat, 7 Mar 2020 17:49:08 +0800 Subject: [PATCH] feature: allow list/export/sort/distict metadata --- cmd/climc/shell/distinctfield.go | 48 +++ cmd/climc/shell/metadata.go | 8 +- go.mod | 4 +- go.sum | 8 +- pkg/apis/list.go | 13 +- pkg/apis/metadata.go | 52 ++++ pkg/cloudcommon/db/db_dispatcher.go | 54 ++-- pkg/cloudcommon/db/interface.go | 5 +- pkg/cloudcommon/db/metadata.go | 353 ++++++++++++++++------- pkg/cloudcommon/db/modelbase.go | 4 +- pkg/cloudcommon/db/orderby.go | 1 - pkg/cloudcommon/db/standalone.go | 83 +++++- pkg/cloudevent/models/initdb.go | 1 + pkg/compute/models/guests.go | 37 +-- pkg/compute/models/initdb.go | 1 + pkg/compute/models/isolated_devices.go | 14 +- pkg/httperrors/consts.go | 2 + pkg/image/models/initdb.go | 3 +- pkg/image/service/handlers.go | 2 + pkg/keystone/models/initdb.go | 2 + pkg/mcclient/options/base.go | 2 + pkg/mcclient/options/metadata.go | 16 +- pkg/util/stringutils2/ignorecase.go | 33 +++ pkg/util/stringutils2/ignorecase_test.go | 81 ++++++ pkg/util/stringutils2/sortedstrings.go | 2 +- vendor/modules.txt | 4 +- 26 files changed, 651 insertions(+), 182 deletions(-) create mode 100644 cmd/climc/shell/distinctfield.go create mode 100644 pkg/apis/metadata.go create mode 100644 pkg/util/stringutils2/ignorecase.go create mode 100644 pkg/util/stringutils2/ignorecase_test.go diff --git a/cmd/climc/shell/distinctfield.go b/cmd/climc/shell/distinctfield.go new file mode 100644 index 0000000000..73cca609c7 --- /dev/null +++ b/cmd/climc/shell/distinctfield.go @@ -0,0 +1,48 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "fmt" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modulebase" +) + +func init() { + type DistinctFieldOption struct { + MODULE string `help:"module name"` + FIELD string `help:"distinct field name to query"` + } + R(&DistinctFieldOption{}, "distinct-field", "Query values of a distinct field for a module", func(s *mcclient.ClientSession, args *DistinctFieldOption) error { + mod, err := modulebase.GetModule(s, args.MODULE) + if err != nil || mod == nil { + if err != nil { + return fmt.Errorf("module %s not found %s", args.MODULE, err) + } + return fmt.Errorf("No module %s found", args.MODULE) + } + params := jsonutils.NewDict() + params.Add(jsonutils.NewString(args.FIELD), "extra_field") + result, err := mod.Get(s, "distinct-field", params) + if err != nil { + return err + } + fmt.Println(result) + return nil + }) +} diff --git a/cmd/climc/shell/metadata.go b/cmd/climc/shell/metadata.go index 22d2fbe9e0..63d6b24d53 100644 --- a/cmd/climc/shell/metadata.go +++ b/cmd/climc/shell/metadata.go @@ -16,6 +16,7 @@ package shell import ( "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modulebase" "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/mcclient/options" ) @@ -43,7 +44,12 @@ func init() { if err != nil { return err } - printObject(result) + listResult := modulebase.ListResult{} + err = result.Unmarshal(&listResult) + if err != nil { + return err + } + printList(&listResult, []string{}) return nil }) diff --git a/go.mod b/go.mod index 7068f3785d..e688b17b61 100644 --- a/go.mod +++ b/go.mod @@ -130,9 +130,9 @@ require ( yunion.io/x/executor v0.0.0-20200227030256-a18417815e74 yunion.io/x/jsonutils v0.0.0-20200303051356-aa609aba0cda yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d - yunion.io/x/pkg v0.0.0-20200312093207-f07f4cb157e0 + yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e - yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57 + yunion.io/x/sqlchemy v0.0.0-20200310015557-9e9a49139db1 yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3 ) diff --git a/go.sum b/go.sum index 7e65d09768..72f7051e85 100644 --- a/go.sum +++ b/go.sum @@ -1114,11 +1114,11 @@ yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/pkg v0.0.0-20200302034534-fdf44d54b070/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= -yunion.io/x/pkg v0.0.0-20200312093207-f07f4cb157e0 h1:ZgNhWrnxnVjw53rjx5uV5yQCYcwM8lpVHsrchmHWQI4= -yunion.io/x/pkg v0.0.0-20200312093207-f07f4cb157e0/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= +yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e h1:rBfX77+VEBVpe6Xxy2gDa4WB7qbtndWvXcrVKzleF84= +yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= -yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57 h1:KtQAuLJ00RSUVqkiRmJ1DiDABiw0U3xxXnzD3lGavaY= -yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI= +yunion.io/x/sqlchemy v0.0.0-20200310015557-9e9a49139db1 h1:CZNbvp1Ddspv97hVP4LMaNiempFvjJeNgLImF2kUPuY= +yunion.io/x/sqlchemy v0.0.0-20200310015557-9e9a49139db1/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI= yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3 h1:bfC8EhXYvyGYldRWlzxiCM39Zfj3s3+zham9mW2h2LE= yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng= diff --git a/pkg/apis/list.go b/pkg/apis/list.go index e7d7bd4314..3f17575f3f 100644 --- a/pkg/apis/list.go +++ b/pkg/apis/list.go @@ -14,14 +14,18 @@ package apis +type ScopedResourceInput struct { + // 指定查询的权限范围，可能值为project, domain or system + Scope string `json:"scope"` +} + type DomainizedResourceListInput struct { // swagger:ignore // Is an admin call? equivalent to scope=system // Deprecated Admin *bool `json:"admin"` - // 指定查询的权限范围，可能值为project, domain or system - Scope string `json:"scope"` + ScopedResourceInput DomainizedResourceInput @@ -164,10 +168,15 @@ type StandaloneResourceListInput struct { // 通过标签过滤 Tags []STag `json:"tags"` + + // 通过标签过滤 + OrderByTag string `json:"order_by_tag"` + // 返回资源的标签不包含特定的用户标签 WithoutUserMeta bool `json:"without_user_meta"` // 返回列表数据中包含资源的标签数据（Metadata） WithMeta *bool `json:"with_meta"` + // 显示所有的资源，包括模拟的资源 ShowEmulated *bool `json:"show_emulated"` diff --git a/pkg/apis/metadata.go b/pkg/apis/metadata.go new file mode 100644 index 0000000000..97ec22e75a --- /dev/null +++ b/pkg/apis/metadata.go @@ -0,0 +1,52 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apis + +type MetadataListInput struct { + ModelBaseListInput + + ScopedResourceInput + + DomainizedResourceInput + + ProjectizedResourceInput + + // 按关联资源类型过滤 + Resources []string `json:"resources"` + + // 仅显示系统标签 + SysMeta *bool `json:"sys_meta"` + + // 仅显示同步下来的标签 + CloudMeta *bool `json:"cloud_meta"` + + // 仅显示用户标签 + UserMeta *bool `json:"user_meta"` + + // 同时显示系统标签 + WithSysMeta *bool `json:"with_sys_meta"` + + // 同时显示同步下来的标签 + WithCloudMeta *bool `json:"with_cloud_meta"` + + // 同时显示用户标签 + WithUserMeta *bool `json:"with_user_meta"` + + // 按Key过滤 + Key []string `json:"key"` + + // 按Value过滤 + Value []string `json:"value"` +} diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index 375fbb36cd..bcb96ba1a4 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -268,7 +268,9 @@ func listItemQueryFiltersRaw(manager IModelManager, return nil, err } if query.Contains("export_keys") { - q, err = manager.ListItemExportKeys(ctx, q, userCred, query) + exportKeys, _ := query.GetString("export_keys") + keys := stringutils2.NewSortedStrings(strings.Split(exportKeys, ",")) + q, err = manager.ListItemExportKeys(ctx, q, userCred, keys) if err != nil { return nil, err } @@ -349,19 +351,26 @@ func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.To return nil, err } defer rows.Close() + + var exportKeys stringutils2.SSortedStrings + if query.Contains("export_keys") { + exportKeyStr, _ := query.GetString("export_keys") + exportKeys = stringutils2.NewSortedStrings(strings.Split(exportKeyStr, ",")) + } + for rows.Next() { item, err := NewModelObject(manager) if err != nil { return nil, err } - if query.Contains("export_keys") { + if exportKeys != nil { extraData := jsonutils.NewDict() RowMap, err := q.Row2Map(rows) if err != nil { return nil, err } - extraKeys := manager.GetExportExtraKeys(ctx, query, RowMap) + extraKeys := manager.GetExportExtraKeys(ctx, exportKeys, RowMap) if extraKeys != nil { extraData.Update(extraKeys) } @@ -386,7 +395,7 @@ func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.To // jsonDict := jsonutils.Marshal(item).(*jsonutils.JSONDict) // jsonDict = jsonDict.CopyIncludes([]string(listF)...) // jsonDict.Update(extraData) - // ignore GetExtraDetails + // ignore GetExtraDetails since release/3.2 /*if showDetails && !query.Contains("export_keys") { extraDict, _ := GetExtraDetails(item, ctx, userCred, query, true) if extraDict != nil { @@ -601,14 +610,6 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok if orderBy == nil { orderBy = []string{} } - if primaryCol != nil && primaryCol.IsNumeric() { - orderBy = append(orderBy, primaryCol.Name()) - } else if manager.TableSpec().ColumnSpec("created_at") != nil { - orderBy = append(orderBy, "created_at") - if primaryCol != nil { - orderBy = append(orderBy, primaryCol.Name()) - } - } for _, orderByField := range orderBy { if pagingConf != nil && utils.IsInStringArray(orderByField, pagingConf.MarkerFields) { // skip markerField in pagingConf @@ -623,6 +624,17 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok } } } + if primaryCol != nil && primaryCol.IsNumeric() { + orderBy = append(orderBy, primaryCol.Name()) + } else if manager.TableSpec().ColumnSpec("created_at") != nil { + orderBy = append(orderBy, "created_at") + if manager.TableSpec().ColumnSpec("name") != nil { + orderBy = append(orderBy, "name") + } + if primaryCol != nil { + orderBy = append(orderBy, primaryCol.Name()) + } + } if pagingConf != nil { q = q.Limit(int(limit) + 1) @@ -826,12 +838,11 @@ func (dispatcher *DBModelDispatcher) tryGetModelProperty(ctx context.Context, pr return nil, nil } userCred := fetchUserCredential(ctx) - params := []reflect.Value{ - reflect.ValueOf(ctx), - reflect.ValueOf(userCred), - reflect.ValueOf(query), + params := []interface{}{ctx, userCred, query} + outs, err := callFunc(funcValue, params...) + if err != nil { + return nil, httperrors.NewInternalServerError("reflect call %s fail %s", allowFuncName, err) } - outs := funcValue.Call(params) if len(outs) != 1 { return nil, httperrors.NewInternalServerError("Invald %s return value", funcName) } @@ -840,12 +851,15 @@ func (dispatcher *DBModelDispatcher) tryGetModelProperty(ctx context.Context, pr } funcValue = modelValue.MethodByName(funcName) - outs = funcValue.Call(params) + outs, err = callFunc(funcValue, params...) + if err != nil { + return nil, httperrors.NewInternalServerError("reflect call %s fail %s", funcName, err) + } if len(outs) != 2 { return nil, httperrors.NewInternalServerError("Invald %s return value", funcName) } - resVal := outs[0].Interface() + resVal := outs[0] errVal := outs[1].Interface() if !gotypes.IsNil(errVal) { return nil, errVal.(error) @@ -853,7 +867,7 @@ func (dispatcher *DBModelDispatcher) tryGetModelProperty(ctx context.Context, pr if gotypes.IsNil(resVal) { return nil, httperrors.NewBadRequestError("No return value, so why query?") } else { - return resVal.(jsonutils.JSONObject), nil + return ValueToJSONObject(resVal), nil } } } diff --git a/pkg/cloudcommon/db/interface.go b/pkg/cloudcommon/db/interface.go index e6f3feb808..05b713b78a 100644 --- a/pkg/cloudcommon/db/interface.go +++ b/pkg/cloudcommon/db/interface.go @@ -27,6 +27,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/object" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/rbacutils" + "yunion.io/x/onecloud/pkg/util/stringutils2" ) type IModelManager interface { @@ -56,8 +57,8 @@ type IModelManager interface { // ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) CustomizeFilterList(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*CustomizeListFilters, error) ExtraSearchConditions(ctx context.Context, q *sqlchemy.SQuery, like string) []sqlchemy.ICondition - GetExportExtraKeys(ctx context.Context, query jsonutils.JSONObject, rowMap map[string]string) *jsonutils.JSONDict - ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) + GetExportExtraKeys(ctx context.Context, keys stringutils2.SSortedStrings, rowMap map[string]string) *jsonutils.JSONDict + ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, keys stringutils2.SSortedStrings) (*sqlchemy.SQuery, error) // OrderByExtraFields dynmically called by dispatcher // OrderByExtraFields(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) diff --git a/pkg/cloudcommon/db/metadata.go b/pkg/cloudcommon/db/metadata.go index 2fc5c23191..83c088a79a 100644 --- a/pkg/cloudcommon/db/metadata.go +++ b/pkg/cloudcommon/db/metadata.go @@ -23,14 +23,18 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/stringutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/cloudcommon/consts" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/policy" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modulebase" "yunion.io/x/onecloud/pkg/util/rbacutils" ) @@ -44,6 +48,8 @@ const ( // TAG_DELETE_RANGE_CLOUD = CLOUD_TAG_PREFIX // "cloud" TAG_DELETE_RANGE_ALL = "all" + + OBJECT_TYPE_ID_SEP = "::" ) type SMetadataManager struct { @@ -53,11 +59,31 @@ type SMetadataManager struct { type SMetadata struct { SModelBase - Id string `width:"128" charset:"ascii" primary:"true" list:"user" get:"user"` // = Column(VARCHAR(128, charset='ascii'), primary_key=True) - Key string `width:"64" charset:"utf8" primary:"true" list:"user" get:"user"` // = Column(VARCHAR(64, charset='ascii'), primary_key=True) - Value string `charset:"utf8" list:"user" get:"user"` // = Column(TEXT(charset='utf8'), nullable=True) - UpdatedAt time.Time `nullable:"false" updated_at:"true"` // = Column(DateTime, default=get_utcnow, nullable=False, onupdate=get_utcnow) - Deleted bool `nullable:"false" default:"false" index:"true"` + // 资源类型 + // example: network + ObjType string `width:"40" charset:"ascii" index:"true" list:"user" get:"user"` + + // 资源ID + // example: 87321a70-1ecb-422a-8b0c-c9aa632a46a7 + ObjId string `width:"88" charset:"ascii" index:"true" list:"user" get:"user"` + + // 资源组合ID + // example: network::87321a70-1ecb-422a-8b0c-c9aa632a46a7 + Id string `width:"128" charset:"ascii" primary:"true" list:"user" get:"user"` + + // 标签KEY + // exmaple: 部门 + Key string `width:"64" charset:"utf8" primary:"true" list:"user" get:"user"` + + // 标签值 + // example: 技术部 + Value string `charset:"utf8" list:"user" get:"user"` + + // 更新时间 + UpdatedAt time.Time `nullable:"false" updated_at:"true"` + + // 是否被删除 + Deleted bool `nullable:"false" default:"false" index:"true"` } var Metadata *SMetadataManager @@ -74,6 +100,35 @@ func init() { Metadata.SetVirtualObject(Metadata) } +func (manager *SMetadataManager) InitializeData() error { + q := manager.RawQuery() + q = q.Filter(sqlchemy.OR( + sqlchemy.IsNullOrEmpty(q.Field("obj_type")), + sqlchemy.IsNullOrEmpty(q.Field("obj_id")), + )) + mds := make([]SMetadata, 0) + err := FetchModelObjects(manager, q, &mds) + if err != nil && err != sql.ErrNoRows { + return errors.Wrap(err, "FetchModelObjects") + } + for i := range mds { + _, err := Update(&mds[i], func() error { + parts := strings.Split(mds[i].Id, OBJECT_TYPE_ID_SEP) + if len(parts) == 2 { + mds[i].ObjType = parts[0] + mds[i].ObjId = parts[1] + return nil + } else { + return errors.Wrapf(httperrors.ErrInvalidFormat, "invlid id format %s", mds[i].Id) + } + }) + if err != nil { + return errors.Wrap(err, "update") + } + } + return nil +} + func (m *SMetadata) GetId() string { return fmt.Sprintf("%s-%s", m.Id, m.Key) } @@ -87,7 +142,7 @@ func (m *SMetadata) GetModelManager() IModelManager { } func GetObjectIdstr(model IModel) string { - return fmt.Sprintf("%s::%s", model.GetModelManager().Keyword(), model.GetId()) + return fmt.Sprintf("%s%s%s", model.GetModelManager().Keyword(), OBJECT_TYPE_ID_SEP, model.GetId()) } func (manager *SMetadataManager) Query(fields ...string) *sqlchemy.SQuery { @@ -107,75 +162,122 @@ func (m *SMetadata) Delete(ctx context.Context, userCred mcclient.TokenCredentia return DeleteModel(ctx, userCred, m) } -func (manager *SMetadataManager) AllowGetPropertyTagValuePairs(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { +func (manager *SMetadataManager) AllowGetPropertyTagValuePairs(ctx context.Context, userCred mcclient.TokenCredential, input apis.MetadataListInput) bool { return true } -func (manager *SMetadataManager) GetPropertyTagValuePairs(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { - q := manager.Query("id", "key", "value") - if key, _ := query.GetString("key"); len(key) > 0 { - q = q.Equals("key", key) - } - if value, _ := query.GetString("value"); len(value) > 0 { - q = q.Equals("value", value) - } - sql, err := manager.ListItemFilter(ctx, q, userCred, query) +func (manager *SMetadataManager) GetPropertyTagValuePairs( + ctx context.Context, + userCred mcclient.TokenCredential, + input apis.MetadataListInput, +) (*modulebase.ListResult, error) { + var err error + q := manager.Query("key", "value") + + q, err = manager.ListItemFilter(ctx, q, userCred, input) if err != nil { - return nil, err + return nil, errors.Wrap(err, "ListItemFilter") } - sql = sql.Asc("key").Asc("value") - metadatas := []struct { - Id string + + q = q.GroupBy(q.Field("key"), q.Field("value")) + if input.Order == string(sqlchemy.SQL_ORDER_DESC) { + q = q.Desc(q.Field("key")).Desc(q.Field("value")) + } else { + q = q.Asc(q.Field("key")).Asc(q.Field("value")) + } + + totalCnt, err := q.CountWithError() + if err != nil { + return nil, errors.Wrap(err, "CountWithError") + } + + if totalCnt == 0 { + emptyList := modulebase.ListResult{Data: []jsonutils.JSONObject{}} + return &emptyList, nil + } + + var maxLimit int64 = consts.GetMaxPagingLimit() + limit := consts.GetDefaultPagingLimit() + if input.Limit != nil { + limit = int64(*input.Limit) + } + offset := int64(0) + if input.Offset != nil { + offset = int64(*input.Offset) + } + if offset < 0 { + offset = int64(totalCnt) + offset + } + if offset < 0 { + offset = 0 + } + if int64(totalCnt) > maxLimit && (limit <= 0 || limit > maxLimit) { + limit = maxLimit + } + if limit > 0 { + q = q.Limit(int(limit)) + } + if offset > 0 { + q = q.Offset(int(offset)) + } + + data, err := manager.metaDataQuery2List(q) + if err != nil { + return nil, errors.Wrap(err, "metadataQuery2List") + } + emptyList := modulebase.ListResult{ + Data: data, + Total: totalCnt, + Limit: int(limit), + Offset: int(offset), + } + return &emptyList, nil +} + +func (manager *SMetadataManager) metaDataQuery2List(q *sqlchemy.SQuery) ([]jsonutils.JSONObject, error) { + metadatas := make([]struct { Key string Value string - }{} - err = sql.All(&metadatas) + }, 0) + err := q.All(&metadatas) if err != nil { - return nil, err + return nil, errors.Wrap(err, "Query.All") } - result := &struct { - Total int - Data []*jsonutils.JSONDict `json:"data,allowempty"` - }{ - Total: 0, - Data: []*jsonutils.JSONDict{}, - } - - statistics := map[string]map[string]map[string][]string{} //map[key][value][resourcetype][]string{ids} - for _, metadata := range metadatas { - if _, ok := statistics[metadata.Key]; !ok { - statistics[metadata.Key] = map[string]map[string][]string{} - } - if _, ok := statistics[metadata.Key][metadata.Value]; !ok { - statistics[metadata.Key][metadata.Value] = map[string][]string{} - } - if resourceInfo := strings.Split(metadata.Id, "::"); len(resourceInfo) == 2 { - resourceType, resourceId := resourceInfo[0], resourceInfo[1] - if _, ok := statistics[metadata.Key][metadata.Value][resourceType]; !ok { - statistics[metadata.Key][metadata.Value][resourceType] = []string{} - } - statistics[metadata.Key][metadata.Value][resourceType] = append(statistics[metadata.Key][metadata.Value][resourceType], resourceId) + ret := make([]jsonutils.JSONObject, len(metadatas)) + for i := range metadatas { + ret[i], err = manager.getKeyValueObjectCount(metadatas[i].Key, metadatas[i].Value) + if err != nil { + return nil, errors.Wrap(err, "getKeyValueObjectCount") } } - for key, v := range statistics { - for value, info := range v { - data := jsonutils.NewDict() - data.Add(jsonutils.NewString(key), "key") - data.Add(jsonutils.NewString(value), "value") - count := 0 - for resourceType, ids := range info { - count += len(ids) - data.Add(jsonutils.NewInt(int64(len(ids))), fmt.Sprintf("%s_count", resourceType)) - } - if count > 0 { - data.Add(jsonutils.NewInt(int64(count)), "count") - result.Data = append(result.Data, data) - } - } + return ret, nil +} + +func (manager *SMetadataManager) getKeyValueObjectCount(key string, value string) (jsonutils.JSONObject, error) { + metadatas := manager.Query().SubQuery() + q := metadatas.Query(metadatas.Field("obj_type"), sqlchemy.COUNT("obj_count")) + q = q.Equals("key", key).Equals("value", value) + q = q.GroupBy("key", "value", "obj_type") + + objectCount := make([]struct { + ObjType string + ObjCount int64 + }, 0) + err := q.All(&objectCount) + if err != nil { + return nil, errors.Wrap(err, "query.All") } - return jsonutils.Marshal(result), nil + + data := jsonutils.NewDict() + data.Add(jsonutils.NewString(key), "key") + data.Add(jsonutils.NewString(value), "value") + for _, oc := range objectCount { + data.Add(jsonutils.NewInt(oc.ObjCount), fmt.Sprintf("%s_count", oc.ObjType)) + } + + return data, nil } func (manager *SMetadataManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { @@ -183,8 +285,27 @@ func (manager *SMetadataManager) AllowListItems(ctx context.Context, userCred mc } // 元数据(标签)列表 -func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { - resources := jsonutils.GetQueryStringArray(query, "resources") +func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, input apis.MetadataListInput) (*sqlchemy.SQuery, error) { + var err error + q, err = manager.SModelBaseManager.ListItemFilter(ctx, q, userCred, input.ModelBaseListInput) + if err != nil { + return nil, errors.Wrap(err, "SModelBaseManager.ListItemFilter") + } + + if len(input.Key) > 0 { + q = q.In("key", input.Key) + } + if len(input.Value) > 0 { + q = q.In("value", input.Value) + } + if len(input.Search) > 0 { + q = q.Filter(sqlchemy.OR( + sqlchemy.Contains(q.Field("key"), input.Search), + sqlchemy.Contains(q.Field("value"), input.Search), + )) + } + + resources := input.Resources if len(resources) == 0 { for resource := range globalTables { resources = append(resources, resource) @@ -200,9 +321,10 @@ func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy continue } resourceView := man.Query().SubQuery() - prefix := sqlchemy.NewStringField(fmt.Sprintf("%s::", man.Keyword())) + prefix := sqlchemy.NewStringField(fmt.Sprintf("%s%s", man.Keyword(), OBJECT_TYPE_ID_SEP)) field := sqlchemy.CONCAT(man.Keyword(), prefix, resourceView.Field("id")) sq := resourceView.Query(field) + query := jsonutils.Marshal(input) ownerId, queryScope, err := FetchCheckQueryOwnerScope(ctx, userCred, query, man, policy.PolicyActionList, true) if err != nil { log.Warningf("FetchCheckQueryOwnerScope.%s error: %v", man.Keyword(), err) @@ -216,18 +338,39 @@ func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy if len(conditions) > 0 { q = q.Filter(sqlchemy.OR(conditions...)) } - for args, prefix := range map[string]string{"sys_meta": SYS_TAG_PREFIX, "cloud_meta": CLOUD_TAG_PREFIX, "user_meta": USER_TAG_PREFIX} { + + if input.SysMeta != nil && *input.SysMeta { + q = q.Filter(sqlchemy.Startswith(q.Field("key"), SYS_TAG_PREFIX)) + } + if input.CloudMeta != nil && *input.CloudMeta { + q = q.Filter(sqlchemy.Startswith(q.Field("key"), CLOUD_TAG_PREFIX)) + } + if input.UserMeta != nil && *input.UserMeta { + q = q.Filter(sqlchemy.Startswith(q.Field("key"), USER_TAG_PREFIX)) + } + + /*for args, prefix := range map[string]string{"sys_meta": SYS_TAG_PREFIX, "cloud_meta": CLOUD_TAG_PREFIX, "user_meta": USER_TAG_PREFIX} { if jsonutils.QueryBoolean(query, args, false) { q = q.Filter(sqlchemy.Startswith(q.Field("key"), prefix)) } - } + }*/ withConditions := []sqlchemy.ICondition{} - for args, prefix := range map[string]string{"with_sys_meta": SYS_TAG_PREFIX, "with_cloud_meta": CLOUD_TAG_PREFIX, "with_user_meta": USER_TAG_PREFIX} { + if input.WithSysMeta != nil && *input.WithSysMeta { + withConditions = append(withConditions, sqlchemy.Startswith(q.Field("key"), SYS_TAG_PREFIX)) + } + if input.WithCloudMeta != nil && *input.WithCloudMeta { + withConditions = append(withConditions, sqlchemy.Startswith(q.Field("key"), CLOUD_TAG_PREFIX)) + } + if input.WithUserMeta != nil && *input.WithUserMeta { + withConditions = append(withConditions, sqlchemy.Startswith(q.Field("key"), USER_TAG_PREFIX)) + } + + /*for args, prefix := range map[string]string{"with_sys_meta": SYS_TAG_PREFIX, "with_cloud_meta": CLOUD_TAG_PREFIX, "with_user_meta": USER_TAG_PREFIX} { if jsonutils.QueryBoolean(query, args, false) { withConditions = append(withConditions, sqlchemy.Startswith(q.Field("key"), prefix)) } - } + }*/ if len(withConditions) > 0 { q = q.Filter(sqlchemy.OR(withConditions...)) @@ -316,50 +459,62 @@ func (manager *SMetadataManager) SetValuesWithLog(ctx context.Context, obj IMode func (manager *SMetadataManager) SetValues(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential) ([]sMetadataChange, error) { idStr := GetObjectIdstr(obj) - lockman.LockObject(ctx, obj) - defer lockman.ReleaseObject(ctx, obj) + // no need to lock + // lockman.LockObject(ctx, obj) + // defer lockman.ReleaseObject(ctx, obj) changes := make([]sMetadataChange, 0) for key, value := range store { + record := SMetadata{} + + err := manager.RawQuery().Equals("id", idStr).Equals("key", key).First(&record) //避免之前设置的tag被删除后再次设置时出现Duplicate entry error + if err != nil { + if errors.Cause(err) != sql.ErrNoRows { + return changes, errors.Wrap(err, "RawQuery") + } else { + record.Deleted = true + } + } + + newRecord := SMetadata{} + + newRecord.ObjId = obj.GetId() + newRecord.ObjType = obj.GetModelManager().Keyword() + newRecord.Id = idStr + newRecord.Key = key + valStr := stringutils.Interface2String(value) valStrLower := strings.ToLower(valStr) if valStrLower == "none" || valStrLower == "null" { - valStr = "" + newRecord.Value = record.Value + newRecord.Deleted = true + } else { + newRecord.Value = valStr + newRecord.Deleted = false } - record := SMetadata{} - err := manager.RawQuery().Equals("id", idStr).Equals("key", key).First(&record) //避免之前设置的tag被删除后再次设置时出现Duplicate entry error + + if record.Deleted == newRecord.Deleted && record.Value == newRecord.Value { + // no changes + continue + } + + err = manager.TableSpec().InsertOrUpdate(&newRecord) if err != nil { - if err == sql.ErrNoRows { + return nil, errors.Wrapf(err, "InsertOrUpdate %s=%s", key, valStr) + } + + if record.Deleted != newRecord.Deleted { + if record.Deleted { + // create changes = append(changes, sMetadataChange{Key: key, NValue: valStr}) - record.Id = idStr - record.Key = key - record.Value = valStr - err = manager.TableSpec().Insert(&record) - if err != nil { - return nil, err - } } else { - return nil, err + // delete + changes = append(changes, sMetadataChange{Key: key, OValue: record.Value}) } } else { - deleted := record.Deleted - oValue := record.Value - _, err := Update(&record, func() error { - record.Deleted = false - record.Value = valStr - return nil - }) - if err != nil { - return nil, err - } - if deleted { - changes = append(changes, sMetadataChange{Key: key, NValue: valStr}) - } else { - if oValue != valStr { - changes = append(changes, sMetadataChange{Key: key, OValue: oValue, NValue: valStr}) - } - } + // change + changes = append(changes, sMetadataChange{Key: key, OValue: record.Value, NValue: valStr}) } } return changes, nil diff --git a/pkg/cloudcommon/db/modelbase.go b/pkg/cloudcommon/db/modelbase.go index c25191e85b..f00240b540 100644 --- a/pkg/cloudcommon/db/modelbase.go +++ b/pkg/cloudcommon/db/modelbase.go @@ -241,12 +241,12 @@ func (manager *SModelBaseManager) InitializeData() error { return nil } -func (manager *SModelBaseManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { +func (manager *SModelBaseManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, keys stringutils2.SSortedStrings) (*sqlchemy.SQuery, error) { q = q.AppendField(q.QueryFields()...) return q, nil } -func (manager *SModelBaseManager) GetExportExtraKeys(ctx context.Context, query jsonutils.JSONObject, rowMap map[string]string) *jsonutils.JSONDict { +func (manager *SModelBaseManager) GetExportExtraKeys(ctx context.Context, keys stringutils2.SSortedStrings, rowMap map[string]string) *jsonutils.JSONDict { return jsonutils.NewDict() } diff --git a/pkg/cloudcommon/db/orderby.go b/pkg/cloudcommon/db/orderby.go index 0a5f9cecdc..425e486ff6 100644 --- a/pkg/cloudcommon/db/orderby.go +++ b/pkg/cloudcommon/db/orderby.go @@ -20,7 +20,6 @@ func NeedOrderQuery(fieldOrders []string) bool { for _, field := range fieldOrders { if sqlchemy.SQL_ORDER_ASC.Equals(field) || sqlchemy.SQL_ORDER_DESC.Equals(field) { return true - break } } return false diff --git a/pkg/cloudcommon/db/standalone.go b/pkg/cloudcommon/db/standalone.go index 0a96506f45..3047258eae 100644 --- a/pkg/cloudcommon/db/standalone.go +++ b/pkg/cloudcommon/db/standalone.go @@ -16,7 +16,6 @@ package db import ( "context" - "fmt" "strings" "yunion.io/x/jsonutils" @@ -200,7 +199,8 @@ func (manager *SStandaloneResourceBaseManager) ListItemFilter( } if len(tags) > 0 { - metadataView := Metadata.Query() + metadataResQ := Metadata.Query().Equals("obj_type", manager.Keyword()).SubQuery() + metadataView := metadataResQ.Query() idx := 0 for key, values := range tags { if idx == 0 { @@ -209,7 +209,7 @@ func (manager *SStandaloneResourceBaseManager) ListItemFilter( metadataView = metadataView.In("value", values) } } else { - subMetataView := Metadata.Query().Equals("key", key) + subMetataView := metadataResQ.Query().Equals("key", key) if len(values) > 0 { subMetataView = subMetataView.In("value", values) } @@ -219,22 +219,13 @@ func (manager *SStandaloneResourceBaseManager) ListItemFilter( idx++ } metadatas := metadataView.SubQuery() - fieldName := fmt.Sprintf("%s_id", manager.Keyword()) - metadataSQ := metadatas.Query( - sqlchemy.REPLACE(fieldName, metadatas.Field("id"), manager.Keyword()+"::", ""), - ) - sq := metadataSQ.Filter(sqlchemy.Like(metadatas.Field("id"), manager.Keyword()+"::%")).Distinct() + sq := metadatas.Query(metadatas.Field("obj_id")).Distinct().SubQuery() q = q.Filter(sqlchemy.In(q.Field("id"), sq)) } if input.WithoutUserMeta { - metadatas := Metadata.Query().SubQuery() - fieldName := fmt.Sprintf("%s_id", manager.Keyword()) - metadataSQ := metadatas.Query( - sqlchemy.REPLACE(fieldName, metadatas.Field("id"), manager.Keyword()+"::", ""), - ) - sq := metadataSQ.Filter(sqlchemy.Like(metadatas.Field("key"), USER_TAG_PREFIX+"%")).Distinct() - + metadatas := Metadata.Query().Equals("obj_type", manager.Keyword()).SubQuery() + sq := metadatas.Query(metadatas.Field("obj_id")).Startswith("key", USER_TAG_PREFIX).Distinct().SubQuery() q.Filter(sqlchemy.NotIn(q.Field("id"), sq)) } @@ -242,6 +233,14 @@ func (manager *SStandaloneResourceBaseManager) ListItemFilter( } func (manager *SStandaloneResourceBaseManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) { + if strings.HasPrefix(field, "tag:") { + tagKey := field[4:] + metaQ := Metadata.Query("obj_id", "value").Equals("obj_type", manager.Keyword()).Equals("key", tagKey).SubQuery() + q = q.AppendField(metaQ.Field("value", field)).Distinct() + q = q.LeftJoin(metaQ, sqlchemy.Equals(q.Field("id"), metaQ.Field("obj_id"))) + q = q.Asc(metaQ.Field("value")) + return q, nil + } q, err := manager.SResourceBaseManager.QueryDistinctExtraField(q, field) if err == nil { return q, nil @@ -259,6 +258,25 @@ func (manager *SStandaloneResourceBaseManager) OrderByExtraFields( if err != nil { return nil, errors.Wrap(err, "SResourceBaseManager.OrderByExtraFields") } + + if len(input.OrderByTag) > 0 { + order := sqlchemy.SQL_ORDER_ASC + tagKey := input.OrderByTag + if stringutils2.HasSuffixIgnoreCase(input.OrderByTag, string(sqlchemy.SQL_ORDER_ASC)) { + tagKey = tagKey[0 : len(tagKey)-len(sqlchemy.SQL_ORDER_ASC)-1] + } else if stringutils2.HasSuffixIgnoreCase(input.OrderByTag, string(sqlchemy.SQL_ORDER_DESC)) { + tagKey = tagKey[0 : len(tagKey)-len(sqlchemy.SQL_ORDER_DESC)-1] + order = sqlchemy.SQL_ORDER_DESC + } + metaQ := Metadata.Query("obj_id", "value").Equals("obj_type", manager.Keyword()).Equals("key", tagKey).SubQuery() + q = q.LeftJoin(metaQ, sqlchemy.Equals(q.Field("id"), metaQ.Field("obj_id"))) + if order == sqlchemy.SQL_ORDER_ASC { + q = q.Asc(metaQ.Field("value")) + } else { + q = q.Desc(metaQ.Field("value")) + } + } + return q, nil } @@ -519,3 +537,38 @@ func (manager *SStandaloneResourceBaseManager) FetchCustomizeColumns( func (manager *SStandaloneResourceBaseManager) GetMetadataHiddenKeys() []string { return nil } + +const ( + TAG_EXPORT_KEY_PREFIX = "tag:" +) + +func (manager *SStandaloneResourceBaseManager) GetExportExtraKeys(ctx context.Context, keys stringutils2.SSortedStrings, rowMap map[string]string) *jsonutils.JSONDict { + res := manager.SResourceBaseManager.GetExportExtraKeys(ctx, keys, rowMap) + + for _, key := range keys { + if strings.HasPrefix(key, TAG_EXPORT_KEY_PREFIX) { + res.Add(jsonutils.NewString(rowMap[key]), key) + } + } + + return res +} + +func (manager *SStandaloneResourceBaseManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, keys stringutils2.SSortedStrings) (*sqlchemy.SQuery, error) { + var err error + q, err = manager.SResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys) + if err != nil { + return nil, errors.Wrap(err, "SResourceBaseManager.ListItemExportKeys") + } + + for _, key := range keys { + if strings.HasPrefix(key, TAG_EXPORT_KEY_PREFIX) { + tagKey := key[len(TAG_EXPORT_KEY_PREFIX):] + metaQ := Metadata.Query("obj_id", "value").Equals("obj_type", manager.Keyword()).Equals("key", tagKey).SubQuery() + q = q.LeftJoin(metaQ, sqlchemy.Equals(q.Field("id"), metaQ.Field("obj_id"))) + q = q.AppendField(metaQ.Field("value", key)) + } + } + + return q, nil +} diff --git a/pkg/cloudevent/models/initdb.go b/pkg/cloudevent/models/initdb.go index ddc91143fd..00913c3c73 100644 --- a/pkg/cloudevent/models/initdb.go +++ b/pkg/cloudevent/models/initdb.go @@ -27,6 +27,7 @@ func InitDB() error { * initialization order matters, do not change the order */ db.TenantCacheManager, + db.Metadata, CloudproviderManager, CloudeventManager, diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 80524d2933..d888e55d3e 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -1701,17 +1701,17 @@ func (self *SGuest) GetExtraDetails(ctx context.Context, userCred mcclient.Token return api.ServerDetails{}, nil } -func (manager *SGuestManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { +func (manager *SGuestManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, keys stringutils2.SSortedStrings) (*sqlchemy.SQuery, error) { var err error - q, err = manager.SVirtualResourceBaseManager.ListItemExportKeys(ctx, q, userCred, query) + q, err = manager.SVirtualResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys) if err != nil { return nil, err } - exportKeys, _ := query.GetString("export_keys") - keys := strings.Split(exportKeys, ",") + // exportKeys, _ := query.GetString("export_keys") + // keys := strings.Split(exportKeys, ",") // guest_id as filter key - if utils.IsInStringArray("ips", keys) { + if keys.Contains("ips") { guestIpsQuery := GuestnetworkManager.Query("guest_id").GroupBy("guest_id") guestIpsQuery.AppendField(sqlchemy.GROUP_CONCAT("concat_ip_addr", guestIpsQuery.Field("ip_addr"))) ipsSubQuery := guestIpsQuery.SubQuery() @@ -1719,7 +1719,7 @@ func (manager *SGuestManager) ListItemExportKeys(ctx context.Context, q *sqlchem q.AppendField(ipsSubQuery.Field("concat_ip_addr")) } - if utils.IsInStringArray("user_tags", keys) { + if keys.Contains("user_tags") { guestUserTagsQuery := db.Metadata.Query().Startswith("id", "server::"). Startswith("key", db.USER_TAG_PREFIX).GroupBy("id") guestUserTagsQuery.AppendField(sqlchemy.SubStr("guest_id", guestUserTagsQuery.Field("id"), len("server::")+1, 0)) @@ -1734,7 +1734,7 @@ func (manager *SGuestManager) ListItemExportKeys(ctx context.Context, q *sqlchem q.AppendField(subQ.Field("user_tags")) } - if utils.IsInStringArray("disk", keys) { + if keys.Contains("disk") { guestDisksQuery := GuestdiskManager.Query("guest_id", "disk_id").GroupBy("guest_id") diskQuery := DiskManager.Query("id", "disk_size").SubQuery() guestDisksQuery.Join(diskQuery, sqlchemy.Equals(diskQuery.Field("id"), guestDisksQuery.Field("disk_id"))) @@ -1743,21 +1743,21 @@ func (manager *SGuestManager) ListItemExportKeys(ctx context.Context, q *sqlchem q.LeftJoin(guestDisksSubQuery, sqlchemy.Equals(q.Field("id"), guestDisksSubQuery.Field("guest_id"))) q.AppendField(guestDisksSubQuery.Field("disk_size")) } - if utils.IsInStringArray("eip", keys) { + if keys.Contains("eip") { eipsQuery := ElasticipManager.Query("associate_id", "ip_addr").Equals("associate_type", "server").GroupBy("associate_id") eipsSubQuery := eipsQuery.SubQuery() q.LeftJoin(eipsSubQuery, sqlchemy.Equals(q.Field("id"), eipsSubQuery.Field("associate_id"))) q.AppendField(eipsSubQuery.Field("ip_addr", "eip")) } - if utils.IsInStringArray("host", keys) { + if keys.Contains("host") { hostQuery := HostManager.Query("id", "name").GroupBy("id") hostSubQuery := hostQuery.SubQuery() q.LeftJoin(hostSubQuery, sqlchemy.Equals(q.Field("host_id"), hostSubQuery.Field("id"))) q.AppendField(hostSubQuery.Field("name", "host")) } - if utils.IsInStringArray("zone", keys) { + if keys.Contains("zone") { zoneQuery := ZoneManager.Query("id", "name").SubQuery() hostQuery := HostManager.Query("id", "zone_id").GroupBy("id") hostQuery.LeftJoin(zoneQuery, sqlchemy.Equals(hostQuery.Field("zone_id"), zoneQuery.Field("id"))) @@ -1768,7 +1768,7 @@ func (manager *SGuestManager) ListItemExportKeys(ctx context.Context, q *sqlchem } // host_id as filter key - if utils.IsInStringArray("region", keys) { + if keys.Contains("region") { zoneQuery := ZoneManager.Query("id", "cloudregion_id").SubQuery() hostQuery := HostManager.Query("id", "zone_id").GroupBy("id") cloudregionQuery := CloudregionManager.Query("id", "name").SubQuery() @@ -1780,7 +1780,8 @@ func (manager *SGuestManager) ListItemExportKeys(ctx context.Context, q *sqlchem q.LeftJoin(hostSubQuery, sqlchemy.Equals(q.Field("host_id"), hostSubQuery.Field("id"))) q.AppendField(hostSubQuery.Field("region")) } - if utils.IsInStringArray("manager", keys) { + + if keys.Contains("manager") { hostQuery := HostManager.Query("id", "manager_id").GroupBy("id") cloudProviderQuery := CloudproviderManager.Query("id", "name").SubQuery() hostQuery.LeftJoin(cloudProviderQuery, sqlchemy.Equals(hostQuery.Field("manager_id"), @@ -1793,10 +1794,10 @@ func (manager *SGuestManager) ListItemExportKeys(ctx context.Context, q *sqlchem return q, nil } -func (manager *SGuestManager) GetExportExtraKeys(ctx context.Context, query jsonutils.JSONObject, rowMap map[string]string) *jsonutils.JSONDict { - res := manager.SVirtualResourceBaseManager.GetExportExtraKeys(ctx, query, rowMap) - exportKeys, _ := query.GetString("export_keys") - keys := strings.Split(exportKeys, ",") +func (manager *SGuestManager) GetExportExtraKeys(ctx context.Context, keys stringutils2.SSortedStrings, rowMap map[string]string) *jsonutils.JSONDict { + res := manager.SVirtualResourceBaseManager.GetExportExtraKeys(ctx, keys, rowMap) + // exportKeys, _ := query.GetString("export_keys") + // keys := strings.Split(exportKeys, ",") if ips, ok := rowMap["concat_ip_addr"]; ok && len(ips) > 0 { res.Set("ips", jsonutils.NewString(ips)) } @@ -1821,7 +1822,7 @@ func (manager *SGuestManager) GetExportExtraKeys(ctx context.Context, query json if userTags, ok := rowMap["user_tags"]; ok && len(userTags) > 0 { res.Set("user_tags", jsonutils.NewString(userTags)) } - if utils.IsInStringArray("tenant", keys) { + if keys.Contains("tenant") { if projectId, ok := rowMap["tenant_id"]; ok { tenant, err := db.TenantCacheManager.FetchTenantById(ctx, projectId) if err == nil { @@ -1829,7 +1830,7 @@ func (manager *SGuestManager) GetExportExtraKeys(ctx context.Context, query json } } } - if utils.IsInStringArray("os_distribution", keys) { + if keys.Contains("os_distribution") { if osType, ok := rowMap["os_type"]; ok { res.Set("os_distribution", jsonutils.NewString(osType)) } diff --git a/pkg/compute/models/initdb.go b/pkg/compute/models/initdb.go index 2fa7b8b185..cbe65c906e 100644 --- a/pkg/compute/models/initdb.go +++ b/pkg/compute/models/initdb.go @@ -27,6 +27,7 @@ func InitDB() error { * initialization order matters, do not change the order */ db.TenantCacheManager, + db.Metadata, QuotaManager, diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index 9342d269ec..3e2e344aab 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -222,20 +222,18 @@ func (self *SIsolatedDevice) AllowDeleteItem(ctx context.Context, userCred mccli return userCred.IsSystemAdmin() } */ -func (manager *SIsolatedDeviceManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { +func (manager *SIsolatedDeviceManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, keys stringutils2.SSortedStrings) (*sqlchemy.SQuery, error) { var err error - q, err = manager.SModelBaseManager.ListItemExportKeys(ctx, q, userCred, query) + q, err = manager.SStandaloneResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys) if err != nil { return nil, err } - exportKeys, _ := query.GetString("export_keys") - keys := strings.Split(exportKeys, ",") - if utils.IsInStringArray("guest", keys) { + if keys.Contains("guest") { guestNameQuery := GuestManager.Query("name", "id").SubQuery() q.LeftJoin(guestNameQuery, sqlchemy.Equals(q.Field("guest_id"), guestNameQuery.Field("id"))) q.AppendField(guestNameQuery.Field("name", "guest")) } - if utils.IsInStringArray("host", keys) { + if keys.Contains("host") { hostNameQuery := HostManager.Query("name", "id").SubQuery() q.LeftJoin(hostNameQuery, sqlchemy.Equals(q.Field("host_id"), hostNameQuery.Field("id"))) q.AppendField(hostNameQuery.Field("name", "host")) @@ -243,8 +241,8 @@ func (manager *SIsolatedDeviceManager) ListItemExportKeys(ctx context.Context, q return q, nil } -func (manager *SIsolatedDeviceManager) GetExportExtraKeys(ctx context.Context, query jsonutils.JSONObject, rowMap map[string]string) *jsonutils.JSONDict { - res := manager.SStandaloneResourceBaseManager.GetExportExtraKeys(ctx, query, rowMap) +func (manager *SIsolatedDeviceManager) GetExportExtraKeys(ctx context.Context, keys stringutils2.SSortedStrings, rowMap map[string]string) *jsonutils.JSONDict { + res := manager.SStandaloneResourceBaseManager.GetExportExtraKeys(ctx, keys, rowMap) if guest, ok := rowMap["guest"]; ok && len(guest) > 0 { res.Set("guest", jsonutils.NewString(guest)) } diff --git a/pkg/httperrors/consts.go b/pkg/httperrors/consts.go index d151e0b5e9..d586d8138e 100644 --- a/pkg/httperrors/consts.go +++ b/pkg/httperrors/consts.go @@ -34,6 +34,7 @@ const ( ErrServerStatus = errors.Error("ServerStatusError") ErrInvalidStatus = errors.ErrInvalidStatus + ErrInvalidFormat = errors.Error("InvalidFormatError") ErrInputParameter = errors.Error("InputParameterError") ErrWeakPassword = errors.Error("WeakPasswordError") @@ -105,6 +106,7 @@ var ( ErrServerStatus: 400, ErrInvalidStatus: 400, + ErrInvalidFormat: 400, ErrInputParameter: 400, ErrWeakPassword: 400, diff --git a/pkg/image/models/initdb.go b/pkg/image/models/initdb.go index 637fbaeef1..f05caefb0c 100644 --- a/pkg/image/models/initdb.go +++ b/pkg/image/models/initdb.go @@ -27,8 +27,9 @@ func InitDB() error { * initialization order matters, do not change the order */ db.TenantCacheManager, - ImageManager, + db.Metadata, + ImageManager, QuotaManager, } { err := manager.InitializeData() diff --git a/pkg/image/service/handlers.go b/pkg/image/service/handlers.go index 122d508a53..7f795e65bd 100644 --- a/pkg/image/service/handlers.go +++ b/pkg/image/service/handlers.go @@ -46,9 +46,11 @@ func InitHandlers(app *appsrv.Application) { taskman.TaskManager, taskman.SubTaskManager, taskman.TaskObjectManager, + db.UserCacheManager, db.TenantCacheManager, db.SharedResourceManager, + models.ImageTagManager, models.ImageMemberManager, models.ImagePropertyManager, diff --git a/pkg/keystone/models/initdb.go b/pkg/keystone/models/initdb.go index e05430b2f6..e86edd782c 100644 --- a/pkg/keystone/models/initdb.go +++ b/pkg/keystone/models/initdb.go @@ -26,6 +26,8 @@ func InitDB() error { * Important!!! * initialization order matters, do not change the order */ + db.Metadata, + RegionManager, ServiceManager, EndpointManager, diff --git a/pkg/mcclient/options/base.go b/pkg/mcclient/options/base.go index 21b64d5a48..15ce40b8a6 100644 --- a/pkg/mcclient/options/base.go +++ b/pkg/mcclient/options/base.go @@ -234,6 +234,8 @@ type BaseListOptions struct { IsManaged *bool `help:"List objects managed by external providers" token:"managed" json:"is_managed"` PagingMarker string `help:"Marker for pagination" json:"paging_marker"` + + OrderByTag string `help:"Order results by tag values, composed by a tag key and order, e.g user:部门:ASC"` } func (opts *BaseListOptions) addTag(prefix, tag string, idx int, params *jsonutils.JSONDict) error { diff --git a/pkg/mcclient/options/metadata.go b/pkg/mcclient/options/metadata.go index bd10616b49..d04d01a903 100644 --- a/pkg/mcclient/options/metadata.go +++ b/pkg/mcclient/options/metadata.go @@ -16,14 +16,22 @@ package options type MetadataListOptions struct { Resources []string `help:"list of resource e.g server、disk、eip、snapshot, empty will show all metadata"` - SysMeta *bool `help:"Show sys metadata only"` - CloudMeta *bool `help:"Show cloud metadata olny"` - UserMeta *bool `help:"Show user metadata olny"` - Admin *bool `help:"Show all metadata"` + + SysMeta *bool `help:"Show sys metadata only"` + CloudMeta *bool `help:"Show cloud metadata olny"` + UserMeta *bool `help:"Show user metadata olny"` + + Admin *bool `help:"Show all metadata"` WithSysMeta *bool `help:"Show sys metadata"` WithCloudMeta *bool `help:"Show cloud metadata"` WithUserMeta *bool `help:"Show user metadata"` + + Key []string `help:"key"` + Value []string `help:"value"` + + Limit *int `help:"limit"` + Offset *int `help:"offset"` } type TagListOptions MetadataListOptions diff --git a/pkg/util/stringutils2/ignorecase.go b/pkg/util/stringutils2/ignorecase.go new file mode 100644 index 0000000000..b493cf2772 --- /dev/null +++ b/pkg/util/stringutils2/ignorecase.go @@ -0,0 +1,33 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stringutils2 + +import ( + "strings" +) + +func HasSuffixIgnoreCase(str string, suffix string) bool { + if len(str) < len(suffix) { + return false + } + return strings.EqualFold(str[len(str)-len(suffix):len(str)], suffix) +} + +func HasPrefixIgnoreCase(str string, prefix string) bool { + if len(str) < len(prefix) { + return false + } + return strings.EqualFold(str[0:len(prefix)], prefix) +} diff --git a/pkg/util/stringutils2/ignorecase_test.go b/pkg/util/stringutils2/ignorecase_test.go new file mode 100644 index 0000000000..f98f6c3fc7 --- /dev/null +++ b/pkg/util/stringutils2/ignorecase_test.go @@ -0,0 +1,81 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stringutils2 + +import ( + "testing" +) + +func TestHasSufffixIgnoreCase(t *testing.T) { + cases := []struct { + Hystack string + Needle string + Want bool + }{ + { + Hystack: "Hyperscacle:ASC", + Needle: ":ASC", + Want: true, + }, + { + Hystack: "Hyperscacle:ASC", + Needle: ":asc", + Want: true, + }, + { + Hystack: "Hyperscacle:ASC", + Needle: "ASCD", + Want: false, + }, + } + + for _, c := range cases { + got := HasSuffixIgnoreCase(c.Hystack, c.Needle) + if got != c.Want { + t.Errorf("HasSuffixIgnoreCase %s %s want %v got %v", c.Hystack, c.Needle, c.Want, got) + } + } +} + +func TestHasPrefixIgnoreCase(t *testing.T) { + cases := []struct { + Hystack string + Needle string + Want bool + }{ + { + Hystack: "Hyperscacle:ASC", + Needle: "Hyper", + Want: true, + }, + { + Hystack: "Hyperscacle:ASC", + Needle: "HYPER", + Want: true, + }, + { + Hystack: "Hyperscacle:ASC", + Needle: "ASCD", + Want: false, + }, + } + + for _, c := range cases { + got := HasPrefixIgnoreCase(c.Hystack, c.Needle) + if got != c.Want { + t.Errorf("HasPrefixIgnoreCase %s %s want %v got %v", c.Hystack, c.Needle, c.Want, got) + } + } +} diff --git a/pkg/util/stringutils2/sortedstrings.go b/pkg/util/stringutils2/sortedstrings.go index 3c9611f8cc..9c7954ace4 100644 --- a/pkg/util/stringutils2/sortedstrings.go +++ b/pkg/util/stringutils2/sortedstrings.go @@ -22,7 +22,7 @@ type SSortedStrings []string func NewSortedStrings(strs []string) SSortedStrings { if strs == nil { - return nil + return SSortedStrings{} } sort.Strings(strs) return SSortedStrings(strs) diff --git a/vendor/modules.txt b/vendor/modules.txt index a421272fd8..e5712bf2d6 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1018,7 +1018,7 @@ yunion.io/x/jsonutils # yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d yunion.io/x/log yunion.io/x/log/hooks -# yunion.io/x/pkg v0.0.0-20200312093207-f07f4cb157e0 +# yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e yunion.io/x/pkg/errors yunion.io/x/pkg/gotypes yunion.io/x/pkg/prettytable @@ -1050,7 +1050,7 @@ yunion.io/x/pkg/util/workqueue yunion.io/x/pkg/utils # yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e yunion.io/x/s3cli -# yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57 +# yunion.io/x/sqlchemy v0.0.0-20200310015557-9e9a49139db1 yunion.io/x/sqlchemy # yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3 yunion.io/x/structarg