Merge pull request #5402 from swordqiu/feature/qj-metadata-cleanup

Feature/qj metadata cleanup

cmd/climc/shell/distinctfield.go

@@ -0,0 +1,48 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shell
+
+import (
+	"fmt"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
+)
+
+func init() {
+	type DistinctFieldOption struct {
+		MODULE string `help:"module name"`
+		FIELD  string `help:"distinct field name to query"`
+	}
+	R(&DistinctFieldOption{}, "distinct-field", "Query values of a distinct field for a module", func(s *mcclient.ClientSession, args *DistinctFieldOption) error {
+		mod, err := modulebase.GetModule(s, args.MODULE)
+		if err != nil || mod == nil {
+			if err != nil {
+				return fmt.Errorf("module %s not found %s", args.MODULE, err)
+			}
+			return fmt.Errorf("No module %s found", args.MODULE)
+		}
+		params := jsonutils.NewDict()
+		params.Add(jsonutils.NewString(args.FIELD), "extra_field")
+		result, err := mod.Get(s, "distinct-field", params)
+		if err != nil {
+			return err
+		}
+		fmt.Println(result)
+		return nil
+	})
+}

cmd/climc/shell/metadata.go

@@ -16,6 +16,7 @@ package shell
 
 import (
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
 )
@@ -43,7 +44,12 @@ func init() {
 		if err != nil {
 			return err
 		}
-		printObject(result)
+		listResult := modulebase.ListResult{}
+		err = result.Unmarshal(&listResult)
+		if err != nil {
+			return err
+		}
+		printList(&listResult, []string{})
 		return nil
 	})
 

go.mod

@@ -130,9 +130,9 @@ require (
 	yunion.io/x/executor v0.0.0-20200227030256-a18417815e74
 	yunion.io/x/jsonutils v0.0.0-20200303051356-aa609aba0cda
 	yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d
-	yunion.io/x/pkg v0.0.0-20200312093207-f07f4cb157e0
+	yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57
+	yunion.io/x/sqlchemy v0.0.0-20200310015557-9e9a49139db1
 	yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3
 )
 

go.sum

@@ -1114,11 +1114,11 @@ yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20200302034534-fdf44d54b070/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20200312093207-f07f4cb157e0 h1:ZgNhWrnxnVjw53rjx5uV5yQCYcwM8lpVHsrchmHWQI4=
-yunion.io/x/pkg v0.0.0-20200312093207-f07f4cb157e0/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e h1:rBfX77+VEBVpe6Xxy2gDa4WB7qbtndWvXcrVKzleF84=
+yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57 h1:KtQAuLJ00RSUVqkiRmJ1DiDABiw0U3xxXnzD3lGavaY=
-yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
+yunion.io/x/sqlchemy v0.0.0-20200310015557-9e9a49139db1 h1:CZNbvp1Ddspv97hVP4LMaNiempFvjJeNgLImF2kUPuY=
+yunion.io/x/sqlchemy v0.0.0-20200310015557-9e9a49139db1/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
 yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3 h1:bfC8EhXYvyGYldRWlzxiCM39Zfj3s3+zham9mW2h2LE=
 yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/apis/list.go

@@ -14,14 +14,18 @@
 
 package apis
 
+type ScopedResourceInput struct {
+	// 指定查询的权限范围，可能值为project, domain or system
+	Scope string `json:"scope"`
+}
+
 type DomainizedResourceListInput struct {
 	// swagger:ignore
 	// Is an admin call? equivalent to scope=system
 	// Deprecated
 	Admin *bool `json:"admin"`
 
-	// 指定查询的权限范围，可能值为project, domain or system
-	Scope string `json:"scope"`
+	ScopedResourceInput
 
 	DomainizedResourceInput
 
@@ -164,10 +168,15 @@ type StandaloneResourceListInput struct {
 
 	// 通过标签过滤
 	Tags []STag `json:"tags"`
+
+	// 通过标签过滤
+	OrderByTag string `json:"order_by_tag"`
+
 	// 返回资源的标签不包含特定的用户标签
 	WithoutUserMeta bool `json:"without_user_meta"`
 	// 返回列表数据中包含资源的标签数据（Metadata）
 	WithMeta *bool `json:"with_meta"`
+
 	// 显示所有的资源，包括模拟的资源
 	ShowEmulated *bool `json:"show_emulated"`
 

pkg/apis/metadata.go

@@ -0,0 +1,52 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package apis
+
+type MetadataListInput struct {
+	ModelBaseListInput
+
+	ScopedResourceInput
+
+	DomainizedResourceInput
+
+	ProjectizedResourceInput
+
+	// 按关联资源类型过滤
+	Resources []string `json:"resources"`
+
+	// 仅显示系统标签
+	SysMeta *bool `json:"sys_meta"`
+
+	// 仅显示同步下来的标签
+	CloudMeta *bool `json:"cloud_meta"`
+
+	// 仅显示用户标签
+	UserMeta *bool `json:"user_meta"`
+
+	// 同时显示系统标签
+	WithSysMeta *bool `json:"with_sys_meta"`
+
+	// 同时显示同步下来的标签
+	WithCloudMeta *bool `json:"with_cloud_meta"`
+
+	// 同时显示用户标签
+	WithUserMeta *bool `json:"with_user_meta"`
+
+	// 按Key过滤
+	Key []string `json:"key"`
+
+	// 按Value过滤
+	Value []string `json:"value"`
+}

pkg/cloudcommon/db/db_dispatcher.go

@@ -268,7 +268,9 @@ func listItemQueryFiltersRaw(manager IModelManager,
 		return nil, err
 	}
 	if query.Contains("export_keys") {
-		q, err = manager.ListItemExportKeys(ctx, q, userCred, query)
+		exportKeys, _ := query.GetString("export_keys")
+		keys := stringutils2.NewSortedStrings(strings.Split(exportKeys, ","))
+		q, err = manager.ListItemExportKeys(ctx, q, userCred, keys)
 		if err != nil {
 			return nil, err
 		}
@@ -349,19 +351,26 @@ func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.To
 		return nil, err
 	}
 	defer rows.Close()
+
+	var exportKeys stringutils2.SSortedStrings
+	if query.Contains("export_keys") {
+		exportKeyStr, _ := query.GetString("export_keys")
+		exportKeys = stringutils2.NewSortedStrings(strings.Split(exportKeyStr, ","))
+	}
+
 	for rows.Next() {
 		item, err := NewModelObject(manager)
 		if err != nil {
 			return nil, err
 		}
 
-		if query.Contains("export_keys") {
+		if exportKeys != nil {
 			extraData := jsonutils.NewDict()
 			RowMap, err := q.Row2Map(rows)
 			if err != nil {
 				return nil, err
 			}
-			extraKeys := manager.GetExportExtraKeys(ctx, query, RowMap)
+			extraKeys := manager.GetExportExtraKeys(ctx, exportKeys, RowMap)
 			if extraKeys != nil {
 				extraData.Update(extraKeys)
 			}
@@ -386,7 +395,7 @@ func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.To
 		// jsonDict := jsonutils.Marshal(item).(*jsonutils.JSONDict)
 		// jsonDict = jsonDict.CopyIncludes([]string(listF)...)
 		// jsonDict.Update(extraData)
-		// ignore GetExtraDetails
+		// ignore GetExtraDetails since release/3.2
 		/*if showDetails && !query.Contains("export_keys") {
 			extraDict, _ := GetExtraDetails(item, ctx, userCred, query, true)
 			if extraDict != nil {
@@ -601,14 +610,6 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 	if orderBy == nil {
 		orderBy = []string{}
 	}
-	if primaryCol != nil && primaryCol.IsNumeric() {
-		orderBy = append(orderBy, primaryCol.Name())
-	} else if manager.TableSpec().ColumnSpec("created_at") != nil {
-		orderBy = append(orderBy, "created_at")
-		if primaryCol != nil {
-			orderBy = append(orderBy, primaryCol.Name())
-		}
-	}
 	for _, orderByField := range orderBy {
 		if pagingConf != nil && utils.IsInStringArray(orderByField, pagingConf.MarkerFields) {
 			// skip markerField in pagingConf
@@ -623,6 +624,17 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 			}
 		}
 	}
+	if primaryCol != nil && primaryCol.IsNumeric() {
+		orderBy = append(orderBy, primaryCol.Name())
+	} else if manager.TableSpec().ColumnSpec("created_at") != nil {
+		orderBy = append(orderBy, "created_at")
+		if manager.TableSpec().ColumnSpec("name") != nil {
+			orderBy = append(orderBy, "name")
+		}
+		if primaryCol != nil {
+			orderBy = append(orderBy, primaryCol.Name())
+		}
+	}
 
 	if pagingConf != nil {
 		q = q.Limit(int(limit) + 1)
@@ -826,12 +838,11 @@ func (dispatcher *DBModelDispatcher) tryGetModelProperty(ctx context.Context, pr
 		return nil, nil
 	}
 	userCred := fetchUserCredential(ctx)
-	params := []reflect.Value{
-		reflect.ValueOf(ctx),
-		reflect.ValueOf(userCred),
-		reflect.ValueOf(query),
+	params := []interface{}{ctx, userCred, query}
+	outs, err := callFunc(funcValue, params...)
+	if err != nil {
+		return nil, httperrors.NewInternalServerError("reflect call %s fail %s", allowFuncName, err)
 	}
-	outs := funcValue.Call(params)
 	if len(outs) != 1 {
 		return nil, httperrors.NewInternalServerError("Invald %s return value", funcName)
 	}
@@ -840,12 +851,15 @@ func (dispatcher *DBModelDispatcher) tryGetModelProperty(ctx context.Context, pr
 	}
 
 	funcValue = modelValue.MethodByName(funcName)
-	outs = funcValue.Call(params)
+	outs, err = callFunc(funcValue, params...)
+	if err != nil {
+		return nil, httperrors.NewInternalServerError("reflect call %s fail %s", funcName, err)
+	}
 	if len(outs) != 2 {
 		return nil, httperrors.NewInternalServerError("Invald %s return value", funcName)
 	}
 
-	resVal := outs[0].Interface()
+	resVal := outs[0]
 	errVal := outs[1].Interface()
 	if !gotypes.IsNil(errVal) {
 		return nil, errVal.(error)
@@ -853,7 +867,7 @@ func (dispatcher *DBModelDispatcher) tryGetModelProperty(ctx context.Context, pr
 		if gotypes.IsNil(resVal) {
 			return nil, httperrors.NewBadRequestError("No return value, so why query?")
 		} else {
-			return resVal.(jsonutils.JSONObject), nil
+			return ValueToJSONObject(resVal), nil
 		}
 	}
 }

pkg/cloudcommon/db/interface.go

@@ -27,6 +27,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/object"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
+	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
 
 type IModelManager interface {
@@ -56,8 +57,8 @@ type IModelManager interface {
 	// ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error)
 	CustomizeFilterList(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*CustomizeListFilters, error)
 	ExtraSearchConditions(ctx context.Context, q *sqlchemy.SQuery, like string) []sqlchemy.ICondition
-	GetExportExtraKeys(ctx context.Context, query jsonutils.JSONObject, rowMap map[string]string) *jsonutils.JSONDict
-	ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error)
+	GetExportExtraKeys(ctx context.Context, keys stringutils2.SSortedStrings, rowMap map[string]string) *jsonutils.JSONDict
+	ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, keys stringutils2.SSortedStrings) (*sqlchemy.SQuery, error)
 	// OrderByExtraFields dynmically called by dispatcher
 	// OrderByExtraFields(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error)
 

pkg/cloudcommon/db/metadata.go

@@ -23,14 +23,18 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/stringutils"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
+	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
@@ -44,6 +48,8 @@ const (
 	// TAG_DELETE_RANGE_CLOUD = CLOUD_TAG_PREFIX // "cloud"
 
 	TAG_DELETE_RANGE_ALL = "all"
+
+	OBJECT_TYPE_ID_SEP = "::"
 )
 
 type SMetadataManager struct {
@@ -53,11 +59,31 @@ type SMetadataManager struct {
 type SMetadata struct {
 	SModelBase
 
-	Id        string    `width:"128" charset:"ascii" primary:"true" list:"user" get:"user"` // = Column(VARCHAR(128, charset='ascii'), primary_key=True)
-	Key       string    `width:"64" charset:"utf8" primary:"true" list:"user" get:"user"`   // = Column(VARCHAR(64, charset='ascii'),  primary_key=True)
-	Value     string    `charset:"utf8" list:"user" get:"user"`                             // = Column(TEXT(charset='utf8'), nullable=True)
-	UpdatedAt time.Time `nullable:"false" updated_at:"true"`                                // = Column(DateTime, default=get_utcnow, nullable=False, onupdate=get_utcnow)
-	Deleted   bool      `nullable:"false" default:"false" index:"true"`
+	// 资源类型
+	// example: network
+	ObjType string `width:"40" charset:"ascii" index:"true" list:"user" get:"user"`
+
+	// 资源ID
+	// example: 87321a70-1ecb-422a-8b0c-c9aa632a46a7
+	ObjId string `width:"88" charset:"ascii" index:"true" list:"user" get:"user"`
+
+	// 资源组合ID
+	// example: network::87321a70-1ecb-422a-8b0c-c9aa632a46a7
+	Id string `width:"128" charset:"ascii" primary:"true" list:"user" get:"user"`
+
+	// 标签KEY
+	// exmaple: 部门
+	Key string `width:"64" charset:"utf8" primary:"true" list:"user" get:"user"`
+
+	// 标签值
+	// example: 技术部
+	Value string `charset:"utf8" list:"user" get:"user"`
+
+	// 更新时间
+	UpdatedAt time.Time `nullable:"false" updated_at:"true"`
+
+	// 是否被删除
+	Deleted bool `nullable:"false" default:"false" index:"true"`
 }
 
 var Metadata *SMetadataManager
@@ -74,6 +100,35 @@ func init() {
 	Metadata.SetVirtualObject(Metadata)
 }
 
+func (manager *SMetadataManager) InitializeData() error {
+	q := manager.RawQuery()
+	q = q.Filter(sqlchemy.OR(
+		sqlchemy.IsNullOrEmpty(q.Field("obj_type")),
+		sqlchemy.IsNullOrEmpty(q.Field("obj_id")),
+	))
+	mds := make([]SMetadata, 0)
+	err := FetchModelObjects(manager, q, &mds)
+	if err != nil && err != sql.ErrNoRows {
+		return errors.Wrap(err, "FetchModelObjects")
+	}
+	for i := range mds {
+		_, err := Update(&mds[i], func() error {
+			parts := strings.Split(mds[i].Id, OBJECT_TYPE_ID_SEP)
+			if len(parts) == 2 {
+				mds[i].ObjType = parts[0]
+				mds[i].ObjId = parts[1]
+				return nil
+			} else {
+				return errors.Wrapf(httperrors.ErrInvalidFormat, "invlid id format %s", mds[i].Id)
+			}
+		})
+		if err != nil {
+			return errors.Wrap(err, "update")
+		}
+	}
+	return nil
+}
+
 func (m *SMetadata) GetId() string {
 	return fmt.Sprintf("%s-%s", m.Id, m.Key)
 }
@@ -87,7 +142,7 @@ func (m *SMetadata) GetModelManager() IModelManager {
 }
 
 func GetObjectIdstr(model IModel) string {
-	return fmt.Sprintf("%s::%s", model.GetModelManager().Keyword(), model.GetId())
+	return fmt.Sprintf("%s%s%s", model.GetModelManager().Keyword(), OBJECT_TYPE_ID_SEP, model.GetId())
 }
 
 func (manager *SMetadataManager) Query(fields ...string) *sqlchemy.SQuery {
@@ -107,75 +162,122 @@ func (m *SMetadata) Delete(ctx context.Context, userCred mcclient.TokenCredentia
 	return DeleteModel(ctx, userCred, m)
 }
 
-func (manager *SMetadataManager) AllowGetPropertyTagValuePairs(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+func (manager *SMetadataManager) AllowGetPropertyTagValuePairs(ctx context.Context, userCred mcclient.TokenCredential, input apis.MetadataListInput) bool {
 	return true
 }
 
-func (manager *SMetadataManager) GetPropertyTagValuePairs(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	q := manager.Query("id", "key", "value")
-	if key, _ := query.GetString("key"); len(key) > 0 {
-		q = q.Equals("key", key)
-	}
-	if value, _ := query.GetString("value"); len(value) > 0 {
-		q = q.Equals("value", value)
-	}
-	sql, err := manager.ListItemFilter(ctx, q, userCred, query)
+func (manager *SMetadataManager) GetPropertyTagValuePairs(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	input apis.MetadataListInput,
+) (*modulebase.ListResult, error) {
+	var err error
+	q := manager.Query("key", "value")
+
+	q, err = manager.ListItemFilter(ctx, q, userCred, input)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "ListItemFilter")
 	}
-	sql = sql.Asc("key").Asc("value")
-	metadatas := []struct {
-		Id    string
+
+	q = q.GroupBy(q.Field("key"), q.Field("value"))
+	if input.Order == string(sqlchemy.SQL_ORDER_DESC) {
+		q = q.Desc(q.Field("key")).Desc(q.Field("value"))
+	} else {
+		q = q.Asc(q.Field("key")).Asc(q.Field("value"))
+	}
+
+	totalCnt, err := q.CountWithError()
+	if err != nil {
+		return nil, errors.Wrap(err, "CountWithError")
+	}
+
+	if totalCnt == 0 {
+		emptyList := modulebase.ListResult{Data: []jsonutils.JSONObject{}}
+		return &emptyList, nil
+	}
+
+	var maxLimit int64 = consts.GetMaxPagingLimit()
+	limit := consts.GetDefaultPagingLimit()
+	if input.Limit != nil {
+		limit = int64(*input.Limit)
+	}
+	offset := int64(0)
+	if input.Offset != nil {
+		offset = int64(*input.Offset)
+	}
+	if offset < 0 {
+		offset = int64(totalCnt) + offset
+	}
+	if offset < 0 {
+		offset = 0
+	}
+	if int64(totalCnt) > maxLimit && (limit <= 0 || limit > maxLimit) {
+		limit = maxLimit
+	}
+	if limit > 0 {
+		q = q.Limit(int(limit))
+	}
+	if offset > 0 {
+		q = q.Offset(int(offset))
+	}
+
+	data, err := manager.metaDataQuery2List(q)
+	if err != nil {
+		return nil, errors.Wrap(err, "metadataQuery2List")
+	}
+	emptyList := modulebase.ListResult{
+		Data:   data,
+		Total:  totalCnt,
+		Limit:  int(limit),
+		Offset: int(offset),
+	}
+	return &emptyList, nil
+}
+
+func (manager *SMetadataManager) metaDataQuery2List(q *sqlchemy.SQuery) ([]jsonutils.JSONObject, error) {
+	metadatas := make([]struct {
 		Key   string
 		Value string
-	}{}
-	err = sql.All(&metadatas)
+	}, 0)
+	err := q.All(&metadatas)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "Query.All")
 	}
 
-	result := &struct {
-		Total int
-		Data  []*jsonutils.JSONDict `json:"data,allowempty"`
-	}{
-		Total: 0,
-		Data:  []*jsonutils.JSONDict{},
-	}
-
-	statistics := map[string]map[string]map[string][]string{} //map[key][value][resourcetype][]string{ids}
-	for _, metadata := range metadatas {
-		if _, ok := statistics[metadata.Key]; !ok {
-			statistics[metadata.Key] = map[string]map[string][]string{}
-		}
-		if _, ok := statistics[metadata.Key][metadata.Value]; !ok {
-			statistics[metadata.Key][metadata.Value] = map[string][]string{}
-		}
-		if resourceInfo := strings.Split(metadata.Id, "::"); len(resourceInfo) == 2 {
-			resourceType, resourceId := resourceInfo[0], resourceInfo[1]
-			if _, ok := statistics[metadata.Key][metadata.Value][resourceType]; !ok {
-				statistics[metadata.Key][metadata.Value][resourceType] = []string{}
-			}
-			statistics[metadata.Key][metadata.Value][resourceType] = append(statistics[metadata.Key][metadata.Value][resourceType], resourceId)
+	ret := make([]jsonutils.JSONObject, len(metadatas))
+	for i := range metadatas {
+		ret[i], err = manager.getKeyValueObjectCount(metadatas[i].Key, metadatas[i].Value)
+		if err != nil {
+			return nil, errors.Wrap(err, "getKeyValueObjectCount")
 		}
 	}
 
-	for key, v := range statistics {
-		for value, info := range v {
-			data := jsonutils.NewDict()
-			data.Add(jsonutils.NewString(key), "key")
-			data.Add(jsonutils.NewString(value), "value")
-			count := 0
-			for resourceType, ids := range info {
-				count += len(ids)
-				data.Add(jsonutils.NewInt(int64(len(ids))), fmt.Sprintf("%s_count", resourceType))
-			}
-			if count > 0 {
-				data.Add(jsonutils.NewInt(int64(count)), "count")
-				result.Data = append(result.Data, data)
-			}
-		}
+	return ret, nil
+}
+
+func (manager *SMetadataManager) getKeyValueObjectCount(key string, value string) (jsonutils.JSONObject, error) {
+	metadatas := manager.Query().SubQuery()
+	q := metadatas.Query(metadatas.Field("obj_type"), sqlchemy.COUNT("obj_count"))
+	q = q.Equals("key", key).Equals("value", value)
+	q = q.GroupBy("key", "value", "obj_type")
+
+	objectCount := make([]struct {
+		ObjType  string
+		ObjCount int64
+	}, 0)
+	err := q.All(&objectCount)
+	if err != nil {
+		return nil, errors.Wrap(err, "query.All")
 	}
-	return jsonutils.Marshal(result), nil
+
+	data := jsonutils.NewDict()
+	data.Add(jsonutils.NewString(key), "key")
+	data.Add(jsonutils.NewString(value), "value")
+	for _, oc := range objectCount {
+		data.Add(jsonutils.NewInt(oc.ObjCount), fmt.Sprintf("%s_count", oc.ObjType))
+	}
+
+	return data, nil
 }
 
 func (manager *SMetadataManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
@@ -183,8 +285,27 @@ func (manager *SMetadataManager) AllowListItems(ctx context.Context, userCred mc
 }
 
 // 元数据(标签)列表
-func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
-	resources := jsonutils.GetQueryStringArray(query, "resources")
+func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, input apis.MetadataListInput) (*sqlchemy.SQuery, error) {
+	var err error
+	q, err = manager.SModelBaseManager.ListItemFilter(ctx, q, userCred, input.ModelBaseListInput)
+	if err != nil {
+		return nil, errors.Wrap(err, "SModelBaseManager.ListItemFilter")
+	}
+
+	if len(input.Key) > 0 {
+		q = q.In("key", input.Key)
+	}
+	if len(input.Value) > 0 {
+		q = q.In("value", input.Value)
+	}
+	if len(input.Search) > 0 {
+		q = q.Filter(sqlchemy.OR(
+			sqlchemy.Contains(q.Field("key"), input.Search),
+			sqlchemy.Contains(q.Field("value"), input.Search),
+		))
+	}
+
+	resources := input.Resources
 	if len(resources) == 0 {
 		for resource := range globalTables {
 			resources = append(resources, resource)
@@ -200,9 +321,10 @@ func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy
 			continue
 		}
 		resourceView := man.Query().SubQuery()
-		prefix := sqlchemy.NewStringField(fmt.Sprintf("%s::", man.Keyword()))
+		prefix := sqlchemy.NewStringField(fmt.Sprintf("%s%s", man.Keyword(), OBJECT_TYPE_ID_SEP))
 		field := sqlchemy.CONCAT(man.Keyword(), prefix, resourceView.Field("id"))
 		sq := resourceView.Query(field)
+		query := jsonutils.Marshal(input)
 		ownerId, queryScope, err := FetchCheckQueryOwnerScope(ctx, userCred, query, man, policy.PolicyActionList, true)
 		if err != nil {
 			log.Warningf("FetchCheckQueryOwnerScope.%s error: %v", man.Keyword(), err)
@@ -216,18 +338,39 @@ func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy
 	if len(conditions) > 0 {
 		q = q.Filter(sqlchemy.OR(conditions...))
 	}
-	for args, prefix := range map[string]string{"sys_meta": SYS_TAG_PREFIX, "cloud_meta": CLOUD_TAG_PREFIX, "user_meta": USER_TAG_PREFIX} {
+
+	if input.SysMeta != nil && *input.SysMeta {
+		q = q.Filter(sqlchemy.Startswith(q.Field("key"), SYS_TAG_PREFIX))
+	}
+	if input.CloudMeta != nil && *input.CloudMeta {
+		q = q.Filter(sqlchemy.Startswith(q.Field("key"), CLOUD_TAG_PREFIX))
+	}
+	if input.UserMeta != nil && *input.UserMeta {
+		q = q.Filter(sqlchemy.Startswith(q.Field("key"), USER_TAG_PREFIX))
+	}
+
+	/*for args, prefix := range map[string]string{"sys_meta": SYS_TAG_PREFIX, "cloud_meta": CLOUD_TAG_PREFIX, "user_meta": USER_TAG_PREFIX} {
 		if jsonutils.QueryBoolean(query, args, false) {
 			q = q.Filter(sqlchemy.Startswith(q.Field("key"), prefix))
 		}
-	}
+	}*/
 
 	withConditions := []sqlchemy.ICondition{}
-	for args, prefix := range map[string]string{"with_sys_meta": SYS_TAG_PREFIX, "with_cloud_meta": CLOUD_TAG_PREFIX, "with_user_meta": USER_TAG_PREFIX} {
+	if input.WithSysMeta != nil && *input.WithSysMeta {
+		withConditions = append(withConditions, sqlchemy.Startswith(q.Field("key"), SYS_TAG_PREFIX))
+	}
+	if input.WithCloudMeta != nil && *input.WithCloudMeta {
+		withConditions = append(withConditions, sqlchemy.Startswith(q.Field("key"), CLOUD_TAG_PREFIX))
+	}
+	if input.WithUserMeta != nil && *input.WithUserMeta {
+		withConditions = append(withConditions, sqlchemy.Startswith(q.Field("key"), USER_TAG_PREFIX))
+	}
+
+	/*for args, prefix := range map[string]string{"with_sys_meta": SYS_TAG_PREFIX, "with_cloud_meta": CLOUD_TAG_PREFIX, "with_user_meta": USER_TAG_PREFIX} {
 		if jsonutils.QueryBoolean(query, args, false) {
 			withConditions = append(withConditions, sqlchemy.Startswith(q.Field("key"), prefix))
 		}
-	}
+	}*/
 
 	if len(withConditions) > 0 {
 		q = q.Filter(sqlchemy.OR(withConditions...))
@@ -316,50 +459,62 @@ func (manager *SMetadataManager) SetValuesWithLog(ctx context.Context, obj IMode
 func (manager *SMetadataManager) SetValues(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential) ([]sMetadataChange, error) {
 	idStr := GetObjectIdstr(obj)
 
-	lockman.LockObject(ctx, obj)
-	defer lockman.ReleaseObject(ctx, obj)
+	// no need to lock
+	// lockman.LockObject(ctx, obj)
+	// defer lockman.ReleaseObject(ctx, obj)
 
 	changes := make([]sMetadataChange, 0)
 	for key, value := range store {
 
+		record := SMetadata{}
+
+		err := manager.RawQuery().Equals("id", idStr).Equals("key", key).First(&record) //避免之前设置的tag被删除后再次设置时出现Duplicate entry error
+		if err != nil {
+			if errors.Cause(err) != sql.ErrNoRows {
+				return changes, errors.Wrap(err, "RawQuery")
+			} else {
+				record.Deleted = true
+			}
+		}
+
+		newRecord := SMetadata{}
+
+		newRecord.ObjId = obj.GetId()
+		newRecord.ObjType = obj.GetModelManager().Keyword()
+		newRecord.Id = idStr
+		newRecord.Key = key
+
 		valStr := stringutils.Interface2String(value)
 		valStrLower := strings.ToLower(valStr)
 		if valStrLower == "none" || valStrLower == "null" {
-			valStr = ""
+			newRecord.Value = record.Value
+			newRecord.Deleted = true
+		} else {
+			newRecord.Value = valStr
+			newRecord.Deleted = false
 		}
-		record := SMetadata{}
-		err := manager.RawQuery().Equals("id", idStr).Equals("key", key).First(&record) //避免之前设置的tag被删除后再次设置时出现Duplicate entry error
+
+		if record.Deleted == newRecord.Deleted && record.Value == newRecord.Value {
+			// no changes
+			continue
+		}
+
+		err = manager.TableSpec().InsertOrUpdate(&newRecord)
 		if err != nil {
-			if err == sql.ErrNoRows {
+			return nil, errors.Wrapf(err, "InsertOrUpdate %s=%s", key, valStr)
+		}
+
+		if record.Deleted != newRecord.Deleted {
+			if record.Deleted {
+				// create
 				changes = append(changes, sMetadataChange{Key: key, NValue: valStr})
-				record.Id = idStr
-				record.Key = key
-				record.Value = valStr
-				err = manager.TableSpec().Insert(&record)
-				if err != nil {
-					return nil, err
-				}
 			} else {
-				return nil, err
+				// delete
+				changes = append(changes, sMetadataChange{Key: key, OValue: record.Value})
 			}
 		} else {
-			deleted := record.Deleted
-			oValue := record.Value
-			_, err := Update(&record, func() error {
-				record.Deleted = false
-				record.Value = valStr
-				return nil
-			})
-			if err != nil {
-				return nil, err
-			}
-			if deleted {
-				changes = append(changes, sMetadataChange{Key: key, NValue: valStr})
-			} else {
-				if oValue != valStr {
-					changes = append(changes, sMetadataChange{Key: key, OValue: oValue, NValue: valStr})
-				}
-			}
+			// change
+			changes = append(changes, sMetadataChange{Key: key, OValue: record.Value, NValue: valStr})
 		}
 	}
 	return changes, nil

pkg/cloudcommon/db/modelbase.go

@@ -241,12 +241,12 @@ func (manager *SModelBaseManager) InitializeData() error {
 	return nil
 }
 
-func (manager *SModelBaseManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
+func (manager *SModelBaseManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, keys stringutils2.SSortedStrings) (*sqlchemy.SQuery, error) {
 	q = q.AppendField(q.QueryFields()...)
 	return q, nil
 }
 
-func (manager *SModelBaseManager) GetExportExtraKeys(ctx context.Context, query jsonutils.JSONObject, rowMap map[string]string) *jsonutils.JSONDict {
+func (manager *SModelBaseManager) GetExportExtraKeys(ctx context.Context, keys stringutils2.SSortedStrings, rowMap map[string]string) *jsonutils.JSONDict {
 	return jsonutils.NewDict()
 }
 

pkg/cloudcommon/db/orderby.go

@@ -20,7 +20,6 @@ func NeedOrderQuery(fieldOrders []string) bool {
 	for _, field := range fieldOrders {
 		if sqlchemy.SQL_ORDER_ASC.Equals(field) || sqlchemy.SQL_ORDER_DESC.Equals(field) {
 			return true
-			break
 		}
 	}
 	return false

pkg/cloudcommon/db/standalone.go

@@ -16,7 +16,6 @@ package db
 
 import (
 	"context"
-	"fmt"
 	"strings"
 
 	"yunion.io/x/jsonutils"
@@ -200,7 +199,8 @@ func (manager *SStandaloneResourceBaseManager) ListItemFilter(
 	}
 
 	if len(tags) > 0 {
-		metadataView := Metadata.Query()
+		metadataResQ := Metadata.Query().Equals("obj_type", manager.Keyword()).SubQuery()
+		metadataView := metadataResQ.Query()
 		idx := 0
 		for key, values := range tags {
 			if idx == 0 {
@@ -209,7 +209,7 @@ func (manager *SStandaloneResourceBaseManager) ListItemFilter(
 					metadataView = metadataView.In("value", values)
 				}
 			} else {
-				subMetataView := Metadata.Query().Equals("key", key)
+				subMetataView := metadataResQ.Query().Equals("key", key)
 				if len(values) > 0 {
 					subMetataView = subMetataView.In("value", values)
 				}
@@ -219,22 +219,13 @@ func (manager *SStandaloneResourceBaseManager) ListItemFilter(
 			idx++
 		}
 		metadatas := metadataView.SubQuery()
-		fieldName := fmt.Sprintf("%s_id", manager.Keyword())
-		metadataSQ := metadatas.Query(
-			sqlchemy.REPLACE(fieldName, metadatas.Field("id"), manager.Keyword()+"::", ""),
-		)
-		sq := metadataSQ.Filter(sqlchemy.Like(metadatas.Field("id"), manager.Keyword()+"::%")).Distinct()
+		sq := metadatas.Query(metadatas.Field("obj_id")).Distinct().SubQuery()
 		q = q.Filter(sqlchemy.In(q.Field("id"), sq))
 	}
 
 	if input.WithoutUserMeta {
-		metadatas := Metadata.Query().SubQuery()
-		fieldName := fmt.Sprintf("%s_id", manager.Keyword())
-		metadataSQ := metadatas.Query(
-			sqlchemy.REPLACE(fieldName, metadatas.Field("id"), manager.Keyword()+"::", ""),
-		)
-		sq := metadataSQ.Filter(sqlchemy.Like(metadatas.Field("key"), USER_TAG_PREFIX+"%")).Distinct()
-
+		metadatas := Metadata.Query().Equals("obj_type", manager.Keyword()).SubQuery()
+		sq := metadatas.Query(metadatas.Field("obj_id")).Startswith("key", USER_TAG_PREFIX).Distinct().SubQuery()
 		q.Filter(sqlchemy.NotIn(q.Field("id"), sq))
 	}
 
@@ -242,6 +233,14 @@ func (manager *SStandaloneResourceBaseManager) ListItemFilter(
 }
 
 func (manager *SStandaloneResourceBaseManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) {
+	if strings.HasPrefix(field, "tag:") {
+		tagKey := field[4:]
+		metaQ := Metadata.Query("obj_id", "value").Equals("obj_type", manager.Keyword()).Equals("key", tagKey).SubQuery()
+		q = q.AppendField(metaQ.Field("value", field)).Distinct()
+		q = q.LeftJoin(metaQ, sqlchemy.Equals(q.Field("id"), metaQ.Field("obj_id")))
+		q = q.Asc(metaQ.Field("value"))
+		return q, nil
+	}
 	q, err := manager.SResourceBaseManager.QueryDistinctExtraField(q, field)
 	if err == nil {
 		return q, nil
@@ -259,6 +258,25 @@ func (manager *SStandaloneResourceBaseManager) OrderByExtraFields(
 	if err != nil {
 		return nil, errors.Wrap(err, "SResourceBaseManager.OrderByExtraFields")
 	}
+
+	if len(input.OrderByTag) > 0 {
+		order := sqlchemy.SQL_ORDER_ASC
+		tagKey := input.OrderByTag
+		if stringutils2.HasSuffixIgnoreCase(input.OrderByTag, string(sqlchemy.SQL_ORDER_ASC)) {
+			tagKey = tagKey[0 : len(tagKey)-len(sqlchemy.SQL_ORDER_ASC)-1]
+		} else if stringutils2.HasSuffixIgnoreCase(input.OrderByTag, string(sqlchemy.SQL_ORDER_DESC)) {
+			tagKey = tagKey[0 : len(tagKey)-len(sqlchemy.SQL_ORDER_DESC)-1]
+			order = sqlchemy.SQL_ORDER_DESC
+		}
+		metaQ := Metadata.Query("obj_id", "value").Equals("obj_type", manager.Keyword()).Equals("key", tagKey).SubQuery()
+		q = q.LeftJoin(metaQ, sqlchemy.Equals(q.Field("id"), metaQ.Field("obj_id")))
+		if order == sqlchemy.SQL_ORDER_ASC {
+			q = q.Asc(metaQ.Field("value"))
+		} else {
+			q = q.Desc(metaQ.Field("value"))
+		}
+	}
+
 	return q, nil
 }
 
@@ -519,3 +537,38 @@ func (manager *SStandaloneResourceBaseManager) FetchCustomizeColumns(
 func (manager *SStandaloneResourceBaseManager) GetMetadataHiddenKeys() []string {
 	return nil
 }
+
+const (
+	TAG_EXPORT_KEY_PREFIX = "tag:"
+)
+
+func (manager *SStandaloneResourceBaseManager) GetExportExtraKeys(ctx context.Context, keys stringutils2.SSortedStrings, rowMap map[string]string) *jsonutils.JSONDict {
+	res := manager.SResourceBaseManager.GetExportExtraKeys(ctx, keys, rowMap)
+
+	for _, key := range keys {
+		if strings.HasPrefix(key, TAG_EXPORT_KEY_PREFIX) {
+			res.Add(jsonutils.NewString(rowMap[key]), key)
+		}
+	}
+
+	return res
+}
+
+func (manager *SStandaloneResourceBaseManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, keys stringutils2.SSortedStrings) (*sqlchemy.SQuery, error) {
+	var err error
+	q, err = manager.SResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys)
+	if err != nil {
+		return nil, errors.Wrap(err, "SResourceBaseManager.ListItemExportKeys")
+	}
+
+	for _, key := range keys {
+		if strings.HasPrefix(key, TAG_EXPORT_KEY_PREFIX) {
+			tagKey := key[len(TAG_EXPORT_KEY_PREFIX):]
+			metaQ := Metadata.Query("obj_id", "value").Equals("obj_type", manager.Keyword()).Equals("key", tagKey).SubQuery()
+			q = q.LeftJoin(metaQ, sqlchemy.Equals(q.Field("id"), metaQ.Field("obj_id")))
+			q = q.AppendField(metaQ.Field("value", key))
+		}
+	}
+
+	return q, nil
+}

pkg/cloudevent/models/initdb.go

@@ -27,6 +27,7 @@ func InitDB() error {
 		 * initialization order matters, do not change the order
 		 */
 		db.TenantCacheManager,
+		db.Metadata,
 
 		CloudproviderManager,
 		CloudeventManager,

pkg/compute/models/guests.go

@@ -1698,17 +1698,17 @@ func (self *SGuest) GetExtraDetails(ctx context.Context, userCred mcclient.Token
 	return api.ServerDetails{}, nil
 }
 
-func (manager *SGuestManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
+func (manager *SGuestManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, keys stringutils2.SSortedStrings) (*sqlchemy.SQuery, error) {
 	var err error
-	q, err = manager.SVirtualResourceBaseManager.ListItemExportKeys(ctx, q, userCred, query)
+	q, err = manager.SVirtualResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys)
 	if err != nil {
 		return nil, err
 	}
 
-	exportKeys, _ := query.GetString("export_keys")
-	keys := strings.Split(exportKeys, ",")
+	// exportKeys, _ := query.GetString("export_keys")
+	// keys := strings.Split(exportKeys, ",")
 	// guest_id as filter key
-	if utils.IsInStringArray("ips", keys) {
+	if keys.Contains("ips") {
 		guestIpsQuery := GuestnetworkManager.Query("guest_id").GroupBy("guest_id")
 		guestIpsQuery.AppendField(sqlchemy.GROUP_CONCAT("concat_ip_addr", guestIpsQuery.Field("ip_addr")))
 		ipsSubQuery := guestIpsQuery.SubQuery()
@@ -1716,7 +1716,7 @@ func (manager *SGuestManager) ListItemExportKeys(ctx context.Context, q *sqlchem
 		q.AppendField(ipsSubQuery.Field("concat_ip_addr"))
 	}
 
-	if utils.IsInStringArray("user_tags", keys) {
+	if keys.Contains("user_tags") {
 		guestUserTagsQuery := db.Metadata.Query().Startswith("id", "server::").
 			Startswith("key", db.USER_TAG_PREFIX).GroupBy("id")
 		guestUserTagsQuery.AppendField(sqlchemy.SubStr("guest_id", guestUserTagsQuery.Field("id"), len("server::")+1, 0))
@@ -1731,7 +1731,7 @@ func (manager *SGuestManager) ListItemExportKeys(ctx context.Context, q *sqlchem
 		q.AppendField(subQ.Field("user_tags"))
 	}
 
-	if utils.IsInStringArray("disk", keys) {
+	if keys.Contains("disk") {
 		guestDisksQuery := GuestdiskManager.Query("guest_id", "disk_id").GroupBy("guest_id")
 		diskQuery := DiskManager.Query("id", "disk_size").SubQuery()
 		guestDisksQuery.Join(diskQuery, sqlchemy.Equals(diskQuery.Field("id"), guestDisksQuery.Field("disk_id")))
@@ -1740,21 +1740,21 @@ func (manager *SGuestManager) ListItemExportKeys(ctx context.Context, q *sqlchem
 		q.LeftJoin(guestDisksSubQuery, sqlchemy.Equals(q.Field("id"), guestDisksSubQuery.Field("guest_id")))
 		q.AppendField(guestDisksSubQuery.Field("disk_size"))
 	}
-	if utils.IsInStringArray("eip", keys) {
+	if keys.Contains("eip") {
 		eipsQuery := ElasticipManager.Query("associate_id", "ip_addr").Equals("associate_type", "server").GroupBy("associate_id")
 		eipsSubQuery := eipsQuery.SubQuery()
 		q.LeftJoin(eipsSubQuery, sqlchemy.Equals(q.Field("id"), eipsSubQuery.Field("associate_id")))
 		q.AppendField(eipsSubQuery.Field("ip_addr", "eip"))
 	}
 
-	if utils.IsInStringArray("host", keys) {
+	if keys.Contains("host") {
 		hostQuery := HostManager.Query("id", "name").GroupBy("id")
 		hostSubQuery := hostQuery.SubQuery()
 		q.LeftJoin(hostSubQuery, sqlchemy.Equals(q.Field("host_id"), hostSubQuery.Field("id")))
 		q.AppendField(hostSubQuery.Field("name", "host"))
 	}
 
-	if utils.IsInStringArray("zone", keys) {
+	if keys.Contains("zone") {
 		zoneQuery := ZoneManager.Query("id", "name").SubQuery()
 		hostQuery := HostManager.Query("id", "zone_id").GroupBy("id")
 		hostQuery.LeftJoin(zoneQuery, sqlchemy.Equals(hostQuery.Field("zone_id"), zoneQuery.Field("id")))
@@ -1765,7 +1765,7 @@ func (manager *SGuestManager) ListItemExportKeys(ctx context.Context, q *sqlchem
 	}
 
 	// host_id as filter key
-	if utils.IsInStringArray("region", keys) {
+	if keys.Contains("region") {
 		zoneQuery := ZoneManager.Query("id", "cloudregion_id").SubQuery()
 		hostQuery := HostManager.Query("id", "zone_id").GroupBy("id")
 		cloudregionQuery := CloudregionManager.Query("id", "name").SubQuery()
@@ -1777,7 +1777,8 @@ func (manager *SGuestManager) ListItemExportKeys(ctx context.Context, q *sqlchem
 		q.LeftJoin(hostSubQuery, sqlchemy.Equals(q.Field("host_id"), hostSubQuery.Field("id")))
 		q.AppendField(hostSubQuery.Field("region"))
 	}
-	if utils.IsInStringArray("manager", keys) {
+
+	if keys.Contains("manager") {
 		hostQuery := HostManager.Query("id", "manager_id").GroupBy("id")
 		cloudProviderQuery := CloudproviderManager.Query("id", "name").SubQuery()
 		hostQuery.LeftJoin(cloudProviderQuery, sqlchemy.Equals(hostQuery.Field("manager_id"),
@@ -1790,10 +1791,10 @@ func (manager *SGuestManager) ListItemExportKeys(ctx context.Context, q *sqlchem
 	return q, nil
 }
 
-func (manager *SGuestManager) GetExportExtraKeys(ctx context.Context, query jsonutils.JSONObject, rowMap map[string]string) *jsonutils.JSONDict {
-	res := manager.SVirtualResourceBaseManager.GetExportExtraKeys(ctx, query, rowMap)
-	exportKeys, _ := query.GetString("export_keys")
-	keys := strings.Split(exportKeys, ",")
+func (manager *SGuestManager) GetExportExtraKeys(ctx context.Context, keys stringutils2.SSortedStrings, rowMap map[string]string) *jsonutils.JSONDict {
+	res := manager.SVirtualResourceBaseManager.GetExportExtraKeys(ctx, keys, rowMap)
+	// exportKeys, _ := query.GetString("export_keys")
+	// keys := strings.Split(exportKeys, ",")
 	if ips, ok := rowMap["concat_ip_addr"]; ok && len(ips) > 0 {
 		res.Set("ips", jsonutils.NewString(ips))
 	}
@@ -1818,7 +1819,7 @@ func (manager *SGuestManager) GetExportExtraKeys(ctx context.Context, query json
 	if userTags, ok := rowMap["user_tags"]; ok && len(userTags) > 0 {
 		res.Set("user_tags", jsonutils.NewString(userTags))
 	}
-	if utils.IsInStringArray("tenant", keys) {
+	if keys.Contains("tenant") {
 		if projectId, ok := rowMap["tenant_id"]; ok {
 			tenant, err := db.TenantCacheManager.FetchTenantById(ctx, projectId)
 			if err == nil {
@@ -1826,7 +1827,7 @@ func (manager *SGuestManager) GetExportExtraKeys(ctx context.Context, query json
 			}
 		}
 	}
-	if utils.IsInStringArray("os_distribution", keys) {
+	if keys.Contains("os_distribution") {
 		if osType, ok := rowMap["os_type"]; ok {
 			res.Set("os_distribution", jsonutils.NewString(osType))
 		}

pkg/compute/models/initdb.go

@@ -27,6 +27,7 @@ func InitDB() error {
 		 * initialization order matters, do not change the order
 		 */
 		db.TenantCacheManager,
+		db.Metadata,
 
 		QuotaManager,
 

pkg/compute/models/isolated_devices.go

@@ -222,20 +222,18 @@ func (self *SIsolatedDevice) AllowDeleteItem(ctx context.Context, userCred mccli
 	return userCred.IsSystemAdmin()
 } */
 
-func (manager *SIsolatedDeviceManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
+func (manager *SIsolatedDeviceManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, keys stringutils2.SSortedStrings) (*sqlchemy.SQuery, error) {
 	var err error
-	q, err = manager.SModelBaseManager.ListItemExportKeys(ctx, q, userCred, query)
+	q, err = manager.SStandaloneResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys)
 	if err != nil {
 		return nil, err
 	}
-	exportKeys, _ := query.GetString("export_keys")
-	keys := strings.Split(exportKeys, ",")
-	if utils.IsInStringArray("guest", keys) {
+	if keys.Contains("guest") {
 		guestNameQuery := GuestManager.Query("name", "id").SubQuery()
 		q.LeftJoin(guestNameQuery, sqlchemy.Equals(q.Field("guest_id"), guestNameQuery.Field("id")))
 		q.AppendField(guestNameQuery.Field("name", "guest"))
 	}
-	if utils.IsInStringArray("host", keys) {
+	if keys.Contains("host") {
 		hostNameQuery := HostManager.Query("name", "id").SubQuery()
 		q.LeftJoin(hostNameQuery, sqlchemy.Equals(q.Field("host_id"), hostNameQuery.Field("id")))
 		q.AppendField(hostNameQuery.Field("name", "host"))
@@ -243,8 +241,8 @@ func (manager *SIsolatedDeviceManager) ListItemExportKeys(ctx context.Context, q
 	return q, nil
 }
 
-func (manager *SIsolatedDeviceManager) GetExportExtraKeys(ctx context.Context, query jsonutils.JSONObject, rowMap map[string]string) *jsonutils.JSONDict {
-	res := manager.SStandaloneResourceBaseManager.GetExportExtraKeys(ctx, query, rowMap)
+func (manager *SIsolatedDeviceManager) GetExportExtraKeys(ctx context.Context, keys stringutils2.SSortedStrings, rowMap map[string]string) *jsonutils.JSONDict {
+	res := manager.SStandaloneResourceBaseManager.GetExportExtraKeys(ctx, keys, rowMap)
 	if guest, ok := rowMap["guest"]; ok && len(guest) > 0 {
 		res.Set("guest", jsonutils.NewString(guest))
 	}

pkg/httperrors/consts.go

@@ -34,6 +34,7 @@ const (
 
 	ErrServerStatus  = errors.Error("ServerStatusError")
 	ErrInvalidStatus = errors.ErrInvalidStatus
+	ErrInvalidFormat = errors.Error("InvalidFormatError")
 
 	ErrInputParameter   = errors.Error("InputParameterError")
 	ErrWeakPassword     = errors.Error("WeakPasswordError")
@@ -105,6 +106,7 @@ var (
 
 		ErrServerStatus:  400,
 		ErrInvalidStatus: 400,
+		ErrInvalidFormat: 400,
 
 		ErrInputParameter:   400,
 		ErrWeakPassword:     400,

pkg/image/models/initdb.go

@@ -27,8 +27,9 @@ func InitDB() error {
 		 * initialization order matters, do not change the order
 		 */
 		db.TenantCacheManager,
-		ImageManager,
+		db.Metadata,
 
+		ImageManager,
 		QuotaManager,
 	} {
 		err := manager.InitializeData()

pkg/image/service/handlers.go

@@ -46,9 +46,11 @@ func InitHandlers(app *appsrv.Application) {
 		taskman.TaskManager,
 		taskman.SubTaskManager,
 		taskman.TaskObjectManager,
+
 		db.UserCacheManager,
 		db.TenantCacheManager,
 		db.SharedResourceManager,
+
 		models.ImageTagManager,
 		models.ImageMemberManager,
 		models.ImagePropertyManager,

pkg/keystone/models/initdb.go

@@ -26,6 +26,8 @@ func InitDB() error {
 		 * Important!!!
 		 * initialization order matters, do not change the order
 		 */
+		db.Metadata,
+
 		RegionManager,
 		ServiceManager,
 		EndpointManager,

pkg/mcclient/options/base.go

@@ -234,6 +234,8 @@ type BaseListOptions struct {
 	IsManaged    *bool    `help:"List objects managed by external providers" token:"managed" json:"is_managed"`
 
 	PagingMarker string `help:"Marker for pagination" json:"paging_marker"`
+
+	OrderByTag string `help:"Order results by tag values, composed by a tag key and order, e.g user:部门:ASC"`
 }
 
 func (opts *BaseListOptions) addTag(prefix, tag string, idx int, params *jsonutils.JSONDict) error {

pkg/mcclient/options/metadata.go

@@ -16,14 +16,22 @@ package options
 
 type MetadataListOptions struct {
 	Resources []string `help:"list of resource e.g server、disk、eip、snapshot, empty will show all metadata"`
-	SysMeta   *bool    `help:"Show sys metadata only"`
-	CloudMeta *bool    `help:"Show cloud metadata olny"`
-	UserMeta  *bool    `help:"Show user metadata olny"`
-	Admin     *bool    `help:"Show all metadata"`
+
+	SysMeta   *bool `help:"Show sys metadata only"`
+	CloudMeta *bool `help:"Show cloud metadata olny"`
+	UserMeta  *bool `help:"Show user metadata olny"`
+
+	Admin *bool `help:"Show all metadata"`
 
 	WithSysMeta   *bool `help:"Show sys metadata"`
 	WithCloudMeta *bool `help:"Show cloud metadata"`
 	WithUserMeta  *bool `help:"Show user metadata"`
+
+	Key   []string `help:"key"`
+	Value []string `help:"value"`
+
+	Limit  *int `help:"limit"`
+	Offset *int `help:"offset"`
 }
 
 type TagListOptions MetadataListOptions

pkg/util/stringutils2/ignorecase.go

@@ -0,0 +1,33 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stringutils2
+
+import (
+	"strings"
+)
+
+func HasSuffixIgnoreCase(str string, suffix string) bool {
+	if len(str) < len(suffix) {
+		return false
+	}
+	return strings.EqualFold(str[len(str)-len(suffix):len(str)], suffix)
+}
+
+func HasPrefixIgnoreCase(str string, prefix string) bool {
+	if len(str) < len(prefix) {
+		return false
+	}
+	return strings.EqualFold(str[0:len(prefix)], prefix)
+}

pkg/util/stringutils2/ignorecase_test.go

@@ -0,0 +1,81 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stringutils2
+
+import (
+	"testing"
+)
+
+func TestHasSufffixIgnoreCase(t *testing.T) {
+	cases := []struct {
+		Hystack string
+		Needle  string
+		Want    bool
+	}{
+		{
+			Hystack: "Hyperscacle:ASC",
+			Needle:  ":ASC",
+			Want:    true,
+		},
+		{
+			Hystack: "Hyperscacle:ASC",
+			Needle:  ":asc",
+			Want:    true,
+		},
+		{
+			Hystack: "Hyperscacle:ASC",
+			Needle:  "ASCD",
+			Want:    false,
+		},
+	}
+
+	for _, c := range cases {
+		got := HasSuffixIgnoreCase(c.Hystack, c.Needle)
+		if got != c.Want {
+			t.Errorf("HasSuffixIgnoreCase %s %s want %v got %v", c.Hystack, c.Needle, c.Want, got)
+		}
+	}
+}
+
+func TestHasPrefixIgnoreCase(t *testing.T) {
+	cases := []struct {
+		Hystack string
+		Needle  string
+		Want    bool
+	}{
+		{
+			Hystack: "Hyperscacle:ASC",
+			Needle:  "Hyper",
+			Want:    true,
+		},
+		{
+			Hystack: "Hyperscacle:ASC",
+			Needle:  "HYPER",
+			Want:    true,
+		},
+		{
+			Hystack: "Hyperscacle:ASC",
+			Needle:  "ASCD",
+			Want:    false,
+		},
+	}
+
+	for _, c := range cases {
+		got := HasPrefixIgnoreCase(c.Hystack, c.Needle)
+		if got != c.Want {
+			t.Errorf("HasPrefixIgnoreCase %s %s want %v got %v", c.Hystack, c.Needle, c.Want, got)
+		}
+	}
+}

pkg/util/stringutils2/sortedstrings.go

@@ -22,7 +22,7 @@ type SSortedStrings []string
 
 func NewSortedStrings(strs []string) SSortedStrings {
 	if strs == nil {
-		return nil
+		return SSortedStrings{}
 	}
 	sort.Strings(strs)
 	return SSortedStrings(strs)

vendor/modules.txt

@@ -1018,7 +1018,7 @@ yunion.io/x/jsonutils
 # yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d
 yunion.io/x/log
 yunion.io/x/log/hooks
-# yunion.io/x/pkg v0.0.0-20200312093207-f07f4cb157e0
+# yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e
 yunion.io/x/pkg/errors
 yunion.io/x/pkg/gotypes
 yunion.io/x/pkg/prettytable
@@ -1050,7 +1050,7 @@ yunion.io/x/pkg/util/workqueue
 yunion.io/x/pkg/utils
 # yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
 yunion.io/x/s3cli
-# yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57
+# yunion.io/x/sqlchemy v0.0.0-20200310015557-9e9a49139db1
 yunion.io/x/sqlchemy
 # yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3
 yunion.io/x/structarg