Merge pull request #1223 in YUNIONIO/onecloud from ~QUXUAN/onecloud:feature/qx-tag to release/2.8.0

* commit '43daa1ceb420d8b6176e92ee7c023d51c9c8e962':
  用户自定义标签加前缀user:
  同步公有云key加上前缀
  限制key必须是中英文、数字或者_
  允许只传key进行过滤
  取消key以_开头key限制
  metadata返回列表使用复数
  支持server disk 过滤
  支持标签

pkg/cloudcommon/db/db_dispatcher.go

@@ -452,7 +452,9 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 		if err == nil && colSpec != nil && colSpec.IsNumeric() {
 			orderBy = []string{"id"}
 		} else {
-			orderBy = []string{"created_at"}
+			if manager.TableSpec().ColumnSpec("created_at") != nil {
+				orderBy = []string{"created_at"}
+			}
 		}
 	}
 	order := sqlchemy.SQL_ORDER_DESC

pkg/cloudcommon/db/metadata.go

@@ -8,14 +8,24 @@ import (
 	"time"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/pkg/util/stringutils"
+	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
+	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
 const (
-	SYSTEM_ADMIN_PREFIX = "__sys_"
+	SYSTEM_ADMIN_PREFIX = "_"
+	CLOUD_TAG_PREFIX    = "ext:"
+	USER_TAG_PREFIX     = "user:"
+
+	TAG_DELETE_RANGE_USER  = "user"
+	TAG_DELETE_RANGE_CLOUD = "cloud"
+	TAG_DELETE_RANGE_SYS   = "sys"
+	TAG_DELETE_RANGE_ALL   = "all"
 )
 
 type SMetadataManager struct {
@@ -25,16 +35,24 @@ type SMetadataManager struct {
 type SMetadata struct {
 	SModelBase
 
-	Id        string    `width:"128" charset:"ascii" primary:"true"` // = Column(VARCHAR(128, charset='ascii'), primary_key=True)
-	Key       string    `width:"64" charset:"ascii" primary:"true"`  // = Column(VARCHAR(64, charset='ascii'),  primary_key=True)
-	Value     string    `charset:"utf8"`                             // = Column(TEXT(charset='utf8'), nullable=True)
-	UpdatedAt time.Time `nullable:"false" updated_at:"true"`         // = Column(DateTime, default=get_utcnow, nullable=False, onupdate=get_utcnow)
+	Id        string    `width:"128" charset:"ascii" primary:"true" list:"user" get:"user"` // = Column(VARCHAR(128, charset='ascii'), primary_key=True)
+	Key       string    `width:"64" charset:"utf8" primary:"true" list:"user" get:"user"`   // = Column(VARCHAR(64, charset='ascii'),  primary_key=True)
+	Value     string    `charset:"utf8" list:"user" get:"user"`                             // = Column(TEXT(charset='utf8'), nullable=True)
+	UpdatedAt time.Time `nullable:"false" updated_at:"true"`                                // = Column(DateTime, default=get_utcnow, nullable=False, onupdate=get_utcnow)
+	Deleted   bool      `nullable:"false" default:"false" index:"true"`
 }
 
 var Metadata *SMetadataManager
+var ResourceMap map[string]*SVirtualResourceBaseManager
 
 func init() {
-	Metadata = &SMetadataManager{SModelBaseManager: NewModelBaseManager(SMetadata{}, "metadata_tbl", "metadata", "metadata")}
+	Metadata = &SMetadataManager{SModelBaseManager: NewModelBaseManager(SMetadata{}, "metadata_tbl", "metadata", "metadatas")}
+	ResourceMap = map[string]*SVirtualResourceBaseManager{
+		"disk":     {SStatusStandaloneResourceBaseManager: NewStatusStandaloneResourceBaseManager(SVirtualResourceBase{}, "disks_tbl", "disk", "disks")},
+		"server":   {SStatusStandaloneResourceBaseManager: NewStatusStandaloneResourceBaseManager(SVirtualResourceBase{}, "guests_tbl", "server", "servers")},
+		"eip":      {SStatusStandaloneResourceBaseManager: NewStatusStandaloneResourceBaseManager(SVirtualResourceBase{}, "elasticips_tbl", "eip", "eips")},
+		"snapshot": {SStatusStandaloneResourceBaseManager: NewStatusStandaloneResourceBaseManager(SVirtualResourceBase{}, "snapshots_tbl", "snpashot", "snpashots")},
+	}
 }
 
 func (m *SMetadata) GetId() string {
@@ -53,6 +71,64 @@ func GetObjectIdstr(model IModel) string {
 	return fmt.Sprintf("%s::%s", model.GetModelManager().Keyword(), model.GetId())
 }
 
+func (manager *SMetadataManager) Query(fields ...string) *sqlchemy.SQuery {
+	return manager.SModelBaseManager.Query(fields...).IsFalse("deleted")
+}
+
+func (manager *SMetadataManager) RawQuery(fields ...string) *sqlchemy.SQuery {
+	return manager.SModelBaseManager.Query(fields...)
+}
+
+func (m *SMetadata) MarkDelete() error {
+	m.Deleted = true
+	return nil
+}
+
+func (m *SMetadata) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	return DeleteModel(ctx, userCred, m)
+}
+
+func (manager *SMetadataManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+	return true
+}
+
+func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
+	resources := jsonutils.GetQueryStringArray(query, "resources")
+	if len(resources) == 0 {
+		for resource := range ResourceMap {
+			resources = append(resources, resource)
+		}
+	}
+	conditions := []sqlchemy.ICondition{}
+	admin := jsonutils.QueryBoolean(query, "admin", false)
+	for _, resource := range resources {
+		if manager, ok := ResourceMap[resource]; ok {
+			resourceView := manager.Query().SubQuery()
+			prefix := sqlchemy.NewStringField(fmt.Sprintf("%s::", manager.Keyword()))
+			field := sqlchemy.CONCAT(manager.Keyword(), prefix, resourceView.Field("id"))
+			sq := resourceView.Query(field)
+			if !admin || !IsAdminAllowList(userCred, manager) {
+				ownerId := manager.GetOwnerId(userCred)
+				if len(ownerId) > 0 {
+					sq = manager.FilterByOwner(sq, ownerId)
+				}
+			}
+			conditions = append(conditions, sqlchemy.In(q.Field("id"), sq))
+		} else {
+			return nil, httperrors.NewInputParameterError("Not support resource %s tag filter", resource)
+		}
+	}
+	if len(conditions) > 0 {
+		q = q.Filter(sqlchemy.OR(conditions...))
+	}
+	for args, prefix := range map[string]string{"sys_meta": SYSTEM_ADMIN_PREFIX, "cloud_meta": CLOUD_TAG_PREFIX, "user_meta": USER_TAG_PREFIX} {
+		if jsonutils.QueryBoolean(query, args, false) {
+			q = q.Filter(sqlchemy.Startswith(q.Field("key"), prefix))
+		}
+	}
+	return q, nil
+}
+
 /* @classmethod
 def get_object_idstr(cls, obj, keygen_func):
 idstr = None
@@ -106,35 +182,42 @@ func (manager *SMetadataManager) RemoveAll(ctx context.Context, model IModel, us
 	lockman.LockObject(ctx, model)
 	defer lockman.ReleaseObject(ctx, model)
 
+	changes := []sMetadataChange{}
 	records := make([]SMetadata, 0)
 	q := manager.Query().Equals("id", idStr)
 	err := FetchModelObjects(manager, q, &records)
 	if err != nil {
 		return fmt.Errorf("find metadata for %s fail: %s", idStr, err)
 	}
-	changes := make([]sMetadataChange, 0)
 	for _, rec := range records {
-		if len(rec.Value) > 0 {
-			_, err := Update(&rec, func() error {
-				rec.Value = ""
-				return nil
-			})
-			if err == nil {
-				changes = append(changes, sMetadataChange{Key: rec.Key, OValue: rec.Value})
-			}
+		if err = rec.Delete(ctx, userCred); err != nil {
+			log.Errorf("remove metadata %v error: %v", rec, err)
+			continue
 		}
+		changes = append(changes, sMetadataChange{Key: rec.Key, OValue: rec.Value})
 	}
 	if len(changes) > 0 {
-		OpsLog.LogEvent(model, ACT_DEL_METADATA, jsonutils.Marshal(changes), userCred)
+		OpsLog.LogEvent(model, ACT_SET_METADATA, jsonutils.Marshal(changes), userCred)
 	}
 	return nil
 }
 
 func (manager *SMetadataManager) SetValue(ctx context.Context, obj IModel, key string, value interface{}, userCred mcclient.TokenCredential) error {
-	return manager.SetAll(ctx, obj, map[string]interface{}{key: value}, userCred)
+	return manager.SetValuesWithLog(ctx, obj, map[string]interface{}{key: value}, userCred)
 }
 
-func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential) error {
+func (manager *SMetadataManager) SetValuesWithLog(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential) error {
+	changes, err := manager.SetValues(ctx, obj, store, userCred)
+	if err != nil {
+		return err
+	}
+	if len(changes) > 0 {
+		OpsLog.LogEvent(obj, ACT_SET_METADATA, jsonutils.Marshal(changes), userCred)
+	}
+	return nil
+}
+
+func (manager *SMetadataManager) SetValues(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential) ([]sMetadataChange, error) {
 	idStr := GetObjectIdstr(obj)
 
 	lockman.LockObject(ctx, obj)
@@ -142,13 +225,17 @@ func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store m
 
 	changes := make([]sMetadataChange, 0)
 	for key, value := range store {
+		if strings.HasPrefix(key, SYSTEM_ADMIN_PREFIX) && (userCred == nil || !IsAdminAllowGetSpec(userCred, obj, "metadata")) {
+			return nil, httperrors.NewForbiddenError("Ordinary users can't set the tags that begin with an underscore")
+		}
+
 		valStr := stringutils.Interface2String(value)
 		valStrLower := strings.ToLower(valStr)
 		if valStrLower == "none" || valStrLower == "null" {
 			valStr = ""
 		}
 		record := SMetadata{}
-		err := manager.Query().Equals("id", idStr).Equals("key", key).First(&record)
+		err := manager.RawQuery().Equals("id", idStr).Equals("key", key).First(&record) //避免之前设置的tag被删除后再次设置时出现Duplicate entry error
 		if err != nil {
 			if err == sql.ErrNoRows {
 				changes = append(changes, sMetadataChange{Key: key, NValue: valStr})
@@ -157,22 +244,70 @@ func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store m
 				record.Value = valStr
 				err = manager.TableSpec().Insert(&record)
 				if err != nil {
-					return err
+					return nil, err
 				}
 			} else {
-				return err
+				return nil, err
 			}
 		} else {
+			deleted := record.Deleted
 			_, err := Update(&record, func() error {
+				record.Deleted = false
 				record.Value = valStr
 				return nil
 			})
 			if err != nil {
-				return err
+				return nil, err
+			}
+			if deleted {
+				changes = append(changes, sMetadataChange{Key: key, NValue: valStr})
+			} else {
+				if record.Value != valStr {
+					changes = append(changes, sMetadataChange{Key: key, OValue: record.Value, NValue: valStr})
+				}
 			}
-			changes = append(changes, sMetadataChange{Key: key, OValue: record.Value, NValue: valStr})
 		}
 	}
+	return changes, nil
+}
+
+func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential, delRange string) error {
+	changes, err := manager.SetValues(ctx, obj, store, userCred)
+	if err != nil {
+		return err
+	}
+
+	idStr := GetObjectIdstr(obj)
+
+	lockman.LockObject(ctx, obj)
+	defer lockman.ReleaseObject(ctx, obj)
+
+	keys := []string{}
+	for key := range store {
+		keys = append(keys, key)
+	}
+
+	records := []SMetadata{}
+	q := manager.Query().Equals("id", idStr)
+	switch delRange {
+	case TAG_DELETE_RANGE_USER:
+		q = q.Like("key", USER_TAG_PREFIX+"%")
+	case TAG_DELETE_RANGE_CLOUD:
+		q = q.Like("key", CLOUD_TAG_PREFIX+"%")
+	case TAG_DELETE_RANGE_SYS:
+		q = q.Like("key", SYSTEM_ADMIN_PREFIX+"%")
+	}
+	q = q.Filter(sqlchemy.NOT(sqlchemy.In(q.Field("key"), keys)))
+	if err := FetchModelObjects(manager, q, &records); err != nil {
+		log.Errorf("failed to fetch metadata error: %v", err)
+	}
+	for _, rec := range records {
+		if err := rec.Delete(ctx, userCred); err != nil {
+			log.Errorf("failed to delete metadata error: %v", err)
+			continue
+		}
+		changes = append(changes, sMetadataChange{Key: rec.Key, OValue: rec.Value})
+	}
 	if len(changes) > 0 {
 		OpsLog.LogEvent(obj, ACT_SET_METADATA, jsonutils.Marshal(changes), userCred)
 	}

pkg/cloudcommon/db/standalone.go

@@ -3,6 +3,8 @@ package db
 import (
 	"context"
 	"database/sql"
+	"fmt"
+	"strings"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -103,6 +105,11 @@ func (manager *SStandaloneResourceBaseManager) FetchByExternalId(idStr string) (
 	}
 }
 
+type STagValue struct {
+	value string
+	exist bool
+}
+
 func (manager *SStandaloneResourceBaseManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
 	q, err := manager.SResourceBaseManager.ListItemFilter(ctx, q, userCred, query)
 	if err != nil {
@@ -114,6 +121,62 @@ func (manager *SStandaloneResourceBaseManager) ListItemFilter(ctx context.Contex
 		q = q.Filter(sqlchemy.IsFalse(q.Field("is_emulated")))
 	}
 
+	tags := map[string]STagValue{}
+	if query.Contains("tags") {
+		idx := 0
+		for {
+			key, _ := query.GetString("tags", fmt.Sprintf("%d", idx), "key")
+			if len(key) == 0 {
+				break
+			}
+			value := STagValue{exist: false}
+			if query.Contains("tags", fmt.Sprintf("%d", idx), "value") {
+				value.value, _ = query.GetString("tags", fmt.Sprintf("%d", idx), "value")
+				value.exist = true
+			}
+			tags[key] = value
+			idx++
+		}
+	}
+
+	if len(tags) > 0 {
+		metadataView := Metadata.Query("id")
+		idx := 0
+		for k, v := range tags {
+			if idx == 0 {
+				metadataView = metadataView.Equals("key", k)
+				if v.exist {
+					metadataView = metadataView.Equals("value", v.value)
+				}
+			} else {
+				subMetataView := Metadata.Query().Equals("key", k)
+				if v.exist {
+					subMetataView = subMetataView.Equals("value", v.value)
+				}
+				sq := subMetataView.SubQuery()
+				metadataView.Join(sq, sqlchemy.Equals(metadataView.Field("id"), sq.Field("id")))
+			}
+			idx++
+		}
+		metadataView = metadataView.Filter(sqlchemy.Like(metadataView.Field("id"), manager.Keyword()+"::%")).Distinct()
+		resourceIds := []string{}
+		rows, err := metadataView.Rows()
+		if err != nil {
+			log.Errorf("query metadata ids error: %v", err)
+			return nil, err
+		}
+		defer rows.Close()
+		for rows.Next() {
+			var metadataID string
+			err = rows.Scan(&metadataID)
+			if err != nil {
+				log.Errorf("get metadata id scan error: %v", err)
+				return nil, err
+			}
+			resourceIds = append(resourceIds, strings.TrimLeft(metadataID, manager.Keyword()+"::"))
+		}
+		q = q.Filter(sqlchemy.In(q.Field("id"), resourceIds))
+	}
 	return q, nil
 }
 
@@ -163,7 +226,15 @@ func (model *SStandaloneResourceBase) SetAllMetadata(ctx context.Context, dictst
 			return httperrors.NewNotSufficientPrivilegeError("not allow to set system key %s", k)
 		}
 	}
-	return Metadata.SetAll(ctx, model, dictstore, userCred)
+	return Metadata.SetValuesWithLog(ctx, model, dictstore, userCred)
+}
+
+func (model *SStandaloneResourceBase) SetUserMetadataValues(ctx context.Context, dictstore map[string]interface{}, userCred mcclient.TokenCredential) error {
+	return Metadata.SetValuesWithLog(ctx, model, dictstore, userCred)
+}
+
+func (model *SStandaloneResourceBase) SetUserMetadataAll(ctx context.Context, dictstore map[string]interface{}, userCred mcclient.TokenCredential) error {
+	return Metadata.SetAll(ctx, model, dictstore, userCred, "user")
 }
 
 func (model *SStandaloneResourceBase) RemoveMetadata(ctx context.Context, key string, userCred mcclient.TokenCredential) error {
@@ -212,6 +283,48 @@ func (model *SStandaloneResourceBase) PerformMetadata(ctx context.Context, userC
 	return nil, err
 }
 
+func (model *SStandaloneResourceBase) AllowPerformUserMetadata(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return IsAdminAllowPerform(userCred, model, "user-metadata")
+}
+
+func (model *SStandaloneResourceBase) PerformUserMetadata(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	dict, ok := data.(*jsonutils.JSONDict)
+	if !ok {
+		return nil, httperrors.NewInputParameterError("input data not key value dict")
+	}
+	dictMap, err := dict.GetMap()
+	if err != nil {
+		return nil, err
+	}
+	dictStore := make(map[string]interface{})
+	for k, v := range dictMap {
+		dictStore["user:"+k], _ = v.GetString()
+	}
+	err = model.SetUserMetadataValues(ctx, dictStore, userCred)
+	return nil, err
+}
+
+func (model *SStandaloneResourceBase) AllowPerformSetUserMetadata(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return IsAdminAllowPerform(userCred, model, "set-user-metadata")
+}
+
+func (model *SStandaloneResourceBase) PerformSetUserMetadata(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	dict, ok := data.(*jsonutils.JSONDict)
+	if !ok {
+		return nil, httperrors.NewInputParameterError("input data not key value dict")
+	}
+	dictMap, err := dict.GetMap()
+	if err != nil {
+		return nil, err
+	}
+	dictStore := make(map[string]interface{})
+	for k, v := range dictMap {
+		dictStore["user:"+k], _ = v.GetString()
+	}
+	err = model.SetUserMetadataAll(ctx, dictStore, userCred)
+	return nil, err
+}
+
 func (model *SStandaloneResourceBase) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
 	extra := model.SResourceBase.GetCustomizeColumns(ctx, userCred, query)
 	withMeta, _ := query.GetString("with_meta")