Merge branch 'master' into feature/implement-vm-rescue

pkg/cloudcommon/cmdline/parser.go

@@ -202,6 +202,16 @@ func ParseDiskConfig(diskStr string, idx int) (*compute.DiskConfig, error) {
 			diskConfig.NVMEDevice = &compute.IsolatedDeviceConfig{
 				Model: str,
 			}
+		case "iops":
+			diskConfig.Iops, _ = strconv.Atoi(str)
+			if err != nil {
+				return nil, errors.Wrapf(err, "parse disk iops %s", str)
+			}
+		case "throughput":
+			diskConfig.Throughput, _ = strconv.Atoi(str)
+			if err != nil {
+				return nil, errors.Wrapf(err, "parse disk iops %s", str)
+			}
 		default:
 			return nil, errors.Errorf("invalid disk description %s", p)
 		}
@@ -239,6 +249,17 @@ func ParseNetworkConfig(desc string, idx int) (*compute.NetworkConfig, error) {
 			netConfig.Mac = netutils.MacUnpackHex(p)
 		} else if strings.HasPrefix(p, "wire=") {
 			netConfig.Wire = p[len("wire="):]
+		} else if strings.HasPrefix(p, "macs=") {
+			macSegs := strings.Split(p[len("macs="):], ",")
+			macs := make([]string, len(macSegs))
+			for i := range macSegs {
+				macs[i] = netutils.MacUnpackHex(macSegs[i])
+			}
+			netConfig.Macs = macs
+		} else if strings.HasPrefix(p, "ips=") {
+			netConfig.Addresses = strings.Split(p[len("ips="):], ",")
+		} else if strings.HasPrefix(p, "ip6s=") {
+			netConfig.Addresses6 = strings.Split(p[len("ip6s="):], ",")
 		} else if p == "[require_designated_ip]" {
 			netConfig.RequireDesignatedIP = true
 		} else if p == "[random_exit]" {
@@ -277,6 +298,18 @@ func ParseNetworkConfig(desc string, idx int) (*compute.NetworkConfig, error) {
 			netConfig.SriovDevice = &compute.IsolatedDeviceConfig{
 				Model: p[len("sriov-nic-model="):],
 			}
+		} else if strings.HasPrefix(p, "rx-traffic-limit=") {
+			var err error
+			netConfig.RxTrafficLimit, err = strconv.ParseInt(p[len("rx-traffic-limit="):], 10, 0)
+			if err != nil {
+				return nil, errors.Wrap(err, "parse rx-traffic-limit")
+			}
+		} else if strings.HasPrefix(p, "tx-traffic-limit=") {
+			var err error
+			netConfig.TxTrafficLimit, err = strconv.ParseInt(p[len("tx-traffic-limit="):], 10, 0)
+			if err != nil {
+				return nil, errors.Wrap(err, "parse tx-traffic-limit")
+			}
 		} else if utils.IsInStringArray(p, compute.ALL_NETWORK_TYPES) {
 			netConfig.NetType = p
 		} else {

pkg/cloudcommon/consts/deployer.go

@@ -16,6 +16,8 @@ package consts
 
 var (
 	deployTempDir = ""
+
+	allowVmSELinux = false
 )
 
 func DeployTempDir() string {
@@ -25,3 +27,11 @@ func DeployTempDir() string {
 func SetDeployTempDir(dir string) {
 	deployTempDir = dir
 }
+
+func AllowVmSELinux() bool {
+	return allowVmSELinux
+}
+
+func SetAllowVmSELinux(allow bool) {
+	allowVmSELinux = allow
+}

pkg/cloudcommon/consts/resource_translate.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package consts
 
 import (

pkg/cloudcommon/cronman/cronman.go

@@ -447,7 +447,7 @@ func (job *SCronJob) runJobInWorker(isStart bool, startTime time.Time) {
 		}
 	}()
 
-	log.Debugf("Cron job: %s started, startTime: %s", job.Name, startTime)
+	log.Debugf("Cron job: %s started, startTime: %s", job.Name, startTime.Format(time.RFC3339))
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_APPNAME, fmt.Sprintf("%s/cron-service", consts.GetServiceName()))
 	ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_TASKNAME, fmt.Sprintf("%s-%d", job.Name, time.Now().Unix()))

pkg/cloudcommon/db/caller.go

@@ -253,6 +253,22 @@ func ValidateCreateData(funcName string, manager IModelManager, ctx context.Cont
 	return mergeInputOutputData(data, resVal), nil
 }
 
+func ExpandBatchCreateData(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict, index int) (*jsonutils.JSONDict, error) {
+	funcName := "ExpandBatchCreateData"
+	ret, err := call(manager, funcName, ctx, userCred, ownerId, query, data, index)
+	if err != nil {
+		return nil, errors.Wrapf(err, "call %s", funcName)
+	}
+	if len(ret) != 2 {
+		return nil, httperrors.NewInternalServerError("Invald %s return value", funcName)
+	}
+	resVal := ret[0]
+	if err := ValueToError(ret[1]); err != nil {
+		return nil, errors.Wrap(err, "ValueToError")
+	}
+	return mergeInputOutputData(data, resVal), nil
+}
+
 func ListItemFilter(manager IModelManager, ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
 	return _callListQueryFilter(manager, "ListItemFilter", ctx, q, userCred, query)
 }

pkg/cloudcommon/db/db_dispatcher.go

@@ -573,6 +573,12 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 	pagingOrderStr, _ := query.GetString("paging_order")
 	pagingOrder := sqlchemy.QueryOrderType(strings.ToUpper(pagingOrderStr))
 
+	// export data only
+	exportLimit, err := query.Int("export_limit")
+	if query.Contains("export_keys") && err == nil {
+		limit = exportLimit
+	}
+
 	var (
 		q           *sqlchemy.SQuery
 		useRawQuery bool
@@ -703,12 +709,6 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 		limit = maxLimit
 	}
 
-	// export data only
-	exportLimit, err := query.Int("export_limit")
-	if query.Contains("export_keys") && err == nil {
-		limit = exportLimit
-	}
-
 	// orders defined in pagingConf should have the highest priority
 	if pagingConf != nil {
 		for _, f := range pagingConf.MarkerFields {
@@ -1427,12 +1427,20 @@ func (dispatcher *DBModelDispatcher) Create(ctx context.Context, query jsonutils
 		notes := model.GetShortDesc(ctx)
 		OpsLog.LogEvent(model, ACT_CREATE, notes, userCred)
 		logclient.AddActionLogWithContext(ctx, model, logclient.ACT_CREATE, notes, userCred, true)
+		CallCreateNotifyHook(ctx, userCred, model)
 	}
-	manager.OnCreateComplete(ctx, []IModel{model}, userCred, ownerId, query, data)
+	manager.OnCreateComplete(ctx, []IModel{model}, userCred, ownerId, query, []jsonutils.JSONObject{data})
 	return getItemDetails(manager, model, ctx, userCred, query)
 }
 
-func expandMultiCreateParams(manager IModelManager, data jsonutils.JSONObject, count int) ([]jsonutils.JSONObject, error) {
+func expandMultiCreateParams(manager IModelManager,
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	ownerId mcclient.IIdentityProvider,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject,
+	count int,
+) ([]jsonutils.JSONObject, error) {
 	jsonDict, ok := data.(*jsonutils.JSONDict)
 	if !ok {
 		return nil, httperrors.NewInputParameterError("body is not a json?")
@@ -1450,7 +1458,16 @@ func expandMultiCreateParams(manager IModelManager, data jsonutils.JSONObject, c
 	}
 	ret := make([]jsonutils.JSONObject, count)
 	for i := 0; i < count; i += 1 {
-		ret[i] = jsonDict.Copy()
+		input, err := ExpandBatchCreateData(manager, ctx, userCred, ownerId, query, jsonDict.Copy(), i)
+		if err != nil {
+			if errors.Cause(err) == MethodNotFoundError {
+				ret[i] = jsonDict.Copy()
+			} else {
+				return nil, errors.Wrap(err, "ExpandBatchCreateData")
+			}
+		} else {
+			ret[i] = input
+		}
 	}
 	return ret, nil
 }
@@ -1508,7 +1525,7 @@ func (dispatcher *DBModelDispatcher) BatchCreate(ctx context.Context, query json
 			return nil, errors.Wrap(err, "manager.BatchPreValidate")
 		}
 
-		multiData, err = expandMultiCreateParams(manager, data, count)
+		multiData, err = expandMultiCreateParams(manager, ctx, userCred, ownerId, query, data, count)
 		if err != nil {
 			return nil, errors.Wrap(err, "expandMultiCreateParams")
 		}
@@ -1517,6 +1534,7 @@ func (dispatcher *DBModelDispatcher) BatchCreate(ctx context.Context, query json
 		ret := make([]sCreateResult, len(multiData))
 		for i := range multiData {
 			var model IModel
+			log.Debugf("batchCreateDoCreateItem %d %s", i, multiData[i].String())
 			model, err = batchCreateDoCreateItem(manager, ctx, userCred, ownerId, query, multiData[i], i+1)
 			if err == nil {
 				ret[i] = sCreateResult{model: model, err: nil}
@@ -1576,7 +1594,7 @@ func (dispatcher *DBModelDispatcher) BatchCreate(ctx context.Context, query json
 		lockman.LockClass(ctx, manager, GetLockClassKey(manager, ownerId))
 		defer lockman.ReleaseClass(ctx, manager, GetLockClassKey(manager, ownerId))
 
-		manager.OnCreateComplete(ctx, models, userCred, ownerId, query, multiData[0])
+		manager.OnCreateComplete(ctx, models, userCred, ownerId, query, multiData)
 	}
 	return results, nil
 }
@@ -1796,6 +1814,9 @@ func (dispatcher *DBModelDispatcher) FetchUpdateHeaderData(ctx context.Context,
 
 func (dispatcher *DBModelDispatcher) Update(ctx context.Context, idStr string, query jsonutils.JSONObject, data jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (jsonutils.JSONObject, error) {
 	userCred := fetchUserCredential(ctx)
+	if data == nil {
+		data = jsonutils.NewDict()
+	}
 	manager := dispatcher.manager.GetMutableInstance(ctx, userCred, query, data)
 	model, err := fetchItem(manager, ctx, userCred, idStr, nil)
 	if err == sql.ErrNoRows {
@@ -1857,6 +1878,7 @@ func DeleteModel(ctx context.Context, userCred mcclient.TokenCredential, item IM
 	if userCred != nil {
 		OpsLog.LogEvent(item, ACT_DELETE, item.GetShortDesc(ctx), userCred)
 		logclient.AddSimpleActionLog(item, logclient.ACT_DELETE, item.GetShortDesc(ctx), userCred, true)
+		CallDeleteNotifyHook(ctx, userCred, item)
 	}
 	if _, ok := item.(IStandaloneModel); ok && len(item.GetId()) > 0 {
 		err := Metadata.RemoveAll(ctx, item, userCred)

pkg/cloudcommon/db/db_joint_dispatcher.go

@@ -190,7 +190,7 @@ func attachItems(
 	}
 	item.PostCreate(ctx, userCred, nil, query, data)
 	OpsLog.LogAttachEvent(ctx, master, slave, userCred, jsonutils.Marshal(item))
-	dispatcher.manager.OnCreateComplete(ctx, []IModel{item}, userCred, nil, query, data)
+	dispatcher.manager.OnCreateComplete(ctx, []IModel{item}, userCred, nil, query, []jsonutils.JSONObject{data})
 	return getItemDetails(dispatcher.JointModelManager(), item, ctx, userCred, query)
 }
 

pkg/cloudcommon/db/enabledbase.go

@@ -33,12 +33,16 @@ type SEnabledResourceBase struct {
 	Enabled tristate.TriState `default:"false" list:"user" create:"optional"`
 }
 
-type IEnabledBase interface {
-	IModel
+type IEnabledBaseInterface interface {
 	SetEnabled(enabled bool)
 	GetEnabled() bool
 }
 
+type IEnabledBase interface {
+	IModel
+	IEnabledBaseInterface
+}
+
 func (m *SEnabledResourceBase) SetEnabled(enabled bool) {
 	if enabled {
 		m.Enabled = tristate.True

pkg/cloudcommon/db/infraresource.go

@@ -47,7 +47,7 @@ func NewInfrasResourceBaseManager(
 
 type SInfrasResourceBase struct {
 	SDomainLevelResourceBase
-	SSharableBaseResource `"is_public=>create":"domain_optional" "public_scope=>create":"domain_optional"`
+	SSharableBaseResource `"is_public->create":"domain_optional" "public_scope->create":"domain_optional"`
 }
 
 func (manager *SInfrasResourceBaseManager) GetIInfrasModelManager() IInfrasModelManager {
@@ -66,10 +66,6 @@ func (model *SInfrasResourceBase) IsShared() bool {
 	return SharableModelIsShared(model)
 }
 
-func (model *SInfrasResourceBase) AllowPerformPublic(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformPublicDomainInput) bool {
-	return true
-}
-
 func (model *SInfrasResourceBase) PerformPublic(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformPublicDomainInput) (jsonutils.JSONObject, error) {
 	err := SharablePerformPublic(model.GetIInfrasModel(), ctx, userCred, apis.PerformPublicProjectInput{PerformPublicDomainInput: input})
 	if err != nil {
@@ -78,10 +74,6 @@ func (model *SInfrasResourceBase) PerformPublic(ctx context.Context, userCred mc
 	return nil, nil
 }
 
-func (model *SInfrasResourceBase) AllowPerformPrivate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformPrivateInput) bool {
-	return true
-}
-
 func (model *SInfrasResourceBase) PerformPrivate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformPrivateInput) (jsonutils.JSONObject, error) {
 	err := SharablePerformPrivate(model.GetIInfrasModel(), ctx, userCred)
 	if err != nil {

pkg/cloudcommon/db/interface.go

@@ -24,6 +24,7 @@ import (
 	"yunion.io/x/pkg/util/rbacscope"
 	"yunion.io/x/sqlchemy"
 
+	"yunion.io/x/onecloud/pkg/apis"
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -95,15 +96,13 @@ type IModelManager interface {
 	// BatchCreateValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error)
 	// ValidateCreateData dynamic called by dispatcher
 	// ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error)
-	OnCreateComplete(ctx context.Context, items []IModel, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject)
+	OnCreateComplete(ctx context.Context, items []IModel, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data []jsonutils.JSONObject)
 	BatchPreValidate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider,
 		query jsonutils.JSONObject, data *jsonutils.JSONDict, count int) error
 
 	OnCreateFailed(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error
 
 	// allow perform action
-	// AllowPerformAction(ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) bool
-	// AllowPerformCheckCreateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool
 	PerformAction(ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error)
 
 	// DoCreate(ctx context.Context, userCred mcclient.TokenCredential, kwargs jsonutils.JSONObject, data jsonutils.JSONObject, realManager IModelManager) (IModel, error)
@@ -167,7 +166,6 @@ type IModel interface {
 	//GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict
 
 	// get hooks
-	// AllowGetDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool
 	GetExtraDetailsHeaders(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) map[string]string
 
 	// before create hooks
@@ -176,7 +174,6 @@ type IModel interface {
 	PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject)
 
 	// allow perform action
-	// AllowPerformAction(ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) bool
 	PerformAction(ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error)
 	PreCheckPerformAction(ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) error
 
@@ -284,6 +281,8 @@ type IStandaloneModel interface {
 	// IsAlterNameUnique(name string, projectId string) bool
 	// GetExternalId() string
 
+	SetName(name string)
+
 	StandaloneModelManager() IStandaloneModelManager
 
 	GetIStandaloneModel() IStandaloneModel
@@ -363,6 +362,7 @@ type IVirtualModel interface {
 	IsOwner(userCred mcclient.TokenCredential) bool
 	// IsAdmin(userCred mcclient.TokenCredential) bool
 
+	SetProjectSrc(apis.TOwnerSource)
 	SyncCloudProjectId(userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider)
 
 	GetIVirtualModel() IVirtualModel

pkg/cloudcommon/db/keystonecache.go

@@ -26,6 +26,7 @@ type SKeystoneCacheObjectManager struct {
 
 type SKeystoneCacheObject struct {
 	SStandaloneResourceBase
+	SPendingDeletedBase
 
 	DomainId string `width:"128" charset:"ascii" nullable:"true"`
 	Domain   string `width:"128" charset:"utf8" nullable:"true"`

pkg/cloudcommon/db/metadata.go

@@ -435,6 +435,9 @@ func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy
 }
 
 func (manager *SMetadataManager) GetStringValue(ctx context.Context, model IModel, key string, userCred mcclient.TokenCredential) string {
+	if !isAllowGetMetadata(ctx, model, userCred) {
+		return ""
+	}
 	if strings.HasPrefix(key, SYSTEM_ADMIN_PREFIX) && (userCred == nil || !IsAllowGetSpec(ctx, rbacscope.ScopeSystem, userCred, model, "metadata")) {
 		return ""
 	}
@@ -448,6 +451,9 @@ func (manager *SMetadataManager) GetStringValue(ctx context.Context, model IMode
 }
 
 func (manager *SMetadataManager) GetJsonValue(ctx context.Context, model IModel, key string, userCred mcclient.TokenCredential) jsonutils.JSONObject {
+	if !isAllowGetMetadata(ctx, model, userCred) {
+		return nil
+	}
 	if strings.HasPrefix(key, SYSTEM_ADMIN_PREFIX) && (userCred == nil || !IsAllowGetSpec(ctx, rbacscope.ScopeSystem, userCred, model, "metadata")) {
 		return nil
 	}
@@ -641,7 +647,26 @@ func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store m
 	return nil
 }
 
+func isAllowGetMetadata(ctx context.Context, obj IModel, userCred mcclient.TokenCredential) bool {
+	if userCred != nil {
+		for _, scope := range []rbacscope.TRbacScope{
+			rbacscope.ScopeSystem,
+			rbacscope.ScopeDomain,
+			rbacscope.ScopeProject,
+		} {
+			if IsAllowGetSpec(ctx, scope, userCred, obj, "metadata") {
+				return true
+			}
+		}
+		return false
+	}
+	return true
+}
+
 func (manager *SMetadataManager) GetAll(ctx context.Context, obj IModel, keys []string, keyPrefix string, userCred mcclient.TokenCredential) (map[string]string, error) {
+	if !isAllowGetMetadata(ctx, obj, userCred) {
+		return map[string]string{}, nil
+	}
 	meta, err := manager.rawGetAll(obj.Keyword(), obj.GetId(), keys, keyPrefix)
 	if err != nil {
 		return nil, errors.Wrap(err, "rawGetAll")
@@ -660,7 +685,7 @@ func (manager *SMetadataManager) rawGetAll(objType, objId string, keys []string,
 	idStr := getObjectIdstr(objType, objId)
 	records := make([]SMetadata, 0)
 	q := manager.Query().Equals("id", idStr)
-	if keys != nil && len(keys) > 0 {
+	if len(keys) > 0 {
 		q = q.In("key", keys)
 	}
 	if len(keyPrefix) > 0 {
@@ -679,15 +704,19 @@ func (manager *SMetadataManager) rawGetAll(objType, objId string, keys []string,
 	return ret, nil
 }
 
-func (manager *SMetadataManager) IsSystemAdminKey(key string) bool {
-	return IsMetadataKeySystemAdmin(key)
+/*func (manager *SMetadataManager) IsSystemAdminKey(key string) bool {
+	return isMetadataKeySystemAdmin(key)
+}*/
+
+func isMetadataLoginKey(key string) bool {
+	return strings.HasPrefix(key, "login_")
 }
 
-func IsMetadataKeySystemAdmin(key string) bool {
+func isMetadataKeySystemAdmin(key string) bool {
 	return strings.HasPrefix(key, SYSTEM_ADMIN_PREFIX)
 }
 
-func IsMetadataKeyPrivateKey(key string) bool {
+func isMetadataKeyPrivateKey(key string) bool {
 	for _, k := range []string{"admin", "project"} {
 		for _, v := range []string{"ssh-private-key", "ssh-public-key"} {
 			if key == fmt.Sprintf("%s-%s", k, v) {
@@ -698,7 +727,7 @@ func IsMetadataKeyPrivateKey(key string) bool {
 	return strings.HasPrefix(key, SYSTEM_ADMIN_PREFIX)
 }
 
-func IsMetadataKeySysTag(key string) bool {
+func isMetadataKeySysTag(key string) bool {
 	return strings.HasPrefix(key, SYS_TAG_PREFIX)
 }
 
@@ -706,11 +735,11 @@ func (manager *SMetadataManager) GetSysadminKey(key string) string {
 	return fmt.Sprintf("%s%s", SYSTEM_ADMIN_PREFIX, key)
 }
 
-func IsMetadataKeyVisiable(key string) bool {
-	return !(IsMetadataKeySysTag(key) || IsMetadataKeySystemAdmin(key) || IsMetadataKeyPrivateKey(key))
+func IsMetadataKeyVisible(key string) bool {
+	return !(isMetadataKeySysTag(key) || isMetadataKeySystemAdmin(key) || isMetadataKeyPrivateKey(key))
 }
 
-func GetVisiableMetadata(ctx context.Context, model IStandaloneModel, userCred mcclient.TokenCredential) (map[string]string, error) {
+func GetVisibleMetadata(ctx context.Context, model IStandaloneModel, userCred mcclient.TokenCredential) (map[string]string, error) {
 	metaData, err := model.GetAllMetadata(ctx, userCred)
 	if err != nil {
 		return nil, err
@@ -719,7 +748,7 @@ func GetVisiableMetadata(ctx context.Context, model IStandaloneModel, userCred m
 		delete(metaData, key)
 	}
 	for key := range metaData {
-		if !IsMetadataKeyVisiable(key) {
+		if !IsMetadataKeyVisible(key) {
 			delete(metaData, key)
 		}
 	}
@@ -731,7 +760,7 @@ func metaList2Map(manager IMetadataBaseModelManager, userCred mcclient.TokenCred
 
 	hiddenKeys := manager.GetMetadataHiddenKeys()
 	for _, meta := range metaList {
-		if IsMetadataKeyVisiable(meta.Key) && !utils.IsInStringArray(meta.Key, hiddenKeys) {
+		if IsMetadataKeyVisible(meta.Key) && !utils.IsInStringArray(meta.Key, hiddenKeys) {
 			metaMap[meta.Key] = meta.Value
 		}
 	}

pkg/cloudcommon/db/metadata_test.go

@@ -43,7 +43,7 @@ func TestIsMetadataKeySystemAdmin(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := IsMetadataKeySystemAdmin(tt.key); got != tt.want {
+			if got := isMetadataKeySystemAdmin(tt.key); got != tt.want {
 				t.Errorf("IsMetadataKeySystemAdmin() = %v, want %v", got, tt.want)
 			}
 		})
@@ -69,7 +69,7 @@ func TestIsMetadataKeySysTag(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := IsMetadataKeySysTag(tt.key); got != tt.want {
+			if got := isMetadataKeySysTag(tt.key); got != tt.want {
 				t.Errorf("IsMetadataKeySysTag() = %v, want %v", got, tt.want)
 			}
 		})
@@ -100,7 +100,7 @@ func TestIsMetadataKeyVisiable(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := IsMetadataKeyVisiable(tt.key); got != tt.want {
+			if got := IsMetadataKeyVisible(tt.key); got != tt.want {
 				t.Errorf("IsMetadataKeyVisiable() = %v, want %v", got, tt.want)
 			}
 		})
@@ -161,7 +161,7 @@ func TestGetVisiableMetadata(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := GetVisiableMetadata(nil, tt.model, nil)
+			got, err := GetVisibleMetadata(nil, tt.model, nil)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("GetVisiableMetadata() error = %v, wantErr %v", err, tt.wantErr)
 				return

pkg/cloudcommon/db/modelbase.go

@@ -271,14 +271,10 @@ func (manager *SModelBaseManager) ValidateCreateData(ctx context.Context, userCr
 	return input, nil
 }
 
-func (manager *SModelBaseManager) OnCreateComplete(ctx context.Context, items []IModel, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) {
+func (manager *SModelBaseManager) OnCreateComplete(ctx context.Context, items []IModel, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data []jsonutils.JSONObject) {
 	// do nothing
 }
 
-func (manager *SModelBaseManager) AllowPerformAction(ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
-	return false
-}
-
 func (manager *SModelBaseManager) PerformAction(ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	return nil, httperrors.NewActionNotFoundError("Action %s not found, please check service version, current version: %s", action, version.GetShortString())
 }
@@ -530,10 +526,6 @@ func (manager *SModelBaseManager) GetPropertySplitableExport(ctx context.Context
 	return nil, httperrors.NewResourceNotFoundError("table %s not found", input.Table)
 }
 
-func (manager *SModelBaseManager) AllowPerformPurgeSplitable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
-	return true
-}
-
 func (manager *SModelBaseManager) PerformPurgeSplitable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PurgeSplitTableInput) (jsonutils.JSONObject, error) {
 	splitable := manager.GetIModelManager().GetImmutableInstance(ctx, userCred, query).GetSplitTable()
 	if splitable == nil {
@@ -606,11 +598,6 @@ func (model *SModelBase) GetShortDescV2(ctx context.Context) *apis.ModelBaseShor
 	return &apis.ModelBaseShortDescDetail{ResName: model.Keyword()}
 }
 
-// get hooks
-func (model *SModelBase) AllowGetDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
-	return false
-}
-
 func (model *SModelBase) GetExtraDetailsHeaders(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) map[string]string {
 	return nil
 }
@@ -624,10 +611,6 @@ func (model *SModelBase) PostCreate(ctx context.Context, userCred mcclient.Token
 
 }
 
-func (model *SModelBase) AllowPerformAction(ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
-	return false
-}
-
 func (model *SModelBase) PerformAction(ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	return nil, httperrors.NewActionNotFoundError("Action %s not found, please check service version, current version: %s", action, version.GetShortString())
 }

pkg/cloudcommon/db/notify_hook.go

@@ -22,9 +22,13 @@ import (
 
 var (
 	updateNotifyHook updateNotifyHookFunc
+	createNotifyHook createNotifyHookFunc
+	deleteNotifyHook deleteNotifyHookFunc
 )
 
 type updateNotifyHookFunc func(ctx context.Context, userCred mcclient.TokenCredential, obj IModel)
+type createNotifyHookFunc func(ctx context.Context, userCred mcclient.TokenCredential, obj IModel)
+type deleteNotifyHookFunc func(ctx context.Context, userCred mcclient.TokenCredential, obj IModel)
 
 func SetUpdateNotifyHook(f updateNotifyHookFunc) {
 	if updateNotifyHook != nil {
@@ -33,9 +37,37 @@ func SetUpdateNotifyHook(f updateNotifyHookFunc) {
 	updateNotifyHook = f
 }
 
+func SetCreateNotifyHook(f createNotifyHookFunc) {
+	if createNotifyHook != nil {
+		panic("createNotifyHook already set")
+	}
+	createNotifyHook = f
+}
+
+func SetDeleteNotifyHook(f deleteNotifyHookFunc) {
+	if deleteNotifyHook != nil {
+		panic("deleteNotifyHook already set")
+	}
+	deleteNotifyHook = f
+}
+
 func CallUpdateNotifyHook(ctx context.Context, userCred mcclient.TokenCredential, obj IModel) {
 	if updateNotifyHook == nil {
 		return
 	}
 	updateNotifyHook(ctx, userCred, obj)
 }
+
+func CallCreateNotifyHook(ctx context.Context, userCred mcclient.TokenCredential, obj IModel) {
+	if createNotifyHook == nil {
+		return
+	}
+	createNotifyHook(ctx, userCred, obj)
+}
+
+func CallDeleteNotifyHook(ctx context.Context, userCred mcclient.TokenCredential, obj IModel) {
+	if deleteNotifyHook == nil {
+		return
+	}
+	deleteNotifyHook(ctx, userCred, obj)
+}

pkg/cloudcommon/db/opslog.go

@@ -46,7 +46,7 @@ type SOpsLogManager struct {
 type SOpsLog struct {
 	SLogBase
 
-	ObjType string `width:"40" charset:"ascii" nullable:"false" list:"user" create:"required"`
+	ObjType string `width:"40" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"`
 	ObjId   string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"`
 	ObjName string `width:"128" charset:"utf8" nullable:"false" list:"user" create:"required"`
 	Action  string `width:"32" charset:"utf8" nullable:"false" list:"user" create:"required"`

pkg/cloudcommon/db/opslog_const.go

@@ -317,6 +317,9 @@ const (
 	ACT_ENCRYPT_FAIL  = "encrypt_fail"
 	ACT_ENCRYPT_DONE  = "encrypted"
 
+	ACT_SYNC_TRAFFIC_LIMIT      = "sync_traffic_limit"
+	ACT_SYNC_TRAFFIC_LIMIT_FAIL = "sync_traffic_limit_fail"
+
 	ACT_BIND   = "bind"
 	ACT_UNBIND = "unbind"
 

pkg/cloudcommon/db/pending_deleted.go

@@ -0,0 +1,125 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package db
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/rbacscope"
+	"yunion.io/x/pkg/util/timeutils"
+	"yunion.io/x/pkg/utils"
+	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
+	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/logclient"
+)
+
+type SPendingDeletedBaseManager struct{}
+
+type SPendingDeletedBase struct {
+	// 资源放入回收站时间
+	PendingDeletedAt time.Time `json:"pending_deleted_at" list:"user" update:"admin"`
+	// 资源是否处于回收站中
+	PendingDeleted bool `nullable:"false" default:"false" index:"true" get:"user" list:"user" json:"pending_deleted"`
+}
+
+// GetPendingDeleted implements IPendingDeltable
+func (base *SPendingDeletedBase) GetPendingDeleted() bool {
+	return base.PendingDeleted
+}
+
+// GetPendingDeletedAt implements IPendingDeltable
+func (base *SPendingDeletedBase) GetPendingDeletedAt() time.Time {
+	return base.PendingDeletedAt
+}
+
+func (base *SPendingDeletedBaseManager) FilterBySystemAttributes(manager IStandaloneModelManager, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject, scope rbacscope.TRbacScope) *sqlchemy.SQuery {
+	var pendingDelete string
+	if query != nil {
+		pendingDelete, _ = query.GetString("pending_delete")
+	}
+	pendingDeleteLower := strings.ToLower(pendingDelete)
+	if pendingDeleteLower == "all" || pendingDeleteLower == "any" || utils.ToBool(pendingDeleteLower) {
+		var isAllow bool
+		allowScope, result := policy.PolicyManager.AllowScope(userCred, consts.GetServiceType(), manager.KeywordPlural(), policy.PolicyActionList, "pending_delete")
+		if result.Result.IsAllow() && !scope.HigherThan(allowScope) {
+			isAllow = true
+		}
+		if !isAllow {
+			pendingDeleteLower = ""
+		}
+	}
+
+	if pendingDeleteLower == "all" || pendingDeleteLower == "any" {
+	} else if utils.ToBool(pendingDeleteLower) {
+		q = q.IsTrue("pending_deleted")
+	} else {
+		q = q.Filter(sqlchemy.OR(sqlchemy.IsNull(q.Field("pending_deleted")), sqlchemy.IsFalse(q.Field("pending_deleted"))))
+	}
+	return q
+}
+
+func (base *SPendingDeletedBase) MarkPendingDelete(model IStandaloneModel, ctx context.Context, userCred mcclient.TokenCredential, newName string) error {
+	if !base.PendingDeleted {
+		_, err := Update(model, func() error {
+			if len(newName) > 0 {
+				model.SetName(newName)
+			}
+			base.PendingDeleted = true
+			base.PendingDeletedAt = timeutils.UtcNow()
+			return nil
+		})
+		if err != nil {
+			log.Errorf("MarkPendingDelete update fail %s", err)
+			return errors.Wrap(err, "MarkPendingDelete.Update")
+		}
+		OpsLog.LogEvent(model, ACT_PENDING_DELETE, model.GetShortDesc(ctx), userCred)
+		logclient.AddSimpleActionLog(model, logclient.ACT_PENDING_DELETE, model.GetShortDesc(ctx), userCred, true)
+	}
+	return nil
+}
+
+func (base *SPendingDeletedBase) MarkCancelPendingDelete(model IStandaloneModel, ctx context.Context, userCred mcclient.TokenCredential) error {
+	manager := model.GetModelManager()
+	ownerId := model.GetOwnerId()
+
+	lockman.LockRawObject(ctx, manager.Keyword(), "name")
+	defer lockman.ReleaseRawObject(ctx, manager.Keyword(), "name")
+
+	newName, err := GenerateName(ctx, manager, ownerId, model.GetName())
+	if err != nil {
+		return errors.Wrapf(err, "GenerateNam")
+	}
+	_, err = Update(model, func() error {
+		model.SetName(newName)
+		base.PendingDeleted = false
+		base.PendingDeletedAt = time.Time{}
+		return nil
+	})
+	if err != nil {
+		return errors.Wrapf(err, "MarkCancelPendingDelete.Update")
+	}
+	OpsLog.LogEvent(model, ACT_CANCEL_DELETE, model.GetShortDesc(ctx), userCred)
+	logclient.AddSimpleActionLog(model, logclient.ACT_CANCEL_DELETE, model.GetShortDesc(ctx), userCred, true)
+	return nil
+}

pkg/cloudcommon/db/project.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/rbacscope"
 	"yunion.io/x/pkg/util/reflectutils"
@@ -189,12 +190,14 @@ func (manager *SProjectizedResourceBaseManager) FetchCustomizeColumns(
 	isList bool,
 ) []apis.ProjectizedResourceInfo {
 	ret := make([]apis.ProjectizedResourceInfo, len(objs))
+	resIds := make([]string, len(objs))
 	if len(fields) == 0 || fields.Contains("project_domain") || fields.Contains("tenant") {
 		projectIds := stringutils2.SSortedStrings{}
 		for i := range objs {
 			var base *SProjectizedResourceBase
 			reflectutils.FindAnonymouStructPointer(objs[i], &base)
 			if base != nil && len(base.ProjectId) > 0 {
+				resIds[i] = getObjectIdstr("project", base.ProjectId)
 				projectIds = stringutils2.Append(projectIds, base.ProjectId)
 			}
 		}
@@ -216,6 +219,26 @@ func (manager *SProjectizedResourceBaseManager) FetchCustomizeColumns(
 			}
 		}
 	}
+
+	if fields == nil || fields.Contains("__meta__") {
+		q := Metadata.Query("id", "key", "value")
+		metaKeyValues := make(map[string][]SMetadata)
+		err := FetchQueryObjectsByIds(q, "id", resIds, &metaKeyValues)
+		if err != nil {
+			log.Errorf("FetchQueryObjectsByIds metadata fail %s", err)
+			return ret
+		}
+
+		for i := range objs {
+			if metaList, ok := metaKeyValues[resIds[i]]; ok {
+				ret[i].ProjectMetadata = map[string]string{}
+				for _, meta := range metaList {
+					ret[i].ProjectMetadata[meta.Key] = meta.Value
+				}
+			}
+		}
+	}
+
 	domainRows := manager.SDomainizedResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
 	for i := range ret {
 		ret[i].DomainizedResourceInfo = domainRows[i]

pkg/cloudcommon/db/rolecache.go

@@ -122,6 +122,7 @@ func (manager *SRoleCacheManager) FetchRoleFromKeystone(ctx context.Context, idS
 	query := jsonutils.NewDict()
 	query.Set("scope", jsonutils.NewString("system"))
 	query.Set("system", jsonutils.JSONTrue)
+	query.Set("pending_delete", jsonutils.NewString("all"))
 
 	s := auth.GetAdminSession(ctx, consts.GetRegion())
 	role, err := modules.RolesV3.GetById(s, idStr, query)

pkg/cloudcommon/db/sharablebase.go

@@ -217,7 +217,47 @@ func SharableManagerValidateCreateData(
 func SharableManagerFilterByOwner(manager IStandaloneModelManager, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, owner mcclient.IIdentityProvider, scope rbacscope.TRbacScope) *sqlchemy.SQuery {
 	if owner != nil {
 		resScope := manager.ResourceScope()
-		if resScope == rbacscope.ScopeProject && scope == rbacscope.ScopeProject {
+		if resScope == rbacscope.ScopeUser {
+			targetProjectId := owner.GetProjectId()
+			if len(targetProjectId) == 0 {
+				targetProjectId = userCred.GetProjectId()
+			}
+			subq := SharedResourceManager.Query("resource_id")
+			subq = subq.Equals("resource_type", manager.Keyword())
+			subq = subq.Equals("target_project_id", targetProjectId)
+			subq = subq.Equals("target_type", SharedTargetProject)
+			subq2 := SharedResourceManager.Query("resource_id")
+			subq2 = subq2.Equals("resource_type", manager.Keyword())
+			subq2 = subq2.Equals("target_project_id", owner.GetProjectDomainId())
+			subq2 = subq2.Equals("target_type", SharedTargetDomain)
+			filters := []sqlchemy.ICondition{
+				sqlchemy.AND(
+					sqlchemy.IsTrue(q.Field("is_public")),
+					sqlchemy.Equals(q.Field("public_scope"), rbacscope.ScopeSystem),
+				),
+				sqlchemy.AND(
+					sqlchemy.IsTrue(q.Field("is_public")),
+					sqlchemy.Equals(q.Field("public_scope"), rbacscope.ScopeDomain),
+					sqlchemy.OR(
+						sqlchemy.In(q.Field("id"), subq2.SubQuery()),
+					),
+				),
+				sqlchemy.In(q.Field("id"), subq.SubQuery()),
+			}
+			ownerUserId := owner.GetUserId()
+			if len(ownerUserId) > 0 {
+				filters = append(filters, sqlchemy.Equals(q.Field("owner_id"), ownerUserId))
+			}
+			q = q.Filter(sqlchemy.OR(filters...))
+			if userCred != nil {
+				result := policy.PolicyManager.Allow(scope, userCred, consts.GetServiceType(), manager.KeywordPlural(), policy.PolicyActionList)
+				if !result.ObjectTags.IsEmpty() {
+					policyTagFilters := tagutils.STagFilters{}
+					policyTagFilters.AddFilters(result.ObjectTags)
+					q = ObjectIdQueryWithTagFilters(q, "id", manager.Keyword(), policyTagFilters)
+				}
+			}
+		} else if resScope == rbacscope.ScopeProject && scope == rbacscope.ScopeProject {
 			ownerProjectId := owner.GetProjectId()
 			if len(ownerProjectId) > 0 {
 				subq := SharedResourceManager.Query("resource_id")

pkg/cloudcommon/db/sharablevirtual.go

@@ -30,7 +30,7 @@ import (
 
 type SSharableVirtualResourceBase struct {
 	SVirtualResourceBase
-	SSharableBaseResource `"is_public=>create":"optional" "public_scope=>create":"optional"`
+	SSharableBaseResource `"is_public->create":"optional" "public_scope->create":"optional"`
 	// IsPublic    bool   `default:"false" nullable:"false" create:"domain_optional" list:"user" json:"is_public"`
 	// PublicScope string `width:"16" charset:"ascii" nullable:"false" default:"system" create:"domain_optional" list:"user" json:"public_scope"`
 }
@@ -60,10 +60,6 @@ func (model *SSharableVirtualResourceBase) IsShared() bool {
 	return SharableModelIsShared(model)
 }
 
-func (model *SSharableVirtualResourceBase) AllowPerformPublic(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformPublicProjectInput) bool {
-	return true
-}
-
 func (model *SSharableVirtualResourceBase) PerformPublic(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformPublicProjectInput) (jsonutils.JSONObject, error) {
 	err := SharablePerformPublic(model.GetISharableVirtualModel(), ctx, userCred, input)
 	if err != nil {
@@ -72,10 +68,6 @@ func (model *SSharableVirtualResourceBase) PerformPublic(ctx context.Context, us
 	return nil, nil
 }
 
-func (model *SSharableVirtualResourceBase) AllowPerformPrivate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformPrivateInput) bool {
-	return true
-}
-
 func (model *SSharableVirtualResourceBase) PerformPrivate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformPrivateInput) (jsonutils.JSONObject, error) {
 	err := SharablePerformPrivate(model.GetISharableVirtualModel(), ctx, userCred)
 	if err != nil {

pkg/cloudcommon/db/sharedresource.go

@@ -99,6 +99,14 @@ func (manager *SSharedResourceManager) shareToTarget(
 	var requireScope rbacscope.TRbacScope
 	resScope := model.GetModelManager().ResourceScope()
 	switch resScope {
+	case rbacscope.ScopeUser:
+		switch targetType {
+		case SharedTargetDomain:
+			// should have system-level privileges
+			requireScope = rbacscope.ScopeSystem
+		case SharedTargetProject:
+			requireScope = rbacscope.ScopeDomain
+		}
 	case rbacscope.ScopeProject:
 		switch targetType {
 		case SharedTargetProject:

pkg/cloudcommon/db/standalone.go

@@ -139,6 +139,10 @@ func (model *SStandaloneResourceBase) GetName() string {
 	return model.Name
 }
 
+func (model *SStandaloneResourceBase) SetName(name string) {
+	model.Name = name
+}
+
 func (model *SStandaloneResourceBase) GetIStandaloneModel() IStandaloneModel {
 	return model.GetVirtualObject().(IStandaloneModel)
 }

pkg/cloudcommon/db/standalone_anon.go

@@ -30,7 +30,6 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
-	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 	"yunion.io/x/onecloud/pkg/util/tagutils"
 )
@@ -48,7 +47,7 @@ type SStandaloneAnonResourceBase struct {
 	Id string `width:"128" charset:"ascii" primary:"true" list:"user" create:"optional" json:"id"`
 
 	// 资源描述信息
-	Description string `width:"256" charset:"utf8" get:"user" list:"user" update:"user" create:"optional" json:"description"`
+	Description string `length:"0" charset:"utf8" get:"user" list:"user" update:"user" create:"optional" json:"description"`
 
 	// 是否是模拟资源, 部分从公有云上同步的资源并不真实存在, 例如宿主机
 	// list 接口默认不会返回这类资源，除非显示指定 is_emulate=true 过滤参数
@@ -393,8 +392,8 @@ func (model *SStandaloneAnonResourceBase) SetClassMetadataAll(ctx context.Contex
 	return nil
 }
 
-func (model *SStandaloneAnonResourceBase) InheritTo(ctx context.Context, dest IClassMetadataSetter) error {
-	return InheritFromTo(ctx, model, dest)
+func (model *SStandaloneAnonResourceBase) InheritTo(ctx context.Context, userCred mcclient.TokenCredential, dest IClassMetadataSetter) error {
+	return InheritFromTo(ctx, userCred, model, dest)
 }
 
 type IClassMetadataSetter interface {
@@ -404,7 +403,7 @@ type IClassMetadataSetter interface {
 	SetClassMetadataAll(context.Context, map[string]string, mcclient.TokenCredential) error
 }
 
-func InheritFromTo(ctx context.Context, src IClassMetadataOwner, dest IClassMetadataSetter) error {
+func InheritFromTo(ctx context.Context, userCred mcclient.TokenCredential, src IClassMetadataOwner, dest IClassMetadataSetter) error {
 	metadata, err := src.GetAllClassMetadata()
 	if err != nil {
 		return errors.Wrap(err, "GetAllClassMetadata")
@@ -422,15 +421,17 @@ func InheritFromTo(ctx context.Context, src IClassMetadataOwner, dest IClassMeta
 			if sv, ok := metadata[k]; ok {
 				if sv != v {
 					// duplicate value for identical key
-					return errors.Wrapf(httperrors.ErrConflict, "destination has another value for class key %s", k)
+					// return errors.Wrapf(httperrors.ErrConflict, "destination has another value for class key %s", k)
+					log.Warningf("replace class metadata %s from %s to %s", k, v, sv)
 				}
 			} else {
 				// no such class key
-				return errors.Wrapf(httperrors.ErrConflict, "destination has extra class key %s", k)
+				metadata[k] = "None"
+				// return errors.Wrapf(httperrors.ErrConflict, "destination has extra class key %s", k)
 			}
 		}
 	}
-	userCred := auth.AdminCredential()
+	// userCred := auth.AdminCredential()
 	return dest.SetClassMetadataAll(ctx, metadata, userCred)
 }
 

pkg/cloudcommon/db/statusbase.go

@@ -17,6 +17,7 @@ package db
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
@@ -101,7 +102,11 @@ func statusBaseSetStatus(model IStatusBaseModel, userCred mcclient.TokenCredenti
 			notes = fmt.Sprintf("%s: %s", notes, reason)
 		}
 		OpsLog.LogEvent(model, ACT_UPDATE_STATUS, notes, userCred)
-		logclient.AddSimpleActionLog(model, logclient.ACT_UPDATE_STATUS, notes, userCred, true)
+		success := true
+		if strings.Contains(status, "fail") || status == apis.STATUS_UNKNOWN {
+			success = false
+		}
+		logclient.AddSimpleActionLog(model, logclient.ACT_UPDATE_STATUS, notes, userCred, success)
 	}
 	return nil
 }
@@ -118,13 +123,9 @@ func StatusBasePerformStatus(model IStatusBaseModel, userCred mcclient.TokenCred
 }
 
 func (model *SStatusResourceBase) IsInStatus(status ...string) bool {
-	return utils.IsInStringArray(model.Status, status)
+	return utils.IsInArray(model.Status, status)
 }
 
-/*func (model *SStatusStandaloneResourceBase) AllowGetDetailsStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
-	return IsAllowGetSpec(rbacutils.ScopeSystem, userCred, model, "status")
-}*/
-
 // 获取资源状态
 func (model *SStatusResourceBase) GetDetailsStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (apis.GetDetailsStatusOutput, error) {
 	ret := apis.GetDetailsStatusOutput{}

pkg/cloudcommon/db/taskman/tasks.go

@@ -124,10 +124,6 @@ func (manager *STaskManager) FilterByName(q *sqlchemy.SQuery, name string) *sqlc
 	return q
 }
 
-func (manager *STaskManager) AllowPerformAction(ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
-	return true
-}
-
 func (manager *STaskManager) PerformAction(ctx context.Context, userCred mcclient.TokenCredential, taskId string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	err := runTask(taskId, data)
 	if err != nil {
@@ -172,10 +168,6 @@ func (manager *STaskManager) FetchTaskById(taskId string) *STask {
 	return manager.fetchTask(taskId)
 }
 
-func (self *STask) AllowGetDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
-	return db.IsAdminAllowGet(ctx, userCred, self) || userCred.GetProjectId() == self.UserCred.GetProjectId()
-}
-
 func (self *STask) AllowUpdateItem(ctx context.Context, userCred mcclient.TokenCredential) bool {
 	return false
 }
@@ -449,36 +441,32 @@ func execITask(taskValue reflect.Value, task *STask, odata jsonutils.JSONObject,
 		data = jsonutils.NewDict()
 	}
 
-	var stageName string
+	stageName := task.Stage
 	if taskFailed {
 		stageName = fmt.Sprintf("%sFailed", task.Stage)
-	} else {
-		stageName = task.Stage
+		if strings.Contains(stageName, "_") {
+			stageName = fmt.Sprintf("%s_failed", task.Stage)
+		}
+	}
+
+	if strings.Contains(stageName, "_") {
+		stageName = utils.Kebab2Camel(stageName, "_")
 	}
 
 	funcValue := taskValue.MethodByName(stageName)
 
 	if !funcValue.IsValid() || funcValue.IsNil() {
-		log.Debugf("Stage %s not found, try kebab to camel and find again", stageName)
+		msg := fmt.Sprintf("Stage %s not found", stageName)
 		if taskFailed {
-			stageName = fmt.Sprintf("%s_failed", task.Stage)
-		}
-		stageName = utils.Kebab2Camel(stageName, "_")
-		funcValue = taskValue.MethodByName(stageName)
-
-		if !funcValue.IsValid() || funcValue.IsNil() {
-			msg := fmt.Sprintf("Stage %s not found", stageName)
-			if taskFailed {
-				// failed handler is optional, ignore the error
-				log.Warningf(msg)
-				msg, _ = data.GetString()
-			} else {
-				log.Errorf(msg)
-			}
-			task.SetStageFailed(ctx, jsonutils.NewString(msg))
-			task.SaveRequestContext(&ctxData)
-			return
+			// failed handler is optional, ignore the error
+			log.Warningf(msg)
+			msg, _ = data.GetString()
+		} else {
+			log.Errorf(msg)
 		}
+		task.SetStageFailed(ctx, jsonutils.NewString(msg))
+		task.SaveRequestContext(&ctxData)
+		return
 	}
 
 	objManager := db.GetModelManager(task.ObjName)

pkg/cloudcommon/db/tenantcache.go

@@ -236,6 +236,7 @@ func (manager *STenantCacheManager) fetchTenantFromKeystone(ctx context.Context,
 	if len(domainId) > 0 {
 		query.Set("domain_id", jsonutils.NewString(domainId))
 	}
+	query.Set("pending_delete", jsonutils.NewString("all"))
 
 	s := auth.GetAdminSession(ctx, consts.GetRegion())
 	tenant, err := modules.Projects.GetById(s, idStr, query)
@@ -335,12 +336,24 @@ func (manager *STenantCacheManager) Save(ctx context.Context, item SCachedTenant
 		log.Errorf("FetchTenantbyId fail %s", err)
 		return nil, errors.Wrapf(err, "TenantCache FetchById %s", item.Id)
 	}
+	// clean cache metadata
+	if item.PendingDeleted && saveMeta {
+		for k := range item.Metadata {
+			if strings.HasPrefix(k, USER_TAG_PREFIX) {
+				item.Metadata[k] = "none"
+			}
+		}
+	}
 	now := time.Now().UTC()
 	if err == nil {
 		obj := objo.(*STenant)
 		if obj.Id == item.Id && obj.Name == item.Name && obj.Domain == item.ProjectDomain && obj.DomainId == item.DomainId {
 			Update(obj, func() error {
 				obj.LastCheck = now
+				obj.PendingDeleted = item.PendingDeleted
+				if obj.PendingDeleted {
+					obj.PendingDeletedAt = item.PendingDeletedAt
+				}
 				return nil
 			})
 			if saveMeta {
@@ -354,6 +367,10 @@ func (manager *STenantCacheManager) Save(ctx context.Context, item SCachedTenant
 			obj.Domain = item.ProjectDomain
 			obj.DomainId = item.DomainId
 			obj.LastCheck = now
+			obj.PendingDeleted = item.PendingDeleted
+			if obj.PendingDeleted {
+				obj.PendingDeletedAt = item.PendingDeletedAt
+			}
 			return nil
 		})
 		if err != nil {
@@ -372,6 +389,10 @@ func (manager *STenantCacheManager) Save(ctx context.Context, item SCachedTenant
 		obj.Domain = item.ProjectDomain
 		obj.DomainId = item.DomainId
 		obj.LastCheck = now
+		obj.PendingDeleted = item.PendingDeleted
+		if obj.PendingDeleted {
+			obj.PendingDeletedAt = item.PendingDeletedAt
+		}
 		err = manager.TableSpec().InsertOrUpdate(ctx, obj)
 		if err != nil {
 			return nil, errors.Wrap(err, "InsertOrUpdate")
@@ -431,6 +452,7 @@ func (manager *STenantCacheManager) fetchDomainTenantsFromKeystone(ctx context.C
 	params := jsonutils.NewDict()
 	params.Add(jsonutils.NewString(domainId), "domain_id")
 	params.Add(jsonutils.JSONTrue, "details")
+	params.Add(jsonutils.NewString("all"), "pending_delete")
 	tenants, err := modules.Projects.List(s, params)
 	if err != nil {
 		return errors.Wrap(err, "Projects.List")
@@ -550,3 +572,29 @@ func (tenant *STenant) GetAllClassMetadata() (map[string]string, error) {
 	}
 	return ret, nil
 }
+
+func (manager *STenantCacheManager) ConvertIds(ids []string, isDomain bool) ([]string, error) {
+	var q *sqlchemy.SQuery
+	if isDomain {
+		q = manager.GetDomainQuery("id")
+	} else {
+		q = manager.GetTenantQuery("id")
+	}
+	q = q.Filter(sqlchemy.OR(
+		sqlchemy.In(q.Field("id"), stringutils2.RemoveUtf8Strings(ids)),
+		sqlchemy.In(q.Field("name"), ids),
+	))
+	q = q.Distinct()
+	results := []struct {
+		Id string
+	}{}
+	err := q.All(&results)
+	if err != nil {
+		return nil, errors.Wrap(err, "query")
+	}
+	ret := make([]string, len(results))
+	for i := range results {
+		ret[i] = results[i].Id
+	}
+	return ret, nil
+}

pkg/cloudcommon/db/tenantcacheitem.go

@@ -14,14 +14,20 @@
 
 package db
 
-import identityapi "yunion.io/x/onecloud/pkg/apis/identity"
+import (
+	"time"
+
+	identityapi "yunion.io/x/onecloud/pkg/apis/identity"
+)
 
 type SCachedTenant struct {
-	Id            string            `json:"id"`
-	Name          string            `json:"name"`
-	DomainId      string            `json:"domain_id"`
-	ProjectDomain string            `json:"project_domain"`
-	Metadata      map[string]string `json:"metadata"`
+	Id               string            `json:"id"`
+	Name             string            `json:"name"`
+	DomainId         string            `json:"domain_id"`
+	ProjectDomain    string            `json:"project_domain"`
+	Metadata         map[string]string `json:"metadata"`
+	PendingDeleted   bool              `json:"pending_deleted"`
+	PendingDeletedAt time.Time         `json:"pending_deleted_at"`
 }
 
 func (s SCachedTenant) objType() string {
@@ -31,3 +37,8 @@ func (s SCachedTenant) objType() string {
 		return "project"
 	}
 }
+
+type SCachedUser struct {
+	SCachedTenant
+	Lang string
+}

pkg/cloudcommon/db/tenantcachesync.go

@@ -62,6 +62,10 @@ func (w *tenantCacheSyncWorker) Run() {
 	if err != nil {
 		log.Errorf("fail to syncProjects %s", err)
 	}
+	err = syncUsers(w.ctx)
+	if err != nil {
+		log.Errorf("fail to syncUsers %s", err)
+	}
 }
 
 func (w *tenantCacheSyncWorker) Dump() string {
@@ -74,23 +78,25 @@ func syncDomains(ctx context.Context) error {
 	query.Add(jsonutils.NewInt(1024), "limit")
 	query.Add(jsonutils.NewString(string(rbacscope.ScopeSystem)), "scope")
 	query.Add(jsonutils.JSONTrue, "details")
+	query.Add(jsonutils.NewString("all"), "pending_delete")
 	total := -1
 	offset := 0
 	for total < 0 || offset < total {
 		query.Set("offset", jsonutils.NewInt(int64(offset)))
 		results, err := modules.Domains.List(s, query)
 		if err != nil {
-			log.Errorf("syncDomain error %s", err)
 			return errors.Wrap(err, "Domains.List")
 		}
 		total = results.Total
 		for i := range results.Data {
 			// update domain cache
 			item := SCachedTenant{}
-			results.Data[i].Unmarshal(&item)
-			item.ProjectDomain = identityapi.KeystoneDomainRoot
-			item.DomainId = identityapi.KeystoneDomainRoot
-			TenantCacheManager.Save(ctx, item, true)
+			err := results.Data[i].Unmarshal(&item)
+			if err == nil {
+				item.ProjectDomain = identityapi.KeystoneDomainRoot
+				item.DomainId = identityapi.KeystoneDomainRoot
+				TenantCacheManager.Save(ctx, item, true)
+			}
 			offset++
 		}
 	}
@@ -103,21 +109,52 @@ func syncProjects(ctx context.Context) error {
 	query.Add(jsonutils.NewInt(1024), "limit")
 	query.Add(jsonutils.NewString(string(rbacscope.ScopeSystem)), "scope")
 	query.Add(jsonutils.JSONTrue, "details")
+	query.Add(jsonutils.NewString("all"), "pending_delete")
 	total := -1
 	offset := 0
 	for total < 0 || offset < total {
 		query.Set("offset", jsonutils.NewInt(int64(offset)))
 		results, err := modules.Projects.List(s, query)
 		if err != nil {
-			log.Errorf("syncProjects error %s", err)
 			return errors.Wrap(err, "Projects.List")
 		}
 		total = results.Total
 		for i := range results.Data {
 			// update project cache
 			item := SCachedTenant{}
-			results.Data[i].Unmarshal(&item)
-			TenantCacheManager.Save(ctx, item, true)
+			err := results.Data[i].Unmarshal(&item)
+			if err == nil {
+				TenantCacheManager.Save(ctx, item, true)
+			}
+			offset++
+		}
+	}
+	return nil
+}
+
+func syncUsers(ctx context.Context) error {
+	s := auth.GetAdminSession(ctx, consts.GetRegion())
+	query := jsonutils.NewDict()
+	query.Add(jsonutils.NewInt(1024), "limit")
+	query.Add(jsonutils.NewString(string(rbacscope.ScopeSystem)), "scope")
+	query.Add(jsonutils.JSONTrue, "details")
+	query.Add(jsonutils.NewString("all"), "pending_delete")
+	total := -1
+	offset := 0
+	for total < 0 || offset < total {
+		query.Set("offset", jsonutils.NewInt(int64(offset)))
+		results, err := modules.UsersV3.List(s, query)
+		if err != nil {
+			return errors.Wrap(err, "UsersV3.List")
+		}
+		total = results.Total
+		for i := range results.Data {
+			// update user cache
+			item := SCachedUser{}
+			err := results.Data[i].Unmarshal(&item)
+			if err == nil {
+				UserCacheManager.Save(ctx, item.Id, item.Name, item.DomainId, item.ProjectDomain, item.Lang)
+			}
 			offset++
 		}
 	}

pkg/cloudcommon/db/usercache.go

@@ -124,6 +124,7 @@ func (manager *SUserCacheManager) FetchUserFromKeystone(ctx context.Context, idS
 	query := jsonutils.NewDict()
 	query.Set("scope", jsonutils.NewString("system"))
 	query.Set("system", jsonutils.JSONTrue)
+	query.Set("pending_delete", jsonutils.NewString("all"))
 
 	s := auth.GetAdminSession(ctx, consts.GetRegion())
 	user, err := modules.UsersV3.GetById(s, idStr, query)

pkg/cloudcommon/db/virtualresource.go

@@ -16,20 +16,16 @@ package db
 
 import (
 	"context"
-	"strings"
-	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/rbacscope"
-	"yunion.io/x/pkg/util/timeutils"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/apis"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
-	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -40,6 +36,7 @@ import (
 type SVirtualResourceBaseManager struct {
 	SStatusStandaloneResourceBaseManager
 	SProjectizedResourceBaseManager
+	SPendingDeletedBaseManager
 }
 
 func NewVirtualResourceBaseManager(dt interface{}, tableName string, keyword string, keywordPlural string) SVirtualResourceBaseManager {
@@ -52,6 +49,7 @@ func NewVirtualResourceBaseManager(dt interface{}, tableName string, keyword str
 type SVirtualResourceBase struct {
 	SStatusStandaloneResourceBase
 	SProjectizedResourceBase
+	SPendingDeletedBase
 
 	// 云上同步资源是否在本地被更改过配置, local: 更改过, cloud: 未更改过
 	// example: local
@@ -60,10 +58,6 @@ type SVirtualResourceBase struct {
 	// 是否是系统资源
 	IsSystem bool `nullable:"true" default:"false" list:"admin" create:"optional" json:"is_system"`
 
-	// 资源放入回收站时间
-	PendingDeletedAt time.Time `json:"pending_deleted_at" list:"user" update:"admin"`
-	// 资源是否处于回收站中
-	PendingDeleted bool `nullable:"false" default:"false" index:"true" get:"user" list:"user" json:"pending_deleted"`
 	// 资源是否被冻结
 	Freezed bool `nullable:"false" default:"false" get:"user" list:"user" json:"freezed"`
 }
@@ -215,29 +209,7 @@ func (model *SVirtualResourceBase) SetProjectInfo(ctx context.Context, userCred
 
 func (manager *SVirtualResourceBaseManager) FilterBySystemAttributes(q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject, scope rbacscope.TRbacScope) *sqlchemy.SQuery {
 	q = manager.SStatusStandaloneResourceBaseManager.FilterBySystemAttributes(q, userCred, query, scope)
-
-	var pendingDelete string
-	if query != nil {
-		pendingDelete, _ = query.GetString("pending_delete")
-	}
-	pendingDeleteLower := strings.ToLower(pendingDelete)
-	if pendingDeleteLower == "all" || pendingDeleteLower == "any" || utils.ToBool(pendingDeleteLower) {
-		var isAllow bool
-		allowScope, result := policy.PolicyManager.AllowScope(userCred, consts.GetServiceType(), manager.KeywordPlural(), policy.PolicyActionList, "pending_delete")
-		if result.Result.IsAllow() && !scope.HigherThan(allowScope) {
-			isAllow = true
-		}
-		if !isAllow {
-			pendingDeleteLower = ""
-		}
-	}
-
-	if pendingDeleteLower == "all" || pendingDeleteLower == "any" {
-	} else if utils.ToBool(pendingDeleteLower) {
-		q = q.IsTrue("pending_deleted")
-	} else {
-		q = q.Filter(sqlchemy.OR(sqlchemy.IsNull(q.Field("pending_deleted")), sqlchemy.IsFalse(q.Field("pending_deleted"))))
-	}
+	q = manager.SPendingDeletedBaseManager.FilterBySystemAttributes(manager.GetIStandaloneModelManager(), q, userCred, query, scope)
 	return q
 }
 
@@ -280,7 +252,7 @@ func (model *SVirtualResourceBase) PostCreate(ctx context.Context, userCred mccl
 		log.Errorf("unable to GetTenantCache: %s", err.Error())
 		return
 	}
-	err = InheritFromTo(ctx, project, model)
+	err = InheritFromTo(ctx, userCred, project, model)
 	if err != nil {
 		log.Errorf("unable to inherit class metadata from poject %s: %s", project.GetId(), err.Error())
 	}
@@ -501,38 +473,20 @@ func (model *SVirtualResourceBase) PerformChangeOwner(ctx context.Context, userC
 }
 
 func (model *SVirtualResourceBase) DoPendingDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
-	return model.MarkPendingDelete(ctx, userCred)
-}
-
-func (model *SVirtualResourceBase) MarkPendingDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
 	if !model.PendingDeleted {
-		_, err := Update(model, func() error {
-			model.PendingDeleted = true
-			model.PendingDeletedAt = timeutils.UtcNow()
-			return nil
-		})
-		if err != nil {
-			log.Errorf("MarkPendingDelete update fail %s", err)
-			return err
-		}
-		vm := model.GetIVirtualModel()
-		OpsLog.LogEvent(model, ACT_PENDING_DELETE, vm.GetShortDesc(ctx), userCred)
-		logclient.AddSimpleActionLog(model, logclient.ACT_PENDING_DELETE, vm.GetShortDesc(ctx), userCred, true)
+		return model.SPendingDeletedBase.MarkPendingDelete(model.GetIStandaloneModel(), ctx, userCred, "")
 	}
 	return nil
 }
 
 func (model *SVirtualResourceBase) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
-	if !model.PendingDeleted {
-		model.DoPendingDelete(ctx, userCred)
+	err := model.DoPendingDelete(ctx, userCred)
+	if err != nil {
+		return errors.Wrap(err, "DoPendingDelete")
 	}
 	return DeleteModel(ctx, userCred, model.GetIVirtualModel())
 }
 
-func (model *SVirtualResourceBase) AllowPerformCancelDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
-	return false
-}
-
 func (model *SVirtualResourceBase) PerformCancelDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	if model.PendingDeleted && !model.Deleted {
 		err := model.DoCancelPendingDelete(ctx, userCred)
@@ -545,11 +499,13 @@ func (model *SVirtualResourceBase) PerformCancelDelete(ctx context.Context, user
 }
 
 func (model *SVirtualResourceBase) DoCancelPendingDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
-	err := model.CancelPendingDelete(ctx, userCred)
-	//if err == nil {
-	//	OpsLog.LogEvent(model, ACT_CANCEL_DELETE, model.GetShortDesc(ctx), userCred)
-	//}
-	return err
+	if model.PendingDeleted && !model.Deleted {
+		err := model.SPendingDeletedBase.MarkCancelPendingDelete(model.GetIStandaloneModel(), ctx, userCred)
+		if err != nil {
+			return errors.Wrap(err, "MarkCancelPendingDelete")
+		}
+	}
+	return nil
 }
 
 func (model *SVirtualResourceBase) VirtualModelManager() IVirtualModelManager {
@@ -560,53 +516,27 @@ func (model *SVirtualResourceBase) GetIVirtualModel() IVirtualModel {
 	return model.GetVirtualObject().(IVirtualModel)
 }
 
-func (model *SVirtualResourceBase) CancelPendingDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
-	if model.PendingDeleted && !model.Deleted {
-		err := model.MarkCancelPendingDelete(ctx, userCred)
-		if err != nil {
-			return errors.Wrap(err, "MarkCancelPendingDelete")
-		}
-	}
-	return nil
-}
-
-func (model *SVirtualResourceBase) MarkCancelPendingDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
-	manager := model.GetModelManager()
-	ownerId := model.GetOwnerId()
-
-	lockman.LockRawObject(ctx, manager.Keyword(), "name")
-	defer lockman.ReleaseRawObject(ctx, manager.Keyword(), "name")
-
-	newName, err := GenerateName(ctx, manager, ownerId, model.Name)
-	if err != nil {
-		return errors.Wrapf(err, "GenerateNam")
-	}
-	_, err = Update(model, func() error {
-		model.Name = newName
-		model.PendingDeleted = false
-		model.PendingDeletedAt = time.Time{}
-		return nil
-	})
-	if err != nil {
-		return errors.Wrapf(err, "MarkCancelPendingDelete.Update")
-	}
-	vm := model.GetIVirtualModel()
-	OpsLog.LogEvent(model, ACT_CANCEL_DELETE, vm.GetShortDesc(ctx), userCred)
-	return nil
-}
-
 func (model *SVirtualResourceBase) GetShortDesc(ctx context.Context) *jsonutils.JSONDict {
 	desc := model.SStatusStandaloneResourceBase.GetShortDesc(ctx)
 	desc.Add(jsonutils.NewString(model.ProjectId), "owner_tenant_id")
 	tc, _ := TenantCacheManager.FetchTenantById(ctx, model.ProjectId)
 	if tc != nil {
 		desc.Add(jsonutils.NewString(tc.GetName()), "owner_tenant")
-		metadata, _ := GetVisiableMetadata(ctx, tc, nil)
+		metadata, _ := GetVisibleMetadata(ctx, tc, nil)
 		desc.Set("project_tags", jsonutils.Marshal(metadata))
 	}
 	return desc
 }
 
+func (model *SVirtualResourceBase) SetProjectSrc(src apis.TOwnerSource) {
+	if model.ProjectSrc != string(src) {
+		Update(model, func() error {
+			model.ProjectSrc = string(apis.OWNER_SOURCE_CLOUD)
+			return nil
+		})
+	}
+}
+
 func (model *SVirtualResourceBase) SyncCloudProjectId(userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider) {
 	if model.ProjectSrc != string(apis.OWNER_SOURCE_LOCAL) && ownerId != nil && len(ownerId.GetProjectId()) > 0 {
 		diff, _ := Update(model, func() error {
@@ -621,16 +551,6 @@ func (model *SVirtualResourceBase) SyncCloudProjectId(userCred mcclient.TokenCre
 	}
 }
 
-// GetPendingDeleted implements IPendingDeltable
-func (model *SVirtualResourceBase) GetPendingDeleted() bool {
-	return model.PendingDeleted
-}
-
-// GetPendingDeletedAt implements IPendingDeltable
-func (model *SVirtualResourceBase) GetPendingDeletedAt() time.Time {
-	return model.PendingDeletedAt
-}
-
 func (manager *SVirtualResourceBaseManager) OrderByExtraFields(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query apis.VirtualResourceListInput) (*sqlchemy.SQuery, error) {
 	q, err := manager.SStatusStandaloneResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.StatusStandaloneResourceListInput)
 	if err != nil {

pkg/cloudcommon/notifyclient/notify.go

@@ -55,6 +55,28 @@ func init() {
 			Action: ActionUpdate,
 		})
 	})
+
+	db.SetCreateNotifyHook(func(ctx context.Context, userCred mcclient.TokenCredential, obj db.IModel) {
+		_, ok := notifyDBHookResources.Load(obj.KeywordPlural())
+		if !ok {
+			return
+		}
+		EventNotify(ctx, userCred, SEventNotifyParam{
+			Obj:    obj,
+			Action: ActionCreate,
+		})
+	})
+
+	db.SetDeleteNotifyHook(func(ctx context.Context, userCred mcclient.TokenCredential, obj db.IModel) {
+		_, ok := notifyDBHookResources.Load(obj.KeywordPlural())
+		if !ok {
+			return
+		}
+		EventNotify(ctx, userCred, SEventNotifyParam{
+			Obj:    obj,
+			Action: ActionDelete,
+		})
+	})
 }
 
 func AddNotifyDBHookResources(keywordPlurals ...string) {

pkg/cloudcommon/types/nic.go

@@ -17,18 +17,20 @@ package types
 import (
 	"net"
 
+	"yunion.io/x/cloudmux/pkg/apis/compute"
 	"yunion.io/x/pkg/util/netutils"
 )
 
 type SNic struct {
-	Type    string   `json:"nic_type"`
+	Type compute.TNicType `json:"nic_type"`
+
 	Domain  string   `json:"domain"`
 	Wire    string   `json:"wire"`
 	IpAddr  string   `json:"ip_addr"`
 	WireId  string   `json:"wire_id"`
 	NetId   string   `json:"net_id"`
-	Rate    int64    `json:"rate"`
-	Mtu     int64    `json:"mtu"`
+	Rate    int      `json:"rate"`
+	Mtu     int16    `json:"mtu"`
 	Mac     string   `json:"mac"`
 	Dns     string   `json:"dns"`
 	Ntp     string   `json:"ntp"`
@@ -37,6 +39,13 @@ type SNic struct {
 	Gateway string   `json:"gateway"`
 	LinkUp  bool     `json:"link_up"`
 	Routes  []SRoute `json:"routes,omitempty"`
+
+	Interface string `json:"interface"`
+	Bridge    string `json:"bridge"`
+
+	VlanId int `json:"vlan_id"`
+
+	Bandwidth int `json:"bandwidth"`
 }
 
 type SRoute []string
@@ -64,7 +73,7 @@ type SServerNic struct {
 	NetId     string   `json:"net_id"`
 	Mac       string   `json:"mac"`
 	BandWidth int      `json:"bw"`
-	Mtu       int      `json:"mtu,omitempty"`
+	Mtu       int16    `json:"mtu,omitempty"`
 	Dns       string   `json:"dns"`
 	Ntp       string   `json:"ntp"`
 	Net       string   `json:"net"`
@@ -72,9 +81,11 @@ type SServerNic struct {
 	Gateway   string   `json:"gateway"`
 	Ifname    string   `json:"ifname"`
 	Routes    []SRoute `json:"routes,omitempty"`
-	NicType   string   `json:"nic_type,omitempty"`
-	LinkUp    bool     `json:"link_up,omitempty"`
-	TeamWith  string   `json:"team_with,omitempty"`
+
+	NicType compute.TNicType `json:"nic_type,omitempty"`
+
+	LinkUp   bool   `json:"link_up,omitempty"`
+	TeamWith string `json:"team_with,omitempty"`
 
 	TeamingMaster *SServerNic   `json:"-"`
 	TeamingSlaves []*SServerNic `json:"-"`
@@ -103,6 +114,6 @@ func (n SServerNic) ToNic() SNic {
 		Gateway: n.Gateway,
 		Routes:  n.Routes,
 		LinkUp:  n.LinkUp,
-		Mtu:     int64(n.Mtu),
+		Mtu:     n.Mtu,
 	}
 }

pkg/cloudcommon/userdata/userdata.go

@@ -18,10 +18,14 @@ import (
 	"bytes"
 	"compress/gzip"
 	"encoding/base64"
-	"fmt"
 	"io/ioutil"
+	"strings"
 
-	"github.com/pkg/errors"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/cloudinit"
+	"yunion.io/x/pkg/util/osprofile"
+
+	"yunion.io/x/onecloud/pkg/httperrors"
 )
 
 const (
@@ -58,16 +62,32 @@ func Decode(encodeUserdata string) (string, error) {
 	return string(data), nil
 }
 
-func ValidateUserdata(data string) error {
+func ValidateUserdata(data string, osType string) error {
 	if len(data) == 0 {
 		return nil
 	}
+	_, err := cloudinit.ParseUserData(data)
+	if err != nil {
+		if osType == osprofile.OS_TYPE_WINDOWS {
+			if strings.HasPrefix(data, "[bat]\n") || strings.HasPrefix(data, "[powershell]\n") {
+				// valid
+			} else {
+				return errors.Wrap(httperrors.ErrInputParameter, "invalid windows scripts")
+			}
+		} else {
+			if strings.HasPrefix(data, "#!/bin/sh\n") || strings.HasPrefix(data, "#!/bin/bash\n") || strings.HasPrefix(data, "#!/usr/bin/env bash") {
+				// valid
+			} else {
+				return errors.Wrap(httperrors.ErrInputParameter, "invalid shell scripts")
+			}
+		}
+	}
 	encodeData, err := Encode(data)
 	if err != nil {
-		return errors.Wrapf(err, "Encode data")
+		return errors.Wrapf(httperrors.ErrInputParameter, "Encode data error %s", err)
 	}
 	if len(encodeData) > UserdataLimitSize {
-		return errors.New(fmt.Sprintf("user data size %d large limit %d", len(encodeData), UserdataLimitSize))
+		return errors.Wrapf(httperrors.ErrInputParameter, "user data size %d large limit %d", len(encodeData), UserdataLimitSize)
 	}
 	return nil
 }