Merge branch 'release/2.4.0' of ssh://git.yunion.io/~quxuan/onecloud into release/2.4.0

pkg/cloudcommon/db/rbac.go

@@ -54,14 +54,14 @@ func isClassActionRbacAllowed(manager IModelManager, userCred mcclient.TokenCred
 	} else {
 		requireAdmin = true
 	}
-	if !requireAdmin {
-		result := policy.PolicyManager.Allow(false, userCred, consts.GetServiceType(),
-			manager.KeywordPlural(), action, extra...)
-		if result == rbacutils.Allow || result == rbacutils.OwnerAllow {
-			return true
-		}
+	// if !requireAdmin {
+	result := policy.PolicyManager.Allow(false, userCred, consts.GetServiceType(),
+		manager.KeywordPlural(), action, extra...)
+	if result == rbacutils.Allow || (!requireAdmin && result == rbacutils.OwnerAllow) {
+		return true
 	}
-	result := policy.PolicyManager.Allow(true, userCred, consts.GetServiceType(),
+	// }
+	result = policy.PolicyManager.Allow(true, userCred, consts.GetServiceType(),
 		manager.KeywordPlural(), action, extra...)
 	return result == rbacutils.Allow
 }
@@ -85,14 +85,14 @@ func isObjectRbacAllowed(manager IModelManager, model IModel, userCred mcclient.
 		requireAdmin = true
 	}
 
-	if !requireAdmin {
-		result := policy.PolicyManager.Allow(false, userCred, consts.GetServiceType(),
-			manager.KeywordPlural(), action, extra...)
-		if result == rbacutils.Allow || (result == rbacutils.OwnerAllow && isOwner) {
-			return true
-		}
+	//if !requireAdmin {
+	result := policy.PolicyManager.Allow(false, userCred, consts.GetServiceType(),
+		manager.KeywordPlural(), action, extra...)
+	if result == rbacutils.Allow || (!requireAdmin && result == rbacutils.OwnerAllow && isOwner) {
+		return true
 	}
-	result := policy.PolicyManager.Allow(true, userCred, consts.GetServiceType(),
+	//}
+	result = policy.PolicyManager.Allow(true, userCred, consts.GetServiceType(),
 		manager.KeywordPlural(), action, extra...)
 	return result == rbacutils.Allow
 }

pkg/cloudcommon/policy/policy.go

@@ -52,12 +52,12 @@ func parseJsonPolicy(obj jsonutils.JSONObject) (string, rbacutils.SRbacPolicy, e
 		return "", policy, err
 	}
 
-	blobStr, err := obj.GetString("blob")
+	blobStr, err := obj.GetString("policy")
 	if err != nil {
 		log.Errorf("get blob error %s", err)
 		return "", policy, err
 	}
-	blob, err := jsonutils.ParseString(blobStr)
+	blob, err := jsonutils.ParseYAML(blobStr)
 	if err != nil {
 		log.Errorf("parse blob json error %s", err)
 		return "", policy, err
@@ -77,14 +77,12 @@ func fetchPolicies() (map[string]rbacutils.SRbacPolicy, map[string]rbacutils.SRb
 	policies := make(map[string]rbacutils.SRbacPolicy)
 	adminPolicies := make(map[string]rbacutils.SRbacPolicy)
 
-	modules.Policies.SetEnableFilter(false)
-
 	offset := 0
 	for {
 		params := jsonutils.NewDict()
 		params.Add(jsonutils.NewInt(2048), "limit")
 		params.Add(jsonutils.NewInt(int64(offset)), "offset")
-		result, err := modules.Policies.ResourceManager.List(s, params)
+		result, err := modules.Policies.List(s, params)
 
 		if err != nil {
 			log.Errorf("fetch policy failed")

pkg/compute/guestdrivers/aliyun.go

@@ -458,7 +458,7 @@ func (self *SAliyunGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Contex
 				if diskInfo[i].AutoDelete {
 					disk.AutoDelete = true
 				}
-				disk.TemplateId = diskInfo[i].TemplateId
+				// disk.TemplateId = diskInfo[i].TemplateId
 				disk.DiskFormat = diskInfo[i].DiskFormat
 				disk.ExpiredAt = diskInfo[i].ExpiredAt
 				if len(diskInfo[i].Metadata) > 0 {

pkg/compute/guestdrivers/aws.go

@@ -69,43 +69,6 @@ func (self *SAwsGuestDriver) ValidateCreateData(ctx context.Context, userCred mc
 	return self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, data)
 }
 
-func fetchAwsIVMinfo(desc SManagedVMCreateConfig, iVM cloudprovider.ICloudVM, guestId string) *jsonutils.JSONDict {
-	data := jsonutils.NewDict()
-	data.Add(jsonutils.NewString(iVM.GetOSType()), "os")
-	if len(desc.OsDistribution) > 0 {
-		data.Add(jsonutils.NewString(desc.OsDistribution), "distro")
-	}
-	if len(desc.OsVersion) > 0 {
-		data.Add(jsonutils.NewString(desc.OsVersion), "version")
-	}
-
-	idisks, err := iVM.GetIDisks()
-
-	if err != nil {
-		log.Errorf("GetiDisks error %s", err)
-	} else {
-		diskInfo := make([]SDiskInfo, len(idisks))
-		for i := 0; i < len(idisks); i += 1 {
-			dinfo := SDiskInfo{}
-			dinfo.Uuid = idisks[i].GetGlobalId()
-			dinfo.Size = idisks[i].GetDiskSizeMB()
-			dinfo.DiskType = idisks[i].GetDiskType()
-			if metaData := idisks[i].GetMetadata(); metaData != nil {
-				dinfo.Metadata = make(map[string]string, 0)
-				if err := metaData.Unmarshal(dinfo.Metadata); err != nil {
-					log.Errorf("Get disk %s metadata info error: %v", idisks[i].GetName(), err)
-				}
-			}
-			diskInfo[i] = dinfo
-		}
-		data.Add(jsonutils.Marshal(&diskInfo), "disks")
-	}
-
-	data.Add(jsonutils.NewString(iVM.GetGlobalId()), "uuid")
-	data.Add(iVM.GetMetadata(), "metadata")
-	return data
-}
-
 func (self *SAwsGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error {
 	config := guest.GetDeployConfigOnHost(ctx, host, task.GetParams())
 	log.Debugf("RequestDeployGuestOnHost: %s", config)
@@ -171,7 +134,7 @@ func (self *SAwsGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest
 				return nil, err
 			}
 
-			data := fetchAwsIVMinfo(desc, iVM, guest.Id)
+			data := fetchIVMinfo(desc, iVM, guest.Id, "root", passwd, action)
 			return data, nil
 		})
 	case "deploy":
@@ -279,6 +242,12 @@ func (self *SAwsGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Context,
 				disk.ExternalId = diskInfo[i].Uuid
 				disk.DiskType = diskInfo[i].DiskType
 				disk.Status = models.DISK_READY
+				disk.BillingType = diskInfo[i].BillingType
+				disk.FsFormat = diskInfo[i].FsFromat
+				disk.AutoDelete = true
+				//disk.TemplateId = diskInfo[i].TemplateId
+				disk.DiskFormat = diskInfo[i].DiskFormat
+				disk.ExpiredAt = diskInfo[i].ExpiredAt
 				if len(diskInfo[i].Metadata) > 0 {
 					for key, value := range diskInfo[i].Metadata {
 						if err := disk.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil {

pkg/compute/guestdrivers/azure.go

@@ -244,7 +244,7 @@ func (self *SAzureGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Context
 				disk.BillingType = diskInfo[i].BillingType
 				disk.FsFormat = diskInfo[i].FsFromat
 				disk.AutoDelete = diskInfo[i].AutoDelete
-				disk.TemplateId = diskInfo[i].TemplateId
+				// disk.TemplateId = diskInfo[i].TemplateId
 				disk.DiskFormat = diskInfo[i].DiskFormat
 				disk.ExpiredAt = diskInfo[i].ExpiredAt
 				if len(diskInfo[i].Metadata) > 0 {

pkg/compute/guestdrivers/qcloud.go

@@ -299,7 +299,7 @@ func (self *SQcloudGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Contex
 				disk.BillingType = diskInfo[i].BillingType
 				disk.FsFormat = diskInfo[i].FsFromat
 				disk.AutoDelete = true
-				disk.TemplateId = diskInfo[i].TemplateId
+				//disk.TemplateId = diskInfo[i].TemplateId
 				disk.DiskFormat = diskInfo[i].DiskFormat
 				disk.ExpiredAt = diskInfo[i].ExpiredAt
 				if len(diskInfo[i].Metadata) > 0 {

pkg/compute/hostdrivers/azure.go

@@ -13,6 +13,7 @@ import (
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/compute/options"
 	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
 type SAzureHostDriver struct {
@@ -28,6 +29,13 @@ func (self *SAzureHostDriver) GetHostType() string {
 	return models.HOST_TYPE_AZURE
 }
 
+func (self *SAzureHostDriver) ValidateUpdateDisk(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	if data.Contains("name") {
+		return nil, httperrors.NewInputParameterError("cannot support change azure disk name")
+	}
+	return data, nil
+}
+
 func (self *SAzureHostDriver) CheckAndSetCacheImage(ctx context.Context, host *models.SHost, storageCache *models.SStoragecache, task taskman.ITask) error {
 	params := task.GetParams()
 	imageId, err := params.GetString("image_id")

pkg/compute/hostdrivers/base.go

@@ -16,6 +16,10 @@ import (
 type SBaseHostDriver struct {
 }
 
+func (self *SBaseHostDriver) ValidateUpdateDisk(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	return data, nil
+}
+
 func (self *SBaseHostDriver) RequestDeleteSnapshotsWithStorage(ctx context.Context, host *models.SHost, snapshot *models.SSnapshot, task taskman.ITask) error {
 	return fmt.Errorf("Not Implement")
 }

pkg/compute/models/disks.go

@@ -235,6 +235,25 @@ func (self *SDisk) CustomizeCreate(ctx context.Context, userCred mcclient.TokenC
 	return self.SSharableVirtualResourceBase.CustomizeCreate(ctx, userCred, ownerProjId, query, data)
 }
 
+func (self *SDisk) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	storage := self.GetStorage()
+	if storage == nil {
+		return nil, httperrors.NewNotFoundError("failed to find storage for disk %s", self.Name)
+	}
+
+	host := storage.GetMasterHost()
+	if host == nil {
+		return nil, httperrors.NewNotFoundError("failed to find host for storage %s with disk %s", storage.Name, self.Name)
+	}
+
+	data, err := host.GetHostDriver().ValidateUpdateDisk(ctx, userCred, data)
+	if err != nil {
+		return nil, err
+	}
+
+	return self.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data)
+}
+
 func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	disk, err := data.Get("disk")
 	if err != nil {

pkg/compute/models/hostdrivers.go

@@ -13,6 +13,7 @@ import (
 type IHostDriver interface {
 	GetHostType() string
 	CheckAndSetCacheImage(ctx context.Context, host *SHost, storagecache *SStoragecache, task taskman.ITask) error
+	ValidateUpdateDisk(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error)
 	RequestPrepareSaveDiskOnHost(ctx context.Context, host *SHost, disk *SDisk, imageId string, task taskman.ITask) error
 	RequestSaveUploadImageOnHost(ctx context.Context, host *SHost, disk *SDisk, imageId string, task taskman.ITask, data jsonutils.JSONObject) error
 	RequestAllocateDiskOnStorage(ctx context.Context, host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask, content *jsonutils.JSONDict) error

pkg/compute/usages/handler.go

@@ -286,19 +286,27 @@ func ReportGeneralUsage(userCred mcclient.TokenCredential, rangeObj db.IStandalo
 		}
 	}
 
+	includeCommon := false
 	if consts.IsRbacEnabled() {
 		if policy.PolicyManager.Allow(false, userCred, consts.GetServiceType(),
 			"usages", policy.PolicyActionGet) == rbacutils.Deny {
-			err = httperrors.NewForbiddenError("not allow to get usages")
-			return
+			if !isAdmin {
+				err = httperrors.NewForbiddenError("not allow to get usages")
+				return
+			}
+		} else {
+			includeCommon = true
 		}
 	}
 
-	commonUsage, err := getCommonGeneralUsage(userCred, rangeObj, hostTypes)
-	if err != nil {
-		return
+	if includeCommon {
+		var commonUsage map[string]interface{}
+		commonUsage, err = getCommonGeneralUsage(userCred, rangeObj, hostTypes)
+		if err != nil {
+			return
+		}
+		count.Include(commonUsage)
 	}
-	count.Include(commonUsage)
 	return
 }
 

pkg/util/qcloud/qcloud.go

@@ -3,6 +3,7 @@ package qcloud
 import (
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
 	tchttp "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/http"
@@ -54,9 +55,9 @@ func jsonRequest(client *common.Client, apiName string, params map[string]string
 
 func vpcRequest(client *common.Client, apiName string, params map[string]string) (jsonutils.JSONObject, error) {
 	domain := "vpc.tencentcloudapi.com"
-	// if region, ok := params["Region"]; ok && strings.HasSuffix(region, "-fsi") {
-	// 	domain = "vpc." + region + ".tencentcloudapi.com"
-	// }
+	if region, ok := params["Region"]; ok && strings.HasSuffix(region, "-fsi") {
+		domain = "vpc." + region + ".tencentcloudapi.com"
+	}
 	return _jsonRequest(client, domain, QCLOUD_API_VERSION, apiName, params)
 }
 
@@ -89,9 +90,24 @@ func _jsonRequest(client *common.Client, domain string, version string, apiName
 	resp := &QcloudResponse{
 		BaseResponse: &tchttp.BaseResponse{},
 	}
-	err := client.Send(req, resp)
-	if err != nil {
-		log.Errorf("request url: %s\nparams: %s\nerror: %v", req.GetDomain(), jsonutils.Marshal(req.GetParams()).PrettyString(), err)
+	for i := 1; i <= 3; i++ {
+		err := client.Send(req, resp)
+		if err == nil {
+			break
+		}
+		needRetry := false
+		for _, msg := range []string{"EOF", "TLS handshake timeout", "Code=InternalError"} {
+			if strings.Index(err.Error(), msg) > 0 {
+				needRetry = true
+				break
+			}
+		}
+		if needRetry && i != 3 {
+			log.Errorf("request url %s\nparams: %s\nerror: %v\nafter %d second try again", req.GetDomain(), jsonutils.Marshal(req.GetParams()).PrettyString(), err, i*10)
+			time.Sleep(time.Second * time.Duration(i*10))
+			continue
+		}
+		log.Errorf("request url: %s\nparams: %s\nresponse: %s\nerror: %v", req.GetDomain(), jsonutils.Marshal(req.GetParams()).PrettyString(), resp.Response, err)
 		return nil, err
 	}
 	return jsonutils.Marshal(resp.Response), nil