From 5e19f0b0c1e04521a4002e4a2be79b4929b91fac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Wed, 14 Nov 2018 20:56:17 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=85=BE=E8=AE=AF?= =?UTF-8?q?=E4=BA=91=E8=AF=B7=E6=B1=82=E5=A4=B1=E8=B4=A5=E9=87=8D=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/util/qcloud/qcloud.go | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/pkg/util/qcloud/qcloud.go b/pkg/util/qcloud/qcloud.go index 898087ef40..02fb2fd061 100644 --- a/pkg/util/qcloud/qcloud.go +++ b/pkg/util/qcloud/qcloud.go @@ -3,6 +3,7 @@ package qcloud import ( "fmt" "strings" + "time" "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common" tchttp "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/http" @@ -54,9 +55,9 @@ func jsonRequest(client *common.Client, apiName string, params map[string]string func vpcRequest(client *common.Client, apiName string, params map[string]string) (jsonutils.JSONObject, error) { domain := "vpc.tencentcloudapi.com" - // if region, ok := params["Region"]; ok && strings.HasSuffix(region, "-fsi") { - // domain = "vpc." + region + ".tencentcloudapi.com" - // } + if region, ok := params["Region"]; ok && strings.HasSuffix(region, "-fsi") { + domain = "vpc." + region + ".tencentcloudapi.com" + } return _jsonRequest(client, domain, QCLOUD_API_VERSION, apiName, params) } @@ -89,9 +90,24 @@ func _jsonRequest(client *common.Client, domain string, version string, apiName resp := &QcloudResponse{ BaseResponse: &tchttp.BaseResponse{}, } - err := client.Send(req, resp) - if err != nil { - log.Errorf("request url: %s\nparams: %s\nerror: %v", req.GetDomain(), jsonutils.Marshal(req.GetParams()).PrettyString(), err) + for i := 1; i <= 3; i++ { + err := client.Send(req, resp) + if err == nil { + break + } + needRetry := false + for _, msg := range []string{"EOF", "TLS handshake timeout", "Code=InternalError"} { + if strings.Index(err.Error(), msg) > 0 { + needRetry = true + break + } + } + if needRetry && i != 3 { + log.Errorf("request url %s\nparams: %s\nerror: %v\nafter %d second try again", req.GetDomain(), jsonutils.Marshal(req.GetParams()).PrettyString(), err, i*10) + time.Sleep(time.Second * time.Duration(i*10)) + continue + } + log.Errorf("request url: %s\nparams: %s\nresponse: %s\nerror: %v", req.GetDomain(), jsonutils.Marshal(req.GetParams()).PrettyString(), resp.Response, err) return nil, err } return jsonutils.Marshal(resp.Response), nil From b4c3ce139fa2a2f6a7c932ac2bee0b9c36b3651f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Wed, 14 Nov 2018 21:46:19 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E9=81=BF=E5=85=8D=E6=9B=B4=E6=96=B0Azure?= =?UTF-8?q?=20disk=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/hostdrivers/azure.go | 8 ++++++++ pkg/compute/hostdrivers/base.go | 5 +++++ pkg/compute/models/disks.go | 19 +++++++++++++++++++ pkg/compute/models/hostdrivers.go | 2 ++ 4 files changed, 34 insertions(+) diff --git a/pkg/compute/hostdrivers/azure.go b/pkg/compute/hostdrivers/azure.go index 76cbb9b5c5..6635e9c879 100644 --- a/pkg/compute/hostdrivers/azure.go +++ b/pkg/compute/hostdrivers/azure.go @@ -13,6 +13,7 @@ import ( "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" ) type SAzureHostDriver struct { @@ -28,6 +29,13 @@ func (self *SAzureHostDriver) GetHostType() string { return models.HOST_TYPE_AZURE } +func (self *SAzureHostDriver) ValidateUpdateDisk(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + if data.Contains("name") { + return nil, httperrors.NewInputParameterError("cannot support change azure disk name") + } + return data, nil +} + func (self *SAzureHostDriver) CheckAndSetCacheImage(ctx context.Context, host *models.SHost, storageCache *models.SStoragecache, task taskman.ITask) error { params := task.GetParams() imageId, err := params.GetString("image_id") diff --git a/pkg/compute/hostdrivers/base.go b/pkg/compute/hostdrivers/base.go index 08f3dd0125..a4c833a9ae 100644 --- a/pkg/compute/hostdrivers/base.go +++ b/pkg/compute/hostdrivers/base.go @@ -7,11 +7,16 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/mcclient" ) type SBaseHostDriver struct { } +func (self *SBaseHostDriver) ValidateUpdateDisk(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + return data, nil +} + func (self *SBaseHostDriver) RequestDeleteSnapshotsWithStorage(ctx context.Context, host *models.SHost, snapshot *models.SSnapshot, task taskman.ITask) error { return fmt.Errorf("Not Implement") } diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 81be9e7eee..62ece7fc90 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -228,6 +228,25 @@ func (self *SDisk) CustomizeCreate(ctx context.Context, userCred mcclient.TokenC return self.SSharableVirtualResourceBase.CustomizeCreate(ctx, userCred, ownerProjId, query, data) } +func (self *SDisk) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + storage := self.GetStorage() + if storage == nil { + return nil, httperrors.NewNotFoundError("failed to find storage for disk %s", self.Name) + } + + host := storage.GetMasterHost() + if host == nil { + return nil, httperrors.NewNotFoundError("failed to find host for storage %s with disk %s", storage.Name, self.Name) + } + + data, err := host.GetHostDriver().ValidateUpdateDisk(ctx, userCred, data) + if err != nil { + return nil, err + } + + return self.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data) +} + func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { if disk, err := data.Get("disk"); err != nil { return nil, err diff --git a/pkg/compute/models/hostdrivers.go b/pkg/compute/models/hostdrivers.go index 294c5f03ea..b1b5752a5f 100644 --- a/pkg/compute/models/hostdrivers.go +++ b/pkg/compute/models/hostdrivers.go @@ -7,11 +7,13 @@ import ( "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/mcclient" ) type IHostDriver interface { GetHostType() string CheckAndSetCacheImage(ctx context.Context, host *SHost, storagecache *SStoragecache, task taskman.ITask) error + ValidateUpdateDisk(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) RequestPrepareSaveDiskOnHost(ctx context.Context, host *SHost, disk *SDisk, imageId string, task taskman.ITask) error RequestSaveUploadImageOnHost(ctx context.Context, host *SHost, disk *SDisk, imageId string, task taskman.ITask, data jsonutils.JSONObject) error RequestAllocateDiskOnStorage(ctx context.Context, host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask, content *jsonutils.JSONDict) error From 0ed39ff2c9ff39c71c76e26241bc86f4df3c5166 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Wed, 14 Nov 2018 22:30:17 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=EF=BC=9A=E6=99=AE?= =?UTF-8?q?=E9=80=9A=E7=94=A8=E6=88=B7=E6=97=A0=E6=B3=95=E8=AE=BF=E9=97=AE?= =?UTF-8?q?admin=E6=9D=83=E9=99=90=E7=9A=84=E8=B5=84=E6=BA=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/cloudcommon/db/rbac.go | 28 ++++++++++++++-------------- pkg/cloudcommon/policy/policy.go | 8 +++----- pkg/compute/usages/handler.go | 20 ++++++++++++++------ 3 files changed, 31 insertions(+), 25 deletions(-) diff --git a/pkg/cloudcommon/db/rbac.go b/pkg/cloudcommon/db/rbac.go index 91ae46f031..15798c9536 100644 --- a/pkg/cloudcommon/db/rbac.go +++ b/pkg/cloudcommon/db/rbac.go @@ -54,14 +54,14 @@ func isClassActionRbacAllowed(manager IModelManager, userCred mcclient.TokenCred } else { requireAdmin = true } - if !requireAdmin { - result := policy.PolicyManager.Allow(false, userCred, consts.GetServiceType(), - manager.KeywordPlural(), action, extra...) - if result == rbacutils.Allow || result == rbacutils.OwnerAllow { - return true - } + // if !requireAdmin { + result := policy.PolicyManager.Allow(false, userCred, consts.GetServiceType(), + manager.KeywordPlural(), action, extra...) + if result == rbacutils.Allow || (!requireAdmin && result == rbacutils.OwnerAllow) { + return true } - result := policy.PolicyManager.Allow(true, userCred, consts.GetServiceType(), + // } + result = policy.PolicyManager.Allow(true, userCred, consts.GetServiceType(), manager.KeywordPlural(), action, extra...) return result == rbacutils.Allow } @@ -85,14 +85,14 @@ func isObjectRbacAllowed(manager IModelManager, model IModel, userCred mcclient. requireAdmin = true } - if !requireAdmin { - result := policy.PolicyManager.Allow(false, userCred, consts.GetServiceType(), - manager.KeywordPlural(), action, extra...) - if result == rbacutils.Allow || (result == rbacutils.OwnerAllow && isOwner) { - return true - } + //if !requireAdmin { + result := policy.PolicyManager.Allow(false, userCred, consts.GetServiceType(), + manager.KeywordPlural(), action, extra...) + if result == rbacutils.Allow || (!requireAdmin && result == rbacutils.OwnerAllow && isOwner) { + return true } - result := policy.PolicyManager.Allow(true, userCred, consts.GetServiceType(), + //} + result = policy.PolicyManager.Allow(true, userCred, consts.GetServiceType(), manager.KeywordPlural(), action, extra...) return result == rbacutils.Allow } diff --git a/pkg/cloudcommon/policy/policy.go b/pkg/cloudcommon/policy/policy.go index dd426e42b5..623b2a7ee0 100644 --- a/pkg/cloudcommon/policy/policy.go +++ b/pkg/cloudcommon/policy/policy.go @@ -52,12 +52,12 @@ func parseJsonPolicy(obj jsonutils.JSONObject) (string, rbacutils.SRbacPolicy, e return "", policy, err } - blobStr, err := obj.GetString("blob") + blobStr, err := obj.GetString("policy") if err != nil { log.Errorf("get blob error %s", err) return "", policy, err } - blob, err := jsonutils.ParseString(blobStr) + blob, err := jsonutils.ParseYAML(blobStr) if err != nil { log.Errorf("parse blob json error %s", err) return "", policy, err @@ -77,14 +77,12 @@ func fetchPolicies() (map[string]rbacutils.SRbacPolicy, map[string]rbacutils.SRb policies := make(map[string]rbacutils.SRbacPolicy) adminPolicies := make(map[string]rbacutils.SRbacPolicy) - modules.Policies.SetEnableFilter(false) - offset := 0 for { params := jsonutils.NewDict() params.Add(jsonutils.NewInt(2048), "limit") params.Add(jsonutils.NewInt(int64(offset)), "offset") - result, err := modules.Policies.ResourceManager.List(s, params) + result, err := modules.Policies.List(s, params) if err != nil { log.Errorf("fetch policy failed") diff --git a/pkg/compute/usages/handler.go b/pkg/compute/usages/handler.go index ae74134055..c8b5797364 100644 --- a/pkg/compute/usages/handler.go +++ b/pkg/compute/usages/handler.go @@ -286,19 +286,27 @@ func ReportGeneralUsage(userCred mcclient.TokenCredential, rangeObj db.IStandalo } } + includeCommon := false if consts.IsRbacEnabled() { if policy.PolicyManager.Allow(false, userCred, consts.GetServiceType(), "usages", policy.PolicyActionGet) == rbacutils.Deny { - err = httperrors.NewForbiddenError("not allow to get usages") - return + if !isAdmin { + err = httperrors.NewForbiddenError("not allow to get usages") + return + } + } else { + includeCommon = true } } - commonUsage, err := getCommonGeneralUsage(userCred, rangeObj, hostTypes) - if err != nil { - return + if includeCommon { + var commonUsage map[string]interface{} + commonUsage, err = getCommonGeneralUsage(userCred, rangeObj, hostTypes) + if err != nil { + return + } + count.Include(commonUsage) } - count.Include(commonUsage) return } From faae16c1d1d88feb1dc2c1c63e432cea9031d7fc Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Thu, 15 Nov 2018 01:19:30 +0800 Subject: [PATCH 4/5] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=EF=BC=9Aaliyun=E7=A3=81?= =?UTF-8?q?=E7=9B=98=E5=88=9B=E5=BB=BA=E5=90=8E=EF=BC=8C=E5=B0=86image=20e?= =?UTF-8?q?xternal=20id=E5=90=8C=E6=AD=A5=E4=B8=BAtemplate=20id=EF=BC=8C?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E6=89=BE=E4=B8=8D=E5=88=B0=E5=AF=B9=E5=BA=94?= =?UTF-8?q?=E7=9A=84image=EF=BC=8C=E5=AF=BC=E8=87=B4=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/guestdrivers/aliyun.go | 2 +- pkg/compute/guestdrivers/azure.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index 51b80b1a34..1a91971127 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -437,7 +437,7 @@ func (self *SAliyunGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Contex if diskInfo[i].AutoDelete { disk.AutoDelete = true } - disk.TemplateId = diskInfo[i].TemplateId + // disk.TemplateId = diskInfo[i].TemplateId disk.DiskFormat = diskInfo[i].DiskFormat disk.ExpiredAt = diskInfo[i].ExpiredAt if len(diskInfo[i].Metadata) > 0 { diff --git a/pkg/compute/guestdrivers/azure.go b/pkg/compute/guestdrivers/azure.go index 92cdd431f0..3526e0d30e 100644 --- a/pkg/compute/guestdrivers/azure.go +++ b/pkg/compute/guestdrivers/azure.go @@ -237,7 +237,7 @@ func (self *SAzureGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Context disk.BillingType = diskInfo[i].BillingType disk.FsFormat = diskInfo[i].FsFromat disk.AutoDelete = diskInfo[i].AutoDelete - disk.TemplateId = diskInfo[i].TemplateId + // disk.TemplateId = diskInfo[i].TemplateId disk.DiskFormat = diskInfo[i].DiskFormat disk.ExpiredAt = diskInfo[i].ExpiredAt if len(diskInfo[i].Metadata) > 0 { From 97f1d1c150f93292234761642e7863b19214aa58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Thu, 15 Nov 2018 11:01:33 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E9=81=BF=E5=85=8D=E8=A6=86=E7=9B=96?= =?UTF-8?q?=E6=9C=AC=E5=9C=B0templateId?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/guestdrivers/aws.go | 45 +++++------------------------- pkg/compute/guestdrivers/qcloud.go | 2 +- 2 files changed, 8 insertions(+), 39 deletions(-) diff --git a/pkg/compute/guestdrivers/aws.go b/pkg/compute/guestdrivers/aws.go index e0bd2ef7bd..76449f0b64 100644 --- a/pkg/compute/guestdrivers/aws.go +++ b/pkg/compute/guestdrivers/aws.go @@ -69,43 +69,6 @@ func (self *SAwsGuestDriver) ValidateCreateData(ctx context.Context, userCred mc return self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, data) } -func fetchAwsIVMinfo(desc SManagedVMCreateConfig, iVM cloudprovider.ICloudVM, guestId string) *jsonutils.JSONDict { - data := jsonutils.NewDict() - data.Add(jsonutils.NewString(iVM.GetOSType()), "os") - if len(desc.OsDistribution) > 0 { - data.Add(jsonutils.NewString(desc.OsDistribution), "distro") - } - if len(desc.OsVersion) > 0 { - data.Add(jsonutils.NewString(desc.OsVersion), "version") - } - - idisks, err := iVM.GetIDisks() - - if err != nil { - log.Errorf("GetiDisks error %s", err) - } else { - diskInfo := make([]SDiskInfo, len(idisks)) - for i := 0; i < len(idisks); i += 1 { - dinfo := SDiskInfo{} - dinfo.Uuid = idisks[i].GetGlobalId() - dinfo.Size = idisks[i].GetDiskSizeMB() - dinfo.DiskType = idisks[i].GetDiskType() - if metaData := idisks[i].GetMetadata(); metaData != nil { - dinfo.Metadata = make(map[string]string, 0) - if err := metaData.Unmarshal(dinfo.Metadata); err != nil { - log.Errorf("Get disk %s metadata info error: %v", idisks[i].GetName(), err) - } - } - diskInfo[i] = dinfo - } - data.Add(jsonutils.Marshal(&diskInfo), "disks") - } - - data.Add(jsonutils.NewString(iVM.GetGlobalId()), "uuid") - data.Add(iVM.GetMetadata(), "metadata") - return data -} - func (self *SAwsGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { config := guest.GetDeployConfigOnHost(ctx, host, task.GetParams()) log.Debugf("RequestDeployGuestOnHost: %s", config) @@ -171,7 +134,7 @@ func (self *SAwsGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest return nil, err } - data := fetchAwsIVMinfo(desc, iVM, guest.Id) + data := fetchIVMinfo(desc, iVM, guest.Id, "root", passwd, action) return data, nil }) case "deploy": @@ -279,6 +242,12 @@ func (self *SAwsGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Context, disk.ExternalId = diskInfo[i].Uuid disk.DiskType = diskInfo[i].DiskType disk.Status = models.DISK_READY + disk.BillingType = diskInfo[i].BillingType + disk.FsFormat = diskInfo[i].FsFromat + disk.AutoDelete = true + //disk.TemplateId = diskInfo[i].TemplateId + disk.DiskFormat = diskInfo[i].DiskFormat + disk.ExpiredAt = diskInfo[i].ExpiredAt if len(diskInfo[i].Metadata) > 0 { for key, value := range diskInfo[i].Metadata { if err := disk.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { diff --git a/pkg/compute/guestdrivers/qcloud.go b/pkg/compute/guestdrivers/qcloud.go index ef0dbc6e6d..9c523c085c 100644 --- a/pkg/compute/guestdrivers/qcloud.go +++ b/pkg/compute/guestdrivers/qcloud.go @@ -299,7 +299,7 @@ func (self *SQcloudGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Contex disk.BillingType = diskInfo[i].BillingType disk.FsFormat = diskInfo[i].FsFromat disk.AutoDelete = true - disk.TemplateId = diskInfo[i].TemplateId + //disk.TemplateId = diskInfo[i].TemplateId disk.DiskFormat = diskInfo[i].DiskFormat disk.ExpiredAt = diskInfo[i].ExpiredAt if len(diskInfo[i].Metadata) > 0 {