feat(region) qcloud redis support multi zones

This commit is contained in:
tangbin
2021-03-10 11:57:15 +08:00
committed by TangBin
parent 765366f8bc
commit 33285eb777
10 changed files with 168 additions and 56 deletions
+6
View File
@@ -159,6 +159,12 @@ type ElasticcacheCreateInput struct {
// 主可用区名称或Id
Zone string `json:"zone"`
// 备可用区名称或Id列表
// 默认副本与主可用区一致
// 支持此参数的云厂商: 腾讯云
// required: false
SlaveZones []string `json:"slave_zones"`
// Ip子网名称或Id,建议使用Id
// required: true
Network string `json:"network"`
+65 -27
View File
@@ -757,24 +757,28 @@ func (manager *SElasticcacheManager) AllowCreateItem(ctx context.Context, userCr
}
func (manager *SElasticcacheManager) BatchCreateValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
input, err := manager.validateCreateData(ctx, userCred, ownerId, query, data)
_data := api.ElasticcacheCreateInput{}
err := data.Unmarshal(&_data)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "ElasticcacheCreateInput.Unmarshal")
}
input, err := manager.validateCreateData(ctx, userCred, ownerId, query, _data)
if err != nil {
return nil, errors.Wrap(err, "validateCreateData")
}
return input, nil
}
func (manager *SElasticcacheManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) {
data := input.JSON(&input)
return manager.validateCreateData(ctx, userCred, ownerId, query, data)
return manager.validateCreateData(ctx, userCred, ownerId, query, input)
}
func (manager *SElasticcacheManager) validateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
func (manager *SElasticcacheManager) validateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) {
var region *SCloudregion
var provider *SCloudprovider
if id, _ := data.GetString("network"); len(id) > 0 {
network, err := db.FetchByIdOrName(NetworkManager, userCred, strings.Split(id, ",")[0])
if len(input.Network) > 0 {
network, err := db.FetchByIdOrName(NetworkManager, userCred, strings.Split(input.Network, ",")[0])
if err != nil {
return nil, fmt.Errorf("getting network failed")
}
@@ -788,34 +792,28 @@ func (manager *SElasticcacheManager) validateCreateData(ctx context.Context, use
}
// postpiad billing cycle
billingType, _ := data.GetString("billing_type")
if billingType == billing_api.BILLING_TYPE_POSTPAID {
billingCycle, _ := data.GetString("duration")
if len(billingCycle) > 0 {
cycle, err := bc.ParseBillingCycle(billingCycle)
if input.BillingType == billing_api.BILLING_TYPE_POSTPAID {
_cycle := input.Duration
if len(_cycle) > 0 {
cycle, err := bc.ParseBillingCycle(_cycle)
if err != nil {
return nil, httperrors.NewInputParameterError("invalid billing_cycle %s", billingCycle)
return nil, httperrors.NewInputParameterError("invalid billing_cycle %s", _cycle)
}
tm := time.Time{}
data.Set("billing_cycle", jsonutils.NewString(cycle.String()))
data.Set("expired_at", jsonutils.NewString(cycle.EndAt(tm).Format("2006-01-02 15:04:05")))
input.BillingCycle = cycle.String()
// .Format("2006-01-02 15:04:05")
input.ExpiredAt = cycle.EndAt(tm)
}
}
input := apis.VirtualResourceCreateInput{}
var err error
err = data.Unmarshal(&input)
input.VirtualResourceCreateInput, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.VirtualResourceCreateInput)
if err != nil {
return nil, httperrors.NewInternalServerError("unmarshal VirtualResourceCreateInput fail %s", err)
return nil, errors.Wrap(err, "SVirtualResourceBaseManager.ValidateCreateData")
}
input, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input)
if err != nil {
return nil, err
}
data.Update(jsonutils.Marshal(input))
ret, err := region.GetDriver().ValidateCreateElasticcacheData(ctx, userCred, nil, data)
ret, err := region.GetDriver().ValidateCreateElasticcacheData(ctx, userCred, nil, input)
if err != nil {
return nil, errors.Wrap(err, "region.GetDriver().ValidateCreateElasticcacheData")
}
@@ -866,6 +864,36 @@ func (self *SElasticcache) StartElasticcacheCreateTask(ctx context.Context, user
return nil
}
func (self *SElasticcache) GetSlaveZones() ([]SZone, error) {
if len(self.SlaveZones) > 0 {
zones := []SZone{}
sz := strings.Split(self.SlaveZones, ",")
err := ZoneManager.Query().In("id", sz).All(&zones)
if err != nil {
return nil, errors.Wrap(err, "GetZones")
}
zoneMap := map[string]SZone{}
for i := range zones {
zoneMap[zones[i].GetId()] = zones[i]
}
ret := make([]SZone, len(sz))
for i := range sz {
z, ok := zoneMap[sz[i]]
if !ok {
return nil, fmt.Errorf("zone %s is not found", sz[i])
}
ret[i] = z
}
return ret, nil
}
return []SZone{}, nil
}
/*func (self *SElasticcache) GetIRegion() (cloudprovider.ICloudRegion, error) {
provider, err := self.GetDriver()
if err != nil {
@@ -1080,11 +1108,21 @@ func (self *SElasticcache) GetCreateQCloudElasticcacheParams(data *jsonutils.JSO
zone := self.GetZone()
if zone != nil {
izone, err := iregion.GetIZoneById(zone.ExternalId)
zones := []SZone{*zone}
// slave zones
sz, err := self.GetSlaveZones()
if err != nil {
return nil, errors.Wrap(err, "elasticcache.GetCreateHuaweiElasticcacheParams.Zone")
return nil, errors.Wrap(err, "GetSlaveZones")
}
zones = append(zones, sz...)
for i := range zones {
izone, err := iregion.GetIZoneById(zones[i].ExternalId)
if err != nil {
return nil, errors.Wrap(err, "elasticcache.GetCreateHuaweiElasticcacheParams.Zone")
}
input.ZoneIds = append(input.ZoneIds, izone.GetId())
}
input.ZoneIds = []string{izone.GetId()}
}
switch self.BillingType {
+1 -1
View File
@@ -200,7 +200,7 @@ type IElasticcacheDriver interface {
AllowCreateElasticcacheBackup(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, elasticcache *SElasticcache) error
AllowUpdateElasticcacheAuthMode(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, elasticcache *SElasticcache) error
ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error)
ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error)
ValidateCreateElasticcacheAccountData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error)
ValidateCreateElasticcacheAclData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error)
ValidateCreateElasticcacheBackupData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error)
+2 -1
View File
@@ -1156,7 +1156,8 @@ func (self *SAliyunRegionDriver) ValidateDBInstanceAccountPrivilege(ctx context.
return nil
}
func (self *SAliyunRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
func (self *SAliyunRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) {
data := input.JSON(input)
zoneV := validators.NewModelIdOrNameValidator("zone", "zone", ownerId)
networkV := validators.NewModelIdOrNameValidator("network", "network", ownerId)
+29 -22
View File
@@ -2286,7 +2286,7 @@ func (self *SHuaWeiRegionDriver) ValidateDBInstanceRecovery(ctx context.Context,
return nil
}
func validatorSlaveZones(ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict, optional bool) error {
func validatorSlaveZones(ownerId mcclient.IIdentityProvider, regionId string, data *jsonutils.JSONDict, optional bool) error {
s, err := data.GetString("slave_zones")
if err != nil {
if optional {
@@ -2299,33 +2299,36 @@ func validatorSlaveZones(ownerId mcclient.IIdentityProvider, data *jsonutils.JSO
zones := strings.Split(s, ",")
ret := []string{}
zoneV := validators.NewModelIdOrNameValidator("zone", "zone", ownerId)
for _, zone := range zones {
for i := range zones {
_data := jsonutils.NewDict()
_data.Add(jsonutils.NewString(zone), "zone")
_data.Set("zone", jsonutils.NewString(zones[i]))
if err := zoneV.Validate(_data); err != nil {
return errors.Wrap(err, "validatorSlaveZones")
} else {
if zoneV.Model.(*models.SZone).GetCloudRegionId() != regionId {
return errors.Wrap(fmt.Errorf("zone %s is not in region %s", zoneV.Model.GetName(), regionId), "GetCloudRegionId")
}
ret = append(ret, zoneV.Model.GetId())
}
}
if sku, err := data.GetString("sku"); err != nil || len(sku) == 0 {
return httperrors.NewMissingParameterError("sku")
} else {
chargeType, _ := data.GetString("charge_type")
_skuModel, err := db.FetchByIdOrName(models.ElasticcacheSkuManager, ownerId, sku)
if err != nil {
return err
}
skuModel := _skuModel.(*models.SElasticcacheSku)
for _, zoneId := range zones {
if err := ValidateElasticcacheSku(zoneId, chargeType, skuModel, nil); err != nil {
return err
}
}
}
//if sku, err := data.GetString("sku"); err != nil || len(sku) == 0 {
// return httperrors.NewMissingParameterError("sku")
//} else {
// chargeType, _ := data.GetString("charge_type")
//
// _skuModel, err := db.FetchByIdOrName(models.ElasticcacheSkuManager, ownerId, sku)
// if err != nil {
// return err
// }
//
// skuModel := _skuModel.(*models.SElasticcacheSku)
// for _, zoneId := range zones {
// if err := ValidateElasticcacheSku(zoneId, chargeType, skuModel, nil); err != nil {
// return err
// }
// }
//}
data.Set("slave_zones", jsonutils.NewString(strings.Join(ret, ",")))
return nil
@@ -2355,7 +2358,8 @@ func ValidateElasticcacheSku(zoneId string, chargeType string, sku *models.SElas
return nil
}
func (self *SHuaWeiRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
func (self *SHuaWeiRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) {
data := input.JSON(input)
zoneV := validators.NewModelIdOrNameValidator("zone", "zone", ownerId)
networkV := validators.NewModelIdOrNameValidator("network", "network", ownerId)
// secgroupV := validators.NewModelIdOrNameValidator("security_group", "secgroup", ownerId)
@@ -2408,7 +2412,10 @@ func (self *SHuaWeiRegionDriver) ValidateCreateElasticcacheData(ctx context.Cont
}
// validate slave zones
if err := validatorSlaveZones(ownerId, data, true); err != nil {
if len(input.SlaveZones) > 0 {
data.Set("slave_zones", jsonutils.NewString(strings.Join(input.SlaveZones, ",")))
}
if err := validatorSlaveZones(ownerId, zoneV.Model.(*models.SZone).GetCloudRegionId(), data, true); err != nil {
return nil, err
}
+2 -2
View File
@@ -1268,8 +1268,8 @@ func (self *SKVMRegionDriver) RequestCreateElasticcache(ctx context.Context, use
return nil
}
func (self *SKVMRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
return data, nil
func (self *SKVMRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) {
return input.JSON(input), nil
}
func (self *SKVMRegionDriver) RequestRestartElasticcache(ctx context.Context, userCred mcclient.TokenCredential, elasticcache *models.SElasticcache, task taskman.ITask) error {
+9 -2
View File
@@ -1910,8 +1910,15 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateElasticcache(ctx co
return nil
}
func (self *SManagedVirtualizationRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
return self.ValidateManagerId(ctx, userCred, data)
func (self *SManagedVirtualizationRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) {
m := jsonutils.NewDict()
m.Set("manager_id", jsonutils.NewString(input.ManagerId))
_, err := self.ValidateManagerId(ctx, userCred, m)
if err != nil {
return nil, errors.Wrap(err, "ValidateManagerId")
}
return input.JSON(input), nil
}
func (self *SManagedVirtualizationRegionDriver) RequestRestartElasticcache(ctx context.Context, userCred mcclient.TokenCredential, ec *models.SElasticcache, task taskman.ITask) error {
+25 -1
View File
@@ -19,6 +19,7 @@ import (
"fmt"
"regexp"
"strconv"
"strings"
"time"
"yunion.io/x/jsonutils"
@@ -1470,7 +1471,8 @@ func (self *SQcloudRegionDriver) GetMaxElasticcacheSecurityGroupCount() int {
return 10
}
func (self *SQcloudRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
func (self *SQcloudRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) {
data := input.JSON(input)
zoneV := validators.NewModelIdOrNameValidator("zone", "zone", ownerId)
networkV := validators.NewModelIdOrNameValidator("network", "network", ownerId)
instanceTypeV := validators.NewModelIdOrNameValidator("instance_type", "elasticcachesku", ownerId)
@@ -1512,6 +1514,28 @@ func (self *SQcloudRegionDriver) ValidateCreateElasticcacheData(ctx context.Cont
data.Set("capacity_mb", jsonutils.NewInt(int64(sku.MemorySizeMB)))
}
// validate slave zones
if len(input.SlaveZones) > 0 {
if len(input.SlaveZones) < sku.ReplicasNum {
padding := make([]string, sku.ReplicasNum-len(input.SlaveZones))
for i := range padding {
padding[i] = zoneId
}
input.SlaveZones = append(input.SlaveZones, padding...)
}
data.Set("slave_zones", jsonutils.NewString(strings.Join(input.SlaveZones, ",")))
}
if err := validatorSlaveZones(ownerId, zoneV.Model.(*models.SZone).GetCloudRegionId(), data, true); err != nil {
return nil, err
}
sz, _ := data.GetString("slave_zones")
if len(strings.Split(sz, ",")) > sku.ReplicasNum {
return nil, fmt.Errorf("the number of slave zones can not beyond redis replicas number")
}
// validate secgroups
secgroups := []string{}
err := data.Unmarshal(&secgroups, "secgroup_ids")
@@ -80,6 +80,7 @@ var zoneMaps = map[int]string{
100002: "ap-guangzhou-2",
100003: "ap-guangzhou-3",
100004: "ap-guangzhou-4",
100006: "ap-guangzhou-6",
110001: "ap-shenzhen-fsi-1",
110002: "ap-shenzhen-fsi-2",
110003: "ap-shenzhen-fsi-3",
@@ -95,11 +96,14 @@ var zoneMaps = map[int]string{
700003: "ap-shanghai-fsi-3",
330001: "ap-nanjing-1",
330002: "ap-nanjing-2",
330003: "ap-nanjing-3",
800001: "ap-beijing-1",
800002: "ap-beijing-2",
800003: "ap-beijing-3",
800004: "ap-beijing-4",
800005: "ap-beijing-5",
800006: "ap-beijing-6",
800007: "ap-beijing-7",
460001: "ap-beijing-fsi-1",
360001: "ap-tianjin-1",
360002: "ap-tianjin-2",
@@ -108,12 +112,14 @@ var zoneMaps = map[int]string{
190001: "ap-chongqing-1",
300001: "ap-hongkong-1",
300002: "ap-hongkong-2",
300003: "ap-hongkong-3",
390001: "ap-taipei-1",
900001: "ap-singapore-1",
230001: "ap-bangkok-1",
210001: "ap-mumbai-1",
210002: "ap-mumbai-2",
180001: "ap-seoul-1",
180002: "ap-seoul-2",
250001: "ap-tokyo-1",
150001: "na-siliconvalley-1",
150002: "na-siliconvalley-2",
@@ -129,6 +135,7 @@ var zoneIdMaps = map[string]int{
"ap-guangzhou-2": 100002,
"ap-guangzhou-3": 100003,
"ap-guangzhou-4": 100004,
"ap-guangzhou-6": 100006,
"ap-shenzhen-fsi-1": 110001,
"ap-shenzhen-fsi-2": 110002,
"ap-shenzhen-fsi-3": 110003,
@@ -144,11 +151,14 @@ var zoneIdMaps = map[string]int{
"ap-shanghai-fsi-3": 700003,
"ap-nanjing-1": 330001,
"ap-nanjing-2": 330002,
"ap-nanjing-3": 330003,
"ap-beijing-1": 800001,
"ap-beijing-2": 800002,
"ap-beijing-3": 800003,
"ap-beijing-4": 800004,
"ap-beijing-5": 800005,
"ap-beijing-6": 800006,
"ap-beijing-7": 800007,
"ap-beijing-fsi-1": 460001,
"ap-tianjin-1": 360001,
"ap-tianjin-2": 360002,
@@ -157,12 +167,14 @@ var zoneIdMaps = map[string]int{
"ap-chongqing-1": 190001,
"ap-hongkong-1": 300001,
"ap-hongkong-2": 300002,
"ap-hongkong-3": 300003,
"ap-taipei-1": 390001,
"ap-singapore-1": 900001,
"ap-bangkok-1": 230001,
"ap-mumbai-1": 210001,
"ap-mumbai-2": 210002,
"ap-seoul-1": 180001,
"ap-seoul-2": 180002,
"ap-tokyo-1": 250001,
"na-siliconvalley-1": 150001,
"na-siliconvalley-2": 150002,
+17
View File
@@ -1013,6 +1013,7 @@ func (region *SRegion) GetIElasticcacheById(id string) (cloudprovider.ICloudElas
return nil, cloudprovider.ErrNotFound
}
// DescribeProductInfo 可以查询在售可用区信息
// https://cloud.tencent.com/document/product/239/20026
func (r *SRegion) CreateIElasticcaches(ec *cloudprovider.SCloudElasticCacheInput) (cloudprovider.ICloudElasticcache, error) {
params := map[string]string{}
@@ -1025,6 +1026,22 @@ func (r *SRegion) CreateIElasticcaches(ec *cloudprovider.SCloudElasticCacheInput
return nil, fmt.Errorf("can't convert zone %s to integer id", ec.ZoneIds[0])
}
if len(ec.ZoneIds) > 1 {
for i := range ec.ZoneIds {
if i == 0 {
params[fmt.Sprintf("NodeSet.%d.NodeType", i)] = "0"
params[fmt.Sprintf("NodeSet.%d.ZoneId", i)] = fmt.Sprintf("%d", zoneId)
} else {
_z, ok := zoneIdMaps[ec.ZoneIds[i]]
if !ok {
return nil, fmt.Errorf("can't convert zone %s to integer id", ec.ZoneIds[i])
}
params[fmt.Sprintf("NodeSet.%d.NodeType", i)] = "1"
params[fmt.Sprintf("NodeSet.%d.ZoneId", i)] = fmt.Sprintf("%d", _z)
}
}
}
spec, err := parseLocalInstanceSpec(ec.InstanceType)
if err != nil {
return nil, errors.Wrap(err, "parseLocalInstanceSpec")