Merge pull request #7380 from ioito/automated-cherry-pick-of-#7378-upstream-release-3.2

Automated cherry pick of #7378: fix: 添加阿里云rds备份恢复限制条件

pkg/apis/compute/dbinstance.go

@@ -101,7 +101,7 @@ type DBInstanceCreateInput struct {
 	// | -----		| ------	| --- |
 	// | 华为云		|ha, single, replica| |
 	// | 阿里云		|basic, high_availability, always_on, finance||
-	// | Google		|SECOND_GEN | FIRST_GEN 目前谷歌已弃用|
+	// | Google		|Zonal, Regional | |
 	// 翻译:
 	// basic: 基础版
 	// high_availability: 高可用
@@ -110,7 +110,8 @@ type DBInstanceCreateInput struct {
 	// ha: 高可用
 	// single: 单机
 	// replica: 只读
-	// SECNOD_GEN: 第二代
+	// Zonal: 单区域
+	// Regional: 区域级
 	// required: true
 	Category string `json:"category"`
 
@@ -174,9 +175,16 @@ type SDBInstanceRecoveryConfigInput struct {
 	DBInstancebackup string `json:"dbinstancebackup" "yunion:deprecated-by":"dbinstancebackup_id"`
 
 	// 备份Id
+	//
+	//
+	// | 平台		| 支持引擎								| 说明		|
+	// | -----		| ------								| ---		|
+	// | 华为云		|MySQL, SQL Server						| 仅SQL Server支持恢复到当前实例			|
+	// | 阿里云		|MySQL, SQL Server						| MySQL要求必须开启单库单表恢复功能 并且只能是MySQL 8.0 高可用版（本地SSD盘）MySQL 5.7 高可用版（本地SSD盘）或MySQL 5.6 高可用版, MySQL仅支持恢复到当前实例|
+	// | Google		|MySQL, PostgreSQL, SQL Server			| PostgreSQL备份恢复时，要求实例不能有副本			|
 	DBInstancebackupId string `json:"dbinstancebackup_id"`
 
-	// 数据库信息, 例如 {"src":"dest"} 是将备份中的src数据库恢复到目标实例的dest数据库中
+	// 数据库信息, 例如 {"src":"dest"} 是将备份中的src数据库恢复到目标实例的dest数据库中, 阿里云此参数为必传
 	// example: {"sdb1":"ddb1"}
 	Databases map[string]string `json:"databases,allowempty"`
 }

pkg/apis/compute/dbinstance_account.go

@@ -70,13 +70,13 @@ type SDBInstanceSetPrivilegesInput struct {
 
 type DBInstancePrivilege struct {
 	// 数据库名称
-	Database string
+	Database string `json:"database"`
 	// 账号名称
-	Account string
+	Account string `json:"account"`
 	// 数据库Id
-	DBInstancedatabaseId string
+	DBInstancedatabaseId string `json:"dbinstancedatabase_id"`
 	// 权限
-	Privileges string
+	Privileges string `json:"privileges"`
 }
 
 type DBInstanceAccountDetails struct {

pkg/compute/regiondrivers/aliyun.go

@@ -976,6 +976,22 @@ func (self *SAliyunRegionDriver) IsSecurityGroupBelongVpc() bool {
 }
 
 func (self *SAliyunRegionDriver) ValidateDBInstanceRecovery(ctx context.Context, userCred mcclient.TokenCredential, instance *models.SDBInstance, backup *models.SDBInstanceBackup, input api.SDBInstanceRecoveryConfigInput) error {
+	if !utils.IsInStringArray(instance.Engine, []string{api.DBINSTANCE_TYPE_MYSQL, api.DBINSTANCE_TYPE_SQLSERVER}) {
+		return httperrors.NewNotSupportedError("Aliyun %s not support recovery", instance.Engine)
+	}
+	if instance.Engine == api.DBINSTANCE_TYPE_MYSQL {
+		if backup.DBInstanceId != instance.Id {
+			return httperrors.NewUnsupportOperationError("Aliyun %s only support recover from it self backups", instance.Engine)
+		}
+		if !((utils.IsInStringArray(instance.EngineVersion, []string{"8.0", "5.7"}) &&
+			instance.StorageType == api.ALIYUN_DBINSTANCE_STORAGE_TYPE_LOCAL_SSD &&
+			instance.Category == api.ALIYUN_DBINSTANCE_CATEGORY_HA) || (instance.EngineVersion == "5.6" && instance.Category == api.ALIYUN_DBINSTANCE_CATEGORY_HA)) {
+			return httperrors.NewUnsupportOperationError("Aliyun %s only 8.0, 5.7 ha local_ssd or 5.6 ha support recovery from it self backups")
+		}
+	}
+	if len(input.Databases) == 0 {
+		return httperrors.NewMissingParameterError("databases")
+	}
 	return nil
 }
 

pkg/compute/regiondrivers/managedvirtual.go

@@ -1480,7 +1480,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx con
 	if len(cache.ExternalId) > 0 {
 		iSecgroup, err = iRegion.GetISecurityGroupById(cache.ExternalId)
 		if err != nil {
-			if err != cloudprovider.ErrNotFound {
+			if errors.Cause(err) != cloudprovider.ErrNotFound {
 				return "", errors.Wrap(err, "iRegion.GetSecurityGroupById")
 			}
 			cache.ExternalId = ""

pkg/compute/tasks/dbinstance_recovery_task.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -89,6 +90,13 @@ func (self *DBInstanceRecoveryTask) OnInit(ctx context.Context, obj db.IStandalo
 		return
 	}
 
+	databases, err := iRds.GetIDBInstanceDatabases()
+	if err != nil {
+		log.Errorf("failed to get dbinstance %s database error: %v", instance.Name, err)
+	} else {
+		models.DBInstanceDatabaseManager.SyncDBInstanceDatabases(ctx, self.UserCred, instance, databases)
+	}
+
 	db.OpsLog.LogEvent(instance, db.ACT_RESTORE, nil, self.GetUserCred())
 	logclient.AddActionLogWithStartable(self, instance, logclient.ACT_RESTORE, backup, self.UserCred, true)
 	instance.SetStatus(self.UserCred, api.DBINSTANCE_RUNNING, "")

pkg/multicloud/aliyun/aliyun.go

@@ -129,7 +129,7 @@ func jsonRequest(client *sdk.Client, domain, apiVersion, apiName string, params
 			}
 			for _, code := range []string{"404 Not Found"} {
 				if strings.Contains(err.Error(), code) {
-					return nil, cloudprovider.ErrNotFound
+					return nil, errors.Wrapf(cloudprovider.ErrNotFound, err.Error())
 				}
 			}
 			for _, code := range []string{

pkg/multicloud/aliyun/dbinstance.go

@@ -762,26 +762,25 @@ func (rds *SDBInstance) ClosePublicConnection() error {
 }
 
 func (rds *SDBInstance) RecoveryFromBackup(conf *cloudprovider.SDBInstanceRecoveryConfig) error {
-	return rds.region.RecoveryDBInstanceFromBackup(rds.DBInstanceId, conf.BackupId, conf.Databases)
+	if len(conf.OriginDBInstanceExternalId) == 0 {
+		conf.OriginDBInstanceExternalId = rds.DBInstanceId
+	}
+	return rds.region.RecoveryDBInstanceFromBackup(conf.OriginDBInstanceExternalId, rds.DBInstanceId, conf.BackupId, conf.Databases)
 }
 
-func (region *SRegion) RecoveryDBInstanceFromBackup(instanceId string, backupId string, databases map[string]string) error {
-	dbs := []string{}
-	for k, v := range databases {
-		dbs = append(dbs, fmt.Sprintf(`"%s":"%s"`, k, v))
-	}
+func (region *SRegion) RecoveryDBInstanceFromBackup(srcId, destId string, backupId string, databases map[string]string) error {
 	params := map[string]string{
-		"RegionId":     region.RegionId,
-		"DBInstanceId": instanceId,
-		"BackupId":     backupId,
-		"DbNames":      strings.Join(dbs, ","),
+		"RegionId":           region.RegionId,
+		"DBInstanceId":       srcId,
+		"TargetDBInstanceId": destId,
+		"BackupId":           backupId,
+		"DbNames":            jsonutils.Marshal(databases).String(),
 	}
 	_, err := region.rdsRequest("RecoveryDBInstance", params)
 	if err != nil {
 		return errors.Wrap(err, "rdsRequest.RecoveryDBInstance")
 	}
 	return nil
-
 }
 
 func (rds *SDBInstance) GetProjectId() string {

pkg/multicloud/aliyun/dbinstance_backup.go

@@ -293,6 +293,9 @@ func (region *SRegion) waitBackupCreateComplete(instanceId, jobId string) error
 			if job.BackupStatus == "Finished" && job.BackupJobId == jobId {
 				return nil
 			}
+			if job.BackupStatus == "Failed" && job.BackupJobId == jobId {
+				return fmt.Errorf("instance %s backup job %s failed", instanceId, jobId)
+			}
 		}
 		time.Sleep(time.Second * 3)
 	}

pkg/multicloud/aliyun/shell/dbinstance.go

@@ -76,6 +76,7 @@ func init() {
 
 	type DBInstanceRecoveryOptions struct {
 		ID        string
+		TARGET    string
 		BACKUP    string
 		Databases []string
 	}
@@ -95,7 +96,7 @@ func init() {
 				}
 			}
 		}
-		return cli.RecoveryDBInstanceFromBackup(args.ID, args.BACKUP, databases)
+		return cli.RecoveryDBInstanceFromBackup(args.ID, args.TARGET, args.BACKUP, databases)
 	})
 
 }

pkg/multicloud/google/dbinstance.go

@@ -520,7 +520,7 @@ func (rds *SDBInstance) CreateIBackup(conf *cloudprovider.SDBInstanceBackupCreat
 	return "", nil
 }
 
-func (region *SRegion) RecoverFromBackup(instanceId string, backupId string) error {
+func (region *SRegion) RecoverFromBackup(instanceName, dest string, backupId string) error {
 	backup, err := region.GetDBInstanceBackup(backupId)
 	if err != nil {
 		return errors.Wrap(err, "GetDBInstanceBackup")
@@ -528,13 +528,23 @@ func (region *SRegion) RecoverFromBackup(instanceId string, backupId string) err
 	body := map[string]interface{}{
 		"restoreBackupContext": map[string]string{
 			"backupRunId": backup.Id,
+			"instanceId":  instanceName,
+			"project":     region.client.projectId,
 		},
 	}
-	return region.rdsDo(instanceId, "restoreBackup", nil, jsonutils.Marshal(body))
+	return region.rdsDo(dest, "restoreBackup", nil, jsonutils.Marshal(body))
 }
 
 func (rds *SDBInstance) RecoveryFromBackup(conf *cloudprovider.SDBInstanceRecoveryConfig) error {
-	return rds.region.RecoverFromBackup(rds.SelfLink, conf.BackupId)
+	instanceName := rds.Name
+	if len(conf.OriginDBInstanceExternalId) > 0 {
+		instance, err := rds.region.GetDBInstance(conf.OriginDBInstanceExternalId)
+		if err != nil {
+			return errors.Wrapf(err, "GetInstance(%s)", conf.OriginDBInstanceExternalId)
+		}
+		instanceName = instance.Name
+	}
+	return rds.region.RecoverFromBackup(instanceName, rds.SelfLink, conf.BackupId)
 }
 
 func (rds *SDBInstance) Reboot() error {
@@ -549,7 +559,7 @@ func (region *SRegion) DeleteDBInstance(id string) error {
 	return region.rdsDelete(id)
 }
 
-func (region *SRegion) CreateRds(name, databaseVersion, category, instanceType, storageType string, diskSizeGb int, vpcId, zoneId, password string) (*SDBInstance, error) {
+func (region *SRegion) CreateRds(name, engine, databaseVersion, category, instanceType, storageType string, diskSizeGb int, vpcId, zoneId, password string) (*SDBInstance, error) {
 	settings := map[string]interface{}{
 		"tier":              instanceType,
 		"storageAutoResize": true,
@@ -558,11 +568,14 @@ func (region *SRegion) CreateRds(name, databaseVersion, category, instanceType,
 	}
 	if utils.IsInStringArray(category, []string{api.GOOGLE_DBINSTANCE_CATEGORY_REGIONAL, api.GOOGLE_DBINSTANCE_CATEGORY_ZONAL}) {
 		settings["availabilityType"] = strings.ToUpper(category)
-		settings["backupConfiguration"] = map[string]interface{}{
-			"enabled":          true,
-			"binaryLogEnabled": true,
-			"startTime":        "19:00",
+		backupConfiguration := map[string]interface{}{
+			"enabled":   true,
+			"startTime": "19:00",
 		}
+		if engine == api.DBINSTANCE_TYPE_MYSQL {
+			backupConfiguration["binaryLogEnabled"] = true
+		}
+		settings["backupConfiguration"] = backupConfiguration
 	}
 	ipConfiguration := map[string]interface{}{
 		"ipv4Enabled": true,
@@ -619,7 +632,7 @@ func (region *SRegion) CreateDBInstance(desc *cloudprovider.SManagedDBInstanceCr
 			desc.ZoneIds = append(desc.ZoneIds, "")
 		}
 		for _, zoneId := range desc.ZoneIds {
-			rds, err = region.CreateRds(desc.Name, databaseVersion, desc.Category, desc.InstanceType, desc.StorageType, desc.DiskSizeGB, desc.VpcId, zoneId, desc.Password)
+			rds, err = region.CreateRds(desc.Name, desc.Engine, databaseVersion, desc.Category, desc.InstanceType, desc.StorageType, desc.DiskSizeGB, desc.VpcId, zoneId, desc.Password)
 			if err == nil {
 				break
 			} else {
@@ -635,7 +648,7 @@ func (region *SRegion) CreateDBInstance(desc *cloudprovider.SManagedDBInstanceCr
 				desc.ZoneIds = append(desc.ZoneIds, "")
 			}
 			for _, zoneId := range spec.ZoneIds {
-				rds, err = region.CreateRds(desc.Name, databaseVersion, desc.Category, desc.InstanceType, desc.StorageType, desc.DiskSizeGB, desc.VpcId, zoneId, desc.Password)
+				rds, err = region.CreateRds(desc.Name, desc.Engine, databaseVersion, desc.Category, desc.InstanceType, desc.StorageType, desc.DiskSizeGB, desc.VpcId, zoneId, desc.Password)
 				if err == nil {
 					break
 				} else {

pkg/multicloud/google/shell/dbinstance.go

@@ -60,8 +60,9 @@ func init() {
 
 	type DBInstanceCreateOptions struct {
 		NAME            string
+		ENGINE          string
 		DATABASEVERSION string
-		BackendType     string `default:"SECOND_GEN" choices:"FIRST_GEN|SECOND_GEN|EXTERNAL"`
+		Category        string `default:"Zonal" choices:"Zonal|Regional"`
 		INSTANCE_TYPE   string
 		STORAGE_TYPE    string
 		DISK_SIZE_GB    int
@@ -71,7 +72,7 @@ func init() {
 	}
 
 	shellutils.R(&DBInstanceCreateOptions{}, "dbinstance-create", "Create dbinstance", func(cli *google.SRegion, args *DBInstanceCreateOptions) error {
-		instance, err := cli.CreateRds(args.NAME, args.DATABASEVERSION, args.BackendType, args.INSTANCE_TYPE, args.STORAGE_TYPE, args.DISK_SIZE_GB, args.VpcId, args.ZoneId, args.Password)
+		instance, err := cli.CreateRds(args.NAME, args.ENGINE, args.DATABASEVERSION, args.Category, args.INSTANCE_TYPE, args.STORAGE_TYPE, args.DISK_SIZE_GB, args.VpcId, args.ZoneId, args.Password)
 		if err != nil {
 			return err
 		}
@@ -90,12 +91,13 @@ func init() {
 	})
 
 	type DBInstanceRecoveryOptions struct {
-		RDS    string
+		NAME   string
+		TARGET string
 		BACKUP string
 	}
 
 	shellutils.R(&DBInstanceRecoveryOptions{}, "dbinstance-restore", "restore dbinstance from backup", func(cli *google.SRegion, args *DBInstanceRecoveryOptions) error {
-		return cli.RecoverFromBackup(args.RDS, args.BACKUP)
+		return cli.RecoverFromBackup(args.NAME, args.TARGET, args.BACKUP)
 	})
 
 }