Merge pull request #5568 from tb365/automated-cherry-pick-of-#5567-upstream-release-3.1

Automated cherry pick of #5567: ctyun bugfix

pkg/compute/models/guests.go

@@ -3437,11 +3437,14 @@ func (self *SGuest) DeleteAllDisksInDB(ctx context.Context, userCred mcclient.To
 		if err != nil {
 			return err
 		}
-		db.OpsLog.LogEvent(disk, db.ACT_DELETE, nil, userCred)
-		db.OpsLog.LogEvent(disk, db.ACT_DELOCATE, nil, userCred)
-		err = disk.RealDelete(ctx, userCred)
-		if err != nil {
-			return err
+
+		if disk != nil {
+			db.OpsLog.LogEvent(disk, db.ACT_DELETE, nil, userCred)
+			db.OpsLog.LogEvent(disk, db.ACT_DELOCATE, nil, userCred)
+			err = disk.RealDelete(ctx, userCred)
+			if err != nil {
+				return err
+			}
 		}
 	}
 	return nil

pkg/multicloud/ctyun/disk.go

@@ -225,6 +225,22 @@ func (self *SDisk) GetIsAutoDelete() bool {
 }
 
 func (self *SDisk) GetTemplateId() string {
+	if len(self.Attachments) > 0 && len(self.Attachments[0].ServerID) > 0 {
+		server, err := self.storage.zone.region.GetVMById(self.Attachments[0].ServerID)
+		if err != nil {
+			log.Errorf("SDisk.GetTemplateId %s", err)
+			return ""
+		}
+
+		image, err := server.GetImage()
+		if err != nil {
+			log.Errorf("SDisk.GetImage %s", err)
+			return ""
+		}
+
+		return image.GetId()
+	}
+
 	return ""
 }
 
@@ -274,54 +290,17 @@ func (self *SDisk) Delete(ctx context.Context) error {
 }
 
 func (self *SDisk) CreateISnapshot(ctx context.Context, name string, desc string) (cloudprovider.ICloudSnapshot, error) {
-	jobId, err := self.storage.zone.region.CreateSnapshot(name, self.GetId(), desc)
-	if err != nil {
-		return nil, errors.Wrap(err, "Disk.CreateISnapshot.CreateSnapshot")
-	}
-
-	snapshotId := ""
-	err = cloudprovider.Wait(10*time.Second, 1800*time.Second, func() (b bool, err error) {
-		statusJson, err := self.storage.zone.region.GetVbsJob(jobId)
-		// ctyun 偶尔会报客户端错误，其实job已经到后台执行了
-		if err != nil {
-			log.Debugf("Ctyun.SDisk.CreateISnapshot.GetVbsJob %s", err)
-			return false, nil
-		}
-
-		if status, _ := statusJson.GetString("status"); status == "SUCCESS" {
-			snapshotId, _ = statusJson.GetString("entities", "snapshot_id")
-			return true, nil
-		} else if status == "FAILED" {
-			return false, fmt.Errorf("CreateSnapshot job %s failed", jobId)
-		} else {
-			return false, nil
-		}
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "Disk.CreateISnapshot.Wait")
-	}
-
-	return self.storage.zone.region.GetSnapshot(self.GetId(), snapshotId)
+	return nil, cloudprovider.ErrNotSupported
 }
 
 // POST http://ctyun-api-url/apiproxy/v3/ondemand/createVBS
 func (self *SDisk) GetISnapshot(idStr string) (cloudprovider.ICloudSnapshot, error) {
-	return self.storage.zone.region.GetSnapshot(self.GetId(), idStr)
+	return nil, cloudprovider.ErrNotFound
 }
 
-// GET http://ctyun-api-url/apiproxy/v3/ondemand/queryVBSs
+// no snapshot api opened
 func (self *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) {
-	snapshots, err := self.storage.zone.region.GetSnapshots(self.GetId())
-	if err != nil {
-		return nil, errors.Wrap(err, "SDisk.GetISnapshots")
-	}
-
-	isnapshots := []cloudprovider.ICloudSnapshot{}
-	for i := range snapshots {
-		isnapshots[i] = &snapshots[i]
-	}
-
-	return isnapshots, nil
+	return []cloudprovider.ICloudSnapshot{}, nil
 }
 
 // POST http://ctyun-api-url/apiproxy/v3/ondemand/updateDiskBackupPolicy
@@ -336,10 +315,10 @@ func (self *SDisk) Resize(ctx context.Context, newSizeMB int64) error {
 	}
 
 	err = cloudprovider.Wait(10*time.Second, 1800*time.Second, func() (b bool, err error) {
-		statusJson, err := self.storage.zone.region.GetVbsJob(jobId)
+		statusJson, err := self.storage.zone.region.GetVolumeJob(jobId)
 		// ctyun 偶尔会报客户端错误，其实job已经到后台执行了
 		if err != nil {
-			log.Debugf("Ctyun.SDisk.Resize.GetVbsJob %s", err)
+			log.Debugf("Ctyun.SDisk.Resize.GetVolumeJob %s", err)
 			return false, nil
 		}
 
@@ -359,32 +338,7 @@ func (self *SDisk) Resize(ctx context.Context, newSizeMB int64) error {
 }
 
 func (self *SDisk) Reset(ctx context.Context, snapshotId string) (string, error) {
-	jobId, err := self.storage.zone.region.RestoreDisk(self.GetId(), snapshotId)
-	if err != nil {
-		return "", errors.Wrap(err, "Disk.Reset")
-	}
-
-	err = cloudprovider.Wait(10*time.Second, 1800*time.Second, func() (b bool, err error) {
-		statusJson, err := self.storage.zone.region.GetVbsJob(jobId)
-		// ctyun 偶尔会报客户端错误，其实job已经到后台执行了
-		if err != nil {
-			log.Debugf("Ctyun.SDisk.Reset.GetVbsJob %s", err)
-			return false, nil
-		}
-
-		if status, _ := statusJson.GetString("status"); status == "SUCCESS" {
-			return true, nil
-		} else if status == "FAILED" {
-			return false, fmt.Errorf("Reset job %s failed", jobId)
-		} else {
-			return false, nil
-		}
-	})
-	if err != nil {
-		return "", errors.Wrap(err, "Disk.Reset.Wait")
-	}
-
-	return self.GetId(), nil
+	return "", cloudprovider.ErrNotSupported
 }
 
 func (self *SDisk) Rebuild(ctx context.Context) error {
@@ -541,7 +495,7 @@ func (self *SRegion) CreateDisk(zoneId, name, diskType, size string) (*SDisk, er
 	return self.GetDisk(diskId)
 }
 
-func (self *SRegion) CreateSnapshot(name, volumeId, desc string) (string, error) {
+func (self *SRegion) CreateDiskBackup(name, volumeId, desc string) (string, error) {
 	params := map[string]jsonutils.JSONObject{
 		"regionId":    jsonutils.NewString(self.GetId()),
 		"volumeId":    jsonutils.NewString(volumeId),
@@ -551,13 +505,13 @@ func (self *SRegion) CreateSnapshot(name, volumeId, desc string) (string, error)
 
 	resp, err := self.client.DoPost("/apiproxy/v3/ondemand/createVBS", params)
 	if err != nil {
-		return "", errors.Wrap(err, "Region.CreateSnapshot.DoPost")
+		return "", errors.Wrap(err, "Region.CreateDiskBackup.DoPost")
 	}
 
 	var jobId string
 	err = resp.Unmarshal(&jobId, "returnObj", "data")
 	if err != nil {
-		return "", errors.Wrap(err, "Region.CreateSnapshot.Unmarshal")
+		return "", errors.Wrap(err, "Region.CreateDiskBackup.Unmarshal")
 	}
 
 	return jobId, nil

pkg/multicloud/ctyun/diskbackup.go

@@ -24,7 +24,7 @@ import (
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 )
 
-type SSnapshot struct {
+type SDiskBackup struct {
 	region *SRegion
 
 	Status           string `json:"status"`
@@ -39,19 +39,19 @@ type SSnapshot struct {
 	CreatedAt        string `json:"created_at"`
 }
 
-func (self *SSnapshot) GetId() string {
+func (self *SDiskBackup) GetId() string {
 	return self.ID
 }
 
-func (self *SSnapshot) GetName() string {
+func (self *SDiskBackup) GetName() string {
 	return self.Name
 }
 
-func (self *SSnapshot) GetGlobalId() string {
+func (self *SDiskBackup) GetGlobalId() string {
 	return self.GetId()
 }
 
-func (self *SSnapshot) GetStatus() string {
+func (self *SDiskBackup) GetStatus() string {
 	switch self.Status {
 	case "available":
 		return api.SNAPSHOT_READY
@@ -68,8 +68,8 @@ func (self *SSnapshot) GetStatus() string {
 	}
 }
 
-func (self *SSnapshot) Refresh() error {
-	snapshot, err := self.region.GetSnapshot(self.VolumeID, self.GetId())
+func (self *SDiskBackup) Refresh() error {
+	snapshot, err := self.region.GetDiskBackup(self.VolumeID, self.GetId())
 	if err != nil {
 		return err
 	}
@@ -81,63 +81,58 @@ func (self *SSnapshot) Refresh() error {
 	return nil
 }
 
-func (self *SSnapshot) IsEmulated() bool {
+func (self *SDiskBackup) IsEmulated() bool {
 	return false
 }
 
-func (self *SSnapshot) GetMetadata() *jsonutils.JSONDict {
+func (self *SDiskBackup) GetMetadata() *jsonutils.JSONDict {
 	return nil
 }
 
-func (self *SSnapshot) GetProjectId() string {
+func (self *SDiskBackup) GetProjectId() string {
 	return ""
 }
 
-func (self *SSnapshot) GetSizeMb() int32 {
+func (self *SDiskBackup) GetSizeMb() int32 {
 	return self.Size * 1024
 }
 
-func (self *SSnapshot) GetDiskId() string {
+func (self *SDiskBackup) GetDiskId() string {
 	return self.VolumeID
 }
 
-func (self *SSnapshot) GetDiskType() string {
+func (self *SDiskBackup) GetDiskType() string {
 	disk, err := self.region.GetDisk(self.VolumeID)
 	if err != nil {
-		log.Debugf("SSnapshot.GetDiskType.GetDisk %s", err)
+		log.Debugf("SDiskBackup.GetDiskType.GetDisk %s", err)
 		return ""
 	}
 
 	return disk.GetDiskType()
 }
 
-func (self *SSnapshot) Delete() error {
-	_, err := self.region.DeleteSnapshot(self.GetId())
-	if err != nil {
-		return errors.Wrap(err, "Snapshot.Delete.DeleteSnapshot")
-	}
-
-	return nil
+func (self *SDiskBackup) Delete() error {
+	return errors.ErrNotSupported
 }
 
-func (self *SRegion) GetSnapshot(diskId string, snapshotId string) (*SSnapshot, error) {
-	snapshots, err := self.GetSnapshots(diskId)
+func (self *SRegion) GetDiskBackup(diskId string, backupId string) (*SDiskBackup, error) {
+	backups, err := self.GetDiskBackups(diskId)
 	if err != nil {
-		return nil, errors.Wrap(err, "SRegion.GetSnapshot.GetSnapshots")
+		return nil, errors.Wrap(err, "SRegion.GetSnapshot.GetDiskBackups")
 	}
 
-	for i := range snapshots {
-		snapshot := snapshots[i]
-		if snapshot.ID == snapshotId || snapshot.Container == snapshotId {
-			snapshot.region = self
-			return &snapshot, nil
+	for i := range backups {
+		backup := backups[i]
+		if backup.ID == backupId || backup.Container == backupId {
+			backup.region = self
+			return &backup, nil
 		}
 	}
 
-	return nil, errors.Wrap(errors.ErrNotFound, "SRegion.GetSnapshot")
+	return nil, errors.Wrap(errors.ErrNotFound, "SRegion.GetDiskBackup")
 }
 
-func (self *SRegion) GetSnapshots(diskId string) ([]SSnapshot, error) {
+func (self *SRegion) GetDiskBackups(diskId string) ([]SDiskBackup, error) {
 	params := map[string]string{
 		"regionId": self.GetId(),
 	}
@@ -148,13 +143,13 @@ func (self *SRegion) GetSnapshots(diskId string) ([]SSnapshot, error) {
 
 	resp, err := self.client.DoGet("/apiproxy/v3/ondemand/queryVBSDetails", params)
 	if err != nil {
-		return nil, errors.Wrap(err, "SRegion.GetSnapshots.DoGet")
+		return nil, errors.Wrap(err, "SRegion.GetDiskBackups.DoGet")
 	}
 
-	ret := make([]SSnapshot, 0)
+	ret := make([]SDiskBackup, 0)
 	err = resp.Unmarshal(&ret, "returnObj", "backups")
 	if err != nil {
-		return nil, errors.Wrap(err, "SRegion.GetSnapshots.Unmarshal")
+		return nil, errors.Wrap(err, "SRegion.GetDiskBackups.Unmarshal")
 	}
 
 	for i := range ret {
@@ -164,7 +159,7 @@ func (self *SRegion) GetSnapshots(diskId string) ([]SSnapshot, error) {
 	return ret, nil
 }
 
-func (self *SRegion) DeleteSnapshot(vbsId string) (string, error) {
+func (self *SRegion) DeleteDiskBackup(vbsId string) (string, error) {
 	params := map[string]jsonutils.JSONObject{
 		"regionId": jsonutils.NewString(self.GetId()),
 		"vbsId":    jsonutils.NewString(vbsId),
@@ -172,20 +167,20 @@ func (self *SRegion) DeleteSnapshot(vbsId string) (string, error) {
 
 	resp, err := self.client.DoPost("/apiproxy/v3/ondemand/deleteVBS", params)
 	if err != nil {
-		return "", errors.Wrap(err, "SRegion.DeleteSnapshot.DoPost")
+		return "", errors.Wrap(err, "SRegion.DeleteDiskBackup.DoPost")
 	}
 
 	var ok bool
 	err = resp.Unmarshal(&ok, "returnObj", "status")
 	if !ok {
 		msg, _ := resp.GetString("message")
-		return "", fmt.Errorf("SRegion.DeleteSnapshot.JobFailed %s", msg)
+		return "", fmt.Errorf("SRegion.DeleteDiskBackup.JobFailed %s", msg)
 	}
 
 	var jobId string
 	err = resp.Unmarshal(&jobId, "returnObj", "data")
 	if err != nil {
-		return "", errors.Wrap(err, "SRegion.DeleteSnapshot.Unmarshal")
+		return "", errors.Wrap(err, "SRegion.DeleteDiskBackup.Unmarshal")
 	}
 
 	return jobId, nil

pkg/multicloud/ctyun/instance.go

@@ -189,6 +189,14 @@ func (self *SInstance) GetMetadata() *jsonutils.JSONDict {
 	priceKey := fmt.Sprintf("%s::%s::%s", self.host.zone.region.GetId(), self.GetInstanceType(), lowerOs)
 	data.Add(jsonutils.NewString(priceKey), "price_key")
 	data.Add(jsonutils.NewString(self.host.zone.GetGlobalId()), "zone_ext_id")
+
+	image, _ := self.GetImage()
+	if image != nil {
+		if meta := image.GetMetadata(); meta != nil {
+			data.Update(meta)
+		}
+	}
+
 	return data
 }
 
@@ -898,6 +906,27 @@ func (self *SRegion) GetVbsJob(jobId string) (jsonutils.JSONObject, error) {
 	return ret, nil
 }
 
+// 查询云硬盘JOB状态信息
+func (self *SRegion) GetVolumeJob(jobId string) (jsonutils.JSONObject, error) {
+	params := map[string]string{
+		"regionId": self.GetId(),
+		"jobId":    jobId,
+	}
+
+	resp, err := self.client.DoGet("/apiproxy/v3/queryVolumeJob", params)
+	if err != nil {
+		return nil, errors.Wrap(err, "SRegion.GetVolumeJob.DoGet")
+	}
+
+	ret := jsonutils.NewDict()
+	err = resp.Unmarshal(&ret, "returnObj")
+	if err != nil {
+		return nil, errors.Wrap(err, "SRegion.GetVolumeJob.Unmarshal")
+	}
+
+	return ret, nil
+}
+
 // POST http://ctyun-api-url/apiproxy/v3/addSecurityGroup 绑定安全组
 func (self *SRegion) AssignSecurityGroup(vmId, securityGroupRuleId string) error {
 	securityParams := jsonutils.NewDict()

pkg/multicloud/ctyun/region.go

@@ -330,32 +330,11 @@ func (self *SRegion) CreateEIP(eip *cloudprovider.SEip) (cloudprovider.ICloudEIP
 }
 
 func (self *SRegion) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) {
-	snapshots, err := self.GetSnapshots("")
-	if err != nil {
-		return nil, errors.Wrap(err, "SRegion.GetISnapshots.GetSnapshots")
-	}
-
-	isnapshots := make([]cloudprovider.ICloudSnapshot, len(snapshots))
-	for i := range snapshots {
-		isnapshots[i] = &snapshots[i]
-	}
-
-	return isnapshots, nil
+	return []cloudprovider.ICloudSnapshot{}, nil
 }
 
 func (self *SRegion) GetISnapshotById(snapshotId string) (cloudprovider.ICloudSnapshot, error) {
-	snapshots, err := self.GetSnapshots("")
-	if err != nil {
-		return nil, errors.Wrap(err, "SRegion.GetISnapshotById.GetSnapshots")
-	}
-
-	for i := range snapshots {
-		if snapshots[i].GetId() == snapshotId {
-			return &snapshots[i], nil
-		}
-	}
-
-	return nil, errors.Wrap(cloudprovider.ErrNotFound, "SRegion.GetISnapshotById")
+	return nil, cloudprovider.ErrNotFound
 }
 
 func (self *SRegion) CreateSnapshotPolicy(*cloudprovider.SnapshotPolicyInput) (string, error) {

pkg/multicloud/ctyun/securitygroup.go

@@ -105,10 +105,11 @@ func (self *SSecurityGroup) GetRulesWithExtId() ([]secrules.SecurityRule, error)
 	return rules, nil
 }
 
-func (self *SRegion) syncSecgroupRules(secgroupId string, rules []secrules.SecurityRule) error {
+func (self *SRegion) syncSecgroupRules(secgroupId string, srules []secrules.SecurityRule) error {
 	var DeleteRules []secrules.SecurityRule
 	var AddRules []secrules.SecurityRule
 
+	rules := SecurityRuleSetToAllowSet(srules)
 	if secgroup, err := self.GetSecurityGroupDetails(secgroupId); err != nil {
 		return errors.Wrapf(err, "syncSecgroupRules.GetSecurityGroupDetails(%s)", secgroupId)
 	} else {

pkg/multicloud/ctyun/shell/disk.go

@@ -15,6 +15,8 @@
 package shell
 
 import (
+	"context"
+
 	"yunion.io/x/onecloud/pkg/multicloud/ctyun"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
 )
@@ -45,4 +47,40 @@ func init() {
 		printObject(disk)
 		return nil
 	})
+
+	type DiskResizeOptions struct {
+		DiskId string `help:"disk id"`
+		Size   int64  `help:"disk size GB"`
+	}
+	shellutils.R(&DiskResizeOptions{}, "disk-resize", "Resize disk", func(cli *ctyun.SRegion, args *DiskResizeOptions) error {
+		disk, err := cli.GetDisk(args.DiskId)
+		if err != nil {
+			return err
+		}
+
+		e := disk.Resize(context.Background(), args.Size*1024)
+		if e != nil {
+			return e
+		}
+		printObject(disk)
+		return nil
+	})
+
+	type VDiskRestoreOptions struct {
+		DiskId     string `help:"disk id"`
+		SnapshotId string `help:"snapshot id"`
+	}
+	shellutils.R(&VDiskRestoreOptions{}, "disk-restore", "Restore disk", func(cli *ctyun.SRegion, args *VDiskRestoreOptions) error {
+		disk, err := cli.GetDisk(args.DiskId)
+		if err != nil {
+			return err
+		}
+
+		_, e := disk.Reset(context.Background(), args.SnapshotId)
+		if e != nil {
+			return e
+		}
+		printObject(disk)
+		return nil
+	})
 }

pkg/multicloud/ctyun/shell/job.go

@@ -32,7 +32,7 @@ func init() {
 		return nil
 	})
 
-	shellutils.R(&VJobShowOptions{}, "vbs-job-show", "Show job", func(cli *ctyun.SRegion, args *VJobShowOptions) error {
+	shellutils.R(&VJobShowOptions{}, "vbs-job-show", "Show vbs job", func(cli *ctyun.SRegion, args *VJobShowOptions) error {
 		job, e := cli.GetVbsJob(args.JOBID)
 		if e != nil {
 			return e
@@ -40,4 +40,13 @@ func init() {
 		printObject(job)
 		return nil
 	})
+
+	shellutils.R(&VJobShowOptions{}, "volume-job-show", "Show volume job", func(cli *ctyun.SRegion, args *VJobShowOptions) error {
+		job, e := cli.GetVolumeJob(args.JOBID)
+		if e != nil {
+			return e
+		}
+		printObject(job)
+		return nil
+	})
 }

pkg/multicloud/ctyun/storage.go

@@ -123,7 +123,7 @@ func (self *SStorage) GetEnabled() bool {
 }
 
 func (self *SStorage) CreateIDisk(name string, sizeGb int, desc string) (cloudprovider.ICloudDisk, error) {
-	disk, err := self.zone.region.CreateDisk(self.zone.GetId(), self.GetName(), self.GetStorageType(), strconv.Itoa(sizeGb))
+	disk, err := self.zone.region.CreateDisk(self.zone.GetId(), name, self.GetStorageType(), strconv.Itoa(sizeGb))
 	if err != nil {
 		return nil, errors.Wrap(err, "Storage.CreateIDisk.CreateDisk")
 	}

pkg/multicloud/huawei/region.go

@@ -734,10 +734,11 @@ func (self *SRegion) CreateSecurityGroup(vpcId string, name string, desc string)
 	return &secgroup, err
 }
 
-func (self *SRegion) syncSecgroupRules(secgroupId string, rules []secrules.SecurityRule) error {
+func (self *SRegion) syncSecgroupRules(secgroupId string, srules []secrules.SecurityRule) error {
 	var DeleteRules []secrules.SecurityRule
 	var AddRules []secrules.SecurityRule
 
+	rules := SecurityRuleSetToAllowSet(srules)
 	if secgroup, err := self.GetSecurityGroupDetails(secgroupId); err != nil {
 		return errors.Wrapf(err, "syncSecgroupRules.GetSecurityGroupDetails(%s)", secgroupId)
 	} else {