Automatic merge from release/2.4.0 -> release/2.5.0

* commit '00d8dba7bda1b63260d1fde8036807463e9ad118':
  限定只允许创建RSA格式密钥
  disk rollback bugfix
  keypair bugfix

pkg/cloudprovider/resources.go

@@ -304,7 +304,7 @@ type ICloudDisk interface {
 	GetISnapshots() ([]ICloudSnapshot, error)
 
 	Resize(ctx context.Context, newSizeMB int64) error
-	Reset(ctx context.Context, snapshotId string) error
+	Reset(ctx context.Context, snapshotId string) (string, error)
 
 	Rebuild(ctx context.Context) error
 }

pkg/compute/hostdrivers/managedvirtual.go

@@ -282,8 +282,10 @@ func (self *SManagedVirtualizationHostDriver) RequestResetDisk(ctx context.Conte
 		return err
 	}
 	taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
-		err := iDisk.Reset(ctx, snapshotId)
-		return nil, err
+		exteranlDiskId, err := iDisk.Reset(ctx, snapshotId)
+		data := jsonutils.NewDict()
+		data.Set("exteranl_disk_id", jsonutils.NewString(exteranlDiskId))
+		return data, err
 	})
 	return nil
 }

pkg/compute/models/keypairs.go

@@ -142,8 +142,15 @@ func (manager *SKeypairManager) ValidateCreateData(ctx context.Context, userCred
 		log.Errorf("invalid public key %s", err)
 		return nil, httperrors.NewInputParameterError("invalid public")
 	}
+
+	// 只允许上传RSA格式密钥。PS: AWS只支持RSA格式。
+	scheme := seclib2.GetPublicKeyScheme(pubKey)
+	if scheme != "RSA" {
+		return nil, httperrors.NewInputParameterError("Unsupported scheme %s", scheme)
+	}
+
 	data.Set("fingerprint", jsonutils.NewString(ssh.FingerprintLegacyMD5(pubKey)))
-	data.Set("scheme", jsonutils.NewString(seclib2.GetPublicKeyScheme(pubKey)))
+	data.Set("scheme", jsonutils.NewString(scheme))
 	data.Set("owner_id", jsonutils.NewString(userCred.GetUserId()))
 
 	return manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)

pkg/compute/tasks/disk_reset_task.go

@@ -28,6 +28,10 @@ func (self *DiskResetTask) TaskFailed(ctx context.Context, disk *models.SDisk, r
 }
 
 func (self *DiskResetTask) TaskCompleted(ctx context.Context, disk *models.SDisk, data *jsonutils.JSONDict) {
+	// data不能为空指针，否则会导致AddActionLog抛空指针异常
+	if data == nil {
+		data = jsonutils.NewDict()
+	}
 	logclient.AddActionLog(disk, logclient.ACT_RESET_DISK, data, self.UserCred, true)
 	self.SetStageComplete(ctx, data)
 }
@@ -81,9 +85,12 @@ func (self *DiskResetTask) OnRequestResetDisk(ctx context.Context, disk *models.
 	snapshotId, _ := self.Params.GetString("snapshot_id")
 	iSnapshot, _ := models.SnapshotManager.FetchById(snapshotId)
 	snapshot := iSnapshot.(*models.SSnapshot)
-	if disk.DiskSize != snapshot.Size {
+
+	externalId, _ := data.GetString("exteranl_disk_id")
+	if disk.DiskSize != snapshot.Size || (len(externalId) > 0 && externalId != disk.GetExternalId() ){
 		_, err := models.DiskManager.TableSpec().Update(disk, func() error {
 			disk.DiskSize = snapshot.Size
+			disk.ExternalId = externalId
 			return nil
 		})
 		if err != nil {

pkg/util/aliyun/disk.go

@@ -368,8 +368,8 @@ func (self *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) {
 	return isnapshots, nil
 }
 
-func (self *SDisk) Reset(ctx context.Context, snapshotId string) error {
-	return self.storage.zone.region.resetDisk(self.DiskId, snapshotId)
+func (self *SDisk) Reset(ctx context.Context, snapshotId string) (string, error) {
+	return "", self.storage.zone.region.resetDisk(self.DiskId, snapshotId)
 }
 
 func (self *SDisk) GetBillingType() string {

pkg/util/aws/disk.go

@@ -223,7 +223,7 @@ func (self *SDisk) Resize(ctx context.Context, newSizeMb int64) error {
 	return self.storage.zone.region.resizeDisk(self.DiskId, newSizeMb)
 }
 
-func (self *SDisk) Reset(ctx context.Context, snapshotId string) error {
+func (self *SDisk) Reset(ctx context.Context, snapshotId string) (string, error) {
 	return self.storage.zone.region.resetDisk(self.DiskId, snapshotId)
 }
 
@@ -395,12 +395,12 @@ func (self *SRegion) resizeDisk(diskId string, sizeMb int64) error {
 	return err
 }
 
-func (self *SRegion) resetDisk(diskId, snapshotId string) error {
+func (self *SRegion) resetDisk(diskId, snapshotId string) (string, error) {
 	// 这里实际是回滚快照
 	disk, err := self.GetDisk(diskId)
 	if err != nil {
 		log.Debugf("resetDisk %s:%s", diskId, err.Error())
-		return err
+		return "", err
 	}
 
 	params := &ec2.CreateVolumeInput{}
@@ -415,7 +415,7 @@ func (self *SRegion) resetDisk(diskId, snapshotId string) error {
 	ret, err := self.ec2Client.CreateVolume(params)
 	if err != nil {
 		log.Debugf("resetDisk %s: %s", params.String(), err.Error())
-		return err
+		return "", err
 	}
 
 	// detach disk
@@ -423,24 +423,24 @@ func (self *SRegion) resetDisk(diskId, snapshotId string) error {
 		err := self.DetachDisk(disk.InstanceId, diskId)
 		if err != nil {
 			log.Debugf("resetDisk %s %s: %s", disk.InstanceId, diskId, err.Error())
-			return err
+			return "", err
 		}
 
 		err = self.ec2Client.WaitUntilVolumeAvailable(&ec2.DescribeVolumesInput{VolumeIds: []*string{&diskId}})
 		if err != nil {
 			log.Debugf("resetDisk :%s", err.Error())
-			return err
+			return "", err
 		}
 	}
 
 	err = self.AttachDisk(disk.InstanceId, *ret.VolumeId, disk.Device)
 	if err != nil {
 		log.Debugf("resetDisk %s %s %s: %s", disk.InstanceId, *ret.VolumeId, disk.Device, err.Error())
-		return err
+		return "", err
 	}
 
 	// 绑定成功后删除原磁盘
-	return self.DeleteDisk(diskId)
+	return StrVal(ret.VolumeId), self.DeleteDisk(diskId)
 }
 
 func (self *SRegion) CreateDisk(zoneId string, category string, name string, sizeGb int, snapshotId string, desc string) (string, error) {
@@ -477,5 +477,6 @@ func (disk *SDisk) GetAccessPath() string {
 }
 
 func (self *SDisk) Rebuild(ctx context.Context) error {
-	return self.storage.zone.region.resetDisk(self.DiskId, "")
+	_, err := self.storage.zone.region.resetDisk(self.DiskId, "")
+	return err
 }

pkg/util/aws/image.go

@@ -201,7 +201,7 @@ func (self *SRegion) GetImageByName(name string) (*SImage, error) {
 		return nil, cloudprovider.ErrNotFound
 	}
 
-	log.Debugf("%d image found match name %", len(images), name)
+	log.Debugf("%d image found match name %s", len(images), name)
 	return &images[0], nil
 }
 

pkg/util/aws/instance.go

@@ -669,7 +669,7 @@ func (self *SRegion) CreateInstance(name string, imageId string, instanceType st
 	if len(res.Instances) == 1 {
 		return *res.Instances[0].InstanceId, nil
 	} else {
-		msg := fmt.Sprintf("CreateInstance fail: %s instance created. ", len(res.Instances))
+		msg := fmt.Sprintf("CreateInstance fail: %d instance created. ", len(res.Instances))
 		log.Errorf(msg)
 		return "", fmt.Errorf(msg)
 	}
@@ -810,7 +810,7 @@ func (self *SRegion) ReplaceSystemDisk(ctx context.Context, instanceId string, i
 	if rootDisk == nil {
 		return "", fmt.Errorf("can not find root disk of instance %s", instanceId)
 	}
-	log.Debugf("ReplaceSystemDisk replace root disk %s", rootDisk)
+	log.Debugf("ReplaceSystemDisk replace root disk %s", rootDisk.DiskId)
 
 	image, err := self.GetImage(imageId)
 	if err != nil {

pkg/util/aws/keypair.go

@@ -1,6 +1,10 @@
 package aws
 
 import (
+	"bytes"
+	"crypto/md5"
+	"crypto/rsa"
+	"crypto/x509"
 	"fmt"
 	"strconv"
 	"time"
@@ -17,8 +21,61 @@ type SKeypair struct {
 	KeyPairName        string
 }
 
+// 只支持计算Openssh ras 格式公钥转换成DER格式后的MD5。
+func md5Fingerprint(publickey string) (string, error) {
+	pk, _, _, _, err := ssh.ParseAuthorizedKey([]byte(publickey))
+	if err != nil {
+		return "", fmt.Errorf("publicKey error %s", err)
+	}
+
+	der := []byte{}
+	cryptoPub, ok := pk.(ssh.CryptoPublicKey)
+	if !ok {
+		return "", fmt.Errorf("public key trans to crypto public key failed")
+	}
+
+	switch pk.Type() {
+	case ssh.KeyAlgoRSA:
+		rsaPK, ok := cryptoPub.CryptoPublicKey().(*rsa.PublicKey)
+		if !ok {
+			return "", fmt.Errorf("crypto public key trans to ras publickey failed")
+		}
+		der, err = x509.MarshalPKIXPublicKey(rsaPK)
+		if err != nil {
+			return "", fmt.Errorf("MarshalPKIXPublicKey ras publickey failed")
+		}
+	default:
+		return "", fmt.Errorf("unsupport public key format.Only ssh-rsa supported")
+	}
+
+	var ret bytes.Buffer
+	fp := md5.Sum(der)
+	for i, b := range fp {
+		ret.WriteString(fmt.Sprintf("%02x", b))
+		if i < len(fp)-1 {
+			ret.WriteString(":")
+		}
+	}
+
+	return ret.String(), nil
+}
+
 func (self *SRegion) GetKeypairs(finger string, name string, offset int, limit int) ([]SKeypair, int, error) {
-	ret, err := self.ec2Client.DescribeKeyPairs(&ec2.DescribeKeyPairsInput{})
+	params := &ec2.DescribeKeyPairsInput{}
+	filters := []*ec2.Filter{}
+	if len(finger) > 0 {
+		filters = AppendSingleValueFilter(filters, "fingerprint", finger)
+	}
+
+	if len(name) > 0 {
+		params.SetKeyNames([]*string{&name})
+	}
+
+	if len(filters) > 0 {
+		params.SetFilters(filters)
+	}
+
+	ret, err := self.ec2Client.DescribeKeyPairs(params)
 	if err != nil {
 		return nil, 0, err
 	}
@@ -35,6 +92,7 @@ func (self *SRegion) GetKeypairs(finger string, name string, offset int, limit i
 	return keypairs, len(keypairs), nil
 }
 
+// Aws貌似不支持ssh-dss格式密钥
 func (self *SRegion) ImportKeypair(name string, pubKey string) (*SKeypair, error) {
 	params := &ec2.ImportKeyPairInput{}
 	params.SetKeyName(name)
@@ -56,13 +114,13 @@ func (self *SRegion) DetachKeyPair(instanceId string, keypairName string) error
 }
 
 func (self *SRegion) lookUpAwsKeypair(publicKey string) (string, error) {
-	pk, _, _, _, err := ssh.ParseAuthorizedKey([]byte(publicKey))
+	// https://docs.amazonaws.cn/AWSEC2/latest/UserGuide/ec2-key-pairs.html
+	fingerprint, err := md5Fingerprint(publicKey)
 	if err != nil {
-		return "", fmt.Errorf("publicKey error %s", err)
+		return "", err
 	}
 
-	fingerprint := ssh.FingerprintLegacyMD5(pk)
-	ks, total, err := self.GetKeypairs(fingerprint, "*", 0, 1)
+	ks, total, err := self.GetKeypairs(fingerprint, "", 0, 1)
 	if total < 1 {
 		return "", fmt.Errorf("keypair not found %s", err)
 	} else {

pkg/util/aws/keypair_test.go

@@ -0,0 +1,30 @@
+package aws
+
+import "testing"
+
+type testPublicKey struct {
+	publickey string
+	fingerprint string
+}
+
+func TestMd5Fingerprint(t *testing.T) {
+	rsa := testPublicKey{
+		"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCBBuv9nsAGNpKVulxNc7zHXEEyiTqYU8J6sTfmB9lmrRea/RO/pUJg1ZGlHKLSbZ5h+d4mquASf8K3s3SQtz/4sBHroRijanO16i0Rk6t5kwcIRzaf11NiImImKgwNiCwZyiK2egAfsjDVEi8H+kSRA0N0PMxRfwOEZ/hNtVaNV7/MwkXylOuWUikGvPpm3sRmelfQoS3Hf055WM1m6POgddbjucq9bjQDW1O4dfDkWuX+385EOtfCBPtfeiAcOBBd+qEjmdfxroQwxHXLkZH7rdoS9jss3fi9P/K0ZpBKswKsed2sxKo9NNYfTDN19Kv8NBOW8W7MxN1po/2gvbd/",
+		"4c:ae:76:94:fb:59:66:8c:a6:07:e2:54:2f:14:19:c5",
+	}
+
+	testKeys := []testPublicKey{rsa}
+
+	for _, k := range testKeys {
+		fingerprint, err := md5Fingerprint(k.publickey)
+		if err != nil {
+			t.Errorf(err.Error())
+			continue
+		}
+
+		if fingerprint !=  k.fingerprint {
+			t.Errorf("ssh-rsa fingerprint is not as expected.%s != %s", fingerprint, k.fingerprint)
+			continue
+		}
+     }
+}

pkg/util/aws/securitygroup.go

@@ -345,9 +345,6 @@ func (self *SRegion) syncSecgroupRules(secgroupId string, rules []secrules.Secur
 		sort.Sort(secrules.SecurityRuleSet(rules))
 		sort.Sort(secrules.SecurityRuleSet(secgroup.Permissions))
 
-		log.Debugf("local security rules %s", rules)
-		log.Debugf("remote security rules %s", secgroup.Permissions)
-
 		i, j := 0, 0
 		for i < len(rules) || j < len(secgroup.Permissions) {
 			if i < len(rules) && j < len(secgroup.Permissions) {

pkg/util/azure/classic_disk.go

@@ -208,8 +208,8 @@ func (self *SClassicDisk) Refresh() error {
 	return nil
 }
 
-func (self *SClassicDisk) Reset(ctx context.Context, snapshotId string) error {
-	return cloudprovider.ErrNotSupported
+func (self *SClassicDisk) Reset(ctx context.Context, snapshotId string) (string, error) {
+	return "", cloudprovider.ErrNotSupported
 }
 
 func (self *SClassicDisk) Resize(ctx context.Context, sizeMb int64) error {

pkg/util/azure/disk.go

@@ -315,8 +315,8 @@ func (region *SRegion) GetSnapShots(diskId string) ([]SSnapshot, error) {
 	return result, nil
 }
 
-func (self *SDisk) Reset(ctx context.Context, snapshotId string) error {
-	return self.storage.zone.region.resetDisk(self.ID, snapshotId)
+func (self *SDisk) Reset(ctx context.Context, snapshotId string) (string, error) {
+	return "", self.storage.zone.region.resetDisk(self.ID, snapshotId)
 }
 
 func (self *SRegion) resetDisk(diskId, snapshotId string) error {

pkg/util/esxi/vdisk.go

@@ -229,8 +229,8 @@ func (disk *SVirtualDisk) Resize(ctx context.Context, newSizeMb int64) error {
 	return err
 }
 
-func (disk *SVirtualDisk) Reset(ctx context.Context, snapshotId string) error {
-	return cloudprovider.ErrNotImplemented
+func (disk *SVirtualDisk) Reset(ctx context.Context, snapshotId string) (string, error) {
+	return "", cloudprovider.ErrNotImplemented
 }
 
 func (disk *SVirtualDisk) GetBillingType() string {

pkg/util/qcloud/disk.go

@@ -344,8 +344,8 @@ func (self *SRegion) ResetDisk(diskId, snapshotId string) error {
 	return nil
 }
 
-func (self *SDisk) Reset(ctx context.Context, snapshotId string) error {
-	return self.storage.zone.region.ResetDisk(self.DiskId, snapshotId)
+func (self *SDisk) Reset(ctx context.Context, snapshotId string) (string, error) {
+	return "", self.storage.zone.region.ResetDisk(self.DiskId, snapshotId)
 }
 
 func (self *SRegion) CreateDisk(zoneId string, category string, name string, sizeGb int, desc string) (string, error) {

pkg/util/qcloud/localdisk.go

@@ -87,8 +87,8 @@ func (self *SLocalDisk) Refresh() error {
 	return nil
 }
 
-func (self *SLocalDisk) Reset(ctx context.Context, snapshotId string) error {
-	return cloudprovider.ErrNotSupported
+func (self *SLocalDisk) Reset(ctx context.Context, snapshotId string) (string, error) {
+	return "", cloudprovider.ErrNotSupported
 }
 
 func (self *SLocalDisk) GetTemplateId() string {