From 5631f5ed68cb4bf08f6d67b6f49e20b214f2affb Mon Sep 17 00:00:00 2001
From: TangBin <tangbin@yunionyun.com>
Date: Mon, 12 Nov 2018 16:05:31 +0800
Subject: [PATCH 01/57] disk create fix

---
 pkg/util/aws/disk.go     |  6 ++++++
 pkg/util/aws/instance.go | 10 ++++++++--
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/pkg/util/aws/disk.go b/pkg/util/aws/disk.go
index edef37c559..2b3535f6ad 100644
--- a/pkg/util/aws/disk.go
+++ b/pkg/util/aws/disk.go
@@ -389,5 +389,11 @@ func (self *SRegion) CreateDisk(zoneId string, category string, name string, siz
 		return "", err
 	}
 
+	paramsWait := &ec2.DescribeVolumesInput{}
+	paramsWait.SetVolumeIds([]*string{ret.VolumeId})
+	err = self.ec2Client.WaitUntilVolumeAvailable(paramsWait)
+	if err != nil {
+		return "", err
+	}
 	return StrVal(ret.VolumeId), nil
 }
diff --git a/pkg/util/aws/instance.go b/pkg/util/aws/instance.go
index 4deb29516e..13748b02fd 100644
--- a/pkg/util/aws/instance.go
+++ b/pkg/util/aws/instance.go
@@ -374,12 +374,18 @@ func (self *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) {
 }
 
 func (self *SInstance) AttachDisk(diskId string) error {
-	// todo：bugfix . self.DeviceNames => self.GetDeviceNames()
 	name, err := NextDeviceName(self.DeviceNames)
 	if err != nil {
 		return err
 	}
-	return self.host.zone.region.AttachDisk(self.InstanceId, diskId, name)
+
+	err = self.host.zone.region.AttachDisk(self.InstanceId, diskId, name)
+	if err != nil {
+		return err
+	}
+
+	self.DeviceNames = append(self.DeviceNames, name)
+	return nil
 }
 
 func (self *SInstance) DetachDisk(diskId string) error {

From 6ec78519cb95235d87ebb766912763af3186cd5b Mon Sep 17 00:00:00 2001
From: TangBin <tangbin@yunionyun.com>
Date: Mon, 12 Nov 2018 17:09:44 +0800
Subject: [PATCH 02/57] detach disk add more log

---
 pkg/util/aws/disk.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pkg/util/aws/disk.go b/pkg/util/aws/disk.go
index 2b3535f6ad..a1a304162c 100644
--- a/pkg/util/aws/disk.go
+++ b/pkg/util/aws/disk.go
@@ -338,6 +338,7 @@ func (self *SRegion) DeleteDisk(diskId string) error {
 	}
 
 	params.SetVolumeId(diskId)
+	log.Debugf("DeleteDisk with params: %s", params.String())
 	_, err = self.ec2Client.DeleteVolume(params)
 	return err
 }

From c8f2510fcba5622a1f1d45b87cc31c32359203f3 Mon Sep 17 00:00:00 2001
From: TangBin <tangbin@yunionyun.com>
Date: Mon, 12 Nov 2018 18:11:24 +0800
Subject: [PATCH 03/57] snapshot fix

---
 pkg/util/aws/snapshot.go | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/pkg/util/aws/snapshot.go b/pkg/util/aws/snapshot.go
index b7a2a9241b..c1a13fc38d 100644
--- a/pkg/util/aws/snapshot.go
+++ b/pkg/util/aws/snapshot.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
 )
@@ -11,9 +12,9 @@ import (
 type SnapshotStatusType string
 
 const (
-	SnapshotStatusAccomplished SnapshotStatusType = "accomplished"
-	SnapshotStatusProgress     SnapshotStatusType = "progressing"
-	SnapshotStatusFailed       SnapshotStatusType = "failed"
+	SnapshotStatusAccomplished SnapshotStatusType = "completed"
+	SnapshotStatusProgress     SnapshotStatusType = "pending"
+	SnapshotStatusFailed       SnapshotStatusType = "error"
 )
 
 type SSnapshot struct {
@@ -176,8 +177,9 @@ func (self *SRegion) CreateSnapshot(diskId, name, desc string) (string, error) {
 	}
 
 	params.SetDescription(desc)
-	_, err := self.ec2Client.CreateSnapshot(params)
-	return "", err
+	log.Debugf("CreateSnapshots with params %s", params)
+	ret, err := self.ec2Client.CreateSnapshot(params)
+	return StrVal(ret.SnapshotId), err
 }
 
 func (self *SRegion) DeleteSnapshot(snapshotId string) error {

From 826717129b0108667085e5879b121802392296a6 Mon Sep 17 00:00:00 2001
From: TangBin <tangbin@yunionyun.com>
Date: Tue, 13 Nov 2018 15:57:05 +0800
Subject: [PATCH 04/57] add more log

---
 pkg/compute/models/cloudaccounts.go  | 1 +
 pkg/compute/models/cloudproviders.go | 2 +-
 pkg/util/aws/aws.go                  | 3 ++-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go
index b6c7672bf4..0c40545812 100644
--- a/pkg/compute/models/cloudaccounts.go
+++ b/pkg/compute/models/cloudaccounts.go
@@ -113,6 +113,7 @@ func (manager *SCloudaccountManager) ValidateCreateData(ctx context.Context, use
 		if err == cloudprovider.ErrNoSuchProvder {
 			return nil, httperrors.NewResourceNotFoundError("no such provider %s", provider)
 		}
+		log.Debugf("ValidateCreateData %s", err.Error())
 		return nil, httperrors.NewInvalidCredentialError("invalid cloud account info")
 	}
 
diff --git a/pkg/compute/models/cloudproviders.go b/pkg/compute/models/cloudproviders.go
index 5b25fb1cbb..fc8607ce7e 100644
--- a/pkg/compute/models/cloudproviders.go
+++ b/pkg/compute/models/cloudproviders.go
@@ -429,7 +429,7 @@ func (self *SCloudprovider) SaveSysInfo(info jsonutils.JSONObject) {
 func (manager *SCloudproviderManager) FetchCloudproviderById(providerId string) *SCloudprovider {
 	providerObj, err := manager.FetchById(providerId)
 	if err != nil {
-		log.Errorf("%s", err)
+		log.Errorf("fetch cloud provider %s: %s", providerId, err)
 		return nil
 	}
 	return providerObj.(*SCloudprovider)
diff --git a/pkg/util/aws/aws.go b/pkg/util/aws/aws.go
index 684e44281c..cbdfc5bf06 100644
--- a/pkg/util/aws/aws.go
+++ b/pkg/util/aws/aws.go
@@ -1,7 +1,7 @@
 package aws
 
 import (
-	"github.com/coredns/coredns/plugin/pkg/log"
+	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
 
@@ -33,6 +33,7 @@ func NewAwsClient(providerId string, providerName string, accessUrl string, acce
 	client := SAwsClient{providerId: providerId, providerName: providerName, accessUrl: accessUrl, accessKey: accessKey, secret: secret}
 	err := client.fetchRegions()
 	if err != nil {
+		log.Debugf("NewAwsClient %s", err.Error())
 		return nil, err
 	}
 	return &client, nil

From 538ee71202fe5723b5f878d20ed329690a95aa95 Mon Sep 17 00:00:00 2001
From: TangBin <tangbin@yunionyun.com>
Date: Tue, 13 Nov 2018 17:45:53 +0800
Subject: [PATCH 05/57] snapshot delete & disk reset

---
 pkg/compute/hostdrivers/aws.go | 16 ++++++++++++
 pkg/util/aws/disk.go           | 47 +++++++++++++++++++++++++++++++---
 pkg/util/aws/snapshot.go       |  7 ++++-
 3 files changed, 66 insertions(+), 4 deletions(-)

diff --git a/pkg/compute/hostdrivers/aws.go b/pkg/compute/hostdrivers/aws.go
index b4022ab211..255282bcc7 100644
--- a/pkg/compute/hostdrivers/aws.go
+++ b/pkg/compute/hostdrivers/aws.go
@@ -188,6 +188,22 @@ func (self *SAwsHostDriver) RequestResizeDiskOnHost(host *models.SHost, storage
 	return nil
 }
 
+func (self *SAwsHostDriver) RequestResetDisk(ctx context.Context, host *models.SHost, disk *models.SDisk, params *jsonutils.JSONDict, task taskman.ITask) error {
+	iDisk, err := disk.GetIDisk()
+	if err != nil {
+		return err
+	}
+	snapshotId, err := params.GetString("snapshot_id")
+	if err != nil {
+		return err
+	}
+	taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
+		err := iDisk.Reset(snapshotId)
+		return nil, err
+	})
+	return nil
+}
+
 func init() {
 	driver := SAwsHostDriver{}
 	models.RegisterHostDriver(&driver)
diff --git a/pkg/util/aws/disk.go b/pkg/util/aws/disk.go
index a1a304162c..02e5c23ae5 100644
--- a/pkg/util/aws/disk.go
+++ b/pkg/util/aws/disk.go
@@ -27,7 +27,7 @@ type SDisk struct {
 	DiskId   string // VolumeId
 
 	DiskName         string // Tag Name
-	Size             int    // Size
+	Size             int    // Size GB
 	Category         string // VolumeType
 	Type             string // system | data
 	Status           string // State
@@ -366,8 +366,49 @@ func (self *SRegion) resizeDisk(diskId string, size int64) error {
 }
 
 func (self *SRegion) resetDisk(diskId, snapshotId string) error {
-	// aws貌似不支持直接重置
-	return cloudprovider.ErrNotImplemented
+	// 这里实际是回滚快照
+	disk, err := self.GetDisk(diskId)
+	if err != nil {
+		log.Debugf("resetDisk %s:%s",diskId, err.Error())
+		return err
+	}
+
+	params := &ec2.CreateVolumeInput{}
+	params.SetSnapshotId(snapshotId)
+	params.SetSize(int64(disk.Size))
+	params.SetVolumeType(disk.Category)
+	params.SetAvailabilityZone(disk.ZoneId)
+	tags,_ := disk.Tags.GetTagSpecifications()
+	params.SetTagSpecifications([]*ec2.TagSpecification{tags})
+	ret, err := self.ec2Client.CreateVolume(params)
+	if err != nil {
+		log.Debugf("resetDisk %s: %s",params.String(), err.Error())
+		return err
+	}
+
+	// detach disk
+	if disk.Status == ec2.VolumeStateInUse {
+		err := self.DetachDisk(disk.InstanceId, diskId)
+		if err != nil {
+			log.Debugf("resetDisk %s %s: %s", disk.InstanceId, diskId, err.Error())
+			return err
+		}
+
+		err = self.ec2Client.WaitUntilVolumeAvailable(&ec2.DescribeVolumesInput{VolumeIds:[]*string{&diskId}})
+		if err != nil {
+			log.Debugf("resetDisk :%s", err.Error())
+			return err
+		}
+	}
+
+	err = self.AttachDisk(disk.InstanceId, *ret.VolumeId, disk.Device)
+	if err != nil {
+		log.Debugf("resetDisk %s %s %s: %s",disk.InstanceId, *ret.VolumeId, disk.Device, err.Error())
+		return err
+	}
+
+	// 绑定成功后删除原磁盘
+	return self.DeleteDisk(diskId)
 }
 
 func (self *SRegion) CreateDisk(zoneId string, category string, name string, sizeGb int, snapshotId string, desc string) (string, error) {
diff --git a/pkg/util/aws/snapshot.go b/pkg/util/aws/snapshot.go
index c1a13fc38d..506044282b 100644
--- a/pkg/util/aws/snapshot.go
+++ b/pkg/util/aws/snapshot.go
@@ -3,6 +3,7 @@ package aws
 import (
 	"fmt"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"strings"
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -90,7 +91,7 @@ func (self *SSnapshot) GetDiskId() string {
 }
 
 func (self *SSnapshot) Delete() error {
-	panic("implement me")
+	return self.region.DeleteSnapshot(self.SnapshotId)
 }
 
 func (self *SSnapshot) GetRegionId() string {
@@ -125,6 +126,10 @@ func (self *SRegion) GetSnapshots(instanceId string, diskId string, snapshotName
 
 	ret, err := self.ec2Client.DescribeSnapshots(params)
 	if err != nil {
+		if strings.Contains(err.Error(), "InvalidSnapshot.NotFound") {
+			return nil, 0, cloudprovider.ErrNotFound
+		}
+
 		return nil, 0, err
 	}
 

From f62c5fa22ed2dd1395b445af2b76029aa99381e1 Mon Sep 17 00:00:00 2001
From: TangBin <tangbin@yunionyun.com>
Date: Tue, 13 Nov 2018 19:19:04 +0800
Subject: [PATCH 06/57] snapshot region id bugfix

---
 pkg/compute/guestdrivers/aws.go    | 8 +++++++-
 pkg/compute/models/cloudregions.go | 2 +-
 pkg/util/aws/snapshot.go           | 3 ++-
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/pkg/compute/guestdrivers/aws.go b/pkg/compute/guestdrivers/aws.go
index 36fa893b8f..ca3d31a513 100644
--- a/pkg/compute/guestdrivers/aws.go
+++ b/pkg/compute/guestdrivers/aws.go
@@ -308,6 +308,12 @@ func (self *SAwsGuestDriver) RequestDiskSnapshot(ctx context.Context, guest *mod
 	if err != nil {
 		return err
 	}
+
+	zone := disk.GetZone()
+	if zone == nil {
+		return fmt.Errorf("can't not fetch zone by disk %s", disk.Id)
+	}
+
 	iSnapshot, _ := models.SnapshotManager.FetchById(snapshotId)
 	snapshot := iSnapshot.(*models.SSnapshot)
 	taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
@@ -318,7 +324,7 @@ func (self *SAwsGuestDriver) RequestDiskSnapshot(ctx context.Context, guest *mod
 		res := jsonutils.NewDict()
 		res.Set("snapshot_id", jsonutils.NewString(cloudSnapshot.GetId()))
 		res.Set("manager_id", jsonutils.NewString(cloudSnapshot.GetManagerId()))
-		res.Set("cloudregion_id", jsonutils.NewString(cloudSnapshot.GetRegionId()))
+		res.Set("cloudregion_id", jsonutils.NewString(zone.CloudregionId))
 		return res, nil
 	})
 	return nil
diff --git a/pkg/compute/models/cloudregions.go b/pkg/compute/models/cloudregions.go
index bf2fcd1eb3..204678c689 100644
--- a/pkg/compute/models/cloudregions.go
+++ b/pkg/compute/models/cloudregions.go
@@ -288,7 +288,7 @@ func (self *SCloudregion) PerformDefaultVpc(ctx context.Context, userCred mcclie
 func (manager *SCloudregionManager) FetchRegionById(id string) *SCloudregion {
 	obj, err := manager.FetchById(id)
 	if err != nil {
-		log.Errorf("%s", err)
+		log.Errorf("region %s %s", id, err)
 		return nil
 	}
 	return obj.(*SCloudregion)
diff --git a/pkg/util/aws/snapshot.go b/pkg/util/aws/snapshot.go
index 506044282b..259fcdca39 100644
--- a/pkg/util/aws/snapshot.go
+++ b/pkg/util/aws/snapshot.go
@@ -95,7 +95,8 @@ func (self *SSnapshot) Delete() error {
 }
 
 func (self *SSnapshot) GetRegionId() string {
-	return self.region.GetId()
+	// 这里特别注意：aws没有有uuid形式的region id
+	return self.region.Get
 }
 
 func (self *SRegion) GetSnapshots(instanceId string, diskId string, snapshotName string, snapshotIds []string, offset int, limit int) ([]SSnapshot, int, error) {

From 5f5a0e2fb7ba8b3484d15f8f142c9349923bd8b4 Mon Sep 17 00:00:00 2001
From: TangBin <tangbin@yunionyun.com>
Date: Tue, 13 Nov 2018 20:14:06 +0800
Subject: [PATCH 07/57] add more log & fix YunionSecRuleToAws\

---
 pkg/util/aws/securitygroup.go | 10 +++++-----
 pkg/util/aws/utils.go         | 10 +++++++++-
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/pkg/util/aws/securitygroup.go b/pkg/util/aws/securitygroup.go
index bade424345..32d69d9355 100644
--- a/pkg/util/aws/securitygroup.go
+++ b/pkg/util/aws/securitygroup.go
@@ -324,7 +324,7 @@ func (self *SRegion) syncSecgroupRules(secgroupId string, rules []secrules.Secur
 				if cmp == 0 {
 					if secgroup.Permissions[j].Description != rules[i].Description {
 						if err := self.updateSecurityGroupRuleDescription(secgroupId, &rules[i]); err != nil {
-							log.Errorf("updateSecurityGroupRuleDescription error %v", rules[i])
+							log.Errorf("updateSecurityGroupRuleDescription %v error: %s", rules[i], err.Error())
 							return err
 						}
 					}
@@ -332,26 +332,26 @@ func (self *SRegion) syncSecgroupRules(secgroupId string, rules []secrules.Secur
 					j += 1
 				} else if cmp > 0 {
 					if err := self.delSecurityGroupRule(secgroupId, &secgroup.Permissions[j]); err != nil {
-						log.Errorf("delSecurityGroupRule error %v", secgroup.Permissions[j])
+						log.Errorf("delSecurityGroupRule %v error: %s", secgroup.Permissions[j], err.Error())
 						return err
 					}
 					j += 1
 				} else {
 					if err := self.addSecurityGroupRules(secgroupId, &rules[i]); err != nil {
-						log.Errorf("addSecurityGroupRule error %v", rules[i])
+						log.Errorf("addSecurityGroupRule %v error: %s", rules[i], err.Error())
 						return err
 					}
 					i += 1
 				}
 			} else if i >= len(rules) {
 				if err := self.delSecurityGroupRule(secgroupId, &secgroup.Permissions[j]); err != nil {
-					log.Errorf("delSecurityGroupRule error %v", secgroup.Permissions[j])
+					log.Errorf("delSecurityGroupRule %v error: %s", secgroup.Permissions[j], err.Error())
 					return err
 				}
 				j += 1
 			} else if j >= len(secgroup.Permissions) {
 				if err := self.addSecurityGroupRules(secgroupId, &rules[i]); err != nil {
-					log.Errorf("addSecurityGroupRule error %v", rules[i])
+					log.Errorf("addSecurityGroupRule %v error: %s", rules[i], err.Error())
 					return err
 				}
 				i += 1
diff --git a/pkg/util/aws/utils.go b/pkg/util/aws/utils.go
index 6db4151d9d..6c9d648769 100644
--- a/pkg/util/aws/utils.go
+++ b/pkg/util/aws/utils.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net"
 	"reflect"
+	"regexp"
 	"strings"
 	"yunion.io/x/jsonutils"
 
@@ -295,6 +296,8 @@ func AwsIpPermissionToYunion(direction secrules.TSecurityRuleDirection, p ec2.Ip
 	return rules, nil
 }
 
+// YunionSecRuleToAws 不能保证无损转换
+// 规则描述如果包含中文等字符，将被丢弃掉
 func YunionSecRuleToAws(rule secrules.SecurityRule) ([]*ec2.IpPermission, error) {
 	if rule.Action == secrules.SecurityRuleDeny {
 		return nil, fmt.Errorf("YunionSecRuleToAws ignored  aws not supported deny rule")
@@ -304,8 +307,13 @@ func YunionSecRuleToAws(rule secrules.SecurityRule) ([]*ec2.IpPermission, error)
 	if iprange == "<nil>" {
 		return nil, fmt.Errorf("YunionSecRuleToAws ignored  ipnet should not be empty")
 	}
+
+	description := ""
+	if match, err := regexp.MatchString("^[\\sa-zA-Z0-9. _:/()#,@\\]\\[+=&;{}!$*-]+$", rule.Description);err == nil && match {
+		description = rule.Description
+	}
 	ipranges := []*ec2.IpRange{}
-	ipranges = append(ipranges, &ec2.IpRange{CidrIp: &iprange, Description: &rule.Description})
+	ipranges = append(ipranges, &ec2.IpRange{CidrIp: &iprange, Description: &description})
 
 	portranges := yunionPortRangeToAws(rule)
 	protocol := yunionProtocolToAws(rule)

From 74c024c52e8cbfaae5824e069a481c7ca8d355f8 Mon Sep 17 00:00:00 2001
From: TangBin <tangbin@yunionyun.com>
Date: Wed, 14 Nov 2018 15:45:14 +0800
Subject: [PATCH 08/57] add latitude & longitude info

---
 pkg/util/aws/disk.go                   | 10 +++---
 pkg/util/aws/latitude_and_longitude.go | 23 +++++++++++++
 pkg/util/aws/region.go                 | 47 ++++++++++++++++++++++++--
 pkg/util/aws/snapshot.go               |  2 +-
 pkg/util/aws/utils.go                  |  2 +-
 5 files changed, 75 insertions(+), 9 deletions(-)
 create mode 100644 pkg/util/aws/latitude_and_longitude.go

diff --git a/pkg/util/aws/disk.go b/pkg/util/aws/disk.go
index 02e5c23ae5..e4c34ef1cc 100644
--- a/pkg/util/aws/disk.go
+++ b/pkg/util/aws/disk.go
@@ -369,7 +369,7 @@ func (self *SRegion) resetDisk(diskId, snapshotId string) error {
 	// 这里实际是回滚快照
 	disk, err := self.GetDisk(diskId)
 	if err != nil {
-		log.Debugf("resetDisk %s:%s",diskId, err.Error())
+		log.Debugf("resetDisk %s:%s", diskId, err.Error())
 		return err
 	}
 
@@ -378,11 +378,11 @@ func (self *SRegion) resetDisk(diskId, snapshotId string) error {
 	params.SetSize(int64(disk.Size))
 	params.SetVolumeType(disk.Category)
 	params.SetAvailabilityZone(disk.ZoneId)
-	tags,_ := disk.Tags.GetTagSpecifications()
+	tags, _ := disk.Tags.GetTagSpecifications()
 	params.SetTagSpecifications([]*ec2.TagSpecification{tags})
 	ret, err := self.ec2Client.CreateVolume(params)
 	if err != nil {
-		log.Debugf("resetDisk %s: %s",params.String(), err.Error())
+		log.Debugf("resetDisk %s: %s", params.String(), err.Error())
 		return err
 	}
 
@@ -394,7 +394,7 @@ func (self *SRegion) resetDisk(diskId, snapshotId string) error {
 			return err
 		}
 
-		err = self.ec2Client.WaitUntilVolumeAvailable(&ec2.DescribeVolumesInput{VolumeIds:[]*string{&diskId}})
+		err = self.ec2Client.WaitUntilVolumeAvailable(&ec2.DescribeVolumesInput{VolumeIds: []*string{&diskId}})
 		if err != nil {
 			log.Debugf("resetDisk :%s", err.Error())
 			return err
@@ -403,7 +403,7 @@ func (self *SRegion) resetDisk(diskId, snapshotId string) error {
 
 	err = self.AttachDisk(disk.InstanceId, *ret.VolumeId, disk.Device)
 	if err != nil {
-		log.Debugf("resetDisk %s %s %s: %s",disk.InstanceId, *ret.VolumeId, disk.Device, err.Error())
+		log.Debugf("resetDisk %s %s %s: %s", disk.InstanceId, *ret.VolumeId, disk.Device, err.Error())
 		return err
 	}
 
diff --git a/pkg/util/aws/latitude_and_longitude.go b/pkg/util/aws/latitude_and_longitude.go
new file mode 100644
index 0000000000..f3082a9e6c
--- /dev/null
+++ b/pkg/util/aws/latitude_and_longitude.go
@@ -0,0 +1,23 @@
+package aws
+
+var LatitudeAndLongitude = map[string]map[string]float32{
+	"ap-south-1":     {"latitude": 19.0759837, "longitude": 72.8776559},
+	"ap-northeast-3": {"latitude": 34.6937378, "longitude": 135.5021651},
+	"us-east-1":      {"latitude": 37.4315734, "longitude": -78.6568942},
+	"us-east-2":      {"latitude": 40.4172871, "longitude": -82.90712300000001},
+	"ap-southeast-2": {"latitude": -33.8688197, "longitude": 151.2092955},
+	"cn-northwest-1": {"latitude": 37.198731, "longitude": 106.1580937},
+	"eu-west-1":      {"latitude": 53.41291, "longitude": -8.24389},
+	"eu-central-1":   {"latitude": 50.1109221, "longitude": 8.6821267},
+	"sa-east-1":      {"latitude": -23.5505199, "longitude": -46.63330939999999},
+	"ap-southeast-1": {"latitude": 1.352083, "longitude": 103.819836},
+	"ca-central-1":   {"latitude": 56.130366, "longitude": -106.346771},
+	"ap-northeast-2": {"latitude": 37.566535, "longitude": 126.9779692},
+	"us-west-2":      {"latitude": 43.8041334, "longitude": -120.5542012},
+	"us-gov-west-1":  {"latitude": 37.09024, "longitude": -95.712891},
+	"us-west-1":      {"latitude": 38.8375215, "longitude": -120.8958242},
+	"cn-north-1":     {"latitude": 39.90419989999999, "longitude": 116.4073963},
+	"ap-northeast-1": {"latitude": 35.7090259, "longitude": 139.7319925},
+	"eu-west-2":      {"latitude": 51.5073509, "longitude": -0.1277583},
+	"eu-west-3":      {"latitude": 48.856614, "longitude": 2.3522219},
+}
diff --git a/pkg/util/aws/region.go b/pkg/util/aws/region.go
index a03f4303ff..a3c8fcb875 100644
--- a/pkg/util/aws/region.go
+++ b/pkg/util/aws/region.go
@@ -2,6 +2,7 @@ package aws
 
 import (
 	"fmt"
+
 	sdk "github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -14,6 +15,28 @@ import (
 	"yunion.io/x/onecloud/pkg/compute/models"
 )
 
+var RegionLocations map[string]string = map[string]string{
+	"us-east-2":      "美国东部(俄亥俄州)",
+	"us-east-1":      "美国东部(弗吉尼亚北部)",
+	"us-west-1":      "美国西部(加利福尼亚北部)",
+	"us-west-2":      "美国西部(俄勒冈)",
+	"ap-south-1":     "亚太地区(孟买)",
+	"ap-northeast-2": "亚太区域(首尔)",
+	"ap-northeast-3": "亚太区域(大阪)",
+	"ap-southeast-1": "亚太区域(新加坡)",
+	"ap-southeast-2": "亚太区域(悉尼)",
+	"ap-northeast-1": "亚太区域(东京)",
+	"ca-central-1":   "加拿大(中部)",
+	"cn-north-1":     "中国(北京)",
+	"cn-northwest-1": "中国(宁夏)",
+	"eu-central-1":   "欧洲(法兰克福)",
+	"eu-west-1":      "欧洲(爱尔兰)",
+	"eu-west-2":      "欧洲(伦敦)",
+	"eu-west-3":      "欧洲(巴黎)",
+	"sa-east-1":      "南美洲(圣保罗)",
+	"us-gov-west-1":  "AWS GovCloud(美国)",
+}
+
 type SRegion struct {
 	client    *SAwsClient
 	ec2Client *ec2.EC2
@@ -164,6 +187,10 @@ func (self *SRegion) GetId() string {
 }
 
 func (self *SRegion) GetName() string {
+	if localName, ok := RegionLocations[self.RegionId]; ok {
+		return fmt.Sprintf("%s %s", CLOUD_PROVIDER_AWS_CN, localName)
+	}
+
 	return fmt.Sprintf("%s %s", CLOUD_PROVIDER_AWS_CN, self.RegionId)
 }
 
@@ -188,11 +215,27 @@ func (self *SRegion) GetMetadata() *jsonutils.JSONDict {
 }
 
 func (self *SRegion) GetLatitude() float32 {
-	return 0.0
+	if data, ok := LatitudeAndLongitude[self.RegionId]; !ok {
+		log.Debugf("Region %s not found in LatitudeAndLongitude", self.RegionId)
+		return 0.0
+	} else if lat, ok := data["latitude"]; !ok {
+		log.Debugf("Region %s's latitude not found in LatitudeAndLongitude", self.RegionId)
+		return 0.0
+	} else {
+		return lat
+	}
 }
 
 func (self *SRegion) GetLongitude() float32 {
-	return 0.0
+	if data, ok := LatitudeAndLongitude[self.RegionId]; !ok {
+		log.Debugf("Region %s not found in LatitudeAndLongitude", self.RegionId)
+		return 0.0
+	} else if lat, ok := data["longitude"]; !ok {
+		log.Debugf("Region %s's latitude not found in LatitudeAndLongitude", self.RegionId)
+		return 0.0
+	} else {
+		return lat
+	}
 }
 
 func (self *SRegion) GetIZones() ([]cloudprovider.ICloudZone, error) {
diff --git a/pkg/util/aws/snapshot.go b/pkg/util/aws/snapshot.go
index 259fcdca39..c69816ecb4 100644
--- a/pkg/util/aws/snapshot.go
+++ b/pkg/util/aws/snapshot.go
@@ -96,7 +96,7 @@ func (self *SSnapshot) Delete() error {
 
 func (self *SSnapshot) GetRegionId() string {
 	// 这里特别注意：aws没有有uuid形式的region id
-	return self.region.Get
+	return self.region.GetId()
 }
 
 func (self *SRegion) GetSnapshots(instanceId string, diskId string, snapshotName string, snapshotIds []string, offset int, limit int) ([]SSnapshot, int, error) {
diff --git a/pkg/util/aws/utils.go b/pkg/util/aws/utils.go
index 6c9d648769..3ac2d34f9e 100644
--- a/pkg/util/aws/utils.go
+++ b/pkg/util/aws/utils.go
@@ -309,7 +309,7 @@ func YunionSecRuleToAws(rule secrules.SecurityRule) ([]*ec2.IpPermission, error)
 	}
 
 	description := ""
-	if match, err := regexp.MatchString("^[\\sa-zA-Z0-9. _:/()#,@\\]\\[+=&;{}!$*-]+$", rule.Description);err == nil && match {
+	if match, err := regexp.MatchString("^[\\sa-zA-Z0-9. _:/()#,@\\]\\[+=&;{}!$*-]+$", rule.Description); err == nil && match {
 		description = rule.Description
 	}
 	ipranges := []*ec2.IpRange{}

From ebbedb9458213019016f3be0152a793a617360a5 Mon Sep 17 00:00:00 2001
From: TangBin <tangbin@yunionyun.com>
Date: Wed, 14 Nov 2018 20:32:16 +0800
Subject: [PATCH 09/57] vpc & securtiy group sync

---
 pkg/util/aws/instance.go      | 27 ++++++-----
 pkg/util/aws/region.go        | 21 ++++++++
 pkg/util/aws/securitygroup.go | 90 +++++++++++++++++++----------------
 pkg/util/aws/vpc.go           | 29 ++++++++++-
 4 files changed, 112 insertions(+), 55 deletions(-)

diff --git a/pkg/util/aws/instance.go b/pkg/util/aws/instance.go
index 13748b02fd..4d693c6ac3 100644
--- a/pkg/util/aws/instance.go
+++ b/pkg/util/aws/instance.go
@@ -273,23 +273,26 @@ func (self *SInstance) SyncSecurityGroup(secgroupId string, name string, rules [
 	if vpc, err := self.getVpc(); err != nil {
 		return err
 	} else if len(secgroupId) == 0 {
-		for index, secgrpId := range self.SecurityGroupIds.SecurityGroupId {
-			if err := vpc.revokeSecurityGroup(secgrpId, self.InstanceId, index == 0); err != nil {
-				return err
-			}
-		}
+		// todo ： 这里应该有问题。aws不能直接删除安全组。且至少选择一个安全组
+		// for index, secgrpId := range self.SecurityGroupIds.SecurityGroupId {
+		// 	if err := vpc.revokeSecurityGroup(secgrpId, self.InstanceId, index == 0); err != nil {
+		// 		return err
+		// 	}
+		// }
+		return nil
 	} else if secgrpId, err := vpc.SyncSecurityGroup(secgroupId, name, rules); err != nil {
 		return err
 	} else if err := vpc.assignSecurityGroup(secgrpId, self.InstanceId); err != nil {
 		return err
 	} else {
-		for _, secgroupId := range self.SecurityGroupIds.SecurityGroupId {
-			if secgroupId != secgrpId {
-				if err := vpc.revokeSecurityGroup(secgroupId, self.InstanceId, false); err != nil {
-					return err
-				}
-			}
-		}
+		// todo ： 这里应该有问题。aws不能直接删除安全组。且至少选择一个安全组
+		// for _, secgroupId := range self.SecurityGroupIds.SecurityGroupId {
+		// 	if secgroupId != secgrpId {
+		// 		if err := vpc.revokeSecurityGroup(secgroupId, self.InstanceId, false); err != nil {
+		// 			return err
+		// 		}
+		// 	}
+		// }
 		self.SecurityGroupIds.SecurityGroupId = []string{secgrpId}
 	}
 	return nil
diff --git a/pkg/util/aws/region.go b/pkg/util/aws/region.go
index a3c8fcb875..558a307604 100644
--- a/pkg/util/aws/region.go
+++ b/pkg/util/aws/region.go
@@ -363,11 +363,32 @@ func (self *SRegion) GetIStoragecacheById(id string) (cloudprovider.ICloudStorag
 }
 
 func (self *SRegion) CreateIVpc(name string, desc string, cidr string) (cloudprovider.ICloudVpc, error) {
+	tagspec := TagSpec{ResourceType: "vpc"}
+	if len(name) > 0 {
+		tagspec.SetNameTag(name)
+	}
+
+	if len(desc) > 0 {
+		tagspec.SetDescTag(desc)
+	}
+
+	spec, err := tagspec.GetTagSpecifications()
+	if err != nil {
+		return nil, err
+	}
+
+	// start create vpc
 	vpc, err := self.ec2Client.CreateVpc(&ec2.CreateVpcInput{CidrBlock: &cidr})
 	if err != nil {
 		return nil, err
 	}
 
+	tagsParams := &ec2.CreateTagsInput{Resources: []*string{vpc.Vpc.VpcId}, Tags: spec.Tags}
+	_, err = self.ec2Client.CreateTags(tagsParams)
+	if err != nil {
+		log.Debugf("CreateIVpc add tag failed %s", err.Error())
+	}
+
 	err = self.fetchInfrastructure()
 	if err != nil {
 		return nil, err
diff --git a/pkg/util/aws/securitygroup.go b/pkg/util/aws/securitygroup.go
index 32d69d9355..29399d61b0 100644
--- a/pkg/util/aws/securitygroup.go
+++ b/pkg/util/aws/securitygroup.go
@@ -27,7 +27,7 @@ type SSecurityGroup struct {
 	VpcId             string
 	SecurityGroupId   string
 	Description       string
-	SecurityGroupName string
+	SecurityGroupName string //对应tag中的name标签
 	Permissions       []secrules.SecurityRule
 	Tags              Tags
 
@@ -117,22 +117,22 @@ func (self *SRegion) addSecurityGroupRule(secGrpId string, rule *secrules.Securi
 		params := &ec2.AuthorizeSecurityGroupIngressInput{}
 		params.SetGroupId(secGrpId)
 		params.SetIpPermissions(ipPermissions)
-		_, err := self.ec2Client.AuthorizeSecurityGroupIngress(params)
-		if err != nil {
-			return err
-		}
+		_, err = self.ec2Client.AuthorizeSecurityGroupIngress(params)
 	}
 
 	if rule.Direction == secrules.SecurityRuleEgress {
 		params := &ec2.AuthorizeSecurityGroupEgressInput{}
 		params.SetGroupId(secGrpId)
 		params.SetIpPermissions(ipPermissions)
-		_, err := self.ec2Client.AuthorizeSecurityGroupEgress(params)
-		if err != nil {
-			return err
-		}
+		_, err = self.ec2Client.AuthorizeSecurityGroupEgress(params)
 	}
-	return nil
+
+	if err != nil && strings.Contains(err.Error(),"InvalidPermission.Duplicate") {
+		log.Debugf("addSecurityGroupRule %s %s", rule.Direction, err.Error())
+		return nil
+	}
+
+	return err
 }
 
 func (self *SRegion) delSecurityGroupRule(secGrpId string, rule *secrules.SecurityRule) error {
@@ -145,20 +145,19 @@ func (self *SRegion) delSecurityGroupRule(secGrpId string, rule *secrules.Securi
 		params := &ec2.RevokeSecurityGroupIngressInput{}
 		params.SetGroupId(secGrpId)
 		params.SetIpPermissions(ipPermissions)
-		_, err := self.ec2Client.RevokeSecurityGroupIngress(params)
-		if err != nil {
-			return err
-		}
+		_, err = self.ec2Client.RevokeSecurityGroupIngress(params)
 	}
 
 	if rule.Direction == secrules.SecurityRuleEgress {
 		params := &ec2.RevokeSecurityGroupEgressInput{}
 		params.SetGroupId(secGrpId)
 		params.SetIpPermissions(ipPermissions)
-		_, err := self.ec2Client.RevokeSecurityGroupEgress(params)
-		if err != nil {
-			return err
-		}
+		_, err = self.ec2Client.RevokeSecurityGroupEgress(params)
+	}
+
+	if err != nil {
+		log.Debugf("delSecurityGroupRule %s %s", rule.Direction, err.Error())
+		return err
 	}
 	return nil
 }
@@ -198,8 +197,10 @@ func (self *SRegion) updateSecurityGroupRuleDescription(secGrpId string, rule *s
 func (self *SRegion) createSecurityGroup(vpcId string, name string, secgroupIdTag string, desc string) (string, error) {
 	params := &ec2.CreateSecurityGroupInput{}
 	params.SetVpcId(vpcId)
+	// 这里的描述aws 上层代码拼接的描述。并非用户提交的描述，用户描述放置在Yunion本地数据库中。）
 	params.SetDescription(desc)
-	params.SetGroupName(name)
+	// 这里使用id作为组名。原因name容易重名、另外有可能包含中文，aws不支持中文
+	params.SetGroupName(secgroupIdTag)
 
 	group, err := self.ec2Client.CreateSecurityGroup(params)
 	if err != nil {
@@ -208,6 +209,8 @@ func (self *SRegion) createSecurityGroup(vpcId string, name string, secgroupIdTa
 
 	tagspec := TagSpec{ResourceType: "security-group"}
 	tagspec.SetTag("id", secgroupIdTag)
+	tagspec.SetNameTag(name)
+	tagspec.SetDescTag(desc)
 	tags, _ := tagspec.GetTagSpecifications()
 	tagParams := &ec2.CreateTagsInput{}
 	tagParams.SetResources([]*string{group.GroupId})
@@ -308,6 +311,9 @@ func (self *SRegion) modifySecurityGroup(secGrpId string, name string, desc stri
 }
 
 func (self *SRegion) syncSecgroupRules(secgroupId string, rules []secrules.SecurityRule) error {
+	var DeleteRules []secrules.SecurityRule
+	var AddRules []secrules.SecurityRule
+
 	if secgroup, err := self.GetSecurityGroupDetails(secgroupId); err != nil {
 		return err
 	} else {
@@ -315,6 +321,9 @@ func (self *SRegion) syncSecgroupRules(secgroupId string, rules []secrules.Secur
 		sort.Sort(secrules.SecurityRuleSet(rules))
 		sort.Sort(secrules.SecurityRuleSet(secgroup.Permissions))
 
+		log.Debugf("local security rules %s",rules)
+		log.Debugf("remote security rules %s",secgroup.Permissions)
+
 		i, j := 0, 0
 		for i < len(rules) || j < len(secgroup.Permissions) {
 			if i < len(rules) && j < len(secgroup.Permissions) {
@@ -322,42 +331,41 @@ func (self *SRegion) syncSecgroupRules(secgroupId string, rules []secrules.Secur
 				ruleStr := rules[i].String()
 				cmp := strings.Compare(permissionStr, ruleStr)
 				if cmp == 0 {
-					if secgroup.Permissions[j].Description != rules[i].Description {
-						if err := self.updateSecurityGroupRuleDescription(secgroupId, &rules[i]); err != nil {
-							log.Errorf("updateSecurityGroupRuleDescription %v error: %s", rules[i], err.Error())
-							return err
-						}
-					}
+					DeleteRules = append(DeleteRules, secgroup.Permissions[j])
+					AddRules = append(AddRules, rules[i])
 					i += 1
 					j += 1
 				} else if cmp > 0 {
-					if err := self.delSecurityGroupRule(secgroupId, &secgroup.Permissions[j]); err != nil {
-						log.Errorf("delSecurityGroupRule %v error: %s", secgroup.Permissions[j], err.Error())
-						return err
-					}
+					DeleteRules = append(DeleteRules, secgroup.Permissions[j])
 					j += 1
 				} else {
-					if err := self.addSecurityGroupRules(secgroupId, &rules[i]); err != nil {
-						log.Errorf("addSecurityGroupRule %v error: %s", rules[i], err.Error())
-						return err
-					}
+					AddRules = append(AddRules, rules[i])
 					i += 1
 				}
 			} else if i >= len(rules) {
-				if err := self.delSecurityGroupRule(secgroupId, &secgroup.Permissions[j]); err != nil {
-					log.Errorf("delSecurityGroupRule %v error: %s", secgroup.Permissions[j], err.Error())
-					return err
-				}
+				DeleteRules = append(DeleteRules, secgroup.Permissions[j])
 				j += 1
 			} else if j >= len(secgroup.Permissions) {
-				if err := self.addSecurityGroupRules(secgroupId, &rules[i]); err != nil {
-					log.Errorf("addSecurityGroupRule %v error: %s", rules[i], err.Error())
-					return err
-				}
+				AddRules = append(AddRules, rules[i])
 				i += 1
 			}
 		}
 	}
+
+	for _, r := range DeleteRules {
+		if err := self.delSecurityGroupRule(secgroupId, &r); err != nil {
+			log.Errorf("delSecurityGroupRule %v error: %s", r, err.Error())
+			return err
+		}
+	}
+
+	for _, r := range AddRules {
+		if err := self.addSecurityGroupRules(secgroupId, &r); err != nil {
+			log.Errorf("addSecurityGroupRule %v error: %s", r, err.Error())
+			return err
+		}
+	}
+
 	return nil
 }
 
diff --git a/pkg/util/aws/vpc.go b/pkg/util/aws/vpc.go
index 9d93352d27..1288c063d4 100644
--- a/pkg/util/aws/vpc.go
+++ b/pkg/util/aws/vpc.go
@@ -240,19 +240,44 @@ func (self *SRegion) getVpc(vpcId string) (*SVpc, error) {
 }
 
 func (self *SRegion) revokeSecurityGroup(secgroupId, instanceId string, keep bool) error {
+	// todo : keep ? 直接使用assignSecurityGroup 即可？
 	return nil
 }
 
 func (self *SRegion) assignSecurityGroup(secgroupId, instanceId string) error {
+	instance, err := self.GetInstance(instanceId)
+	if err != nil {
+		return err
+	}
+
+	for _, eth := range instance.NetworkInterfaces.NetworkInterface {
+		params := &ec2.ModifyNetworkInterfaceAttributeInput{}
+		params.SetNetworkInterfaceId(eth.NetworkInterfaceId)
+		params.SetGroups([]*string{&secgroupId})
+
+		_, err := self.ec2Client.ModifyNetworkInterfaceAttribute(params)
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
 func (self *SRegion) deleteSecurityGroup(secGrpId string) error {
-	return nil
+	params := &ec2.DeleteSecurityGroupInput{}
+	params.SetGroupId(secGrpId)
+
+	_, err := self.ec2Client.DeleteSecurityGroup(params)
+	return err
 }
 
 func (self *SRegion) DeleteVpc(vpcId string) error {
-	return nil
+	params := &ec2.DeleteVpcInput{}
+	params.SetVpcId(vpcId)
+
+	_, err := self.ec2Client.DeleteVpc(params)
+	return err
 }
 
 func (self *SRegion) GetVpcs(vpcId []string, offset int, limit int) ([]SVpc, int, error) {

From 010f5c1f979ec933d606e21faa826e9acb76f810 Mon Sep 17 00:00:00 2001
From: TangBin <tangbin@yunionyun.com>
Date: Thu, 15 Nov 2018 18:00:40 +0800
Subject: [PATCH 10/57] vpc fix

---
 pkg/util/aws/vpc.go | 17 ++++-------------
 1 file changed, 4 insertions(+), 13 deletions(-)

diff --git a/pkg/util/aws/vpc.go b/pkg/util/aws/vpc.go
index 1288c063d4..72b35b3a80 100644
--- a/pkg/util/aws/vpc.go
+++ b/pkg/util/aws/vpc.go
@@ -108,19 +108,7 @@ func (self *SVpc) GetManagerId() string {
 }
 
 func (self *SVpc) Delete() error {
-	err := self.fetchSecurityGroups()
-	if err != nil {
-		log.Errorf("fetchSecurityGroup for VPC delete fail %s", err)
-		return err
-	}
-	for i := 0; i < len(self.secgroups); i += 1 {
-		secgroup := self.secgroups[i].(*SSecurityGroup)
-		err := self.region.deleteSecurityGroup(secgroup.SecurityGroupId)
-		if err != nil {
-			log.Errorf("deleteSecurityGroup for VPC delete fail %s", err)
-			return err
-		}
-	}
+    // 删除vpc会同步删除关联的安全组
 	return self.region.DeleteVpc(self.VpcId)
 }
 
@@ -287,6 +275,9 @@ func (self *SRegion) GetVpcs(vpcId []string, offset int, limit int) ([]SVpc, int
 	}
 	ret, err := self.ec2Client.DescribeVpcs(params)
 	if err != nil {
+		if strings.Contains(err.Error(), "InvalidVpcID.NotFound") {
+			return nil, 0, cloudprovider.ErrNotFound
+		}
 		return nil, 0, err
 	}
 

From 9bf73e99d915cb89bea190ece65709f520ca87d1 Mon Sep 17 00:00:00 2001
From: TangBin <tangbin@yunionyun.com>
Date: Thu, 15 Nov 2018 19:26:35 +0800
Subject: [PATCH 11/57] return err

---
 pkg/compute/models/cloudaccounts.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go
index 0c40545812..14c2e97f39 100644
--- a/pkg/compute/models/cloudaccounts.go
+++ b/pkg/compute/models/cloudaccounts.go
@@ -114,7 +114,7 @@ func (manager *SCloudaccountManager) ValidateCreateData(ctx context.Context, use
 			return nil, httperrors.NewResourceNotFoundError("no such provider %s", provider)
 		}
 		log.Debugf("ValidateCreateData %s", err.Error())
-		return nil, httperrors.NewInvalidCredentialError("invalid cloud account info")
+		return nil, httperrors.NewInputParameterError("invalid cloud account info")
 	}
 
 	return manager.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)

From ca0e3977c70cb4ad521e023098ae598f3cbfb4e2 Mon Sep 17 00:00:00 2001
From: Qiu Jian <qiujian@yunionyun.com>
Date: Thu, 15 Nov 2018 23:41:04 +0800
Subject: [PATCH 12/57] minor fixes

---
 pkg/cloudcommon/db/db_dispatcher.go | 68 ++++++++++++++++++-----------
 pkg/cloudcommon/db/taskman/tasks.go | 19 +++++---
 pkg/cloudcommon/policy/policy.go    | 57 +++++++++++++++++++++---
 pkg/cloutpost/options/options.go    | 21 +++++++++
 pkg/cloutpost/service/handlers.go   |  7 +++
 pkg/cloutpost/service/service.go    | 53 ++++++++++++++++++++++
 pkg/util/rbacutils/rabc.go          |  4 +-
 7 files changed, 191 insertions(+), 38 deletions(-)
 create mode 100644 pkg/cloutpost/options/options.go
 create mode 100644 pkg/cloutpost/service/handlers.go
 create mode 100644 pkg/cloutpost/service/service.go

diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go
index 1f4b543a69..89c08637c2 100644
--- a/pkg/cloudcommon/db/db_dispatcher.go
+++ b/pkg/cloudcommon/db/db_dispatcher.go
@@ -479,14 +479,14 @@ func listItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 	if err != nil {
 		return nil, httperrors.NewGeneralError(err)
 	}
-	retConut := len(retList)
+	retCount := len(retList)
 
 	// apply customizeFilters
 	retList, err = customizeFilters.DoApply(retList)
 	if err != nil {
 		return nil, httperrors.NewGeneralError(err)
 	}
-	if len(retList) != retConut {
+	if len(retList) != retCount {
 		totalCnt = int64(len(retList))
 	}
 	paginate := false
@@ -500,7 +500,7 @@ func listItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 func calculateListResult(data []jsonutils.JSONObject, total, limit, offset int64, paginate bool) *modules.ListResult {
 	if paginate {
 		// do offset first
-		if offset != 0 {
+		if offset > 0 {
 			if total > offset {
 				data = data[offset:]
 			} else {
@@ -508,11 +508,13 @@ func calculateListResult(data []jsonutils.JSONObject, total, limit, offset int64
 			}
 		}
 		// do limit
-		if total > limit {
+		if limit > 0 && total > limit {
 			data = data[:limit]
 		}
 	}
+
 	retResult := modules.ListResult{Data: data, Total: int(total), Limit: int(limit), Offset: int(offset)}
+
 	return &retResult
 }
 
@@ -957,6 +959,34 @@ func (dispatcher *DBModelDispatcher) BatchCreate(ctx context.Context, query json
 	return results, nil
 }
 
+func managerPerformCheckCreateData(
+	manager IModelManager,
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	action string,
+	ownerProjId string,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject,
+) (jsonutils.JSONObject, error) {
+	body, err := data.(*jsonutils.JSONDict).Get(manager.Keyword())
+	if err != nil {
+		return nil, httperrors.NewGeneralError(err)
+	}
+	bodyDict := body.(*jsonutils.JSONDict)
+
+	var isAllow bool
+	if consts.IsRbacEnabled() {
+		isAllow = isClassActionRbacAllowed(manager, userCred, ownerProjId, policy.PolicyActionPerform, action)
+	} else {
+		isAllow = manager.AllowPerformCheckCreateData(ctx, userCred, query, data)
+	}
+	if !isAllow {
+		return nil, httperrors.NewForbiddenError("not allow to perform %s", action)
+	}
+
+	return manager.ValidateCreateData(ctx, userCred, ownerProjId, query, bodyDict)
+}
+
 func (dispatcher *DBModelDispatcher) PerformClassAction(ctx context.Context, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	userCred := fetchUserCredential(ctx)
 
@@ -969,25 +999,8 @@ func (dispatcher *DBModelDispatcher) PerformClassAction(ctx context.Context, act
 	defer lockman.ReleaseClass(ctx, dispatcher.modelManager, ownerProjId)
 
 	if action == "check-create-data" {
-		manager := dispatcher.modelManager
-
-		body, err := data.(*jsonutils.JSONDict).Get(manager.Keyword())
-		if err != nil {
-			return nil, httperrors.NewGeneralError(err)
-		}
-		data := body.(*jsonutils.JSONDict)
-
-		var isAllow bool
-		if consts.IsRbacEnabled() {
-			isAllow = isClassActionRbacAllowed(manager, userCred, ownerProjId, policy.PolicyActionPerform, action)
-		} else {
-			isAllow = manager.AllowPerformCheckCreateData(ctx, userCred, query, data)
-		}
-		if !isAllow {
-			return nil, httperrors.NewForbiddenError("not allow to perform %s", action)
-		}
-
-		return manager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)
+		return managerPerformCheckCreateData(dispatcher.modelManager,
+			ctx, userCred, action, ownerProjId, query, data)
 	}
 
 	managerValue := reflect.ValueOf(dispatcher.modelManager)
@@ -1022,8 +1035,8 @@ func objectPerformAction(dispatcher *DBModelDispatcher, model IModel, modelValue
 
 	isGeneral := false
 	funcName := fmt.Sprintf("Perform%s", utils.Kebab2Camel(action, "-"))
-
 	funcValue := modelValue.MethodByName(funcName)
+
 	if !funcValue.IsValid() || funcValue.IsNil() {
 		funcValue = modelValue.MethodByName(generalFuncName)
 		if !funcValue.IsValid() || funcValue.IsNil() {
@@ -1057,7 +1070,12 @@ func objectPerformAction(dispatcher *DBModelDispatcher, model IModel, modelValue
 
 	var isAllow bool
 	if consts.IsRbacEnabled() {
-		isAllow = isObjectRbacAllowed(dispatcher.modelManager, model, userCred, policy.PolicyActionPerform, action)
+		if model == nil {
+			ownerProjId, _ := fetchOwnerProjectId(ctx, dispatcher.modelManager, userCred, data)
+			isAllow = isClassActionRbacAllowed(dispatcher.modelManager, userCred, ownerProjId, policy.PolicyActionPerform, action)
+		} else {
+			isAllow = isObjectRbacAllowed(dispatcher.modelManager, model, userCred, policy.PolicyActionPerform, action)
+		}
 	} else {
 		allowFuncName := "Allow" + funcName
 		allowFuncValue := modelValue.MethodByName(allowFuncName)
diff --git a/pkg/cloudcommon/db/taskman/tasks.go b/pkg/cloudcommon/db/taskman/tasks.go
index 9b4f175e46..5460fff7cc 100644
--- a/pkg/cloudcommon/db/taskman/tasks.go
+++ b/pkg/cloudcommon/db/taskman/tasks.go
@@ -89,10 +89,6 @@ func (manager *STaskManager) FilterByName(q *sqlchemy.SQuery, name string) *sqlc
 	return q
 }
 
-func (manager *STaskManager) FilterByOwner(q *sqlchemy.SQuery, owner string) *sqlchemy.SQuery {
-	return q
-}
-
 func (manager *STaskManager) AllowPerformAction(ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
 	return true
 }
@@ -105,6 +101,19 @@ func (manager *STaskManager) PerformAction(ctx context.Context, userCred mcclien
 	return resp, nil
 }
 
+func (manager *STaskManager) GetOwnerId(userCred mcclient.IIdentityProvider) string {
+	return userCred.GetProjectId()
+}
+
+func (self *STask) GetOwnerProjectId() string {
+	return self.UserCred.GetProjectId()
+}
+
+func (manager *STaskManager) FilterByOwner(q *sqlchemy.SQuery, owner string) *sqlchemy.SQuery {
+	q = q.Contains("user_cred", owner)
+	return q
+}
+
 func (self *STask) AllowGetDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
 	return userCred.IsSystemAdmin() || userCred.GetProjectId() == self.UserCred.GetProjectId()
 }
@@ -414,7 +423,7 @@ func execITask(taskValue reflect.Value, task *STask, odata jsonutils.JSONObject,
 		return
 	}
 
-	log.Debugf("Call %s %s: %s with %s", task.TaskName, stageName, funcValue, params)
+	log.Debugf("Call %s %s", task.TaskName, stageName)
 	funcValue.Call(params)
 
 	// call save request context
diff --git a/pkg/cloudcommon/policy/policy.go b/pkg/cloudcommon/policy/policy.go
index 623b2a7ee0..c5a14b5e64 100644
--- a/pkg/cloudcommon/policy/policy.go
+++ b/pkg/cloudcommon/policy/policy.go
@@ -30,8 +30,37 @@ const (
 var (
 	PolicyManager *SPolicyManager
 
-	PolicyFailedRetryInterval = 15 * time.Second
-	PolicyRefreshInterval     = 15 * time.Minute
+	defaultRules = []rbacutils.SRbacRule{
+		{
+			Resource: "tasks",
+			Action:   PolicyActionPerform,
+			Result:   rbacutils.Allow,
+		},
+		{
+			Service:  "compute",
+			Resource: "zones",
+			Action:   PolicyActionList,
+			Result:   rbacutils.Allow,
+		},
+		{
+			Service:  "compute",
+			Resource: "zones",
+			Action:   PolicyActionGet,
+			Result:   rbacutils.Allow,
+		},
+		{
+			Service:  "compute",
+			Resource: "cloudregions",
+			Action:   PolicyActionList,
+			Result:   rbacutils.Allow,
+		},
+		{
+			Service:  "compute",
+			Resource: "cloudregions",
+			Action:   PolicyActionGet,
+			Result:   rbacutils.Allow,
+		},
+	}
 )
 
 func init() {
@@ -41,7 +70,11 @@ func init() {
 type SPolicyManager struct {
 	policies      map[string]rbacutils.SRbacPolicy
 	adminPolicies map[string]rbacutils.SRbacPolicy
+	defaultPolicy *rbacutils.SRbacPolicy
 	lastSync      time.Time
+
+	failedRetryInterval time.Duration
+	refreshInterval     time.Duration
 }
 
 func parseJsonPolicy(obj jsonutils.JSONObject) (string, rbacutils.SRbacPolicy, error) {
@@ -114,8 +147,13 @@ func fetchPolicies() (map[string]rbacutils.SRbacPolicy, map[string]rbacutils.SRb
 
 func (manager *SPolicyManager) start(refreshInterval time.Duration, retryInterval time.Duration) {
 	log.Infof("PolicyManager start to fetch policies ...")
-	PolicyRefreshInterval = refreshInterval
-	PolicyFailedRetryInterval = retryInterval
+	manager.refreshInterval = refreshInterval
+	manager.failedRetryInterval = retryInterval
+	if len(defaultRules) > 0 {
+		manager.defaultPolicy = &rbacutils.SRbacPolicy{
+			Rules: rbacutils.CompactRules(defaultRules),
+		}
+	}
 	manager.sync()
 }
 
@@ -124,13 +162,14 @@ func (manager *SPolicyManager) sync() {
 	policies, adminPolicies, err := fetchPolicies()
 	if err != nil {
 		log.Errorf("sync policy fail %s", err)
-		time.AfterFunc(PolicyFailedRetryInterval, manager.sync)
+		time.AfterFunc(manager.failedRetryInterval, manager.sync)
 		return
 	}
 	manager.policies = policies
 	manager.adminPolicies = adminPolicies
+
 	manager.lastSync = time.Now()
-	time.AfterFunc(PolicyRefreshInterval, manager.sync)
+	time.AfterFunc(manager.refreshInterval, manager.sync)
 }
 
 func (manager *SPolicyManager) Allow(isAdmin bool, userCred mcclient.TokenCredential, service string, resource string, action string, extra ...string) rbacutils.TRbacResult {
@@ -152,6 +191,12 @@ func (manager *SPolicyManager) Allow(isAdmin bool, userCred mcclient.TokenCreden
 			currentPriv = result
 		}
 	}
+	if manager.defaultPolicy != nil {
+		result := manager.defaultPolicy.Allow(userCredJson, service, resource, action, extra...)
+		if result.IsHigherPrivilege(currentPriv) {
+			currentPriv = result
+		}
+	}
 	if consts.IsRbacDebug() {
 		log.Debugf("[RBAC: %v] %s %s %s %#v permission %s", isAdmin, service, resource, action, extra, currentPriv)
 	}
diff --git a/pkg/cloutpost/options/options.go b/pkg/cloutpost/options/options.go
new file mode 100644
index 0000000000..ec3a78860b
--- /dev/null
+++ b/pkg/cloutpost/options/options.go
@@ -0,0 +1,21 @@
+package options
+
+import (
+	"yunion.io/x/onecloud/pkg/cloudcommon"
+	"yunion.io/x/onecloud/pkg/cloudcommon/etcd"
+)
+
+type CloudSyncOptions struct {
+	Provider    string `help:"Public cloud provider" choices:"Aliyun|Azure|Aws|Qcloud"`
+	Environment string `help:"environment of public cloud"`
+	Cloudregion string `help:"region of public cloud"`
+	Zone        string `help:"availability zone of public cloud"`
+
+	etcd.SEtcdOptions
+
+	cloudcommon.Options
+}
+
+var (
+	Options CloudSyncOptions
+)
diff --git a/pkg/cloutpost/service/handlers.go b/pkg/cloutpost/service/handlers.go
new file mode 100644
index 0000000000..f1c362799b
--- /dev/null
+++ b/pkg/cloutpost/service/handlers.go
@@ -0,0 +1,7 @@
+package service
+
+import "yunion.io/x/onecloud/pkg/appsrv"
+
+func initHandlers(app *appsrv.Application) {
+
+}
diff --git a/pkg/cloutpost/service/service.go b/pkg/cloutpost/service/service.go
new file mode 100644
index 0000000000..243e311ada
--- /dev/null
+++ b/pkg/cloutpost/service/service.go
@@ -0,0 +1,53 @@
+package service
+
+import (
+	"os"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
+	"yunion.io/x/onecloud/pkg/cloudcommon/etcd"
+	"yunion.io/x/onecloud/pkg/cloudcommon/etcd/models"
+	"yunion.io/x/onecloud/pkg/cloutpost/options"
+)
+
+const (
+	SERVICE_TYPE = "cloutpost"
+)
+
+func StartService() {
+	consts.SetServiceType(SERVICE_TYPE)
+
+	cloudcommon.ParseOptions(&options.Options, &options.Options.Options, os.Args, "cloutpost.conf")
+
+	cloudcommon.InitAuth(&options.Options.Options, func() {
+		log.Infof("Auth complete!!")
+	})
+
+	err := etcd.InitDefaultEtcdClient(&options.Options.SEtcdOptions)
+	if err != nil {
+		log.Fatalf("init etcd fail: %s", err)
+	}
+	defer etcd.CloseDefaultEtcdClient()
+
+	app := cloudcommon.InitApp(&options.Options.Options)
+
+	initHandlers(app)
+
+	err = models.ServiceRegistryManager.Register(
+		app.GetContext(),
+		options.Options.Address,
+		options.Options.Port,
+		options.Options.Provider,
+		options.Options.Environment,
+		options.Options.Cloudregion,
+		options.Options.Zone,
+		SERVICE_TYPE,
+	)
+
+	if err != nil {
+		log.Fatalf("fail to register service %s", err)
+	}
+
+	cloudcommon.ServeForever(app, &options.Options.Options)
+}
diff --git a/pkg/util/rbacutils/rabc.go b/pkg/util/rbacutils/rabc.go
index ebf1fb911c..39530086ee 100644
--- a/pkg/util/rbacutils/rabc.go
+++ b/pkg/util/rbacutils/rabc.go
@@ -153,7 +153,7 @@ func (policy *SRbacPolicy) GetMatchRule(service string, resource string, action
 	return matchRule
 }
 
-func compactRules(rules []SRbacRule) []SRbacRule {
+func CompactRules(rules []SRbacRule) []SRbacRule {
 	output := make([]SRbacRule, 1)
 	output[0] = rules[0]
 	for i := 1; i < len(rules); i += 1 {
@@ -190,7 +190,7 @@ func (policy *SRbacPolicy) Decode(policyJson jsonutils.JSONObject) error {
 		return err
 	}
 
-	policy.Rules = compactRules(rules)
+	policy.Rules = CompactRules(rules)
 
 	return nil
 }

From 2734d2f5db59a87a116b7b0e888b77bb3b0fee66 Mon Sep 17 00:00:00 2001
From: Qiu Jian <qiujian@yunionyun.com>
Date: Fri, 16 Nov 2018 01:32:01 +0800
Subject: [PATCH 13/57] =?UTF-8?q?=E5=A2=9E=E5=8A=A0user/guest=E4=B8=A4?=
 =?UTF-8?q?=E4=B8=AA=E6=9D=83=E5=88=A9=E7=AD=89=E7=BA=A7?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/cloudcommon/db/db_dispatcher.go  | 10 ++++--
 pkg/cloudcommon/db/quotas/handler.go | 10 +++---
 pkg/cloudcommon/db/rbac.go           | 45 +++++++++++++++++-------
 pkg/cloudcommon/policy/policy.go     | 24 ++++++++-----
 pkg/compute/usages/handler.go        |  2 +-
 pkg/mcclient/auth/middleware.go      | 23 +++++++++---
 pkg/util/rbacutils/rabc.go           | 52 +++++++++++++++++-----------
 7 files changed, 110 insertions(+), 56 deletions(-)

diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go
index 89c08637c2..41671468af 100644
--- a/pkg/cloudcommon/db/db_dispatcher.go
+++ b/pkg/cloudcommon/db/db_dispatcher.go
@@ -57,12 +57,16 @@ func (dispatcher *DBModelDispatcher) ContextKeywordPlural() []string {
 }
 
 func (dispatcher *DBModelDispatcher) Filter(f appsrv.FilterHandler) appsrv.FilterHandler {
-	return auth.Authenticate(f)
+	if consts.IsRbacEnabled() {
+		return auth.AuthenticateWithDelayDecision(f, true)
+	} else {
+		return auth.Authenticate(f)
+	}
 }
 
 func fetchUserCredential(ctx context.Context) mcclient.TokenCredential {
 	token := auth.FetchUserCredential(ctx)
-	if token == nil {
+	if token == nil && ! consts.IsRbacEnabled() {
 		log.Fatalf("user token credential not found?")
 	}
 	return token
@@ -727,7 +731,7 @@ func fetchOwnerProjectId(ctx context.Context, manager IModelManager, userCred mc
 	if consts.IsRbacEnabled() {
 		result := policy.PolicyManager.Allow(true, userCred,
 			consts.GetServiceType(), policy.PolicyDelegation, "")
-		if result == rbacutils.Allow {
+		if result == rbacutils.AdminAllow {
 			isAllow = true
 		}
 	} else {
diff --git a/pkg/cloudcommon/db/quotas/handler.go b/pkg/cloudcommon/db/quotas/handler.go
index 6d16c8c708..ed27a32b72 100644
--- a/pkg/cloudcommon/db/quotas/handler.go
+++ b/pkg/cloudcommon/db/quotas/handler.go
@@ -91,7 +91,7 @@ func getQuotaHanlder(ctx context.Context, w http.ResponseWriter, r *http.Request
 		if consts.IsRbacEnabled() {
 			result := policy.PolicyManager.Allow(true, userCred, consts.GetServiceType(),
 				policy.PolicyDelegation, policy.PolicyActionGet)
-			isAllow = result == rbacutils.Allow
+			isAllow = result == rbacutils.AdminAllow
 		} else {
 			isAllow = userCred.IsSystemAdmin()
 		}
@@ -101,7 +101,7 @@ func getQuotaHanlder(ctx context.Context, w http.ResponseWriter, r *http.Request
 		}
 		if consts.IsRbacEnabled() {
 			if policy.PolicyManager.Allow(true, userCred, consts.GetServiceType(),
-				"quotas", policy.PolicyActionGet) != rbacutils.Allow {
+				"quotas", policy.PolicyActionGet) != rbacutils.AdminAllow {
 				httperrors.ForbiddenError(w, "not allow to query quota")
 				return
 			}
@@ -138,7 +138,7 @@ func setQuotaHanlder(ctx context.Context, w http.ResponseWriter, r *http.Request
 	var isAllow bool
 	if consts.IsRbacEnabled() {
 		isAllow = policy.PolicyManager.Allow(true, userCred, consts.GetServiceType(),
-			"quotas", policy.PolicyActionUpdate) == rbacutils.Allow
+			"quotas", policy.PolicyActionUpdate) == rbacutils.AdminAllow
 	} else {
 		isAllow = userCred.IsSystemAdmin()
 	}
@@ -201,7 +201,7 @@ func checkQuotaHanlder(ctx context.Context, w http.ResponseWriter, r *http.Reque
 	isAllow := false
 	if consts.IsRbacEnabled() {
 		isAllow = policy.PolicyManager.Allow(true, userCred, consts.GetServiceType(),
-			policy.PolicyDelegation, policy.PolicyActionGet) == rbacutils.Allow
+			policy.PolicyDelegation, policy.PolicyActionGet) == rbacutils.AdminAllow
 	} else {
 		isAllow = userCred.IsSystemAdmin()
 	}
@@ -211,7 +211,7 @@ func checkQuotaHanlder(ctx context.Context, w http.ResponseWriter, r *http.Reque
 	}
 	if consts.IsRbacEnabled() {
 		if policy.PolicyManager.Allow(true, userCred, consts.GetServiceType(),
-			"quotas", policy.PolicyActionGet) != rbacutils.Allow {
+			"quotas", policy.PolicyActionGet) != rbacutils.AdminAllow {
 			httperrors.ForbiddenError(w, "not allow to query quota")
 			return
 		}
diff --git a/pkg/cloudcommon/db/rbac.go b/pkg/cloudcommon/db/rbac.go
index 15798c9536..d2565e2bd2 100644
--- a/pkg/cloudcommon/db/rbac.go
+++ b/pkg/cloudcommon/db/rbac.go
@@ -28,14 +28,22 @@ func isListRbacAllowedInternal(manager IModelManager, resource string, userCred
 	}
 	result := policy.PolicyManager.Allow(false, userCred, consts.GetServiceType(),
 		resource, policy.PolicyActionList)
-	log.Debugf("allow list for non-admin %s %v ownerId: %s", result, requireAdmin, ownerId)
-	if (result == rbacutils.OwnerAllow && !requireAdmin) || (result == rbacutils.Allow && requireAdmin) {
+
+	switch {
+	case result == rbacutils.GuestAllow:
+		return true
+	case result == rbacutils.UserAllow && userCred != nil && userCred.IsValid():
+		return true
+	case result == rbacutils.OwnerAllow && !requireAdmin:
+		return true
+	case result == rbacutils.AdminAllow && requireAdmin:
 		return true
 	}
+
 	result = policy.PolicyManager.Allow(true, userCred, consts.GetServiceType(),
 		resource, policy.PolicyActionList)
-	log.Debugf("allow list for admin %s %v ownerId: %s", result, requireAdmin, ownerId)
-	return result == rbacutils.Allow
+
+	return result == rbacutils.AdminAllow
 }
 
 func isJointListRbacAllowed(manager IJointModelManager, userCred mcclient.TokenCredential, isAdminMode bool) bool {
@@ -54,16 +62,23 @@ func isClassActionRbacAllowed(manager IModelManager, userCred mcclient.TokenCred
 	} else {
 		requireAdmin = true
 	}
-	// if !requireAdmin {
+
 	result := policy.PolicyManager.Allow(false, userCred, consts.GetServiceType(),
 		manager.KeywordPlural(), action, extra...)
-	if result == rbacutils.Allow || (!requireAdmin && result == rbacutils.OwnerAllow) {
+	switch {
+	case result == rbacutils.GuestAllow:
+		return true
+	case result == rbacutils.UserAllow && userCred != nil && userCred.IsValid():
+		return true
+	case result == rbacutils.OwnerAllow && !requireAdmin:
+		return true
+	case result == rbacutils.AdminAllow && requireAdmin:
 		return true
 	}
-	// }
+
 	result = policy.PolicyManager.Allow(true, userCred, consts.GetServiceType(),
 		manager.KeywordPlural(), action, extra...)
-	return result == rbacutils.Allow
+	return result == rbacutils.AdminAllow
 }
 
 func isObjectRbacAllowed(manager IModelManager, model IModel, userCred mcclient.TokenCredential, action string, extra ...string) bool {
@@ -85,16 +100,22 @@ func isObjectRbacAllowed(manager IModelManager, model IModel, userCred mcclient.
 		requireAdmin = true
 	}
 
-	//if !requireAdmin {
 	result := policy.PolicyManager.Allow(false, userCred, consts.GetServiceType(),
 		manager.KeywordPlural(), action, extra...)
-	if result == rbacutils.Allow || (!requireAdmin && result == rbacutils.OwnerAllow && isOwner) {
+	switch {
+	case result == rbacutils.GuestAllow:
+		return true
+	case result == rbacutils.UserAllow && userCred != nil && userCred.IsValid():
+		return true
+	case result == rbacutils.OwnerAllow && isOwner && !requireAdmin:
+		return true
+	case result == rbacutils.AdminAllow && (requireAdmin || isOwner):
 		return true
 	}
-	//}
+
 	result = policy.PolicyManager.Allow(true, userCred, consts.GetServiceType(),
 		manager.KeywordPlural(), action, extra...)
-	return result == rbacutils.Allow
+	return result == rbacutils.AdminAllow
 }
 
 func isJointObjectRbacAllowed(manager IJointModelManager, item IJointModel, userCred mcclient.TokenCredential, action string, extra ...string) bool {
diff --git a/pkg/cloudcommon/policy/policy.go b/pkg/cloudcommon/policy/policy.go
index c5a14b5e64..489ffbbbbe 100644
--- a/pkg/cloudcommon/policy/policy.go
+++ b/pkg/cloudcommon/policy/policy.go
@@ -34,31 +34,31 @@ var (
 		{
 			Resource: "tasks",
 			Action:   PolicyActionPerform,
-			Result:   rbacutils.Allow,
+			Result:   rbacutils.UserAllow,
 		},
 		{
 			Service:  "compute",
 			Resource: "zones",
 			Action:   PolicyActionList,
-			Result:   rbacutils.Allow,
+			Result:   rbacutils.UserAllow,
 		},
 		{
 			Service:  "compute",
 			Resource: "zones",
 			Action:   PolicyActionGet,
-			Result:   rbacutils.Allow,
+			Result:   rbacutils.UserAllow,
 		},
 		{
 			Service:  "compute",
 			Resource: "cloudregions",
 			Action:   PolicyActionList,
-			Result:   rbacutils.Allow,
+			Result:   rbacutils.UserAllow,
 		},
 		{
 			Service:  "compute",
 			Resource: "cloudregions",
 			Action:   PolicyActionGet,
-			Result:   rbacutils.Allow,
+			Result:   rbacutils.UserAllow,
 		},
 	}
 )
@@ -187,13 +187,13 @@ func (manager *SPolicyManager) Allow(isAdmin bool, userCred mcclient.TokenCreden
 	currentPriv := rbacutils.Deny
 	for _, p := range policies {
 		result := p.Allow(userCredJson, service, resource, action, extra...)
-		if result.IsHigherPrivilege(currentPriv) {
+		if currentPriv.StricterThan(result) {
 			currentPriv = result
 		}
 	}
 	if manager.defaultPolicy != nil {
 		result := manager.defaultPolicy.Allow(userCredJson, service, resource, action, extra...)
-		if result.IsHigherPrivilege(currentPriv) {
+		if currentPriv.StricterThan(result) {
 			currentPriv = result
 		}
 	}
@@ -210,8 +210,10 @@ func (manager *SPolicyManager) explainPolicy(userCred mcclient.TokenCredential,
 	}
 	isAdmin, _ := policySeq[0].Bool()
 	if !consts.IsRbacEnabled() {
-		if !isAdmin || (isAdmin && userCred.IsSystemAdmin()) {
-			return rbacutils.Allow, nil
+		if !isAdmin {
+			return rbacutils.OwnerAllow, nil
+		} else if isAdmin && userCred.IsSystemAdmin() {
+			return rbacutils.AdminAllow, nil
 		} else {
 			return rbacutils.Deny, httperrors.NewForbiddenError("operation not allowed")
 		}
@@ -255,6 +257,10 @@ func (manager *SPolicyManager) ExplainRpc(userCred mcclient.TokenCredential, par
 }
 
 func (manager *SPolicyManager) IsAdminCapable(userCred mcclient.TokenCredential) bool {
+	if ! consts.IsRbacEnabled() && userCred.IsSystemAdmin() {
+		return true
+	}
+
 	userCredJson := userCred.ToJson()
 	for _, p := range manager.adminPolicies {
 		match, _ := conditionparser.Eval(p.Condition, userCredJson)
diff --git a/pkg/compute/usages/handler.go b/pkg/compute/usages/handler.go
index c8b5797364..a1e9a57b9e 100644
--- a/pkg/compute/usages/handler.go
+++ b/pkg/compute/usages/handler.go
@@ -272,7 +272,7 @@ func ReportGeneralUsage(userCred mcclient.TokenCredential, rangeObj db.IStandalo
 
 	if consts.IsRbacEnabled() {
 		if policy.PolicyManager.Allow(true, userCred, consts.GetServiceType(),
-			"usages", policy.PolicyActionGet) == rbacutils.Allow {
+			"usages", policy.PolicyActionGet) == rbacutils.AdminAllow {
 			isAdmin = true
 		}
 	} else {
diff --git a/pkg/mcclient/auth/middleware.go b/pkg/mcclient/auth/middleware.go
index f779b132ff..fa14ac898c 100644
--- a/pkg/mcclient/auth/middleware.go
+++ b/pkg/mcclient/auth/middleware.go
@@ -17,25 +17,38 @@ const (
 )
 
 func Authenticate(f appsrv.FilterHandler) appsrv.FilterHandler {
+	return AuthenticateWithDelayDecision(f, false)
+}
+
+func AuthenticateWithDelayDecision(f appsrv.FilterHandler, delayDecision bool) appsrv.FilterHandler {
 	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 		tokenStr := r.Header.Get(mcclient.AUTH_TOKEN)
 		if len(tokenStr) == 0 {
-			httperrors.UnauthorizedError(w, "Unauthorized")
-			return
+			log.Errorf("no auth_token found!")
+			if !delayDecision {
+				httperrors.UnauthorizedError(w, "Unauthorized")
+				return
+			}
 		}
 		token, err := Verify(tokenStr)
 		if err != nil {
 			log.Errorf("Verify token failed: %s", err)
-			httperrors.UnauthorizedError(w, "InvalidToken")
-			return
+			if !delayDecision {
+				httperrors.UnauthorizedError(w, "InvalidToken")
+				return
+			}
 		}
-		ctx = context.WithValue(ctx, AUTH_TOKEN, token)
+		if token != nil {
+			ctx = context.WithValue(ctx, AUTH_TOKEN, token)
+		}
+
 		if taskId := r.Header.Get(mcclient.TASK_ID); taskId != "" {
 			ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_TASK_ID, taskId)
 		}
 		if taskNotifyUrl := r.Header.Get(mcclient.TASK_NOTIFY_URL); taskNotifyUrl != "" {
 			ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_TASK_NOTIFY_URL, taskNotifyUrl)
 		}
+
 		f(ctx, w, r)
 	}
 }
diff --git a/pkg/util/rbacutils/rabc.go b/pkg/util/rbacutils/rabc.go
index 39530086ee..aedb7716a5 100644
--- a/pkg/util/rbacutils/rabc.go
+++ b/pkg/util/rbacutils/rabc.go
@@ -13,29 +13,29 @@ type TRbacResult string
 const (
 	WILD_MATCH = "*"
 
-	Allow      = TRbacResult("allow")
+	AdminAllow = TRbacResult("allow")
 	OwnerAllow = TRbacResult("owner")
+	UserAllow  = TRbacResult("user")
+	GuestAllow = TRbacResult("guest")
 	Deny       = TRbacResult("deny")
 )
 
-func (result TRbacResult) IsHigherPrivilege(r2 TRbacResult) bool {
-	switch result {
-	case Allow:
-		if r2 == Allow {
-			return false
-		} else {
-			return true
-		}
-	case OwnerAllow:
-		if r2 == Deny {
-			return true
-		} else {
-			return false
-		}
-	case Deny:
-		return false
+var (
+	strictness = map[TRbacResult]int{
+		Deny:       0,
+		AdminAllow: 1,
+		OwnerAllow: 2,
+		UserAllow:  3,
+		GuestAllow: 4,
 	}
-	return false
+)
+
+func (r TRbacResult) Strictness() int {
+	return strictness[r]
+}
+
+func (r1 TRbacResult) StricterThan(r2 TRbacResult) bool {
+	return r1.Strictness() < r2.Strictness()
 }
 
 type SRbacPolicy struct {
@@ -79,6 +79,10 @@ func (rule *SRbacRule) contains(rule2 *SRbacRule) bool {
 	return true
 }
 
+func (rule *SRbacRule) stricterThan(r2 *SRbacRule) bool {
+	return rule.Result.StricterThan(r2.Result)
+}
+
 func (rule *SRbacRule) match(service string, resource string, action string, extra ...string) (bool, int, int) {
 	matched := 0
 	weight := 0
@@ -144,7 +148,9 @@ func (policy *SRbacPolicy) GetMatchRule(service string, resource string, action
 	var matchRule *SRbacRule
 	for i := 0; i < len(policy.Rules); i += 1 {
 		match, matchCnt, weight := policy.Rules[i].match(service, resource, action, extra...)
-		if match && (maxMatchCnt < matchCnt || (maxMatchCnt == matchCnt && minWeight > weight)) {
+		if match && (maxMatchCnt < matchCnt ||
+			(maxMatchCnt == matchCnt && minWeight > weight) ||
+			(maxMatchCnt == matchCnt && minWeight == weight && matchRule.stricterThan(&policy.Rules[i])))  {
 			maxMatchCnt = matchCnt
 			minWeight = weight
 			matchRule = &policy.Rules[i]
@@ -208,10 +214,14 @@ func decode(rules jsonutils.JSONObject, decodeRule SRbacRule, level int) ([]SRba
 		ruleJsonStr := rules.(*jsonutils.JSONString)
 		ruleStr, _ := ruleJsonStr.GetString()
 		switch ruleStr {
-		case string(Allow):
-			decodeRule.Result = Allow
+		case string(AdminAllow):
+			decodeRule.Result = AdminAllow
 		case string(OwnerAllow):
 			decodeRule.Result = OwnerAllow
+		case string(UserAllow):
+			decodeRule.Result = UserAllow
+		case string(GuestAllow):
+			decodeRule.Result = GuestAllow
 		case string(Deny):
 			decodeRule.Result = Deny
 		default:

From c434cd5669fbdeca517b3a41b1505abc34ee97ad Mon Sep 17 00:00:00 2001
From: Qiu Jian <qiujian@yunionyun.com>
Date: Fri, 16 Nov 2018 01:33:18 +0800
Subject: [PATCH 14/57] make rbac default enabled

---
 pkg/cloudcommon/options.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/cloudcommon/options.go b/pkg/cloudcommon/options.go
index 0106a3e6c9..fa7730c4a9 100644
--- a/pkg/cloudcommon/options.go
+++ b/pkg/cloudcommon/options.go
@@ -40,7 +40,7 @@ type Options struct {
 	SslCertfile string `help:"ssl certification file"`
 	SslKeyfile  string `help:"ssl certification key file"`
 
-	EnableRbac                       bool `help:"Switch on Role-based Access Control" default:"false"`
+	EnableRbac                       bool `help:"Switch on Role-based Access Control" default:"true"`
 	RbacDebug                        bool `help:"turn on rbac debug log" default:"false"`
 	RbacPolicySyncPeriodSeconds      int  `help:"policy sync interval in seconds, default 15 minutes" default:"900"`
 	RbacPolicySyncFailedRetrySeconds int  `help:"seconds to wait after a failed sync, default 30 seconds" default:"30"`

From ca73b5442e79a6b7d8af991afb890981e7974939 Mon Sep 17 00:00:00 2001
From: Qiu Jian <qiujian@yunionyun.com>
Date: Fri, 16 Nov 2018 01:35:33 +0800
Subject: [PATCH 15/57] skip cloutpost

---
 pkg/cloutpost/options/options.go  | 21 ------------
 pkg/cloutpost/service/handlers.go |  7 ----
 pkg/cloutpost/service/service.go  | 53 -------------------------------
 3 files changed, 81 deletions(-)
 delete mode 100644 pkg/cloutpost/options/options.go
 delete mode 100644 pkg/cloutpost/service/handlers.go
 delete mode 100644 pkg/cloutpost/service/service.go

diff --git a/pkg/cloutpost/options/options.go b/pkg/cloutpost/options/options.go
deleted file mode 100644
index ec3a78860b..0000000000
--- a/pkg/cloutpost/options/options.go
+++ /dev/null
@@ -1,21 +0,0 @@
-package options
-
-import (
-	"yunion.io/x/onecloud/pkg/cloudcommon"
-	"yunion.io/x/onecloud/pkg/cloudcommon/etcd"
-)
-
-type CloudSyncOptions struct {
-	Provider    string `help:"Public cloud provider" choices:"Aliyun|Azure|Aws|Qcloud"`
-	Environment string `help:"environment of public cloud"`
-	Cloudregion string `help:"region of public cloud"`
-	Zone        string `help:"availability zone of public cloud"`
-
-	etcd.SEtcdOptions
-
-	cloudcommon.Options
-}
-
-var (
-	Options CloudSyncOptions
-)
diff --git a/pkg/cloutpost/service/handlers.go b/pkg/cloutpost/service/handlers.go
deleted file mode 100644
index f1c362799b..0000000000
--- a/pkg/cloutpost/service/handlers.go
+++ /dev/null
@@ -1,7 +0,0 @@
-package service
-
-import "yunion.io/x/onecloud/pkg/appsrv"
-
-func initHandlers(app *appsrv.Application) {
-
-}
diff --git a/pkg/cloutpost/service/service.go b/pkg/cloutpost/service/service.go
deleted file mode 100644
index 243e311ada..0000000000
--- a/pkg/cloutpost/service/service.go
+++ /dev/null
@@ -1,53 +0,0 @@
-package service
-
-import (
-	"os"
-	"yunion.io/x/log"
-
-	"yunion.io/x/onecloud/pkg/cloudcommon"
-	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
-	"yunion.io/x/onecloud/pkg/cloudcommon/etcd"
-	"yunion.io/x/onecloud/pkg/cloudcommon/etcd/models"
-	"yunion.io/x/onecloud/pkg/cloutpost/options"
-)
-
-const (
-	SERVICE_TYPE = "cloutpost"
-)
-
-func StartService() {
-	consts.SetServiceType(SERVICE_TYPE)
-
-	cloudcommon.ParseOptions(&options.Options, &options.Options.Options, os.Args, "cloutpost.conf")
-
-	cloudcommon.InitAuth(&options.Options.Options, func() {
-		log.Infof("Auth complete!!")
-	})
-
-	err := etcd.InitDefaultEtcdClient(&options.Options.SEtcdOptions)
-	if err != nil {
-		log.Fatalf("init etcd fail: %s", err)
-	}
-	defer etcd.CloseDefaultEtcdClient()
-
-	app := cloudcommon.InitApp(&options.Options.Options)
-
-	initHandlers(app)
-
-	err = models.ServiceRegistryManager.Register(
-		app.GetContext(),
-		options.Options.Address,
-		options.Options.Port,
-		options.Options.Provider,
-		options.Options.Environment,
-		options.Options.Cloudregion,
-		options.Options.Zone,
-		SERVICE_TYPE,
-	)
-
-	if err != nil {
-		log.Fatalf("fail to register service %s", err)
-	}
-
-	cloudcommon.ServeForever(app, &options.Options.Options)
-}

From 4034ee517e55bda87a71d81d59d3b770474e3792 Mon Sep 17 00:00:00 2001
From: Qiu Jian <qiujian@yunionyun.com>
Date: Fri, 16 Nov 2018 01:43:01 +0800
Subject: [PATCH 16/57] make fmt

---
 pkg/cloudcommon/db/db_dispatcher.go | 2 +-
 pkg/cloudcommon/policy/policy.go    | 2 +-
 pkg/util/rbacutils/rabc.go          | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go
index 41671468af..c2d72e15f2 100644
--- a/pkg/cloudcommon/db/db_dispatcher.go
+++ b/pkg/cloudcommon/db/db_dispatcher.go
@@ -66,7 +66,7 @@ func (dispatcher *DBModelDispatcher) Filter(f appsrv.FilterHandler) appsrv.Filte
 
 func fetchUserCredential(ctx context.Context) mcclient.TokenCredential {
 	token := auth.FetchUserCredential(ctx)
-	if token == nil && ! consts.IsRbacEnabled() {
+	if token == nil && !consts.IsRbacEnabled() {
 		log.Fatalf("user token credential not found?")
 	}
 	return token
diff --git a/pkg/cloudcommon/policy/policy.go b/pkg/cloudcommon/policy/policy.go
index 489ffbbbbe..7517dfa087 100644
--- a/pkg/cloudcommon/policy/policy.go
+++ b/pkg/cloudcommon/policy/policy.go
@@ -257,7 +257,7 @@ func (manager *SPolicyManager) ExplainRpc(userCred mcclient.TokenCredential, par
 }
 
 func (manager *SPolicyManager) IsAdminCapable(userCred mcclient.TokenCredential) bool {
-	if ! consts.IsRbacEnabled() && userCred.IsSystemAdmin() {
+	if !consts.IsRbacEnabled() && userCred.IsSystemAdmin() {
 		return true
 	}
 
diff --git a/pkg/util/rbacutils/rabc.go b/pkg/util/rbacutils/rabc.go
index aedb7716a5..47c8ff6d69 100644
--- a/pkg/util/rbacutils/rabc.go
+++ b/pkg/util/rbacutils/rabc.go
@@ -150,7 +150,7 @@ func (policy *SRbacPolicy) GetMatchRule(service string, resource string, action
 		match, matchCnt, weight := policy.Rules[i].match(service, resource, action, extra...)
 		if match && (maxMatchCnt < matchCnt ||
 			(maxMatchCnt == matchCnt && minWeight > weight) ||
-			(maxMatchCnt == matchCnt && minWeight == weight && matchRule.stricterThan(&policy.Rules[i])))  {
+			(maxMatchCnt == matchCnt && minWeight == weight && matchRule.stricterThan(&policy.Rules[i]))) {
 			maxMatchCnt = matchCnt
 			minWeight = weight
 			matchRule = &policy.Rules[i]

From 6d27d05ae78de72e58e5d29e76f99330dd68bcac Mon Sep 17 00:00:00 2001
From: Yousong Zhou <zhouyousong@yunionyun.com>
Date: Fri, 16 Nov 2018 03:34:26 +0000
Subject: [PATCH 17/57] networks: add "port_used" field to details listing

---
 pkg/compute/models/networks.go | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/pkg/compute/models/networks.go b/pkg/compute/models/networks.go
index 62737711e5..c1ddf8b93d 100644
--- a/pkg/compute/models/networks.go
+++ b/pkg/compute/models/networks.go
@@ -145,7 +145,12 @@ func (self *SNetwork) ValidateDeleteCondition(ctx context.Context) error {
 }
 
 func (self *SNetwork) GetTotalNicCount() int {
-	return self.GetGuestnicsCount() + self.GetGroupNicsCount() + self.GetBaremetalNicsCount() + self.GetReservedNicsCount()
+	total := self.GetGuestnicsCount() +
+		self.GetGroupNicsCount() +
+		self.GetBaremetalNicsCount() +
+		self.GetReservedNicsCount() +
+		self.GetLoadbalancerIpsCount()
+	return total
 }
 
 func (self *SNetwork) GetGuestnicsCount() int {
@@ -164,6 +169,10 @@ func (self *SNetwork) GetReservedNicsCount() int {
 	return ReservedipManager.Query().Equals("network_id", self.Id).Count()
 }
 
+func (self *SNetwork) GetLoadbalancerIpsCount() int {
+	return LoadbalancernetworkManager.Query().Equals("network_id", self.Id).Count()
+}
+
 func (self *SNetwork) GetUsedAddresses() map[string]bool {
 	used := make(map[string]bool)
 
@@ -818,6 +827,7 @@ func (self *SNetwork) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSOND
 		extra.Add(jsonutils.JSONFalse, "exit")
 	}
 	extra.Add(jsonutils.NewInt(int64(self.getIPRange().AddressCount())), "ports")
+	extra.Add(jsonutils.NewInt(int64(self.GetTotalNicCount())), "ports_used")
 	extra.Add(jsonutils.NewInt(int64(self.GetGuestnicsCount())), "vnics")
 	extra.Add(jsonutils.NewInt(int64(self.GetBaremetalNicsCount())), "bm_vnics")
 	extra.Add(jsonutils.NewInt(int64(self.GetGroupNicsCount())), "group_vnics")

From 4c4b93aa4b67470dabb2fccdc2400f64cc315de9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= <quxuan@yunionyun.com>
Date: Fri, 16 Nov 2018 14:53:06 +0800
Subject: [PATCH 18/57] =?UTF-8?q?=E9=81=BF=E5=85=8D=E5=85=AC=E6=9C=89?=
 =?UTF-8?q?=E4=BA=91=E8=B4=A6=E5=8F=B7=E8=AE=A4=E8=AF=81=E5=A4=B1=E8=B4=A5?=
 =?UTF-8?q?=E5=90=8E=E8=BF=94=E5=9B=9E=E7=B3=BB=E7=BB=9F=E7=99=BB=E5=BD=95?=
 =?UTF-8?q?=E7=95=8C=E9=9D=A2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/compute/models/cloudaccounts.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go
index 4441e2a808..0655cac3e1 100644
--- a/pkg/compute/models/cloudaccounts.go
+++ b/pkg/compute/models/cloudaccounts.go
@@ -147,7 +147,7 @@ func (manager *SCloudaccountManager) ValidateCreateData(ctx context.Context, use
 		if err == cloudprovider.ErrNoSuchProvder {
 			return nil, httperrors.NewResourceNotFoundError("no such provider %s", provider)
 		}
-		return nil, httperrors.NewInvalidCredentialError("invalid cloud account info")
+		return nil, httperrors.NewInputParameterError("invalid cloud account info")
 	}
 
 	return manager.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)

From 2e868ed7c118cce8b18f387e19580288b5429ce8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= <quxuan@yunionyun.com>
Date: Fri, 16 Nov 2018 15:28:32 +0800
Subject: [PATCH 19/57] =?UTF-8?q?=E9=81=BF=E5=85=8D=E5=85=AC=E6=9C=89?=
 =?UTF-8?q?=E4=BA=91=E6=89=BE=E4=B8=8D=E5=88=B0=E5=AF=B9=E5=BA=94=E7=9A=84?=
 =?UTF-8?q?cloudregion?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/compute/guestdrivers/managedvirtual.go | 7 ++++++-
 pkg/compute/models/snapshots.go            | 8 ++++++++
 pkg/util/aliyun/snapshot.go                | 2 +-
 pkg/util/azure/snapshot.go                 | 4 +++-
 4 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/pkg/compute/guestdrivers/managedvirtual.go b/pkg/compute/guestdrivers/managedvirtual.go
index f3e7458173..36d1f7f73d 100644
--- a/pkg/compute/guestdrivers/managedvirtual.go
+++ b/pkg/compute/guestdrivers/managedvirtual.go
@@ -369,7 +369,12 @@ func (self *SManagedVirtualizedGuestDriver) RequestDiskSnapshot(ctx context.Cont
 		res := jsonutils.NewDict()
 		res.Set("snapshot_id", jsonutils.NewString(cloudSnapshot.GetId()))
 		res.Set("manager_id", jsonutils.NewString(cloudSnapshot.GetManagerId()))
-		res.Set("cloudregion_id", jsonutils.NewString(cloudSnapshot.GetRegionId()))
+		extId := cloudSnapshot.GetRegionId()
+		cloudregion, err := models.CloudregionManager.FetchByExternalId(extId)
+		if err != nil {
+			return nil, err
+		}
+		res.Set("cloudregion_id", jsonutils.NewString(cloudregion.GetId()))
 		return res, nil
 	})
 	return nil
diff --git a/pkg/compute/models/snapshots.go b/pkg/compute/models/snapshots.go
index 491381fe77..bc0e70fef6 100644
--- a/pkg/compute/models/snapshots.go
+++ b/pkg/compute/models/snapshots.go
@@ -439,8 +439,16 @@ func totalSnapshotCount(projectId string) int {
 
 func (self *SSnapshot) SyncWithCloudSnapshot(userCred mcclient.TokenCredential, ext cloudprovider.ICloudSnapshot) error {
 	_, err := self.GetModelManager().TableSpec().Update(self, func() error {
+		self.Name = ext.GetName()
 		self.Status = ext.GetStatus()
 		self.DiskType = ext.GetDiskType()
+
+		cloudregionId := ext.GetRegionId()
+		region, err := CloudregionManager.FetchByExternalId(cloudregionId)
+		if err != nil {
+			return err
+		}
+		self.CloudregionId = region.GetId()
 		return nil
 	})
 	if err != nil {
diff --git a/pkg/util/aliyun/snapshot.go b/pkg/util/aliyun/snapshot.go
index e0cada0372..515115e8fb 100644
--- a/pkg/util/aliyun/snapshot.go
+++ b/pkg/util/aliyun/snapshot.go
@@ -56,7 +56,7 @@ func (self *SSnapshot) GetManagerId() string {
 }
 
 func (self *SSnapshot) GetRegionId() string {
-	return self.region.GetId()
+	return self.region.GetGlobalId()
 }
 
 func (self *SSnapshot) GetSize() int32 {
diff --git a/pkg/util/azure/snapshot.go b/pkg/util/azure/snapshot.go
index 429b1451c8..8804fb7832 100644
--- a/pkg/util/azure/snapshot.go
+++ b/pkg/util/azure/snapshot.go
@@ -154,9 +154,11 @@ func (self *SRegion) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) {
 	classicSnapshots = append(classicSnapshots, _classicSnapshots...)
 	isnapshots := make([]cloudprovider.ICloudSnapshot, len(snapshots)+len(classicSnapshots))
 	for i := 0; i < len(snapshots); i++ {
+		snapshots[i].region = self
 		isnapshots[i] = &snapshots[i]
 	}
 	for i := 0; i < len(classicSnapshots); i++ {
+		classicSnapshots[i].region = self
 		isnapshots[len(snapshots)+i] = &classicSnapshots[i]
 	}
 	return isnapshots, nil
@@ -171,7 +173,7 @@ func (self *SSnapshot) GetManagerId() string {
 }
 
 func (self *SSnapshot) GetRegionId() string {
-	return self.region.GetId()
+	return self.region.GetGlobalId()
 }
 
 func (self *SSnapshot) GetDiskType() string {

From de45f88830a3d549ca45167876c2a7c75b186acb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= <quxuan@yunionyun.com>
Date: Fri, 16 Nov 2018 17:16:58 +0800
Subject: [PATCH 20/57] =?UTF-8?q?Azure=E4=B8=BB=E6=9C=BA=E5=90=8D=E7=A7=B0?=
 =?UTF-8?q?=E5=B8=A6=E4=B8=8Aregion=E4=BF=A1=E6=81=AF,=E9=81=BF=E5=85=8D?=
 =?UTF-8?q?=E5=90=8D=E7=A7=B0=E9=87=8D=E5=A4=8D?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/util/azure/classic_host.go | 2 +-
 pkg/util/azure/host.go         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/util/azure/classic_host.go b/pkg/util/azure/classic_host.go
index e932ec8d35..89ddfc3373 100644
--- a/pkg/util/azure/classic_host.go
+++ b/pkg/util/azure/classic_host.go
@@ -26,7 +26,7 @@ func (self *SClassicHost) GetId() string {
 }
 
 func (self *SClassicHost) GetName() string {
-	return fmt.Sprintf("%s-classic", self.zone.region.client.subscriptionId)
+	return fmt.Sprintf("%s/%s-classic", self.zone.region.GetName(), self.zone.region.client.subscriptionId)
 }
 
 func (self *SClassicHost) GetGlobalId() string {
diff --git a/pkg/util/azure/host.go b/pkg/util/azure/host.go
index 2fe2f279cc..147d5a8dd0 100644
--- a/pkg/util/azure/host.go
+++ b/pkg/util/azure/host.go
@@ -23,7 +23,7 @@ func (self *SHost) GetId() string {
 }
 
 func (self *SHost) GetName() string {
-	return fmt.Sprintf("%s", self.zone.region.client.subscriptionId)
+	return fmt.Sprintf("%s/%s", self.zone.region.GetName(), self.zone.region.client.subscriptionId)
 }
 
 func (self *SHost) GetGlobalId() string {

From 110fdc1d24cc0816bcc4b406340751ea70b4c0be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= <quxuan@yunionyun.com>
Date: Fri, 16 Nov 2018 17:23:03 +0800
Subject: [PATCH 21/57] =?UTF-8?q?=E5=8E=BB=E9=99=A4=E4=B8=BB=E6=9C=BA?=
 =?UTF-8?q?=E5=90=8D=E4=B8=AD=E6=96=87=E5=AD=97=E7=AC=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/util/azure/classic_host.go | 2 +-
 pkg/util/azure/host.go         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/util/azure/classic_host.go b/pkg/util/azure/classic_host.go
index 89ddfc3373..d20ed16f4e 100644
--- a/pkg/util/azure/classic_host.go
+++ b/pkg/util/azure/classic_host.go
@@ -26,7 +26,7 @@ func (self *SClassicHost) GetId() string {
 }
 
 func (self *SClassicHost) GetName() string {
-	return fmt.Sprintf("%s/%s-classic", self.zone.region.GetName(), self.zone.region.client.subscriptionId)
+	return fmt.Sprintf("%s/%s-classic", self.zone.region.GetGlobalId(), self.zone.region.client.subscriptionId)
 }
 
 func (self *SClassicHost) GetGlobalId() string {
diff --git a/pkg/util/azure/host.go b/pkg/util/azure/host.go
index 147d5a8dd0..089e714fd3 100644
--- a/pkg/util/azure/host.go
+++ b/pkg/util/azure/host.go
@@ -23,7 +23,7 @@ func (self *SHost) GetId() string {
 }
 
 func (self *SHost) GetName() string {
-	return fmt.Sprintf("%s/%s", self.zone.region.GetName(), self.zone.region.client.subscriptionId)
+	return fmt.Sprintf("%s/%s", self.zone.region.GetGlobalId(), self.zone.region.client.subscriptionId)
 }
 
 func (self *SHost) GetGlobalId() string {

From 503bec34c86f2b00ed54736ec988ab5a4f11bc9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= <quxuan@yunionyun.com>
Date: Fri, 16 Nov 2018 16:55:17 +0800
Subject: [PATCH 22/57] =?UTF-8?q?=E5=8E=BB=E9=99=A4=E4=B8=8D=E9=9C=80?=
 =?UTF-8?q?=E8=A6=81=E6=8E=A5=E5=8F=A3?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/cloudprovider/resources.go                |  2 --
 pkg/compute/guestdrivers/managedvirtual.go    |  7 -------
 pkg/compute/models/snapshots.go               | 21 ++++++++-----------
 pkg/compute/tasks/guest_disk_snapshot_task.go | 13 +++---------
 pkg/util/aliyun/snapshot.go                   |  8 -------
 pkg/util/azure/snapshot.go                    |  8 -------
 6 files changed, 12 insertions(+), 47 deletions(-)

diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go
index 0935ffb38f..2a9df62487 100644
--- a/pkg/cloudprovider/resources.go
+++ b/pkg/cloudprovider/resources.go
@@ -249,12 +249,10 @@ type ICloudDisk interface {
 
 type ICloudSnapshot interface {
 	ICloudResource
-	GetManagerId() string
 	GetSize() int32
 	GetDiskId() string
 	GetDiskType() string
 	Delete() error
-	GetRegionId() string
 }
 
 type ICloudVpc interface {
diff --git a/pkg/compute/guestdrivers/managedvirtual.go b/pkg/compute/guestdrivers/managedvirtual.go
index 36d1f7f73d..30dbc10618 100644
--- a/pkg/compute/guestdrivers/managedvirtual.go
+++ b/pkg/compute/guestdrivers/managedvirtual.go
@@ -368,13 +368,6 @@ func (self *SManagedVirtualizedGuestDriver) RequestDiskSnapshot(ctx context.Cont
 		}
 		res := jsonutils.NewDict()
 		res.Set("snapshot_id", jsonutils.NewString(cloudSnapshot.GetId()))
-		res.Set("manager_id", jsonutils.NewString(cloudSnapshot.GetManagerId()))
-		extId := cloudSnapshot.GetRegionId()
-		cloudregion, err := models.CloudregionManager.FetchByExternalId(extId)
-		if err != nil {
-			return nil, err
-		}
-		res.Set("cloudregion_id", jsonutils.NewString(cloudregion.GetId()))
 		return res, nil
 	})
 	return nil
diff --git a/pkg/compute/models/snapshots.go b/pkg/compute/models/snapshots.go
index bc0e70fef6..367f05bb71 100644
--- a/pkg/compute/models/snapshots.go
+++ b/pkg/compute/models/snapshots.go
@@ -273,6 +273,7 @@ func (self *SSnapshotManager) CreateSnapshot(ctx context.Context, userCred mccli
 		return nil, err
 	}
 	disk := iDisk.(*SDisk)
+	storage := disk.GetStorage()
 	snapshot := &SSnapshot{}
 	snapshot.SetModelManager(self)
 	snapshot.ProjectId = userCred.GetProjectId()
@@ -282,6 +283,8 @@ func (self *SSnapshotManager) CreateSnapshot(ctx context.Context, userCred mccli
 	snapshot.DiskType = disk.DiskType
 	snapshot.Location = location
 	snapshot.CreatedBy = createdBy
+	snapshot.ManagerId = storage.ManagerId
+	snapshot.CloudregionId = storage.getZone().GetRegion().GetId()
 	snapshot.Name = name
 	snapshot.Status = SNAPSHOT_CREATING
 	err = SnapshotManager.TableSpec().Insert(snapshot)
@@ -437,18 +440,12 @@ func totalSnapshotCount(projectId string) int {
 	return count
 }
 
-func (self *SSnapshot) SyncWithCloudSnapshot(userCred mcclient.TokenCredential, ext cloudprovider.ICloudSnapshot) error {
+func (self *SSnapshot) SyncWithCloudSnapshot(userCred mcclient.TokenCredential, ext cloudprovider.ICloudSnapshot, region *SCloudregion) error {
 	_, err := self.GetModelManager().TableSpec().Update(self, func() error {
 		self.Name = ext.GetName()
 		self.Status = ext.GetStatus()
 		self.DiskType = ext.GetDiskType()
-
-		cloudregionId := ext.GetRegionId()
-		region, err := CloudregionManager.FetchByExternalId(cloudregionId)
-		if err != nil {
-			return err
-		}
-		self.CloudregionId = region.GetId()
+		self.CloudregionId = region.Id
 		return nil
 	})
 	if err != nil {
@@ -457,7 +454,7 @@ func (self *SSnapshot) SyncWithCloudSnapshot(userCred mcclient.TokenCredential,
 	return err
 }
 
-func (manager *SSnapshotManager) newFromCloudSnapshot(userCred mcclient.TokenCredential, extSnapshot cloudprovider.ICloudSnapshot, region *SCloudregion) (*SSnapshot, error) {
+func (manager *SSnapshotManager) newFromCloudSnapshot(userCred mcclient.TokenCredential, extSnapshot cloudprovider.ICloudSnapshot, region *SCloudregion, provider *SCloudprovider) (*SSnapshot, error) {
 	snapshot := SSnapshot{}
 	snapshot.SetModelManager(manager)
 
@@ -475,7 +472,7 @@ func (manager *SSnapshotManager) newFromCloudSnapshot(userCred mcclient.TokenCre
 
 	snapshot.DiskType = extSnapshot.GetDiskType()
 	snapshot.Size = int(extSnapshot.GetSize()) * 1024
-	snapshot.ManagerId = extSnapshot.GetManagerId()
+	snapshot.ManagerId = provider.Id
 	snapshot.CloudregionId = region.Id
 
 	snapshot.ProjectId = userCred.GetProjectId()
@@ -526,7 +523,7 @@ func (manager *SSnapshotManager) SyncSnapshots(ctx context.Context, userCred mcc
 		}
 	}
 	for i := 0; i < len(commondb); i += 1 {
-		err = commondb[i].SyncWithCloudSnapshot(userCred, commonext[i])
+		err = commondb[i].SyncWithCloudSnapshot(userCred, commonext[i], region)
 		if err != nil {
 			syncResult.UpdateError(err)
 		} else {
@@ -534,7 +531,7 @@ func (manager *SSnapshotManager) SyncSnapshots(ctx context.Context, userCred mcc
 		}
 	}
 	for i := 0; i < len(added); i += 1 {
-		_, err := manager.newFromCloudSnapshot(userCred, added[i], region)
+		_, err := manager.newFromCloudSnapshot(userCred, added[i], region, provider)
 		if err != nil {
 			syncResult.AddError(err)
 		} else {
diff --git a/pkg/compute/tasks/guest_disk_snapshot_task.go b/pkg/compute/tasks/guest_disk_snapshot_task.go
index 9c9e7aa7ea..4f5f230ef2 100644
--- a/pkg/compute/tasks/guest_disk_snapshot_task.go
+++ b/pkg/compute/tasks/guest_disk_snapshot_task.go
@@ -52,32 +52,25 @@ func (self *GuestDiskSnapshotTask) DoDiskSnapshot(ctx context.Context, guest *mo
 
 func (self *GuestDiskSnapshotTask) OnDiskSnapshotComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
 	res := data.(*jsonutils.JSONDict)
+	snapshotId, _ := self.Params.GetString("snapshot_id")
+	iSnapshot, _ := models.SnapshotManager.FetchById(snapshotId)
+	snapshot := iSnapshot.(*models.SSnapshot)
 	if guest.Hypervisor == models.HYPERVISOR_KVM {
 		location, err := res.GetString("location")
 		if err != nil {
 			log.Infof("OnDiskSnapshotComplete called with data no location")
 			return
 		}
-		snapshotId, _ := self.Params.GetString("snapshot_id")
-		iSnapshot, _ := models.SnapshotManager.FetchById(snapshotId)
-		snapshot := iSnapshot.(*models.SSnapshot)
 		models.SnapshotManager.TableSpec().Update(snapshot, func() error {
 			snapshot.Location = location
 			snapshot.Status = models.SNAPSHOT_READY
 			return nil
 		})
 	} else {
-		snapshotId, _ := self.Params.GetString("snapshot_id")
-		iSnapshot, _ := models.SnapshotManager.FetchById(snapshotId)
-		snapshot := iSnapshot.(*models.SSnapshot)
 		extSnapshotId, _ := data.GetString("snapshot_id")
-		cloudregionId, _ := data.GetString("cloudregion_id")
-		managerId, _ := data.GetString("manager_id")
 		models.SnapshotManager.TableSpec().Update(snapshot, func() error {
-			snapshot.CloudregionId = cloudregionId
 			snapshot.ExternalId = extSnapshotId
 			snapshot.Status = models.SNAPSHOT_READY
-			snapshot.ManagerId = managerId
 			return nil
 		})
 	}
diff --git a/pkg/util/aliyun/snapshot.go b/pkg/util/aliyun/snapshot.go
index 515115e8fb..87d9523970 100644
--- a/pkg/util/aliyun/snapshot.go
+++ b/pkg/util/aliyun/snapshot.go
@@ -51,14 +51,6 @@ func (self *SSnapshot) GetStatus() string {
 	}
 }
 
-func (self *SSnapshot) GetManagerId() string {
-	return self.region.client.providerId
-}
-
-func (self *SSnapshot) GetRegionId() string {
-	return self.region.GetGlobalId()
-}
-
 func (self *SSnapshot) GetSize() int32 {
 	return self.SourceDiskSize
 }
diff --git a/pkg/util/azure/snapshot.go b/pkg/util/azure/snapshot.go
index 8804fb7832..ccc78658bd 100644
--- a/pkg/util/azure/snapshot.go
+++ b/pkg/util/azure/snapshot.go
@@ -168,14 +168,6 @@ func (self *SSnapshot) GetDiskId() string {
 	return self.Properties.CreationData.SourceResourceID
 }
 
-func (self *SSnapshot) GetManagerId() string {
-	return self.region.client.providerId
-}
-
-func (self *SSnapshot) GetRegionId() string {
-	return self.region.GetGlobalId()
-}
-
 func (self *SSnapshot) GetDiskType() string {
 	return ""
 }

From f65e031597ad86224320a699850022807c52feab Mon Sep 17 00:00:00 2001
From: Zexi Li <lizexi@yunionyun.com>
Date: Fri, 16 Nov 2018 18:43:50 +0800
Subject: [PATCH 23/57] usage: fix schedtag usage panic

---
 pkg/compute/models/usage.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/compute/models/usage.go b/pkg/compute/models/usage.go
index e45bce93de..d3bc2be701 100644
--- a/pkg/compute/models/usage.go
+++ b/pkg/compute/models/usage.go
@@ -37,7 +37,7 @@ func AttachUsageQuery(
 	case "vcenter":
 		q = q.Filter(sqlchemy.Equals(hosts.Field("manager_id"), rangeObjId))
 	case "schedtag":
-		aggHosts := SchedtagManager.Query().SubQuery()
+		aggHosts := HostschedtagManager.Query().SubQuery()
 		q = q.Join(aggHosts, sqlchemy.AND(
 			sqlchemy.Equals(hosts.Field("id"), aggHosts.Field("host_id")),
 			sqlchemy.IsFalse(aggHosts.Field("deleted")))).

From 76c60926c6ee6f77fe9303ee1c6fb38b1319783f Mon Sep 17 00:00:00 2001
From: Yousong Zhou <zhouyousong@yunionyun.com>
Date: Fri, 16 Nov 2018 12:17:30 +0000
Subject: [PATCH 24/57] loadbalancerlisteners: clear out acl_id when
 acl_status=off

---
 pkg/compute/models/loadbalancerlisteners.go | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/pkg/compute/models/loadbalancerlisteners.go b/pkg/compute/models/loadbalancerlisteners.go
index 137f594698..6f91f5860e 100644
--- a/pkg/compute/models/loadbalancerlisteners.go
+++ b/pkg/compute/models/loadbalancerlisteners.go
@@ -295,6 +295,7 @@ func (lblis *SLoadbalancerListener) ValidateUpdateData(ctx context.Context, user
 	ownerProjId := lblis.GetOwnerProjectId()
 	backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", ownerProjId)
 	aclStatusV := validators.NewStringChoicesValidator("acl_status", LB_BOOL_VALUES)
+	aclStatusV.Default(lblis.AclStatus)
 	aclTypeV := validators.NewStringChoicesValidator("acl_type", LB_ACL_TYPES)
 	aclV := validators.NewModelIdOrNameValidator("acl", "loadbalanceracl", ownerProjId)
 	certV := validators.NewModelIdOrNameValidator("certificate", "loadbalancercertificate", ownerProjId)
@@ -344,6 +345,12 @@ func (lblis *SLoadbalancerListener) ValidateUpdateData(ctx context.Context, user
 			return nil, err
 		}
 	}
+	{
+		// clear out acl_id when acl_status is off
+		if aclStatusV.Value == LB_BOOL_OFF {
+			data.Set("acl_id", jsonutils.NewString(""))
+		}
+	}
 	{
 		if backendGroup, ok := backendGroupV.Model.(*SLoadbalancerBackendGroup); ok && backendGroup.LoadbalancerId != lblis.LoadbalancerId {
 			return nil, httperrors.NewInputParameterError("backend group %s(%s) belongs to loadbalancer %s instead of %s",

From 5568a009d18656804c8249b1b21b617fdb7e119c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= <quxuan@yunionyun.com>
Date: Fri, 16 Nov 2018 20:51:29 +0800
Subject: [PATCH 25/57] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=86=B2=E7=AA=81?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/util/azure/host.go | 51 +++++++++++++++++++++++++++++++++---------
 1 file changed, 40 insertions(+), 11 deletions(-)

diff --git a/pkg/util/azure/host.go b/pkg/util/azure/host.go
index 6c32c3e25e..4dd6fff0ee 100644
--- a/pkg/util/azure/host.go
+++ b/pkg/util/azure/host.go
@@ -42,18 +42,47 @@ func (self *SHost) Refresh() error {
 	return nil
 }
 
-func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, networkId string, ipAddr string, desc string, passwd string, storageType string, diskSizes []int, publicKey string, secgroupId string, userData string) (cloudprovider.ICloudVM, error) {
-	nicId := ""
-	if net := self.zone.getNetworkById(networkId); net == nil {
-		return nil, fmt.Errorf("invalid network ID %s", networkId)
-	} else if nic, err := self.zone.region.CreateNetworkInterface(fmt.Sprintf("%s-ipconfig", name), ipAddr, net.GetId(), secgroupId); err != nil {
-		return nil, err
-	} else {
-		nicId = nic.ID
-	}
-	vmId, err := self._createVM(name, imgId, int32(sysDiskSize), cpu, memMB, nicId, ipAddr, desc, passwd, storageType, diskSizes, publicKey, userData)
+func (self *SHost) searchNetorkInterface(IPAddr string, networkId string, secgroupId string) (*SInstanceNic, error) {
+	interfaces, err := self.zone.region.GetNetworkInterfaces()
 	if err != nil {
-		self.zone.region.DeleteNetworkInterface(nicId)
+		return nil, err
+	}
+	for i, nic := range interfaces {
+		for _, ipConf := range nic.Properties.IPConfigurations {
+			if ipConf.Properties.PrivateIPAddress == IPAddr && networkId == ipConf.Properties.Subnet.ID && ipConf.Properties.PrivateIPAllocationMethod == "Static" {
+				if nic.Properties.NetworkSecurityGroup == nil || nic.Properties.NetworkSecurityGroup.ID != secgroupId {
+					nic.Properties.NetworkSecurityGroup = &SSecurityGroup{ID: secgroupId}
+					if err := self.zone.region.client.Update(jsonutils.Marshal(nic), nil); err != nil {
+						log.Errorf("assign secgroup %s for nic %s failed %d")
+						return nil, err
+					}
+				}
+				return &interfaces[i], nil
+			}
+		}
+	}
+	return nil, cloudprovider.ErrNotFound
+}
+
+func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, networkId string, ipAddr string, desc string, passwd string, storageType string, diskSizes []int, publicKey string, secgroupId string, userData string) (cloudprovider.ICloudVM, error) {
+	net := self.zone.getNetworkById(networkId)
+	if net == nil {
+		return nil, fmt.Errorf("invalid network ID %s", networkId)
+	}
+	nic, err := self.searchNetorkInterface(ipAddr, net.GetId(), secgroupId)
+	if err != nil {
+		if err == cloudprovider.ErrNotFound {
+			nic, err = self.zone.region.CreateNetworkInterface(fmt.Sprintf("%s-ipconfig", name), ipAddr, net.GetId(), secgroupId)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			return nil, err
+		}
+	}
+	vmId, err := self._createVM(name, imgId, int32(sysDiskSize), cpu, memMB, nic.ID, ipAddr, desc, passwd, storageType, diskSizes, publicKey, userData)
+	if err != nil {
+		self.zone.region.DeleteNetworkInterface(nic.ID)
 		return nil, err
 	}
 	if vm, err := self.zone.region.GetInstance(vmId); err != nil {

From 54dda9496ffdefc5f331d1b0d8bd3c203542583b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= <quxuan@yunionyun.com>
Date: Fri, 16 Nov 2018 20:42:30 +0800
Subject: [PATCH 26/57] =?UTF-8?q?=E9=87=8D=E5=A4=8D=E4=BD=BF=E7=94=A8?=
 =?UTF-8?q?=E5=B7=B2=E5=88=86=E9=85=8D=E7=9A=84=E6=9C=AA=E4=BD=BF=E7=94=A8?=
 =?UTF-8?q?=E7=BD=91=E5=8D=A1?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/util/azure/host.go | 52 +++++++++++++++++++++++++++++++++---------
 1 file changed, 41 insertions(+), 11 deletions(-)

diff --git a/pkg/util/azure/host.go b/pkg/util/azure/host.go
index 089e714fd3..a32a5ca6df 100644
--- a/pkg/util/azure/host.go
+++ b/pkg/util/azure/host.go
@@ -42,18 +42,48 @@ func (self *SHost) Refresh() error {
 	return nil
 }
 
-func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, networkId string, ipAddr string, desc string, passwd string, storageType string, diskSizes []int, publicKey string, secgroupId string) (cloudprovider.ICloudVM, error) {
-	nicId := ""
-	if net := self.zone.getNetworkById(networkId); net == nil {
-		return nil, fmt.Errorf("invalid network ID %s", networkId)
-	} else if nic, err := self.zone.region.CreateNetworkInterface(fmt.Sprintf("%s-ipconfig", name), ipAddr, net.GetId(), secgroupId); err != nil {
-		return nil, err
-	} else {
-		nicId = nic.ID
-	}
-	vmId, err := self._createVM(name, imgId, int32(sysDiskSize), cpu, memMB, nicId, ipAddr, desc, passwd, storageType, diskSizes, publicKey)
+func (self *SHost) searchNetorkInterface(IPAddr string, networkId string, secgroupId string) (*SInstanceNic, error) {
+	interfaces, err := self.zone.region.GetNetworkInterfaces()
 	if err != nil {
-		self.zone.region.DeleteNetworkInterface(nicId)
+		return nil, err
+	}
+	for i, nic := range interfaces {
+		for _, ipConf := range nic.Properties.IPConfigurations {
+			if ipConf.Properties.PrivateIPAddress == IPAddr && networkId == ipConf.Properties.Subnet.ID && ipConf.Properties.PrivateIPAllocationMethod == "Static" {
+				if nic.Properties.NetworkSecurityGroup == nil || nic.Properties.NetworkSecurityGroup.ID != secgroupId {
+					nic.Properties.NetworkSecurityGroup = &SSecurityGroup{ID: secgroupId}
+					if err := self.zone.region.client.Update(jsonutils.Marshal(nic), nil); err != nil {
+						log.Errorf("assign secgroup %s for nic %s failed %d")
+						return nil, err
+					}
+				}
+				return &interfaces[i], nil
+			}
+		}
+	}
+	return nil, cloudprovider.ErrNotFound
+}
+
+func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, networkId string, ipAddr string, desc string, passwd string, storageType string, diskSizes []int, publicKey string, secgroupId string) (cloudprovider.ICloudVM, error) {
+	net := self.zone.getNetworkById(networkId)
+	if net == nil {
+		return nil, fmt.Errorf("invalid network ID %s", networkId)
+	}
+	nic, err := self.searchNetorkInterface(ipAddr, net.GetId(), secgroupId)
+	if err != nil {
+		if err == cloudprovider.ErrNotFound {
+			nic, err = self.zone.region.CreateNetworkInterface(fmt.Sprintf("%s-ipconfig", name), ipAddr, net.GetId(), secgroupId)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			return nil, err
+		}
+	}
+
+	vmId, err := self._createVM(name, imgId, int32(sysDiskSize), cpu, memMB, nic.ID, ipAddr, desc, passwd, storageType, diskSizes, publicKey)
+	if err != nil {
+		self.zone.region.DeleteNetworkInterface(nic.ID)
 		return nil, err
 	}
 	if vm, err := self.zone.region.GetInstance(vmId); err != nil {

From 97162f59658467c76f85d06af4c916567638df56 Mon Sep 17 00:00:00 2001
From: Yousong Zhou <zhouyousong@yunionyun.com>
Date: Fri, 16 Nov 2018 14:33:50 +0000
Subject: [PATCH 27/57] loadbalancerlisteners: more complete acl fields
 validation

---
 pkg/compute/models/loadbalancerlisteners.go | 36 ++++++++++++---------
 1 file changed, 21 insertions(+), 15 deletions(-)

diff --git a/pkg/compute/models/loadbalancerlisteners.go b/pkg/compute/models/loadbalancerlisteners.go
index 6f91f5860e..607fefa3f7 100644
--- a/pkg/compute/models/loadbalancerlisteners.go
+++ b/pkg/compute/models/loadbalancerlisteners.go
@@ -260,16 +260,8 @@ func (man *SLoadbalancerListenerManager) ValidateCreateData(ctx context.Context,
 			}
 		}
 	}
-	{
-		if aclStatusV.Value == LB_BOOL_ON {
-			acl := aclV.Model.(*SLoadbalancerAcl)
-			if acl == nil {
-				return nil, fmt.Errorf("missing acl")
-			}
-			if len(aclTypeV.Value) == 0 {
-				return nil, fmt.Errorf("missing acl_type")
-			}
-		}
+	if err := man.validateAcl(aclStatusV, aclTypeV, aclV, data); err != nil {
+		return nil, err
 	}
 	return man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)
 }
@@ -287,6 +279,20 @@ func (man *SLoadbalancerListenerManager) checkTypeV(listenerType string) validat
 	return nil
 }
 
+func (man *SLoadbalancerListenerManager) validateAcl(aclStatusV *validators.ValidatorStringChoices, aclTypeV *validators.ValidatorStringChoices, aclV *validators.ValidatorModelIdOrName, data *jsonutils.JSONDict) error {
+	if aclStatusV.Value == LB_BOOL_ON {
+		if aclV.Model == nil {
+			return httperrors.NewInputParameterError("missing acl")
+		}
+		if len(aclTypeV.Value) == 0 {
+			return httperrors.NewInputParameterError("missing acl_type")
+		}
+	} else {
+		data.Set("acl_id", jsonutils.NewString(""))
+	}
+	return nil
+}
+
 func (lblis *SLoadbalancerListener) AllowPerformStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
 	return lblis.IsOwner(userCred) || userCred.IsSystemAdmin()
 }
@@ -297,6 +303,9 @@ func (lblis *SLoadbalancerListener) ValidateUpdateData(ctx context.Context, user
 	aclStatusV := validators.NewStringChoicesValidator("acl_status", LB_BOOL_VALUES)
 	aclStatusV.Default(lblis.AclStatus)
 	aclTypeV := validators.NewStringChoicesValidator("acl_type", LB_ACL_TYPES)
+	if LB_ACL_TYPES.Has(lblis.AclType) {
+		aclTypeV.Default(lblis.AclType)
+	}
 	aclV := validators.NewModelIdOrNameValidator("acl", "loadbalanceracl", ownerProjId)
 	certV := validators.NewModelIdOrNameValidator("certificate", "loadbalancercertificate", ownerProjId)
 	tlsCipherPolicyV := validators.NewStringChoicesValidator("tls_cipher_policy", LB_TLS_CIPHER_POLICIES).Default(LB_TLS_CIPHER_POLICY_1_2)
@@ -345,11 +354,8 @@ func (lblis *SLoadbalancerListener) ValidateUpdateData(ctx context.Context, user
 			return nil, err
 		}
 	}
-	{
-		// clear out acl_id when acl_status is off
-		if aclStatusV.Value == LB_BOOL_OFF {
-			data.Set("acl_id", jsonutils.NewString(""))
-		}
+	if err := LoadbalancerListenerManager.validateAcl(aclStatusV, aclTypeV, aclV, data); err != nil {
+		return nil, err
 	}
 	{
 		if backendGroup, ok := backendGroupV.Model.(*SLoadbalancerBackendGroup); ok && backendGroup.LoadbalancerId != lblis.LoadbalancerId {

From ddbbb24fcf4bb95fa32eb4f33fd989d198a36ed8 Mon Sep 17 00:00:00 2001
From: Yousong Zhou <zhouyousong@yunionyun.com>
Date: Fri, 16 Nov 2018 14:51:07 +0000
Subject: [PATCH 28/57] loadbalancers: exclude pending_deleted when checking
 uniqueness

---
 pkg/compute/models/loadbalancerlistenerrules.go | 1 +
 pkg/compute/models/loadbalancerlisteners.go     | 1 +
 2 files changed, 2 insertions(+)

diff --git a/pkg/compute/models/loadbalancerlistenerrules.go b/pkg/compute/models/loadbalancerlistenerrules.go
index e4f5455e0c..71be2e4b89 100644
--- a/pkg/compute/models/loadbalancerlistenerrules.go
+++ b/pkg/compute/models/loadbalancerlistenerrules.go
@@ -43,6 +43,7 @@ type SLoadbalancerListenerRule struct {
 
 func loadbalancerListenerRuleCheckUniqueness(ctx context.Context, lbls *SLoadbalancerListener, domain, path string) error {
 	q := LoadbalancerListenerRuleManager.Query().
+		IsFalse("pending_deleted").
 		Equals("listener_id", lbls.Id).
 		Equals("domain", domain).
 		Equals("path", path)
diff --git a/pkg/compute/models/loadbalancerlisteners.go b/pkg/compute/models/loadbalancerlisteners.go
index 607fefa3f7..7ce180e9c1 100644
--- a/pkg/compute/models/loadbalancerlisteners.go
+++ b/pkg/compute/models/loadbalancerlisteners.go
@@ -104,6 +104,7 @@ type SLoadbalancerListener struct {
 
 func (man *SLoadbalancerListenerManager) checkListenerUniqueness(ctx context.Context, lb *SLoadbalancer, listenerType string, listenerPort int64) error {
 	q := man.Query().
+		IsFalse("pending_deleted").
 		Equals("loadbalancer_id", lb.Id).
 		Equals("listener_port", listenerPort)
 	switch listenerType {

From 3a04d0c3d312a842a2379713068158c925c46310 Mon Sep 17 00:00:00 2001
From: Qiu Jian <qiujian@yunionyun.com>
Date: Sat, 17 Nov 2018 08:40:19 +0800
Subject: [PATCH 29/57] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=20sched-policy-show=20?=
 =?UTF-8?q?command?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 cmd/climc/shell/schedpolicies.go | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/cmd/climc/shell/schedpolicies.go b/cmd/climc/shell/schedpolicies.go
index fac45935b7..758b6c0f86 100644
--- a/cmd/climc/shell/schedpolicies.go
+++ b/cmd/climc/shell/schedpolicies.go
@@ -111,6 +111,18 @@ func init() {
 		return nil
 	})
 
+	type SchedpoliciesShowOptions struct {
+		ID string `help:"ID or name of the sched policy"`
+	}
+	R(&SchedpoliciesShowOptions{}, "sched-policy-show", "show details of a sched policy", func(s *mcclient.ClientSession, args *SchedpoliciesShowOptions) error {
+		result, err := modules.Schedpolicies.Get(s, args.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	type SchedpoliciesEvaluateOptions struct {
 		ID     string `help:"ID or name of the sched policy"`
 		SERVER string `help:"ID or name of the server"`

From bc1fae8ebb672b39b46e4907f2cf3e2b1fdcfd2a Mon Sep 17 00:00:00 2001
From: TangBin <tangbin@yunionyun.com>
Date: Sat, 17 Nov 2018 10:25:25 +0800
Subject: [PATCH 30/57] rebuild root disk

---
 Gopkg.lock                                    |  21 +-
 pkg/util/aws/image.go                         |  11 +
 pkg/util/aws/region.go                        |  22 +-
 pkg/util/aws/storagecache.go                  |  29 +-
 .../service/s3/s3iface/interface.go           | 403 +++++++++
 .../aws-sdk-go/service/s3/s3manager/batch.go  | 529 ++++++++++++
 .../service/s3/s3manager/bucket_region.go     |  88 ++
 .../aws-sdk-go/service/s3/s3manager/doc.go    |   3 +
 .../service/s3/s3manager/download.go          | 555 ++++++++++++
 .../aws-sdk-go/service/s3/s3manager/upload.go | 802 ++++++++++++++++++
 10 files changed, 2429 insertions(+), 34 deletions(-)
 create mode 100644 vendor/github.com/aws/aws-sdk-go/service/s3/s3iface/interface.go
 create mode 100644 vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/batch.go
 create mode 100644 vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/bucket_region.go
 create mode 100644 vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/doc.go
 create mode 100644 vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/download.go
 create mode 100644 vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/upload.go

diff --git a/Gopkg.lock b/Gopkg.lock
index 414c3a0e55..7ec8fb6624 100644
--- a/Gopkg.lock
+++ b/Gopkg.lock
@@ -121,7 +121,7 @@
   version = "0.11.0"
 
 [[projects]]
-  digest = "1:55bf2a4da68caa693d660683e53bab3e651a940cd46354b997ab68bec7920e23"
+  digest = "1:ae41c49d4812dd31a848045447637fd0f10cac09af75e1d20825a241c626f3fa"
   name = "github.com/aws/aws-sdk-go"
   packages = [
     "aws",
@@ -159,6 +159,8 @@
     "service/ec2",
     "service/iam",
     "service/s3",
+    "service/s3/s3iface",
+    "service/s3/s3manager",
     "service/sts",
   ]
   pruneopts = "UT"
@@ -1292,14 +1294,6 @@
   revision = "1638f8970cefaa404ff3a62950f88b08292b2696"
   version = "v9.0.0"
 
-[[projects]]
-  digest = "1:7719608fe0b52a4ece56c2dde37bedd95b938677d1ab0f84b8a7852e4c59f849"
-  name = "sigs.k8s.io/yaml"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "fd68e9863619f6ec2fdd8625fe1f02e7c877e480"
-  version = "v1.1.0"
-
 [[projects]]
   digest = "1:356a33d2d627cbb232a1d71cc42051fbac0e5e6a1717679f110054baa02472e5"
   name = "k8s.io/kubernetes"
@@ -1311,6 +1305,14 @@
   revision = "17c77c7898218073f14c8d573582e8d2313dc740"
   version = "v1.12.2"
 
+[[projects]]
+  digest = "1:7719608fe0b52a4ece56c2dde37bedd95b938677d1ab0f84b8a7852e4c59f849"
+  name = "sigs.k8s.io/yaml"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "fd68e9863619f6ec2fdd8625fe1f02e7c877e480"
+  version = "v1.1.0"
+
 [[projects]]
   branch = "master"
   digest = "1:54554b3c72f4fcbd3c27c5137f9acd9d153d7e7150d77cd509af50c75fcb41b9"
@@ -1409,6 +1411,7 @@
     "github.com/aws/aws-sdk-go/service/ec2",
     "github.com/aws/aws-sdk-go/service/iam",
     "github.com/aws/aws-sdk-go/service/s3",
+    "github.com/aws/aws-sdk-go/service/s3/s3manager",
     "github.com/bitly/go-simplejson",
     "github.com/c-bata/go-prompt",
     "github.com/coredns/coredns/core/dnsserver",
diff --git a/pkg/util/aws/image.go b/pkg/util/aws/image.go
index 8389e38b9a..700e43692b 100644
--- a/pkg/util/aws/image.go
+++ b/pkg/util/aws/image.go
@@ -2,6 +2,7 @@ package aws
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"yunion.io/x/jsonutils"
@@ -189,6 +190,8 @@ func (self *SRegion) GetImageByName(name string) (*SImage, error) {
 	if len(images) == 0 {
 		return nil, cloudprovider.ErrNotFound
 	}
+
+	log.Debugf("%d image found match name %", len(images), name)
 	return &images[0], nil
 }
 
@@ -230,8 +233,16 @@ func (self *SRegion) GetImages(status ImageStatusType, owner ImageOwnerType, ima
 	if len(imageId) > 0 {
 		params.SetImageIds(ConvertedList(imageId))
 	}
+
+	if len(filters) > 0 {
+		params.SetFilters(filters)
+	}
+
 	ret, err := self.ec2Client.DescribeImages(params)
 	if err != nil {
+		if strings.Contains(err.Error(), ".NotFound") {
+			return nil, 0, cloudprovider.ErrNotFound
+		}
 		return nil, 0, err
 	}
 
diff --git a/pkg/util/aws/region.go b/pkg/util/aws/region.go
index 558a307604..910bae7767 100644
--- a/pkg/util/aws/region.go
+++ b/pkg/util/aws/region.go
@@ -59,12 +59,16 @@ func (self *SRegion) GetClient() *SAwsClient {
 	return self.client
 }
 
+func (self *SRegion) getAwsSession() (*session.Session, error) {
+	return session.NewSession(&sdk.Config{
+		Region:      sdk.String(self.RegionId),
+		Credentials: credentials.NewStaticCredentials(self.client.accessKey, self.client.secret, ""),
+	})
+}
+
 func (self *SRegion) getEc2Client() (*ec2.EC2, error) {
 	if self.ec2Client == nil {
-		s, err := session.NewSession(&sdk.Config{
-			Region:      sdk.String(self.RegionId),
-			Credentials: credentials.NewStaticCredentials(self.client.accessKey, self.client.secret, ""),
-		})
+		s, err := self.getAwsSession()
 
 		if err != nil {
 			return nil, err
@@ -79,10 +83,7 @@ func (self *SRegion) getEc2Client() (*ec2.EC2, error) {
 
 func (self *SRegion) getIamClient() (*iam.IAM, error) {
 	if self.iamClient == nil {
-		s, err := session.NewSession(&sdk.Config{
-			Region:      sdk.String(self.RegionId),
-			Credentials: credentials.NewStaticCredentials(self.client.accessKey, self.client.secret, ""),
-		})
+		s, err := self.getAwsSession()
 
 		if err != nil {
 			return nil, err
@@ -96,10 +97,7 @@ func (self *SRegion) getIamClient() (*iam.IAM, error) {
 
 func (self *SRegion) getS3Client() (*s3.S3, error) {
 	if self.s3Client == nil {
-		s, err := session.NewSession(&sdk.Config{
-			Region:      sdk.String(self.RegionId),
-			Credentials: credentials.NewStaticCredentials(self.client.accessKey, self.client.secret, ""),
-		})
+		s, err := self.getAwsSession()
 
 		if err != nil {
 			return nil, err
diff --git a/pkg/util/aws/storagecache.go b/pkg/util/aws/storagecache.go
index 20760bc86c..eaed1175b2 100644
--- a/pkg/util/aws/storagecache.go
+++ b/pkg/util/aws/storagecache.go
@@ -1,12 +1,11 @@
 package aws
 
 import (
-	"bytes"
 	"fmt"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/iam"
 	"github.com/aws/aws-sdk-go/service/s3"
-	"io/ioutil"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"strings"
 	"time"
 	"yunion.io/x/jsonutils"
@@ -148,17 +147,20 @@ func (self *SStoragecache) uploadImage(userCred mcclient.TokenCredential, imageI
 		return "", err
 	}
 
-	s3Client, err := self.region.getS3Client()
-	if err != nil {
-		return "", nil
+	// uploader to aws s3
+	input := &s3manager.UploadInput{
+		Bucket: &bucketName,
+		Key: &imageId,
+		Body: reader,
 	}
-	// 内存？
-	f, err := ioutil.ReadAll(reader)
-	params := &s3.PutObjectInput{}
-	params.SetBucket(bucketName)
-	params.SetKey(imageId)
-	params.SetBody(bytes.NewReader(f))
-	_, err = s3Client.PutObject(params)
+
+	awsSession, err := self.region.getAwsSession()
+	if err != nil {
+		log.Debugf("uploadImage %s", err.Error())
+		return "", fmt.Errorf("get aws session failed")
+	}
+	uploader := s3manager.NewUploader(awsSession)
+	_, err = uploader.Upload(input)
 	if err != nil {
 		return "", nil
 	}
@@ -183,6 +185,7 @@ func (self *SStoragecache) uploadImage(userCred mcclient.TokenCredential, imageI
 
 		imageName = fmt.Sprintf("%s-%d", imageBaseName, nameIdx)
 		nameIdx += 1
+		log.Debugf("uploadImage Match remote name %s", imageName)
 	}
 
 	task, err := self.region.ImportImage(imageName, osArch, osType, osDist, diskFormat, bucketName, imageId)
@@ -194,6 +197,7 @@ func (self *SStoragecache) uploadImage(userCred mcclient.TokenCredential, imageI
 
 	// todo:// 等待镜像导入完成
 	for i := 1; i < 120; i++ {
+		time.Sleep(2 * time.Minute)
 		ret, err := self.region.ec2Client.DescribeImportImageTasks(&ec2.DescribeImportImageTasksInput{ImportTaskIds: []*string{&task.TaskId}})
 		if err != nil {
 			return "", err
@@ -210,7 +214,6 @@ func (self *SStoragecache) uploadImage(userCred mcclient.TokenCredential, imageI
 				return *item.ImageId, nil
 			}
 		}
-		time.Sleep(1 * time.Minute)
 	}
 
 	return task.ImageId, fmt.Errorf("uploadImage uncompleted: %s", task)
diff --git a/vendor/github.com/aws/aws-sdk-go/service/s3/s3iface/interface.go b/vendor/github.com/aws/aws-sdk-go/service/s3/s3iface/interface.go
new file mode 100644
index 0000000000..cc427882b8
--- /dev/null
+++ b/vendor/github.com/aws/aws-sdk-go/service/s3/s3iface/interface.go
@@ -0,0 +1,403 @@
+// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT.
+
+// Package s3iface provides an interface to enable mocking the Amazon Simple Storage Service service client
+// for testing your code.
+//
+// It is important to note that this interface will have breaking changes
+// when the service model is updated and adds new API operations, paginators,
+// and waiters.
+package s3iface
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+// S3API provides an interface to enable mocking the
+// s3.S3 service client's API operation,
+// paginators, and waiters. This make unit testing your code that calls out
+// to the SDK's service client's calls easier.
+//
+// The best way to use this interface is so the SDK's service client's calls
+// can be stubbed out for unit testing your code with the SDK without needing
+// to inject custom request handlers into the SDK's request pipeline.
+//
+//    // myFunc uses an SDK service client to make a request to
+//    // Amazon Simple Storage Service.
+//    func myFunc(svc s3iface.S3API) bool {
+//        // Make svc.AbortMultipartUpload request
+//    }
+//
+//    func main() {
+//        sess := session.New()
+//        svc := s3.New(sess)
+//
+//        myFunc(svc)
+//    }
+//
+// In your _test.go file:
+//
+//    // Define a mock struct to be used in your unit tests of myFunc.
+//    type mockS3Client struct {
+//        s3iface.S3API
+//    }
+//    func (m *mockS3Client) AbortMultipartUpload(input *s3.AbortMultipartUploadInput) (*s3.AbortMultipartUploadOutput, error) {
+//        // mock response/functionality
+//    }
+//
+//    func TestMyFunc(t *testing.T) {
+//        // Setup Test
+//        mockSvc := &mockS3Client{}
+//
+//        myfunc(mockSvc)
+//
+//        // Verify myFunc's functionality
+//    }
+//
+// It is important to note that this interface will have breaking changes
+// when the service model is updated and adds new API operations, paginators,
+// and waiters. Its suggested to use the pattern above for testing, or using
+// tooling to generate mocks to satisfy the interfaces.
+type S3API interface {
+	AbortMultipartUpload(*s3.AbortMultipartUploadInput) (*s3.AbortMultipartUploadOutput, error)
+	AbortMultipartUploadWithContext(aws.Context, *s3.AbortMultipartUploadInput, ...request.Option) (*s3.AbortMultipartUploadOutput, error)
+	AbortMultipartUploadRequest(*s3.AbortMultipartUploadInput) (*request.Request, *s3.AbortMultipartUploadOutput)
+
+	CompleteMultipartUpload(*s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
+	CompleteMultipartUploadWithContext(aws.Context, *s3.CompleteMultipartUploadInput, ...request.Option) (*s3.CompleteMultipartUploadOutput, error)
+	CompleteMultipartUploadRequest(*s3.CompleteMultipartUploadInput) (*request.Request, *s3.CompleteMultipartUploadOutput)
+
+	CopyObject(*s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
+	CopyObjectWithContext(aws.Context, *s3.CopyObjectInput, ...request.Option) (*s3.CopyObjectOutput, error)
+	CopyObjectRequest(*s3.CopyObjectInput) (*request.Request, *s3.CopyObjectOutput)
+
+	CreateBucket(*s3.CreateBucketInput) (*s3.CreateBucketOutput, error)
+	CreateBucketWithContext(aws.Context, *s3.CreateBucketInput, ...request.Option) (*s3.CreateBucketOutput, error)
+	CreateBucketRequest(*s3.CreateBucketInput) (*request.Request, *s3.CreateBucketOutput)
+
+	CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
+	CreateMultipartUploadWithContext(aws.Context, *s3.CreateMultipartUploadInput, ...request.Option) (*s3.CreateMultipartUploadOutput, error)
+	CreateMultipartUploadRequest(*s3.CreateMultipartUploadInput) (*request.Request, *s3.CreateMultipartUploadOutput)
+
+	DeleteBucket(*s3.DeleteBucketInput) (*s3.DeleteBucketOutput, error)
+	DeleteBucketWithContext(aws.Context, *s3.DeleteBucketInput, ...request.Option) (*s3.DeleteBucketOutput, error)
+	DeleteBucketRequest(*s3.DeleteBucketInput) (*request.Request, *s3.DeleteBucketOutput)
+
+	DeleteBucketAnalyticsConfiguration(*s3.DeleteBucketAnalyticsConfigurationInput) (*s3.DeleteBucketAnalyticsConfigurationOutput, error)
+	DeleteBucketAnalyticsConfigurationWithContext(aws.Context, *s3.DeleteBucketAnalyticsConfigurationInput, ...request.Option) (*s3.DeleteBucketAnalyticsConfigurationOutput, error)
+	DeleteBucketAnalyticsConfigurationRequest(*s3.DeleteBucketAnalyticsConfigurationInput) (*request.Request, *s3.DeleteBucketAnalyticsConfigurationOutput)
+
+	DeleteBucketCors(*s3.DeleteBucketCorsInput) (*s3.DeleteBucketCorsOutput, error)
+	DeleteBucketCorsWithContext(aws.Context, *s3.DeleteBucketCorsInput, ...request.Option) (*s3.DeleteBucketCorsOutput, error)
+	DeleteBucketCorsRequest(*s3.DeleteBucketCorsInput) (*request.Request, *s3.DeleteBucketCorsOutput)
+
+	DeleteBucketEncryption(*s3.DeleteBucketEncryptionInput) (*s3.DeleteBucketEncryptionOutput, error)
+	DeleteBucketEncryptionWithContext(aws.Context, *s3.DeleteBucketEncryptionInput, ...request.Option) (*s3.DeleteBucketEncryptionOutput, error)
+	DeleteBucketEncryptionRequest(*s3.DeleteBucketEncryptionInput) (*request.Request, *s3.DeleteBucketEncryptionOutput)
+
+	DeleteBucketInventoryConfiguration(*s3.DeleteBucketInventoryConfigurationInput) (*s3.DeleteBucketInventoryConfigurationOutput, error)
+	DeleteBucketInventoryConfigurationWithContext(aws.Context, *s3.DeleteBucketInventoryConfigurationInput, ...request.Option) (*s3.DeleteBucketInventoryConfigurationOutput, error)
+	DeleteBucketInventoryConfigurationRequest(*s3.DeleteBucketInventoryConfigurationInput) (*request.Request, *s3.DeleteBucketInventoryConfigurationOutput)
+
+	DeleteBucketLifecycle(*s3.DeleteBucketLifecycleInput) (*s3.DeleteBucketLifecycleOutput, error)
+	DeleteBucketLifecycleWithContext(aws.Context, *s3.DeleteBucketLifecycleInput, ...request.Option) (*s3.DeleteBucketLifecycleOutput, error)
+	DeleteBucketLifecycleRequest(*s3.DeleteBucketLifecycleInput) (*request.Request, *s3.DeleteBucketLifecycleOutput)
+
+	DeleteBucketMetricsConfiguration(*s3.DeleteBucketMetricsConfigurationInput) (*s3.DeleteBucketMetricsConfigurationOutput, error)
+	DeleteBucketMetricsConfigurationWithContext(aws.Context, *s3.DeleteBucketMetricsConfigurationInput, ...request.Option) (*s3.DeleteBucketMetricsConfigurationOutput, error)
+	DeleteBucketMetricsConfigurationRequest(*s3.DeleteBucketMetricsConfigurationInput) (*request.Request, *s3.DeleteBucketMetricsConfigurationOutput)
+
+	DeleteBucketPolicy(*s3.DeleteBucketPolicyInput) (*s3.DeleteBucketPolicyOutput, error)
+	DeleteBucketPolicyWithContext(aws.Context, *s3.DeleteBucketPolicyInput, ...request.Option) (*s3.DeleteBucketPolicyOutput, error)
+	DeleteBucketPolicyRequest(*s3.DeleteBucketPolicyInput) (*request.Request, *s3.DeleteBucketPolicyOutput)
+
+	DeleteBucketReplication(*s3.DeleteBucketReplicationInput) (*s3.DeleteBucketReplicationOutput, error)
+	DeleteBucketReplicationWithContext(aws.Context, *s3.DeleteBucketReplicationInput, ...request.Option) (*s3.DeleteBucketReplicationOutput, error)
+	DeleteBucketReplicationRequest(*s3.DeleteBucketReplicationInput) (*request.Request, *s3.DeleteBucketReplicationOutput)
+
+	DeleteBucketTagging(*s3.DeleteBucketTaggingInput) (*s3.DeleteBucketTaggingOutput, error)
+	DeleteBucketTaggingWithContext(aws.Context, *s3.DeleteBucketTaggingInput, ...request.Option) (*s3.DeleteBucketTaggingOutput, error)
+	DeleteBucketTaggingRequest(*s3.DeleteBucketTaggingInput) (*request.Request, *s3.DeleteBucketTaggingOutput)
+
+	DeleteBucketWebsite(*s3.DeleteBucketWebsiteInput) (*s3.DeleteBucketWebsiteOutput, error)
+	DeleteBucketWebsiteWithContext(aws.Context, *s3.DeleteBucketWebsiteInput, ...request.Option) (*s3.DeleteBucketWebsiteOutput, error)
+	DeleteBucketWebsiteRequest(*s3.DeleteBucketWebsiteInput) (*request.Request, *s3.DeleteBucketWebsiteOutput)
+
+	DeleteObject(*s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
+	DeleteObjectWithContext(aws.Context, *s3.DeleteObjectInput, ...request.Option) (*s3.DeleteObjectOutput, error)
+	DeleteObjectRequest(*s3.DeleteObjectInput) (*request.Request, *s3.DeleteObjectOutput)
+
+	DeleteObjectTagging(*s3.DeleteObjectTaggingInput) (*s3.DeleteObjectTaggingOutput, error)
+	DeleteObjectTaggingWithContext(aws.Context, *s3.DeleteObjectTaggingInput, ...request.Option) (*s3.DeleteObjectTaggingOutput, error)
+	DeleteObjectTaggingRequest(*s3.DeleteObjectTaggingInput) (*request.Request, *s3.DeleteObjectTaggingOutput)
+
+	DeleteObjects(*s3.DeleteObjectsInput) (*s3.DeleteObjectsOutput, error)
+	DeleteObjectsWithContext(aws.Context, *s3.DeleteObjectsInput, ...request.Option) (*s3.DeleteObjectsOutput, error)
+	DeleteObjectsRequest(*s3.DeleteObjectsInput) (*request.Request, *s3.DeleteObjectsOutput)
+
+	GetBucketAccelerateConfiguration(*s3.GetBucketAccelerateConfigurationInput) (*s3.GetBucketAccelerateConfigurationOutput, error)
+	GetBucketAccelerateConfigurationWithContext(aws.Context, *s3.GetBucketAccelerateConfigurationInput, ...request.Option) (*s3.GetBucketAccelerateConfigurationOutput, error)
+	GetBucketAccelerateConfigurationRequest(*s3.GetBucketAccelerateConfigurationInput) (*request.Request, *s3.GetBucketAccelerateConfigurationOutput)
+
+	GetBucketAcl(*s3.GetBucketAclInput) (*s3.GetBucketAclOutput, error)
+	GetBucketAclWithContext(aws.Context, *s3.GetBucketAclInput, ...request.Option) (*s3.GetBucketAclOutput, error)
+	GetBucketAclRequest(*s3.GetBucketAclInput) (*request.Request, *s3.GetBucketAclOutput)
+
+	GetBucketAnalyticsConfiguration(*s3.GetBucketAnalyticsConfigurationInput) (*s3.GetBucketAnalyticsConfigurationOutput, error)
+	GetBucketAnalyticsConfigurationWithContext(aws.Context, *s3.GetBucketAnalyticsConfigurationInput, ...request.Option) (*s3.GetBucketAnalyticsConfigurationOutput, error)
+	GetBucketAnalyticsConfigurationRequest(*s3.GetBucketAnalyticsConfigurationInput) (*request.Request, *s3.GetBucketAnalyticsConfigurationOutput)
+
+	GetBucketCors(*s3.GetBucketCorsInput) (*s3.GetBucketCorsOutput, error)
+	GetBucketCorsWithContext(aws.Context, *s3.GetBucketCorsInput, ...request.Option) (*s3.GetBucketCorsOutput, error)
+	GetBucketCorsRequest(*s3.GetBucketCorsInput) (*request.Request, *s3.GetBucketCorsOutput)
+
+	GetBucketEncryption(*s3.GetBucketEncryptionInput) (*s3.GetBucketEncryptionOutput, error)
+	GetBucketEncryptionWithContext(aws.Context, *s3.GetBucketEncryptionInput, ...request.Option) (*s3.GetBucketEncryptionOutput, error)
+	GetBucketEncryptionRequest(*s3.GetBucketEncryptionInput) (*request.Request, *s3.GetBucketEncryptionOutput)
+
+	GetBucketInventoryConfiguration(*s3.GetBucketInventoryConfigurationInput) (*s3.GetBucketInventoryConfigurationOutput, error)
+	GetBucketInventoryConfigurationWithContext(aws.Context, *s3.GetBucketInventoryConfigurationInput, ...request.Option) (*s3.GetBucketInventoryConfigurationOutput, error)
+	GetBucketInventoryConfigurationRequest(*s3.GetBucketInventoryConfigurationInput) (*request.Request, *s3.GetBucketInventoryConfigurationOutput)
+
+	GetBucketLifecycle(*s3.GetBucketLifecycleInput) (*s3.GetBucketLifecycleOutput, error)
+	GetBucketLifecycleWithContext(aws.Context, *s3.GetBucketLifecycleInput, ...request.Option) (*s3.GetBucketLifecycleOutput, error)
+	GetBucketLifecycleRequest(*s3.GetBucketLifecycleInput) (*request.Request, *s3.GetBucketLifecycleOutput)
+
+	GetBucketLifecycleConfiguration(*s3.GetBucketLifecycleConfigurationInput) (*s3.GetBucketLifecycleConfigurationOutput, error)
+	GetBucketLifecycleConfigurationWithContext(aws.Context, *s3.GetBucketLifecycleConfigurationInput, ...request.Option) (*s3.GetBucketLifecycleConfigurationOutput, error)
+	GetBucketLifecycleConfigurationRequest(*s3.GetBucketLifecycleConfigurationInput) (*request.Request, *s3.GetBucketLifecycleConfigurationOutput)
+
+	GetBucketLocation(*s3.GetBucketLocationInput) (*s3.GetBucketLocationOutput, error)
+	GetBucketLocationWithContext(aws.Context, *s3.GetBucketLocationInput, ...request.Option) (*s3.GetBucketLocationOutput, error)
+	GetBucketLocationRequest(*s3.GetBucketLocationInput) (*request.Request, *s3.GetBucketLocationOutput)
+
+	GetBucketLogging(*s3.GetBucketLoggingInput) (*s3.GetBucketLoggingOutput, error)
+	GetBucketLoggingWithContext(aws.Context, *s3.GetBucketLoggingInput, ...request.Option) (*s3.GetBucketLoggingOutput, error)
+	GetBucketLoggingRequest(*s3.GetBucketLoggingInput) (*request.Request, *s3.GetBucketLoggingOutput)
+
+	GetBucketMetricsConfiguration(*s3.GetBucketMetricsConfigurationInput) (*s3.GetBucketMetricsConfigurationOutput, error)
+	GetBucketMetricsConfigurationWithContext(aws.Context, *s3.GetBucketMetricsConfigurationInput, ...request.Option) (*s3.GetBucketMetricsConfigurationOutput, error)
+	GetBucketMetricsConfigurationRequest(*s3.GetBucketMetricsConfigurationInput) (*request.Request, *s3.GetBucketMetricsConfigurationOutput)
+
+	GetBucketNotification(*s3.GetBucketNotificationConfigurationRequest) (*s3.NotificationConfigurationDeprecated, error)
+	GetBucketNotificationWithContext(aws.Context, *s3.GetBucketNotificationConfigurationRequest, ...request.Option) (*s3.NotificationConfigurationDeprecated, error)
+	GetBucketNotificationRequest(*s3.GetBucketNotificationConfigurationRequest) (*request.Request, *s3.NotificationConfigurationDeprecated)
+
+	GetBucketNotificationConfiguration(*s3.GetBucketNotificationConfigurationRequest) (*s3.NotificationConfiguration, error)
+	GetBucketNotificationConfigurationWithContext(aws.Context, *s3.GetBucketNotificationConfigurationRequest, ...request.Option) (*s3.NotificationConfiguration, error)
+	GetBucketNotificationConfigurationRequest(*s3.GetBucketNotificationConfigurationRequest) (*request.Request, *s3.NotificationConfiguration)
+
+	GetBucketPolicy(*s3.GetBucketPolicyInput) (*s3.GetBucketPolicyOutput, error)
+	GetBucketPolicyWithContext(aws.Context, *s3.GetBucketPolicyInput, ...request.Option) (*s3.GetBucketPolicyOutput, error)
+	GetBucketPolicyRequest(*s3.GetBucketPolicyInput) (*request.Request, *s3.GetBucketPolicyOutput)
+
+	GetBucketReplication(*s3.GetBucketReplicationInput) (*s3.GetBucketReplicationOutput, error)
+	GetBucketReplicationWithContext(aws.Context, *s3.GetBucketReplicationInput, ...request.Option) (*s3.GetBucketReplicationOutput, error)
+	GetBucketReplicationRequest(*s3.GetBucketReplicationInput) (*request.Request, *s3.GetBucketReplicationOutput)
+
+	GetBucketRequestPayment(*s3.GetBucketRequestPaymentInput) (*s3.GetBucketRequestPaymentOutput, error)
+	GetBucketRequestPaymentWithContext(aws.Context, *s3.GetBucketRequestPaymentInput, ...request.Option) (*s3.GetBucketRequestPaymentOutput, error)
+	GetBucketRequestPaymentRequest(*s3.GetBucketRequestPaymentInput) (*request.Request, *s3.GetBucketRequestPaymentOutput)
+
+	GetBucketTagging(*s3.GetBucketTaggingInput) (*s3.GetBucketTaggingOutput, error)
+	GetBucketTaggingWithContext(aws.Context, *s3.GetBucketTaggingInput, ...request.Option) (*s3.GetBucketTaggingOutput, error)
+	GetBucketTaggingRequest(*s3.GetBucketTaggingInput) (*request.Request, *s3.GetBucketTaggingOutput)
+
+	GetBucketVersioning(*s3.GetBucketVersioningInput) (*s3.GetBucketVersioningOutput, error)
+	GetBucketVersioningWithContext(aws.Context, *s3.GetBucketVersioningInput, ...request.Option) (*s3.GetBucketVersioningOutput, error)
+	GetBucketVersioningRequest(*s3.GetBucketVersioningInput) (*request.Request, *s3.GetBucketVersioningOutput)
+
+	GetBucketWebsite(*s3.GetBucketWebsiteInput) (*s3.GetBucketWebsiteOutput, error)
+	GetBucketWebsiteWithContext(aws.Context, *s3.GetBucketWebsiteInput, ...request.Option) (*s3.GetBucketWebsiteOutput, error)
+	GetBucketWebsiteRequest(*s3.GetBucketWebsiteInput) (*request.Request, *s3.GetBucketWebsiteOutput)
+
+	GetObject(*s3.GetObjectInput) (*s3.GetObjectOutput, error)
+	GetObjectWithContext(aws.Context, *s3.GetObjectInput, ...request.Option) (*s3.GetObjectOutput, error)
+	GetObjectRequest(*s3.GetObjectInput) (*request.Request, *s3.GetObjectOutput)
+
+	GetObjectAcl(*s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
+	GetObjectAclWithContext(aws.Context, *s3.GetObjectAclInput, ...request.Option) (*s3.GetObjectAclOutput, error)
+	GetObjectAclRequest(*s3.GetObjectAclInput) (*request.Request, *s3.GetObjectAclOutput)
+
+	GetObjectTagging(*s3.GetObjectTaggingInput) (*s3.GetObjectTaggingOutput, error)
+	GetObjectTaggingWithContext(aws.Context, *s3.GetObjectTaggingInput, ...request.Option) (*s3.GetObjectTaggingOutput, error)
+	GetObjectTaggingRequest(*s3.GetObjectTaggingInput) (*request.Request, *s3.GetObjectTaggingOutput)
+
+	GetObjectTorrent(*s3.GetObjectTorrentInput) (*s3.GetObjectTorrentOutput, error)
+	GetObjectTorrentWithContext(aws.Context, *s3.GetObjectTorrentInput, ...request.Option) (*s3.GetObjectTorrentOutput, error)
+	GetObjectTorrentRequest(*s3.GetObjectTorrentInput) (*request.Request, *s3.GetObjectTorrentOutput)
+
+	HeadBucket(*s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
+	HeadBucketWithContext(aws.Context, *s3.HeadBucketInput, ...request.Option) (*s3.HeadBucketOutput, error)
+	HeadBucketRequest(*s3.HeadBucketInput) (*request.Request, *s3.HeadBucketOutput)
+
+	HeadObject(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
+	HeadObjectWithContext(aws.Context, *s3.HeadObjectInput, ...request.Option) (*s3.HeadObjectOutput, error)
+	HeadObjectRequest(*s3.HeadObjectInput) (*request.Request, *s3.HeadObjectOutput)
+
+	ListBucketAnalyticsConfigurations(*s3.ListBucketAnalyticsConfigurationsInput) (*s3.ListBucketAnalyticsConfigurationsOutput, error)
+	ListBucketAnalyticsConfigurationsWithContext(aws.Context, *s3.ListBucketAnalyticsConfigurationsInput, ...request.Option) (*s3.ListBucketAnalyticsConfigurationsOutput, error)
+	ListBucketAnalyticsConfigurationsRequest(*s3.ListBucketAnalyticsConfigurationsInput) (*request.Request, *s3.ListBucketAnalyticsConfigurationsOutput)
+
+	ListBucketInventoryConfigurations(*s3.ListBucketInventoryConfigurationsInput) (*s3.ListBucketInventoryConfigurationsOutput, error)
+	ListBucketInventoryConfigurationsWithContext(aws.Context, *s3.ListBucketInventoryConfigurationsInput, ...request.Option) (*s3.ListBucketInventoryConfigurationsOutput, error)
+	ListBucketInventoryConfigurationsRequest(*s3.ListBucketInventoryConfigurationsInput) (*request.Request, *s3.ListBucketInventoryConfigurationsOutput)
+
+	ListBucketMetricsConfigurations(*s3.ListBucketMetricsConfigurationsInput) (*s3.ListBucketMetricsConfigurationsOutput, error)
+	ListBucketMetricsConfigurationsWithContext(aws.Context, *s3.ListBucketMetricsConfigurationsInput, ...request.Option) (*s3.ListBucketMetricsConfigurationsOutput, error)
+	ListBucketMetricsConfigurationsRequest(*s3.ListBucketMetricsConfigurationsInput) (*request.Request, *s3.ListBucketMetricsConfigurationsOutput)
+
+	ListBuckets(*s3.ListBucketsInput) (*s3.ListBucketsOutput, error)
+	ListBucketsWithContext(aws.Context, *s3.ListBucketsInput, ...request.Option) (*s3.ListBucketsOutput, error)
+	ListBucketsRequest(*s3.ListBucketsInput) (*request.Request, *s3.ListBucketsOutput)
+
+	ListMultipartUploads(*s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error)
+	ListMultipartUploadsWithContext(aws.Context, *s3.ListMultipartUploadsInput, ...request.Option) (*s3.ListMultipartUploadsOutput, error)
+	ListMultipartUploadsRequest(*s3.ListMultipartUploadsInput) (*request.Request, *s3.ListMultipartUploadsOutput)
+
+	ListMultipartUploadsPages(*s3.ListMultipartUploadsInput, func(*s3.ListMultipartUploadsOutput, bool) bool) error
+	ListMultipartUploadsPagesWithContext(aws.Context, *s3.ListMultipartUploadsInput, func(*s3.ListMultipartUploadsOutput, bool) bool, ...request.Option) error
+
+	ListObjectVersions(*s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
+	ListObjectVersionsWithContext(aws.Context, *s3.ListObjectVersionsInput, ...request.Option) (*s3.ListObjectVersionsOutput, error)
+	ListObjectVersionsRequest(*s3.ListObjectVersionsInput) (*request.Request, *s3.ListObjectVersionsOutput)
+
+	ListObjectVersionsPages(*s3.ListObjectVersionsInput, func(*s3.ListObjectVersionsOutput, bool) bool) error
+	ListObjectVersionsPagesWithContext(aws.Context, *s3.ListObjectVersionsInput, func(*s3.ListObjectVersionsOutput, bool) bool, ...request.Option) error
+
+	ListObjects(*s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
+	ListObjectsWithContext(aws.Context, *s3.ListObjectsInput, ...request.Option) (*s3.ListObjectsOutput, error)
+	ListObjectsRequest(*s3.ListObjectsInput) (*request.Request, *s3.ListObjectsOutput)
+
+	ListObjectsPages(*s3.ListObjectsInput, func(*s3.ListObjectsOutput, bool) bool) error
+	ListObjectsPagesWithContext(aws.Context, *s3.ListObjectsInput, func(*s3.ListObjectsOutput, bool) bool, ...request.Option) error
+
+	ListObjectsV2(*s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
+	ListObjectsV2WithContext(aws.Context, *s3.ListObjectsV2Input, ...request.Option) (*s3.ListObjectsV2Output, error)
+	ListObjectsV2Request(*s3.ListObjectsV2Input) (*request.Request, *s3.ListObjectsV2Output)
+
+	ListObjectsV2Pages(*s3.ListObjectsV2Input, func(*s3.ListObjectsV2Output, bool) bool) error
+	ListObjectsV2PagesWithContext(aws.Context, *s3.ListObjectsV2Input, func(*s3.ListObjectsV2Output, bool) bool, ...request.Option) error
+
+	ListParts(*s3.ListPartsInput) (*s3.ListPartsOutput, error)
+	ListPartsWithContext(aws.Context, *s3.ListPartsInput, ...request.Option) (*s3.ListPartsOutput, error)
+	ListPartsRequest(*s3.ListPartsInput) (*request.Request, *s3.ListPartsOutput)
+
+	ListPartsPages(*s3.ListPartsInput, func(*s3.ListPartsOutput, bool) bool) error
+	ListPartsPagesWithContext(aws.Context, *s3.ListPartsInput, func(*s3.ListPartsOutput, bool) bool, ...request.Option) error
+
+	PutBucketAccelerateConfiguration(*s3.PutBucketAccelerateConfigurationInput) (*s3.PutBucketAccelerateConfigurationOutput, error)
+	PutBucketAccelerateConfigurationWithContext(aws.Context, *s3.PutBucketAccelerateConfigurationInput, ...request.Option) (*s3.PutBucketAccelerateConfigurationOutput, error)
+	PutBucketAccelerateConfigurationRequest(*s3.PutBucketAccelerateConfigurationInput) (*request.Request, *s3.PutBucketAccelerateConfigurationOutput)
+
+	PutBucketAcl(*s3.PutBucketAclInput) (*s3.PutBucketAclOutput, error)
+	PutBucketAclWithContext(aws.Context, *s3.PutBucketAclInput, ...request.Option) (*s3.PutBucketAclOutput, error)
+	PutBucketAclRequest(*s3.PutBucketAclInput) (*request.Request, *s3.PutBucketAclOutput)
+
+	PutBucketAnalyticsConfiguration(*s3.PutBucketAnalyticsConfigurationInput) (*s3.PutBucketAnalyticsConfigurationOutput, error)
+	PutBucketAnalyticsConfigurationWithContext(aws.Context, *s3.PutBucketAnalyticsConfigurationInput, ...request.Option) (*s3.PutBucketAnalyticsConfigurationOutput, error)
+	PutBucketAnalyticsConfigurationRequest(*s3.PutBucketAnalyticsConfigurationInput) (*request.Request, *s3.PutBucketAnalyticsConfigurationOutput)
+
+	PutBucketCors(*s3.PutBucketCorsInput) (*s3.PutBucketCorsOutput, error)
+	PutBucketCorsWithContext(aws.Context, *s3.PutBucketCorsInput, ...request.Option) (*s3.PutBucketCorsOutput, error)
+	PutBucketCorsRequest(*s3.PutBucketCorsInput) (*request.Request, *s3.PutBucketCorsOutput)
+
+	PutBucketEncryption(*s3.PutBucketEncryptionInput) (*s3.PutBucketEncryptionOutput, error)
+	PutBucketEncryptionWithContext(aws.Context, *s3.PutBucketEncryptionInput, ...request.Option) (*s3.PutBucketEncryptionOutput, error)
+	PutBucketEncryptionRequest(*s3.PutBucketEncryptionInput) (*request.Request, *s3.PutBucketEncryptionOutput)
+
+	PutBucketInventoryConfiguration(*s3.PutBucketInventoryConfigurationInput) (*s3.PutBucketInventoryConfigurationOutput, error)
+	PutBucketInventoryConfigurationWithContext(aws.Context, *s3.PutBucketInventoryConfigurationInput, ...request.Option) (*s3.PutBucketInventoryConfigurationOutput, error)
+	PutBucketInventoryConfigurationRequest(*s3.PutBucketInventoryConfigurationInput) (*request.Request, *s3.PutBucketInventoryConfigurationOutput)
+
+	PutBucketLifecycle(*s3.PutBucketLifecycleInput) (*s3.PutBucketLifecycleOutput, error)
+	PutBucketLifecycleWithContext(aws.Context, *s3.PutBucketLifecycleInput, ...request.Option) (*s3.PutBucketLifecycleOutput, error)
+	PutBucketLifecycleRequest(*s3.PutBucketLifecycleInput) (*request.Request, *s3.PutBucketLifecycleOutput)
+
+	PutBucketLifecycleConfiguration(*s3.PutBucketLifecycleConfigurationInput) (*s3.PutBucketLifecycleConfigurationOutput, error)
+	PutBucketLifecycleConfigurationWithContext(aws.Context, *s3.PutBucketLifecycleConfigurationInput, ...request.Option) (*s3.PutBucketLifecycleConfigurationOutput, error)
+	PutBucketLifecycleConfigurationRequest(*s3.PutBucketLifecycleConfigurationInput) (*request.Request, *s3.PutBucketLifecycleConfigurationOutput)
+
+	PutBucketLogging(*s3.PutBucketLoggingInput) (*s3.PutBucketLoggingOutput, error)
+	PutBucketLoggingWithContext(aws.Context, *s3.PutBucketLoggingInput, ...request.Option) (*s3.PutBucketLoggingOutput, error)
+	PutBucketLoggingRequest(*s3.PutBucketLoggingInput) (*request.Request, *s3.PutBucketLoggingOutput)
+
+	PutBucketMetricsConfiguration(*s3.PutBucketMetricsConfigurationInput) (*s3.PutBucketMetricsConfigurationOutput, error)
+	PutBucketMetricsConfigurationWithContext(aws.Context, *s3.PutBucketMetricsConfigurationInput, ...request.Option) (*s3.PutBucketMetricsConfigurationOutput, error)
+	PutBucketMetricsConfigurationRequest(*s3.PutBucketMetricsConfigurationInput) (*request.Request, *s3.PutBucketMetricsConfigurationOutput)
+
+	PutBucketNotification(*s3.PutBucketNotificationInput) (*s3.PutBucketNotificationOutput, error)
+	PutBucketNotificationWithContext(aws.Context, *s3.PutBucketNotificationInput, ...request.Option) (*s3.PutBucketNotificationOutput, error)
+	PutBucketNotificationRequest(*s3.PutBucketNotificationInput) (*request.Request, *s3.PutBucketNotificationOutput)
+
+	PutBucketNotificationConfiguration(*s3.PutBucketNotificationConfigurationInput) (*s3.PutBucketNotificationConfigurationOutput, error)
+	PutBucketNotificationConfigurationWithContext(aws.Context, *s3.PutBucketNotificationConfigurationInput, ...request.Option) (*s3.PutBucketNotificationConfigurationOutput, error)
+	PutBucketNotificationConfigurationRequest(*s3.PutBucketNotificationConfigurationInput) (*request.Request, *s3.PutBucketNotificationConfigurationOutput)
+
+	PutBucketPolicy(*s3.PutBucketPolicyInput) (*s3.PutBucketPolicyOutput, error)
+	PutBucketPolicyWithContext(aws.Context, *s3.PutBucketPolicyInput, ...request.Option) (*s3.PutBucketPolicyOutput, error)
+	PutBucketPolicyRequest(*s3.PutBucketPolicyInput) (*request.Request, *s3.PutBucketPolicyOutput)
+
+	PutBucketReplication(*s3.PutBucketReplicationInput) (*s3.PutBucketReplicationOutput, error)
+	PutBucketReplicationWithContext(aws.Context, *s3.PutBucketReplicationInput, ...request.Option) (*s3.PutBucketReplicationOutput, error)
+	PutBucketReplicationRequest(*s3.PutBucketReplicationInput) (*request.Request, *s3.PutBucketReplicationOutput)
+
+	PutBucketRequestPayment(*s3.PutBucketRequestPaymentInput) (*s3.PutBucketRequestPaymentOutput, error)
+	PutBucketRequestPaymentWithContext(aws.Context, *s3.PutBucketRequestPaymentInput, ...request.Option) (*s3.PutBucketRequestPaymentOutput, error)
+	PutBucketRequestPaymentRequest(*s3.PutBucketRequestPaymentInput) (*request.Request, *s3.PutBucketRequestPaymentOutput)
+
+	PutBucketTagging(*s3.PutBucketTaggingInput) (*s3.PutBucketTaggingOutput, error)
+	PutBucketTaggingWithContext(aws.Context, *s3.PutBucketTaggingInput, ...request.Option) (*s3.PutBucketTaggingOutput, error)
+	PutBucketTaggingRequest(*s3.PutBucketTaggingInput) (*request.Request, *s3.PutBucketTaggingOutput)
+
+	PutBucketVersioning(*s3.PutBucketVersioningInput) (*s3.PutBucketVersioningOutput, error)
+	PutBucketVersioningWithContext(aws.Context, *s3.PutBucketVersioningInput, ...request.Option) (*s3.PutBucketVersioningOutput, error)
+	PutBucketVersioningRequest(*s3.PutBucketVersioningInput) (*request.Request, *s3.PutBucketVersioningOutput)
+
+	PutBucketWebsite(*s3.PutBucketWebsiteInput) (*s3.PutBucketWebsiteOutput, error)
+	PutBucketWebsiteWithContext(aws.Context, *s3.PutBucketWebsiteInput, ...request.Option) (*s3.PutBucketWebsiteOutput, error)
+	PutBucketWebsiteRequest(*s3.PutBucketWebsiteInput) (*request.Request, *s3.PutBucketWebsiteOutput)
+
+	PutObject(*s3.PutObjectInput) (*s3.PutObjectOutput, error)
+	PutObjectWithContext(aws.Context, *s3.PutObjectInput, ...request.Option) (*s3.PutObjectOutput, error)
+	PutObjectRequest(*s3.PutObjectInput) (*request.Request, *s3.PutObjectOutput)
+
+	PutObjectAcl(*s3.PutObjectAclInput) (*s3.PutObjectAclOutput, error)
+	PutObjectAclWithContext(aws.Context, *s3.PutObjectAclInput, ...request.Option) (*s3.PutObjectAclOutput, error)
+	PutObjectAclRequest(*s3.PutObjectAclInput) (*request.Request, *s3.PutObjectAclOutput)
+
+	PutObjectTagging(*s3.PutObjectTaggingInput) (*s3.PutObjectTaggingOutput, error)
+	PutObjectTaggingWithContext(aws.Context, *s3.PutObjectTaggingInput, ...request.Option) (*s3.PutObjectTaggingOutput, error)
+	PutObjectTaggingRequest(*s3.PutObjectTaggingInput) (*request.Request, *s3.PutObjectTaggingOutput)
+
+	RestoreObject(*s3.RestoreObjectInput) (*s3.RestoreObjectOutput, error)
+	RestoreObjectWithContext(aws.Context, *s3.RestoreObjectInput, ...request.Option) (*s3.RestoreObjectOutput, error)
+	RestoreObjectRequest(*s3.RestoreObjectInput) (*request.Request, *s3.RestoreObjectOutput)
+
+	SelectObjectContent(*s3.SelectObjectContentInput) (*s3.SelectObjectContentOutput, error)
+	SelectObjectContentWithContext(aws.Context, *s3.SelectObjectContentInput, ...request.Option) (*s3.SelectObjectContentOutput, error)
+	SelectObjectContentRequest(*s3.SelectObjectContentInput) (*request.Request, *s3.SelectObjectContentOutput)
+
+	UploadPart(*s3.UploadPartInput) (*s3.UploadPartOutput, error)
+	UploadPartWithContext(aws.Context, *s3.UploadPartInput, ...request.Option) (*s3.UploadPartOutput, error)
+	UploadPartRequest(*s3.UploadPartInput) (*request.Request, *s3.UploadPartOutput)
+
+	UploadPartCopy(*s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error)
+	UploadPartCopyWithContext(aws.Context, *s3.UploadPartCopyInput, ...request.Option) (*s3.UploadPartCopyOutput, error)
+	UploadPartCopyRequest(*s3.UploadPartCopyInput) (*request.Request, *s3.UploadPartCopyOutput)
+
+	WaitUntilBucketExists(*s3.HeadBucketInput) error
+	WaitUntilBucketExistsWithContext(aws.Context, *s3.HeadBucketInput, ...request.WaiterOption) error
+
+	WaitUntilBucketNotExists(*s3.HeadBucketInput) error
+	WaitUntilBucketNotExistsWithContext(aws.Context, *s3.HeadBucketInput, ...request.WaiterOption) error
+
+	WaitUntilObjectExists(*s3.HeadObjectInput) error
+	WaitUntilObjectExistsWithContext(aws.Context, *s3.HeadObjectInput, ...request.WaiterOption) error
+
+	WaitUntilObjectNotExists(*s3.HeadObjectInput) error
+	WaitUntilObjectNotExistsWithContext(aws.Context, *s3.HeadObjectInput, ...request.WaiterOption) error
+}
+
+var _ S3API = (*s3.S3)(nil)
diff --git a/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/batch.go b/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/batch.go
new file mode 100644
index 0000000000..18215574bf
--- /dev/null
+++ b/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/batch.go
@@ -0,0 +1,529 @@
+package s3manager
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/client"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+)
+
+const (
+	// DefaultBatchSize is the batch size we initialize when constructing a batch delete client.
+	// This value is used when calling DeleteObjects. This represents how many objects to delete
+	// per DeleteObjects call.
+	DefaultBatchSize = 100
+)
+
+// BatchError will contain the key and bucket of the object that failed to
+// either upload or download.
+type BatchError struct {
+	Errors  Errors
+	code    string
+	message string
+}
+
+// Errors is a typed alias for a slice of errors to satisfy the error
+// interface.
+type Errors []Error
+
+func (errs Errors) Error() string {
+	buf := bytes.NewBuffer(nil)
+	for i, err := range errs {
+		buf.WriteString(err.Error())
+		if i+1 < len(errs) {
+			buf.WriteString("\n")
+		}
+	}
+	return buf.String()
+}
+
+// Error will contain the original error, bucket, and key of the operation that failed
+// during batch operations.
+type Error struct {
+	OrigErr error
+	Bucket  *string
+	Key     *string
+}
+
+func newError(err error, bucket, key *string) Error {
+	return Error{
+		err,
+		bucket,
+		key,
+	}
+}
+
+func (err *Error) Error() string {
+	origErr := ""
+	if err.OrigErr != nil {
+		origErr = ":\n" + err.OrigErr.Error()
+	}
+	return fmt.Sprintf("failed to perform batch operation on %q to %q%s",
+		aws.StringValue(err.Key),
+		aws.StringValue(err.Bucket),
+		origErr,
+	)
+}
+
+// NewBatchError will return a BatchError that satisfies the awserr.Error interface.
+func NewBatchError(code, message string, err []Error) awserr.Error {
+	return &BatchError{
+		Errors:  err,
+		code:    code,
+		message: message,
+	}
+}
+
+// Code will return the code associated with the batch error.
+func (err *BatchError) Code() string {
+	return err.code
+}
+
+// Message will return the message associated with the batch error.
+func (err *BatchError) Message() string {
+	return err.message
+}
+
+func (err *BatchError) Error() string {
+	return awserr.SprintError(err.Code(), err.Message(), "", err.Errors)
+}
+
+// OrigErr will return the original error. Which, in this case, will always be nil
+// for batched operations.
+func (err *BatchError) OrigErr() error {
+	return err.Errors
+}
+
+// BatchDeleteIterator is an interface that uses the scanner pattern to
+// iterate through what needs to be deleted.
+type BatchDeleteIterator interface {
+	Next() bool
+	Err() error
+	DeleteObject() BatchDeleteObject
+}
+
+// DeleteListIterator is an alternative iterator for the BatchDelete client. This will
+// iterate through a list of objects and delete the objects.
+//
+// Example:
+//	iter := &s3manager.DeleteListIterator{
+//		Client: svc,
+//		Input: &s3.ListObjectsInput{
+//			Bucket:  aws.String("bucket"),
+//			MaxKeys: aws.Int64(5),
+//		},
+//		Paginator: request.Pagination{
+//			NewRequest: func() (*request.Request, error) {
+//				var inCpy *ListObjectsInput
+//				if input != nil {
+//					tmp := *input
+//					inCpy = &tmp
+//				}
+//				req, _ := c.ListObjectsRequest(inCpy)
+//				return req, nil
+//			},
+//		},
+//	}
+//
+//	batcher := s3manager.NewBatchDeleteWithClient(svc)
+//	if err := batcher.Delete(aws.BackgroundContext(), iter); err != nil {
+//		return err
+//	}
+type DeleteListIterator struct {
+	Bucket    *string
+	Paginator request.Pagination
+	objects   []*s3.Object
+}
+
+// NewDeleteListIterator will return a new DeleteListIterator.
+func NewDeleteListIterator(svc s3iface.S3API, input *s3.ListObjectsInput, opts ...func(*DeleteListIterator)) BatchDeleteIterator {
+	iter := &DeleteListIterator{
+		Bucket: input.Bucket,
+		Paginator: request.Pagination{
+			NewRequest: func() (*request.Request, error) {
+				var inCpy *s3.ListObjectsInput
+				if input != nil {
+					tmp := *input
+					inCpy = &tmp
+				}
+				req, _ := svc.ListObjectsRequest(inCpy)
+				return req, nil
+			},
+		},
+	}
+
+	for _, opt := range opts {
+		opt(iter)
+	}
+	return iter
+}
+
+// Next will use the S3API client to iterate through a list of objects.
+func (iter *DeleteListIterator) Next() bool {
+	if len(iter.objects) > 0 {
+		iter.objects = iter.objects[1:]
+	}
+
+	if len(iter.objects) == 0 && iter.Paginator.Next() {
+		iter.objects = iter.Paginator.Page().(*s3.ListObjectsOutput).Contents
+	}
+
+	return len(iter.objects) > 0
+}
+
+// Err will return the last known error from Next.
+func (iter *DeleteListIterator) Err() error {
+	return iter.Paginator.Err()
+}
+
+// DeleteObject will return the current object to be deleted.
+func (iter *DeleteListIterator) DeleteObject() BatchDeleteObject {
+	return BatchDeleteObject{
+		Object: &s3.DeleteObjectInput{
+			Bucket: iter.Bucket,
+			Key:    iter.objects[0].Key,
+		},
+	}
+}
+
+// BatchDelete will use the s3 package's service client to perform a batch
+// delete.
+type BatchDelete struct {
+	Client    s3iface.S3API
+	BatchSize int
+}
+
+// NewBatchDeleteWithClient will return a new delete client that can delete a batched amount of
+// objects.
+//
+// Example:
+//	batcher := s3manager.NewBatchDeleteWithClient(client, size)
+//
+//	objects := []BatchDeleteObject{
+//		{
+//			Object:	&s3.DeleteObjectInput {
+//				Key: aws.String("key"),
+//				Bucket: aws.String("bucket"),
+//			},
+//		},
+//	}
+//
+//	if err := batcher.Delete(aws.BackgroundContext(), &s3manager.DeleteObjectsIterator{
+//		Objects: objects,
+//	}); err != nil {
+//		return err
+//	}
+func NewBatchDeleteWithClient(client s3iface.S3API, options ...func(*BatchDelete)) *BatchDelete {
+	svc := &BatchDelete{
+		Client:    client,
+		BatchSize: DefaultBatchSize,
+	}
+
+	for _, opt := range options {
+		opt(svc)
+	}
+
+	return svc
+}
+
+// NewBatchDelete will return a new delete client that can delete a batched amount of
+// objects.
+//
+// Example:
+//	batcher := s3manager.NewBatchDelete(sess, size)
+//
+//	objects := []BatchDeleteObject{
+//		{
+//			Object:	&s3.DeleteObjectInput {
+//				Key: aws.String("key"),
+//				Bucket: aws.String("bucket"),
+//			},
+//		},
+//	}
+//
+//	if err := batcher.Delete(aws.BackgroundContext(), &s3manager.DeleteObjectsIterator{
+//		Objects: objects,
+//	}); err != nil {
+//		return err
+//	}
+func NewBatchDelete(c client.ConfigProvider, options ...func(*BatchDelete)) *BatchDelete {
+	client := s3.New(c)
+	return NewBatchDeleteWithClient(client, options...)
+}
+
+// BatchDeleteObject is a wrapper object for calling the batch delete operation.
+type BatchDeleteObject struct {
+	Object *s3.DeleteObjectInput
+	// After will run after each iteration during the batch process. This function will
+	// be executed whether or not the request was successful.
+	After func() error
+}
+
+// DeleteObjectsIterator is an interface that uses the scanner pattern to iterate
+// through a series of objects to be deleted.
+type DeleteObjectsIterator struct {
+	Objects []BatchDeleteObject
+	index   int
+	inc     bool
+}
+
+// Next will increment the default iterator's index and and ensure that there
+// is another object to iterator to.
+func (iter *DeleteObjectsIterator) Next() bool {
+	if iter.inc {
+		iter.index++
+	} else {
+		iter.inc = true
+	}
+	return iter.index < len(iter.Objects)
+}
+
+// Err will return an error. Since this is just used to satisfy the BatchDeleteIterator interface
+// this will only return nil.
+func (iter *DeleteObjectsIterator) Err() error {
+	return nil
+}
+
+// DeleteObject will return the BatchDeleteObject at the current batched index.
+func (iter *DeleteObjectsIterator) DeleteObject() BatchDeleteObject {
+	object := iter.Objects[iter.index]
+	return object
+}
+
+// Delete will use the iterator to queue up objects that need to be deleted.
+// Once the batch size is met, this will call the deleteBatch function.
+func (d *BatchDelete) Delete(ctx aws.Context, iter BatchDeleteIterator) error {
+	var errs []Error
+	objects := []BatchDeleteObject{}
+	var input *s3.DeleteObjectsInput
+
+	for iter.Next() {
+		o := iter.DeleteObject()
+
+		if input == nil {
+			input = initDeleteObjectsInput(o.Object)
+		}
+
+		parity := hasParity(input, o)
+		if parity {
+			input.Delete.Objects = append(input.Delete.Objects, &s3.ObjectIdentifier{
+				Key:       o.Object.Key,
+				VersionId: o.Object.VersionId,
+			})
+			objects = append(objects, o)
+		}
+
+		if len(input.Delete.Objects) == d.BatchSize || !parity {
+			if err := deleteBatch(ctx, d, input, objects); err != nil {
+				errs = append(errs, err...)
+			}
+
+			objects = objects[:0]
+			input = nil
+
+			if !parity {
+				objects = append(objects, o)
+				input = initDeleteObjectsInput(o.Object)
+				input.Delete.Objects = append(input.Delete.Objects, &s3.ObjectIdentifier{
+					Key:       o.Object.Key,
+					VersionId: o.Object.VersionId,
+				})
+			}
+		}
+	}
+
+	// iter.Next() could return false (above) plus populate iter.Err()
+	if iter.Err() != nil {
+		errs = append(errs, newError(iter.Err(), nil, nil))
+	}
+
+	if input != nil && len(input.Delete.Objects) > 0 {
+		if err := deleteBatch(ctx, d, input, objects); err != nil {
+			errs = append(errs, err...)
+		}
+	}
+
+	if len(errs) > 0 {
+		return NewBatchError("BatchedDeleteIncomplete", "some objects have failed to be deleted.", errs)
+	}
+	return nil
+}
+
+func initDeleteObjectsInput(o *s3.DeleteObjectInput) *s3.DeleteObjectsInput {
+	return &s3.DeleteObjectsInput{
+		Bucket:       o.Bucket,
+		MFA:          o.MFA,
+		RequestPayer: o.RequestPayer,
+		Delete:       &s3.Delete{},
+	}
+}
+
+const (
+	// ErrDeleteBatchFailCode represents an error code which will be returned
+	// only when DeleteObjects.Errors has an error that does not contain a code.
+	ErrDeleteBatchFailCode       = "DeleteBatchError"
+	errDefaultDeleteBatchMessage = "failed to delete"
+)
+
+// deleteBatch will delete a batch of items in the objects parameters.
+func deleteBatch(ctx aws.Context, d *BatchDelete, input *s3.DeleteObjectsInput, objects []BatchDeleteObject) []Error {
+	errs := []Error{}
+
+	if result, err := d.Client.DeleteObjectsWithContext(ctx, input); err != nil {
+		for i := 0; i < len(input.Delete.Objects); i++ {
+			errs = append(errs, newError(err, input.Bucket, input.Delete.Objects[i].Key))
+		}
+	} else if len(result.Errors) > 0 {
+		for i := 0; i < len(result.Errors); i++ {
+			code := ErrDeleteBatchFailCode
+			msg := errDefaultDeleteBatchMessage
+			if result.Errors[i].Message != nil {
+				msg = *result.Errors[i].Message
+			}
+			if result.Errors[i].Code != nil {
+				code = *result.Errors[i].Code
+			}
+
+			errs = append(errs, newError(awserr.New(code, msg, err), input.Bucket, result.Errors[i].Key))
+		}
+	}
+	for _, object := range objects {
+		if object.After == nil {
+			continue
+		}
+		if err := object.After(); err != nil {
+			errs = append(errs, newError(err, object.Object.Bucket, object.Object.Key))
+		}
+	}
+
+	return errs
+}
+
+func hasParity(o1 *s3.DeleteObjectsInput, o2 BatchDeleteObject) bool {
+	if o1.Bucket != nil && o2.Object.Bucket != nil {
+		if *o1.Bucket != *o2.Object.Bucket {
+			return false
+		}
+	} else if o1.Bucket != o2.Object.Bucket {
+		return false
+	}
+
+	if o1.MFA != nil && o2.Object.MFA != nil {
+		if *o1.MFA != *o2.Object.MFA {
+			return false
+		}
+	} else if o1.MFA != o2.Object.MFA {
+		return false
+	}
+
+	if o1.RequestPayer != nil && o2.Object.RequestPayer != nil {
+		if *o1.RequestPayer != *o2.Object.RequestPayer {
+			return false
+		}
+	} else if o1.RequestPayer != o2.Object.RequestPayer {
+		return false
+	}
+
+	return true
+}
+
+// BatchDownloadIterator is an interface that uses the scanner pattern to iterate
+// through a series of objects to be downloaded.
+type BatchDownloadIterator interface {
+	Next() bool
+	Err() error
+	DownloadObject() BatchDownloadObject
+}
+
+// BatchDownloadObject contains all necessary information to run a batch operation once.
+type BatchDownloadObject struct {
+	Object *s3.GetObjectInput
+	Writer io.WriterAt
+	// After will run after each iteration during the batch process. This function will
+	// be executed whether or not the request was successful.
+	After func() error
+}
+
+// DownloadObjectsIterator implements the BatchDownloadIterator interface and allows for batched
+// download of objects.
+type DownloadObjectsIterator struct {
+	Objects []BatchDownloadObject
+	index   int
+	inc     bool
+}
+
+// Next will increment the default iterator's index and and ensure that there
+// is another object to iterator to.
+func (batcher *DownloadObjectsIterator) Next() bool {
+	if batcher.inc {
+		batcher.index++
+	} else {
+		batcher.inc = true
+	}
+	return batcher.index < len(batcher.Objects)
+}
+
+// DownloadObject will return the BatchDownloadObject at the current batched index.
+func (batcher *DownloadObjectsIterator) DownloadObject() BatchDownloadObject {
+	object := batcher.Objects[batcher.index]
+	return object
+}
+
+// Err will return an error. Since this is just used to satisfy the BatchDeleteIterator interface
+// this will only return nil.
+func (batcher *DownloadObjectsIterator) Err() error {
+	return nil
+}
+
+// BatchUploadIterator is an interface that uses the scanner pattern to
+// iterate through what needs to be uploaded.
+type BatchUploadIterator interface {
+	Next() bool
+	Err() error
+	UploadObject() BatchUploadObject
+}
+
+// UploadObjectsIterator implements the BatchUploadIterator interface and allows for batched
+// upload of objects.
+type UploadObjectsIterator struct {
+	Objects []BatchUploadObject
+	index   int
+	inc     bool
+}
+
+// Next will increment the default iterator's index and and ensure that there
+// is another object to iterator to.
+func (batcher *UploadObjectsIterator) Next() bool {
+	if batcher.inc {
+		batcher.index++
+	} else {
+		batcher.inc = true
+	}
+	return batcher.index < len(batcher.Objects)
+}
+
+// Err will return an error. Since this is just used to satisfy the BatchUploadIterator interface
+// this will only return nil.
+func (batcher *UploadObjectsIterator) Err() error {
+	return nil
+}
+
+// UploadObject will return the BatchUploadObject at the current batched index.
+func (batcher *UploadObjectsIterator) UploadObject() BatchUploadObject {
+	object := batcher.Objects[batcher.index]
+	return object
+}
+
+// BatchUploadObject contains all necessary information to run a batch operation once.
+type BatchUploadObject struct {
+	Object *UploadInput
+	// After will run after each iteration during the batch process. This function will
+	// be executed whether or not the request was successful.
+	After func() error
+}
diff --git a/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/bucket_region.go b/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/bucket_region.go
new file mode 100644
index 0000000000..f61665a58a
--- /dev/null
+++ b/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/bucket_region.go
@@ -0,0 +1,88 @@
+package s3manager
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/client"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+)
+
+// GetBucketRegion will attempt to get the region for a bucket using the
+// regionHint to determine which AWS partition to perform the query on.
+//
+// The request will not be signed, and will not use your AWS credentials.
+//
+// A "NotFound" error code will be returned if the bucket does not exist in the
+// AWS partition the regionHint belongs to. If the regionHint parameter is an
+// empty string GetBucketRegion will fallback to the ConfigProvider's region
+// config. If the regionHint is empty, and the ConfigProvider does not have a
+// region value, an error will be returned..
+//
+// For example to get the region of a bucket which exists in "eu-central-1"
+// you could provide a region hint of "us-west-2".
+//
+//    sess := session.Must(session.NewSession())
+//
+//    bucket := "my-bucket"
+//    region, err := s3manager.GetBucketRegion(ctx, sess, bucket, "us-west-2")
+//    if err != nil {
+//        if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" {
+//             fmt.Fprintf(os.Stderr, "unable to find bucket %s's region not found\n", bucket)
+//        }
+//        return err
+//    }
+//    fmt.Printf("Bucket %s is in %s region\n", bucket, region)
+//
+func GetBucketRegion(ctx aws.Context, c client.ConfigProvider, bucket, regionHint string, opts ...request.Option) (string, error) {
+	var cfg aws.Config
+	if len(regionHint) != 0 {
+		cfg.Region = aws.String(regionHint)
+	}
+	svc := s3.New(c, &cfg)
+	return GetBucketRegionWithClient(ctx, svc, bucket, opts...)
+}
+
+const bucketRegionHeader = "X-Amz-Bucket-Region"
+
+// GetBucketRegionWithClient is the same as GetBucketRegion with the exception
+// that it takes a S3 service client instead of a Session. The regionHint is
+// derived from the region the S3 service client was created in.
+//
+// See GetBucketRegion for more information.
+func GetBucketRegionWithClient(ctx aws.Context, svc s3iface.S3API, bucket string, opts ...request.Option) (string, error) {
+	req, _ := svc.HeadBucketRequest(&s3.HeadBucketInput{
+		Bucket: aws.String(bucket),
+	})
+	req.Config.S3ForcePathStyle = aws.Bool(true)
+	req.Config.Credentials = credentials.AnonymousCredentials
+	req.SetContext(ctx)
+
+	// Disable HTTP redirects to prevent an invalid 301 from eating the response
+	// because Go's HTTP client will fail, and drop the response if an 301 is
+	// received without a location header. S3 will return a 301 without the
+	// location header for HeadObject API calls.
+	req.DisableFollowRedirects = true
+
+	var bucketRegion string
+	req.Handlers.Send.PushBack(func(r *request.Request) {
+		bucketRegion = r.HTTPResponse.Header.Get(bucketRegionHeader)
+		if len(bucketRegion) == 0 {
+			return
+		}
+		r.HTTPResponse.StatusCode = 200
+		r.HTTPResponse.Status = "OK"
+		r.Error = nil
+	})
+
+	req.ApplyOptions(opts...)
+
+	if err := req.Send(); err != nil {
+		return "", err
+	}
+
+	bucketRegion = s3.NormalizeBucketLocation(bucketRegion)
+
+	return bucketRegion, nil
+}
diff --git a/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/doc.go b/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/doc.go
new file mode 100644
index 0000000000..229c0d63bd
--- /dev/null
+++ b/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/doc.go
@@ -0,0 +1,3 @@
+// Package s3manager provides utilities to upload and download objects from
+// S3 concurrently. Helpful for when working with large objects.
+package s3manager
diff --git a/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/download.go b/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/download.go
new file mode 100644
index 0000000000..4a9ad65e49
--- /dev/null
+++ b/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/download.go
@@ -0,0 +1,555 @@
+package s3manager
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/awsutil"
+	"github.com/aws/aws-sdk-go/aws/client"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+)
+
+// DefaultDownloadPartSize is the default range of bytes to get at a time when
+// using Download().
+const DefaultDownloadPartSize = 1024 * 1024 * 5
+
+// DefaultDownloadConcurrency is the default number of goroutines to spin up
+// when using Download().
+const DefaultDownloadConcurrency = 5
+
+// The Downloader structure that calls Download(). It is safe to call Download()
+// on this structure for multiple objects and across concurrent goroutines.
+// Mutating the Downloader's properties is not safe to be done concurrently.
+type Downloader struct {
+	// The buffer size (in bytes) to use when buffering data into chunks and
+	// sending them as parts to S3. The minimum allowed part size is 5MB, and
+	// if this value is set to zero, the DefaultDownloadPartSize value will be used.
+	//
+	// PartSize is ignored if the Range input parameter is provided.
+	PartSize int64
+
+	// The number of goroutines to spin up in parallel when sending parts.
+	// If this is set to zero, the DefaultDownloadConcurrency value will be used.
+	//
+	// Concurrency of 1 will download the parts sequentially.
+	//
+	// Concurrency is ignored if the Range input parameter is provided.
+	Concurrency int
+
+	// An S3 client to use when performing downloads.
+	S3 s3iface.S3API
+
+	// List of request options that will be passed down to individual API
+	// operation requests made by the downloader.
+	RequestOptions []request.Option
+}
+
+// WithDownloaderRequestOptions appends to the Downloader's API request options.
+func WithDownloaderRequestOptions(opts ...request.Option) func(*Downloader) {
+	return func(d *Downloader) {
+		d.RequestOptions = append(d.RequestOptions, opts...)
+	}
+}
+
+// NewDownloader creates a new Downloader instance to downloads objects from
+// S3 in concurrent chunks. Pass in additional functional options  to customize
+// the downloader behavior. Requires a client.ConfigProvider in order to create
+// a S3 service client. The session.Session satisfies the client.ConfigProvider
+// interface.
+//
+// Example:
+//     // The session the S3 Downloader will use
+//     sess := session.Must(session.NewSession())
+//
+//     // Create a downloader with the session and default options
+//     downloader := s3manager.NewDownloader(sess)
+//
+//     // Create a downloader with the session and custom options
+//     downloader := s3manager.NewDownloader(sess, func(d *s3manager.Downloader) {
+//          d.PartSize = 64 * 1024 * 1024 // 64MB per part
+//     })
+func NewDownloader(c client.ConfigProvider, options ...func(*Downloader)) *Downloader {
+	d := &Downloader{
+		S3:          s3.New(c),
+		PartSize:    DefaultDownloadPartSize,
+		Concurrency: DefaultDownloadConcurrency,
+	}
+	for _, option := range options {
+		option(d)
+	}
+
+	return d
+}
+
+// NewDownloaderWithClient creates a new Downloader instance to downloads
+// objects from S3 in concurrent chunks. Pass in additional functional
+// options to customize the downloader behavior. Requires a S3 service client
+// to make S3 API calls.
+//
+// Example:
+//     // The session the S3 Downloader will use
+//     sess := session.Must(session.NewSession())
+//
+//     // The S3 client the S3 Downloader will use
+//     s3Svc := s3.new(sess)
+//
+//     // Create a downloader with the s3 client and default options
+//     downloader := s3manager.NewDownloaderWithClient(s3Svc)
+//
+//     // Create a downloader with the s3 client and custom options
+//     downloader := s3manager.NewDownloaderWithClient(s3Svc, func(d *s3manager.Downloader) {
+//          d.PartSize = 64 * 1024 * 1024 // 64MB per part
+//     })
+func NewDownloaderWithClient(svc s3iface.S3API, options ...func(*Downloader)) *Downloader {
+	d := &Downloader{
+		S3:          svc,
+		PartSize:    DefaultDownloadPartSize,
+		Concurrency: DefaultDownloadConcurrency,
+	}
+	for _, option := range options {
+		option(d)
+	}
+
+	return d
+}
+
+type maxRetrier interface {
+	MaxRetries() int
+}
+
+// Download downloads an object in S3 and writes the payload into w using
+// concurrent GET requests.
+//
+// Additional functional options can be provided to configure the individual
+// download. These options are copies of the Downloader instance Download is called from.
+// Modifying the options will not impact the original Downloader instance.
+//
+// It is safe to call this method concurrently across goroutines.
+//
+// The w io.WriterAt can be satisfied by an os.File to do multipart concurrent
+// downloads, or in memory []byte wrapper using aws.WriteAtBuffer.
+//
+// Specifying a Downloader.Concurrency of 1 will cause the Downloader to
+// download the parts from S3 sequentially.
+//
+// If the GetObjectInput's Range value is provided that will cause the downloader
+// to perform a single GetObjectInput request for that object's range. This will
+// caused the part size, and concurrency configurations to be ignored.
+func (d Downloader) Download(w io.WriterAt, input *s3.GetObjectInput, options ...func(*Downloader)) (n int64, err error) {
+	return d.DownloadWithContext(aws.BackgroundContext(), w, input, options...)
+}
+
+// DownloadWithContext downloads an object in S3 and writes the payload into w
+// using concurrent GET requests.
+//
+// DownloadWithContext is the same as Download with the additional support for
+// Context input parameters. The Context must not be nil. A nil Context will
+// cause a panic. Use the Context to add deadlining, timeouts, etc. The
+// DownloadWithContext may create sub-contexts for individual underlying
+// requests.
+//
+// Additional functional options can be provided to configure the individual
+// download. These options are copies of the Downloader instance Download is
+// called from. Modifying the options will not impact the original Downloader
+// instance. Use the WithDownloaderRequestOptions helper function to pass in request
+// options that will be applied to all API operations made with this downloader.
+//
+// The w io.WriterAt can be satisfied by an os.File to do multipart concurrent
+// downloads, or in memory []byte wrapper using aws.WriteAtBuffer.
+//
+// Specifying a Downloader.Concurrency of 1 will cause the Downloader to
+// download the parts from S3 sequentially.
+//
+// It is safe to call this method concurrently across goroutines.
+//
+// If the GetObjectInput's Range value is provided that will cause the downloader
+// to perform a single GetObjectInput request for that object's range. This will
+// caused the part size, and concurrency configurations to be ignored.
+func (d Downloader) DownloadWithContext(ctx aws.Context, w io.WriterAt, input *s3.GetObjectInput, options ...func(*Downloader)) (n int64, err error) {
+	impl := downloader{w: w, in: input, cfg: d, ctx: ctx}
+
+	for _, option := range options {
+		option(&impl.cfg)
+	}
+	impl.cfg.RequestOptions = append(impl.cfg.RequestOptions, request.WithAppendUserAgent("S3Manager"))
+
+	if s, ok := d.S3.(maxRetrier); ok {
+		impl.partBodyMaxRetries = s.MaxRetries()
+	}
+
+	impl.totalBytes = -1
+	if impl.cfg.Concurrency == 0 {
+		impl.cfg.Concurrency = DefaultDownloadConcurrency
+	}
+
+	if impl.cfg.PartSize == 0 {
+		impl.cfg.PartSize = DefaultDownloadPartSize
+	}
+
+	return impl.download()
+}
+
+// DownloadWithIterator will download a batched amount of objects in S3 and writes them
+// to the io.WriterAt specificed in the iterator.
+//
+// Example:
+//	svc := s3manager.NewDownloader(session)
+//
+//	fooFile, err := os.Open("/tmp/foo.file")
+//	if err != nil {
+//		return err
+//	}
+//
+//	barFile, err := os.Open("/tmp/bar.file")
+//	if err != nil {
+//		return err
+//	}
+//
+//	objects := []s3manager.BatchDownloadObject {
+//		{
+//			Object: &s3.GetObjectInput {
+//				Bucket: aws.String("bucket"),
+//				Key: aws.String("foo"),
+//			},
+//			Writer: fooFile,
+//		},
+//		{
+//			Object: &s3.GetObjectInput {
+//				Bucket: aws.String("bucket"),
+//				Key: aws.String("bar"),
+//			},
+//			Writer: barFile,
+//		},
+//	}
+//
+//	iter := &s3manager.DownloadObjectsIterator{Objects: objects}
+//	if err := svc.DownloadWithIterator(aws.BackgroundContext(), iter); err != nil {
+//		return err
+//	}
+func (d Downloader) DownloadWithIterator(ctx aws.Context, iter BatchDownloadIterator, opts ...func(*Downloader)) error {
+	var errs []Error
+	for iter.Next() {
+		object := iter.DownloadObject()
+		if _, err := d.DownloadWithContext(ctx, object.Writer, object.Object, opts...); err != nil {
+			errs = append(errs, newError(err, object.Object.Bucket, object.Object.Key))
+		}
+
+		if object.After == nil {
+			continue
+		}
+
+		if err := object.After(); err != nil {
+			errs = append(errs, newError(err, object.Object.Bucket, object.Object.Key))
+		}
+	}
+
+	if len(errs) > 0 {
+		return NewBatchError("BatchedDownloadIncomplete", "some objects have failed to download.", errs)
+	}
+	return nil
+}
+
+// downloader is the implementation structure used internally by Downloader.
+type downloader struct {
+	ctx aws.Context
+	cfg Downloader
+
+	in *s3.GetObjectInput
+	w  io.WriterAt
+
+	wg sync.WaitGroup
+	m  sync.Mutex
+
+	pos        int64
+	totalBytes int64
+	written    int64
+	err        error
+
+	partBodyMaxRetries int
+}
+
+// download performs the implementation of the object download across ranged
+// GETs.
+func (d *downloader) download() (n int64, err error) {
+	// If range is specified fall back to single download of that range
+	// this enables the functionality of ranged gets with the downloader but
+	// at the cost of no multipart downloads.
+	if rng := aws.StringValue(d.in.Range); len(rng) > 0 {
+		d.downloadRange(rng)
+		return d.written, d.err
+	}
+
+	// Spin off first worker to check additional header information
+	d.getChunk()
+
+	if total := d.getTotalBytes(); total >= 0 {
+		// Spin up workers
+		ch := make(chan dlchunk, d.cfg.Concurrency)
+
+		for i := 0; i < d.cfg.Concurrency; i++ {
+			d.wg.Add(1)
+			go d.downloadPart(ch)
+		}
+
+		// Assign work
+		for d.getErr() == nil {
+			if d.pos >= total {
+				break // We're finished queuing chunks
+			}
+
+			// Queue the next range of bytes to read.
+			ch <- dlchunk{w: d.w, start: d.pos, size: d.cfg.PartSize}
+			d.pos += d.cfg.PartSize
+		}
+
+		// Wait for completion
+		close(ch)
+		d.wg.Wait()
+	} else {
+		// Checking if we read anything new
+		for d.err == nil {
+			d.getChunk()
+		}
+
+		// We expect a 416 error letting us know we are done downloading the
+		// total bytes. Since we do not know the content's length, this will
+		// keep grabbing chunks of data until the range of bytes specified in
+		// the request is out of range of the content. Once, this happens, a
+		// 416 should occur.
+		e, ok := d.err.(awserr.RequestFailure)
+		if ok && e.StatusCode() == http.StatusRequestedRangeNotSatisfiable {
+			d.err = nil
+		}
+	}
+
+	// Return error
+	return d.written, d.err
+}
+
+// downloadPart is an individual goroutine worker reading from the ch channel
+// and performing a GetObject request on the data with a given byte range.
+//
+// If this is the first worker, this operation also resolves the total number
+// of bytes to be read so that the worker manager knows when it is finished.
+func (d *downloader) downloadPart(ch chan dlchunk) {
+	defer d.wg.Done()
+	for {
+		chunk, ok := <-ch
+		if !ok {
+			break
+		}
+		if d.getErr() != nil {
+			// Drain the channel if there is an error, to prevent deadlocking
+			// of download producer.
+			continue
+		}
+
+		if err := d.downloadChunk(chunk); err != nil {
+			d.setErr(err)
+		}
+	}
+}
+
+// getChunk grabs a chunk of data from the body.
+// Not thread safe. Should only used when grabbing data on a single thread.
+func (d *downloader) getChunk() {
+	if d.getErr() != nil {
+		return
+	}
+
+	chunk := dlchunk{w: d.w, start: d.pos, size: d.cfg.PartSize}
+	d.pos += d.cfg.PartSize
+
+	if err := d.downloadChunk(chunk); err != nil {
+		d.setErr(err)
+	}
+}
+
+// downloadRange downloads an Object given the passed in Byte-Range value.
+// The chunk used down download the range will be configured for that range.
+func (d *downloader) downloadRange(rng string) {
+	if d.getErr() != nil {
+		return
+	}
+
+	chunk := dlchunk{w: d.w, start: d.pos}
+	// Ranges specified will short circuit the multipart download
+	chunk.withRange = rng
+
+	if err := d.downloadChunk(chunk); err != nil {
+		d.setErr(err)
+	}
+
+	// Update the position based on the amount of data received.
+	d.pos = d.written
+}
+
+// downloadChunk downloads the chunk from s3
+func (d *downloader) downloadChunk(chunk dlchunk) error {
+	in := &s3.GetObjectInput{}
+	awsutil.Copy(in, d.in)
+
+	// Get the next byte range of data
+	in.Range = aws.String(chunk.ByteRange())
+
+	var n int64
+	var err error
+	for retry := 0; retry <= d.partBodyMaxRetries; retry++ {
+		var resp *s3.GetObjectOutput
+		resp, err = d.cfg.S3.GetObjectWithContext(d.ctx, in, d.cfg.RequestOptions...)
+		if err != nil {
+			return err
+		}
+		d.setTotalBytes(resp) // Set total if not yet set.
+
+		n, err = io.Copy(&chunk, resp.Body)
+		resp.Body.Close()
+		if err == nil {
+			break
+		}
+
+		chunk.cur = 0
+		logMessage(d.cfg.S3, aws.LogDebugWithRequestRetries,
+			fmt.Sprintf("DEBUG: object part body download interrupted %s, err, %v, retrying attempt %d",
+				aws.StringValue(in.Key), err, retry))
+	}
+
+	d.incrWritten(n)
+
+	return err
+}
+
+func logMessage(svc s3iface.S3API, level aws.LogLevelType, msg string) {
+	s, ok := svc.(*s3.S3)
+	if !ok {
+		return
+	}
+
+	if s.Config.Logger == nil {
+		return
+	}
+
+	if s.Config.LogLevel.Matches(level) {
+		s.Config.Logger.Log(msg)
+	}
+}
+
+// getTotalBytes is a thread-safe getter for retrieving the total byte status.
+func (d *downloader) getTotalBytes() int64 {
+	d.m.Lock()
+	defer d.m.Unlock()
+
+	return d.totalBytes
+}
+
+// setTotalBytes is a thread-safe setter for setting the total byte status.
+// Will extract the object's total bytes from the Content-Range if the file
+// will be chunked, or Content-Length. Content-Length is used when the response
+// does not include a Content-Range. Meaning the object was not chunked. This
+// occurs when the full file fits within the PartSize directive.
+func (d *downloader) setTotalBytes(resp *s3.GetObjectOutput) {
+	d.m.Lock()
+	defer d.m.Unlock()
+
+	if d.totalBytes >= 0 {
+		return
+	}
+
+	if resp.ContentRange == nil {
+		// ContentRange is nil when the full file contents is provided, and
+		// is not chunked. Use ContentLength instead.
+		if resp.ContentLength != nil {
+			d.totalBytes = *resp.ContentLength
+			return
+		}
+	} else {
+		parts := strings.Split(*resp.ContentRange, "/")
+
+		total := int64(-1)
+		var err error
+		// Checking for whether or not a numbered total exists
+		// If one does not exist, we will assume the total to be -1, undefined,
+		// and sequentially download each chunk until hitting a 416 error
+		totalStr := parts[len(parts)-1]
+		if totalStr != "*" {
+			total, err = strconv.ParseInt(totalStr, 10, 64)
+			if err != nil {
+				d.err = err
+				return
+			}
+		}
+
+		d.totalBytes = total
+	}
+}
+
+func (d *downloader) incrWritten(n int64) {
+	d.m.Lock()
+	defer d.m.Unlock()
+
+	d.written += n
+}
+
+// getErr is a thread-safe getter for the error object
+func (d *downloader) getErr() error {
+	d.m.Lock()
+	defer d.m.Unlock()
+
+	return d.err
+}
+
+// setErr is a thread-safe setter for the error object
+func (d *downloader) setErr(e error) {
+	d.m.Lock()
+	defer d.m.Unlock()
+
+	d.err = e
+}
+
+// dlchunk represents a single chunk of data to write by the worker routine.
+// This structure also implements an io.SectionReader style interface for
+// io.WriterAt, effectively making it an io.SectionWriter (which does not
+// exist).
+type dlchunk struct {
+	w     io.WriterAt
+	start int64
+	size  int64
+	cur   int64
+
+	// specifies the byte range the chunk should be downloaded with.
+	withRange string
+}
+
+// Write wraps io.WriterAt for the dlchunk, writing from the dlchunk's start
+// position to its end (or EOF).
+//
+// If a range is specified on the dlchunk the size will be ignored when writing.
+// as the total size may not of be known ahead of time.
+func (c *dlchunk) Write(p []byte) (n int, err error) {
+	if c.cur >= c.size && len(c.withRange) == 0 {
+		return 0, io.EOF
+	}
+
+	n, err = c.w.WriteAt(p, c.start+c.cur)
+	c.cur += int64(n)
+
+	return
+}
+
+// ByteRange returns a HTTP Byte-Range header value that should be used by the
+// client to request the chunk's range.
+func (c *dlchunk) ByteRange() string {
+	if len(c.withRange) != 0 {
+		return c.withRange
+	}
+
+	return fmt.Sprintf("bytes=%d-%d", c.start, c.start+c.size-1)
+}
diff --git a/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/upload.go b/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/upload.go
new file mode 100644
index 0000000000..0ccf634c5b
--- /dev/null
+++ b/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/upload.go
@@ -0,0 +1,802 @@
+package s3manager
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/awsutil"
+	"github.com/aws/aws-sdk-go/aws/client"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+)
+
+// MaxUploadParts is the maximum allowed number of parts in a multi-part upload
+// on Amazon S3.
+const MaxUploadParts = 10000
+
+// MinUploadPartSize is the minimum allowed part size when uploading a part to
+// Amazon S3.
+const MinUploadPartSize int64 = 1024 * 1024 * 5
+
+// DefaultUploadPartSize is the default part size to buffer chunks of a
+// payload into.
+const DefaultUploadPartSize = MinUploadPartSize
+
+// DefaultUploadConcurrency is the default number of goroutines to spin up when
+// using Upload().
+const DefaultUploadConcurrency = 5
+
+// A MultiUploadFailure wraps a failed S3 multipart upload. An error returned
+// will satisfy this interface when a multi part upload failed to upload all
+// chucks to S3. In the case of a failure the UploadID is needed to operate on
+// the chunks, if any, which were uploaded.
+//
+// Example:
+//
+//     u := s3manager.NewUploader(opts)
+//     output, err := u.upload(input)
+//     if err != nil {
+//         if multierr, ok := err.(s3manager.MultiUploadFailure); ok {
+//             // Process error and its associated uploadID
+//             fmt.Println("Error:", multierr.Code(), multierr.Message(), multierr.UploadID())
+//         } else {
+//             // Process error generically
+//             fmt.Println("Error:", err.Error())
+//         }
+//     }
+//
+type MultiUploadFailure interface {
+	awserr.Error
+
+	// Returns the upload id for the S3 multipart upload that failed.
+	UploadID() string
+}
+
+// So that the Error interface type can be included as an anonymous field
+// in the multiUploadError struct and not conflict with the error.Error() method.
+type awsError awserr.Error
+
+// A multiUploadError wraps the upload ID of a failed s3 multipart upload.
+// Composed of BaseError for code, message, and original error
+//
+// Should be used for an error that occurred failing a S3 multipart upload,
+// and a upload ID is available. If an uploadID is not available a more relevant
+type multiUploadError struct {
+	awsError
+
+	// ID for multipart upload which failed.
+	uploadID string
+}
+
+// Error returns the string representation of the error.
+//
+// See apierr.BaseError ErrorWithExtra for output format
+//
+// Satisfies the error interface.
+func (m multiUploadError) Error() string {
+	extra := fmt.Sprintf("upload id: %s", m.uploadID)
+	return awserr.SprintError(m.Code(), m.Message(), extra, m.OrigErr())
+}
+
+// String returns the string representation of the error.
+// Alias for Error to satisfy the stringer interface.
+func (m multiUploadError) String() string {
+	return m.Error()
+}
+
+// UploadID returns the id of the S3 upload which failed.
+func (m multiUploadError) UploadID() string {
+	return m.uploadID
+}
+
+// UploadInput contains all input for upload requests to Amazon S3.
+type UploadInput struct {
+	// The canned ACL to apply to the object.
+	ACL *string `location:"header" locationName:"x-amz-acl" type:"string"`
+
+	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
+
+	// Specifies caching behavior along the request/reply chain.
+	CacheControl *string `location:"header" locationName:"Cache-Control" type:"string"`
+
+	// Specifies presentational information for the object.
+	ContentDisposition *string `location:"header" locationName:"Content-Disposition" type:"string"`
+
+	// Specifies what content encodings have been applied to the object and thus
+	// what decoding mechanisms must be applied to obtain the media-type referenced
+	// by the Content-Type header field.
+	ContentEncoding *string `location:"header" locationName:"Content-Encoding" type:"string"`
+
+	// The language the content is in.
+	ContentLanguage *string `location:"header" locationName:"Content-Language" type:"string"`
+
+	// The base64-encoded 128-bit MD5 digest of the part data.
+	ContentMD5 *string `location:"header" locationName:"Content-MD5" type:"string"`
+
+	// A standard MIME type describing the format of the object data.
+	ContentType *string `location:"header" locationName:"Content-Type" type:"string"`
+
+	// The date and time at which the object is no longer cacheable.
+	Expires *time.Time `location:"header" locationName:"Expires" type:"timestamp" timestampFormat:"rfc822"`
+
+	// Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object.
+	GrantFullControl *string `location:"header" locationName:"x-amz-grant-full-control" type:"string"`
+
+	// Allows grantee to read the object data and its metadata.
+	GrantRead *string `location:"header" locationName:"x-amz-grant-read" type:"string"`
+
+	// Allows grantee to read the object ACL.
+	GrantReadACP *string `location:"header" locationName:"x-amz-grant-read-acp" type:"string"`
+
+	// Allows grantee to write the ACL for the applicable object.
+	GrantWriteACP *string `location:"header" locationName:"x-amz-grant-write-acp" type:"string"`
+
+	Key *string `location:"uri" locationName:"Key" type:"string" required:"true"`
+
+	// A map of metadata to store with the object in S3.
+	Metadata map[string]*string `location:"headers" locationName:"x-amz-meta-" type:"map"`
+
+	// Confirms that the requester knows that she or he will be charged for the
+	// request. Bucket owners need not specify this parameter in their requests.
+	// Documentation on downloading objects from requester pays buckets can be found
+	// at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
+	RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string"`
+
+	// Specifies the algorithm to use to when encrypting the object (e.g., AES256,
+	// aws:kms).
+	SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"`
+
+	// Specifies the customer-provided encryption key for Amazon S3 to use in encrypting
+	// data. This value is used to store the object and then it is discarded; Amazon
+	// does not store the encryption key. The key must be appropriate for use with
+	// the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm
+	// header.
+	SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"`
+
+	// Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321.
+	// Amazon S3 uses this header for a message integrity check to ensure the encryption
+	// key was transmitted without error.
+	SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"`
+
+	// Specifies the AWS KMS key ID to use for object encryption. All GET and PUT
+	// requests for an object protected by AWS KMS will fail if not made via SSL
+	// or using SigV4. Documentation on configuring any of the officially supported
+	// AWS SDKs and CLI can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version
+	SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"`
+
+	// The Server-side encryption algorithm used when storing this object in S3
+	// (e.g., AES256, aws:kms).
+	ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string"`
+
+	// The type of storage to use for the object. Defaults to 'STANDARD'.
+	StorageClass *string `location:"header" locationName:"x-amz-storage-class" type:"string"`
+
+	// The tag-set for the object. The tag-set must be encoded as URL Query parameters
+	Tagging *string `location:"header" locationName:"x-amz-tagging" type:"string"`
+
+	// If the bucket is configured as a website, redirects requests for this object
+	// to another object in the same bucket or to an external URL. Amazon S3 stores
+	// the value of this header in the object metadata.
+	WebsiteRedirectLocation *string `location:"header" locationName:"x-amz-website-redirect-location" type:"string"`
+
+	// The readable body payload to send to S3.
+	Body io.Reader
+}
+
+// UploadOutput represents a response from the Upload() call.
+type UploadOutput struct {
+	// The URL where the object was uploaded to.
+	Location string
+
+	// The version of the object that was uploaded. Will only be populated if
+	// the S3 Bucket is versioned. If the bucket is not versioned this field
+	// will not be set.
+	VersionID *string
+
+	// The ID for a multipart upload to S3. In the case of an error the error
+	// can be cast to the MultiUploadFailure interface to extract the upload ID.
+	UploadID string
+}
+
+// WithUploaderRequestOptions appends to the Uploader's API request options.
+func WithUploaderRequestOptions(opts ...request.Option) func(*Uploader) {
+	return func(u *Uploader) {
+		u.RequestOptions = append(u.RequestOptions, opts...)
+	}
+}
+
+// The Uploader structure that calls Upload(). It is safe to call Upload()
+// on this structure for multiple objects and across concurrent goroutines.
+// Mutating the Uploader's properties is not safe to be done concurrently.
+type Uploader struct {
+	// The buffer size (in bytes) to use when buffering data into chunks and
+	// sending them as parts to S3. The minimum allowed part size is 5MB, and
+	// if this value is set to zero, the DefaultUploadPartSize value will be used.
+	PartSize int64
+
+	// The number of goroutines to spin up in parallel per call to Upload when
+	// sending parts. If this is set to zero, the DefaultUploadConcurrency value
+	// will be used.
+	//
+	// The concurrency pool is not shared between calls to Upload.
+	Concurrency int
+
+	// Setting this value to true will cause the SDK to avoid calling
+	// AbortMultipartUpload on a failure, leaving all successfully uploaded
+	// parts on S3 for manual recovery.
+	//
+	// Note that storing parts of an incomplete multipart upload counts towards
+	// space usage on S3 and will add additional costs if not cleaned up.
+	LeavePartsOnError bool
+
+	// MaxUploadParts is the max number of parts which will be uploaded to S3.
+	// Will be used to calculate the partsize of the object to be uploaded.
+	// E.g: 5GB file, with MaxUploadParts set to 100, will upload the file
+	// as 100, 50MB parts.
+	// With a limited of s3.MaxUploadParts (10,000 parts).
+	//
+	// Defaults to package const's MaxUploadParts value.
+	MaxUploadParts int
+
+	// The client to use when uploading to S3.
+	S3 s3iface.S3API
+
+	// List of request options that will be passed down to individual API
+	// operation requests made by the uploader.
+	RequestOptions []request.Option
+}
+
+// NewUploader creates a new Uploader instance to upload objects to S3. Pass In
+// additional functional options to customize the uploader's behavior. Requires a
+// client.ConfigProvider in order to create a S3 service client. The session.Session
+// satisfies the client.ConfigProvider interface.
+//
+// Example:
+//     // The session the S3 Uploader will use
+//     sess := session.Must(session.NewSession())
+//
+//     // Create an uploader with the session and default options
+//     uploader := s3manager.NewUploader(sess)
+//
+//     // Create an uploader with the session and custom options
+//     uploader := s3manager.NewUploader(session, func(u *s3manager.Uploader) {
+//          u.PartSize = 64 * 1024 * 1024 // 64MB per part
+//     })
+func NewUploader(c client.ConfigProvider, options ...func(*Uploader)) *Uploader {
+	u := &Uploader{
+		S3:                s3.New(c),
+		PartSize:          DefaultUploadPartSize,
+		Concurrency:       DefaultUploadConcurrency,
+		LeavePartsOnError: false,
+		MaxUploadParts:    MaxUploadParts,
+	}
+
+	for _, option := range options {
+		option(u)
+	}
+
+	return u
+}
+
+// NewUploaderWithClient creates a new Uploader instance to upload objects to S3. Pass in
+// additional functional options to customize the uploader's behavior. Requires
+// a S3 service client to make S3 API calls.
+//
+// Example:
+//     // The session the S3 Uploader will use
+//     sess := session.Must(session.NewSession())
+//
+//     // S3 service client the Upload manager will use.
+//     s3Svc := s3.New(sess)
+//
+//     // Create an uploader with S3 client and default options
+//     uploader := s3manager.NewUploaderWithClient(s3Svc)
+//
+//     // Create an uploader with S3 client and custom options
+//     uploader := s3manager.NewUploaderWithClient(s3Svc, func(u *s3manager.Uploader) {
+//          u.PartSize = 64 * 1024 * 1024 // 64MB per part
+//     })
+func NewUploaderWithClient(svc s3iface.S3API, options ...func(*Uploader)) *Uploader {
+	u := &Uploader{
+		S3:                svc,
+		PartSize:          DefaultUploadPartSize,
+		Concurrency:       DefaultUploadConcurrency,
+		LeavePartsOnError: false,
+		MaxUploadParts:    MaxUploadParts,
+	}
+
+	for _, option := range options {
+		option(u)
+	}
+
+	return u
+}
+
+// Upload uploads an object to S3, intelligently buffering large files into
+// smaller chunks and sending them in parallel across multiple goroutines. You
+// can configure the buffer size and concurrency through the Uploader's parameters.
+//
+// Additional functional options can be provided to configure the individual
+// upload. These options are copies of the Uploader instance Upload is called from.
+// Modifying the options will not impact the original Uploader instance.
+//
+// Use the WithUploaderRequestOptions helper function to pass in request
+// options that will be applied to all API operations made with this uploader.
+//
+// It is safe to call this method concurrently across goroutines.
+//
+// Example:
+//     // Upload input parameters
+//     upParams := &s3manager.UploadInput{
+//         Bucket: &bucketName,
+//         Key:    &keyName,
+//         Body:   file,
+//     }
+//
+//     // Perform an upload.
+//     result, err := uploader.Upload(upParams)
+//
+//     // Perform upload with options different than the those in the Uploader.
+//     result, err := uploader.Upload(upParams, func(u *s3manager.Uploader) {
+//          u.PartSize = 10 * 1024 * 1024 // 10MB part size
+//          u.LeavePartsOnError = true    // Don't delete the parts if the upload fails.
+//     })
+func (u Uploader) Upload(input *UploadInput, options ...func(*Uploader)) (*UploadOutput, error) {
+	return u.UploadWithContext(aws.BackgroundContext(), input, options...)
+}
+
+// UploadWithContext uploads an object to S3, intelligently buffering large
+// files into smaller chunks and sending them in parallel across multiple
+// goroutines. You can configure the buffer size and concurrency through the
+// Uploader's parameters.
+//
+// UploadWithContext is the same as Upload with the additional support for
+// Context input parameters. The Context must not be nil. A nil Context will
+// cause a panic. Use the context to add deadlining, timeouts, etc. The
+// UploadWithContext may create sub-contexts for individual underlying requests.
+//
+// Additional functional options can be provided to configure the individual
+// upload. These options are copies of the Uploader instance Upload is called from.
+// Modifying the options will not impact the original Uploader instance.
+//
+// Use the WithUploaderRequestOptions helper function to pass in request
+// options that will be applied to all API operations made with this uploader.
+//
+// It is safe to call this method concurrently across goroutines.
+func (u Uploader) UploadWithContext(ctx aws.Context, input *UploadInput, opts ...func(*Uploader)) (*UploadOutput, error) {
+	i := uploader{in: input, cfg: u, ctx: ctx}
+
+	for _, opt := range opts {
+		opt(&i.cfg)
+	}
+	i.cfg.RequestOptions = append(i.cfg.RequestOptions, request.WithAppendUserAgent("S3Manager"))
+
+	return i.upload()
+}
+
+// UploadWithIterator will upload a batched amount of objects to S3. This operation uses
+// the iterator pattern to know which object to upload next. Since this is an interface this
+// allows for custom defined functionality.
+//
+// Example:
+//	svc:= s3manager.NewUploader(sess)
+//
+//	objects := []BatchUploadObject{
+//		{
+//			Object:	&s3manager.UploadInput {
+//				Key: aws.String("key"),
+//				Bucket: aws.String("bucket"),
+//			},
+//		},
+//	}
+//
+//	iter := &s3manager.UploadObjectsIterator{Objects: objects}
+//	if err := svc.UploadWithIterator(aws.BackgroundContext(), iter); err != nil {
+//		return err
+//	}
+func (u Uploader) UploadWithIterator(ctx aws.Context, iter BatchUploadIterator, opts ...func(*Uploader)) error {
+	var errs []Error
+	for iter.Next() {
+		object := iter.UploadObject()
+		if _, err := u.UploadWithContext(ctx, object.Object, opts...); err != nil {
+			s3Err := Error{
+				OrigErr: err,
+				Bucket:  object.Object.Bucket,
+				Key:     object.Object.Key,
+			}
+
+			errs = append(errs, s3Err)
+		}
+
+		if object.After == nil {
+			continue
+		}
+
+		if err := object.After(); err != nil {
+			s3Err := Error{
+				OrigErr: err,
+				Bucket:  object.Object.Bucket,
+				Key:     object.Object.Key,
+			}
+
+			errs = append(errs, s3Err)
+		}
+	}
+
+	if len(errs) > 0 {
+		return NewBatchError("BatchedUploadIncomplete", "some objects have failed to upload.", errs)
+	}
+	return nil
+}
+
+// internal structure to manage an upload to S3.
+type uploader struct {
+	ctx aws.Context
+	cfg Uploader
+
+	in *UploadInput
+
+	readerPos int64 // current reader position
+	totalSize int64 // set to -1 if the size is not known
+
+	bufferPool sync.Pool
+}
+
+// internal logic for deciding whether to upload a single part or use a
+// multipart upload.
+func (u *uploader) upload() (*UploadOutput, error) {
+	u.init()
+
+	if u.cfg.PartSize < MinUploadPartSize {
+		msg := fmt.Sprintf("part size must be at least %d bytes", MinUploadPartSize)
+		return nil, awserr.New("ConfigError", msg, nil)
+	}
+
+	// Do one read to determine if we have more than one part
+	reader, _, part, err := u.nextReader()
+	if err == io.EOF { // single part
+		return u.singlePart(reader)
+	} else if err != nil {
+		return nil, awserr.New("ReadRequestBody", "read upload data failed", err)
+	}
+
+	mu := multiuploader{uploader: u}
+	return mu.upload(reader, part)
+}
+
+// init will initialize all default options.
+func (u *uploader) init() {
+	if u.cfg.Concurrency == 0 {
+		u.cfg.Concurrency = DefaultUploadConcurrency
+	}
+	if u.cfg.PartSize == 0 {
+		u.cfg.PartSize = DefaultUploadPartSize
+	}
+	if u.cfg.MaxUploadParts == 0 {
+		u.cfg.MaxUploadParts = MaxUploadParts
+	}
+
+	u.bufferPool = sync.Pool{
+		New: func() interface{} { return make([]byte, u.cfg.PartSize) },
+	}
+
+	// Try to get the total size for some optimizations
+	u.initSize()
+}
+
+// initSize tries to detect the total stream size, setting u.totalSize. If
+// the size is not known, totalSize is set to -1.
+func (u *uploader) initSize() {
+	u.totalSize = -1
+
+	switch r := u.in.Body.(type) {
+	case io.Seeker:
+		n, err := aws.SeekerLen(r)
+		if err != nil {
+			return
+		}
+		u.totalSize = n
+
+		// Try to adjust partSize if it is too small and account for
+		// integer division truncation.
+		if u.totalSize/u.cfg.PartSize >= int64(u.cfg.MaxUploadParts) {
+			// Add one to the part size to account for remainders
+			// during the size calculation. e.g odd number of bytes.
+			u.cfg.PartSize = (u.totalSize / int64(u.cfg.MaxUploadParts)) + 1
+		}
+	}
+}
+
+// nextReader returns a seekable reader representing the next packet of data.
+// This operation increases the shared u.readerPos counter, but note that it
+// does not need to be wrapped in a mutex because nextReader is only called
+// from the main thread.
+func (u *uploader) nextReader() (io.ReadSeeker, int, []byte, error) {
+	type readerAtSeeker interface {
+		io.ReaderAt
+		io.ReadSeeker
+	}
+	switch r := u.in.Body.(type) {
+	case readerAtSeeker:
+		var err error
+
+		n := u.cfg.PartSize
+		if u.totalSize >= 0 {
+			bytesLeft := u.totalSize - u.readerPos
+
+			if bytesLeft <= u.cfg.PartSize {
+				err = io.EOF
+				n = bytesLeft
+			}
+		}
+
+		reader := io.NewSectionReader(r, u.readerPos, n)
+		u.readerPos += n
+
+		return reader, int(n), nil, err
+
+	default:
+		part := u.bufferPool.Get().([]byte)
+		n, err := readFillBuf(r, part)
+		u.readerPos += int64(n)
+
+		return bytes.NewReader(part[0:n]), n, part, err
+	}
+}
+
+func readFillBuf(r io.Reader, b []byte) (offset int, err error) {
+	for offset < len(b) && err == nil {
+		var n int
+		n, err = r.Read(b[offset:])
+		offset += n
+	}
+
+	return offset, err
+}
+
+// singlePart contains upload logic for uploading a single chunk via
+// a regular PutObject request. Multipart requests require at least two
+// parts, or at least 5MB of data.
+func (u *uploader) singlePart(buf io.ReadSeeker) (*UploadOutput, error) {
+	params := &s3.PutObjectInput{}
+	awsutil.Copy(params, u.in)
+	params.Body = buf
+
+	// Need to use request form because URL generated in request is
+	// used in return.
+	req, out := u.cfg.S3.PutObjectRequest(params)
+	req.SetContext(u.ctx)
+	req.ApplyOptions(u.cfg.RequestOptions...)
+	if err := req.Send(); err != nil {
+		return nil, err
+	}
+
+	url := req.HTTPRequest.URL.String()
+	return &UploadOutput{
+		Location:  url,
+		VersionID: out.VersionId,
+	}, nil
+}
+
+// internal structure to manage a specific multipart upload to S3.
+type multiuploader struct {
+	*uploader
+	wg       sync.WaitGroup
+	m        sync.Mutex
+	err      error
+	uploadID string
+	parts    completedParts
+}
+
+// keeps track of a single chunk of data being sent to S3.
+type chunk struct {
+	buf  io.ReadSeeker
+	part []byte
+	num  int64
+}
+
+// completedParts is a wrapper to make parts sortable by their part number,
+// since S3 required this list to be sent in sorted order.
+type completedParts []*s3.CompletedPart
+
+func (a completedParts) Len() int           { return len(a) }
+func (a completedParts) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a completedParts) Less(i, j int) bool { return *a[i].PartNumber < *a[j].PartNumber }
+
+// upload will perform a multipart upload using the firstBuf buffer containing
+// the first chunk of data.
+func (u *multiuploader) upload(firstBuf io.ReadSeeker, firstPart []byte) (*UploadOutput, error) {
+	params := &s3.CreateMultipartUploadInput{}
+	awsutil.Copy(params, u.in)
+
+	// Create the multipart
+	resp, err := u.cfg.S3.CreateMultipartUploadWithContext(u.ctx, params, u.cfg.RequestOptions...)
+	if err != nil {
+		return nil, err
+	}
+	u.uploadID = *resp.UploadId
+
+	// Create the workers
+	ch := make(chan chunk, u.cfg.Concurrency)
+	for i := 0; i < u.cfg.Concurrency; i++ {
+		u.wg.Add(1)
+		go u.readChunk(ch)
+	}
+
+	// Send part 1 to the workers
+	var num int64 = 1
+	ch <- chunk{buf: firstBuf, part: firstPart, num: num}
+
+	// Read and queue the rest of the parts
+	for u.geterr() == nil && err == nil {
+		num++
+		// This upload exceeded maximum number of supported parts, error now.
+		if num > int64(u.cfg.MaxUploadParts) || num > int64(MaxUploadParts) {
+			var msg string
+			if num > int64(u.cfg.MaxUploadParts) {
+				msg = fmt.Sprintf("exceeded total allowed configured MaxUploadParts (%d). Adjust PartSize to fit in this limit",
+					u.cfg.MaxUploadParts)
+			} else {
+				msg = fmt.Sprintf("exceeded total allowed S3 limit MaxUploadParts (%d). Adjust PartSize to fit in this limit",
+					MaxUploadParts)
+			}
+			u.seterr(awserr.New("TotalPartsExceeded", msg, nil))
+			break
+		}
+
+		var reader io.ReadSeeker
+		var nextChunkLen int
+		var part []byte
+		reader, nextChunkLen, part, err = u.nextReader()
+
+		if err != nil && err != io.EOF {
+			u.seterr(awserr.New(
+				"ReadRequestBody",
+				"read multipart upload data failed",
+				err))
+			break
+		}
+
+		if nextChunkLen == 0 {
+			// No need to upload empty part, if file was empty to start
+			// with empty single part would of been created and never
+			// started multipart upload.
+			break
+		}
+
+		ch <- chunk{buf: reader, part: part, num: num}
+	}
+
+	// Close the channel, wait for workers, and complete upload
+	close(ch)
+	u.wg.Wait()
+	complete := u.complete()
+
+	if err := u.geterr(); err != nil {
+		return nil, &multiUploadError{
+			awsError: awserr.New(
+				"MultipartUpload",
+				"upload multipart failed",
+				err),
+			uploadID: u.uploadID,
+		}
+	}
+	return &UploadOutput{
+		Location:  aws.StringValue(complete.Location),
+		VersionID: complete.VersionId,
+		UploadID:  u.uploadID,
+	}, nil
+}
+
+// readChunk runs in worker goroutines to pull chunks off of the ch channel
+// and send() them as UploadPart requests.
+func (u *multiuploader) readChunk(ch chan chunk) {
+	defer u.wg.Done()
+	for {
+		data, ok := <-ch
+
+		if !ok {
+			break
+		}
+
+		if u.geterr() == nil {
+			if err := u.send(data); err != nil {
+				u.seterr(err)
+			}
+		}
+	}
+}
+
+// send performs an UploadPart request and keeps track of the completed
+// part information.
+func (u *multiuploader) send(c chunk) error {
+	params := &s3.UploadPartInput{
+		Bucket:               u.in.Bucket,
+		Key:                  u.in.Key,
+		Body:                 c.buf,
+		UploadId:             &u.uploadID,
+		SSECustomerAlgorithm: u.in.SSECustomerAlgorithm,
+		SSECustomerKey:       u.in.SSECustomerKey,
+		PartNumber:           &c.num,
+	}
+	resp, err := u.cfg.S3.UploadPartWithContext(u.ctx, params, u.cfg.RequestOptions...)
+	// put the byte array back into the pool to conserve memory
+	u.bufferPool.Put(c.part)
+	if err != nil {
+		return err
+	}
+
+	n := c.num
+	completed := &s3.CompletedPart{ETag: resp.ETag, PartNumber: &n}
+
+	u.m.Lock()
+	u.parts = append(u.parts, completed)
+	u.m.Unlock()
+
+	return nil
+}
+
+// geterr is a thread-safe getter for the error object
+func (u *multiuploader) geterr() error {
+	u.m.Lock()
+	defer u.m.Unlock()
+
+	return u.err
+}
+
+// seterr is a thread-safe setter for the error object
+func (u *multiuploader) seterr(e error) {
+	u.m.Lock()
+	defer u.m.Unlock()
+
+	u.err = e
+}
+
+// fail will abort the multipart unless LeavePartsOnError is set to true.
+func (u *multiuploader) fail() {
+	if u.cfg.LeavePartsOnError {
+		return
+	}
+
+	params := &s3.AbortMultipartUploadInput{
+		Bucket:   u.in.Bucket,
+		Key:      u.in.Key,
+		UploadId: &u.uploadID,
+	}
+	_, err := u.cfg.S3.AbortMultipartUploadWithContext(u.ctx, params, u.cfg.RequestOptions...)
+	if err != nil {
+		logMessage(u.cfg.S3, aws.LogDebug, fmt.Sprintf("failed to abort multipart upload, %v", err))
+	}
+}
+
+// complete successfully completes a multipart upload and returns the response.
+func (u *multiuploader) complete() *s3.CompleteMultipartUploadOutput {
+	if u.geterr() != nil {
+		u.fail()
+		return nil
+	}
+
+	// Parts must be sorted in PartNumber order.
+	sort.Sort(u.parts)
+
+	params := &s3.CompleteMultipartUploadInput{
+		Bucket:          u.in.Bucket,
+		Key:             u.in.Key,
+		UploadId:        &u.uploadID,
+		MultipartUpload: &s3.CompletedMultipartUpload{Parts: u.parts},
+	}
+	resp, err := u.cfg.S3.CompleteMultipartUploadWithContext(u.ctx, params, u.cfg.RequestOptions...)
+	if err != nil {
+		u.seterr(err)
+		u.fail()
+	}
+
+	return resp
+}

From ab4bb74e1747ae1835e45bd3866eb306c38da3ad Mon Sep 17 00:00:00 2001
From: TangBin <tangbin@yunionyun.com>
Date: Sat, 17 Nov 2018 10:26:19 +0800
Subject: [PATCH 31/57] make fmt

---
 pkg/util/aws/securitygroup.go | 6 +++---
 pkg/util/aws/storagecache.go  | 4 ++--
 pkg/util/aws/vpc.go           | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/pkg/util/aws/securitygroup.go b/pkg/util/aws/securitygroup.go
index 29399d61b0..3fc04daf36 100644
--- a/pkg/util/aws/securitygroup.go
+++ b/pkg/util/aws/securitygroup.go
@@ -127,7 +127,7 @@ func (self *SRegion) addSecurityGroupRule(secGrpId string, rule *secrules.Securi
 		_, err = self.ec2Client.AuthorizeSecurityGroupEgress(params)
 	}
 
-	if err != nil && strings.Contains(err.Error(),"InvalidPermission.Duplicate") {
+	if err != nil && strings.Contains(err.Error(), "InvalidPermission.Duplicate") {
 		log.Debugf("addSecurityGroupRule %s %s", rule.Direction, err.Error())
 		return nil
 	}
@@ -321,8 +321,8 @@ func (self *SRegion) syncSecgroupRules(secgroupId string, rules []secrules.Secur
 		sort.Sort(secrules.SecurityRuleSet(rules))
 		sort.Sort(secrules.SecurityRuleSet(secgroup.Permissions))
 
-		log.Debugf("local security rules %s",rules)
-		log.Debugf("remote security rules %s",secgroup.Permissions)
+		log.Debugf("local security rules %s", rules)
+		log.Debugf("remote security rules %s", secgroup.Permissions)
 
 		i, j := 0, 0
 		for i < len(rules) || j < len(secgroup.Permissions) {
diff --git a/pkg/util/aws/storagecache.go b/pkg/util/aws/storagecache.go
index eaed1175b2..b1a42471bd 100644
--- a/pkg/util/aws/storagecache.go
+++ b/pkg/util/aws/storagecache.go
@@ -150,8 +150,8 @@ func (self *SStoragecache) uploadImage(userCred mcclient.TokenCredential, imageI
 	// uploader to aws s3
 	input := &s3manager.UploadInput{
 		Bucket: &bucketName,
-		Key: &imageId,
-		Body: reader,
+		Key:    &imageId,
+		Body:   reader,
 	}
 
 	awsSession, err := self.region.getAwsSession()
diff --git a/pkg/util/aws/vpc.go b/pkg/util/aws/vpc.go
index 72b35b3a80..eb0ed59eab 100644
--- a/pkg/util/aws/vpc.go
+++ b/pkg/util/aws/vpc.go
@@ -108,7 +108,7 @@ func (self *SVpc) GetManagerId() string {
 }
 
 func (self *SVpc) Delete() error {
-    // 删除vpc会同步删除关联的安全组
+	// 删除vpc会同步删除关联的安全组
 	return self.region.DeleteVpc(self.VpcId)
 }
 

From 8e9105d11540c96f5e41419cd1cbcb9a5c088678 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= <quxuan@yunionyun.com>
Date: Sat, 17 Nov 2018 12:15:31 +0800
Subject: [PATCH 32/57] =?UTF-8?q?=E4=BD=BF=E7=94=A8=E7=B3=BB=E7=BB=9F?=
 =?UTF-8?q?=E5=AF=86=E9=92=A5?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/webconsole/command/ssh_command.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/webconsole/command/ssh_command.go b/pkg/webconsole/command/ssh_command.go
index 3988c2a03b..05aa9fea17 100644
--- a/pkg/webconsole/command/ssh_command.go
+++ b/pkg/webconsole/command/ssh_command.go
@@ -31,7 +31,7 @@ type SSHtoolSol struct {
 func getCommand(ctx context.Context, userCred mcclient.TokenCredential, ip string) (string, *BaseCommand, error) {
 	cmd := NewBaseCommand(o.Options.SshToolPath)
 	s := auth.GetAdminSession(o.Options.Region, "v2")
-	key, err := modules.Sshkeypairs.GetById(s, userCred.GetProjectId(), jsonutils.NewDict())
+	key, err := modules.Sshkeypairs.GetById(s, userCred.GetProjectId(), jsonutils.Marshal(map[string]bool{"admin": true}))
 	if err != nil {
 		return "", nil, err
 	}

From c62424aa2cd323a2bcd65f144ccba2b8d669cd93 Mon Sep 17 00:00:00 2001
From: Qiu Jian <qiujian@yunionyun.com>
Date: Sat, 17 Nov 2018 12:54:56 +0800
Subject: [PATCH 33/57] change yunionroot => cloudroot

---
 pkg/util/ansible/const.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/util/ansible/const.go b/pkg/util/ansible/const.go
index 06ac34ff47..0f74502789 100644
--- a/pkg/util/ansible/const.go
+++ b/pkg/util/ansible/const.go
@@ -1,5 +1,5 @@
 package ansible
 
 const (
-	PUBLIC_CLOUD_ANSIBLE_USER = "yunionroot"
+	PUBLIC_CLOUD_ANSIBLE_USER = "cloudroot"
 )

From fc0e60c5ec50b20fdc896389ae65fa464b4448d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= <quxuan@yunionyun.com>
Date: Sat, 17 Nov 2018 13:02:10 +0800
Subject: [PATCH 34/57] =?UTF-8?q?azure=E4=BD=BF=E7=94=A8=E7=BB=9F=E4=B8=80?=
 =?UTF-8?q?=E7=99=BB=E5=BD=95=E7=94=A8=E6=88=B7=E5=90=8D?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/compute/guestdrivers/aws.go   |  6 ++++--
 pkg/compute/guestdrivers/azure.go | 11 ++++-------
 pkg/util/azure/classic_host.go    |  4 ----
 pkg/util/azure/host.go            |  3 ++-
 4 files changed, 10 insertions(+), 14 deletions(-)

diff --git a/pkg/compute/guestdrivers/aws.go b/pkg/compute/guestdrivers/aws.go
index 163a7278e0..ff6bb28987 100644
--- a/pkg/compute/guestdrivers/aws.go
+++ b/pkg/compute/guestdrivers/aws.go
@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"time"
 
+	"yunion.io/x/onecloud/pkg/util/ansible"
+
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
@@ -158,7 +160,7 @@ func (self *SAwsGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest
 				return nil, err
 			}
 
-			data := fetchIVMinfo(desc, iVM, guest.Id, DEFAULT_USER, passwd, action)
+			data := fetchIVMinfo(desc, iVM, guest.Id, ansible.PUBLIC_CLOUD_ANSIBLE_USER, passwd, action)
 			return data, nil
 		})
 	case "rebuild":
@@ -210,7 +212,7 @@ func (self *SAwsGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest
 				}
 			}
 
-			data := fetchIVMinfo(desc, iVM, guest.Id, DEFAULT_USER, passwd, action)
+			data := fetchIVMinfo(desc, iVM, guest.Id, ansible.PUBLIC_CLOUD_ANSIBLE_USER, passwd, action)
 
 			return data, nil
 		})
diff --git a/pkg/compute/guestdrivers/azure.go b/pkg/compute/guestdrivers/azure.go
index 139dbad0e5..6afc543997 100644
--- a/pkg/compute/guestdrivers/azure.go
+++ b/pkg/compute/guestdrivers/azure.go
@@ -9,6 +9,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/ansible"
 	"yunion.io/x/onecloud/pkg/util/seclib2"
 	"yunion.io/x/pkg/utils"
 
@@ -22,10 +23,6 @@ type SAzureGuestDriver struct {
 	SManagedVirtualizedGuestDriver
 }
 
-const (
-	DEFAULT_USER = "yunion"
-)
-
 func init() {
 	driver := SAzureGuestDriver{}
 	models.RegisterGuestDriver(&driver)
@@ -169,7 +166,7 @@ func (self *SAzureGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gue
 					return nil, err
 				}
 
-				data := fetchIVMinfo(desc, iVM, guest.Id, DEFAULT_USER, passwd, action)
+				data := fetchIVMinfo(desc, iVM, guest.Id, ansible.PUBLIC_CLOUD_ANSIBLE_USER, passwd, action)
 				return data, nil
 			}
 		})
@@ -192,7 +189,7 @@ func (self *SAzureGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gue
 			if err != nil {
 				return nil, err
 			}
-			data := fetchIVMinfo(desc, iVM, guest.Id, DEFAULT_USER, passwd, action)
+			data := fetchIVMinfo(desc, iVM, guest.Id, ansible.PUBLIC_CLOUD_ANSIBLE_USER, passwd, action)
 			return data, nil
 		})
 	} else if action == "rebuild" {
@@ -209,7 +206,7 @@ func (self *SAzureGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gue
 			}
 
 			log.Debugf("VMrebuildRoot %s, and status is ready", iVM.GetGlobalId())
-			data := fetchIVMinfo(desc, iVM, guest.Id, DEFAULT_USER, passwd, action)
+			data := fetchIVMinfo(desc, iVM, guest.Id, ansible.PUBLIC_CLOUD_ANSIBLE_USER, passwd, action)
 
 			return data, nil
 		})
diff --git a/pkg/util/azure/classic_host.go b/pkg/util/azure/classic_host.go
index 1a943b7dd1..f2c7aa5654 100644
--- a/pkg/util/azure/classic_host.go
+++ b/pkg/util/azure/classic_host.go
@@ -13,10 +13,6 @@ type SClassicHost struct {
 	zone *SZone
 }
 
-const (
-	DEFAULT_USER = "yunion"
-)
-
 func (self *SClassicHost) GetMetadata() *jsonutils.JSONDict {
 	return nil
 }
diff --git a/pkg/util/azure/host.go b/pkg/util/azure/host.go
index 4dd6fff0ee..970a497cd7 100644
--- a/pkg/util/azure/host.go
+++ b/pkg/util/azure/host.go
@@ -8,6 +8,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/util/ansible"
 )
 
 type SHost struct {
@@ -118,7 +119,7 @@ func (self *SHost) _createVM(name string, imgId string, sysDiskSize int32, cpu i
 			},
 			OsProfile: OsProfile{
 				ComputerName:  name,
-				AdminUsername: DEFAULT_USER,
+				AdminUsername: ansible.PUBLIC_CLOUD_ANSIBLE_USER,
 				AdminPassword: passwd,
 				CustomData:    userData,
 			},

From aff50b0de444707c4578246f44cca7819992e77b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= <quxuan@yunionyun.com>
Date: Sat, 17 Nov 2018 14:28:26 +0800
Subject: [PATCH 35/57] =?UTF-8?q?=E2=80=9D=E5=88=9A=E5=AF=BC=E5=85=A5?=
 =?UTF-8?q?=E8=BF=9B=E6=9D=A5=E7=9A=84=E6=96=B0=E8=AE=A2=E9=98=85=E9=BB=98?=
 =?UTF-8?q?=E8=AE=A4=E7=8A=B6=E6=80=81=E6=9C=AAconnected=E2=80=9C?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/compute/models/cloudaccounts.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go
index 0655cac3e1..32b4762ff5 100644
--- a/pkg/compute/models/cloudaccounts.go
+++ b/pkg/compute/models/cloudaccounts.go
@@ -386,6 +386,7 @@ func (self *SCloudaccount) ImportSubAccount(ctx context.Context, userCred mcclie
 	newCloudprovider.CloudaccountId = self.Id
 	newCloudprovider.Provider = self.Provider
 	newCloudprovider.Enabled = true
+	newCloudprovider.Status = CLOUD_PROVIDER_CONNECTED
 	newCloudprovider.Name = subAccount.Name
 	if !autoCreateProject {
 		newCloudprovider.ProjectId = auth.AdminCredential().GetProjectId()

From 04917c93453febb0843a178ac9f1b3c9327fe3f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= <quxuan@yunionyun.com>
Date: Sat, 17 Nov 2018 15:31:30 +0800
Subject: [PATCH 36/57] =?UTF-8?q?=E5=88=B7=E6=96=B0scheduler=E7=BC=93?=
 =?UTF-8?q?=E5=AD=98?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/compute/models/hosts.go                    | 18 +++++++++++++++---
 .../tasks/cloud_provider_sync_info_task.go     |  2 +-
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go
index 5a52ab1505..79028ba701 100644
--- a/pkg/compute/models/hosts.go
+++ b/pkg/compute/models/hosts.go
@@ -958,7 +958,7 @@ func (manager *SHostManager) getHostsByZone(zone *SZone, provider *SCloudprovide
 	return hosts, nil
 }
 
-func (manager *SHostManager) SyncHosts(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, zone *SZone, hosts []cloudprovider.ICloudHost) ([]SHost, []cloudprovider.ICloudHost, compare.SyncResult) {
+func (manager *SHostManager) SyncHosts(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, zone *SZone, hosts []cloudprovider.ICloudHost, projectSync bool) ([]SHost, []cloudprovider.ICloudHost, compare.SyncResult) {
 	localHosts := make([]SHost, 0)
 	remoteHosts := make([]cloudprovider.ICloudHost, 0)
 	syncResult := compare.SyncResult{}
@@ -1002,7 +1002,7 @@ func (manager *SHostManager) SyncHosts(ctx context.Context, userCred mcclient.To
 		}
 	}
 	for i := 0; i < len(commondb); i += 1 {
-		err = commondb[i].syncWithCloudHost(commonext[i])
+		err = commondb[i].syncWithCloudHost(commonext[i], projectSync)
 		if err != nil {
 			syncResult.UpdateError(err)
 		} else {
@@ -1025,7 +1025,7 @@ func (manager *SHostManager) SyncHosts(ctx context.Context, userCred mcclient.To
 	return localHosts, remoteHosts, syncResult
 }
 
-func (self *SHost) syncWithCloudHost(extHost cloudprovider.ICloudHost) error {
+func (self *SHost) syncWithCloudHost(extHost cloudprovider.ICloudHost, projectSync bool) error {
 	_, err := self.GetModelManager().TableSpec().Update(self, func() error {
 		self.Name = extHost.GetName()
 		self.Status = extHost.GetStatus()
@@ -1052,6 +1052,13 @@ func (self *SHost) syncWithCloudHost(extHost cloudprovider.ICloudHost) error {
 	if err != nil {
 		log.Errorf("syncWithCloudZone error %s", err)
 	}
+
+	if projectSync {
+		if err := HostManager.ClearSchedDescCache(self.Id); err != nil {
+			log.Errorf("ClearSchedDescCache for host %s error %v", self.Name, err)
+		}
+	}
+
 	return err
 }
 
@@ -1089,6 +1096,11 @@ func (manager *SHostManager) newFromCloudHost(extHost cloudprovider.ICloudHost,
 		log.Errorf("newFromCloudHost fail %s", err)
 		return nil, err
 	}
+
+	if err := manager.ClearSchedDescCache(host.Id); err != nil {
+		log.Errorf("ClearSchedDescCache for host %s error %v", host.Name, err)
+	}
+
 	return &host, nil
 }
 
diff --git a/pkg/compute/tasks/cloud_provider_sync_info_task.go b/pkg/compute/tasks/cloud_provider_sync_info_task.go
index 18e75d4d03..85a3d60bb3 100644
--- a/pkg/compute/tasks/cloud_provider_sync_info_task.go
+++ b/pkg/compute/tasks/cloud_provider_sync_info_task.go
@@ -343,7 +343,7 @@ func syncZoneHosts(ctx context.Context, provider *models.SCloudprovider, task *C
 		logSyncFailed(provider, task, msg)
 		return
 	}
-	localHosts, remoteHosts, result := models.HostManager.SyncHosts(ctx, task.UserCred, provider, localZone, hosts)
+	localHosts, remoteHosts, result := models.HostManager.SyncHosts(ctx, task.UserCred, provider, localZone, hosts, syncRange.ProjectSync)
 	msg := result.Result()
 	notes := fmt.Sprintf("SyncHosts for zone %s result: %s", localZone.Name, msg)
 	log.Infof(notes)

From c6c1b17c3edba2104ceaf646c60e2964c358aaea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= <quxuan@yunionyun.com>
Date: Sat, 17 Nov 2018 15:53:11 +0800
Subject: [PATCH 37/57] =?UTF-8?q?=E5=8F=96=E6=B6=88=E5=88=9B=E5=BB=BA?=
 =?UTF-8?q?=E5=85=AC=E6=9C=89=E4=BA=91vpc=E9=99=90=E5=88=B6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/compute/models/vpcs.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/pkg/compute/models/vpcs.go b/pkg/compute/models/vpcs.go
index 2ec8e1bd24..639cbd6f10 100644
--- a/pkg/compute/models/vpcs.go
+++ b/pkg/compute/models/vpcs.go
@@ -344,10 +344,6 @@ func (manager *SVpcManager) ValidateCreateData(ctx context.Context, userCred mcc
 		return nil, httperrors.NewInputParameterError("Invalid cloudregion_id")
 	}
 	if region.isManaged() {
-		if region.GetVpcCount() >= MAX_VPC_PER_REGION {
-			return nil, httperrors.NewNotAcceptableError("Too many vpcs per region")
-		}
-
 		managerStr, _ := data.GetString("manager_id")
 		if len(managerStr) == 0 {
 			managerStr, _ = data.GetString("manager")

From a49747bebe6e6271938dd8cd4573d4e5ad1d12f2 Mon Sep 17 00:00:00 2001
From: TangBin <tangbin@yunionyun.com>
Date: Sat, 17 Nov 2018 16:07:06 +0800
Subject: [PATCH 38/57] rebuild root fix

---
 pkg/util/aws/instance.go | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/pkg/util/aws/instance.go b/pkg/util/aws/instance.go
index 4d693c6ac3..bfc699822d 100644
--- a/pkg/util/aws/instance.go
+++ b/pkg/util/aws/instance.go
@@ -5,8 +5,8 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/coredns/coredns/plugin/pkg/log"
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/pkg/util/osprofile"
@@ -420,7 +420,6 @@ func (self *SRegion) GetInstances(zoneId string, ids []string, offset int, limit
 		log.Errorf("GetInstances fail %s", err)
 		return nil, 0, err
 	}
-
 	instances := []SInstance{}
 	for _, reservation := range res.Reservations {
 		for _, instance := range reservation.Instances {
@@ -473,8 +472,16 @@ func (self *SRegion) GetInstances(zoneId string, ids []string, offset int, limit
 				productCodes = append(productCodes, *p.ProductCodeId)
 			}
 
+			szone, err := self.getZoneById(*instance.Placement.AvailabilityZone)
+			if err != nil {
+				return nil, 0, err
+			}
+
+			host := szone.getHost()
+
 			sinstance := SInstance{
 				RegionId:          self.RegionId,
+				host:              host,
 				ZoneId:            *instance.Placement.AvailabilityZone,
 				InstanceId:        *instance.InstanceId,
 				ImageId:           *instance.ImageId,

From 6b14c8d95f52ae209bf3847640e6cabdb4601b46 Mon Sep 17 00:00:00 2001
From: TangBin <tangbin@yunionyun.com>
Date: Sat, 17 Nov 2018 16:21:31 +0800
Subject: [PATCH 39/57] remove useless log

---
 pkg/util/aws/instance.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pkg/util/aws/instance.go b/pkg/util/aws/instance.go
index bfc699822d..02ff642841 100644
--- a/pkg/util/aws/instance.go
+++ b/pkg/util/aws/instance.go
@@ -423,7 +423,6 @@ func (self *SRegion) GetInstances(zoneId string, ids []string, offset int, limit
 	instances := []SInstance{}
 	for _, reservation := range res.Reservations {
 		for _, instance := range reservation.Instances {
-			log.Debugf("GetInstances %s", instance.String())
 			if err := FillZero(instance); err != nil {
 				return nil, 0, err
 			}

From 532f5ae1affe7283b2ba8e1b59e22f0ee387831c Mon Sep 17 00:00:00 2001
From: Qiu Jian <qiujian@yunionyun.com>
Date: Sat, 17 Nov 2018 17:27:05 +0800
Subject: [PATCH 40/57] =?UTF-8?q?=E8=99=9A=E6=8B=9F=E6=9C=BA=E5=90=AF?=
 =?UTF-8?q?=E5=8A=A8=E6=97=B6=E5=A6=82=E6=9E=9C=E5=B7=B2=E7=BB=8F=E5=90=AF?=
 =?UTF-8?q?=E5=8A=A8=EF=BC=8C=E4=BC=9A=E4=B8=A2=E5=A4=B1start=E7=9A=84ospl?=
 =?UTF-8?q?og=E4=BA=8B=E4=BB=B6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/compute/models/disks.go           | 2 ++
 pkg/compute/models/guests.go          | 1 +
 pkg/compute/tasks/guest_start_task.go | 5 +++--
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go
index 13fad325ae..c0a992900b 100644
--- a/pkg/compute/models/disks.go
+++ b/pkg/compute/models/disks.go
@@ -1101,6 +1101,7 @@ type DiskInfo struct {
 	MediumType string
 	Driver     string
 	Cache      string
+	DiskType   string
 }
 
 func (self *SDisk) ToDiskInfo() DiskInfo {
@@ -1110,6 +1111,7 @@ func (self *SDisk) ToDiskInfo() DiskInfo {
 		MountPoint: self.GetMountPoint(),
 		Format:     self.DiskFormat,
 		Size:       int64(self.DiskSize),
+		DiskType:   self.DiskType,
 	}
 	storage := self.GetStorage()
 	if storage == nil {
diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go
index a8911ac012..0212771866 100644
--- a/pkg/compute/models/guests.go
+++ b/pkg/compute/models/guests.go
@@ -3913,6 +3913,7 @@ func (self *SGuest) GetSpec(checkStatus bool) *jsonutils.JSONDict {
 		diskSpec.Set("size", jsonutils.NewInt(info.Size))
 		diskSpec.Set("backend", jsonutils.NewString(info.Backend))
 		diskSpec.Set("medium_type", jsonutils.NewString(info.MediumType))
+		diskSpec.Set("disk_type", jsonutils.NewString(info.DiskType))
 		diskSpecs.Add(diskSpec)
 	}
 	spec.Set("disk", diskSpecs)
diff --git a/pkg/compute/tasks/guest_start_task.go b/pkg/compute/tasks/guest_start_task.go
index 99dae67cf3..2eb1ca3db7 100644
--- a/pkg/compute/tasks/guest_start_task.go
+++ b/pkg/compute/tasks/guest_start_task.go
@@ -48,8 +48,9 @@ func (self *GuestStartTask) startStart(ctx context.Context, guest *models.SGuest
 		self.onStartGuestFailed(ctx, guest, err)
 	} else {
 		if result != nil && jsonutils.QueryBoolean(result, "is_running", false) {
-			guest.SetStatus(self.UserCred, models.VM_RUNNING, "start")
-			self.taskComplete(ctx, guest)
+			self.OnStartComplete(ctx, guest, nil)
+			// guest.SetStatus(self.UserCred, models.VM_RUNNING, "start")
+			// self.taskComplete(ctx, guest)
 		}
 	}
 }

From db0f7656aa8735f8c2341a3dae8ccd877eaae149 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= <quxuan@yunionyun.com>
Date: Sat, 17 Nov 2018 20:29:45 +0800
Subject: [PATCH 41/57] =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E6=9C=AA=E6=B3=A8?=
 =?UTF-8?q?=E5=86=8C=E6=97=B6=EF=BC=8C=E5=B0=9D=E8=AF=95=E6=B3=A8=E5=86=8C?=
 =?UTF-8?q?=E6=9C=8D=E5=8A=A1?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/util/azure/azure.go         | 143 +++++++++++++++++++++++++-------
 pkg/util/azure/debug.go         |   4 +
 pkg/util/azure/service.go       |  55 ++++++++++++
 pkg/util/azure/shell/service.go |  41 +++++++++
 pkg/util/azure/shell/vpc.go     |  19 +++++
 pkg/util/azure/vpc.go           |   6 +-
 6 files changed, 239 insertions(+), 29 deletions(-)
 create mode 100644 pkg/util/azure/service.go
 create mode 100644 pkg/util/azure/shell/service.go

diff --git a/pkg/util/azure/azure.go b/pkg/util/azure/azure.go
index 791e6d3fa1..85120b53c1 100644
--- a/pkg/util/azure/azure.go
+++ b/pkg/util/azure/azure.go
@@ -100,8 +100,10 @@ func (self *SAzureClient) getDefaultClient() (*autorest.Client, error) {
 		return nil, err
 	}
 	client.Authorizer = authorizer
-	// client.RequestInspector = LogRequest()
-	// client.ResponseInspector = LogResponse()
+	if DEBUG {
+		client.RequestInspector = LogRequest()
+		client.ResponseInspector = LogResponse()
+	}
 	return &client, nil
 }
 
@@ -110,7 +112,7 @@ func (self *SAzureClient) jsonRequest(method, url string, body string) (jsonutil
 	if err != nil {
 		return nil, err
 	}
-	return jsonRequest(cli, method, self.domain, url, body)
+	return jsonRequest(cli, method, self.domain, url, self.subscriptionId, body)
 }
 
 func (self *SAzureClient) Get(resourceId string, params []string, retVal interface{}) error {
@@ -125,7 +127,7 @@ func (self *SAzureClient) Get(resourceId string, params []string, retVal interfa
 	if err != nil {
 		return err
 	}
-	body, err := jsonRequest(cli, "GET", self.domain, path, "")
+	body, err := jsonRequest(cli, "GET", self.domain, path, self.subscriptionId, "")
 	if err != nil {
 		return err
 	}
@@ -145,7 +147,7 @@ func (self *SAzureClient) ListVmSizes(location string) (jsonutils.JSONObject, er
 		return nil, fmt.Errorf("need subscription id")
 	}
 	url := fmt.Sprintf("/subscriptions/%s/providers/Microsoft.Compute/locations/%s/vmSizes", self.subscriptionId, location)
-	return jsonRequest(cli, "GET", self.domain, url, "")
+	return jsonRequest(cli, "GET", self.domain, url, self.subscriptionId, "")
 }
 
 func (self *SAzureClient) ListClassicDisks() (jsonutils.JSONObject, error) {
@@ -157,7 +159,7 @@ func (self *SAzureClient) ListClassicDisks() (jsonutils.JSONObject, error) {
 		return nil, fmt.Errorf("need subscription id")
 	}
 	url := fmt.Sprintf("/subscriptions/%s/services/disks", self.subscriptionId)
-	return jsonRequest(cli, "GET", self.domain, url, "")
+	return jsonRequest(cli, "GET", self.domain, url, self.subscriptionId, "")
 }
 
 func (self *SAzureClient) ListAll(resourceType string, retVal interface{}) error {
@@ -172,7 +174,7 @@ func (self *SAzureClient) ListAll(resourceType string, retVal interface{}) error
 	if len(resourceType) > 0 {
 		url += fmt.Sprintf("/providers/%s", resourceType)
 	}
-	body, err := jsonRequest(cli, "GET", self.domain, url, "")
+	body, err := jsonRequest(cli, "GET", self.domain, url, self.subscriptionId, "")
 	if err != nil {
 		return err
 	}
@@ -187,7 +189,7 @@ func (self *SAzureClient) ListSubscriptions() (jsonutils.JSONObject, error) {
 	if err != nil {
 		return nil, err
 	}
-	return jsonRequest(cli, "GET", self.domain, "/subscriptions", "")
+	return jsonRequest(cli, "GET", self.domain, "/subscriptions", self.subscriptionId, "")
 }
 
 func (self *SAzureClient) List(golbalResource string, retVal interface{}) error {
@@ -202,7 +204,7 @@ func (self *SAzureClient) List(golbalResource string, retVal interface{}) error
 	if len(self.subscriptionId) > 0 && len(golbalResource) > 0 {
 		url += fmt.Sprintf("/%s", golbalResource)
 	}
-	body, err := jsonRequest(cli, "GET", self.domain, url, "")
+	body, err := jsonRequest(cli, "GET", self.domain, url, self.subscriptionId, "")
 	if err != nil {
 		return err
 	}
@@ -218,7 +220,7 @@ func (self *SAzureClient) ListByTypeWithResourceGroup(resourceGroupName string,
 		return fmt.Errorf("Missing subscription Info")
 	}
 	url := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/%s", self.subscriptionId, resourceGroupName, Type)
-	body, err := jsonRequest(cli, "GET", self.domain, url, "")
+	body, err := jsonRequest(cli, "GET", self.domain, url, self.subscriptionId, "")
 	if err != nil {
 		return err
 	}
@@ -230,7 +232,7 @@ func (self *SAzureClient) Delete(resourceId string) error {
 	if err != nil {
 		return err
 	}
-	_, err = jsonRequest(cli, "DELETE", self.domain, resourceId, "")
+	_, err = jsonRequest(cli, "DELETE", self.domain, resourceId, self.subscriptionId, "")
 	return err
 }
 
@@ -240,7 +242,7 @@ func (self *SAzureClient) PerformAction(resourceId string, action string, body s
 		return nil, err
 	}
 	url := fmt.Sprintf("%s/%s", resourceId, action)
-	return jsonRequest(cli, "POST", self.domain, url, body)
+	return jsonRequest(cli, "POST", self.domain, url, self.subscriptionId, body)
 }
 
 func (self *SAzureClient) fetchResourceGroup(cli *autorest.Client, location string) error {
@@ -255,7 +257,7 @@ func (self *SAzureClient) fetchResourceGroup(cli *autorest.Client, location stri
 	if len(self.ressourceGroups) == 0 {
 		//Create Default resourceGroup
 		_url := fmt.Sprintf("/subscriptions/%s/resourcegroups/Default", self.subscriptionId)
-		body, err := jsonRequest(cli, "PUT", self.domain, _url, fmt.Sprintf(`{"name": "Default", "location": "%s"}`, location))
+		body, err := jsonRequest(cli, "PUT", self.domain, _url, self.subscriptionId, fmt.Sprintf(`{"name": "Default", "location": "%s"}`, location))
 		if err != nil {
 			return err
 		}
@@ -281,6 +283,18 @@ func (self *SAzureClient) checkParams(body jsonutils.JSONObject, params []string
 	return result, nil
 }
 
+type AzureErrorDetail struct {
+	Code    string `json:"code,omitempty"`
+	Message string `json:"message,omitempty"`
+	Target  string `json:"target,omitempty"`
+}
+
+type AzureError struct {
+	Code    string             `json:"code,omitempty"`
+	Details []AzureErrorDetail `json:"details,omitempty"`
+	Message string             `json:"message,omitempty"`
+}
+
 func (self *SAzureClient) Create(body jsonutils.JSONObject, retVal interface{}) error {
 	cli, err := self.getDefaultClient()
 	if err != nil {
@@ -301,11 +315,14 @@ func (self *SAzureClient) Create(body jsonutils.JSONObject, retVal interface{})
 		return fmt.Errorf("Create Default resourceGroup error?")
 	}
 	url := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/%s/%s", self.subscriptionId, self.ressourceGroups[0].Name, params["type"], params["name"])
-	result, err := jsonRequest(cli, "PUT", self.domain, url, body.String())
+	result, err := jsonRequest(cli, "PUT", self.domain, url, self.subscriptionId, body.String())
 	if err != nil {
 		return err
 	}
-	return result.Unmarshal(retVal)
+	if retVal != nil {
+		return result.Unmarshal(retVal)
+	}
+	return nil
 }
 
 func (self *SAzureClient) CheckNameAvailability(Type string, body string) (jsonutils.JSONObject, error) {
@@ -317,7 +334,7 @@ func (self *SAzureClient) CheckNameAvailability(Type string, body string) (jsonu
 		return nil, fmt.Errorf("Missing subscription ID")
 	}
 	url := fmt.Sprintf("/subscriptions/%s/providers/%s/checkNameAvailability", self.subscriptionId, Type)
-	return jsonRequest(cli, "POST", self.domain, url, body)
+	return jsonRequest(cli, "POST", self.domain, url, self.subscriptionId, body)
 }
 
 func (self *SAzureClient) Update(body jsonutils.JSONObject, retVal interface{}) error {
@@ -326,7 +343,7 @@ func (self *SAzureClient) Update(body jsonutils.JSONObject, retVal interface{})
 		return err
 	}
 	url, err := body.GetString("id")
-	result, err := jsonRequest(cli, "PUT", self.domain, url, body.String())
+	result, err := jsonRequest(cli, "PUT", self.domain, url, self.subscriptionId, body.String())
 	if err != nil {
 		return err
 	}
@@ -336,8 +353,86 @@ func (self *SAzureClient) Update(body jsonutils.JSONObject, retVal interface{})
 	return nil
 }
 
-func jsonRequest(client *autorest.Client, method, domain, baseUrl string, body string) (jsonutils.JSONObject, error) {
-	return _jsonRequest(client, method, domain, baseUrl, body)
+func waitRegisterComplete(client *autorest.Client, domain, subscriptionId string, serviceType string) error {
+	for i := 1; i < 10; i++ {
+		result, err := _jsonRequest(client, "GET", domain, fmt.Sprintf("/subscriptions/%s/providers", subscriptionId), "")
+		if err != nil {
+			return err
+		}
+		value, err := result.GetArray("value")
+		if err != nil {
+			return err
+		}
+		for _, v := range value {
+			namespace, _ := v.GetString("namespace")
+			if namespace == serviceType {
+				state, _ := v.GetString("registrationState")
+				if state == "Registered" {
+					return nil
+				}
+				log.Debugf("service %s state %s waite %d second ...", serviceType, state, i*10)
+			}
+		}
+		time.Sleep(time.Second * time.Duration(i*10))
+	}
+	return fmt.Errorf("wait service %s register timeout", serviceType)
+}
+
+func registerService(client *autorest.Client, domain, subscriptionId string, serviceType string) error {
+	registryUrl := fmt.Sprintf("/subscriptions/%s/providers/%s/register", subscriptionId, serviceType)
+	result, err := _jsonRequest(client, "POST", domain, registryUrl, "")
+	if err != nil || result.Contains("error") {
+		return fmt.Errorf("failed to register %s service", serviceType)
+	}
+	if state, _ := result.GetString("registrationState"); state == "Registered" {
+		return nil
+	}
+	return waitRegisterComplete(client, domain, subscriptionId, serviceType)
+}
+
+func recoverFromError(client *autorest.Client, domain, subscriptionId string, azureErr AzureError) bool {
+	switch azureErr.Code {
+	case "SubscriptionNotRegistered":
+		services := []string{"Microsoft.Network"}
+		for _, service := range services {
+			if err := registerService(client, domain, subscriptionId, service); err != nil {
+				log.Errorf("register %s error: %v", service, err)
+				return false
+			}
+		}
+		return true
+	case "MissingSubscriptionRegistration":
+		for _, detail := range azureErr.Details {
+			log.Errorf("The subscription is not registered to use namespace '%s', try register it", detail.Target)
+			if err := registerService(client, domain, subscriptionId, detail.Target); err != nil {
+				log.Errorf("register %s error: %v", detail.Target, err)
+				return false
+			}
+		}
+		return true
+	default:
+		return false
+	}
+	return false
+}
+
+func jsonRequest(client *autorest.Client, method, domain, baseUrl string, subscriptionId string, body string) (jsonutils.JSONObject, error) {
+	result, err := _jsonRequest(client, method, domain, baseUrl, body)
+	if err != nil {
+		return nil, err
+	}
+	if result.Contains("error") {
+		azureError := AzureError{}
+		if err := result.Unmarshal(&azureError, "error"); err != nil {
+			return nil, fmt.Errorf(result.String())
+		}
+		if recoverFromError(client, domain, subscriptionId, azureError) {
+			return _jsonRequest(client, method, domain, baseUrl, body)
+		}
+		log.Errorf("Azure %s request: %s \nbody: %s error: %v", method, baseUrl, body, result.String())
+		return nil, fmt.Errorf(result.String())
+	}
+	return result, nil
 }
 
 func waitForComplatetion(client *autorest.Client, req *http.Request, resp *http.Response, timeout time.Duration) (jsonutils.JSONObject, error) {
@@ -464,15 +559,7 @@ func _jsonRequest(client *autorest.Client, method, domain, baseURL, body string)
 		return nil, err
 	}
 	_data := strings.Replace(string(data), "\r", "", -1)
-	result, err = jsonutils.Parse([]byte(_data))
-	if err != nil {
-		return nil, err
-	}
-	if result.Contains("error") {
-		log.Errorf("Azure %s request: %s \nbody: %s error: %v", req.Method, req.URL.String(), body, result.String())
-		return nil, fmt.Errorf(result.String())
-	}
-	return result, nil
+	return jsonutils.Parse([]byte(_data))
 }
 
 func (self *SAzureClient) UpdateAccount(tenantId, secret, envName string) error {
diff --git a/pkg/util/azure/debug.go b/pkg/util/azure/debug.go
index 8da73e4983..57beb087d0 100644
--- a/pkg/util/azure/debug.go
+++ b/pkg/util/azure/debug.go
@@ -9,6 +9,10 @@ import (
 	"github.com/Azure/go-autorest/autorest"
 )
 
+const (
+	DEBUG = false
+)
+
 func LogRequest() autorest.PrepareDecorator {
 	return func(p autorest.Preparer) autorest.Preparer {
 		return autorest.PreparerFunc(func(r *http.Request) (*http.Request, error) {
diff --git a/pkg/util/azure/service.go b/pkg/util/azure/service.go
new file mode 100644
index 0000000000..c5fa17abe7
--- /dev/null
+++ b/pkg/util/azure/service.go
@@ -0,0 +1,55 @@
+package azure
+
+import (
+	"fmt"
+)
+
+type SServices struct {
+	Value []SService `json:"value,omitempty"`
+}
+
+type SService struct {
+	ID                string         `json:"id,omitempty"`
+	Namespace         string         `json:"namespace,omitempty"`
+	RegistrationState string         `json:"registrationState,omitempty"`
+	ResourceTypes     []ResourceType `json:"resourceTypes,omitempty"`
+}
+
+type ResourceType struct {
+	ApiVersions  []string `json:"apiVersions,omitempty"`
+	Capabilities string   `json:"capabilities,omitempty"`
+	Locations    []string `json:"locations,omitempty"`
+	ResourceType string   `json:"locations,omitempty"`
+}
+
+func (self *SRegion) ListServices() ([]SService, error) {
+	services := []SService{}
+	return services, self.client.List("providers", &services)
+}
+
+func (self *SRegion) SerciceShow(serviceType string) (*SService, error) {
+	service := SService{}
+	return &service, self.client.Get("providers/"+serviceType, []string{}, &service)
+}
+
+func (self *SRegion) serviceOperation(resourceType, operation string) error {
+	services, err := self.ListServices()
+	if err != nil {
+		return err
+	}
+	for _, service := range services {
+		if service.Namespace == resourceType {
+			_, err := self.client.jsonRequest("POST", fmt.Sprintf("%s/%s", service.ID, operation), "")
+			return err
+		}
+	}
+	return fmt.Errorf("failed to find namespace: %s", resourceType)
+}
+
+func (self *SRegion) ServiceRegister(resourceType string) error {
+	return self.serviceOperation(resourceType, "register")
+}
+
+func (self *SRegion) ServiceUnRegister(resourceType string) error {
+	return self.serviceOperation(resourceType, "unregister")
+}
diff --git a/pkg/util/azure/shell/service.go b/pkg/util/azure/shell/service.go
new file mode 100644
index 0000000000..2f1c26bb5f
--- /dev/null
+++ b/pkg/util/azure/shell/service.go
@@ -0,0 +1,41 @@
+package shell
+
+import (
+	"yunion.io/x/onecloud/pkg/util/azure"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	type ServiceListOptions struct {
+	}
+	shellutils.R(&ServiceListOptions{}, "service-list", "List providers", func(cli *azure.SRegion, args *ServiceListOptions) error {
+		services, err := cli.ListServices()
+		if err != nil {
+			return err
+		}
+		printList(services, len(services), 0, 0, []string{})
+		return nil
+	})
+
+	type ServiceOptions struct {
+		NAME string `help:"Name for service register"`
+	}
+
+	shellutils.R(&ServiceOptions{}, "service-register", "Register service", func(cli *azure.SRegion, args *ServiceOptions) error {
+		return cli.ServiceRegister(args.NAME)
+	})
+
+	shellutils.R(&ServiceOptions{}, "service-unregister", "Unregister service", func(cli *azure.SRegion, args *ServiceOptions) error {
+		return cli.ServiceUnRegister(args.NAME)
+	})
+
+	shellutils.R(&ServiceOptions{}, "service-show", "Show service detail", func(cli *azure.SRegion, args *ServiceOptions) error {
+		service, err := cli.SerciceShow(args.NAME)
+		if err != nil {
+			return err
+		}
+		printObject(service)
+		return nil
+	})
+
+}
diff --git a/pkg/util/azure/shell/vpc.go b/pkg/util/azure/shell/vpc.go
index 6c068fdc0b..552a22c654 100644
--- a/pkg/util/azure/shell/vpc.go
+++ b/pkg/util/azure/shell/vpc.go
@@ -32,4 +32,23 @@ func init() {
 		printObject(vpc)
 		return nil
 	})
+
+	shellutils.R(&VpcOptions{}, "vpc-delete", "Delete vpc", func(cli *azure.SRegion, args *VpcOptions) error {
+		return cli.DeleteVpc(args.ID)
+	})
+
+	type VpcCreateOptions struct {
+		NAME string `help:"vpc Name"`
+		CIDR string `help:"vpc cidr"`
+		Desc string `help:"vpc description"`
+	}
+
+	shellutils.R(&VpcCreateOptions{}, "vpc-create", "Create vpc", func(cli *azure.SRegion, args *VpcCreateOptions) error {
+		vpc, err := cli.CreateIVpc(args.NAME, args.Desc, args.CIDR)
+		if err != nil {
+			return err
+		}
+		printObject(vpc)
+		return nil
+	})
 }
diff --git a/pkg/util/azure/vpc.go b/pkg/util/azure/vpc.go
index b75d14bbde..07060c4a4c 100644
--- a/pkg/util/azure/vpc.go
+++ b/pkg/util/azure/vpc.go
@@ -77,7 +77,11 @@ func (self *SVpc) GetCidrBlock() string {
 }
 
 func (self *SVpc) Delete() error {
-	return self.region.client.Delete(self.ID)
+	return self.region.DeleteVpc(self.ID)
+}
+
+func (self *SRegion) DeleteVpc(vpcId string) error {
+	return self.client.Delete(vpcId)
 }
 
 func (self *SVpc) getSecurityGroups() ([]SSecurityGroup, error) {

From 69d38d333ad0437f26000aba1db7d28c12f7c148 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= <quxuan@yunionyun.com>
Date: Mon, 19 Nov 2018 15:04:03 +0800
Subject: [PATCH 42/57] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=8F=82=E6=95=B0?=
 =?UTF-8?q?=E7=BC=BA=E5=B0=91=E9=97=AE=E9=A2=98?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/util/azure/resourcesku.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pkg/util/azure/resourcesku.go b/pkg/util/azure/resourcesku.go
index b26207fe74..2c6e6e80af 100644
--- a/pkg/util/azure/resourcesku.go
+++ b/pkg/util/azure/resourcesku.go
@@ -2,6 +2,7 @@ package azure
 
 import (
 	"fmt"
+
 	"yunion.io/x/pkg/utils"
 )
 
@@ -108,7 +109,7 @@ func (self *SAzureClient) ListResourceSkus() ([]SResourceSku, error) {
 	url := fmt.Sprintf("/subscriptions/%s/providers/Microsoft.Compute/skus?api-version=2017-09-01", self.subscriptionId)
 	skus := make([]SResourceSku, 0)
 	for {
-		body, err := jsonRequest(cli, "GET", self.domain, url, "")
+		body, err := jsonRequest(cli, "GET", self.domain, url, self.subscriptionId, "")
 		if err != nil {
 			return nil, err
 		}

From d7653387561d3a37e44325a8a962dd0fb19e3d73 Mon Sep 17 00:00:00 2001
From: Zexi Li <lizexi@yunionyun.com>
Date: Mon, 19 Nov 2018 18:05:04 +0800
Subject: [PATCH 43/57] climc: kube cluster restart agents

---
 cmd/climc/shell/k8s/clusters.go     | 13 +++++++++++++
 pkg/mcclient/options/k8s/cluster.go | 18 ++++++++++++++++++
 2 files changed, 31 insertions(+)

diff --git a/cmd/climc/shell/k8s/clusters.go b/cmd/climc/shell/k8s/clusters.go
index ebfed4b12f..6fcbaf7b9b 100644
--- a/cmd/climc/shell/k8s/clusters.go
+++ b/cmd/climc/shell/k8s/clusters.go
@@ -203,4 +203,17 @@ func initCluster() {
 		printObject(ret)
 		return nil
 	})
+
+	R(&o.ClusterRestartAgentsOptions{}, cmdN("restart-agent"), "Restart node agents in cluster", func(s *mcclient.ClientSession, args *o.ClusterRestartAgentsOptions) error {
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
+		ret, err := k8s.Clusters.PerformAction(s, args.ID, "restart-agent", params)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
 }
diff --git a/pkg/mcclient/options/k8s/cluster.go b/pkg/mcclient/options/k8s/cluster.go
index b26974cc58..27207247d7 100644
--- a/pkg/mcclient/options/k8s/cluster.go
+++ b/pkg/mcclient/options/k8s/cluster.go
@@ -194,3 +194,21 @@ func (o ClusterDeleteNodesOptions) Params() (*jsonutils.JSONDict, error) {
 	params.Add(nodesArray, "nodes")
 	return params, nil
 }
+
+type ClusterRestartAgentsOptions struct {
+	ClusterDeleteNodesOptions
+	All bool `help:"Restart all nodes agent"`
+}
+
+func (o ClusterRestartAgentsOptions) Params() (*jsonutils.JSONDict, error) {
+	params, err := o.ClusterDeleteNodesOptions.Params()
+	if err != nil {
+		return nil, err
+	}
+	all := jsonutils.JSONFalse
+	if o.All {
+		all = jsonutils.JSONTrue
+	}
+	params.Add(all, "all")
+	return params, nil
+}

From 4af0ce699ccfad0d55061e0e978f804e976268ba Mon Sep 17 00:00:00 2001
From: Yousong Zhou <zhouyousong@yunionyun.com>
Date: Mon, 19 Nov 2018 10:37:48 +0000
Subject: [PATCH 44/57] loadbalancers: use httperrors for parameter validation
 results

---
 pkg/compute/models/loadbalanceracls.go   | 9 +++++----
 pkg/compute/models/loadbalanceragents.go | 2 +-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/pkg/compute/models/loadbalanceracls.go b/pkg/compute/models/loadbalanceracls.go
index 67dd190c6c..94fd544b2a 100644
--- a/pkg/compute/models/loadbalanceracls.go
+++ b/pkg/compute/models/loadbalanceracls.go
@@ -14,6 +14,7 @@ import (
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
@@ -33,16 +34,16 @@ func (aclEntry *SLoadbalancerAclEntry) Validate(data *jsonutils.JSONDict) error
 	} else {
 		ip := net.ParseIP(aclEntry.Cidr).To4()
 		if ip == nil {
-			return fmt.Errorf("invalid addr %s", aclEntry.Cidr)
+			return httperrors.NewInputParameterError("invalid addr %s", aclEntry.Cidr)
 		}
 	}
 	if commentLimit := 128; len(aclEntry.Comment) > commentLimit {
-		return fmt.Errorf("comment too long (%d>=%d)",
+		return httperrors.NewInputParameterError("comment too long (%d>=%d)",
 			len(aclEntry.Comment), commentLimit)
 	}
 	for _, r := range aclEntry.Comment {
 		if !unicode.IsPrint(r) {
-			return fmt.Errorf("comment contains non-printable char: %v", r)
+			return httperrors.NewInputParameterError("comment contains non-printable char: %v", r)
 		}
 	}
 	return nil
@@ -68,7 +69,7 @@ func (aclEntries *SLoadbalancerAclEntries) Validate(data *jsonutils.JSONDict) er
 		}
 		if _, ok := found[aclEntry.Cidr]; ok {
 			// error so that the user has a chance to deal with comments
-			return fmt.Errorf("acl cidr duplicate %s", aclEntry.Cidr)
+			return httperrors.NewInputParameterError("acl cidr duplicate %s", aclEntry.Cidr)
 		}
 		found[aclEntry.Cidr] = true
 	}
diff --git a/pkg/compute/models/loadbalanceragents.go b/pkg/compute/models/loadbalanceragents.go
index d73e6ed37c..29269063fd 100644
--- a/pkg/compute/models/loadbalanceragents.go
+++ b/pkg/compute/models/loadbalanceragents.go
@@ -154,7 +154,7 @@ func (p *SLoadbalancerAgentParamsTelegraf) Validate(data *jsonutils.JSONDict) er
 	if p.InfluxDbOutputUrl != "" {
 		_, err := url.Parse(p.InfluxDbOutputUrl)
 		if err != nil {
-			return err
+			return httperrors.NewInputParameterError("telegraf params: invalid influxdb url: %s", err)
 		}
 	}
 	if p.HaproxyInputInterval <= 0 {

From f9ac0c020483b77b448a678de7b81e9d34fc158f Mon Sep 17 00:00:00 2001
From: Yousong Zhou <zhouyousong@yunionyun.com>
Date: Mon, 19 Nov 2018 10:16:28 +0000
Subject: [PATCH 45/57] validators: errors: retain msg fmt and params

---
 pkg/cloudcommon/validators/errors.go | 46 ++++++++++++----------------
 1 file changed, 20 insertions(+), 26 deletions(-)

diff --git a/pkg/cloudcommon/validators/errors.go b/pkg/cloudcommon/validators/errors.go
index 9045fc93ed..48f4564ddd 100644
--- a/pkg/cloudcommon/validators/errors.go
+++ b/pkg/cloudcommon/validators/errors.go
@@ -54,65 +54,59 @@ func (ve *ValidateError) Error() string {
 
 // TODO let each validator provide the error
 func newMissingKeyError(key string) error {
-	msg := fmt.Sprintf("missing %q", key)
-	return newError(ERR_MISSING_KEY, msg)
+	return newError(ERR_MISSING_KEY, "missing %q", key)
 }
 
 func newGeneralError(key string, err error) error {
-	msg := fmt.Sprintf("general error for %q: %s", key, err)
-	return newError(ERR_GENERAL, msg)
+	return newError(ERR_GENERAL, "general error for %q: %s", key, err)
 }
 
 func newInvalidTypeError(key string, typ string, err error) error {
-	msg := fmt.Sprintf("expecting %s type for %q: %s", typ, key, err)
-	return newError(ERR_INVALID_TYPE, msg)
+	return newError(ERR_INVALID_TYPE, "expecting %s type for %q: %s", typ, key, err)
 }
 
 func newInvalidChoiceError(key string, choices Choices, choice string) error {
-	msg := fmt.Sprintf("invalid %q, want %s, got %s", key, choices, choice)
-	return newError(ERR_INVALID_CHOICE, msg)
+	return newError(ERR_INVALID_CHOICE, "invalid %q, want %s, got %s", key, choices, choice)
 }
 
 func newNotInRangeError(key string, value, lower, upper int64) error {
-	msg := fmt.Sprintf("invalid %q: %d, want [%d,%d]", key, value, lower, upper)
-	return newError(ERR_NOT_IN_RANGE, msg)
+	return newError(ERR_NOT_IN_RANGE, "invalid %q: %d, want [%d,%d]", key, value, lower, upper)
 }
 
 func newInvalidValueError(key string, value string) error {
-	msg := fmt.Sprintf("invalid %q: %s", key, value)
-	return newError(ERR_INVALID_VALUE, msg)
+	return newError(ERR_INVALID_VALUE, "invalid %q: %s", key, value)
 }
 
 func newModelManagerError(modelKeyword string) error {
-	msg := fmt.Sprintf("internal error: getting model manager for %q failed",
-		modelKeyword)
-	return newError(ERR_MODEL_MANAGER, msg)
+	return newError(ERR_MODEL_MANAGER, "failed getting model manager for %q", modelKeyword)
 }
 
 func newModelNotFoundError(modelKeyword, idOrName string, err error) error {
-	msg := fmt.Sprintf("cannot find %q with id/name %q",
-		modelKeyword, idOrName)
+	errFmt := "cannot find %q with id/name %q"
+	params := []interface{}{modelKeyword, idOrName}
 	if err != sql.ErrNoRows {
-		msg += ": " + err.Error()
+		errFmt += ": %s"
+		params = append(params, err.Error())
 	}
-	return newError(ERR_MODEL_NOT_FOUND, msg)
+	return newError(ERR_MODEL_NOT_FOUND, errFmt, params...)
 }
 
-func newError(typ ErrType, msg string) error {
-	err := &ValidateError{
-		ErrType: typ,
-		Msg:     msg,
-	}
+func newError(typ ErrType, errFmt string, params ...interface{}) error {
+	errFmt = fmt.Sprintf("%s: %s", typ, errFmt)
 	if returnHttpError {
 		switch typ {
 		case ERR_SUCCESS:
 			return nil
 		case ERR_GENERAL, ERR_MODEL_MANAGER:
-			return httperrors.NewInternalServerError(msg)
+			return httperrors.NewInternalServerError(errFmt, params...)
 		default:
-			return httperrors.NewInputParameterError(msg)
+			return httperrors.NewInputParameterError(errFmt, params...)
 		}
 	}
+	err := &ValidateError{
+		ErrType: typ,
+		Msg:     fmt.Sprintf(errFmt, params...),
+	}
 	return err
 }
 

From 751525d77937e3e9805b66bf4a3f34bd34355bef Mon Sep 17 00:00:00 2001
From: Yousong Zhou <zhouyousong@yunionyun.com>
Date: Mon, 19 Nov 2018 10:38:19 +0000
Subject: [PATCH 46/57] validators: add newInvalidStructError()

To retain fmt and parameter information

	{"class":"InputParameterError","code":400,"data":{"fields":["acl_entries","192.168.0.256"],"id":"Invalid value error: invalid %q: invalid addr %s"},"details":"Invalid value error: invalid \"acl_entries\": invalid addr 192.168.0.256"}
	{"class":"InputParameterError","code":400,"data":{"fields":[],"id":"invalid \"acl_entries\": invalid addr 192.168.0.256"},"details":"invalid \"acl_entries\": invalid addr 192.168.0.256"}
---
 pkg/cloudcommon/validators/errors.go     | 14 ++++++++++++++
 pkg/cloudcommon/validators/validators.go |  2 +-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/pkg/cloudcommon/validators/errors.go b/pkg/cloudcommon/validators/errors.go
index 48f4564ddd..9f32801303 100644
--- a/pkg/cloudcommon/validators/errors.go
+++ b/pkg/cloudcommon/validators/errors.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 )
 
 var returnHttpError = true
@@ -77,6 +78,19 @@ func newInvalidValueError(key string, value string) error {
 	return newError(ERR_INVALID_VALUE, "invalid %q: %s", key, value)
 }
 
+func newInvalidStructError(key string, err error) error {
+	errFmt := "invalid %q: "
+	params := []interface{}{key}
+	jsonClientErr, ok := err.(*httputils.JSONClientError)
+	if ok {
+		errFmt += jsonClientErr.Data.Id
+		for _, f := range jsonClientErr.Data.Fields {
+			params = append(params, f)
+		}
+	}
+	return newError(ERR_INVALID_VALUE, errFmt, params...)
+}
+
 func newModelManagerError(modelKeyword string) error {
 	return newError(ERR_MODEL_MANAGER, "failed getting model manager for %q", modelKeyword)
 }
diff --git a/pkg/cloudcommon/validators/validators.go b/pkg/cloudcommon/validators/validators.go
index 9594138a5b..c1af053458 100644
--- a/pkg/cloudcommon/validators/validators.go
+++ b/pkg/cloudcommon/validators/validators.go
@@ -511,7 +511,7 @@ func (v *ValidatorStruct) Validate(data *jsonutils.JSONDict) error {
 	if valueValidator, ok := v.Value.(IValidatorBase); ok {
 		err = valueValidator.Validate(data)
 		if err != nil {
-			return newInvalidValueError(v.Key, err.Error())
+			return newInvalidStructError(v.Key, err)
 		}
 	}
 	data.Set(v.Key, jsonutils.Marshal(v.Value))

From ff5071eba26efd3e5f66249d28d5e1d66270afe2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= <quxuan@yunionyun.com>
Date: Mon, 19 Nov 2018 20:26:24 +0800
Subject: [PATCH 47/57] =?UTF-8?q?priceKey=E6=90=BA=E5=B8=A6region=E4=BF=A1?=
 =?UTF-8?q?=E6=81=AF?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/util/azure/classic_instance.go | 3 ++-
 pkg/util/azure/instance.go         | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/pkg/util/azure/classic_instance.go b/pkg/util/azure/classic_instance.go
index 388e9f3620..03f622e386 100644
--- a/pkg/util/azure/classic_instance.go
+++ b/pkg/util/azure/classic_instance.go
@@ -113,7 +113,8 @@ type SClassicInstance struct {
 
 func (self *SClassicInstance) GetMetadata() *jsonutils.JSONDict {
 	data := jsonutils.NewDict()
-	data.Add(jsonutils.NewString(self.Properties.HardwareProfile.Size), "price_key")
+	priceKey := fmt.Sprintf("%s::%s", self.Properties.HardwareProfile.Size, self.host.zone.region.Name)
+	data.Add(jsonutils.NewString(priceKey), "price_key")
 	if self.Properties.NetworkProfile.NetworkSecurityGroup != nil {
 		data.Add(jsonutils.NewString(self.Properties.NetworkProfile.NetworkSecurityGroup.ID), "secgroupId")
 	}
diff --git a/pkg/util/azure/instance.go b/pkg/util/azure/instance.go
index aff99f53ee..5da04a5f94 100644
--- a/pkg/util/azure/instance.go
+++ b/pkg/util/azure/instance.go
@@ -224,7 +224,8 @@ func (self *SInstance) GetMetadata() *jsonutils.JSONDict {
 		data.Add(jsonutils.NewString(loginKey), "login_key")
 	}
 
-	data.Add(jsonutils.NewString(self.Properties.HardwareProfile.VMSize), "price_key")
+	priceKey := fmt.Sprintf("%s::%s", self.Properties.HardwareProfile.VMSize, self.host.zone.region.Name)
+	data.Add(jsonutils.NewString(priceKey), "price_key")
 	if nics, err := self.getNics(); err == nil {
 		for _, nic := range nics {
 			if nic.Properties.NetworkSecurityGroup != nil {

From 5071e8ae6a3b1570437127d894745d7586374c14 Mon Sep 17 00:00:00 2001
From: ningyu <ningyu@yunionyun.com>
Date: Mon, 19 Nov 2018 20:56:53 +0800
Subject: [PATCH 48/57] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0cloudmon?=
 =?UTF-8?q?=E6=A8=A1=E5=9D=97=E5=92=8Cclimc=20underutilizedinstances-list?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 cmd/climc/shell/underutilizedinstances.go     | 34 +++++++++++++++++++
 pkg/mcclient/modules/managers.go              |  9 +++++
 .../modules/mod_underutilizedinstances.go     | 13 +++++++
 3 files changed, 56 insertions(+)
 create mode 100644 cmd/climc/shell/underutilizedinstances.go
 create mode 100644 pkg/mcclient/modules/mod_underutilizedinstances.go

diff --git a/cmd/climc/shell/underutilizedinstances.go b/cmd/climc/shell/underutilizedinstances.go
new file mode 100644
index 0000000000..a8bebb1e87
--- /dev/null
+++ b/cmd/climc/shell/underutilizedinstances.go
@@ -0,0 +1,34 @@
+package shell
+
+import (
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func init() {
+
+	type UnderutilizedInstancesListOptions struct {
+		options.BaseListOptions
+	}
+	R(&UnderutilizedInstancesListOptions{}, "underutilizedinstances-list", "List underutilized instances", func(s *mcclient.ClientSession, args *UnderutilizedInstancesListOptions) error {
+		var params *jsonutils.JSONDict
+		{
+			var err error
+			params, err = args.BaseListOptions.Params()
+			if err != nil {
+				return err
+
+			}
+		}
+		result, err := modules.UnderutilizedInstances.List(s, params)
+		if err != nil {
+			return err
+		}
+
+		printList(result, modules.UnderutilizedInstances.GetColumns(s))
+		return nil
+	})
+
+}
diff --git a/pkg/mcclient/modules/managers.go b/pkg/mcclient/modules/managers.go
index cf51952d23..6172530fc7 100644
--- a/pkg/mcclient/modules/managers.go
+++ b/pkg/mcclient/modules/managers.go
@@ -25,6 +25,15 @@ func NewMonitorManager(keyword, keywordPlural string, columns, adminColumns []st
 		Keyword: keyword, KeywordPlural: keywordPlural}
 }
 
+func NewCloudmonManager(keyword, keywordPlural string, columns, adminColumns []string) ResourceManager {
+	return ResourceManager{
+		BaseManager: BaseManager{columns: columns,
+			adminColumns: adminColumns,
+			version:      "v1",
+			serviceType:  "cloudmon"},
+		Keyword: keyword, KeywordPlural: keywordPlural}
+}
+
 func NewNotifyManager(keyword, keywordPlural string, columns, adminColumns []string) ResourceManager {
 	return ResourceManager{
 		BaseManager: BaseManager{columns: columns,
diff --git a/pkg/mcclient/modules/mod_underutilizedinstances.go b/pkg/mcclient/modules/mod_underutilizedinstances.go
new file mode 100644
index 0000000000..8c0e6af42f
--- /dev/null
+++ b/pkg/mcclient/modules/mod_underutilizedinstances.go
@@ -0,0 +1,13 @@
+package modules
+
+var (
+	UnderutilizedInstances ResourceManager
+)
+
+func init() {
+	UnderutilizedInstances = NewCloudmonManager("underutilizedinstance", "underutilizedinstances",
+		[]string{"id", "vm_id", "vm_name", "time", "advices", "vm_cpu", "vm_disk", "vm_memory", "vm_provider"},
+		[]string{})
+
+	register(&UnderutilizedInstances)
+}

From d1680ade9512ee37e6e8d4a0624ed9f813a3b7ac Mon Sep 17 00:00:00 2001
From: ningyu <ningyu@yunionyun.com>
Date: Mon, 19 Nov 2018 21:11:46 +0800
Subject: [PATCH 49/57] =?UTF-8?q?fix:=20=E4=BF=AE=E6=94=B9=E6=9C=AA?=
 =?UTF-8?q?=E5=85=85=E5=88=86=E5=88=A9=E7=94=A8=E7=9A=84=E5=AE=9E=E4=BE=8B?=
 =?UTF-8?q?=E5=88=97=E8=A1=A8API=E7=9A=84climc=E5=91=BD=E5=90=8D=EF=BC=8C?=
 =?UTF-8?q?=E7=9B=AE=E7=9A=84=E6=98=AF=E9=81=BF=E5=85=8D=E8=BF=87=E9=95=BF?=
 =?UTF-8?q?=E4=B8=8D=E6=98=93=E8=AF=BB?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 cmd/climc/shell/underutilizedinstances.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/climc/shell/underutilizedinstances.go b/cmd/climc/shell/underutilizedinstances.go
index a8bebb1e87..84fd17ed08 100644
--- a/cmd/climc/shell/underutilizedinstances.go
+++ b/cmd/climc/shell/underutilizedinstances.go
@@ -12,7 +12,7 @@ func init() {
 	type UnderutilizedInstancesListOptions struct {
 		options.BaseListOptions
 	}
-	R(&UnderutilizedInstancesListOptions{}, "underutilizedinstances-list", "List underutilized instances", func(s *mcclient.ClientSession, args *UnderutilizedInstancesListOptions) error {
+	R(&UnderutilizedInstancesListOptions{}, "underutilized-instances-list", "List underutilized instances", func(s *mcclient.ClientSession, args *UnderutilizedInstancesListOptions) error {
 		var params *jsonutils.JSONDict
 		{
 			var err error

From 7f51e6d919b153c3892ea9942e192a1857f59c79 Mon Sep 17 00:00:00 2001
From: TangBin <tangbin@yunionyun.com>
Date: Tue, 20 Nov 2018 10:33:19 +0800
Subject: [PATCH 50/57] fix confilict

---
 cmd/climc/shell/cloudmeta.go          | 16 ++++++++++++++++
 pkg/mcclient/modules/managers.go      |  8 ++++++++
 pkg/mcclient/modules/mod_cloudmeta.go | 13 +++++++++++++
 pkg/mcclient/modules/resource.go      |  2 +-
 pkg/mcclient/options/servers.go       |  2 +-
 5 files changed, 39 insertions(+), 2 deletions(-)
 create mode 100644 cmd/climc/shell/cloudmeta.go
 create mode 100644 pkg/mcclient/modules/mod_cloudmeta.go

diff --git a/cmd/climc/shell/cloudmeta.go b/cmd/climc/shell/cloudmeta.go
new file mode 100644
index 0000000000..6fa020a639
--- /dev/null
+++ b/cmd/climc/shell/cloudmeta.go
@@ -0,0 +1,16 @@
+package shell
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+func init() {
+	type CloudmetaOptions struct {
+		PROVIDER_ID string `help:"provider_id"`
+		REGION_ID   string 	`help:"region_id"`
+		ZONE_ID   string 	`help:"zone_id"`
+	}
+	R(&CloudmetaOptions{}, "instance-type-list", "query backend service for its version", func(s *mcclient.ClientSession, args *CloudmetaOptions) error {
+		return nil
+	})
+}
diff --git a/pkg/mcclient/modules/managers.go b/pkg/mcclient/modules/managers.go
index cf51952d23..ed6e17a0c4 100644
--- a/pkg/mcclient/modules/managers.go
+++ b/pkg/mcclient/modules/managers.go
@@ -139,3 +139,11 @@ func NewWebsocketManager(keyword, keywordPlural string, columns, adminColumns []
 			serviceType:  "websocket"},
 		Keyword: keyword, KeywordPlural: keywordPlural}
 }
+
+func NewCloudmetaManager(keyword, keywordPlural string, columns, adminColumns []string) ResourceManager {
+	return ResourceManager{
+		BaseManager: BaseManager{columns: columns,
+			adminColumns: adminColumns,
+			serviceType:  "cloudmeta"},
+		Keyword: keyword, KeywordPlural: keywordPlural}
+}
\ No newline at end of file
diff --git a/pkg/mcclient/modules/mod_cloudmeta.go b/pkg/mcclient/modules/mod_cloudmeta.go
new file mode 100644
index 0000000000..e1f5ca497e
--- /dev/null
+++ b/pkg/mcclient/modules/mod_cloudmeta.go
@@ -0,0 +1,13 @@
+package modules
+
+var (
+	Cloudmeta ResourceManager
+)
+
+func init() {
+	Cloudmeta = NewCloudmetaManager("cloudmeta", "cloudmetas",
+		[]string{},
+		[]string{})
+
+	register(&Cloudmeta)
+}
diff --git a/pkg/mcclient/modules/resource.go b/pkg/mcclient/modules/resource.go
index 65528750c4..35399fa2a2 100644
--- a/pkg/mcclient/modules/resource.go
+++ b/pkg/mcclient/modules/resource.go
@@ -297,7 +297,7 @@ func (this *ResourceManager) params2Body(s *mcclient.ClientSession, params jsonu
 	return body
 }
 
-func (this *ResourceManager) Create(session *mcclient.ClientSession, params jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+func (this *ResourceManager)Create(session *mcclient.ClientSession, params jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	return this.CreateInContexts(session, params, nil)
 }
 
diff --git a/pkg/mcclient/options/servers.go b/pkg/mcclient/options/servers.go
index 89924038bf..dd486bb278 100644
--- a/pkg/mcclient/options/servers.go
+++ b/pkg/mcclient/options/servers.go
@@ -18,7 +18,7 @@ type ServerListOptions struct {
 	Gpu           *bool  `help:"Show gpu servers"`
 	Secgroup      string `help:"Secgroup ID or Name"`
 	AdminSecgroup string `help:"AdminSecgroup ID or Name"`
-	Hypervisor    string `help:"Show server of hypervisor" choices:"kvm|esxi|container|baremetal|aliyun|azure"`
+	Hypervisor    string `help:"Show server of hypervisor" choices:"kvm|esxi|container|baremetal|aliyun|azure|aws"`
 	Manager       string `help:"Show servers imported from manager"`
 	Region        string `help:"Show servers in cloudregion"`
 	WithEip       *bool  `help:"Show Servers with EIP"`

From 5652ac4dfb6fe25ced3e4da6136789674403d31f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= <quxuan@yunionyun.com>
Date: Tue, 20 Nov 2018 14:42:50 +0800
Subject: [PATCH 51/57] =?UTF-8?q?=E4=BF=AE=E5=A4=8D:=20=E4=BB=BB=E5=8A=A1?=
 =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E6=97=B6=E4=B8=8D=E5=8F=8A=E6=97=B6=E9=87=8A?=
 =?UTF-8?q?=E6=94=BEpending-usage-quota?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/compute/tasks/guest_create_task.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pkg/compute/tasks/guest_create_task.go b/pkg/compute/tasks/guest_create_task.go
index a57f49afca..916195563d 100644
--- a/pkg/compute/tasks/guest_create_task.go
+++ b/pkg/compute/tasks/guest_create_task.go
@@ -50,6 +50,7 @@ func (self *GuestCreateTask) OnDiskPreparedFailed(ctx context.Context, obj db.IS
 	guest.SetStatus(self.UserCred, models.VM_DISK_FAILED, "allocation failed")
 	db.OpsLog.LogEvent(guest, db.ACT_ALLOCATE_FAIL, data, self.UserCred)
 	notifyclient.NotifySystemError(guest.Id, guest.Name, models.VM_DISK_FAILED, data.String())
+	self.SetStageFailed(ctx, data.String())
 }
 
 func (self *GuestCreateTask) OnDiskPrepared(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
@@ -77,6 +78,7 @@ func (self *GuestCreateTask) OnCdromPreparedFailed(ctx context.Context, obj db.I
 	guest.SetStatus(self.UserCred, models.VM_DISK_FAILED, "")
 	db.OpsLog.LogEvent(guest, db.ACT_ALLOCATE_FAIL, data, self.UserCred)
 	notifyclient.NotifySystemError(guest.Id, guest.Name, models.VM_DISK_FAILED, fmt.Sprintf("cdrom_failed %s", data))
+	self.SetStageFailed(ctx, fmt.Sprintf("cdrom_failed %s", data))
 }
 
 func (self *GuestCreateTask) StartDeployGuest(ctx context.Context, guest *models.SGuest) {
@@ -103,6 +105,7 @@ func (self *GuestCreateTask) OnDeployGuestDescCompleteFailed(ctx context.Context
 	guest.SetStatus(self.UserCred, models.VM_DEPLOY_FAILED, "deploy_failed")
 	db.OpsLog.LogEvent(guest, db.ACT_ALLOCATE_FAIL, data, self.UserCred)
 	notifyclient.NotifySystemError(guest.Id, guest.Name, models.VM_DEPLOY_FAILED, data.String())
+	self.SetStageFailed(ctx, data.String())
 }
 
 func (self *GuestCreateTask) OnAutoStartGuest(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {

From 4f63b4ad587375a9e715e0d80f3b53edc9b6b6b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= <quxuan@yunionyun.com>
Date: Tue, 20 Nov 2018 16:00:16 +0800
Subject: [PATCH 52/57] =?UTF-8?q?=E9=81=BF=E5=85=8D=E5=9B=A0snapshot?=
 =?UTF-8?q?=E6=89=BE=E4=B8=8D=E5=88=B0=E8=80=8C=E5=88=A0=E9=99=A4=E5=A4=B1?=
 =?UTF-8?q?=E8=B4=A5?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/compute/models/snapshots.go               | 24 ++++++-------------
 pkg/compute/tasks/guest_disk_snapshot_task.go | 10 +++++---
 pkg/util/aliyun/snapshot.go                   | 13 ++++++----
 3 files changed, 22 insertions(+), 25 deletions(-)

diff --git a/pkg/compute/models/snapshots.go b/pkg/compute/models/snapshots.go
index 367f05bb71..94085c7074 100644
--- a/pkg/compute/models/snapshots.go
+++ b/pkg/compute/models/snapshots.go
@@ -9,7 +9,6 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/util/compare"
-	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
@@ -319,29 +318,20 @@ func (self *SSnapshot) CustomizeDelete(ctx context.Context, userCred mcclient.To
 	if self.Status == SNAPSHOT_DELETING {
 		return fmt.Errorf("Cannot delete snapshot in status %s", self.Status)
 	}
-	if self.Status == SNAPSHOT_UNKNOWN {
-		return self.RealDelete(ctx, userCred)
-	}
 	if len(self.ExternalId) == 0 {
-		if utils.IsInStringArray(self.Status, []string{SNAPSHOT_FAILED}) {
-			return self.RealDelete(ctx, userCred)
-		}
 		if self.CreatedBy == MANUAL {
 			if !self.FakeDeleted {
 				return self.FakeDelete()
-			} else {
-				_, err := SnapshotManager.GetConvertSnapshot(self)
-				if err != nil {
-					return fmt.Errorf("Cannot delete snapshot: %s, disk need at least one of snapshot as backing file", err.Error())
-				}
-				return self.StartSnapshotDeleteTask(ctx, userCred, false, "")
 			}
-		} else {
-			return fmt.Errorf("Cannot delete snapshot created by %s", self.CreatedBy)
+			_, err := SnapshotManager.GetConvertSnapshot(self)
+			if err != nil {
+				return fmt.Errorf("Cannot delete snapshot: %s, disk need at least one of snapshot as backing file", err.Error())
+			}
+			return self.StartSnapshotDeleteTask(ctx, userCred, false, "")
 		}
-	} else {
-		return self.StartSnapshotDeleteTask(ctx, userCred, false, "")
+		return fmt.Errorf("Cannot delete snapshot created by %s", self.CreatedBy)
 	}
+	return self.StartSnapshotDeleteTask(ctx, userCred, false, "")
 }
 
 func (self *SSnapshot) AllowPerformDeleted(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
diff --git a/pkg/compute/tasks/guest_disk_snapshot_task.go b/pkg/compute/tasks/guest_disk_snapshot_task.go
index 4f5f230ef2..3f5fb8b7e9 100644
--- a/pkg/compute/tasks/guest_disk_snapshot_task.go
+++ b/pkg/compute/tasks/guest_disk_snapshot_task.go
@@ -162,12 +162,16 @@ func (self *SnapshotDeleteTask) deleteExternalSnapshot(ctx context.Context, snap
 	}
 	cloudSnapshot, err := cloudRegion.GetISnapshotById(snapshot.ExternalId)
 	if err != nil {
+		if err == cloudprovider.ErrNotFound {
+			return nil
+		}
 		log.Errorln(err, cloudSnapshot)
 		return err
 	}
-	cloudSnapshot.Delete()
-	err = cloudprovider.WaitDeleted(cloudSnapshot, 10*time.Second, 300*time.Second)
-	return err
+	if err := cloudSnapshot.Delete(); err != nil {
+		return err
+	}
+	return cloudprovider.WaitDeleted(cloudSnapshot, 10*time.Second, 300*time.Second)
 }
 
 func (self *SnapshotDeleteTask) StartReloadDisk(ctx context.Context, snapshot *models.SSnapshot, guest *models.SGuest) {
diff --git a/pkg/util/aliyun/snapshot.go b/pkg/util/aliyun/snapshot.go
index 87d9523970..b85837420a 100644
--- a/pkg/util/aliyun/snapshot.go
+++ b/pkg/util/aliyun/snapshot.go
@@ -167,11 +167,14 @@ func (self *SRegion) GetSnapshots(instanceId string, diskId string, snapshotName
 }
 
 func (self *SRegion) GetISnapshotById(snapshotId string) (cloudprovider.ICloudSnapshot, error) {
-	if snapshots, total, err := self.GetSnapshots("", "", "", []string{snapshotId}, 0, 1); err != nil {
+	snapshots, total, err := self.GetSnapshots("", "", "", []string{snapshotId}, 0, 1)
+	if err != nil {
 		return nil, err
-	} else if total != 1 {
-		return nil, cloudprovider.ErrNotFound
-	} else {
-		return &snapshots[0], nil
 	}
+	if total == 0 {
+		return nil, cloudprovider.ErrNotFound
+	} else if total > 1 {
+		return nil, cloudprovider.ErrDuplicateId
+	}
+	return &snapshots[0], nil
 }

From a62f7e1db2fb038ecb915e200ab4710fa4a88235 Mon Sep 17 00:00:00 2001
From: Yousong Zhou <zhouyousong@yunionyun.com>
Date: Tue, 20 Nov 2018 09:12:40 +0000
Subject: [PATCH 53/57] cloudproviders: reword error message in create data
 validation

---
 pkg/compute/models/cloudproviders.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/compute/models/cloudproviders.go b/pkg/compute/models/cloudproviders.go
index 5b25fb1cbb..18724288ec 100644
--- a/pkg/compute/models/cloudproviders.go
+++ b/pkg/compute/models/cloudproviders.go
@@ -130,7 +130,7 @@ func (self *SCloudprovider) ValidateUpdateData(ctx context.Context, userCred mcc
 }
 
 func (self *SCloudproviderManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
-	return nil, httperrors.NewUnsupportOperationError("Not support create cloudprovider, please considir create cloudaccount")
+	return nil, httperrors.NewUnsupportOperationError("Directly creating cloudprovider is not supported, create cloudaccount instead")
 }
 
 func (self *SCloudprovider) getPassword() (string, error) {

From cf2b42f78b5abbff4c4b1a91e7c3bcc4a54ce6ef Mon Sep 17 00:00:00 2001
From: Yousong Zhou <zhouyousong@yunionyun.com>
Date: Tue, 20 Nov 2018 09:12:08 +0000
Subject: [PATCH 54/57] climc: cloudprovider: drop cloud-provider-create
 command

It's not allowed since release/2.3.0
---
 cmd/climc/shell/cloudproviders.go | 32 -------------------------------
 1 file changed, 32 deletions(-)

diff --git a/cmd/climc/shell/cloudproviders.go b/cmd/climc/shell/cloudproviders.go
index 9ac65dfcf0..f35da1d11f 100644
--- a/cmd/climc/shell/cloudproviders.go
+++ b/cmd/climc/shell/cloudproviders.go
@@ -31,38 +31,6 @@ func init() {
 		return nil
 	})
 
-	type CloudproviderCreateOptions struct {
-		NAME      string `help:"Name of cloud provider"`
-		ACCOUNT   string `help:"Account to access the cloud provider, tenantId/subscriptionId for Azure"`
-		SECRET    string `help:"Secret to access the cloud provider, clientId/clientScret for Azure"`
-		PROVIDER  string `help:"Driver for cloud provider" choices:"VMware|Aliyun|Azure"`
-		AccessURL string `helo:"hello" metavar:"Azure choices: <AzureGermanCloud、AzureChinaCloud、AzureUSGovernmentCloud、AzurePublicCloud>"`
-		Desc      string `help:"Description"`
-		Enabled   bool   `help:"Enabled the provider automatically"`
-	}
-	R(&CloudproviderCreateOptions{}, "cloud-provider-create", "Create a cloud provider", func(s *mcclient.ClientSession, args *CloudproviderCreateOptions) error {
-		params := jsonutils.NewDict()
-		params.Add(jsonutils.NewString(args.NAME), "name")
-		params.Add(jsonutils.NewString(args.ACCOUNT), "account")
-		params.Add(jsonutils.NewString(args.SECRET), "secret")
-		params.Add(jsonutils.NewString(args.PROVIDER), "provider")
-		if args.Enabled {
-			params.Add(jsonutils.JSONTrue, "enabled")
-		}
-		if len(args.AccessURL) > 0 {
-			params.Add(jsonutils.NewString(args.AccessURL), "access_url")
-		}
-		if len(args.Desc) > 0 {
-			params.Add(jsonutils.NewString(args.Desc), "description")
-		}
-		result, err := modules.Cloudproviders.Create(s, params)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
 	type CloudproviderUpdateOptions struct {
 		ID        string `help:"ID or Name of cloud provider"`
 		Name      string `help:"New name to update"`

From f1b9ea55e70803d126c727e7973b5ce2fa17255b Mon Sep 17 00:00:00 2001
From: Qiu Jian <qiujian@yunionyun.com>
Date: Tue, 20 Nov 2018 17:35:58 +0800
Subject: [PATCH 55/57] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=EF=BC=9A=E5=89=8D?=
 =?UTF-8?q?=E7=AB=AFcheck=20compute=20servers=20perform=20start=E6=9D=83?=
 =?UTF-8?q?=E9=99=90=E6=8A=A5out=20of=20index=E9=94=99=E8=AF=AF?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/cloudcommon/policy/policy.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pkg/cloudcommon/policy/policy.go b/pkg/cloudcommon/policy/policy.go
index 7517dfa087..56418e3550 100644
--- a/pkg/cloudcommon/policy/policy.go
+++ b/pkg/cloudcommon/policy/policy.go
@@ -233,7 +233,8 @@ func (manager *SPolicyManager) explainPolicy(userCred mcclient.TokenCredential,
 	}
 	if len(policySeq) > 4 {
 		for i := 4; i < len(policySeq); i += 1 {
-			extra[i-4], _ = policySeq[i].GetString()
+			ev, _ := policySeq[i].GetString()
+			extra = append(extra, ev)
 		}
 	}
 

From aece52666d51f45d929aa4f96044bcbcee7b504e Mon Sep 17 00:00:00 2001
From: ningyu <ningyu@yunionyun.com>
Date: Tue, 20 Nov 2018 21:46:36 +0800
Subject: [PATCH 56/57] =?UTF-8?q?refacotr:=20=E9=85=8D=E7=BD=AE=E5=BB=BA?=
 =?UTF-8?q?=E8=AE=AE=EF=BC=8C=E5=A2=9E=E5=8A=A0=E9=98=88=E5=80=BC=E5=8F=82?=
 =?UTF-8?q?=E6=95=B0=E5=92=8C=E5=BD=93=E5=89=8D=E7=9A=84TOP95=E5=80=BC?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/mcclient/modules/mod_underutilizedinstances.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/mcclient/modules/mod_underutilizedinstances.go b/pkg/mcclient/modules/mod_underutilizedinstances.go
index 8c0e6af42f..0fde51c30e 100644
--- a/pkg/mcclient/modules/mod_underutilizedinstances.go
+++ b/pkg/mcclient/modules/mod_underutilizedinstances.go
@@ -6,7 +6,7 @@ var (
 
 func init() {
 	UnderutilizedInstances = NewCloudmonManager("underutilizedinstance", "underutilizedinstances",
-		[]string{"id", "vm_id", "vm_name", "time", "advices", "vm_cpu", "vm_disk", "vm_memory", "vm_provider"},
+		[]string{"id", "vm_id", "vm_name", "datetime_str", "vm_cpu", "vm_disk", "vm_memory", "vm_provider", "cpu_usage_threshold", "netio_rx_bps_threshold", "netio_tx_bps_threshold", "stastics_details"},
 		[]string{})
 
 	register(&UnderutilizedInstances)

From bd415e8da9d8dc76e64ae769a79bbe25bf55fcf0 Mon Sep 17 00:00:00 2001
From: Qiu Jian <qiujian@yunionyun.com>
Date: Wed, 21 Nov 2018 21:21:24 +0800
Subject: [PATCH 57/57] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=EF=BC=9Avmware?=
 =?UTF-8?q?=E4=B8=BB=E6=9C=BA=E8=8E=B7=E5=8F=96=E8=B4=A6=E5=8F=B7=E4=BF=A1?=
 =?UTF-8?q?=E6=81=AF=E5=A4=B1=E8=B4=A5?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pkg/compute/models/cloudproviders.go | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/pkg/compute/models/cloudproviders.go b/pkg/compute/models/cloudproviders.go
index 18724288ec..19359fe19e 100644
--- a/pkg/compute/models/cloudproviders.go
+++ b/pkg/compute/models/cloudproviders.go
@@ -400,7 +400,15 @@ func (self *SCloudprovider) getAccount() (SAccount, error) {
 
 	cloudaccount := self.GetCloudaccount()
 	if cloudaccount == nil {
-		return account, fmt.Errorf("fail to find cloudaccount???")
+		// legacy mode
+		passwd, err := self.getPassword()
+		if err != nil {
+			return account, err
+		}
+		account.Account = self.Account
+		account.AccessUrl = self.AccessUrl
+		account.Secret = passwd
+		return account, nil // fmt.Errorf("fail to find cloudaccount???")
 	}
 
 	passwd, err := cloudaccount.getPassword()