Merge branch 'release/2.2.0' of ssh://git.yunion.io/~quxuan/onecloud into feature/qx-azure

cmd/climc/shell/zones.go

@@ -101,8 +101,8 @@ func init() {
 		return nil
 	})
 
-	R(&ZoneShowOptions{}, "zone-capabilities", "Show zone's capacibilities", func(s *mcclient.ClientSession, args *ZoneShowOptions) error {
-		result, err := modules.Zones.GetSpecific(s, args.ID, "capabilities", nil)
+	R(&ZoneShowOptions{}, "zone-capability", "Show zone's capacibilities", func(s *mcclient.ClientSession, args *ZoneShowOptions) error {
+		result, err := modules.Zones.GetSpecific(s, args.ID, "capability", nil)
 		if err != nil {
 			return err
 		}

pkg/compute/guestdrivers/azure.go

@@ -101,9 +101,7 @@ func (self *SAzureGuestDriver) GetJsonDescAtHost(ctx context.Context, guest *mod
 	for i := 0; i < len(disks); i += 1 {
 		disk := disks[i].GetDisk()
 		if i == 0 {
-			log.Debugf("disk: %v", disk)
 			storage := disk.GetStorage()
-			log.Debugf("disk storage: %v", storage)
 			config.StorageType = storage.StorageType
 			cache := storage.GetStoragecache()
 			imageId := disk.GetTemplateId()
@@ -141,7 +139,7 @@ func (self *SAzureGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gue
 				desc.IpAddr, desc.Description, passwd, desc.StorageType, desc.DataDisks, desc.PublicKey); err != nil {
 				return nil, err
 			} else {
-				log.Debugf("VMcreated %s, wait status ready ...", iVM.GetGlobalId())
+				log.Debugf("VMcreated %s, wait status running ...", iVM.GetGlobalId())
 
 				if iVM, err = ihost.GetIVMById(iVM.GetGlobalId()); err != nil {
 					log.Errorf("cannot find vm %s", err)
@@ -247,7 +245,7 @@ func (self *SAzureGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Context
 			log.Errorf(msg)
 			return fmt.Errorf(msg)
 		}
-		for i := 0; i < len(diskInfo); i += 1 {
+		for i := 0; i < len(diskInfo); i++ {
 			disk := disks[i].GetDisk()
 			_, err = disk.GetModelManager().TableSpec().Update(disk, func() error {
 				disk.DiskSize = diskInfo[i].Size

pkg/compute/hostdrivers/aliyun.go

@@ -6,11 +6,11 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/httperrors"
-	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 )
 
 type SAliyunHostDriver struct {
@@ -37,7 +37,6 @@ func (self *SAliyunHostDriver) CheckAndSetCacheImage(ctx context.Context, host *
 	osType, _ := params.GetString("os_type")
 	osDist, _ := params.GetString("os_distribution")
 
-
 	isForce := jsonutils.QueryBoolean(params, "is_force", false)
 	userCred := task.GetUserCred()
 	taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {

pkg/compute/hostdrivers/azure.go

@@ -8,6 +8,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/httperrors"
 )
@@ -48,10 +49,11 @@ func (self *SAzureHostDriver) CheckAndSetCacheImage(ctx context.Context, host *m
 		}
 
 		extImgId, err := iStorageCache.UploadImage(userCred, imageId, osArch, osType, osDist, scimg.ExternalId, isForce)
-
 		if err != nil {
 			return nil, err
 		} else {
+			scimg.SetExternalId(extImgId)
+
 			ret := jsonutils.NewDict()
 			ret.Add(jsonutils.NewString(extImgId), "image_id")
 			return ret, nil
@@ -103,16 +105,19 @@ func (self *SAzureHostDriver) RequestAllocateDiskOnStorage(ctx context.Context,
 }
 
 func (self *SAzureHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error {
+	data := jsonutils.NewDict()
 	if iCloudStorage, err := storage.GetIStorage(); err != nil {
 		return err
 	} else if iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()); err != nil {
+		if err == cloudprovider.ErrNotFound {
+			task.ScheduleRun(data)
+			return nil
+		}
 		return err
 	} else if err := iDisk.Delete(); err != nil {
 		return err
-	} else {
-		data := jsonutils.NewDict()
-		task.ScheduleRun(data)
 	}
+	task.ScheduleRun(data)
 	return nil
 }
 

pkg/util/azure/disk.go

@@ -127,7 +127,13 @@ func (self *SRegion) GetDisk(diskId string) (*SDisk, error) {
 	computeClient := compute.NewDisksClientWithBaseURI(self.client.baseUrl, self.client.subscriptionId)
 	computeClient.Authorizer = self.client.authorizer
 	_, resourceGroup, diskName := pareResourceGroupWithName(diskId, DISK_RESOURCE)
+	if len(diskId) == 0 {
+		return nil, cloudprovider.ErrNotFound
+	}
 	if _disk, err := computeClient.Get(context.Background(), resourceGroup, diskName); err != nil {
+		if _disk.Response.StatusCode == 404 {
+			return nil, cloudprovider.ErrNotFound
+		}
 		return nil, err
 	} else if err := jsonutils.Update(&disk, _disk); err != nil {
 		return nil, err

pkg/util/azure/host.go

@@ -3,7 +3,6 @@ package azure
 import (
 	"context"
 	"fmt"
-	"strings"
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute"
@@ -17,6 +16,10 @@ type SHost struct {
 	zone *SZone
 }
 
+const (
+	DEFAULT_USER = "yunion"
+)
+
 func (self *SHost) GetMetadata() *jsonutils.JSONDict {
 	return nil
 }
@@ -46,8 +49,17 @@ func (self *SHost) Refresh() error {
 }
 
 func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, networkId string, ipAddr string, desc string, passwd string, storageType string, diskSizes []int, publicKey string) (cloudprovider.ICloudVM, error) {
-	vmId, err := self._createVM(name, imgId, sysDiskSize, cpu, memMB, networkId, ipAddr, desc, passwd, storageType, diskSizes, publicKey)
+	nicId := ""
+	if net := self.zone.getNetworkById(networkId); net == nil {
+		return nil, fmt.Errorf("invalid network ID %s", networkId)
+	} else if nic, err := self.zone.region.CreateNetworkInterface(fmt.Sprintf("%s-ipconfig", name), ipAddr, net.GetId()); err != nil {
+		return nil, err
+	} else {
+		nicId = nic.ID
+	}
+	vmId, err := self._createVM(name, imgId, sysDiskSize, cpu, memMB, nicId, ipAddr, desc, passwd, storageType, diskSizes, publicKey)
 	if err != nil {
+		self.zone.region.DeleteNetworkInterface(nicId)
 		return nil, err
 	}
 	if vm, err := self.zone.region.GetInstance(vmId); err != nil {
@@ -57,15 +69,7 @@ func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int,
 	}
 }
 
-func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, networkId string, ipAddr string, desc string, passwd string, storageType string, diskSizes []int, publicKey string) (string, error) {
-	nicId := ""
-	if net := self.zone.getNetworkById(networkId); net == nil {
-		return "", fmt.Errorf("invalid network ID %s", networkId)
-	} else if nic, err := self.zone.region.CreateNetworkInterface(fmt.Sprintf("%s-ipconfig", name), ipAddr, net.GetId()); err != nil {
-		return "", err
-	} else {
-		nicId = nic.ID
-	}
+func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, nicId string, ipAddr string, desc string, passwd string, storageType string, diskSizes []int, publicKey string) (string, error) {
 	computeClient := compute.NewVirtualMachinesClientWithBaseURI(self.zone.region.client.baseUrl, self.zone.region.client.subscriptionId)
 	computeClient.Authorizer = self.zone.region.client.authorizer
 
@@ -102,7 +106,7 @@ func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int
 		})
 	}
 
-	AdminUsername := "yunion"
+	AdminUsername := DEFAULT_USER
 
 	NetworkInterfaceReferences := []compute.NetworkInterfaceReference{
 		compute.NetworkInterfaceReference{ID: &nicId},
@@ -111,6 +115,14 @@ func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int
 	osType := compute.OperatingSystemTypes(image.GetOsType())
 	DiskSizeGB := int32(sysDiskSize)
 
+	// bootDiagnostics := true
+	// diagnosticsProfile := compute.DiagnosticsProfile{
+	// 	BootDiagnostics: &compute.BootDiagnostics{
+	// 		Enabled: &bootDiagnostics,
+	// 		//StorageURI:
+	// 	},
+	// }
+
 	properties := compute.VirtualMachineProperties{
 		HardwareProfile: &compute.HardwareProfile{},
 		StorageProfile: &compute.StorageProfile{
@@ -127,6 +139,7 @@ func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int
 			},
 			DataDisks: &DataDisks,
 		},
+
 		OsProfile: &compute.OSProfile{
 			ComputerName:  &name,
 			AdminUsername: &AdminUsername,
@@ -146,7 +159,7 @@ func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int
 	}
 
 	params := compute.VirtualMachine{Location: &self.zone.region.Name, Name: &name, VirtualMachineProperties: &properties}
-	log.Debugf("Create instance params: %s", jsonutils.Marshal(params).PrettyString())
+	//log.Debugf("Create instance params: %s", jsonutils.Marshal(params).PrettyString())
 	for _, profile := range self.zone.region.getHardwareProfile(cpu, memMB) {
 		params.HardwareProfile.VMSize = compute.VirtualMachineSizeTypes(profile)
 		log.Debugf("Try HardwareProfile : %s", profile)
@@ -154,21 +167,16 @@ func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int
 		result, err := computeClient.CreateOrUpdate(context.Background(), resourceGroup, instanceName, params)
 		if err != nil {
 			log.Errorf("Failed for %s: %s", profile, err)
-		} else if err := result.WaitForCompletion(context.Background(), computeClient.Client); err != nil {
-			if strings.Index(err.Error(), "OSProvisioningTimedOut") == -1 {
-				return "", err
-			} else if instance, err := self.zone.region.GetInstance(instanceId); err != nil {
-				return "", err
-			} else {
-				return instance.ID, nil
-			}
-		} else if vm, err := result.Result(computeClient); err != nil {
+		} else if _, err := result.Done(computeClient.Client); err != nil {
+			return "", err
+		} else if instance, err := self.zone.region.GetInstance(instanceId); err != nil {
+			return "", err
+		} else if err = cloudprovider.WaitStatus(instance, models.VM_RUNNING, time.Second*5, time.Second*1800); err != nil {
 			return "", err
 		} else {
-			return *vm.ID, nil
+			return instance.ID, nil
 		}
 	}
-	self.zone.region.DeleteNetworkInterface(nicId)
 	return "", fmt.Errorf("Failed to create, specification not supported")
 }
 

pkg/util/azure/image.go

@@ -123,6 +123,9 @@ func (self *SRegion) GetImage(imageId string) (*SImage, error) {
 	image := SImage{}
 	imageClient := compute.NewImagesClientWithBaseURI(self.client.baseUrl, self.SubscriptionID)
 	imageClient.Authorizer = self.client.authorizer
+	if len(imageId) == 0 {
+		return nil, cloudprovider.ErrNotFound
+	}
 	_, resourceGroup, imageName := pareResourceGroupWithName(imageId, IMAGE_RESOURCE)
 	if result, err := imageClient.Get(context.Background(), resourceGroup, imageName, ""); err != nil {
 		if result.Response.StatusCode == 404 {

pkg/util/azure/instance.go

@@ -166,7 +166,8 @@ type SInstance struct {
 }
 
 func (self *SRegion) GetInstance(instanceId string) (*SInstance, error) {
-	instance := SInstance{}
+	zone := self.izones[0].(*SZone)
+	instance := SInstance{host: zone.getHost()}
 	computeClient := compute.NewVirtualMachinesClientWithBaseURI(self.client.baseUrl, self.client.subscriptionId)
 	computeClient.Authorizer = self.client.authorizer
 	_, resourceGroup, instanceName := pareResourceGroupWithName(instanceId, INSTANCE_RESOURCE)
@@ -296,11 +297,14 @@ func (self *SInstance) GetStatus() string {
 				case "stopping":
 					return models.VM_START_STOP
 				default:
+					log.Errorf("Unknow instance status %s", code[1])
 					return models.VM_UNKNOWN
 				}
-
 			}
 		}
+		if statuses.Level == "Error" {
+			log.Errorf("Find error code: [%s] message: %s", statuses.Code, statuses.Message)
+		}
 	}
 	return models.VM_UNKNOWN
 }
@@ -314,13 +318,18 @@ func (self *SInstance) AttachDisk(diskId string) error {
 }
 
 func (region *SRegion) UpdateInstance(instanceId string, params compute.VirtualMachineUpdate) error {
-	computeClient := compute.NewVirtualMachinesClientWithBaseURI(region.client.baseUrl, region.client.subscriptionId)
-	computeClient.Authorizer = region.client.authorizer
-	_, resourceGroup, instanceName := pareResourceGroupWithName(instanceId, INSTANCE_RESOURCE)
-	if result, err := computeClient.Update(context.Background(), resourceGroup, instanceName, params); err != nil {
-		return err
-	} else if err := result.WaitForCompletion(context.Background(), computeClient.Client); err != nil {
+	if instance, err := region.GetInstance(instanceId); err != nil {
 		return err
+	} else {
+		computeClient := compute.NewVirtualMachinesClientWithBaseURI(region.client.baseUrl, region.client.subscriptionId)
+		computeClient.Authorizer = region.client.authorizer
+		_, resourceGroup, instanceName := pareResourceGroupWithName(instanceId, INSTANCE_RESOURCE)
+		if _, err := computeClient.Update(context.Background(), resourceGroup, instanceName, params); err != nil {
+			return err
+		}
+		if err := cloudprovider.WaitStatus(instance, instance.GetStatus(), time.Second*5, time.Second*1800); err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -508,20 +517,16 @@ func (self *SInstance) RebuildRoot(imageId string) error {
 }
 
 func (region *SRegion) RebuildRoot(instanceId string) error {
-	_, resourceGroup, instanceName := pareResourceGroupWithName(instanceId, INSTANCE_RESOURCE)
-	computeClient := compute.NewVirtualMachinesClientWithBaseURI(region.client.baseUrl, region.client.subscriptionId)
-	computeClient.Authorizer = region.client.authorizer
-	if result, err := computeClient.Redeploy(context.Background(), resourceGroup, instanceName); err != nil {
+	if instance, err := region.GetInstance(instanceId); err != nil {
 		return err
-	} else if err := result.WaitForCompletion(context.Background(), computeClient.Client); err != nil {
-		if strings.Index(err.Error(), "OSProvisioningTimedOut") > 0 {
-			if instance, err := region.GetInstance(instanceId); err != nil {
-				return err
-			} else if status := instance.GetStatus(); status == models.VM_RUNNING {
-				region.StopVM(instanceId, true)
-			}
-			return nil
-		} else {
+	} else {
+		_, resourceGroup, instanceName := pareResourceGroupWithName(instanceId, INSTANCE_RESOURCE)
+		computeClient := compute.NewVirtualMachinesClientWithBaseURI(region.client.baseUrl, region.client.subscriptionId)
+		computeClient.Authorizer = region.client.authorizer
+		if _, err := computeClient.Redeploy(context.Background(), resourceGroup, instanceName); err != nil {
+			return err
+		}
+		if err := cloudprovider.WaitStatus(instance, instance.GetStatus(), time.Second*5, time.Second*1800); err != nil {
 			return err
 		}
 	}
@@ -694,13 +699,20 @@ func (self *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) {
 }
 
 func (self *SRegion) StartVM(instanceId string) error {
-	_, resourceGroup, instanceName := pareResourceGroupWithName(instanceId, INSTANCE_RESOURCE)
-	computeClient := compute.NewVirtualMachinesClientWithBaseURI(self.client.baseUrl, self.client.subscriptionId)
-	computeClient.Authorizer = self.client.authorizer
-	if result, err := computeClient.Start(context.Background(), resourceGroup, instanceName); err != nil {
-		return err
-	} else if err := result.WaitForCompletion(context.Background(), computeClient.Client); err != nil {
+	if instance, err := self.GetInstance(instanceId); err != nil {
 		return err
+	} else if status := instance.GetStatus(); status == models.VM_RUNNING {
+		return nil
+	} else {
+		_, resourceGroup, instanceName := pareResourceGroupWithName(instanceId, INSTANCE_RESOURCE)
+		computeClient := compute.NewVirtualMachinesClientWithBaseURI(self.client.baseUrl, self.client.subscriptionId)
+		computeClient.Authorizer = self.client.authorizer
+		if _, err := computeClient.Start(context.Background(), resourceGroup, instanceName); err != nil {
+			return err
+		}
+		if err = cloudprovider.WaitStatus(instance, models.VM_RUNNING, time.Second*5, time.Second*1800); err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -724,18 +736,42 @@ func (self *SRegion) StopVM(instanceId string, isForce bool) error {
 }
 
 func (self *SRegion) doStopVM(instanceId string, isForce bool) error {
-	_, resourceGroup, instanceName := pareResourceGroupWithName(instanceId, INSTANCE_RESOURCE)
-	computeClient := compute.NewVirtualMachinesClientWithBaseURI(self.client.baseUrl, self.client.subscriptionId)
-	computeClient.Authorizer = self.client.authorizer
-	if result, err := computeClient.PowerOff(context.Background(), resourceGroup, instanceName); err != nil {
-		return err
-	} else if err := result.WaitForCompletion(context.Background(), computeClient.Client); err != nil {
+	if instance, err := self.GetInstance(instanceId); err != nil {
 		return err
+	} else {
+		_, resourceGroup, instanceName := pareResourceGroupWithName(instanceId, INSTANCE_RESOURCE)
+		computeClient := compute.NewVirtualMachinesClientWithBaseURI(self.client.baseUrl, self.client.subscriptionId)
+		computeClient.Authorizer = self.client.authorizer
+		if _, err := computeClient.PowerOff(context.Background(), resourceGroup, instanceName); err != nil {
+			return err
+		}
+		if err = cloudprovider.WaitStatus(instance, models.VM_READY, time.Second*5, time.Second*1800); err != nil {
+			return err
+		}
 	}
 	return nil
 }
 
 func (self *SInstance) SyncSecurityGroup(secgroupId string, name string, rules []secrules.SecurityRule) error {
+	nics, err := self.getNics()
+	if err != nil {
+		return err
+	}
+	if len(secgroupId) == 0 {
+		for _, nic := range nics {
+			if err := nic.revokeSecurityGroup(); err != nil {
+				return err
+			}
+		}
+	} else if extId, err := self.host.zone.region.syncSecurityGroup(secgroupId, name, rules); err != nil {
+		return err
+	} else {
+		for _, nic := range nics {
+			if err := nic.assignSecurityGroup(extId); err != nil {
+				return err
+			}
+		}
+	}
 	return nil
 }
 

pkg/util/azure/instancenic.go

@@ -80,6 +80,46 @@ func (self *SInstanceNic) GetDriver() string {
 	return "virtio"
 }
 
+func (self *SInstanceNic) updateSecurityGroup(secgroupId string) error {
+	region := self.instance.host.zone.region
+	nicClient := network.NewInterfacesClientWithBaseURI(region.client.baseUrl, region.SubscriptionID)
+	nicClient.Authorizer = region.client.authorizer
+	_, resourceGroup, nicName := pareResourceGroupWithName(self.ID, NIC_RESOURCE)
+	iPConfigurations := []network.InterfaceIPConfiguration{
+		network.InterfaceIPConfiguration{
+			Name: &nicName,
+			ID:   &self.ID,
+			InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{
+				Subnet: &network.Subnet{ID: &self.Properties.IPConfigurations[0].Properties.Subnet.ID},
+			},
+		},
+	}
+	params := network.Interface{
+		Location: &region.Name,
+		InterfacePropertiesFormat: &network.InterfacePropertiesFormat{
+			IPConfigurations:     &iPConfigurations,
+			NetworkSecurityGroup: &network.SecurityGroup{},
+		},
+	}
+	if len(secgroupId) > 0 {
+		params.InterfacePropertiesFormat.NetworkSecurityGroup.ID = &secgroupId
+	}
+	if result, err := nicClient.CreateOrUpdate(context.Background(), resourceGroup, nicName, params); err != nil {
+		return err
+	} else if err := result.WaitForCompletion(context.Background(), nicClient.Client); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (self *SInstanceNic) revokeSecurityGroup() error {
+	return self.updateSecurityGroup("")
+}
+
+func (self *SInstanceNic) assignSecurityGroup(secgroupId string) error {
+	return self.updateSecurityGroup(secgroupId)
+}
+
 func (self *SInstanceNic) GetINetwork() cloudprovider.ICloudNetwork {
 	if wires, err := self.instance.host.GetIWires(); err != nil {
 		log.Errorf("GetINetwork error: %v", err)

pkg/util/azure/securitygroup.go

@@ -3,10 +3,14 @@ package azure
 import (
 	"context"
 	"fmt"
+	"sort"
 	"strings"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/pkg/util/secrules"
+	"yunion.io/x/pkg/utils"
 
 	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network"
 )
@@ -50,6 +54,25 @@ type SecurityRules struct {
 	ID         string
 }
 
+type SecurityRulesSet []SecurityRules
+
+func (v SecurityRulesSet) Len() int {
+	return len(v)
+}
+
+func (v SecurityRulesSet) Swap(i, j int) {
+	v[i], v[j] = v[j], v[i]
+}
+
+func (v SecurityRulesSet) Less(i, j int) bool {
+	if v[i].Properties.Priority < v[j].Properties.Priority {
+		return true
+	} else if v[i].Properties.Priority == v[j].Properties.Priority {
+		return strings.Compare(v[i].Properties.String(), v[j].Properties.String()) <= 0
+	}
+	return false
+}
+
 type Interface struct {
 }
 
@@ -66,6 +89,7 @@ type SSecurityGroup struct {
 	ID         string
 	Name       string
 	Location   string
+	Tags       map[string]string
 }
 
 func (self *SecurityRulePropertiesFormat) String() string {
@@ -117,7 +141,6 @@ func (self *SecurityRulePropertiesFormat) String() string {
 			result += fmt.Sprintf(" %s", port)
 		}
 	}
-	//log.Debugf("result: %s", result)
 	return result
 }
 
@@ -130,7 +153,8 @@ func (self *SSecurityGroup) GetMetadata() *jsonutils.JSONDict {
 }
 
 func (self *SSecurityGroup) GetGlobalId() string {
-	return fmt.Sprintf("%s/%s", self.vpc.GetGlobalId(), self.Name)
+	globalId, _, _ := pareResourceGroupWithName(self.ID, SECGRP_RESOURCE)
+	return globalId
 }
 
 func (self *SSecurityGroup) GetDescription() string {
@@ -138,21 +162,22 @@ func (self *SSecurityGroup) GetDescription() string {
 }
 
 func (self *SSecurityGroup) GetName() string {
-	return self.Name
+	return strings.TrimPrefix(self.Name, self.vpc.region.Name+"-")
 }
 
 func (self *SSecurityGroup) GetRules() ([]secrules.SecurityRule, error) {
 	rules := make([]secrules.SecurityRule, 0)
+	sort.Sort(SecurityRulesSet(self.Properties.SecurityRules))
+	priority := 100
+
 	for _, _rule := range self.Properties.SecurityRules {
 		for _, ruleString := range strings.Split(_rule.Properties.String(), ";") {
 			if rule, err := secrules.ParseSecurityRule(ruleString); err != nil {
 				return rules, err
 			} else {
-				rule.Priority = 100 - int(_rule.Properties.Priority)
+				rule.Priority = priority
+				priority--
 				rule.Description = _rule.Properties.Description
-				if rule.Priority < 0 {
-					rule.Priority = 1
-				}
 				rules = append(rules, *rule)
 			}
 		}
@@ -168,14 +193,242 @@ func (self *SSecurityGroup) IsEmulated() bool {
 	return false
 }
 
+func (region *SRegion) CreateSecurityGroup(secName string) (*SSecurityGroup, error) {
+	securityName := fmt.Sprintf("%s-%s", region.Name, secName)
+	globalId, resourceGroup, securityName := pareResourceGroupWithName(securityName, SECGRP_RESOURCE)
+	secClient := network.NewSecurityGroupsClientWithBaseURI(region.client.baseUrl, region.SubscriptionID)
+	secClient.Authorizer = region.client.authorizer
+	params := network.SecurityGroup{
+		Location: &region.Name,
+		Name:     &securityName,
+	}
+	if result, err := secClient.CreateOrUpdate(context.Background(), resourceGroup, securityName, params); err != nil {
+		return nil, err
+	} else if result.WaitForCompletion(context.Background(), secClient.Client); err != nil {
+		return nil, err
+	}
+	return region.GetSecurityGroupDetails(globalId)
+}
+
+func (region *SRegion) GetSecurityGroups() ([]SSecurityGroup, error) {
+	secgroups := make([]SSecurityGroup, 0)
+	secClient := network.NewSecurityGroupsClientWithBaseURI(region.client.baseUrl, region.SubscriptionID)
+	secClient.Authorizer = region.client.authorizer
+	secgroupIds := make([]string, 0)
+	if result, err := secClient.ListAll(context.Background()); err != nil {
+		return nil, err
+	} else {
+		for _, secgrp := range result.Values() {
+			if *secgrp.Location == region.Name {
+				securityGroup := SSecurityGroup{}
+				if secgroupId, ok := secgrp.Tags["id"]; ok {
+					if utils.IsInStringArray(*secgroupId, secgroupIds) {
+						continue
+					} else {
+						secgroupIds = append(secgroupIds, *secgroupId)
+					}
+				}
+				if err := jsonutils.Update(&securityGroup, secgrp); err != nil {
+					return nil, err
+				}
+				securityGroup.Name = strings.TrimPrefix(securityGroup.Name, region.Name+"-")
+				secgroups = append(secgroups, securityGroup)
+			}
+		}
+	}
+	return secgroups, nil
+}
+
+func (region *SRegion) GetSecurityGroupDetails(secgroupId string) (*SSecurityGroup, error) {
+	sec := SSecurityGroup{}
+	secClient := network.NewSecurityGroupsClientWithBaseURI(region.client.baseUrl, region.SubscriptionID)
+	secClient.Authorizer = region.client.authorizer
+	_, resourceGroup, secName := pareResourceGroupWithName(secgroupId, SECGRP_RESOURCE)
+	if len(secName) == 0 {
+		return nil, cloudprovider.ErrNotFound
+	}
+	if result, err := secClient.Get(context.Background(), resourceGroup, secName, ""); err != nil {
+		if result.Response.StatusCode == 404 {
+			return nil, cloudprovider.ErrNotFound
+		}
+		return nil, err
+	} else if err := jsonutils.Update(&sec, result); err != nil {
+		return nil, err
+	}
+	return &sec, nil
+}
+
 func (self *SSecurityGroup) Refresh() error {
-	_, resourceGroup, secgrpName := pareResourceGroupWithName(self.ID, SECGRP_RESOURCE)
-	networkClient := network.NewSecurityGroupsClientWithBaseURI(self.vpc.region.client.baseUrl, self.vpc.region.SubscriptionID)
-	networkClient.Authorizer = self.vpc.region.client.authorizer
-	if secgrp, err := networkClient.Get(context.Background(), resourceGroup, secgrpName, ""); err != nil {
+	if sec, err := self.vpc.region.GetSecurityGroupDetails(self.ID); err != nil {
 		return err
-	} else if err := jsonutils.Update(self, secgrp); err != nil {
+	} else if err := jsonutils.Update(self, sec); err != nil {
 		return err
 	}
 	return nil
 }
+
+func (region *SRegion) addTagToSecurityGroup(secgroupId, value string) error {
+	secClient := network.NewSecurityGroupsClientWithBaseURI(region.client.baseUrl, region.SubscriptionID)
+	secClient.Authorizer = region.client.authorizer
+	_, resourceGroup, secName := pareResourceGroupWithName(secgroupId, SECGRP_RESOURCE)
+	params := network.TagsObject{Tags: map[string]*string{"id": &value}}
+	if result, err := secClient.UpdateTags(context.Background(), resourceGroup, secName, params); err != nil {
+		return err
+	} else if result.WaitForCompletion(context.Background(), secClient.Client); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (region *SRegion) checkSecurityGroup(name, secgroupId string) (*SSecurityGroup, error) {
+	secName := fmt.Sprintf("%s-%s", region.Name, name)
+	globalId, _, _ := pareResourceGroupWithName(secName, SECGRP_RESOURCE)
+	if _, err := region.GetSecurityGroupDetails(globalId); err != nil {
+		if err == cloudprovider.ErrNotFound {
+			if _, err := region.CreateSecurityGroup(name); err != nil {
+				return nil, err
+			} else if err := region.addTagToSecurityGroup(globalId, secgroupId); err != nil {
+				return nil, err
+			}
+		}
+	} else if err := region.addTagToSecurityGroup(globalId, secgroupId); err != nil {
+		return nil, err
+	}
+	return region.GetSecurityGroupDetails(globalId)
+}
+
+func convertRulePort(rule secrules.SecurityRule) *[]string {
+	ports := []string{}
+	for i := 0; i < len(rule.Ports); i++ {
+		ports = append(ports, fmt.Sprintf("%d", rule.Ports[i]))
+	}
+	if rule.PortStart > 0 && rule.PortEnd < 65535 {
+		ports = append(ports, fmt.Sprintf("%d-%d", rule.PortStart, rule.PortEnd))
+	}
+	return &ports
+}
+
+func convertSecurityGroupRule(rule secrules.SecurityRule) *network.SecurityRule {
+	name := strings.Replace(rule.String(), ":", "_", -1)
+	name = strings.Replace(name, " ", "_", -1)
+	name = strings.Replace(name, "-", "_", -1)
+	destRule := network.SecurityRule{
+		Name: &name,
+		SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{},
+	}
+	protocol := network.SecurityRuleProtocolAsterisk
+	if len(rule.Protocol) == 0 || rule.Protocol == secrules.PROTO_ANY {
+		protocol = network.SecurityRuleProtocolAsterisk
+	} else if rule.Protocol == secrules.PROTO_TCP {
+		protocol = network.SecurityRuleProtocolTCP
+	} else if rule.Protocol == secrules.PROTO_UDP {
+		protocol = network.SecurityRuleProtocolUDP
+	} else {
+		return nil
+	}
+	destRule.SecurityRulePropertiesFormat.Protocol = protocol
+	destRule.SecurityRulePropertiesFormat.Description = &rule.Description
+	direction := network.SecurityRuleDirectionInbound
+	if rule.Direction == secrules.SecurityRuleEgress {
+		direction = network.SecurityRuleDirectionOutbound
+	}
+	destRule.SecurityRulePropertiesFormat.Direction = direction
+	ipAddr := rule.IPNet.String()
+	ports := convertRulePort(rule)
+	if len(*ports) == 0 {
+		port := "*"
+		destRule.SecurityRulePropertiesFormat.SourcePortRange = &port
+		destRule.SecurityRulePropertiesFormat.DestinationPortRange = &port
+	} else {
+		destRule.SecurityRulePropertiesFormat.SourcePortRanges = ports
+		destRule.SecurityRulePropertiesFormat.DestinationPortRanges = ports
+	}
+	destRule.SecurityRulePropertiesFormat.DestinationAddressPrefix = &ipAddr
+	destRule.SecurityRulePropertiesFormat.SourceAddressPrefix = &ipAddr
+
+	access := network.SecurityRuleAccessAllow
+	if rule.Action == secrules.SecurityRuleDeny {
+		access = network.SecurityRuleAccessDeny
+	}
+	destRule.SecurityRulePropertiesFormat.Access = access
+	priority := int32(rule.Priority)
+	destRule.SecurityRulePropertiesFormat.Priority = &priority
+	return &destRule
+}
+
+func (region *SRegion) updateSecurityGroupRules(secgroupId string, rules []secrules.SecurityRule) (string, error) {
+	_, resourceGroup, secName := pareResourceGroupWithName(secgroupId, SECGRP_RESOURCE)
+	secClient := network.NewSecurityGroupsClientWithBaseURI(region.client.baseUrl, region.SubscriptionID)
+	secClient.Authorizer = region.client.authorizer
+	securityRules := []network.SecurityRule{}
+	priority := int32(100)
+	for i := 0; i < len(rules); i++ {
+		rules[i].Priority = int(priority)
+		if rule := convertSecurityGroupRule(rules[i]); rule != nil {
+			securityRules = append(securityRules, *rule)
+			priority++
+		}
+	}
+
+	params := network.SecurityGroup{
+		Location: &region.Name,
+		SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{
+			SecurityRules: &securityRules,
+		},
+	}
+	if result, err := secClient.CreateOrUpdate(context.Background(), resourceGroup, secName, params); err != nil {
+		return "", err
+	} else if err := result.WaitForCompletion(context.Background(), secClient.Client); err != nil {
+		return "", err
+	}
+	return secgroupId, nil
+}
+
+func (self *SRegion) syncSecgroupRules(secgroupId string, rules []secrules.SecurityRule) (string, error) {
+	if secgroup, err := self.GetSecurityGroupDetails(secgroupId); err != nil {
+		return "", err
+	} else {
+		sort.Sort(secrules.SecurityRuleSet(rules))
+		sort.Sort(SecurityRulesSet(secgroup.Properties.SecurityRules))
+
+		newRules := []secrules.SecurityRule{}
+
+		i, j := 0, 0
+		for i < len(rules) || j < len(secgroup.Properties.SecurityRules) {
+			if i < len(rules) && j < len(secgroup.Properties.SecurityRules) {
+				srcRule := secgroup.Properties.SecurityRules[j].Properties.String()
+				destRule := rules[i].String()
+				cmp := strings.Compare(srcRule, destRule)
+				if cmp == 0 {
+					i++
+					j++
+				} else if cmp > 0 {
+					// remove srcRule
+					j++
+				} else {
+					// add destRule
+					newRules = append(newRules, rules[i])
+					i++
+				}
+			} else if i >= len(rules) {
+				// del other rules
+				j++
+			} else if j >= len(secgroup.Properties.SecurityRules) {
+				// add rule
+				newRules = append(newRules, rules[i])
+				i++
+			}
+		}
+		return self.updateSecurityGroupRules(secgroup.ID, newRules)
+	}
+}
+
+func (self *SRegion) syncSecurityGroup(secgroupId, name string, rules []secrules.SecurityRule) (string, error) {
+	if secgroup, err := self.checkSecurityGroup(name, secgroupId); err != nil {
+		log.Errorf("check err: %v", err)
+		return "", err
+	} else {
+
+		return self.syncSecgroupRules(secgroup.ID, rules)
+	}
+}

pkg/util/azure/shell/secgroup.go

@@ -0,0 +1,47 @@
+package shell
+
+import (
+	"yunion.io/x/onecloud/pkg/util/azure"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	type SecurityGroupListOptions struct {
+		Limit  int `help:"page size"`
+		Offset int `help:"page offset"`
+	}
+	shellutils.R(&SecurityGroupListOptions{}, "security-group-list", "List security group", func(cli *azure.SRegion, args *SecurityGroupListOptions) error {
+		if secgrps, err := cli.GetSecurityGroups(); err != nil {
+			return err
+		} else {
+			printList(secgrps, len(secgrps), args.Offset, args.Limit, []string{})
+			return nil
+		}
+	})
+
+	type SecurityGroupShowOptions struct {
+		ID string `help:"ID or name of security group"`
+	}
+	shellutils.R(&SecurityGroupShowOptions{}, "security-group-show", "Show details of a security group", func(cli *azure.SRegion, args *SecurityGroupShowOptions) error {
+		if secgrp, err := cli.GetSecurityGroupDetails(args.ID); err != nil {
+			return err
+		} else {
+			printObject(secgrp)
+			return nil
+		}
+	})
+
+	type SecurityGroupCreateOptions struct {
+		NAME string `help:"Security Group name"`
+	}
+
+	shellutils.R(&SecurityGroupCreateOptions{}, "security-group-create", "Create security group", func(cli *azure.SRegion, args *SecurityGroupCreateOptions) error {
+		if secgrp, err := cli.CreateSecurityGroup(args.NAME); err != nil {
+			return err
+		} else {
+			printObject(secgrp)
+			return nil
+		}
+	})
+
+}

pkg/util/azure/storagecache.go

@@ -101,7 +101,24 @@ func (self *SStoragecache) UploadImage(userCred mcclient.TokenCredential, imageI
 	return self.uploadImage(userCred, imageId, osArch, osType, osDist, isForce)
 }
 
-func (self *SRegion) CreateStorageAccount(resourceGroup, storageAccount string) error {
+func (self *SRegion) checkBootDiagnosticStorageAccount() (string, error) {
+	storageAccount := fmt.Sprintf("%s-boot", self.Name)
+	resourceGroup := defaultResourceGroups[STORAGE_RESOURCE]
+	storageClinet := storageaccount.NewAccountsClientWithBaseURI(self.client.baseUrl, self.SubscriptionID)
+	storageClinet.Authorizer = self.client.authorizer
+	if result, err := storageClinet.ListByResourceGroup(context.Background(), resourceGroup); err != nil {
+		return "", err
+	} else {
+		for _, _storage := range *result.Value {
+			if *_storage.Name == storageAccount {
+				return *_storage.ID, nil
+			}
+		}
+		return self.CreateStorageAccount(resourceGroup, storageAccount)
+	}
+}
+
+func (self *SRegion) CreateStorageAccount(resourceGroup, storageAccount string) (string, error) {
 	storageClinet := storageaccount.NewAccountsClientWithBaseURI(self.client.baseUrl, self.SubscriptionID)
 	storageClinet.Authorizer = self.client.authorizer
 	sku := storageaccount.Sku{Name: storageaccount.SkuName("Standard_GRS")}
@@ -113,11 +130,11 @@ func (self *SRegion) CreateStorageAccount(resourceGroup, storageAccount string)
 		storageAccount = fmt.Sprintf("%s%s", self.Name, DefaultStorageAccount)
 	}
 	if result, err := storageClinet.Create(context.Background(), resourceGroup, storageAccount, params); err != nil {
-		return err
+		return "", err
 	} else if err := result.WaitForCompletion(context.Background(), storageClinet.Client); err != nil {
-		return err
+		return "", err
 	}
-	return nil
+	return self.getStorageAccountId(resourceGroup, storageAccount)
 }
 
 func (self *SRegion) checkStorageContainer(storageAccount, accessKey, containerName string) error {
@@ -133,10 +150,18 @@ func (self *SRegion) checkStorageContainer(storageAccount, accessKey, containerN
 	return nil
 }
 
-func (self *SRegion) isStorageAccountExist(resourceGroup, storageAccount string) bool {
+func (self *SRegion) getStorageAccountId(resourceGroup, storageAccount string) (string, error) {
 	storageClinet := storageaccount.NewAccountsClientWithBaseURI(self.client.baseUrl, self.SubscriptionID)
 	storageClinet.Authorizer = self.client.authorizer
-	if _, err := storageClinet.GetProperties(context.Background(), resourceGroup, storageAccount); err != nil {
+	if _storage, err := storageClinet.GetProperties(context.Background(), resourceGroup, storageAccount); err != nil {
+		return "", err
+	} else {
+		return *_storage.ID, nil
+	}
+}
+
+func (self *SRegion) isStorageAccountExist(resourceGroup, storageAccount string) bool {
+	if _, err := self.getStorageAccountId(resourceGroup, storageAccount); err != nil {
 		return false
 	}
 	return true
@@ -171,7 +196,7 @@ func (self *SRegion) CheckBlobContainer(resourceGroup, storageAccount, blobName
 	if err := self.client.fetchAzueResourceGroup(); err != nil {
 		return err
 	} else if !self.isStorageAccountExist(resourceGroup, storageAccount) {
-		if err := self.CreateStorageAccount(resourceGroup, storageAccount); err != nil {
+		if _, err := self.CreateStorageAccount(resourceGroup, storageAccount); err != nil {
 			return err
 		}
 	}
@@ -306,7 +331,7 @@ func (self *SRegion) uploadContainerFileByPath(storageAccount, accessKey, contai
 			BlobServiceClient:     blobServiceClient,
 			ContainerName:         containerName,
 			BlobName:              blobName,
-			Parallelism:           4,
+			Parallelism:           3,
 			Resume:                false,
 			MD5Hash:               []byte(""), //localMetaData.FileMetaData.MD5Hash,
 		}

pkg/util/azure/vpc.go

@@ -91,24 +91,27 @@ func (self *SVpc) Delete() error {
 	return nil
 }
 
+func (self *SVpc) getSecurityGroups() ([]SSecurityGroup, error) {
+	if securityGroups, err := self.region.GetSecurityGroups(); err != nil {
+		return nil, err
+	} else {
+		for i := 0; i < len(securityGroups); i++ {
+			securityGroups[i].vpc = self
+		}
+		return securityGroups, nil
+	}
+}
+
 func (self *SVpc) fetchSecurityGroups() error {
 	self.secgroups = make([]cloudprovider.ICloudSecurityGroup, 0)
-	networkClient := network.NewSecurityGroupsClientWithBaseURI(self.region.client.baseUrl, self.region.SubscriptionID)
-	networkClient.Authorizer = self.region.client.authorizer
-	if secgrpList, err := networkClient.ListAll(context.Background()); err != nil {
+	if secgrps, err := self.getSecurityGroups(); err != nil {
 		return err
 	} else {
-		for _, secgrp := range secgrpList.Values() {
-			securityGroup := SSecurityGroup{vpc: self}
-			if *secgrp.Location == self.Location {
-				if err := jsonutils.Update(&securityGroup, secgrp); err != nil {
-					return err
-				}
-				self.secgroups = append(self.secgroups, &securityGroup)
-			}
+		for i := 0; i < len(secgrps); i++ {
+			self.secgroups = append(self.secgroups, &secgrps[i])
 		}
+		return nil
 	}
-	return nil
 }
 
 func (self *SVpc) getWire() *SWire {

pkg/util/azure/wire.go

@@ -145,7 +145,6 @@ func (self *SWire) getNetworkById(networkId string) *SNetwork {
 		for i := 0; i < len(networks); i++ {
 			network := networks[i].(*SNetwork)
 			_globalId, _, _ := pareResourceGroupWithName(network.ID, VPC_RESOURCE)
-			log.Errorf("Compaire %s => %s", globalId, _globalId)
 			if globalId == _globalId {
 				return network
 			}