diff --git a/cmd/climc/shell/zones.go b/cmd/climc/shell/zones.go index 4ca0b2d254..2ac7c8dd69 100644 --- a/cmd/climc/shell/zones.go +++ b/cmd/climc/shell/zones.go @@ -101,8 +101,8 @@ func init() { return nil }) - R(&ZoneShowOptions{}, "zone-capabilities", "Show zone's capacibilities", func(s *mcclient.ClientSession, args *ZoneShowOptions) error { - result, err := modules.Zones.GetSpecific(s, args.ID, "capabilities", nil) + R(&ZoneShowOptions{}, "zone-capability", "Show zone's capacibilities", func(s *mcclient.ClientSession, args *ZoneShowOptions) error { + result, err := modules.Zones.GetSpecific(s, args.ID, "capability", nil) if err != nil { return err } diff --git a/pkg/compute/guestdrivers/azure.go b/pkg/compute/guestdrivers/azure.go index 02f594819c..102e166041 100644 --- a/pkg/compute/guestdrivers/azure.go +++ b/pkg/compute/guestdrivers/azure.go @@ -101,9 +101,7 @@ func (self *SAzureGuestDriver) GetJsonDescAtHost(ctx context.Context, guest *mod for i := 0; i < len(disks); i += 1 { disk := disks[i].GetDisk() if i == 0 { - log.Debugf("disk: %v", disk) storage := disk.GetStorage() - log.Debugf("disk storage: %v", storage) config.StorageType = storage.StorageType cache := storage.GetStoragecache() imageId := disk.GetTemplateId() @@ -141,7 +139,7 @@ func (self *SAzureGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gue desc.IpAddr, desc.Description, passwd, desc.StorageType, desc.DataDisks, desc.PublicKey); err != nil { return nil, err } else { - log.Debugf("VMcreated %s, wait status ready ...", iVM.GetGlobalId()) + log.Debugf("VMcreated %s, wait status running ...", iVM.GetGlobalId()) if iVM, err = ihost.GetIVMById(iVM.GetGlobalId()); err != nil { log.Errorf("cannot find vm %s", err) @@ -247,7 +245,7 @@ func (self *SAzureGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Context log.Errorf(msg) return fmt.Errorf(msg) } - for i := 0; i < len(diskInfo); i += 1 { + for i := 0; i < len(diskInfo); i++ { disk := disks[i].GetDisk() _, err = disk.GetModelManager().TableSpec().Update(disk, func() error { disk.DiskSize = diskInfo[i].Size diff --git a/pkg/compute/hostdrivers/aliyun.go b/pkg/compute/hostdrivers/aliyun.go index 2a7ad29a72..c9f4bedcbd 100644 --- a/pkg/compute/hostdrivers/aliyun.go +++ b/pkg/compute/hostdrivers/aliyun.go @@ -6,11 +6,11 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/httperrors" - "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" ) type SAliyunHostDriver struct { @@ -37,7 +37,6 @@ func (self *SAliyunHostDriver) CheckAndSetCacheImage(ctx context.Context, host * osType, _ := params.GetString("os_type") osDist, _ := params.GetString("os_distribution") - isForce := jsonutils.QueryBoolean(params, "is_force", false) userCred := task.GetUserCred() taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { diff --git a/pkg/compute/hostdrivers/azure.go b/pkg/compute/hostdrivers/azure.go index 4990830042..15fa2dece8 100644 --- a/pkg/compute/hostdrivers/azure.go +++ b/pkg/compute/hostdrivers/azure.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/httperrors" ) @@ -48,10 +49,11 @@ func (self *SAzureHostDriver) CheckAndSetCacheImage(ctx context.Context, host *m } extImgId, err := iStorageCache.UploadImage(userCred, imageId, osArch, osType, osDist, scimg.ExternalId, isForce) - if err != nil { return nil, err } else { + scimg.SetExternalId(extImgId) + ret := jsonutils.NewDict() ret.Add(jsonutils.NewString(extImgId), "image_id") return ret, nil @@ -103,16 +105,19 @@ func (self *SAzureHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, } func (self *SAzureHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error { + data := jsonutils.NewDict() if iCloudStorage, err := storage.GetIStorage(); err != nil { return err } else if iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()); err != nil { + if err == cloudprovider.ErrNotFound { + task.ScheduleRun(data) + return nil + } return err } else if err := iDisk.Delete(); err != nil { return err - } else { - data := jsonutils.NewDict() - task.ScheduleRun(data) } + task.ScheduleRun(data) return nil } diff --git a/pkg/util/azure/disk.go b/pkg/util/azure/disk.go index ebee8edfcc..085945ca00 100644 --- a/pkg/util/azure/disk.go +++ b/pkg/util/azure/disk.go @@ -127,7 +127,13 @@ func (self *SRegion) GetDisk(diskId string) (*SDisk, error) { computeClient := compute.NewDisksClientWithBaseURI(self.client.baseUrl, self.client.subscriptionId) computeClient.Authorizer = self.client.authorizer _, resourceGroup, diskName := pareResourceGroupWithName(diskId, DISK_RESOURCE) + if len(diskId) == 0 { + return nil, cloudprovider.ErrNotFound + } if _disk, err := computeClient.Get(context.Background(), resourceGroup, diskName); err != nil { + if _disk.Response.StatusCode == 404 { + return nil, cloudprovider.ErrNotFound + } return nil, err } else if err := jsonutils.Update(&disk, _disk); err != nil { return nil, err diff --git a/pkg/util/azure/host.go b/pkg/util/azure/host.go index 4390b516d3..e36b818c64 100644 --- a/pkg/util/azure/host.go +++ b/pkg/util/azure/host.go @@ -3,7 +3,6 @@ package azure import ( "context" "fmt" - "strings" "time" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute" @@ -17,6 +16,10 @@ type SHost struct { zone *SZone } +const ( + DEFAULT_USER = "yunion" +) + func (self *SHost) GetMetadata() *jsonutils.JSONDict { return nil } @@ -46,8 +49,17 @@ func (self *SHost) Refresh() error { } func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, networkId string, ipAddr string, desc string, passwd string, storageType string, diskSizes []int, publicKey string) (cloudprovider.ICloudVM, error) { - vmId, err := self._createVM(name, imgId, sysDiskSize, cpu, memMB, networkId, ipAddr, desc, passwd, storageType, diskSizes, publicKey) + nicId := "" + if net := self.zone.getNetworkById(networkId); net == nil { + return nil, fmt.Errorf("invalid network ID %s", networkId) + } else if nic, err := self.zone.region.CreateNetworkInterface(fmt.Sprintf("%s-ipconfig", name), ipAddr, net.GetId()); err != nil { + return nil, err + } else { + nicId = nic.ID + } + vmId, err := self._createVM(name, imgId, sysDiskSize, cpu, memMB, nicId, ipAddr, desc, passwd, storageType, diskSizes, publicKey) if err != nil { + self.zone.region.DeleteNetworkInterface(nicId) return nil, err } if vm, err := self.zone.region.GetInstance(vmId); err != nil { @@ -57,15 +69,7 @@ func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, } } -func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, networkId string, ipAddr string, desc string, passwd string, storageType string, diskSizes []int, publicKey string) (string, error) { - nicId := "" - if net := self.zone.getNetworkById(networkId); net == nil { - return "", fmt.Errorf("invalid network ID %s", networkId) - } else if nic, err := self.zone.region.CreateNetworkInterface(fmt.Sprintf("%s-ipconfig", name), ipAddr, net.GetId()); err != nil { - return "", err - } else { - nicId = nic.ID - } +func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, nicId string, ipAddr string, desc string, passwd string, storageType string, diskSizes []int, publicKey string) (string, error) { computeClient := compute.NewVirtualMachinesClientWithBaseURI(self.zone.region.client.baseUrl, self.zone.region.client.subscriptionId) computeClient.Authorizer = self.zone.region.client.authorizer @@ -102,7 +106,7 @@ func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int }) } - AdminUsername := "yunion" + AdminUsername := DEFAULT_USER NetworkInterfaceReferences := []compute.NetworkInterfaceReference{ compute.NetworkInterfaceReference{ID: &nicId}, @@ -111,6 +115,14 @@ func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int osType := compute.OperatingSystemTypes(image.GetOsType()) DiskSizeGB := int32(sysDiskSize) + // bootDiagnostics := true + // diagnosticsProfile := compute.DiagnosticsProfile{ + // BootDiagnostics: &compute.BootDiagnostics{ + // Enabled: &bootDiagnostics, + // //StorageURI: + // }, + // } + properties := compute.VirtualMachineProperties{ HardwareProfile: &compute.HardwareProfile{}, StorageProfile: &compute.StorageProfile{ @@ -127,6 +139,7 @@ func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int }, DataDisks: &DataDisks, }, + OsProfile: &compute.OSProfile{ ComputerName: &name, AdminUsername: &AdminUsername, @@ -146,7 +159,7 @@ func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int } params := compute.VirtualMachine{Location: &self.zone.region.Name, Name: &name, VirtualMachineProperties: &properties} - log.Debugf("Create instance params: %s", jsonutils.Marshal(params).PrettyString()) + //log.Debugf("Create instance params: %s", jsonutils.Marshal(params).PrettyString()) for _, profile := range self.zone.region.getHardwareProfile(cpu, memMB) { params.HardwareProfile.VMSize = compute.VirtualMachineSizeTypes(profile) log.Debugf("Try HardwareProfile : %s", profile) @@ -154,21 +167,16 @@ func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int result, err := computeClient.CreateOrUpdate(context.Background(), resourceGroup, instanceName, params) if err != nil { log.Errorf("Failed for %s: %s", profile, err) - } else if err := result.WaitForCompletion(context.Background(), computeClient.Client); err != nil { - if strings.Index(err.Error(), "OSProvisioningTimedOut") == -1 { - return "", err - } else if instance, err := self.zone.region.GetInstance(instanceId); err != nil { - return "", err - } else { - return instance.ID, nil - } - } else if vm, err := result.Result(computeClient); err != nil { + } else if _, err := result.Done(computeClient.Client); err != nil { + return "", err + } else if instance, err := self.zone.region.GetInstance(instanceId); err != nil { + return "", err + } else if err = cloudprovider.WaitStatus(instance, models.VM_RUNNING, time.Second*5, time.Second*1800); err != nil { return "", err } else { - return *vm.ID, nil + return instance.ID, nil } } - self.zone.region.DeleteNetworkInterface(nicId) return "", fmt.Errorf("Failed to create, specification not supported") } diff --git a/pkg/util/azure/image.go b/pkg/util/azure/image.go index ee49d984b6..153d2bc877 100644 --- a/pkg/util/azure/image.go +++ b/pkg/util/azure/image.go @@ -123,6 +123,9 @@ func (self *SRegion) GetImage(imageId string) (*SImage, error) { image := SImage{} imageClient := compute.NewImagesClientWithBaseURI(self.client.baseUrl, self.SubscriptionID) imageClient.Authorizer = self.client.authorizer + if len(imageId) == 0 { + return nil, cloudprovider.ErrNotFound + } _, resourceGroup, imageName := pareResourceGroupWithName(imageId, IMAGE_RESOURCE) if result, err := imageClient.Get(context.Background(), resourceGroup, imageName, ""); err != nil { if result.Response.StatusCode == 404 { diff --git a/pkg/util/azure/instance.go b/pkg/util/azure/instance.go index 952be055bb..4ece64b3db 100644 --- a/pkg/util/azure/instance.go +++ b/pkg/util/azure/instance.go @@ -166,7 +166,8 @@ type SInstance struct { } func (self *SRegion) GetInstance(instanceId string) (*SInstance, error) { - instance := SInstance{} + zone := self.izones[0].(*SZone) + instance := SInstance{host: zone.getHost()} computeClient := compute.NewVirtualMachinesClientWithBaseURI(self.client.baseUrl, self.client.subscriptionId) computeClient.Authorizer = self.client.authorizer _, resourceGroup, instanceName := pareResourceGroupWithName(instanceId, INSTANCE_RESOURCE) @@ -296,11 +297,14 @@ func (self *SInstance) GetStatus() string { case "stopping": return models.VM_START_STOP default: + log.Errorf("Unknow instance status %s", code[1]) return models.VM_UNKNOWN } - } } + if statuses.Level == "Error" { + log.Errorf("Find error code: [%s] message: %s", statuses.Code, statuses.Message) + } } return models.VM_UNKNOWN } @@ -314,13 +318,18 @@ func (self *SInstance) AttachDisk(diskId string) error { } func (region *SRegion) UpdateInstance(instanceId string, params compute.VirtualMachineUpdate) error { - computeClient := compute.NewVirtualMachinesClientWithBaseURI(region.client.baseUrl, region.client.subscriptionId) - computeClient.Authorizer = region.client.authorizer - _, resourceGroup, instanceName := pareResourceGroupWithName(instanceId, INSTANCE_RESOURCE) - if result, err := computeClient.Update(context.Background(), resourceGroup, instanceName, params); err != nil { - return err - } else if err := result.WaitForCompletion(context.Background(), computeClient.Client); err != nil { + if instance, err := region.GetInstance(instanceId); err != nil { return err + } else { + computeClient := compute.NewVirtualMachinesClientWithBaseURI(region.client.baseUrl, region.client.subscriptionId) + computeClient.Authorizer = region.client.authorizer + _, resourceGroup, instanceName := pareResourceGroupWithName(instanceId, INSTANCE_RESOURCE) + if _, err := computeClient.Update(context.Background(), resourceGroup, instanceName, params); err != nil { + return err + } + if err := cloudprovider.WaitStatus(instance, instance.GetStatus(), time.Second*5, time.Second*1800); err != nil { + return err + } } return nil } @@ -508,20 +517,16 @@ func (self *SInstance) RebuildRoot(imageId string) error { } func (region *SRegion) RebuildRoot(instanceId string) error { - _, resourceGroup, instanceName := pareResourceGroupWithName(instanceId, INSTANCE_RESOURCE) - computeClient := compute.NewVirtualMachinesClientWithBaseURI(region.client.baseUrl, region.client.subscriptionId) - computeClient.Authorizer = region.client.authorizer - if result, err := computeClient.Redeploy(context.Background(), resourceGroup, instanceName); err != nil { + if instance, err := region.GetInstance(instanceId); err != nil { return err - } else if err := result.WaitForCompletion(context.Background(), computeClient.Client); err != nil { - if strings.Index(err.Error(), "OSProvisioningTimedOut") > 0 { - if instance, err := region.GetInstance(instanceId); err != nil { - return err - } else if status := instance.GetStatus(); status == models.VM_RUNNING { - region.StopVM(instanceId, true) - } - return nil - } else { + } else { + _, resourceGroup, instanceName := pareResourceGroupWithName(instanceId, INSTANCE_RESOURCE) + computeClient := compute.NewVirtualMachinesClientWithBaseURI(region.client.baseUrl, region.client.subscriptionId) + computeClient.Authorizer = region.client.authorizer + if _, err := computeClient.Redeploy(context.Background(), resourceGroup, instanceName); err != nil { + return err + } + if err := cloudprovider.WaitStatus(instance, instance.GetStatus(), time.Second*5, time.Second*1800); err != nil { return err } } @@ -694,13 +699,20 @@ func (self *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) { } func (self *SRegion) StartVM(instanceId string) error { - _, resourceGroup, instanceName := pareResourceGroupWithName(instanceId, INSTANCE_RESOURCE) - computeClient := compute.NewVirtualMachinesClientWithBaseURI(self.client.baseUrl, self.client.subscriptionId) - computeClient.Authorizer = self.client.authorizer - if result, err := computeClient.Start(context.Background(), resourceGroup, instanceName); err != nil { - return err - } else if err := result.WaitForCompletion(context.Background(), computeClient.Client); err != nil { + if instance, err := self.GetInstance(instanceId); err != nil { return err + } else if status := instance.GetStatus(); status == models.VM_RUNNING { + return nil + } else { + _, resourceGroup, instanceName := pareResourceGroupWithName(instanceId, INSTANCE_RESOURCE) + computeClient := compute.NewVirtualMachinesClientWithBaseURI(self.client.baseUrl, self.client.subscriptionId) + computeClient.Authorizer = self.client.authorizer + if _, err := computeClient.Start(context.Background(), resourceGroup, instanceName); err != nil { + return err + } + if err = cloudprovider.WaitStatus(instance, models.VM_RUNNING, time.Second*5, time.Second*1800); err != nil { + return err + } } return nil } @@ -724,18 +736,42 @@ func (self *SRegion) StopVM(instanceId string, isForce bool) error { } func (self *SRegion) doStopVM(instanceId string, isForce bool) error { - _, resourceGroup, instanceName := pareResourceGroupWithName(instanceId, INSTANCE_RESOURCE) - computeClient := compute.NewVirtualMachinesClientWithBaseURI(self.client.baseUrl, self.client.subscriptionId) - computeClient.Authorizer = self.client.authorizer - if result, err := computeClient.PowerOff(context.Background(), resourceGroup, instanceName); err != nil { - return err - } else if err := result.WaitForCompletion(context.Background(), computeClient.Client); err != nil { + if instance, err := self.GetInstance(instanceId); err != nil { return err + } else { + _, resourceGroup, instanceName := pareResourceGroupWithName(instanceId, INSTANCE_RESOURCE) + computeClient := compute.NewVirtualMachinesClientWithBaseURI(self.client.baseUrl, self.client.subscriptionId) + computeClient.Authorizer = self.client.authorizer + if _, err := computeClient.PowerOff(context.Background(), resourceGroup, instanceName); err != nil { + return err + } + if err = cloudprovider.WaitStatus(instance, models.VM_READY, time.Second*5, time.Second*1800); err != nil { + return err + } } return nil } func (self *SInstance) SyncSecurityGroup(secgroupId string, name string, rules []secrules.SecurityRule) error { + nics, err := self.getNics() + if err != nil { + return err + } + if len(secgroupId) == 0 { + for _, nic := range nics { + if err := nic.revokeSecurityGroup(); err != nil { + return err + } + } + } else if extId, err := self.host.zone.region.syncSecurityGroup(secgroupId, name, rules); err != nil { + return err + } else { + for _, nic := range nics { + if err := nic.assignSecurityGroup(extId); err != nil { + return err + } + } + } return nil } diff --git a/pkg/util/azure/instancenic.go b/pkg/util/azure/instancenic.go index 226df9930c..fc989add1b 100644 --- a/pkg/util/azure/instancenic.go +++ b/pkg/util/azure/instancenic.go @@ -80,6 +80,46 @@ func (self *SInstanceNic) GetDriver() string { return "virtio" } +func (self *SInstanceNic) updateSecurityGroup(secgroupId string) error { + region := self.instance.host.zone.region + nicClient := network.NewInterfacesClientWithBaseURI(region.client.baseUrl, region.SubscriptionID) + nicClient.Authorizer = region.client.authorizer + _, resourceGroup, nicName := pareResourceGroupWithName(self.ID, NIC_RESOURCE) + iPConfigurations := []network.InterfaceIPConfiguration{ + network.InterfaceIPConfiguration{ + Name: &nicName, + ID: &self.ID, + InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ + Subnet: &network.Subnet{ID: &self.Properties.IPConfigurations[0].Properties.Subnet.ID}, + }, + }, + } + params := network.Interface{ + Location: ®ion.Name, + InterfacePropertiesFormat: &network.InterfacePropertiesFormat{ + IPConfigurations: &iPConfigurations, + NetworkSecurityGroup: &network.SecurityGroup{}, + }, + } + if len(secgroupId) > 0 { + params.InterfacePropertiesFormat.NetworkSecurityGroup.ID = &secgroupId + } + if result, err := nicClient.CreateOrUpdate(context.Background(), resourceGroup, nicName, params); err != nil { + return err + } else if err := result.WaitForCompletion(context.Background(), nicClient.Client); err != nil { + return err + } + return nil +} + +func (self *SInstanceNic) revokeSecurityGroup() error { + return self.updateSecurityGroup("") +} + +func (self *SInstanceNic) assignSecurityGroup(secgroupId string) error { + return self.updateSecurityGroup(secgroupId) +} + func (self *SInstanceNic) GetINetwork() cloudprovider.ICloudNetwork { if wires, err := self.instance.host.GetIWires(); err != nil { log.Errorf("GetINetwork error: %v", err) diff --git a/pkg/util/azure/securitygroup.go b/pkg/util/azure/securitygroup.go index c84756cddf..41b87a2eb9 100644 --- a/pkg/util/azure/securitygroup.go +++ b/pkg/util/azure/securitygroup.go @@ -3,10 +3,14 @@ package azure import ( "context" "fmt" + "sort" "strings" "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/pkg/util/secrules" + "yunion.io/x/pkg/utils" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network" ) @@ -50,6 +54,25 @@ type SecurityRules struct { ID string } +type SecurityRulesSet []SecurityRules + +func (v SecurityRulesSet) Len() int { + return len(v) +} + +func (v SecurityRulesSet) Swap(i, j int) { + v[i], v[j] = v[j], v[i] +} + +func (v SecurityRulesSet) Less(i, j int) bool { + if v[i].Properties.Priority < v[j].Properties.Priority { + return true + } else if v[i].Properties.Priority == v[j].Properties.Priority { + return strings.Compare(v[i].Properties.String(), v[j].Properties.String()) <= 0 + } + return false +} + type Interface struct { } @@ -66,6 +89,7 @@ type SSecurityGroup struct { ID string Name string Location string + Tags map[string]string } func (self *SecurityRulePropertiesFormat) String() string { @@ -117,7 +141,6 @@ func (self *SecurityRulePropertiesFormat) String() string { result += fmt.Sprintf(" %s", port) } } - //log.Debugf("result: %s", result) return result } @@ -130,7 +153,8 @@ func (self *SSecurityGroup) GetMetadata() *jsonutils.JSONDict { } func (self *SSecurityGroup) GetGlobalId() string { - return fmt.Sprintf("%s/%s", self.vpc.GetGlobalId(), self.Name) + globalId, _, _ := pareResourceGroupWithName(self.ID, SECGRP_RESOURCE) + return globalId } func (self *SSecurityGroup) GetDescription() string { @@ -138,21 +162,22 @@ func (self *SSecurityGroup) GetDescription() string { } func (self *SSecurityGroup) GetName() string { - return self.Name + return strings.TrimPrefix(self.Name, self.vpc.region.Name+"-") } func (self *SSecurityGroup) GetRules() ([]secrules.SecurityRule, error) { rules := make([]secrules.SecurityRule, 0) + sort.Sort(SecurityRulesSet(self.Properties.SecurityRules)) + priority := 100 + for _, _rule := range self.Properties.SecurityRules { for _, ruleString := range strings.Split(_rule.Properties.String(), ";") { if rule, err := secrules.ParseSecurityRule(ruleString); err != nil { return rules, err } else { - rule.Priority = 100 - int(_rule.Properties.Priority) + rule.Priority = priority + priority-- rule.Description = _rule.Properties.Description - if rule.Priority < 0 { - rule.Priority = 1 - } rules = append(rules, *rule) } } @@ -168,14 +193,242 @@ func (self *SSecurityGroup) IsEmulated() bool { return false } +func (region *SRegion) CreateSecurityGroup(secName string) (*SSecurityGroup, error) { + securityName := fmt.Sprintf("%s-%s", region.Name, secName) + globalId, resourceGroup, securityName := pareResourceGroupWithName(securityName, SECGRP_RESOURCE) + secClient := network.NewSecurityGroupsClientWithBaseURI(region.client.baseUrl, region.SubscriptionID) + secClient.Authorizer = region.client.authorizer + params := network.SecurityGroup{ + Location: ®ion.Name, + Name: &securityName, + } + if result, err := secClient.CreateOrUpdate(context.Background(), resourceGroup, securityName, params); err != nil { + return nil, err + } else if result.WaitForCompletion(context.Background(), secClient.Client); err != nil { + return nil, err + } + return region.GetSecurityGroupDetails(globalId) +} + +func (region *SRegion) GetSecurityGroups() ([]SSecurityGroup, error) { + secgroups := make([]SSecurityGroup, 0) + secClient := network.NewSecurityGroupsClientWithBaseURI(region.client.baseUrl, region.SubscriptionID) + secClient.Authorizer = region.client.authorizer + secgroupIds := make([]string, 0) + if result, err := secClient.ListAll(context.Background()); err != nil { + return nil, err + } else { + for _, secgrp := range result.Values() { + if *secgrp.Location == region.Name { + securityGroup := SSecurityGroup{} + if secgroupId, ok := secgrp.Tags["id"]; ok { + if utils.IsInStringArray(*secgroupId, secgroupIds) { + continue + } else { + secgroupIds = append(secgroupIds, *secgroupId) + } + } + if err := jsonutils.Update(&securityGroup, secgrp); err != nil { + return nil, err + } + securityGroup.Name = strings.TrimPrefix(securityGroup.Name, region.Name+"-") + secgroups = append(secgroups, securityGroup) + } + } + } + return secgroups, nil +} + +func (region *SRegion) GetSecurityGroupDetails(secgroupId string) (*SSecurityGroup, error) { + sec := SSecurityGroup{} + secClient := network.NewSecurityGroupsClientWithBaseURI(region.client.baseUrl, region.SubscriptionID) + secClient.Authorizer = region.client.authorizer + _, resourceGroup, secName := pareResourceGroupWithName(secgroupId, SECGRP_RESOURCE) + if len(secName) == 0 { + return nil, cloudprovider.ErrNotFound + } + if result, err := secClient.Get(context.Background(), resourceGroup, secName, ""); err != nil { + if result.Response.StatusCode == 404 { + return nil, cloudprovider.ErrNotFound + } + return nil, err + } else if err := jsonutils.Update(&sec, result); err != nil { + return nil, err + } + return &sec, nil +} + func (self *SSecurityGroup) Refresh() error { - _, resourceGroup, secgrpName := pareResourceGroupWithName(self.ID, SECGRP_RESOURCE) - networkClient := network.NewSecurityGroupsClientWithBaseURI(self.vpc.region.client.baseUrl, self.vpc.region.SubscriptionID) - networkClient.Authorizer = self.vpc.region.client.authorizer - if secgrp, err := networkClient.Get(context.Background(), resourceGroup, secgrpName, ""); err != nil { + if sec, err := self.vpc.region.GetSecurityGroupDetails(self.ID); err != nil { return err - } else if err := jsonutils.Update(self, secgrp); err != nil { + } else if err := jsonutils.Update(self, sec); err != nil { return err } return nil } + +func (region *SRegion) addTagToSecurityGroup(secgroupId, value string) error { + secClient := network.NewSecurityGroupsClientWithBaseURI(region.client.baseUrl, region.SubscriptionID) + secClient.Authorizer = region.client.authorizer + _, resourceGroup, secName := pareResourceGroupWithName(secgroupId, SECGRP_RESOURCE) + params := network.TagsObject{Tags: map[string]*string{"id": &value}} + if result, err := secClient.UpdateTags(context.Background(), resourceGroup, secName, params); err != nil { + return err + } else if result.WaitForCompletion(context.Background(), secClient.Client); err != nil { + return err + } + return nil +} + +func (region *SRegion) checkSecurityGroup(name, secgroupId string) (*SSecurityGroup, error) { + secName := fmt.Sprintf("%s-%s", region.Name, name) + globalId, _, _ := pareResourceGroupWithName(secName, SECGRP_RESOURCE) + if _, err := region.GetSecurityGroupDetails(globalId); err != nil { + if err == cloudprovider.ErrNotFound { + if _, err := region.CreateSecurityGroup(name); err != nil { + return nil, err + } else if err := region.addTagToSecurityGroup(globalId, secgroupId); err != nil { + return nil, err + } + } + } else if err := region.addTagToSecurityGroup(globalId, secgroupId); err != nil { + return nil, err + } + return region.GetSecurityGroupDetails(globalId) +} + +func convertRulePort(rule secrules.SecurityRule) *[]string { + ports := []string{} + for i := 0; i < len(rule.Ports); i++ { + ports = append(ports, fmt.Sprintf("%d", rule.Ports[i])) + } + if rule.PortStart > 0 && rule.PortEnd < 65535 { + ports = append(ports, fmt.Sprintf("%d-%d", rule.PortStart, rule.PortEnd)) + } + return &ports +} + +func convertSecurityGroupRule(rule secrules.SecurityRule) *network.SecurityRule { + name := strings.Replace(rule.String(), ":", "_", -1) + name = strings.Replace(name, " ", "_", -1) + name = strings.Replace(name, "-", "_", -1) + destRule := network.SecurityRule{ + Name: &name, + SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{}, + } + protocol := network.SecurityRuleProtocolAsterisk + if len(rule.Protocol) == 0 || rule.Protocol == secrules.PROTO_ANY { + protocol = network.SecurityRuleProtocolAsterisk + } else if rule.Protocol == secrules.PROTO_TCP { + protocol = network.SecurityRuleProtocolTCP + } else if rule.Protocol == secrules.PROTO_UDP { + protocol = network.SecurityRuleProtocolUDP + } else { + return nil + } + destRule.SecurityRulePropertiesFormat.Protocol = protocol + destRule.SecurityRulePropertiesFormat.Description = &rule.Description + direction := network.SecurityRuleDirectionInbound + if rule.Direction == secrules.SecurityRuleEgress { + direction = network.SecurityRuleDirectionOutbound + } + destRule.SecurityRulePropertiesFormat.Direction = direction + ipAddr := rule.IPNet.String() + ports := convertRulePort(rule) + if len(*ports) == 0 { + port := "*" + destRule.SecurityRulePropertiesFormat.SourcePortRange = &port + destRule.SecurityRulePropertiesFormat.DestinationPortRange = &port + } else { + destRule.SecurityRulePropertiesFormat.SourcePortRanges = ports + destRule.SecurityRulePropertiesFormat.DestinationPortRanges = ports + } + destRule.SecurityRulePropertiesFormat.DestinationAddressPrefix = &ipAddr + destRule.SecurityRulePropertiesFormat.SourceAddressPrefix = &ipAddr + + access := network.SecurityRuleAccessAllow + if rule.Action == secrules.SecurityRuleDeny { + access = network.SecurityRuleAccessDeny + } + destRule.SecurityRulePropertiesFormat.Access = access + priority := int32(rule.Priority) + destRule.SecurityRulePropertiesFormat.Priority = &priority + return &destRule +} + +func (region *SRegion) updateSecurityGroupRules(secgroupId string, rules []secrules.SecurityRule) (string, error) { + _, resourceGroup, secName := pareResourceGroupWithName(secgroupId, SECGRP_RESOURCE) + secClient := network.NewSecurityGroupsClientWithBaseURI(region.client.baseUrl, region.SubscriptionID) + secClient.Authorizer = region.client.authorizer + securityRules := []network.SecurityRule{} + priority := int32(100) + for i := 0; i < len(rules); i++ { + rules[i].Priority = int(priority) + if rule := convertSecurityGroupRule(rules[i]); rule != nil { + securityRules = append(securityRules, *rule) + priority++ + } + } + + params := network.SecurityGroup{ + Location: ®ion.Name, + SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{ + SecurityRules: &securityRules, + }, + } + if result, err := secClient.CreateOrUpdate(context.Background(), resourceGroup, secName, params); err != nil { + return "", err + } else if err := result.WaitForCompletion(context.Background(), secClient.Client); err != nil { + return "", err + } + return secgroupId, nil +} + +func (self *SRegion) syncSecgroupRules(secgroupId string, rules []secrules.SecurityRule) (string, error) { + if secgroup, err := self.GetSecurityGroupDetails(secgroupId); err != nil { + return "", err + } else { + sort.Sort(secrules.SecurityRuleSet(rules)) + sort.Sort(SecurityRulesSet(secgroup.Properties.SecurityRules)) + + newRules := []secrules.SecurityRule{} + + i, j := 0, 0 + for i < len(rules) || j < len(secgroup.Properties.SecurityRules) { + if i < len(rules) && j < len(secgroup.Properties.SecurityRules) { + srcRule := secgroup.Properties.SecurityRules[j].Properties.String() + destRule := rules[i].String() + cmp := strings.Compare(srcRule, destRule) + if cmp == 0 { + i++ + j++ + } else if cmp > 0 { + // remove srcRule + j++ + } else { + // add destRule + newRules = append(newRules, rules[i]) + i++ + } + } else if i >= len(rules) { + // del other rules + j++ + } else if j >= len(secgroup.Properties.SecurityRules) { + // add rule + newRules = append(newRules, rules[i]) + i++ + } + } + return self.updateSecurityGroupRules(secgroup.ID, newRules) + } +} + +func (self *SRegion) syncSecurityGroup(secgroupId, name string, rules []secrules.SecurityRule) (string, error) { + if secgroup, err := self.checkSecurityGroup(name, secgroupId); err != nil { + log.Errorf("check err: %v", err) + return "", err + } else { + + return self.syncSecgroupRules(secgroup.ID, rules) + } +} diff --git a/pkg/util/azure/shell/secgroup.go b/pkg/util/azure/shell/secgroup.go new file mode 100644 index 0000000000..ff2fc17219 --- /dev/null +++ b/pkg/util/azure/shell/secgroup.go @@ -0,0 +1,47 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/util/azure" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type SecurityGroupListOptions struct { + Limit int `help:"page size"` + Offset int `help:"page offset"` + } + shellutils.R(&SecurityGroupListOptions{}, "security-group-list", "List security group", func(cli *azure.SRegion, args *SecurityGroupListOptions) error { + if secgrps, err := cli.GetSecurityGroups(); err != nil { + return err + } else { + printList(secgrps, len(secgrps), args.Offset, args.Limit, []string{}) + return nil + } + }) + + type SecurityGroupShowOptions struct { + ID string `help:"ID or name of security group"` + } + shellutils.R(&SecurityGroupShowOptions{}, "security-group-show", "Show details of a security group", func(cli *azure.SRegion, args *SecurityGroupShowOptions) error { + if secgrp, err := cli.GetSecurityGroupDetails(args.ID); err != nil { + return err + } else { + printObject(secgrp) + return nil + } + }) + + type SecurityGroupCreateOptions struct { + NAME string `help:"Security Group name"` + } + + shellutils.R(&SecurityGroupCreateOptions{}, "security-group-create", "Create security group", func(cli *azure.SRegion, args *SecurityGroupCreateOptions) error { + if secgrp, err := cli.CreateSecurityGroup(args.NAME); err != nil { + return err + } else { + printObject(secgrp) + return nil + } + }) + +} diff --git a/pkg/util/azure/storagecache.go b/pkg/util/azure/storagecache.go index 4e4c1712d6..df67d5dfe2 100644 --- a/pkg/util/azure/storagecache.go +++ b/pkg/util/azure/storagecache.go @@ -101,7 +101,24 @@ func (self *SStoragecache) UploadImage(userCred mcclient.TokenCredential, imageI return self.uploadImage(userCred, imageId, osArch, osType, osDist, isForce) } -func (self *SRegion) CreateStorageAccount(resourceGroup, storageAccount string) error { +func (self *SRegion) checkBootDiagnosticStorageAccount() (string, error) { + storageAccount := fmt.Sprintf("%s-boot", self.Name) + resourceGroup := defaultResourceGroups[STORAGE_RESOURCE] + storageClinet := storageaccount.NewAccountsClientWithBaseURI(self.client.baseUrl, self.SubscriptionID) + storageClinet.Authorizer = self.client.authorizer + if result, err := storageClinet.ListByResourceGroup(context.Background(), resourceGroup); err != nil { + return "", err + } else { + for _, _storage := range *result.Value { + if *_storage.Name == storageAccount { + return *_storage.ID, nil + } + } + return self.CreateStorageAccount(resourceGroup, storageAccount) + } +} + +func (self *SRegion) CreateStorageAccount(resourceGroup, storageAccount string) (string, error) { storageClinet := storageaccount.NewAccountsClientWithBaseURI(self.client.baseUrl, self.SubscriptionID) storageClinet.Authorizer = self.client.authorizer sku := storageaccount.Sku{Name: storageaccount.SkuName("Standard_GRS")} @@ -113,11 +130,11 @@ func (self *SRegion) CreateStorageAccount(resourceGroup, storageAccount string) storageAccount = fmt.Sprintf("%s%s", self.Name, DefaultStorageAccount) } if result, err := storageClinet.Create(context.Background(), resourceGroup, storageAccount, params); err != nil { - return err + return "", err } else if err := result.WaitForCompletion(context.Background(), storageClinet.Client); err != nil { - return err + return "", err } - return nil + return self.getStorageAccountId(resourceGroup, storageAccount) } func (self *SRegion) checkStorageContainer(storageAccount, accessKey, containerName string) error { @@ -133,10 +150,18 @@ func (self *SRegion) checkStorageContainer(storageAccount, accessKey, containerN return nil } -func (self *SRegion) isStorageAccountExist(resourceGroup, storageAccount string) bool { +func (self *SRegion) getStorageAccountId(resourceGroup, storageAccount string) (string, error) { storageClinet := storageaccount.NewAccountsClientWithBaseURI(self.client.baseUrl, self.SubscriptionID) storageClinet.Authorizer = self.client.authorizer - if _, err := storageClinet.GetProperties(context.Background(), resourceGroup, storageAccount); err != nil { + if _storage, err := storageClinet.GetProperties(context.Background(), resourceGroup, storageAccount); err != nil { + return "", err + } else { + return *_storage.ID, nil + } +} + +func (self *SRegion) isStorageAccountExist(resourceGroup, storageAccount string) bool { + if _, err := self.getStorageAccountId(resourceGroup, storageAccount); err != nil { return false } return true @@ -171,7 +196,7 @@ func (self *SRegion) CheckBlobContainer(resourceGroup, storageAccount, blobName if err := self.client.fetchAzueResourceGroup(); err != nil { return err } else if !self.isStorageAccountExist(resourceGroup, storageAccount) { - if err := self.CreateStorageAccount(resourceGroup, storageAccount); err != nil { + if _, err := self.CreateStorageAccount(resourceGroup, storageAccount); err != nil { return err } } @@ -306,7 +331,7 @@ func (self *SRegion) uploadContainerFileByPath(storageAccount, accessKey, contai BlobServiceClient: blobServiceClient, ContainerName: containerName, BlobName: blobName, - Parallelism: 4, + Parallelism: 3, Resume: false, MD5Hash: []byte(""), //localMetaData.FileMetaData.MD5Hash, } diff --git a/pkg/util/azure/vpc.go b/pkg/util/azure/vpc.go index f1f85fdb94..2fc06a3b29 100644 --- a/pkg/util/azure/vpc.go +++ b/pkg/util/azure/vpc.go @@ -91,24 +91,27 @@ func (self *SVpc) Delete() error { return nil } +func (self *SVpc) getSecurityGroups() ([]SSecurityGroup, error) { + if securityGroups, err := self.region.GetSecurityGroups(); err != nil { + return nil, err + } else { + for i := 0; i < len(securityGroups); i++ { + securityGroups[i].vpc = self + } + return securityGroups, nil + } +} + func (self *SVpc) fetchSecurityGroups() error { self.secgroups = make([]cloudprovider.ICloudSecurityGroup, 0) - networkClient := network.NewSecurityGroupsClientWithBaseURI(self.region.client.baseUrl, self.region.SubscriptionID) - networkClient.Authorizer = self.region.client.authorizer - if secgrpList, err := networkClient.ListAll(context.Background()); err != nil { + if secgrps, err := self.getSecurityGroups(); err != nil { return err } else { - for _, secgrp := range secgrpList.Values() { - securityGroup := SSecurityGroup{vpc: self} - if *secgrp.Location == self.Location { - if err := jsonutils.Update(&securityGroup, secgrp); err != nil { - return err - } - self.secgroups = append(self.secgroups, &securityGroup) - } + for i := 0; i < len(secgrps); i++ { + self.secgroups = append(self.secgroups, &secgrps[i]) } + return nil } - return nil } func (self *SVpc) getWire() *SWire { diff --git a/pkg/util/azure/wire.go b/pkg/util/azure/wire.go index 72ecd4ca37..79a843e5ff 100644 --- a/pkg/util/azure/wire.go +++ b/pkg/util/azure/wire.go @@ -145,7 +145,6 @@ func (self *SWire) getNetworkById(networkId string) *SNetwork { for i := 0; i < len(networks); i++ { network := networks[i].(*SNetwork) _globalId, _, _ := pareResourceGroupWithName(network.ID, VPC_RESOURCE) - log.Errorf("Compaire %s => %s", globalId, _globalId) if globalId == _globalId { return network }