diff --git a/pkg/compute/models/cloudregions.go b/pkg/compute/models/cloudregions.go index 6909a5c512..44a1901655 100644 --- a/pkg/compute/models/cloudregions.go +++ b/pkg/compute/models/cloudregions.go @@ -309,7 +309,7 @@ func (manager *SCloudregionManager) ListItemFilter(ctx context.Context, q *sqlch wires := WireManager.Query().SubQuery() vpcs := VpcManager.Query().SubQuery() - sq := vpcs.Query(vpcs.Field("cloudregion_id")) + sq := vpcs.Query(sqlchemy.DISTINCT("cloudregion_id", vpcs.Field("cloudregion_id"))) sq = sq.Join(wires, sqlchemy.Equals(vpcs.Field("id"), wires.Field("vpc_id"))) sq = sq.Join(networks, sqlchemy.Equals(wires.Field("id"), networks.Field("wire_id"))) sq = sq.Filter(sqlchemy.Equals(networks.Field("status"), NETWORK_STATUS_AVAILABLE)) diff --git a/pkg/util/azure/azure.go b/pkg/util/azure/azure.go index de4a4d0f25..b6bf6d9617 100644 --- a/pkg/util/azure/azure.go +++ b/pkg/util/azure/azure.go @@ -31,7 +31,6 @@ const ( DISK_RESOURCE = "disk" INSTANCE_RESOURCE = "instance" VPC_RESOURCE = "vpc" - NETWORK_RESOURCE = "network" NIC_RESOURCE = "nic" IMAGE_RESOURCE = "image" STORAGE_RESOURCE = "storage" @@ -43,7 +42,6 @@ var defaultResourceGroups = map[string]string{ DISK_RESOURCE: "YunionDiskResource", INSTANCE_RESOURCE: "YunionInstanceResource", VPC_RESOURCE: "YunionVpcResource", - NETWORK_RESOURCE: "YunionNetworkResource", NIC_RESOURCE: "YunionNicInterface", IMAGE_RESOURCE: "YunionImageResource", STORAGE_RESOURCE: "YunionStorageResource", @@ -93,7 +91,8 @@ func NewAzureClient(providerId string, providerName string, accessKey string, se func pareResourceGroupWithName(s string, resourceType string) (string, string, string) { valid := regexp.MustCompile("resourceGroups/(.+)/providers/.+/(.+)$") if resourceGroups := valid.FindStringSubmatch(s); len(resourceGroups) == 3 { - return s, resourceGroups[1], resourceGroups[2] + globalId := fmt.Sprintf("resourceGroups/%s/providers/%s/%s", resourceGroups[1], resourceType, resourceGroups[2]) + return globalId, resourceGroups[1], resourceGroups[2] } if len(s) == 0 { log.Errorf("pareResourceGroupWithName[%s] error", resourceType) diff --git a/pkg/util/azure/network.go b/pkg/util/azure/network.go index 4eb311f473..57a8a4e16a 100644 --- a/pkg/util/azure/network.go +++ b/pkg/util/azure/network.go @@ -35,7 +35,7 @@ func (self *SNetwork) GetName() string { } func (self *SNetwork) GetGlobalId() string { - globalId, _, _ := pareResourceGroupWithName(self.ID, NETWORK_RESOURCE) + globalId, _, _ := pareResourceGroupWithName(self.ID, VPC_RESOURCE) return globalId } diff --git a/pkg/util/azure/storagecache.go b/pkg/util/azure/storagecache.go index e33202887b..4e4c1712d6 100644 --- a/pkg/util/azure/storagecache.go +++ b/pkg/util/azure/storagecache.go @@ -266,39 +266,6 @@ func (self *SRegion) ListContainerFiles(resourceGroup, storageAccount, blobName } } -func (self *SRegion) uploadContainerFileByReader(storageAccount, accessKey, containerName, fileName string, content io.Reader, size int64) (string, error) { - if client, err := storage.NewBasicClientOnSovereignCloud(storageAccount, accessKey, self.client.env); err != nil { - return "", err - } else { - blob := client.GetBlobService() - container := blob.GetContainerReference(containerName) - blobClient := container.GetBlobReference(fileName) - if _, err := blobClient.DeleteIfExists(&storage.DeleteBlobOptions{}); err != nil { - return "", err - } - blobClient.Properties.ContentLength = size - if err := blobClient.PutPageBlob(&storage.PutBlobOptions{}); err != nil { - return "", err - } - var readed int64 = 0 - for i := 0; i < int(size/DefaultReadBlockSize); i++ { - if err := blobClient.WriteRange(storage.BlobRange{Start: uint64(readed), End: uint64(readed + DefaultReadBlockSize - 1)}, content, &storage.PutPageOptions{}); err != nil { - return "", err - } - readed += DefaultReadBlockSize - log.Debugf("Upload %s %f%% to %s", fileName, float64(readed)/float64(size)*100, storageAccount) - } - if extraSize := size % DefaultReadBlockSize; extraSize > 0 { - log.Debugf("Upload %s extra size: %d to %s", fileName, extraSize, storageAccount) - if err := blobClient.WriteRange(storage.BlobRange{Start: uint64(readed), End: uint64(readed + extraSize - 1)}, content, &storage.PutPageOptions{}); err != nil { - return "", err - } - } - log.Debugf("Upload %s complate", fileName) - return blobClient.GetURL(), nil - } -} - func (self *SRegion) uploadContainerFileByPath(storageAccount, accessKey, containerName, localVHDPath string) (string, error) { if err := ensureVHDSanity(localVHDPath); err != nil { return "", err @@ -339,7 +306,7 @@ func (self *SRegion) uploadContainerFileByPath(storageAccount, accessKey, contai BlobServiceClient: blobServiceClient, ContainerName: containerName, BlobName: blobName, - Parallelism: 3, + Parallelism: 4, Resume: false, MD5Hash: []byte(""), //localMetaData.FileMetaData.MD5Hash, } @@ -407,7 +374,6 @@ func (self *SStoragecache) uploadImage(userCred mcclient.TokenCredential, imageI blobURI, err := self.region.uploadContainerFileByPath(storageAccount, accessKey, DefaultBlobContainer, tmpFile) os.Remove(tmpFile) - //blobURI, err := self.region.uploadContainerFileByReader(storageAccount, accessKey, DefaultBlobContainer, imageNameOnBlob, reader, size) if err != nil { log.Errorf("uploadContainerFileByPath error: %v", err) return "", err diff --git a/pkg/util/azure/vpc.go b/pkg/util/azure/vpc.go index 7e50cf305b..f1f85fdb94 100644 --- a/pkg/util/azure/vpc.go +++ b/pkg/util/azure/vpc.go @@ -119,15 +119,18 @@ func (self *SVpc) getWire() *SWire { } func (self *SVpc) fetchNetworks() error { - self.Refresh() - for i := 0; i < len(self.Properties.Subnets); i++ { - _network := self.Properties.Subnets[i] - wire := self.getWire() - network := SNetwork{wire: wire, Name: _network.Name, ID: _network.ID} - if err := jsonutils.Update(&network, _network); err != nil { - return err + if vpc, err := self.region.getVpc(self.ID); err != nil { + return err + } else { + for i := 0; i < len(vpc.Properties.Subnets); i++ { + _network := vpc.Properties.Subnets[i] + wire := self.getWire() + network := SNetwork{wire: wire, Name: _network.Name, ID: _network.ID} + if err := jsonutils.Update(&network, _network); err != nil { + return err + } + wire.addNetwork(&network) } - wire.addNetwork(&network) } return nil } @@ -194,13 +197,23 @@ func (self *SVpc) GetStatus() string { return "disabled" } -func (self *SVpc) Refresh() error { - _, resourceGroup, vpcName := pareResourceGroupWithName(self.ID, VPC_RESOURCE) - vpcClient := network.NewVirtualNetworksClientWithBaseURI(self.region.client.baseUrl, self.region.SubscriptionID) - vpcClient.Authorizer = self.region.client.authorizer +func (region *SRegion) getVpc(vpcId string) (*SVpc, error) { + vpc := SVpc{} + _, resourceGroup, vpcName := pareResourceGroupWithName(vpcId, VPC_RESOURCE) + vpcClient := network.NewVirtualNetworksClientWithBaseURI(region.client.baseUrl, region.SubscriptionID) + vpcClient.Authorizer = region.client.authorizer if result, err := vpcClient.Get(context.Background(), resourceGroup, vpcName, ""); err != nil { - return cloudprovider.ErrNotFound - } else if err := jsonutils.Update(self, result); err != nil { + return nil, cloudprovider.ErrNotFound + } else if err := jsonutils.Update(&vpc, result); err != nil { + return nil, err + } + return &vpc, nil +} + +func (self *SVpc) Refresh() error { + if vpc, err := self.region.getVpc(self.ID); err != nil { + return err + } else if err := jsonutils.Update(self, vpc); err != nil { return err } return nil diff --git a/pkg/util/azure/wire.go b/pkg/util/azure/wire.go index b26d95bc1a..72ecd4ca37 100644 --- a/pkg/util/azure/wire.go +++ b/pkg/util/azure/wire.go @@ -78,7 +78,8 @@ func (self *SRegion) createNetwork(vpc *SVpc, subnetName string, cidr string, de networkClient := network.NewVirtualNetworksClientWithBaseURI(self.client.baseUrl, self.SubscriptionID) networkClient.Authorizer = self.client.authorizer - networkId, resourceGroup, vpcName := pareResourceGroupWithName(vpc.ID, VPC_RESOURCE) + _, resourceGroup, vpcName := pareResourceGroupWithName(vpc.ID, VPC_RESOURCE) + networkId, _, _ := pareResourceGroupWithName(subnetName, VPC_RESOURCE) if result, err := networkClient.CreateOrUpdate(context.Background(), resourceGroup, vpcName, params); err != nil { return "", err } else if err := result.WaitForCompletion(context.Background(), networkClient.Client); err != nil { @@ -120,10 +121,8 @@ func (self *SWire) GetINetworkById(netid string) (cloudprovider.ICloudNetwork, e } func (self *SWire) GetINetworks() ([]cloudprovider.ICloudNetwork, error) { - if self.inetworks == nil { - if err := self.vpc.fetchNetworks(); err != nil { - return nil, err - } + if err := self.vpc.fetchNetworks(); err != nil { + return nil, err } return self.inetworks, nil } @@ -141,12 +140,13 @@ func (self *SWire) getNetworkById(networkId string) *SNetwork { log.Errorf("getNetworkById error: %v", err) return nil } else { - _, resourceGroup, networkName := pareResourceGroupWithName(networkId, NETWORK_RESOURCE) + globalId, _, _ := pareResourceGroupWithName(networkId, VPC_RESOURCE) log.Debugf("search for networks %d", len(networks)) for i := 0; i < len(networks); i++ { network := networks[i].(*SNetwork) - _, _resourceGroup, _networkName := pareResourceGroupWithName(network.ID, NETWORK_RESOURCE) - if resourceGroup == _resourceGroup && networkName == _networkName { + _globalId, _, _ := pareResourceGroupWithName(network.ID, VPC_RESOURCE) + log.Errorf("Compaire %s => %s", globalId, _globalId) + if globalId == _globalId { return network } }