From e74f31b7c0a8971a800fb6cbf83e620a53d54682 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Thu, 14 Mar 2019 09:46:29 +0800 Subject: [PATCH 1/6] fix: cleanup project sync codes --- cmd/azurecli/main.go | 3 +- cmd/climc/shell/external_projects.go | 14 ++- pkg/cloudcommon/db/interface.go | 2 + pkg/cloudcommon/db/opslog.go | 1 + pkg/cloudcommon/db/virtualresource.go | 27 ++++- pkg/cloudprovider/resources.go | 1 + pkg/compute/models/cloudproviderregions.go | 10 ++ pkg/compute/models/cloudproviders.go | 109 ++++++++++++++++-- pkg/compute/models/cloudsync.go | 38 +++--- pkg/compute/models/disks.go | 45 +++----- pkg/compute/models/elasticips.go | 42 ++----- pkg/compute/models/external_projects.go | 28 ++++- pkg/compute/models/guests.go | 46 +++----- pkg/compute/models/host_recycle.go | 4 +- pkg/compute/models/hosts.go | 27 ++--- pkg/compute/models/loadbalanceracls.go | 48 +++----- .../models/loadbalancerbackendgroups.go | 38 ++---- pkg/compute/models/loadbalancerbackends.go | 37 ++---- .../models/loadbalancercertificates.go | 46 +++----- .../models/loadbalancerlistenerrules.go | 26 +++-- pkg/compute/models/loadbalancerlisteners.go | 37 ++---- pkg/compute/models/loadbalancers.go | 37 ++---- pkg/compute/models/networks.go | 42 ++----- pkg/compute/models/secgroups.go | 45 +++----- pkg/compute/models/snapshots.go | 41 ++----- pkg/compute/models/syncutils.go | 9 -- pkg/compute/regiondrivers/managedvirtual.go | 14 +-- pkg/compute/regiondrivers/qcloud.go | 6 +- .../tasks/cloud_provider_delete_task.go | 36 ++++++ pkg/compute/tasks/eip_allocate_task.go | 2 +- pkg/compute/tasks/eip_syncstatus_task.go | 2 +- pkg/compute/tasks/network_create_task.go | 2 +- pkg/mcclient/modules/mod_externalprojects.go | 6 +- pkg/util/aliyun/loadbalancerlistenerrule.go | 4 + pkg/util/azure/azure.go | 40 ++++++- pkg/util/azure/provider/provider.go | 2 +- pkg/util/azure/resourcegroup.go | 25 +++- pkg/util/azure/shell/resourcegroup.go | 33 ++++-- pkg/util/logclient/logclient.go | 1 + pkg/util/printutils/printjson.go | 2 +- pkg/util/qcloud/loadbalancer_listenerrule.go | 4 + pkg/util/qcloud/project.go | 18 ++- pkg/util/qcloud/qcloud.go | 3 +- 43 files changed, 538 insertions(+), 465 deletions(-) create mode 100644 pkg/compute/tasks/cloud_provider_delete_task.go diff --git a/cmd/azurecli/main.go b/cmd/azurecli/main.go index ffc636a49c..5130a024bf 100644 --- a/cmd/azurecli/main.go +++ b/cmd/azurecli/main.go @@ -16,6 +16,7 @@ import ( type BaseOptions struct { Help bool `help:"Show help"` + Debug bool `help:"debug mode"` DirectoryID string `help:"Azure account Directory ID/Tenant ID" default:"$AZURE_DIRECTORY_ID"` SubscriptionID string `help:"Azure account subscription ID" default:"$AZURE_SUBSCRIPTION_ID"` ApplicationID string `help:"Azure application ID" default:"$AZURE_APPLICATION_ID"` @@ -88,7 +89,7 @@ func newClient(options *BaseOptions) (*azure.SRegion, error) { account := fmt.Sprintf("%s/%s", options.DirectoryID, options.SubscriptionID) secret := fmt.Sprintf("%s/%s", options.ApplicationID, options.ApplicationKey) - cli, err := azure.NewAzureClient("", "", account, secret, options.CloudEnv) + cli, err := azure.NewAzureClient("", "", account, secret, options.CloudEnv, options.Debug) if err != nil { return nil, err } diff --git a/cmd/climc/shell/external_projects.go b/cmd/climc/shell/external_projects.go index 409cd5e5f1..fd6bb8b259 100644 --- a/cmd/climc/shell/external_projects.go +++ b/cmd/climc/shell/external_projects.go @@ -21,7 +21,19 @@ func init() { if err != nil { return err } - printList(result, modules.Disks.GetColumns(s)) + printList(result, modules.ExternalProjects.GetColumns(s)) + return nil + }) + + type ExternalProjectShowOptions struct { + ID string `help:"ID"` + } + R(&ExternalProjectShowOptions{}, "external-project-show", "Show details of project mapping", func(s *mcclient.ClientSession, args *ExternalProjectShowOptions) error { + info, err := modules.ExternalProjects.Get(s, args.ID, nil) + if err != nil { + return err + } + printObject(info) return nil }) diff --git a/pkg/cloudcommon/db/interface.go b/pkg/cloudcommon/db/interface.go index 2d526dc079..5c50f843c8 100644 --- a/pkg/cloudcommon/db/interface.go +++ b/pkg/cloudcommon/db/interface.go @@ -180,6 +180,8 @@ type IVirtualModel interface { IsOwner(userCred mcclient.TokenCredential) bool // IsAdmin(userCred mcclient.TokenCredential) bool + + SyncCloudProjectId(userCred mcclient.TokenCredential, projectId string) } type ISharableVirtualModelManager interface { diff --git a/pkg/cloudcommon/db/opslog.go b/pkg/cloudcommon/db/opslog.go index 5aefdac2d2..902d042b0e 100644 --- a/pkg/cloudcommon/db/opslog.go +++ b/pkg/cloudcommon/db/opslog.go @@ -115,6 +115,7 @@ const ( ACT_SYNC_STATUS = "sync_status" ACT_CHANGE_OWNER = "change_owner" + ACT_SYNC_OWNER = "sync_owner" ACT_RESERVE_IP = "reserve_ip" ACT_RELEASE_IP = "release_ip" diff --git a/pkg/cloudcommon/db/virtualresource.go b/pkg/cloudcommon/db/virtualresource.go index 8224741df5..c5e5dabdde 100644 --- a/pkg/cloudcommon/db/virtualresource.go +++ b/pkg/cloudcommon/db/virtualresource.go @@ -17,9 +17,11 @@ import ( "yunion.io/x/onecloud/pkg/util/logclient" ) +type TProjectSource string + const ( - PROJECT_SOURCE_LOCAL = "local" - PROJECT_SOURCE_CLOUD = "cloud" + PROJECT_SOURCE_LOCAL = TProjectSource("local") + PROJECT_SOURCE_CLOUD = TProjectSource("cloud") ) type SVirtualResourceBaseManager struct { @@ -222,7 +224,7 @@ func (model *SVirtualResourceBase) PerformChangeOwner(ctx context.Context, userC if tobj.GetId() == model.ProjectId { // do nothing Update(model, func() error { - model.ProjectSrc = PROJECT_SOURCE_LOCAL + model.ProjectSrc = string(PROJECT_SOURCE_LOCAL) return nil }) return nil, nil @@ -241,7 +243,7 @@ func (model *SVirtualResourceBase) PerformChangeOwner(ctx context.Context, userC } diff, err := Update(model, func() error { model.ProjectId = tobj.GetId() - model.ProjectSrc = PROJECT_SOURCE_LOCAL + model.ProjectSrc = string(PROJECT_SOURCE_LOCAL) return nil }) if err != nil { @@ -347,3 +349,20 @@ func (model *SVirtualResourceBase) GetShortDesc(ctx context.Context) *jsonutils. } return desc } + +func (model *SVirtualResourceBase) SyncCloudProjectId(userCred mcclient.TokenCredential, projectId string) { + if model.ProjectSrc != string(PROJECT_SOURCE_LOCAL) && len(projectId) > 0 { + diff, _ := Update(model, func() error { + model.ProjectSrc = string(PROJECT_SOURCE_CLOUD) + if len(projectId) > 0 { + model.ProjectId = projectId + } + return nil + }) + diffStr := sqlchemy.UpdateDiffString(diff) + if len(diffStr) > 0 { + OpsLog.LogEvent(model, ACT_SYNC_OWNER, diffStr, userCred) + logclient.AddSimpleActionLog(model, logclient.ACT_SYNC_CLOUD_OWNER, diffStr, userCred, true) + } + } +} diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index 9c5a8cff1a..781d471c71 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -490,6 +490,7 @@ type ICloudLoadbalancerListener interface { type ICloudLoadbalancerListenerRule interface { ICloudResource + IVirtualResource GetDomain() string GetPath() string diff --git a/pkg/compute/models/cloudproviderregions.go b/pkg/compute/models/cloudproviderregions.go index a0bb278a73..a2c0d2c5fb 100644 --- a/pkg/compute/models/cloudproviderregions.go +++ b/pkg/compute/models/cloudproviderregions.go @@ -338,3 +338,13 @@ func (cpr *SCloudproviderregion) isEmptyPublicCloud() bool { } return true } + +func (cprm *SCloudproviderregionManager) fetchRecordsForCloudprovider(manager *SCloudprovider) ([]SCloudproviderregion, error) { + q := cprm.Query().Equals("cloudprovider_id", manager.Id) + recs := make([]SCloudproviderregion, 0) + err := db.FetchModelObjects(cprm, q, &recs) + if err != nil { + return nil, err + } + return recs, nil +} diff --git a/pkg/compute/models/cloudproviders.go b/pkg/compute/models/cloudproviders.go index 9288c047d9..374ebfdf17 100644 --- a/pkg/compute/models/cloudproviders.go +++ b/pkg/compute/models/cloudproviders.go @@ -26,9 +26,13 @@ import ( ) const ( - CLOUD_PROVIDER_INIT = "init" - CLOUD_PROVIDER_CONNECTED = "connected" - CLOUD_PROVIDER_DISCONNECTED = "disconnected" + CLOUD_PROVIDER_INIT = "init" + CLOUD_PROVIDER_CONNECTED = "connected" + CLOUD_PROVIDER_DISCONNECTED = "disconnected" + CLOUD_PROVIDER_START_DELETE = "start_delete" + CLOUD_PROVIDER_DELETING = "deleting" + CLOUD_PROVIDER_DELETED = "deleted" + CLOUD_PROVIDER_DELETE_FAILED = "delete_failed" CLOUD_PROVIDER_SYNC_STATUS_QUEUED = "queued" CLOUD_PROVIDER_SYNC_STATUS_SYNCING = "syncing" @@ -219,6 +223,14 @@ func (self *SCloudprovider) getLoadbalancerCount() int { return LoadbalancerManager.Query().Equals("manager_id", self.Id).Count() } +func (self *SCloudprovider) getExternalProjectCount() int { + return ExternalProjectManager.Query().Equals("manager_id", self.Id).Count() +} + +func (self *SCloudprovider) getSyncRegionCount() int { + return CloudproviderRegionManager.Query().Equals("cloudprovider_id", self.Id).Count() +} + func (self *SCloudprovider) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { return self.SEnabledStatusStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, data) } @@ -305,12 +317,13 @@ func (self *SCloudprovider) saveProject(userCred mcclient.TokenCredential, proje } type SSyncRange struct { - Force bool - FullSync bool - ProjectSync bool - Region []string - Zone []string - Host []string + Force bool + FullSync bool + // ProjectSync bool + + Region []string + Zone []string + Host []string } func (sr *SSyncRange) NeedSyncInfo() bool { @@ -489,7 +502,11 @@ func (self *SCloudprovider) PerformChangeProject(ctx context.Context, userCred m return nil, httperrors.NewGeneralError(err) } - return nil, self.StartSyncCloudProviderInfoTask(ctx, userCred, &SSyncRange{FullSync: true, ProjectSync: true}, "") + if self.GetCloudaccount().EnableAutoSync { // no need to sync rightnow, will do it in auto sync + return nil, nil + } + + return nil, self.StartSyncCloudProviderInfoTask(ctx, userCred, &SSyncRange{FullSync: true}, "") } func (self *SCloudprovider) markStartSync(userCred mcclient.TokenCredential) error { @@ -594,6 +611,8 @@ type SCloudproviderUsage struct { EipCount int SnapshotCount int LoadbalancerCount int + ProjectCount int + SyncRegionCount int } func (usage *SCloudproviderUsage) isEmpty() bool { @@ -618,6 +637,12 @@ func (usage *SCloudproviderUsage) isEmpty() bool { if usage.LoadbalancerCount > 0 { return false } + /*if usage.ProjectCount > 0 { + return false + } + if usage.SyncRegionCount > 0 { + return false + }*/ return true } @@ -632,6 +657,8 @@ func (self *SCloudprovider) getUsage() *SCloudproviderUsage { usage.EipCount = self.getEipCount() usage.SnapshotCount = self.getSnapshotCount() usage.LoadbalancerCount = self.getLoadbalancerCount() + usage.ProjectCount = self.getExternalProjectCount() + usage.SyncRegionCount = self.getSyncRegionCount() return &usage } @@ -868,3 +895,65 @@ func (self *SCloudprovider) IsAvailable() bool { } return true } + +func (self *SCloudprovider) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + // override + log.Infof("cloud provider delete do nothing") + return nil +} + +func (self *SCloudprovider) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error { + err := self.deleteSyncedRegions(ctx, userCred) + if err != nil { + return err + } + err = self.deleteProjectsMapping(ctx, userCred) + if err != nil { + return err + } + return self.SEnabledStatusStandaloneResourceBase.Delete(ctx, userCred) +} + +func (self *SCloudprovider) deleteSyncedRegions(ctx context.Context, userCred mcclient.TokenCredential) error { + cprs, err := CloudproviderRegionManager.fetchRecordsForCloudprovider(self) + if err != nil { + return err + } + for i := range cprs { + err = cprs[i].Detach(ctx, userCred) + if err != nil { + return err + } + } + return nil +} + +func (self *SCloudprovider) deleteProjectsMapping(ctx context.Context, userCred mcclient.TokenCredential) error { + projs, err := ExternalProjectManager.getProjectsByProvider(self) + if err != nil { + return err + } + for i := range projs { + err = projs[i].Delete(ctx, userCred) + if err != nil { + return err + } + } + return nil +} + +func (self *SCloudprovider) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { + return self.StartDiskCloudproviderTask(ctx, userCred, "") +} + +func (self *SCloudprovider) StartDiskCloudproviderTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { + params := jsonutils.NewDict() + task, err := taskman.TaskManager.NewTask(ctx, "CloudProviderDeleteTask", self, userCred, params, parentTaskId, "", nil) + if err != nil { + log.Errorf("%s", err) + return err + } + self.SetStatus(userCred, CLOUD_PROVIDER_START_DELETE, "StartDiskCloudproviderTask") + task.ScheduleRun(nil) + return nil +} diff --git a/pkg/compute/models/cloudsync.go b/pkg/compute/models/cloudsync.go index 33e5b16526..1f21b419a9 100644 --- a/pkg/compute/models/cloudsync.go +++ b/pkg/compute/models/cloudsync.go @@ -122,7 +122,7 @@ func syncRegionEips(ctx context.Context, userCred mcclient.TokenCredential, sync return } - result := ElasticipManager.SyncEips(ctx, userCred, provider, localRegion, eips, provider.ProjectId, syncRange.ProjectSync) + result := ElasticipManager.SyncEips(ctx, userCred, provider, localRegion, eips, provider.ProjectId) syncResults.Add(ElasticipManager, result) @@ -176,7 +176,7 @@ func syncVpcSecGroup(ctx context.Context, userCred mcclient.TokenCredential, syn return } - _, _, result := SecurityGroupManager.SyncSecgroups(ctx, userCred, secgroups, localVpc, provider.ProjectId, syncRange.ProjectSync) + _, _, result := SecurityGroupManager.SyncSecgroups(ctx, userCred, secgroups, localVpc, provider.ProjectId) syncResults.Add(SecurityGroupManager, result) @@ -246,7 +246,7 @@ func syncWireNetworks(ctx context.Context, userCred mcclient.TokenCredential, sy log.Errorf(msg) return } - _, _, result := NetworkManager.SyncNetworks(ctx, userCred, localWire, nets, provider.ProjectId, syncRange.ProjectSync) + _, _, result := NetworkManager.SyncNetworks(ctx, userCred, localWire, nets, provider.ProjectId) if syncResults != nil { syncResults.Add(NetworkManager, result) @@ -326,7 +326,7 @@ func syncStorageDisks(ctx context.Context, userCred mcclient.TokenCredential, sy log.Errorf(msg) return } - _, _, result := DiskManager.SyncDisks(ctx, userCred, driver, localStorage, disks, provider.ProjectId, syncRange.ProjectSync) + _, _, result := DiskManager.SyncDisks(ctx, userCred, driver, localStorage, disks, provider.ProjectId) syncResults.Add(DiskManager, result) @@ -347,7 +347,7 @@ func syncZoneHosts(ctx context.Context, userCred mcclient.TokenCredential, syncR log.Errorf(msg) return nil } - localHosts, remoteHosts, result := HostManager.SyncHosts(ctx, userCred, provider, localZone, hosts, syncRange.ProjectSync) + localHosts, remoteHosts, result := HostManager.SyncHosts(ctx, userCred, provider, localZone, hosts) syncResults.Add(HostManager, result) @@ -438,7 +438,7 @@ func syncHostVMs(ctx context.Context, userCred mcclient.TokenCredential, syncRes log.Errorf(msg) return } - localVMs, remoteVMs, result := localHost.SyncHostVMs(ctx, userCred, driver, vms, provider.ProjectId, syncRange.ProjectSync) + localVMs, remoteVMs, result := localHost.SyncHostVMs(ctx, userCred, driver, vms, provider.ProjectId) syncResults.Add(GuestManager, result) @@ -460,10 +460,6 @@ func syncHostVMs(ctx context.Context, userCred mcclient.TokenCredential, syncRes syncVMEip(ctx, userCred, provider, &localVMs[i], remoteVMs[i]) }() - - if localVMs[i].Status == VM_RUNNING { - db.OpsLog.LogEvent(&localVMs[i], db.ACT_START, localVMs[i].GetShortDesc(ctx), userCred) - } } } @@ -492,7 +488,7 @@ func syncVMDisks(ctx context.Context, userCred mcclient.TokenCredential, provide log.Errorf(msg) return } - result := localVM.SyncVMDisks(ctx, userCred, driver, host, disks, provider.ProjectId, syncRange.ProjectSync) + result := localVM.SyncVMDisks(ctx, userCred, driver, host, disks, provider.ProjectId) msg := result.Result() notes := fmt.Sprintf("syncVMDisks for VM %s result: %s", localVM.Name, msg) log.Infof(notes) @@ -700,7 +696,7 @@ func syncRegionSnapshots(ctx context.Context, userCred mcclient.TokenCredential, return } - result := SnapshotManager.SyncSnapshots(ctx, userCred, provider, localRegion, snapshots, provider.ProjectId, syncRange.ProjectSync) + result := SnapshotManager.SyncSnapshots(ctx, userCred, provider, localRegion, snapshots, provider.ProjectId) syncResults.Add(SnapshotManager, result) @@ -809,7 +805,7 @@ func syncOnPremiseCloudProviderInfo( return err } - localHosts, remoteHosts, result := HostManager.SyncHosts(ctx, userCred, provider, nil, ihosts, syncRange.ProjectSync) + localHosts, remoteHosts, result := HostManager.SyncHosts(ctx, userCred, provider, nil, ihosts) syncResults.Add(HostManager, result) @@ -877,3 +873,19 @@ func (manager *SCloudproviderregionManager) initAllRecords() { }) } } + +func SyncCloudProject(userCred mcclient.TokenCredential, model db.IVirtualModel, projectId string, extModel cloudprovider.IVirtualResource, managerId string) { + newId := "" + if extProjectId := extModel.GetProjectId(); len(extProjectId) > 0 { + extProject, err := ExternalProjectManager.GetProject(extProjectId, managerId) + if err != nil { + log.Errorf(err.Error()) + } else { + newId = extProject.ProjectId + } + } + if len(newId) == 0 && len(projectId) > 0 { + newId = projectId + } + model.SyncCloudProjectId(userCred, newId) +} diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 45ead3cd4f..d222167782 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -917,8 +917,9 @@ func (manager *SDiskManager) getDisksByStorage(storage *SStorage) ([]SDisk, erro return disks, nil } -func (manager *SDiskManager) syncCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, provider cloudprovider.ICloudProvider, vdisk cloudprovider.ICloudDisk, index int, projectId string, projectSync bool) (*SDisk, error) { - ownerProjId := getSyncOwnerProjectId(manager, userCred, projectId, projectSync) +func (manager *SDiskManager) syncCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, provider cloudprovider.ICloudProvider, vdisk cloudprovider.ICloudDisk, index int, projectId string) (*SDisk, error) { + ownerProjId := projectId + lockman.LockClass(ctx, manager, ownerProjId) defer lockman.ReleaseClass(ctx, manager, ownerProjId) @@ -939,7 +940,7 @@ func (manager *SDiskManager) syncCloudDisk(ctx context.Context, userCred mcclien } } else { disk := diskObj.(*SDisk) - err = disk.syncWithCloudDisk(ctx, userCred, provider, vdisk, index, projectId, projectSync) + err = disk.syncWithCloudDisk(ctx, userCred, provider, vdisk, index, ownerProjId) if err != nil { return nil, err } @@ -947,8 +948,9 @@ func (manager *SDiskManager) syncCloudDisk(ctx context.Context, userCred mcclien } } -func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.TokenCredential, provider cloudprovider.ICloudProvider, storage *SStorage, disks []cloudprovider.ICloudDisk, projectId string, projectSync bool) ([]SDisk, []cloudprovider.ICloudDisk, compare.SyncResult) { - syncOwnerId := getSyncOwnerProjectId(manager, userCred, projectId, projectSync) +func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.TokenCredential, provider cloudprovider.ICloudProvider, storage *SStorage, disks []cloudprovider.ICloudDisk, projectId string) ([]SDisk, []cloudprovider.ICloudDisk, compare.SyncResult) { + syncOwnerId := projectId + lockman.LockClass(ctx, manager, syncOwnerId) defer lockman.ReleaseClass(ctx, manager, syncOwnerId) @@ -983,7 +985,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].syncWithCloudDisk(ctx, userCred, provider, commonext[i], -1, projectId, projectSync) + err = commondb[i].syncWithCloudDisk(ctx, userCred, provider, commonext[i], -1, projectId) if err != nil { syncResult.UpdateError(err) } else { @@ -1016,7 +1018,7 @@ func (self *SDisk) syncRemoveCloudDisk(ctx context.Context, userCred mcclient.To return self.SetStatus(userCred, DISK_UNKNOWN, "missing original disk after sync") } -func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, provider cloudprovider.ICloudProvider, extDisk cloudprovider.ICloudDisk, index int, projectId string, projectSync bool) error { +func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, provider cloudprovider.ICloudProvider, extDisk cloudprovider.ICloudDisk, index int, projectId string) error { recycle := false guests := self.GetGuests() if provider.GetFactory().IsSupportPrepaidResources() && len(guests) == 1 && guests[0].IsPrepaidRecycle() { @@ -1049,20 +1051,6 @@ func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.Toke self.ExpiredAt = extDisk.GetExpiredAt() } - if projectSync && self.ProjectSrc != db.PROJECT_SOURCE_LOCAL { - self.ProjectSrc = db.PROJECT_SOURCE_CLOUD - if len(projectId) > 0 { - self.ProjectId = projectId - } - if extProjectId := extDisk.GetProjectId(); len(extProjectId) > 0 { - extProject, err := ExternalProjectManager.GetProject(extProjectId, storage.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - self.ProjectId = extProject.ProjectId - } - } - } return nil }) if err != nil { @@ -1072,6 +1060,8 @@ func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.Toke db.OpsLog.LogSyncUpdate(self, diff, userCred) + SyncCloudProject(userCred, self, projectId, extDisk, storage.ManagerId) + return nil } @@ -1084,17 +1074,6 @@ func (manager *SDiskManager) newFromCloudDisk(ctx context.Context, userCred mccl disk.ExternalId = extDisk.GetGlobalId() disk.StorageId = storage.Id - disk.ProjectSrc = db.PROJECT_SOURCE_CLOUD - disk.ProjectId = projectId - if extProjectId := extDisk.GetProjectId(); len(extProjectId) > 0 { - externalProject, err := ExternalProjectManager.GetProject(extProjectId, storage.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - disk.ProjectId = externalProject.ProjectId - } - } - disk.DiskFormat = extDisk.GetDiskFormat() disk.DiskSize = extDisk.GetDiskSizeMB() disk.AutoDelete = extDisk.GetIsAutoDelete() @@ -1119,6 +1098,8 @@ func (manager *SDiskManager) newFromCloudDisk(ctx context.Context, userCred mccl db.OpsLog.LogEvent(&disk, db.ACT_CREATE, disk.GetShortDesc(ctx), userCred) + SyncCloudProject(userCred, &disk, projectId, extDisk, storage.ManagerId) + return &disk, nil } diff --git a/pkg/compute/models/elasticips.go b/pkg/compute/models/elasticips.go index b81b081b42..814287b0d3 100644 --- a/pkg/compute/models/elasticips.go +++ b/pkg/compute/models/elasticips.go @@ -206,8 +206,9 @@ func (self *SElasticip) GetShortDesc(ctx context.Context) *jsonutils.JSONDict { return desc } -func (manager *SElasticipManager) SyncEips(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, eips []cloudprovider.ICloudEIP, projectId string, projectSync bool) compare.SyncResult { - ownerProjId := getSyncOwnerProjectId(manager, userCred, projectId, projectSync) +func (manager *SElasticipManager) SyncEips(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, eips []cloudprovider.ICloudEIP, projectId string) compare.SyncResult { + ownerProjId := projectId + lockman.LockClass(ctx, manager, ownerProjId) defer lockman.ReleaseClass(ctx, manager, ownerProjId) @@ -248,7 +249,7 @@ func (manager *SElasticipManager) SyncEips(ctx context.Context, userCred mcclien } } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithCloudEip(ctx, userCred, provider, commonext[i], projectId, projectSync) + err = commondb[i].SyncWithCloudEip(ctx, userCred, provider, commonext[i], projectId) if err != nil { syncResult.UpdateError(err) } else { @@ -316,7 +317,7 @@ func (self *SElasticip) SyncInstanceWithCloudEip(ctx context.Context, userCred m return nil } -func (self *SElasticip) SyncWithCloudEip(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, ext cloudprovider.ICloudEIP, projectId string, projectSync bool) error { +func (self *SElasticip) SyncWithCloudEip(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, ext cloudprovider.ICloudEIP, projectId string) error { diff, err := db.UpdateWithLock(ctx, self, func() error { // self.Name = ext.GetName() @@ -327,20 +328,7 @@ func (self *SElasticip) SyncWithCloudEip(ctx context.Context, userCred mcclient. self.ExternalId = ext.GetGlobalId() // self.ManagerId = ext.GetManagerId() self.IsEmulated = ext.IsEmulated() - if projectSync && self.ProjectSrc != db.PROJECT_SOURCE_LOCAL { - self.ProjectSrc = db.PROJECT_SOURCE_CLOUD - if len(projectId) > 0 { - self.ProjectId = projectId - } - if extProjectId := ext.GetProjectId(); len(extProjectId) > 0 { - extProject, err := ExternalProjectManager.GetProject(extProjectId, self.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - self.ProjectId = extProject.ProjectId - } - } - } + self.ChargeType = ext.GetInternetChargeType() factory, _ := provider.GetProviderFactory() @@ -356,6 +344,9 @@ func (self *SElasticip) SyncWithCloudEip(ctx context.Context, userCred mcclient. return err } db.OpsLog.LogSyncUpdate(self, diff, userCred) + + SyncCloudProject(userCred, self, projectId, ext, self.ManagerId) + return nil } @@ -373,18 +364,6 @@ func (manager *SElasticipManager) newFromCloudEip(ctx context.Context, userCred eip.CloudregionId = region.Id eip.ChargeType = extEip.GetInternetChargeType() - eip.ProjectSrc = db.PROJECT_SOURCE_CLOUD - eip.ProjectId = projectId - - if extProjectId := extEip.GetProjectId(); len(extProjectId) > 0 { - externalProject, err := ExternalProjectManager.GetProject(extProjectId, eip.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - eip.ProjectId = externalProject.ProjectId - } - } - err := manager.TableSpec().Insert(&eip) if err != nil { log.Errorf("newFromCloudEip fail %s", err) @@ -392,6 +371,9 @@ func (manager *SElasticipManager) newFromCloudEip(ctx context.Context, userCred } db.OpsLog.LogEvent(&eip, db.ACT_CREATE, eip.GetShortDesc(ctx), userCred) + + SyncCloudProject(userCred, &eip, projectId, extEip, eip.ManagerId) + return &eip, nil } diff --git a/pkg/compute/models/external_projects.go b/pkg/compute/models/external_projects.go index 30dbe140a0..a2a5dd6e4c 100644 --- a/pkg/compute/models/external_projects.go +++ b/pkg/compute/models/external_projects.go @@ -14,6 +14,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/sqlchemy" ) type SExternalProjectManager struct { @@ -75,7 +76,16 @@ func (manager *SExternalProjectManager) getProjectsByProvider(provider *SCloudpr return projects, nil } +func (self *SExternalProject) getCloudProviderInfo() SCloudProviderInfo { + provider := self.GetCloudprovider() + return MakeCloudProviderInfo(nil, nil, provider) +} + func (self *SExternalProject) getMoreDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, extra *jsonutils.JSONDict) *jsonutils.JSONDict { + info := self.getCloudProviderInfo() + + extra.Update(jsonutils.Marshal(&info)) + tenant, err := db.TenantCacheManager.FetchTenantById(ctx, self.ProjectId) if err == nil { extra.Add(jsonutils.NewString(tenant.GetName()), "tenant") @@ -185,7 +195,7 @@ func (manager *SExternalProjectManager) newFromCloudProject(ctx context.Context, project := SExternalProject{} project.SetModelManager(manager) - project.Name = extProject.GetName() + project.Name = db.GenerateName(manager, manager.GetOwnerId(userCred), extProject.GetName()) project.ExternalId = extProject.GetGlobalId() project.IsEmulated = extProject.IsEmulated() project.ManagerId = provider.Id @@ -200,3 +210,19 @@ func (manager *SExternalProjectManager) newFromCloudProject(ctx context.Context, db.OpsLog.LogEvent(&project, db.ACT_CREATE, project.GetShortDesc(ctx), userCred) return &project, nil } + +func (manager *SExternalProjectManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { + var err error + q, err = managedResourceFilterByAccount(q, query, "", nil) + if err != nil { + return nil, err + } + q = managedResourceFilterByCloudType(q, query, "", nil) + + q, err = manager.SStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query) + if err != nil { + return nil, err + } + + return q, nil +} diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 5c50fc8103..35ac4c84e2 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -1829,7 +1829,7 @@ func (self *SGuest) syncRemoveCloudVM(ctx context.Context, userCred mcclient.Tok return self.SetStatus(userCred, VM_UNKNOWN, "Sync lost") } -func (self *SGuest) syncWithCloudVM(ctx context.Context, userCred mcclient.TokenCredential, provider cloudprovider.ICloudProvider, host *SHost, extVM cloudprovider.ICloudVM, projectId string, projectSync bool) error { +func (self *SGuest) syncWithCloudVM(ctx context.Context, userCred mcclient.TokenCredential, provider cloudprovider.ICloudProvider, host *SHost, extVM cloudprovider.ICloudVM, projectId string) error { recycle := false if provider.GetFactory().IsSupportPrepaidResources() && self.IsPrepaidRecycle() { @@ -1870,21 +1870,6 @@ func (self *SGuest) syncWithCloudVM(ctx context.Context, userCred mcclient.Token self.VmemSize = extVM.GetVmemSizeMB() } - if projectSync && self.ProjectSrc != db.PROJECT_SOURCE_LOCAL { - self.ProjectSrc = db.PROJECT_SOURCE_CLOUD - if len(projectId) > 0 { - self.ProjectId = projectId - } - if extProjectId := extVM.GetProjectId(); len(extProjectId) > 0 { - extProject, err := ExternalProjectManager.GetProject(extProjectId, host.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - self.ProjectId = extProject.ProjectId - } - } - } - self.Hypervisor = extVM.GetHypervisor() self.IsEmulated = extVM.IsEmulated() @@ -1923,9 +1908,11 @@ func (self *SGuest) syncWithCloudVM(ctx context.Context, userCred mcclient.Token db.OpsLog.LogSyncUpdate(self, diff, userCred) + SyncCloudProject(userCred, self, projectId, extVM, host.ManagerId) + if provider.GetFactory().IsSupportPrepaidResources() && recycle { vhost := self.GetHost() - err = vhost.syncWithCloudPrepaidVM(extVM, host, projectSync) + err = vhost.syncWithCloudPrepaidVM(extVM, host) if err != nil { return err } @@ -1990,17 +1977,6 @@ func (manager *SGuestManager) newCloudVM(ctx context.Context, userCred mcclient. guest.VmemSize = extVM.GetVmemSizeMB() } - guest.ProjectSrc = db.PROJECT_SOURCE_CLOUD - guest.ProjectId = projectId - if extProjectId := extVM.GetProjectId(); len(extProjectId) > 0 { - externalProject, err := ExternalProjectManager.GetProject(extProjectId, host.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - guest.ProjectId = externalProject.ProjectId - } - } - extraSecgroups := []*SSecurityGroup{} if metaData != nil && metaData.Contains("secgroupIds") { secgroupIds := []string{} @@ -2035,6 +2011,12 @@ func (manager *SGuestManager) newCloudVM(ctx context.Context, userCred mcclient. db.OpsLog.LogEvent(&guest, db.ACT_CREATE, guest.GetShortDesc(ctx), userCred) + SyncCloudProject(userCred, &guest, projectId, extVM, host.ManagerId) + + if guest.Status == VM_RUNNING { + db.OpsLog.LogEvent(&guest, db.ACT_START, guest.GetShortDesc(ctx), userCred) + } + return &guest, nil } @@ -2349,7 +2331,7 @@ type sSyncDiskPair struct { vdisk cloudprovider.ICloudDisk } -func (self *SGuest) SyncVMDisks(ctx context.Context, userCred mcclient.TokenCredential, provider cloudprovider.ICloudProvider, host *SHost, vdisks []cloudprovider.ICloudDisk, projectId string, projectSync bool) compare.SyncResult { +func (self *SGuest) SyncVMDisks(ctx context.Context, userCred mcclient.TokenCredential, provider cloudprovider.ICloudProvider, host *SHost, vdisks []cloudprovider.ICloudDisk, projectId string) compare.SyncResult { result := compare.SyncResult{} newdisks := make([]sSyncDiskPair, 0) @@ -2357,7 +2339,7 @@ func (self *SGuest) SyncVMDisks(ctx context.Context, userCred mcclient.TokenCred if len(vdisks[i].GetGlobalId()) == 0 { continue } - disk, err := DiskManager.syncCloudDisk(ctx, userCred, provider, vdisks[i], i, projectId, projectSync) + disk, err := DiskManager.syncCloudDisk(ctx, userCred, provider, vdisks[i], i, projectId) if err != nil { log.Errorf("syncCloudDisk error: %v", err) result.Error(err) @@ -3701,7 +3683,7 @@ func (self *SGuest) doExternalSync(ctx context.Context, userCred mcclient.TokenC if err != nil { return err } - return self.syncWithCloudVM(ctx, userCred, iprovider, host, iVM, "", false) + return self.syncWithCloudVM(ctx, userCred, iprovider, host, iVM, "") } func (manager *SGuestManager) DeleteExpiredPrepaidServers(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) { @@ -3788,7 +3770,7 @@ func (self *SGuest) SyncVMEip(ctx context.Context, userCred mcclient.TokenCreden } } else { // do nothing - err := eip.SyncWithCloudEip(ctx, userCred, provider, extEip, projectId, false) + err := eip.SyncWithCloudEip(ctx, userCred, provider, extEip, projectId) if err != nil { result.UpdateError(err) } else { diff --git a/pkg/compute/models/host_recycle.go b/pkg/compute/models/host_recycle.go index ae2a495c36..c54e7a9381 100644 --- a/pkg/compute/models/host_recycle.go +++ b/pkg/compute/models/host_recycle.go @@ -639,7 +639,7 @@ func (host *SHost) RebuildRecycledGuest(ctx context.Context, userCred mcclient.T return err } - err = guest.syncWithCloudVM(ctx, userCred, iprovider, &oHost, extVM, "", false) + err = guest.syncWithCloudVM(ctx, userCred, iprovider, &oHost, extVM, "") if err != nil { log.Errorf("guest.syncWithCloudVM fail %s", err) return err @@ -659,7 +659,7 @@ func (host *SHost) RebuildRecycledGuest(ctx context.Context, userCred mcclient.T log.Errorf("disk.SetExternalId fail %s", err) return err } - err = disk.syncWithCloudDisk(ctx, userCred, iprovider, idisks[i], i, "", false) + err = disk.syncWithCloudDisk(ctx, userCred, iprovider, idisks[i], i, guest.ProjectId) if err != nil { log.Errorf("disk.syncWithCloudDisk fail %s", err) return err diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 3bfcd86a42..1bf655a361 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -1255,7 +1255,7 @@ func (manager *SHostManager) getHostsByZoneProvider(zone *SZone, provider *SClou return hosts, nil } -func (manager *SHostManager) SyncHosts(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, zone *SZone, hosts []cloudprovider.ICloudHost, projectSync bool) ([]SHost, []cloudprovider.ICloudHost, compare.SyncResult) { +func (manager *SHostManager) SyncHosts(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, zone *SZone, hosts []cloudprovider.ICloudHost) ([]SHost, []cloudprovider.ICloudHost, compare.SyncResult) { lockman.LockClass(ctx, manager, manager.GetOwnerId(userCred)) defer lockman.ReleaseClass(ctx, manager, manager.GetOwnerId(userCred)) @@ -1292,7 +1292,7 @@ func (manager *SHostManager) SyncHosts(ctx context.Context, userCred mcclient.To } } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].syncWithCloudHost(ctx, userCred, commonext[i], projectSync) + err = commondb[i].syncWithCloudHost(ctx, userCred, commonext[i]) if err != nil { syncResult.UpdateError(err) } else { @@ -1333,7 +1333,7 @@ func (self *SHost) syncRemoveCloudHost(ctx context.Context, userCred mcclient.To return err } -func (self *SHost) syncWithCloudHost(ctx context.Context, userCred mcclient.TokenCredential, extHost cloudprovider.ICloudHost, projectSync bool) error { +func (self *SHost) syncWithCloudHost(ctx context.Context, userCred mcclient.TokenCredential, extHost cloudprovider.ICloudHost) error { diff, err := db.UpdateWithLock(ctx, self, func() error { // self.Name = extHost.GetName() @@ -1368,16 +1368,14 @@ func (self *SHost) syncWithCloudHost(ctx context.Context, userCred mcclient.Toke db.OpsLog.LogSyncUpdate(self, diff, userCred) - if projectSync { - if err := HostManager.ClearSchedDescCache(self.Id); err != nil { - log.Errorf("ClearSchedDescCache for host %s error %v", self.Name, err) - } + if err := HostManager.ClearSchedDescCache(self.Id); err != nil { + log.Errorf("ClearSchedDescCache for host %s error %v", self.Name, err) } return nil } -func (self *SHost) syncWithCloudPrepaidVM(extVM cloudprovider.ICloudVM, host *SHost, projectSync bool) error { +func (self *SHost) syncWithCloudPrepaidVM(extVM cloudprovider.ICloudVM, host *SHost) error { _, err := self.SaveUpdates(func() error { self.CpuCount = extVM.GetVcpuCount() @@ -1394,10 +1392,8 @@ func (self *SHost) syncWithCloudPrepaidVM(extVM cloudprovider.ICloudVM, host *SH log.Errorf("syncWithCloudZone error %s", err) } - if projectSync { - if err := HostManager.ClearSchedDescCache(self.Id); err != nil { - log.Errorf("ClearSchedDescCache for host %s error %v", self.Name, err) - } + if err := HostManager.ClearSchedDescCache(self.Id); err != nil { + log.Errorf("ClearSchedDescCache for host %s error %v", self.Name, err) } return err @@ -1710,8 +1706,9 @@ func (self *SHost) newCloudHostWire(ctx context.Context, userCred mcclient.Token return err } -func (self *SHost) SyncHostVMs(ctx context.Context, userCred mcclient.TokenCredential, iprovider cloudprovider.ICloudProvider, vms []cloudprovider.ICloudVM, projectId string, projectSync bool) ([]SGuest, []cloudprovider.ICloudVM, compare.SyncResult) { - syncOwnerId := getSyncOwnerProjectId(GuestManager, userCred, projectId, projectSync) +func (self *SHost) SyncHostVMs(ctx context.Context, userCred mcclient.TokenCredential, iprovider cloudprovider.ICloudProvider, vms []cloudprovider.ICloudVM, projectId string) ([]SGuest, []cloudprovider.ICloudVM, compare.SyncResult) { + syncOwnerId := projectId + lockman.LockClass(ctx, GuestManager, syncOwnerId) defer lockman.ReleaseClass(ctx, GuestManager, syncOwnerId) @@ -1749,7 +1746,7 @@ func (self *SHost) SyncHostVMs(ctx context.Context, userCred mcclient.TokenCrede } for i := 0; i < len(commondb); i += 1 { - err := commondb[i].syncWithCloudVM(ctx, userCred, iprovider, self, commonext[i], syncOwnerId, projectSync) + err := commondb[i].syncWithCloudVM(ctx, userCred, iprovider, self, commonext[i], syncOwnerId) if err != nil { syncResult.UpdateError(err) } else { diff --git a/pkg/compute/models/loadbalanceracls.go b/pkg/compute/models/loadbalanceracls.go index 478b51c37f..57e4e40842 100644 --- a/pkg/compute/models/loadbalanceracls.go +++ b/pkg/compute/models/loadbalanceracls.go @@ -321,7 +321,8 @@ func (man *SLoadbalancerAclManager) getLoadbalancerAclsByRegion(region *SCloudre } func (man *SLoadbalancerAclManager) SyncLoadbalancerAcls(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, acls []cloudprovider.ICloudLoadbalancerAcl, syncRange *SSyncRange) compare.SyncResult { - ownerProjId := getSyncOwnerProjectId(man, userCred, provider.ProjectId, syncRange.ProjectSync) + ownerProjId := provider.ProjectId + lockman.LockClass(ctx, man, ownerProjId) defer lockman.ReleaseClass(ctx, man, ownerProjId) @@ -353,7 +354,7 @@ func (man *SLoadbalancerAclManager) SyncLoadbalancerAcls(ctx context.Context, us } } for i := 0; i < len(commondb); i++ { - err = commondb[i].SyncWithCloudLoadbalancerAcl(ctx, userCred, commonext[i], provider.ProjectId, syncRange.ProjectSync) + err = commondb[i].SyncWithCloudLoadbalancerAcl(ctx, userCred, commonext[i], provider.ProjectId) if err != nil { syncResult.UpdateError(err) } else { @@ -395,51 +396,38 @@ func (man *SLoadbalancerAclManager) newFromCloudLoadbalancerAcl(ctx context.Cont acl.ManagerId = provider.Id acl.CloudregionId = region.Id - acl.ProjectSrc = db.PROJECT_SOURCE_CLOUD - acl.ProjectId = projectId - - if extProjectId := extAcl.GetProjectId(); len(extProjectId) > 0 { - externalProject, err := ExternalProjectManager.GetProject(extProjectId, acl.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - acl.ProjectId = externalProject.ProjectId - } - } - acl.AclEntries = &SLoadbalancerAclEntries{} for _, entry := range extAcl.GetAclEntries() { *acl.AclEntries = append(*acl.AclEntries, &SLoadbalancerAclEntry{Cidr: entry.CIDR, Comment: entry.Comment}) } - return &acl, man.TableSpec().Insert(&acl) + err := man.TableSpec().Insert(&acl) + if err != nil { + log.Errorf("newFromCloudLoadbalancerAcl fail %s", err) + return nil, err + } + + db.OpsLog.LogEvent(&acl, db.ACT_CREATE, acl.GetShortDesc(ctx), userCred) + + SyncCloudProject(userCred, &acl, projectId, extAcl, acl.ManagerId) + + return &acl, nil } -func (acl *SLoadbalancerAcl) SyncWithCloudLoadbalancerAcl(ctx context.Context, userCred mcclient.TokenCredential, extAcl cloudprovider.ICloudLoadbalancerAcl, projectId string, projectSync bool) error { +func (acl *SLoadbalancerAcl) SyncWithCloudLoadbalancerAcl(ctx context.Context, userCred mcclient.TokenCredential, extAcl cloudprovider.ICloudLoadbalancerAcl, projectId string) error { diff, err := db.UpdateWithLock(ctx, acl, func() error { acl.Name = extAcl.GetName() acl.AclEntries = &SLoadbalancerAclEntries{} for _, entry := range extAcl.GetAclEntries() { *acl.AclEntries = append(*acl.AclEntries, &SLoadbalancerAclEntry{Cidr: entry.CIDR, Comment: entry.Comment}) } - if projectSync && acl.ProjectSrc != db.PROJECT_SOURCE_LOCAL { - acl.ProjectSrc = db.PROJECT_SOURCE_CLOUD - if len(projectId) > 0 { - acl.ProjectId = projectId - } - if extProjectId := extAcl.GetProjectId(); len(extProjectId) > 0 { - extProject, err := ExternalProjectManager.GetProject(extProjectId, acl.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - acl.ProjectId = extProject.ProjectId - } - } - } return nil }) if err != nil { return err } db.OpsLog.LogSyncUpdate(acl, diff, userCred) + + SyncCloudProject(userCred, acl, projectId, extAcl, acl.ManagerId) + return nil } diff --git a/pkg/compute/models/loadbalancerbackendgroups.go b/pkg/compute/models/loadbalancerbackendgroups.go index e9b8827845..93deaeda5e 100644 --- a/pkg/compute/models/loadbalancerbackendgroups.go +++ b/pkg/compute/models/loadbalancerbackendgroups.go @@ -327,7 +327,8 @@ func (man *SLoadbalancerBackendGroupManager) getLoadbalancerBackendgroupsByLoadb } func (man *SLoadbalancerBackendGroupManager) SyncLoadbalancerBackendgroups(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, lb *SLoadbalancer, lbbgs []cloudprovider.ICloudLoadbalancerBackendGroup, syncRange *SSyncRange) ([]SLoadbalancerBackendGroup, []cloudprovider.ICloudLoadbalancerBackendGroup, compare.SyncResult) { - syncOwnerId := getSyncOwnerProjectId(man, userCred, provider.ProjectId, syncRange.ProjectSync) + syncOwnerId := provider.ProjectId + lockman.LockClass(ctx, man, syncOwnerId) defer lockman.ReleaseClass(ctx, man, syncOwnerId) @@ -361,7 +362,7 @@ func (man *SLoadbalancerBackendGroupManager) SyncLoadbalancerBackendgroups(ctx c } } for i := 0; i < len(commondb); i++ { - err = commondb[i].SyncWithCloudLoadbalancerBackendgroup(ctx, userCred, lb, commonext[i], provider.ProjectId, syncRange.ProjectSync) + err = commondb[i].SyncWithCloudLoadbalancerBackendgroup(ctx, userCred, lb, commonext[i], provider.ProjectId) if err != nil { syncResult.UpdateError(err) } else { @@ -408,25 +409,10 @@ func (lbbg *SLoadbalancerBackendGroup) syncRemoveCloudLoadbalancerBackendgroup(c return err } -func (lbbg *SLoadbalancerBackendGroup) SyncWithCloudLoadbalancerBackendgroup(ctx context.Context, userCred mcclient.TokenCredential, lb *SLoadbalancer, extLoadbalancerBackendgroup cloudprovider.ICloudLoadbalancerBackendGroup, projectId string, projectSync bool) error { +func (lbbg *SLoadbalancerBackendGroup) SyncWithCloudLoadbalancerBackendgroup(ctx context.Context, userCred mcclient.TokenCredential, lb *SLoadbalancer, extLoadbalancerBackendgroup cloudprovider.ICloudLoadbalancerBackendGroup, projectId string) error { diff, err := db.UpdateWithLock(ctx, lbbg, func() error { lbbg.Type = extLoadbalancerBackendgroup.GetType() lbbg.Status = extLoadbalancerBackendgroup.GetStatus() - - if projectSync && lbbg.ProjectSrc != db.PROJECT_SOURCE_LOCAL { - lbbg.ProjectSrc = db.PROJECT_SOURCE_CLOUD - if len(projectId) > 0 { - lbbg.ProjectId = projectId - } - if extProjectId := extLoadbalancerBackendgroup.GetProjectId(); len(extProjectId) > 0 { - extProject, err := ExternalProjectManager.GetProject(extProjectId, lb.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - lbbg.ProjectId = extProject.ProjectId - } - } - } return nil }) if err != nil { @@ -434,6 +420,8 @@ func (lbbg *SLoadbalancerBackendGroup) SyncWithCloudLoadbalancerBackendgroup(ctx } db.OpsLog.LogSyncUpdate(lbbg, diff, userCred) + SyncCloudProject(userCred, lbbg, projectId, extLoadbalancerBackendgroup, lb.ManagerId) + if extLoadbalancerBackendgroup.IsDefault() { diff, err := db.UpdateWithLock(ctx, lb, func() error { lb.BackendGroupId = lbbg.Id @@ -469,18 +457,6 @@ func (man *SLoadbalancerBackendGroupManager) newFromCloudLoadbalancerBackendgrou lbbg.Type = extLoadbalancerBackendgroup.GetType() lbbg.Status = extLoadbalancerBackendgroup.GetStatus() - lbbg.ProjectSrc = db.PROJECT_SOURCE_CLOUD - lbbg.ProjectId = projectId - - if extProjectId := extLoadbalancerBackendgroup.GetProjectId(); len(extProjectId) > 0 { - externalProject, err := ExternalProjectManager.GetProject(extProjectId, lb.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - lbbg.ProjectId = externalProject.ProjectId - } - } - err := man.TableSpec().Insert(lbbg) if err != nil { return nil, err @@ -488,6 +464,8 @@ func (man *SLoadbalancerBackendGroupManager) newFromCloudLoadbalancerBackendgrou db.OpsLog.LogEvent(lbbg, db.ACT_CREATE, lbbg.GetShortDesc(ctx), userCred) + SyncCloudProject(userCred, lbbg, projectId, extLoadbalancerBackendgroup, lb.ManagerId) + if extLoadbalancerBackendgroup.IsDefault() { _, err := db.Update(lb, func() error { lb.BackendGroupId = lbbg.Id diff --git a/pkg/compute/models/loadbalancerbackends.go b/pkg/compute/models/loadbalancerbackends.go index 95548a8568..2ef3b42b5d 100644 --- a/pkg/compute/models/loadbalancerbackends.go +++ b/pkg/compute/models/loadbalancerbackends.go @@ -272,7 +272,8 @@ func (lbb *SLoadbalancerBackend) ValidateDeleteCondition(ctx context.Context) er } func (man *SLoadbalancerBackendManager) SyncLoadbalancerBackends(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, loadbalancerBackendgroup *SLoadbalancerBackendGroup, lbbs []cloudprovider.ICloudLoadbalancerBackend, syncRange *SSyncRange) compare.SyncResult { - syncOwnerId := getSyncOwnerProjectId(man, userCred, provider.ProjectId, syncRange.ProjectSync) + syncOwnerId := provider.ProjectId + lockman.LockClass(ctx, man, syncOwnerId) defer lockman.ReleaseClass(ctx, man, syncOwnerId) @@ -304,7 +305,7 @@ func (man *SLoadbalancerBackendManager) SyncLoadbalancerBackends(ctx context.Con } } for i := 0; i < len(commondb); i++ { - err = commondb[i].SyncWithCloudLoadbalancerBackend(ctx, userCred, commonext[i], provider.ProjectId, syncRange.ProjectSync) + err = commondb[i].SyncWithCloudLoadbalancerBackend(ctx, userCred, commonext[i], provider.ProjectId) if err != nil { syncResult.UpdateError(err) } else { @@ -362,28 +363,17 @@ func (lbb *SLoadbalancerBackend) syncRemoveCloudLoadbalancerBackend(ctx context. return err } -func (lbb *SLoadbalancerBackend) SyncWithCloudLoadbalancerBackend(ctx context.Context, userCred mcclient.TokenCredential, extLoadbalancerBackend cloudprovider.ICloudLoadbalancerBackend, projectId string, projectSync bool) error { +func (lbb *SLoadbalancerBackend) SyncWithCloudLoadbalancerBackend(ctx context.Context, userCred mcclient.TokenCredential, extLoadbalancerBackend cloudprovider.ICloudLoadbalancerBackend, projectId string) error { diff, err := db.UpdateWithLock(ctx, lbb, func() error { - if projectSync && lbb.ProjectSrc != db.PROJECT_SOURCE_LOCAL { - lbb.ProjectSrc = db.PROJECT_SOURCE_CLOUD - if len(projectId) > 0 { - lbb.ProjectId = projectId - } - if extProjectId := extLoadbalancerBackend.GetProjectId(); len(extProjectId) > 0 { - extProject, err := ExternalProjectManager.GetProject(extProjectId, lbb.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - lbb.ProjectId = extProject.ProjectId - } - } - } return lbb.constructFieldsFromCloudLoadbalancerBackend(extLoadbalancerBackend) }) if err != nil { return err } db.OpsLog.LogSyncUpdate(lbb, diff, userCred) + + SyncCloudProject(userCred, lbb, projectId, extLoadbalancerBackend, lbb.ManagerId) + return nil } @@ -403,17 +393,6 @@ func (man *SLoadbalancerBackendManager) newFromCloudLoadbalancerBackend(ctx cont return nil, err } - lbb.ProjectSrc = db.PROJECT_SOURCE_CLOUD - lbb.ProjectId = projectId - if extProjectId := extLoadbalancerBackend.GetProjectId(); len(extProjectId) > 0 { - externalProject, err := ExternalProjectManager.GetProject(extProjectId, loadbalancerBackendgroup.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - lbb.ProjectId = externalProject.ProjectId - } - } - err := man.TableSpec().Insert(lbb) if err != nil { @@ -422,6 +401,8 @@ func (man *SLoadbalancerBackendManager) newFromCloudLoadbalancerBackend(ctx cont db.OpsLog.LogEvent(lbb, db.ACT_CREATE, lbb.GetShortDesc(ctx), userCred) + SyncCloudProject(userCred, lbb, projectId, extLoadbalancerBackend, loadbalancerBackendgroup.ManagerId) + return lbb, nil } diff --git a/pkg/compute/models/loadbalancercertificates.go b/pkg/compute/models/loadbalancercertificates.go index 81e12b6ec9..6d0dd7be43 100644 --- a/pkg/compute/models/loadbalancercertificates.go +++ b/pkg/compute/models/loadbalancercertificates.go @@ -301,7 +301,8 @@ func (man *SLoadbalancerCertificateManager) getLoadbalancerCertificatesByRegion( } func (man *SLoadbalancerCertificateManager) SyncLoadbalancerCertificates(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, certificates []cloudprovider.ICloudLoadbalancerCertificate, syncRange *SSyncRange) compare.SyncResult { - ownerProjId := getSyncOwnerProjectId(man, userCred, provider.ProjectId, syncRange.ProjectSync) + ownerProjId := provider.ProjectId + lockman.LockClass(ctx, man, ownerProjId) defer lockman.ReleaseClass(ctx, man, ownerProjId) @@ -333,7 +334,7 @@ func (man *SLoadbalancerCertificateManager) SyncLoadbalancerCertificates(ctx con } } for i := 0; i < len(commondb); i++ { - err = commondb[i].SyncWithCloudLoadbalancerCertificate(ctx, userCred, commonext[i], provider.ProjectId, syncRange.ProjectSync) + err = commondb[i].SyncWithCloudLoadbalancerCertificate(ctx, userCred, commonext[i], provider.ProjectId) if err != nil { syncResult.UpdateError(err) } else { @@ -367,19 +368,17 @@ func (man *SLoadbalancerCertificateManager) newFromCloudLoadbalancerCertificate( lbcert.Fingerprint = extCertificate.GetFingerprint() lbcert.NotAfter = extCertificate.GetExpireTime() - lbcert.ProjectSrc = db.PROJECT_SOURCE_CLOUD - lbcert.ProjectId = projectId - - if extProjectId := extCertificate.GetProjectId(); len(extProjectId) > 0 { - externalProject, err := ExternalProjectManager.GetProject(extProjectId, lbcert.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - lbcert.ProjectId = externalProject.ProjectId - } + err := man.TableSpec().Insert(&lbcert) + if err != nil { + log.Errorf("newFromCloudLoadbalancerCertificate fail %s", err) + return nil, err } - return &lbcert, man.TableSpec().Insert(&lbcert) + db.OpsLog.LogEvent(&lbcert, db.ACT_CREATE, lbcert.GetShortDesc(ctx), userCred) + + SyncCloudProject(userCred, &lbcert, projectId, extCertificate, lbcert.ManagerId) + + return &lbcert, nil } func (lbcert *SLoadbalancerCertificate) syncRemoveCloudLoadbalancerCertificate(ctx context.Context, userCred mcclient.TokenCredential) error { @@ -395,34 +394,21 @@ func (lbcert *SLoadbalancerCertificate) syncRemoveCloudLoadbalancerCertificate(c return err } -func (lbcert *SLoadbalancerCertificate) SyncWithCloudLoadbalancerCertificate(ctx context.Context, userCred mcclient.TokenCredential, extCertificate cloudprovider.ICloudLoadbalancerCertificate, projectId string, projectSync bool) error { +func (lbcert *SLoadbalancerCertificate) SyncWithCloudLoadbalancerCertificate(ctx context.Context, userCred mcclient.TokenCredential, extCertificate cloudprovider.ICloudLoadbalancerCertificate, projectId string) error { diff, err := db.UpdateWithLock(ctx, lbcert, func() error { lbcert.Name = extCertificate.GetName() lbcert.CommonName = extCertificate.GetCommonName() lbcert.SubjectAlternativeNames = extCertificate.GetSubjectAlternativeNames() lbcert.Fingerprint = extCertificate.GetFingerprint() lbcert.NotAfter = extCertificate.GetExpireTime() - - if projectSync && lbcert.ProjectSrc != db.PROJECT_SOURCE_LOCAL { - lbcert.ProjectSrc = db.PROJECT_SOURCE_CLOUD - if len(projectId) > 0 { - lbcert.ProjectId = projectId - } - if extProjectId := extCertificate.GetProjectId(); len(extProjectId) > 0 { - extProject, err := ExternalProjectManager.GetProject(extProjectId, lbcert.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - lbcert.ProjectId = extProject.ProjectId - } - } - } - return nil }) if err != nil { return err } db.OpsLog.LogSyncUpdate(lbcert, diff, userCred) + + SyncCloudProject(userCred, lbcert, projectId, extCertificate, lbcert.ManagerId) + return nil } diff --git a/pkg/compute/models/loadbalancerlistenerrules.go b/pkg/compute/models/loadbalancerlistenerrules.go index 4b2f9f1d7d..1c7d5e1b17 100644 --- a/pkg/compute/models/loadbalancerlistenerrules.go +++ b/pkg/compute/models/loadbalancerlistenerrules.go @@ -283,7 +283,8 @@ func (man *SLoadbalancerListenerRuleManager) getLoadbalancerListenerRulesByListe } func (man *SLoadbalancerListenerRuleManager) SyncLoadbalancerListenerRules(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, listener *SLoadbalancerListener, rules []cloudprovider.ICloudLoadbalancerListenerRule, syncRange *SSyncRange) compare.SyncResult { - syncOwnerId := getSyncOwnerProjectId(man, userCred, provider.ProjectId, syncRange.ProjectSync) + syncOwnerId := provider.ProjectId + lockman.LockClass(ctx, man, syncOwnerId) defer lockman.ReleaseClass(ctx, man, syncOwnerId) @@ -315,7 +316,7 @@ func (man *SLoadbalancerListenerRuleManager) SyncLoadbalancerListenerRules(ctx c } } for i := 0; i < len(commondb); i++ { - err = commondb[i].SyncWithCloudLoadbalancerListenerRule(ctx, userCred, commonext[i], provider.ProjectId, syncRange.ProjectSync) + err = commondb[i].SyncWithCloudLoadbalancerListenerRule(ctx, userCred, commonext[i], provider.ProjectId) if err != nil { syncResult.UpdateError(err) } else { @@ -370,10 +371,18 @@ func (man *SLoadbalancerListenerRuleManager) newFromCloudLoadbalancerListenerRul lbr.Name = db.GenerateName(man, projectId, extRule.GetName()) lbr.constructFieldsFromCloudListenerRule(userCred, extRule) - lbr.ProjectSrc = listener.ProjectSrc - lbr.ProjectId = listener.ProjectId + err := man.TableSpec().Insert(lbr) - return lbr, man.TableSpec().Insert(lbr) + if err != nil { + log.Errorf("newFromCloudLoadbalancerListenerRule fail %s", err) + return nil, err + } + + db.OpsLog.LogEvent(lbr, db.ACT_CREATE, lbr.GetShortDesc(ctx), userCred) + + SyncCloudProject(userCred, lbr, projectId, extRule, listener.ManagerId) + + return lbr, nil } func (lbr *SLoadbalancerListenerRule) syncRemoveCloudLoadbalancerListenerRule(ctx context.Context, userCred mcclient.TokenCredential) error { @@ -389,19 +398,20 @@ func (lbr *SLoadbalancerListenerRule) syncRemoveCloudLoadbalancerListenerRule(ct return err } -func (lbr *SLoadbalancerListenerRule) SyncWithCloudLoadbalancerListenerRule(ctx context.Context, userCred mcclient.TokenCredential, extRule cloudprovider.ICloudLoadbalancerListenerRule, projectId string, projectSync bool) error { +func (lbr *SLoadbalancerListenerRule) SyncWithCloudLoadbalancerListenerRule(ctx context.Context, userCred mcclient.TokenCredential, extRule cloudprovider.ICloudLoadbalancerListenerRule, projectId string) error { listener := lbr.GetLoadbalancerListener() diff, err := db.UpdateWithLock(ctx, lbr, func() error { lbr.constructFieldsFromCloudListenerRule(userCred, extRule) lbr.ManagerId = listener.ManagerId - lbr.ProjectSrc = listener.ProjectSrc - lbr.ProjectId = listener.ProjectId return nil }) if err != nil { return err } db.OpsLog.LogSyncUpdate(lbr, diff, userCred) + + SyncCloudProject(userCred, lbr, projectId, extRule, listener.ManagerId) + return nil } diff --git a/pkg/compute/models/loadbalancerlisteners.go b/pkg/compute/models/loadbalancerlisteners.go index 705be1d2e1..14232c80ae 100644 --- a/pkg/compute/models/loadbalancerlisteners.go +++ b/pkg/compute/models/loadbalancerlisteners.go @@ -673,7 +673,8 @@ func (man *SLoadbalancerListenerManager) getLoadbalancerListenersByLoadbalancer( } func (man *SLoadbalancerListenerManager) SyncLoadbalancerListeners(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, lb *SLoadbalancer, listeners []cloudprovider.ICloudLoadbalancerListener, syncRange *SSyncRange) ([]SLoadbalancerListener, []cloudprovider.ICloudLoadbalancerListener, compare.SyncResult) { - syncOwnerId := getSyncOwnerProjectId(man, userCred, provider.ProjectId, syncRange.ProjectSync) + syncOwnerId := provider.ProjectId + lockman.LockClass(ctx, man, syncOwnerId) defer lockman.ReleaseClass(ctx, man, syncOwnerId) @@ -707,7 +708,7 @@ func (man *SLoadbalancerListenerManager) SyncLoadbalancerListeners(ctx context.C } } for i := 0; i < len(commondb); i++ { - err = commondb[i].SyncWithCloudLoadbalancerListener(ctx, userCred, lb, commonext[i], provider.ProjectId, syncRange.ProjectSync) + err = commondb[i].SyncWithCloudLoadbalancerListener(ctx, userCred, lb, commonext[i], provider.ProjectId) if err != nil { syncResult.UpdateError(err) } else { @@ -804,23 +805,9 @@ func (lblis *SLoadbalancerListener) syncRemoveCloudLoadbalancerListener(ctx cont return err } -func (lblis *SLoadbalancerListener) SyncWithCloudLoadbalancerListener(ctx context.Context, userCred mcclient.TokenCredential, lb *SLoadbalancer, extListener cloudprovider.ICloudLoadbalancerListener, projectId string, projectSync bool) error { +func (lblis *SLoadbalancerListener) SyncWithCloudLoadbalancerListener(ctx context.Context, userCred mcclient.TokenCredential, lb *SLoadbalancer, extListener cloudprovider.ICloudLoadbalancerListener, projectId string) error { diff, err := db.UpdateWithLock(ctx, lblis, func() error { lblis.constructFieldsFromCloudListener(userCred, lb, extListener) - if projectSync && lblis.ProjectSrc != db.PROJECT_SOURCE_LOCAL { - lblis.ProjectSrc = db.PROJECT_SOURCE_CLOUD - if len(projectId) > 0 { - lblis.ProjectId = projectId - } - if extProjectId := extListener.GetProjectId(); len(extProjectId) > 0 { - extProject, err := ExternalProjectManager.GetProject(extProjectId, lblis.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - lblis.ProjectId = extProject.ProjectId - } - } - } return nil }) if err != nil { @@ -829,6 +816,8 @@ func (lblis *SLoadbalancerListener) SyncWithCloudLoadbalancerListener(ctx contex db.OpsLog.LogSyncUpdate(lblis, diff, userCred) + SyncCloudProject(userCred, lblis, projectId, extListener, lblis.ManagerId) + return nil } @@ -843,18 +832,6 @@ func (man *SLoadbalancerListenerManager) newFromCloudLoadbalancerListener(ctx co lblis.constructFieldsFromCloudListener(userCred, lb, extListener) - lblis.ProjectSrc = db.PROJECT_SOURCE_CLOUD - lblis.ProjectId = projectId - - if extProjectId := extListener.GetProjectId(); len(extProjectId) > 0 { - externalProject, err := ExternalProjectManager.GetProject(extProjectId, lblis.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - lblis.ProjectId = externalProject.ProjectId - } - } - err := man.TableSpec().Insert(lblis) if err != nil { return nil, err @@ -862,6 +839,8 @@ func (man *SLoadbalancerListenerManager) newFromCloudLoadbalancerListener(ctx co db.OpsLog.LogEvent(lblis, db.ACT_CREATE, lblis.GetShortDesc(ctx), userCred) + SyncCloudProject(userCred, lblis, projectId, extListener, lblis.ManagerId) + return lblis, nil } diff --git a/pkg/compute/models/loadbalancers.go b/pkg/compute/models/loadbalancers.go index 2e0e8d723f..f9b00cbf5a 100644 --- a/pkg/compute/models/loadbalancers.go +++ b/pkg/compute/models/loadbalancers.go @@ -458,7 +458,8 @@ func (man *SLoadbalancerManager) getLoadbalancersByRegion(region *SCloudregion, } func (man *SLoadbalancerManager) SyncLoadbalancers(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, lbs []cloudprovider.ICloudLoadbalancer, syncRange *SSyncRange) ([]SLoadbalancer, []cloudprovider.ICloudLoadbalancer, compare.SyncResult) { - ownerProjId := getSyncOwnerProjectId(man, userCred, provider.ProjectId, syncRange.ProjectSync) + ownerProjId := provider.ProjectId + lockman.LockClass(ctx, man, ownerProjId) defer lockman.ReleaseClass(ctx, man, ownerProjId) @@ -492,7 +493,7 @@ func (man *SLoadbalancerManager) SyncLoadbalancers(ctx context.Context, userCred } } for i := 0; i < len(commondb); i++ { - err = commondb[i].SyncWithCloudLoadbalancer(ctx, userCred, commonext[i], provider.ProjectId, syncRange.ProjectSync) + err = commondb[i].SyncWithCloudLoadbalancer(ctx, userCred, commonext[i], provider.ProjectId) if err != nil { syncResult.UpdateError(err) } else { @@ -546,18 +547,6 @@ func (man *SLoadbalancerManager) newFromCloudLoadbalancer(ctx context.Context, u } } - lb.ProjectSrc = db.PROJECT_SOURCE_CLOUD - lb.ProjectId = projectId - - if extProjectId := extLb.GetProjectId(); len(extProjectId) > 0 { - externalProject, err := ExternalProjectManager.GetProject(extProjectId, lb.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - lb.ProjectId = externalProject.ProjectId - } - } - if extLb.GetMetadata() != nil { lb.LBInfo = extLb.GetMetadata() } @@ -569,6 +558,8 @@ func (man *SLoadbalancerManager) newFromCloudLoadbalancer(ctx context.Context, u db.OpsLog.LogEvent(&lb, db.ACT_CREATE, lb.GetShortDesc(ctx), userCred) + SyncCloudProject(userCred, &lb, projectId, extLb, lb.ManagerId) + lb.syncLoadbalancerNetwork(ctx, userCred) return &lb, nil } @@ -600,7 +591,7 @@ func (lb *SLoadbalancer) syncLoadbalancerNetwork(ctx context.Context, userCred m } } -func (lb *SLoadbalancer) SyncWithCloudLoadbalancer(ctx context.Context, userCred mcclient.TokenCredential, extLb cloudprovider.ICloudLoadbalancer, projectId string, projectSync bool) error { +func (lb *SLoadbalancer) SyncWithCloudLoadbalancer(ctx context.Context, userCred mcclient.TokenCredential, extLb cloudprovider.ICloudLoadbalancer, projectId string) error { lockman.LockObject(ctx, lb) defer lockman.ReleaseObject(ctx, lb) @@ -615,25 +606,13 @@ func (lb *SLoadbalancer) SyncWithCloudLoadbalancer(ctx context.Context, userCred lb.LBInfo = extLb.GetMetadata() } - if projectSync && lb.ProjectSrc != db.PROJECT_SOURCE_LOCAL { - lb.ProjectSrc = db.PROJECT_SOURCE_CLOUD - if len(projectId) > 0 { - lb.ProjectId = projectId - } - if extProjectId := extLb.GetProjectId(); len(extProjectId) > 0 { - extProject, err := ExternalProjectManager.GetProject(extProjectId, lb.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - lb.ProjectId = extProject.ProjectId - } - } - } return nil }) db.OpsLog.LogSyncUpdate(lb, diff, userCred) + SyncCloudProject(userCred, lb, projectId, extLb, lb.ManagerId) + lb.syncLoadbalancerNetwork(ctx, userCred) return err diff --git a/pkg/compute/models/networks.go b/pkg/compute/models/networks.go index a8c38d40a7..2ad4ad7894 100644 --- a/pkg/compute/models/networks.go +++ b/pkg/compute/models/networks.go @@ -455,8 +455,9 @@ func (manager *SNetworkManager) getNetworksByWire(wire *SWire) ([]SNetwork, erro return nets, nil } -func (manager *SNetworkManager) SyncNetworks(ctx context.Context, userCred mcclient.TokenCredential, wire *SWire, nets []cloudprovider.ICloudNetwork, projectId string, projectSync bool) ([]SNetwork, []cloudprovider.ICloudNetwork, compare.SyncResult) { - ownerProjId := getSyncOwnerProjectId(manager, userCred, projectId, projectSync) +func (manager *SNetworkManager) SyncNetworks(ctx context.Context, userCred mcclient.TokenCredential, wire *SWire, nets []cloudprovider.ICloudNetwork, projectId string) ([]SNetwork, []cloudprovider.ICloudNetwork, compare.SyncResult) { + ownerProjId := projectId + lockman.LockClass(ctx, manager, ownerProjId) defer lockman.ReleaseClass(ctx, manager, ownerProjId) @@ -497,7 +498,7 @@ func (manager *SNetworkManager) SyncNetworks(ctx context.Context, userCred mccli } } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithCloudNetwork(ctx, userCred, commonext[i], projectId, projectSync) + err = commondb[i].SyncWithCloudNetwork(ctx, userCred, commonext[i], projectId) if err != nil { syncResult.UpdateError(err) } else { @@ -536,7 +537,7 @@ func (self *SNetwork) syncRemoveCloudNetwork(ctx context.Context, userCred mccli return err } -func (self *SNetwork) SyncWithCloudNetwork(ctx context.Context, userCred mcclient.TokenCredential, extNet cloudprovider.ICloudNetwork, projectId string, projectSync bool) error { +func (self *SNetwork) SyncWithCloudNetwork(ctx context.Context, userCred mcclient.TokenCredential, extNet cloudprovider.ICloudNetwork, projectId string) error { vpc := self.GetWire().getVpc() diff, err := db.UpdateWithLock(ctx, self, func() error { extNet.Refresh() @@ -551,20 +552,6 @@ func (self *SNetwork) SyncWithCloudNetwork(ctx context.Context, userCred mcclien self.AllocTimoutSeconds = extNet.GetAllocTimeoutSeconds() - if projectSync && self.ProjectSrc != db.PROJECT_SOURCE_LOCAL { - self.ProjectSrc = db.PROJECT_SOURCE_CLOUD - if len(projectId) > 0 { - self.ProjectId = projectId - } - if extProjectId := extNet.GetProjectId(); len(extProjectId) > 0 { - extProject, err := ExternalProjectManager.GetProject(extProjectId, vpc.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - self.ProjectId = extProject.ProjectId - } - } - } return nil }) if err != nil { @@ -572,6 +559,9 @@ func (self *SNetwork) SyncWithCloudNetwork(ctx context.Context, userCred mcclien return err } db.OpsLog.LogSyncUpdate(self, diff, userCred) + + SyncCloudProject(userCred, self, projectId, extNet, vpc.ManagerId) + return nil } @@ -592,19 +582,6 @@ func (manager *SNetworkManager) newFromCloudNetwork(ctx context.Context, userCre net.AllocTimoutSeconds = extNet.GetAllocTimeoutSeconds() - net.ProjectSrc = db.PROJECT_SOURCE_CLOUD - net.ProjectId = projectId - - vpc := wire.getVpc() - if extProjectId := extNet.GetProjectId(); len(extProjectId) > 0 { - externalProject, err := ExternalProjectManager.GetProject(extProjectId, vpc.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - net.ProjectId = externalProject.ProjectId - } - } - err := manager.TableSpec().Insert(&net) if err != nil { log.Errorf("newFromCloudZone fail %s", err) @@ -613,6 +590,9 @@ func (manager *SNetworkManager) newFromCloudNetwork(ctx context.Context, userCre db.OpsLog.LogEvent(&net, db.ACT_CREATE, net.GetShortDesc(ctx), userCred) + vpc := wire.getVpc() + SyncCloudProject(userCred, &net, projectId, extNet, vpc.ManagerId) + return &net, nil } diff --git a/pkg/compute/models/secgroups.go b/pkg/compute/models/secgroups.go index a5eb4347cf..6fce2bb90b 100644 --- a/pkg/compute/models/secgroups.go +++ b/pkg/compute/models/secgroups.go @@ -263,28 +263,20 @@ func (manager *SSecurityGroupManager) getSecurityGroups() ([]SSecurityGroup, err } } -func (self *SSecurityGroup) SyncWithCloudSecurityGroup(userCred mcclient.TokenCredential, extSec cloudprovider.ICloudSecurityGroup, vpc *SVpc, projectId string, projectSync bool) error { - if _, err := db.Update(self, func() error { +func (self *SSecurityGroup) SyncWithCloudSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, extSec cloudprovider.ICloudSecurityGroup, vpc *SVpc, projectId string) error { + if _, err := db.UpdateWithLock(ctx, self, func() error { extSec.Refresh() self.Name = extSec.GetName() self.Description = extSec.GetDescription() - self.ProjectId = userCred.GetProjectId() - if projectSync && self.ProjectSrc != db.PROJECT_SOURCE_LOCAL { - if extProjectId := extSec.GetProjectId(); len(extProjectId) > 0 { - extProject, err := ExternalProjectManager.GetProject(extProjectId, vpc.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - self.ProjectId = extProject.ProjectId - } - } - } + return nil }); err != nil { log.Errorf("syncWithCloudSecurityGroup error %s", err) return err } + SyncCloudProject(userCred, self, projectId, extSec, vpc.ManagerId) + if secgroupcache := SecurityGroupCacheManager.Register(context.Background(), userCred, self.Id, extSec.GetVpcId(), vpc.CloudregionId, vpc.ManagerId); secgroupcache != nil { if err := secgroupcache.SetExternalId(userCred, self.ExternalId); err != nil { log.Errorf("set secgroupcache %s externalId error: %v", secgroupcache.Id, err) @@ -294,7 +286,7 @@ func (self *SSecurityGroup) SyncWithCloudSecurityGroup(userCred mcclient.TokenCr return nil } -func (manager *SSecurityGroupManager) newFromCloudVpc(userCred mcclient.TokenCredential, extSec cloudprovider.ICloudSecurityGroup, vpc *SVpc, projectId string) (*SSecurityGroup, bool, error) { +func (manager *SSecurityGroupManager) newFromCloudVpc(ctx context.Context, userCred mcclient.TokenCredential, extSec cloudprovider.ICloudSecurityGroup, vpc *SVpc, projectId string) (*SSecurityGroup, bool, error) { if secgroup, exist := SecurityGroupCacheManager.CheckExist(context.Background(), userCred, extSec.GetGlobalId(), extSec.GetVpcId(), vpc.CloudregionId, vpc.ManagerId); exist { if secgroup.GetGuestsCount() == 0 { return secgroup, true, nil @@ -309,25 +301,14 @@ func (manager *SSecurityGroupManager) newFromCloudVpc(userCred mcclient.TokenCre secgroup.ExternalId = extSec.GetGlobalId() secgroup.Description = extSec.GetDescription() - secgroup.ProjectSrc = db.PROJECT_SOURCE_CLOUD - secgroup.ProjectId = userCred.GetProjectId() - if len(projectId) > 0 { - secgroup.ProjectId = projectId - } - - if extProjectId := extSec.GetProjectId(); len(extProjectId) > 0 { - externalProject, err := ExternalProjectManager.GetProject(extProjectId, vpc.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - secgroup.ProjectId = externalProject.ProjectId - } - } - if err := manager.TableSpec().Insert(&secgroup); err != nil { return nil, true, err } + db.OpsLog.LogEvent(&secgroup, db.ACT_CREATE, secgroup.GetShortDesc(ctx), userCred) + + SyncCloudProject(userCred, &secgroup, projectId, extSec, vpc.ManagerId) + if secgroupcache := SecurityGroupCacheManager.Register(context.Background(), userCred, secgroup.Id, extSec.GetVpcId(), vpc.CloudregionId, vpc.ManagerId); secgroupcache != nil { if err := secgroupcache.SetExternalId(userCred, secgroup.ExternalId); err != nil { log.Errorf("set secgroupcache %s externalId error: %v", secgroupcache.Id, err) @@ -337,7 +318,7 @@ func (manager *SSecurityGroupManager) newFromCloudVpc(userCred mcclient.TokenCre return &secgroup, true, nil } -func (manager *SSecurityGroupManager) SyncSecgroups(ctx context.Context, userCred mcclient.TokenCredential, secgroups []cloudprovider.ICloudSecurityGroup, vpc *SVpc, projectId string, projectSync bool) ([]SSecurityGroup, []cloudprovider.ICloudSecurityGroup, compare.SyncResult) { +func (manager *SSecurityGroupManager) SyncSecgroups(ctx context.Context, userCred mcclient.TokenCredential, secgroups []cloudprovider.ICloudSecurityGroup, vpc *SVpc, projectId string) ([]SSecurityGroup, []cloudprovider.ICloudSecurityGroup, compare.SyncResult) { lockman.LockClass(ctx, manager, manager.GetOwnerId(userCred)) defer lockman.ReleaseClass(ctx, manager, manager.GetOwnerId(userCred)) @@ -366,7 +347,7 @@ func (manager *SSecurityGroupManager) SyncSecgroups(ctx context.Context, userCre syncResult.Error(err) continue } - if err := commondb[i].SyncWithCloudSecurityGroup(userCred, commonext[i], vpc, projectId, projectSync); err != nil { + if err := commondb[i].SyncWithCloudSecurityGroup(ctx, userCred, commonext[i], vpc, projectId); err != nil { syncResult.UpdateError(err) continue } @@ -382,7 +363,7 @@ func (manager *SSecurityGroupManager) SyncSecgroups(ctx context.Context, userCre syncResult.AddError(err) continue } - new, ruleSync, err := manager.newFromCloudVpc(userCred, added[i], vpc, projectId) + new, ruleSync, err := manager.newFromCloudVpc(ctx, userCred, added[i], vpc, projectId) if err != nil { syncResult.AddError(err) continue diff --git a/pkg/compute/models/snapshots.go b/pkg/compute/models/snapshots.go index f2d4567f67..0de7849aa4 100644 --- a/pkg/compute/models/snapshots.go +++ b/pkg/compute/models/snapshots.go @@ -545,25 +545,12 @@ func (self *SSnapshot) syncRemoveCloudSnapshot(ctx context.Context, userCred mcc } // Only sync snapshot status -func (self *SSnapshot) SyncWithCloudSnapshot(ctx context.Context, userCred mcclient.TokenCredential, ext cloudprovider.ICloudSnapshot, projectId string, projectSync bool, region *SCloudregion) error { +func (self *SSnapshot) SyncWithCloudSnapshot(ctx context.Context, userCred mcclient.TokenCredential, ext cloudprovider.ICloudSnapshot, projectId string, region *SCloudregion) error { diff, err := db.UpdateWithLock(ctx, self, func() error { // self.Name = ext.GetName() self.Status = ext.GetStatus() self.DiskType = ext.GetDiskType() - if projectSync && self.ProjectSrc != db.PROJECT_SOURCE_LOCAL { - self.ProjectSrc = db.PROJECT_SOURCE_CLOUD - if len(projectId) > 0 { - self.ProjectId = projectId - } - if extProjectId := ext.GetProjectId(); len(extProjectId) > 0 { - extProject, err := ExternalProjectManager.GetProject(extProjectId, self.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - self.ProjectId = extProject.ProjectId - } - } - } + self.CloudregionId = region.Id return nil }) @@ -572,6 +559,9 @@ func (self *SSnapshot) SyncWithCloudSnapshot(ctx context.Context, userCred mccli return err } db.OpsLog.LogSyncUpdate(self, diff, userCred) + + SyncCloudProject(userCred, self, projectId, ext, self.ManagerId) + return nil } @@ -596,18 +586,6 @@ func (manager *SSnapshotManager) newFromCloudSnapshot(ctx context.Context, userC snapshot.ManagerId = provider.Id snapshot.CloudregionId = region.Id - snapshot.ProjectSrc = db.PROJECT_SOURCE_CLOUD - snapshot.ProjectId = projectId - - if extProjectId := extSnapshot.GetProjectId(); len(extProjectId) > 0 { - externalProject, err := ExternalProjectManager.GetProject(extProjectId, snapshot.ManagerId) - if err != nil { - log.Errorf(err.Error()) - } else { - snapshot.ProjectId = externalProject.ProjectId - } - } - err := manager.TableSpec().Insert(&snapshot) if err != nil { log.Errorf("newFromCloudEip fail %s", err) @@ -616,6 +594,8 @@ func (manager *SSnapshotManager) newFromCloudSnapshot(ctx context.Context, userC db.OpsLog.LogEvent(&snapshot, db.ACT_CREATE, snapshot.GetShortDesc(ctx), userCred) + SyncCloudProject(userCred, &snapshot, projectId, extSnapshot, snapshot.ManagerId) + return &snapshot, nil } @@ -632,8 +612,9 @@ func (manager *SSnapshotManager) getProviderSnapshotsByRegion(region *SCloudregi return snapshots, nil } -func (manager *SSnapshotManager) SyncSnapshots(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, snapshots []cloudprovider.ICloudSnapshot, projectId string, projectSync bool) compare.SyncResult { - syncOwnerProjId := getSyncOwnerProjectId(manager, userCred, projectId, projectSync) +func (manager *SSnapshotManager) SyncSnapshots(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, snapshots []cloudprovider.ICloudSnapshot, projectId string) compare.SyncResult { + syncOwnerProjId := projectId + lockman.LockClass(ctx, manager, syncOwnerProjId) defer lockman.ReleaseClass(ctx, manager, syncOwnerProjId) @@ -662,7 +643,7 @@ func (manager *SSnapshotManager) SyncSnapshots(ctx context.Context, userCred mcc } } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithCloudSnapshot(ctx, userCred, commonext[i], projectId, projectSync, region) + err = commondb[i].SyncWithCloudSnapshot(ctx, userCred, commonext[i], projectId, region) if err != nil { syncResult.UpdateError(err) } else { diff --git a/pkg/compute/models/syncutils.go b/pkg/compute/models/syncutils.go index e2a2300e95..af2559b7c6 100644 --- a/pkg/compute/models/syncutils.go +++ b/pkg/compute/models/syncutils.go @@ -5,19 +5,10 @@ import ( "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/mcclient" ) -func getSyncOwnerProjectId(manager db.IModelManager, userCred mcclient.TokenCredential, projectId string, projectSync bool) string { - ownerProjId := manager.GetOwnerId(userCred) - if projectSync && ownerProjId != projectId { - ownerProjId = projectId - } - return ownerProjId -} - type IMetadataSetter interface { SetAllMetadata(ctx context.Context, meta map[string]interface{}, userCred mcclient.TokenCredential) error } diff --git a/pkg/compute/regiondrivers/managedvirtual.go b/pkg/compute/regiondrivers/managedvirtual.go index dd6c1b2be2..4ad1d13d26 100644 --- a/pkg/compute/regiondrivers/managedvirtual.go +++ b/pkg/compute/regiondrivers/managedvirtual.go @@ -126,7 +126,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateLoadbalancer(ctx co if err := lb.SetExternalId(userCred, iLoadbalancer.GetGlobalId()); err != nil { return nil, err } - if err := lb.SyncWithCloudLoadbalancer(ctx, userCred, iLoadbalancer, "", false); err != nil { + if err := lb.SyncWithCloudLoadbalancer(ctx, userCred, iLoadbalancer, ""); err != nil { return nil, err } lbbgs, err := iLoadbalancer.GetILoadBalancerBackendGroups() @@ -234,7 +234,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateLoadbalancerAcl(ctx if err := lbacl.SetExternalId(userCred, iLoadbalancerAcl.GetGlobalId()); err != nil { return nil, err } - return nil, lbacl.SyncWithCloudLoadbalancerAcl(ctx, userCred, iLoadbalancerAcl, "", false) + return nil, lbacl.SyncWithCloudLoadbalancerAcl(ctx, userCred, iLoadbalancerAcl, "") }) return nil } @@ -302,7 +302,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateLoadbalancerCertifi if err := lbcert.SetExternalId(userCred, iLoadbalancerCert.GetGlobalId()); err != nil { return nil, err } - return nil, lbcert.SyncWithCloudLoadbalancerCertificate(ctx, userCred, iLoadbalancerCert, "", false) + return nil, lbcert.SyncWithCloudLoadbalancerCertificate(ctx, userCred, iLoadbalancerCert, "") }) return nil } @@ -432,7 +432,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateLoadbalancerBackend if err := lbb.SetExternalId(userCred, iLoadbalancerBackend.GetGlobalId()); err != nil { return nil, err } - return nil, lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, iLoadbalancerBackend, "", false) + return nil, lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, iLoadbalancerBackend, "") }) return nil } @@ -496,7 +496,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateLoadbalancerListene if err := lblis.SetExternalId(userCred, iListener.GetGlobalId()); err != nil { return nil, err } - return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, "", false) + return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, "") }) return nil } @@ -581,7 +581,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncLoadbalancerListener( if err := iListener.Refresh(); err != nil { return nil, err } - return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, "", false) + return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, "") }) return nil } @@ -678,7 +678,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateLoadbalancerListene if err := lbr.SetExternalId(userCred, iListenerRule.GetGlobalId()); err != nil { return nil, err } - return nil, lbr.SyncWithCloudLoadbalancerListenerRule(ctx, userCred, iListenerRule, "", false) + return nil, lbr.SyncWithCloudLoadbalancerListenerRule(ctx, userCred, iListenerRule, "") }) return nil } diff --git a/pkg/compute/regiondrivers/qcloud.go b/pkg/compute/regiondrivers/qcloud.go index 80fca286c3..adeab9732b 100644 --- a/pkg/compute/regiondrivers/qcloud.go +++ b/pkg/compute/regiondrivers/qcloud.go @@ -112,7 +112,7 @@ func (self *SQcloudRegionDriver) RequestCreateLoadbalancerBackend(ctx context.Co if err := lbb.SetExternalId(userCred, iLoadbalancerBackend.GetGlobalId()); err != nil { return nil, err } - return nil, lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, iLoadbalancerBackend, "", false) + return nil, lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, iLoadbalancerBackend, "") }) return nil } @@ -218,7 +218,7 @@ func (self *SQcloudRegionDriver) RequestCreateLoadbalancerListener(ctx context.C } } - return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, "", false) + return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, "") }) return nil } @@ -290,7 +290,7 @@ func (self *SQcloudRegionDriver) RequestCreateLoadbalancerListenerRule(ctx conte } } - return nil, lbr.SyncWithCloudLoadbalancerListenerRule(ctx, userCred, iListenerRule, "", false) + return nil, lbr.SyncWithCloudLoadbalancerListenerRule(ctx, userCred, iListenerRule, "") }) return nil } diff --git a/pkg/compute/tasks/cloud_provider_delete_task.go b/pkg/compute/tasks/cloud_provider_delete_task.go new file mode 100644 index 0000000000..2f0980255f --- /dev/null +++ b/pkg/compute/tasks/cloud_provider_delete_task.go @@ -0,0 +1,36 @@ +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +type CloudProviderDeleteTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(CloudProviderDeleteTask{}) +} + +func (self *CloudProviderDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + provider := obj.(*models.SCloudprovider) + + provider.SetStatus(self.UserCred, models.CLOUD_PROVIDER_DELETING, "StartDiskCloudproviderTask") + + err := provider.RealDelete(ctx, self.UserCred) + if err != nil { + provider.SetStatus(self.UserCred, models.CLOUD_PROVIDER_DELETE_FAILED, "StartDiskCloudproviderTask") + self.SetStageFailed(ctx, err.Error()) + return + } + + provider.SetStatus(self.UserCred, models.CLOUD_PROVIDER_DELETED, "StartDiskCloudproviderTask") + + self.SetStageComplete(ctx, nil) +} diff --git a/pkg/compute/tasks/eip_allocate_task.go b/pkg/compute/tasks/eip_allocate_task.go index f129671239..5de5247ee6 100644 --- a/pkg/compute/tasks/eip_allocate_task.go +++ b/pkg/compute/tasks/eip_allocate_task.go @@ -53,7 +53,7 @@ func (self *EipAllocateTask) OnInit(ctx context.Context, obj db.IStandaloneModel return } - err = eip.SyncWithCloudEip(ctx, self.UserCred, eip.GetCloudprovider(), extEip, "", false) + err = eip.SyncWithCloudEip(ctx, self.UserCred, eip.GetCloudprovider(), extEip, "") if err != nil { msg := fmt.Sprintf("sync eip fail %s", err) diff --git a/pkg/compute/tasks/eip_syncstatus_task.go b/pkg/compute/tasks/eip_syncstatus_task.go index 0978a05041..8367b421fb 100644 --- a/pkg/compute/tasks/eip_syncstatus_task.go +++ b/pkg/compute/tasks/eip_syncstatus_task.go @@ -38,7 +38,7 @@ func (self *EipSyncstatusTask) OnInit(ctx context.Context, obj db.IStandaloneMod return } - err = eip.SyncWithCloudEip(ctx, self.UserCred, eip.GetCloudprovider(), extEip, "", false) + err = eip.SyncWithCloudEip(ctx, self.UserCred, eip.GetCloudprovider(), extEip, "") if err != nil { msg := fmt.Sprintf("fail to sync eip status %s", err) eip.SetStatus(self.UserCred, models.EIP_STATUS_UNKNOWN, msg) diff --git a/pkg/compute/tasks/network_create_task.go b/pkg/compute/tasks/network_create_task.go index 9755087724..897ed19a89 100644 --- a/pkg/compute/tasks/network_create_task.go +++ b/pkg/compute/tasks/network_create_task.go @@ -64,7 +64,7 @@ func (self *NetworkCreateTask) OnInit(ctx context.Context, obj db.IStandaloneMod return } - err = network.SyncWithCloudNetwork(ctx, self.UserCred, inet, "", false) + err = network.SyncWithCloudNetwork(ctx, self.UserCred, inet, "") if err != nil { self.taskFailed(ctx, network, "SyncWithCloudNetwork", err) diff --git a/pkg/mcclient/modules/mod_externalprojects.go b/pkg/mcclient/modules/mod_externalprojects.go index 96cdd268c2..eecf84753a 100644 --- a/pkg/mcclient/modules/mod_externalprojects.go +++ b/pkg/mcclient/modules/mod_externalprojects.go @@ -5,8 +5,10 @@ var ( ) func init() { - ExternalProjects = NewComputeManager("externalproject", "externalprojects", - []string{"ID", "Name", "ExternalId", "Created_at", "TenantId", "Tenant"}, + ExternalProjects = NewComputeManager( + "externalproject", + "externalprojects", + []string{"ID", "Name", "External_Id", "Tenant_id", "Tenant", "Manager_id", "Manager"}, []string{}) registerComputeV2(&ExternalProjects) diff --git a/pkg/util/aliyun/loadbalancerlistenerrule.go b/pkg/util/aliyun/loadbalancerlistenerrule.go index ac69dbefdc..4e7d7fd5f2 100644 --- a/pkg/util/aliyun/loadbalancerlistenerrule.go +++ b/pkg/util/aliyun/loadbalancerlistenerrule.go @@ -72,6 +72,10 @@ func (lbr *SLoadbalancerListenerRule) GetPath() string { return lbr.Url } +func (lbr *SLoadbalancerListenerRule) GetProjectId() string { + return "" +} + func (lbr *SLoadbalancerListenerRule) GetBackendGroupId() string { return lbr.VServerGroupId } diff --git a/pkg/util/azure/azure.go b/pkg/util/azure/azure.go index bac88333b7..e2c94aaff9 100644 --- a/pkg/util/azure/azure.go +++ b/pkg/util/azure/azure.go @@ -44,6 +44,8 @@ type SAzureClient struct { env azureenv.Environment authorizer autorest.Authorizer iregions []cloudprovider.ICloudRegion + + debug bool } var DEFAULT_API_VERSION = map[string]string{ @@ -71,14 +73,16 @@ var DEFAULT_API_VERSION = map[string]string{ "Microsoft.Compute/locations": "2018-06-01", } -func NewAzureClient(providerId string, providerName string, accessKey string, secret string, envName string) (*SAzureClient, error) { +func NewAzureClient(providerId string, providerName string, accessKey string, secret string, envName string, debug bool) (*SAzureClient, error) { clientInfo := strings.Split(secret, "/") accountInfo := strings.Split(accessKey, "/") if len(clientInfo) >= 2 && len(accountInfo) >= 1 { - client := SAzureClient{providerId: providerId, + client := SAzureClient{ + providerId: providerId, providerName: providerName, secret: secret, envName: envName, + debug: debug, } client.clientId, client.clientScret = clientInfo[0], strings.Join(clientInfo[1:], "/") client.tenantId = accountInfo[0] @@ -125,6 +129,38 @@ func (self *SAzureClient) jsonRequest(method, url string, body string) (jsonutil return jsonRequest(cli, method, self.domain, url, self.subscriptionId, body) } +func (self *SAzureClient) Put(url string, body jsonutils.JSONObject) error { + cli, err := self.getDefaultClient() + if err != nil { + return err + } + resp, err := jsonRequest(cli, "PUT", self.domain, url, self.subscriptionId, body.String()) + if err != nil { + return err + } + if self.debug { + log.Debugf("%s", resp) + } + fmt.Println(resp) + return nil +} + +func (self *SAzureClient) Patch(url string, body jsonutils.JSONObject) error { + cli, err := self.getDefaultClient() + if err != nil { + return err + } + resp, err := jsonRequest(cli, "PATCH", self.domain, url, self.subscriptionId, body.String()) + if err != nil { + return err + } + if self.debug { + log.Debugf("%s", resp) + } + fmt.Println(resp) + return nil +} + func (self *SAzureClient) Get(resourceId string, params []string, retVal interface{}) error { if len(resourceId) == 0 { return cloudprovider.ErrNotFound diff --git a/pkg/util/azure/provider/provider.go b/pkg/util/azure/provider/provider.go index e628671d20..2a23cd78be 100644 --- a/pkg/util/azure/provider/provider.go +++ b/pkg/util/azure/provider/provider.go @@ -83,7 +83,7 @@ func (self *SAzureProviderFactory) ValidateUpdateCloudaccountCredential(ctx cont } func (self *SAzureProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) { - if client, err := azure.NewAzureClient(providerId, providerName, account, secret, url); err != nil { + if client, err := azure.NewAzureClient(providerId, providerName, account, secret, url, false); err != nil { return nil, err } else { return &SAzureProvider{ diff --git a/pkg/util/azure/resourcegroup.go b/pkg/util/azure/resourcegroup.go index f1fccd2960..981a3eeb5f 100644 --- a/pkg/util/azure/resourcegroup.go +++ b/pkg/util/azure/resourcegroup.go @@ -1,6 +1,7 @@ package azure import ( + "fmt" "yunion.io/x/jsonutils" ) @@ -23,12 +24,26 @@ func (self *SRegion) GetResourceGroups() ([]SResourceGroup, error) { func (self *SRegion) GetResourceGroupDetail(groupName string) (*SResourceGroup, error) { resourceGroup := SResourceGroup{} - return &resourceGroup, self.client.Get("resourcegroups/"+groupName, []string{}, &resourceGroup) + idStr := fmt.Sprintf("subscriptions/%s/resourcegroups/%s", self.SubscriptionID, groupName) + return &resourceGroup, self.client.Get(idStr, []string{}, &resourceGroup) } -func (self *SRegion) CreateResourceGroup(groupName string) (*SResourceGroup, error) { - resourceGroup := SResourceGroup{Name: groupName, Location: self.Name} - return &resourceGroup, self.client.Create(jsonutils.Marshal(resourceGroup), &resourceGroup) +// not support update, resource group name is immutable??? +func (self *SRegion) UpdateResourceGroup(groupName string, newName string) error { + resourceGroup := SResourceGroup{Name: newName} + idStr := fmt.Sprintf("subscriptions/%s/resourcegroups/%s", self.SubscriptionID, groupName) + return self.client.Patch(idStr, jsonutils.Marshal(&resourceGroup)) +} + +func (self *SRegion) CreateResourceGroup(groupName string) error { + resourceGroup := SResourceGroup{Location: self.Name} + idStr := fmt.Sprintf("subscriptions/%s/resourcegroups/%s", self.SubscriptionID, groupName) + return self.client.Put(idStr, jsonutils.Marshal(resourceGroup)) +} + +func (self *SRegion) DeleteResourceGroup(groupName string) error { + idStr := fmt.Sprintf("subscriptions/%s/resourcegroups/%s", self.SubscriptionID, groupName) + return self.client.Delete(idStr) } func (r *SResourceGroup) GetName() string { @@ -44,7 +59,7 @@ func (r *SResourceGroup) GetGlobalId() string { } func (r *SResourceGroup) GetStatus() string { - return "" + return r.Properties.ProvisioningState } func (r *SResourceGroup) GetMetadata() *jsonutils.JSONDict { diff --git a/pkg/util/azure/shell/resourcegroup.go b/pkg/util/azure/shell/resourcegroup.go index 66d5f9e0d2..1a63b9a8eb 100644 --- a/pkg/util/azure/shell/resourcegroup.go +++ b/pkg/util/azure/shell/resourcegroup.go @@ -10,7 +10,7 @@ func init() { Limit int `help:"page size"` Offset int `help:"page offset"` } - shellutils.R(&ResourceGroupListOptions{}, "group-list", "List group", func(cli *azure.SRegion, args *ResourceGroupListOptions) error { + shellutils.R(&ResourceGroupListOptions{}, "resource-group-list", "List group", func(cli *azure.SRegion, args *ResourceGroupListOptions) error { if groups, err := cli.GetResourceGroups(); err != nil { return err } else { @@ -23,7 +23,7 @@ func init() { GROUP string `help:"ResourceGrop Name"` } - shellutils.R(&ResourceGroupOptions{}, "group-show", "Show group detail", func(cli *azure.SRegion, args *ResourceGroupOptions) error { + shellutils.R(&ResourceGroupOptions{}, "resource-group-show", "Show group detail", func(cli *azure.SRegion, args *ResourceGroupOptions) error { if group, err := cli.GetResourceGroupDetail(args.GROUP); err != nil { return err } else { @@ -32,13 +32,32 @@ func init() { } }) - shellutils.R(&ResourceGroupOptions{}, "group-create", "Create group", func(cli *azure.SRegion, args *ResourceGroupOptions) error { - if group, err := cli.CreateResourceGroup(args.GROUP); err != nil { + shellutils.R(&ResourceGroupOptions{}, "resource-group-create", "Create resource group", func(cli *azure.SRegion, args *ResourceGroupOptions) error { + err := cli.CreateResourceGroup(args.GROUP) + if err != nil { return err - } else { - printObject(group) - return nil } + return nil + }) + + shellutils.R(&ResourceGroupOptions{}, "resource-group-delete", "Delete resource group", func(cli *azure.SRegion, args *ResourceGroupOptions) error { + err := cli.DeleteResourceGroup(args.GROUP) + if err != nil { + return err + } + return nil + }) + + type ResourceGroupUpdateOptions struct { + GROUP string `help:"Name of resource group to update"` + NAME string `help:"New name of resource group"` + } + shellutils.R(&ResourceGroupUpdateOptions{}, "resource-group-update", "Update resource group detail", func(cli *azure.SRegion, args *ResourceGroupUpdateOptions) error { + err := cli.UpdateResourceGroup(args.GROUP, args.NAME) + if err != nil { + return err + } + return nil }) } diff --git a/pkg/util/logclient/logclient.go b/pkg/util/logclient/logclient.go index 7005da8f77..9f91814e9a 100644 --- a/pkg/util/logclient/logclient.go +++ b/pkg/util/logclient/logclient.go @@ -28,6 +28,7 @@ const ( ACT_BM_UNMAINTENANCE = "退出离线状态" ACT_CANCEL_DELETE = "恢复" ACT_CHANGE_OWNER = "更改项目" + ACT_SYNC_CLOUD_OWNER = "同步云项目" ACT_CLOUD_FULLSYNC = "全量同步" ACT_CLOUD_SYNC = "同步" ACT_CREATE = "创建" diff --git a/pkg/util/printutils/printjson.go b/pkg/util/printutils/printjson.go index 32bc3ce345..9e3a9dcbe0 100644 --- a/pkg/util/printutils/printjson.go +++ b/pkg/util/printutils/printjson.go @@ -43,6 +43,7 @@ func PrintJSONList(list *modules.ListResult, columns []string) { } } } + fmt.Println(colsWithData) pt := prettytable.NewPrettyTable(colsWithData) rows := make([][]string, 0) for _, obj := range list.Data { @@ -58,7 +59,6 @@ func PrintJSONList(list *modules.ListResult, columns []string) { } rows = append(rows, row) } - fmt.Print(pt.GetString(rows)) if list.Total == 0 { list.Total = len(list.Data) } diff --git a/pkg/util/qcloud/loadbalancer_listenerrule.go b/pkg/util/qcloud/loadbalancer_listenerrule.go index fe1e8a8bf8..6e1cd7646b 100755 --- a/pkg/util/qcloud/loadbalancer_listenerrule.go +++ b/pkg/util/qcloud/loadbalancer_listenerrule.go @@ -79,6 +79,10 @@ func (self *SLBListenerRule) GetPath() string { return self.URL } +func (self *SLBListenerRule) GetProjectId() string { + return "" +} + func (self *SLBListenerRule) GetBackendGroup() *SLBBackendGroup { t := self.listener.GetListenerType() if t == consts.LB_LISTENER_TYPE_HTTP || t == consts.LB_LISTENER_TYPE_HTTPS { diff --git a/pkg/util/qcloud/project.go b/pkg/util/qcloud/project.go index a4ce9b33d7..0d8da0c856 100644 --- a/pkg/util/qcloud/project.go +++ b/pkg/util/qcloud/project.go @@ -8,22 +8,28 @@ import ( ) type SProject struct { + client *SQcloudClient + ProjectName string `json:"projectName"` - ProjectId string `json:projectId` - CreateTime time.Time `json:createTime` + ProjectId string `json:"projectId"` + CreateTime time.Time `json:"createTime"` CreateorUin int `json:"creatorUin"` ProjectInfo string `json:"projectInfo"` } func (p *SProject) GetId() string { - if strings.Index(p.ProjectId, ".") != -1 { - return strings.Split(p.ProjectId, ".")[0] + var pId string + pos := strings.Index(p.ProjectId, ".") + if pos >= 0 { + pId = p.ProjectId[:pos] + } else { + pId = p.ProjectId } - return "" + return pId } func (p *SProject) GetGlobalId() string { - return p.GetId() + return p.client.providerId + "/" + p.GetId() } func (p *SProject) GetMetadata() *jsonutils.JSONDict { diff --git a/pkg/util/qcloud/qcloud.go b/pkg/util/qcloud/qcloud.go index 94db99aebd..da0b2ba53d 100644 --- a/pkg/util/qcloud/qcloud.go +++ b/pkg/util/qcloud/qcloud.go @@ -490,10 +490,11 @@ func (client *SQcloudClient) GetIProjects() ([]cloudprovider.ICloudProject, erro projects = append(projects, SProject{ ProjectId: "0", ProjectName: "默认项目", - CreateTime: time.Time{}, + // CreateTime: time.Time{}, }) iprojects := []cloudprovider.ICloudProject{} for i := 0; i < len(projects); i++ { + projects[i].client = client iprojects = append(iprojects, &projects[i]) } return iprojects, nil From fc8d0c450a1b5433d4d8390234a1e69247933e54 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Thu, 14 Mar 2019 09:48:38 +0800 Subject: [PATCH 2/6] minor fixes --- pkg/util/printutils/printjson.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/printutils/printjson.go b/pkg/util/printutils/printjson.go index 9e3a9dcbe0..32bc3ce345 100644 --- a/pkg/util/printutils/printjson.go +++ b/pkg/util/printutils/printjson.go @@ -43,7 +43,6 @@ func PrintJSONList(list *modules.ListResult, columns []string) { } } } - fmt.Println(colsWithData) pt := prettytable.NewPrettyTable(colsWithData) rows := make([][]string, 0) for _, obj := range list.Data { @@ -59,6 +58,7 @@ func PrintJSONList(list *modules.ListResult, columns []string) { } rows = append(rows, row) } + fmt.Print(pt.GetString(rows)) if list.Total == 0 { list.Total = len(list.Data) } From 62a8e1eb7f37b69047481f2474a1507f1fd76d74 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Thu, 14 Mar 2019 11:08:56 +0800 Subject: [PATCH 3/6] not allow to delete cloud-account and provider when auto sync enabled --- pkg/compute/models/cloudaccounts.go | 3 +++ pkg/compute/models/cloudproviders.go | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go index 7f9090168c..ea9a7b63b1 100644 --- a/pkg/compute/models/cloudaccounts.go +++ b/pkg/compute/models/cloudaccounts.go @@ -119,6 +119,9 @@ func (self *SCloudaccount) getCloudprovidersInternal(enabled tristate.TriState) } func (self *SCloudaccount) ValidateDeleteCondition(ctx context.Context) error { + if self.EnableAutoSync { + return httperrors.NewInvalidStatusError("automatic syncing is enabled") + } if self.Enabled { return httperrors.NewInvalidStatusError("account is enabled") } diff --git a/pkg/compute/models/cloudproviders.go b/pkg/compute/models/cloudproviders.go index 374ebfdf17..6e0650103a 100644 --- a/pkg/compute/models/cloudproviders.go +++ b/pkg/compute/models/cloudproviders.go @@ -137,6 +137,10 @@ func (self *SCloudprovider) AllowDeleteItem(ctx context.Context, userCred mcclie } func (self *SCloudprovider) ValidateDeleteCondition(ctx context.Context) error { + account := self.GetCloudaccount() + if account != nil && account.EnableAutoSync { + return httperrors.NewInvalidStatusError("auto syncing is enabled on account") + } if self.Enabled { return httperrors.NewInvalidStatusError("provider is enabled") } From f3554fe30f3b1fe30a1ea832190f7d2d4747a384 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Thu, 14 Mar 2019 12:13:37 +0800 Subject: [PATCH 4/6] make capabilities optional --- pkg/compute/models/zones.go | 11 ++++++----- pkg/compute/options/options.go | 9 +++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/pkg/compute/models/zones.go b/pkg/compute/models/zones.go index f941b176c3..e9ab6f5669 100644 --- a/pkg/compute/models/zones.go +++ b/pkg/compute/models/zones.go @@ -12,6 +12,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -727,23 +728,23 @@ func (self *SZone) isSchedPolicySupported() bool { } func (self *SZone) getMinNicCount() int { - return 1 + return options.Options.MinNicCount } func (self *SZone) getMaxNicCount() int { if self.isManaged() { - return 1 + return options.Options.MaxManagedNicCount } else { - return 8 + return options.Options.MaxNormalNicCount } } func (self *SZone) getMinDataDiskCount() int { - return 0 + return options.Options.MinDataDiskCount } func (self *SZone) getMaxDataDiskCount() int { - return 6 + return options.Options.MaxDataDiskCount } func (manager *SZoneManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { diff --git a/pkg/compute/options/options.go b/pkg/compute/options/options.go index 86014296e3..90b926e2aa 100644 --- a/pkg/compute/options/options.go +++ b/pkg/compute/options/options.go @@ -78,10 +78,19 @@ type ComputeOptions struct { MinimalIpAddrReusedIntervalSeconds int `help:"Minimal seconds when a release IP address can be reallocate" default:"30"` + SCapabilityOptions cloudcommon.CommonOptions cloudcommon.DBOptions } +type SCapabilityOptions struct { + MinDataDiskCount int `help:"Minimal data disk count" default:"0"` + MaxDataDiskCount int `help:"Maximal data disk count" default:"12"` + MinNicCount int `help:"Minimal nic count" default:"1"` + MaxNormalNicCount int `help:"Maximal nic count" default:"8"` + MaxManagedNicCount int `help:"Maximal managed nic count" default:"1"` +} + var ( Options ComputeOptions ) From 60ea022492b345843869c29f7a29670c092b773a Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Thu, 14 Mar 2019 12:19:03 +0800 Subject: [PATCH 5/6] minor fixes --- pkg/compute/models/cloudproviders.go | 6 +++--- pkg/util/azure/azure.go | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/compute/models/cloudproviders.go b/pkg/compute/models/cloudproviders.go index 6e0650103a..f82e20b401 100644 --- a/pkg/compute/models/cloudproviders.go +++ b/pkg/compute/models/cloudproviders.go @@ -947,17 +947,17 @@ func (self *SCloudprovider) deleteProjectsMapping(ctx context.Context, userCred } func (self *SCloudprovider) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { - return self.StartDiskCloudproviderTask(ctx, userCred, "") + return self.StartCloudproviderDeleteTask(ctx, userCred, "") } -func (self *SCloudprovider) StartDiskCloudproviderTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { +func (self *SCloudprovider) StartCloudproviderDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { params := jsonutils.NewDict() task, err := taskman.TaskManager.NewTask(ctx, "CloudProviderDeleteTask", self, userCred, params, parentTaskId, "", nil) if err != nil { log.Errorf("%s", err) return err } - self.SetStatus(userCred, CLOUD_PROVIDER_START_DELETE, "StartDiskCloudproviderTask") + self.SetStatus(userCred, CLOUD_PROVIDER_START_DELETE, "StartCloudproviderDeleteTask") task.ScheduleRun(nil) return nil } diff --git a/pkg/util/azure/azure.go b/pkg/util/azure/azure.go index e2c94aaff9..332c671b79 100644 --- a/pkg/util/azure/azure.go +++ b/pkg/util/azure/azure.go @@ -141,7 +141,6 @@ func (self *SAzureClient) Put(url string, body jsonutils.JSONObject) error { if self.debug { log.Debugf("%s", resp) } - fmt.Println(resp) return nil } @@ -157,7 +156,6 @@ func (self *SAzureClient) Patch(url string, body jsonutils.JSONObject) error { if self.debug { log.Debugf("%s", resp) } - fmt.Println(resp) return nil } From a9bbdddc15a709ba6611d4939aa42462ea9723d0 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Thu, 14 Mar 2019 16:51:34 +0800 Subject: [PATCH 6/6] fix: qcloud startVM do not retry implicitly --- pkg/cloudcommon/db/virtualresource.go | 7 +++--- pkg/util/qcloud/eip.go | 2 +- pkg/util/qcloud/image.go | 8 +++---- pkg/util/qcloud/instance.go | 26 +++++++++++----------- pkg/util/qcloud/instancetype.go | 2 +- pkg/util/qcloud/keypair.go | 10 ++++----- pkg/util/qcloud/qcloud.go | 32 +++++++++++++++------------ pkg/util/qcloud/region.go | 14 ++++++------ pkg/util/qcloud/zone.go | 2 +- 9 files changed, 53 insertions(+), 50 deletions(-) diff --git a/pkg/cloudcommon/db/virtualresource.go b/pkg/cloudcommon/db/virtualresource.go index c5e5dabdde..e41e6bdfbb 100644 --- a/pkg/cloudcommon/db/virtualresource.go +++ b/pkg/cloudcommon/db/virtualresource.go @@ -359,10 +359,9 @@ func (model *SVirtualResourceBase) SyncCloudProjectId(userCred mcclient.TokenCre } return nil }) - diffStr := sqlchemy.UpdateDiffString(diff) - if len(diffStr) > 0 { - OpsLog.LogEvent(model, ACT_SYNC_OWNER, diffStr, userCred) - logclient.AddSimpleActionLog(model, logclient.ACT_SYNC_CLOUD_OWNER, diffStr, userCred, true) + if len(diff) > 0 { + OpsLog.LogEvent(model, ACT_SYNC_OWNER, diff, userCred) + logclient.AddSimpleActionLog(model, logclient.ACT_SYNC_CLOUD_OWNER, diff, userCred, true) } } } diff --git a/pkg/util/qcloud/eip.go b/pkg/util/qcloud/eip.go index e30942b643..3987d6f82c 100644 --- a/pkg/util/qcloud/eip.go +++ b/pkg/util/qcloud/eip.go @@ -327,7 +327,7 @@ func (region *SRegion) UpdateInstanceBandwidth(instanceId string, bw int) error params["InstanceIds.0"] = instanceId params["InternetAccessible.InternetMaxBandwidthOut"] = fmt.Sprintf("%d", bw) - _, err := region.cvmRequest("ResetInstancesInternetMaxBandwidth", params) + _, err := region.cvmRequest("ResetInstancesInternetMaxBandwidth", params, true) return err } diff --git a/pkg/util/qcloud/image.go b/pkg/util/qcloud/image.go index 84bd67ab7d..bfc3cae931 100644 --- a/pkg/util/qcloud/image.go +++ b/pkg/util/qcloud/image.go @@ -79,7 +79,7 @@ func (self *SRegion) GetImages(status string, owner string, imageIds []string, n } images := make([]SImage, 0) - body, err := self.cvmRequest("DescribeImages", params) + body, err := self.cvmRequest("DescribeImages", params, true) if err != nil { return nil, 0, err } @@ -222,7 +222,7 @@ func (self *SRegion) DeleteImage(imageId string) error { params := make(map[string]string) params["ImageIds.0"] = imageId - _, err := self.cvmRequest("DeleteImages", params) + _, err := self.cvmRequest("DeleteImages", params, true) return err } @@ -262,7 +262,7 @@ type SupportImageSet struct { } func (self *SRegion) GetSupportImageSet() (*SupportImageSet, error) { - body, err := self.cvmRequest("DescribeImportImageOs", map[string]string{}) + body, err := self.cvmRequest("DescribeImportImageOs", map[string]string{}, true) if err != nil { return nil, err } @@ -336,7 +336,7 @@ func (self *SRegion) ImportImage(name string, osArch, osDist, osVersion string, log.Debugf("Upload image with params %#v", params) - if _, err := self.cvmRequest("ImportImage", params); err != nil { + if _, err := self.cvmRequest("ImportImage", params, true); err != nil { return nil, err } for i := 0; i < 8; i++ { diff --git a/pkg/util/qcloud/instance.go b/pkg/util/qcloud/instance.go index f94d81ddff..0f66e78ee7 100644 --- a/pkg/util/qcloud/instance.go +++ b/pkg/util/qcloud/instance.go @@ -126,7 +126,7 @@ func (self *SRegion) GetInstances(zoneId string, ids []string, offset int, limit params["Filters.0.Values.0"] = zoneId } } - body, err := self.cvmRequest("DescribeInstances", params) + body, err := self.cvmRequest("DescribeInstances", params, true) if err != nil { return nil, 0, err } @@ -529,7 +529,7 @@ func (self *SRegion) CreateInstance(name string, imageId string, instanceType st params["ClientToken"] = utils.GenRequestId(20) // log.Errorf("create params: %s", jsonutils.Marshal(params).PrettyString()) instanceIdSet := []string{} - body, err := self.cvmRequest("RunInstances", params) + body, err := self.cvmRequest("RunInstances", params, true) if err != nil { log.Errorf("RunInstances fail %s", err) return "", err @@ -542,7 +542,7 @@ func (self *SRegion) CreateInstance(name string, imageId string, instanceType st } func (self *SRegion) doStartVM(instanceId string) error { - return self.instanceOperation(instanceId, "StartInstances", nil) + return self.instanceOperation(instanceId, "StartInstances", nil, false) } func (self *SRegion) doStopVM(instanceId string, isForce bool) error { @@ -552,12 +552,12 @@ func (self *SRegion) doStopVM(instanceId string, isForce bool) error { } else { params["ForceStop"] = "false" } - return self.instanceOperation(instanceId, "StopInstances", params) + return self.instanceOperation(instanceId, "StopInstances", params, true) } func (self *SRegion) doDeleteVM(instanceId string) error { params := make(map[string]string) - err := self.instanceOperation(instanceId, "TerminateInstances", params) + err := self.instanceOperation(instanceId, "TerminateInstances", params, true) if err != nil && cloudprovider.IsError(err, []string{"InvalidInstanceId.NotFound"}) { return nil } @@ -641,7 +641,7 @@ func (self *SRegion) DeployVM(instanceId string, name string, password string, k } } if len(password) > 0 { - return self.instanceOperation(instanceId, "ResetInstancesPassword", map[string]string{"Password": password}) + return self.instanceOperation(instanceId, "ResetInstancesPassword", map[string]string{"Password": password}, true) } return nil } @@ -660,7 +660,7 @@ func (self *SRegion) UpdateVM(instanceId string, hostname string) error { } func (self *SRegion) modifyInstanceAttribute(instanceId string, params map[string]string) error { - return self.instanceOperation(instanceId, "ModifyInstancesAttribute", params) + return self.instanceOperation(instanceId, "ModifyInstancesAttribute", params, true) } func (self *SRegion) ReplaceSystemDisk(instanceId string, imageId string, passwd string, keypairName string, sysDiskSizeGB int) error { @@ -680,7 +680,7 @@ func (self *SRegion) ReplaceSystemDisk(instanceId string, imageId string, passwd if sysDiskSizeGB > 0 { params["SystemDisk.DiskSize"] = fmt.Sprintf("%d", sysDiskSizeGB) } - _, err := self.cvmRequest("ResetInstance", params) + _, err := self.cvmRequest("ResetInstance", params, true) return err } @@ -694,7 +694,7 @@ func (self *SRegion) ChangeVMConfig(zoneId string, instanceId string, ncpu int, for _, instancetype := range instanceTypes { params["InstanceType"] = instancetype.InstanceType - err := self.instanceOperation(instanceId, "ResetInstancesType", params) + err := self.instanceOperation(instanceId, "ResetInstancesType", params, true) if err != nil { log.Errorf("Failed for %s: %s", instancetype.InstanceType, err) } else { @@ -710,7 +710,7 @@ func (self *SRegion) ChangeVMConfig2(zoneId string, instanceId string, instanceT params := make(map[string]string) params["InstanceType"] = instanceType - err := self.instanceOperation(instanceId, "ResetInstancesType", params) + err := self.instanceOperation(instanceId, "ResetInstancesType", params, true) if err != nil { log.Errorf("Failed for %s: %s", instanceType, err) return fmt.Errorf("Failed to change vm config, specification not supported") @@ -746,7 +746,7 @@ func (self *SRegion) AttachDisk(instanceId string, diskId string) error { func (self *SInstance) AssignSecurityGroup(secgroupId string) error { params := map[string]string{"SecurityGroups.0": secgroupId} - return self.host.zone.region.instanceOperation(self.InstanceId, "ModifyInstancesAttribute", params) + return self.host.zone.region.instanceOperation(self.InstanceId, "ModifyInstancesAttribute", params, true) } func (self *SInstance) SetSecurityGroups(secgroupIds []string) error { @@ -754,7 +754,7 @@ func (self *SInstance) SetSecurityGroups(secgroupIds []string) error { for i := 0; i < len(secgroupIds); i++ { params[fmt.Sprintf("SecurityGroups.%d", i)] = secgroupIds[i] } - return self.host.zone.region.instanceOperation(self.InstanceId, "ModifyInstancesAttribute", params) + return self.host.zone.region.instanceOperation(self.InstanceId, "ModifyInstancesAttribute", params, true) } func (self *SInstance) GetIEIP() (cloudprovider.ICloudEIP, error) { @@ -814,7 +814,7 @@ func (region *SRegion) RenewInstances(instanceId []string, bc billing.SBillingCy params["InstanceChargePrepaid.RenewFlag"] = "NOTIFY_AND_MANUAL_RENEW" params["RenewPortableDataDisk"] = "TRUE" params["ClientToken"] = utils.GenRequestId(20) - _, err := region.cvmRequest("RenewInstances", params) + _, err := region.cvmRequest("RenewInstances", params, true) if err != nil { log.Errorf("RenewInstance fail %s", err) return err diff --git a/pkg/util/qcloud/instancetype.go b/pkg/util/qcloud/instancetype.go index 09b2f2cb72..f8183d8188 100644 --- a/pkg/util/qcloud/instancetype.go +++ b/pkg/util/qcloud/instancetype.go @@ -22,7 +22,7 @@ func (self *SRegion) GetInstanceTypes() ([]SInstanceType, error) { params := make(map[string]string) params["Region"] = self.Region - body, err := self.cvmRequest("DescribeInstanceTypeConfigs", params) + body, err := self.cvmRequest("DescribeInstanceTypeConfigs", params, true) if err != nil { log.Errorf("DescribeInstanceTypeConfigs fail %s", err) return nil, err diff --git a/pkg/util/qcloud/keypair.go b/pkg/util/qcloud/keypair.go index 53f1df735e..5e868332e1 100644 --- a/pkg/util/qcloud/keypair.go +++ b/pkg/util/qcloud/keypair.go @@ -41,7 +41,7 @@ func (self *SRegion) GetKeypairs(name string, keyIds []string, offset int, limit } } - body, err := self.cvmRequest("DescribeKeyPairs", params) + body, err := self.cvmRequest("DescribeKeyPairs", params, true) if err != nil { log.Errorf("GetKeypairs fail %s", err) return nil, 0, err @@ -63,7 +63,7 @@ func (self *SRegion) ImportKeypair(name string, pubKey string) (*SKeypair, error params["ProjectId"] = "0" params["KeyName"] = name - body, err := self.cvmRequest("ImportKeyPair", params) + body, err := self.cvmRequest("ImportKeyPair", params, true) if err != nil { log.Errorf("ImportKeypair fail %s", err) return nil, err @@ -87,7 +87,7 @@ func (self *SRegion) AttachKeypair(instanceId string, keypairId string) error { params := map[string]string{} params["InstanceIds.0"] = instanceId params["KeyIds.0"] = keypairId - _, err := self.cvmRequest("AssociateInstancesKeyPairs", params) + _, err := self.cvmRequest("AssociateInstancesKeyPairs", params, true) return err } @@ -95,7 +95,7 @@ func (self *SRegion) DetachKeyPair(instanceId string, keypairId string) error { params := make(map[string]string) params["InstanceIds.0"] = instanceId params["KeyIds.0"] = keypairId - _, err := self.cvmRequest("DisassociateInstancesKeyPairs", params) + _, err := self.cvmRequest("DisassociateInstancesKeyPairs", params, true) return err } @@ -103,7 +103,7 @@ func (self *SRegion) CreateKeyPair(name string) (*SKeypair, error) { params := make(map[string]string) params["KeyName"] = name params["ProjectId"] = "0" - body, err := self.cvmRequest("CreateKeyPair", params) + body, err := self.cvmRequest("CreateKeyPair", params, true) keypair := SKeypair{} err = body.Unmarshal(&keypair, "KeyPair") if err != nil { diff --git a/pkg/util/qcloud/qcloud.go b/pkg/util/qcloud/qcloud.go index da0b2ba53d..770668ddd6 100644 --- a/pkg/util/qcloud/qcloud.go +++ b/pkg/util/qcloud/qcloud.go @@ -65,19 +65,19 @@ func apiDomain(product string, params map[string]string) string { } } -func jsonRequest(client *common.Client, apiName string, params map[string]string, debug bool) (jsonutils.JSONObject, error) { +func jsonRequest(client *common.Client, apiName string, params map[string]string, debug bool, retry bool) (jsonutils.JSONObject, error) { domain := apiDomain("cvm", params) - return _jsonRequest(client, domain, QCLOUD_API_VERSION, apiName, params, debug) + return _jsonRequest(client, domain, QCLOUD_API_VERSION, apiName, params, debug, retry) } func vpcRequest(client *common.Client, apiName string, params map[string]string, debug bool) (jsonutils.JSONObject, error) { domain := apiDomain("vpc", params) - return _jsonRequest(client, domain, QCLOUD_API_VERSION, apiName, params, debug) + return _jsonRequest(client, domain, QCLOUD_API_VERSION, apiName, params, debug, true) } func cbsRequest(client *common.Client, apiName string, params map[string]string, debug bool) (jsonutils.JSONObject, error) { domain := apiDomain("cbs", params) - return _jsonRequest(client, domain, QCLOUD_API_VERSION, apiName, params, debug) + return _jsonRequest(client, domain, QCLOUD_API_VERSION, apiName, params, debug, true) } func accountRequest(client *common.Client, apiName string, params map[string]string, debug bool) (jsonutils.JSONObject, error) { @@ -88,7 +88,7 @@ func accountRequest(client *common.Client, apiName string, params map[string]str // loadbalancer服务 api 3.0 func clbRequest(client *common.Client, apiName string, params map[string]string, debug bool) (jsonutils.JSONObject, error) { domain := apiDomain("clb", params) - return _jsonRequest(client, domain, QCLOUD_CLB_API_VERSION, apiName, params, debug) + return _jsonRequest(client, domain, QCLOUD_CLB_API_VERSION, apiName, params, debug, true) } // loadbalancer服务 api 2017 @@ -105,7 +105,7 @@ func wssRequest(client *common.Client, apiName string, params map[string]string, func billingRequest(client *common.Client, apiName string, params map[string]string, debug bool) (jsonutils.JSONObject, error) { domain := "billing.tencentcloudapi.com" - return _jsonRequest(client, domain, QCLOUD_BILLING_API_VERSION, apiName, params, debug) + return _jsonRequest(client, domain, QCLOUD_BILLING_API_VERSION, apiName, params, debug, true) } // ============phpJsonRequest============ @@ -205,7 +205,7 @@ func (r *QcloudResponse) GetResponse() *interface{} { return r.Response } -func _jsonRequest(client *common.Client, domain string, version string, apiName string, params map[string]string, debug bool) (jsonutils.JSONObject, error) { +func _jsonRequest(client *common.Client, domain string, version string, apiName string, params map[string]string, debug bool, retry bool) (jsonutils.JSONObject, error) { req := &tchttp.BaseRequest{} if region, ok := params["Region"]; ok { client = client.Init(region) @@ -225,7 +225,7 @@ func _jsonRequest(client *common.Client, domain string, version string, apiName resp := &QcloudResponse{ BaseResponse: &tchttp.BaseResponse{}, } - return _baseJsonRequest(client, req, resp, debug) + return _baseJsonRequest(client, req, resp, debug, retry) } // 老版本腾讯云api。 适用于类似 https://cvm.api.qcloud.com/v2/index.php 这样的带/v2/index.php路径的接口 @@ -247,11 +247,15 @@ func _phpJsonRequest(client *common.Client, resp qcloudResponse, domain string, req.GetParams()[k] = v } - return _baseJsonRequest(client, req, resp, debug) + return _baseJsonRequest(client, req, resp, debug, true) } -func _baseJsonRequest(client *common.Client, req tchttp.Request, resp qcloudResponse, debug bool) (jsonutils.JSONObject, error) { - for i := 1; i <= 3; i++ { +func _baseJsonRequest(client *common.Client, req tchttp.Request, resp qcloudResponse, debug bool, retry bool) (jsonutils.JSONObject, error) { + tryMax := 1 + if retry { + tryMax = 3 + } + for i := 1; i <= tryMax; i++ { err := client.Send(req, resp) if err == nil { break @@ -347,16 +351,16 @@ func (client *SQcloudClient) billingRequest(apiName string, params map[string]st return billingRequest(cli, apiName, params, client.Debug) } -func (client *SQcloudClient) jsonRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { +func (client *SQcloudClient) jsonRequest(apiName string, params map[string]string, retry bool) (jsonutils.JSONObject, error) { cli, err := client.getDefaultClient() if err != nil { return nil, err } - return jsonRequest(cli, apiName, params, client.Debug) + return jsonRequest(cli, apiName, params, client.Debug, retry) } func (client *SQcloudClient) fetchRegions() error { - body, err := client.jsonRequest("DescribeRegions", nil) + body, err := client.jsonRequest("DescribeRegions", nil, true) if err != nil { log.Errorf("fetchRegions fail %s", err) return err diff --git a/pkg/util/qcloud/region.go b/pkg/util/qcloud/region.go index 31ebc63569..4e53ead733 100644 --- a/pkg/util/qcloud/region.go +++ b/pkg/util/qcloud/region.go @@ -391,7 +391,7 @@ func (self *SRegion) updateInstance(instId string, name, desc, passwd, hostname if len(hostname) > 0 { params["HostName"] = hostname } - _, err := self.cvmRequest("ModifyInstanceAttribute", params) + _, err := self.cvmRequest("ModifyInstanceAttribute", params, true) return err } @@ -463,7 +463,7 @@ func (self *SRegion) GetIZones() ([]cloudprovider.ICloudZone, error) { func (self *SRegion) _fetchZones() error { params := make(map[string]string) zones := make([]SZone, 0) - body, err := self.cvmRequest("DescribeZones", params) + body, err := self.cvmRequest("DescribeZones", params, true) if err != nil { return err } @@ -597,9 +597,9 @@ func (self *SRegion) vpcRequest(apiName string, params map[string]string) (jsonu return self.client.vpcRequest(apiName, params) } -func (self *SRegion) cvmRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { +func (self *SRegion) cvmRequest(apiName string, params map[string]string, retry bool) (jsonutils.JSONObject, error) { params["Region"] = self.Region - return self.client.jsonRequest(apiName, params) + return self.client.jsonRequest(apiName, params, retry) } func (self *SRegion) accountRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { @@ -744,7 +744,7 @@ func (self *SRegion) CreateInstanceSimple(name string, imgId string, cpu int, me return nil, fmt.Errorf("cannot find network %s", networkId) } -func (self *SRegion) instanceOperation(instanceId string, opname string, extra map[string]string) error { +func (self *SRegion) instanceOperation(instanceId string, opname string, extra map[string]string, retry bool) error { params := make(map[string]string) params["InstanceIds.0"] = instanceId if extra != nil && len(extra) > 0 { @@ -752,7 +752,7 @@ func (self *SRegion) instanceOperation(instanceId string, opname string, extra m params[k] = v } } - _, err := self.cvmRequest(opname, params) + _, err := self.cvmRequest(opname, params, retry) return err } @@ -763,7 +763,7 @@ func (self *SRegion) DeleteSecurityGroup(vpcId string, secgroupId string) error func (self *SRegion) GetInstanceVNCUrl(instanceId string) (string, error) { params := make(map[string]string) params["InstanceId"] = instanceId - body, err := self.cvmRequest("DescribeInstanceVncUrl", params) + body, err := self.cvmRequest("DescribeInstanceVncUrl", params, true) if err != nil { return "", err } diff --git a/pkg/util/qcloud/zone.go b/pkg/util/qcloud/zone.go index fe45963a9c..1e4150e6fd 100644 --- a/pkg/util/qcloud/zone.go +++ b/pkg/util/qcloud/zone.go @@ -221,7 +221,7 @@ func (self *SZone) fetchInstanceTypes() { params["Region"] = self.region.Region params["Filters.0.Name"] = "zone" params["Filters.0.Values.0"] = self.Zone - if body, err := self.region.cvmRequest("DescribeInstanceTypeConfigs", params); err != nil { + if body, err := self.region.cvmRequest("DescribeInstanceTypeConfigs", params, true); err != nil { log.Errorf("DescribeInstanceTypeConfigs error: %v", err) } else if configSet, err := body.GetArray("InstanceTypeConfigSet"); err != nil { log.Errorf("Get InstanceTypeConfigSet error: %v", err)