diff --git a/cmd/climc/shell/cloudaccounts.go b/cmd/climc/shell/cloudaccounts.go index 3de4da23c8..0b2dd60c95 100644 --- a/cmd/climc/shell/cloudaccounts.go +++ b/cmd/climc/shell/cloudaccounts.go @@ -31,14 +31,15 @@ func init() { }) type CloudaccountCreateOptions struct { - NAME string `help:"Name of cloud account"` - ACCOUNT string `help:"Account to access the cloud account"` - SECRET string `help:"Secret to access the cloud account, clientId/clientScret for Azure"` - PROVIDER string `help:"Driver for cloud account" choices:"VMware|Aliyun|Azure|Qcloud"` - AccessURL string `helo:"hello" metavar:"Azure choices: "` - Desc string `help:"Description"` - Enabled bool `help:"Enabled the account automatically"` - Import bool `help:"Import all sub account automatically"` + NAME string `help:"Name of cloud account"` + ACCOUNT string `help:"Account to access the cloud account"` + SECRET string `help:"Secret to access the cloud account, clientId/clientScret for Azure"` + PROVIDER string `help:"Driver for cloud account" choices:"VMware|Aliyun|Azure|Qcloud"` + AccessURL string `helo:"hello" metavar:"Azure choices: "` + Desc string `help:"Description"` + Enabled bool `help:"Enabled the account automatically"` + EnableProject bool `help:"Enable the account with same name project"` + Import bool `help:"Import all sub account automatically"` } R(&CloudaccountCreateOptions{}, "cloud-account-create", "Create a cloud account", func(s *mcclient.ClientSession, args *CloudaccountCreateOptions) error { params := jsonutils.NewDict() @@ -49,6 +50,9 @@ func init() { if args.Enabled { params.Add(jsonutils.JSONTrue, "enabled") } + if args.EnableProject { + params.Add(jsonutils.JSONTrue, "enable_project") + } if args.Import { params.Add(jsonutils.JSONTrue, "import") } @@ -106,15 +110,6 @@ func init() { return nil }) - R(&CloudaccountShowOptions{}, "cloud-account-sub-accounts", "Get sub accounts of a cloud account", func(s *mcclient.ClientSession, args *CloudaccountShowOptions) error { - result, err := modules.Cloudaccounts.PerformAction(s, args.ID, "get-sub-accounts", nil) - if err != nil { - return err - } - printObject(result) - return nil - }) - R(&CloudaccountShowOptions{}, "cloud-account-delete", "Delete a cloud account", func(s *mcclient.ClientSession, args *CloudaccountShowOptions) error { result, err := modules.Cloudaccounts.Delete(s, args.ID, nil) if err != nil { @@ -152,11 +147,12 @@ func init() { }) type CloudaccountImportOptions struct { - ID string `help:"ID or Name of cloud account"` - Enabled bool `help:"Import sub accounts with enabled status"` + ID string `help:"ID or Name of cloud account"` + Enabled bool `help:"Import sub accounts with enabled status"` + EnableProject bool `help:"Import sub account with project"` } R(&CloudaccountImportOptions{}, "cloud-account-import", "Import sub cloud account", func(s *mcclient.ClientSession, args *CloudaccountImportOptions) error { - result, err := modules.Cloudaccounts.PerformAction(s, args.ID, "import", jsonutils.Marshal(map[string]bool{"enabled": args.Enabled})) + result, err := modules.Cloudaccounts.PerformAction(s, args.ID, "import", jsonutils.Marshal(map[string]bool{"enabled": args.Enabled, "enable_project": args.EnableProject})) if err != nil { return err } diff --git a/cmd/climc/shell/cloudproviders.go b/cmd/climc/shell/cloudproviders.go index f8eb5bf518..9ac65dfcf0 100644 --- a/cmd/climc/shell/cloudproviders.go +++ b/cmd/climc/shell/cloudproviders.go @@ -91,6 +91,19 @@ func init() { return nil }) + type CloudproviderChangeProjectOptions struct { + ID string `help:"ID or Name of cloud provider"` + TENANT string `help:"ID or Name of tenant"` + } + R(&CloudproviderChangeProjectOptions{}, "cloud-provider-change-project", "Change project for provider", func(s *mcclient.ClientSession, args *CloudproviderChangeProjectOptions) error { + result, err := modules.Cloudproviders.PerformAction(s, args.ID, "change-project", jsonutils.Marshal(map[string]string{"project": args.TENANT})) + if err != nil { + return err + } + printObject(result) + return nil + }) + type CloudproviderShowOptions struct { ID string `help:"ID or Name of cloud provider"` } diff --git a/pkg/cloudcommon/db/taskman/subtasks.go b/pkg/cloudcommon/db/taskman/subtasks.go index d4d0bf50ed..2ef4e56966 100644 --- a/pkg/cloudcommon/db/taskman/subtasks.go +++ b/pkg/cloudcommon/db/taskman/subtasks.go @@ -37,6 +37,7 @@ type SSubTask struct { func (manager *SSubTaskmanager) GetSubTask(ptaskId string, subtaskId string) *SSubTask { subtask := SSubTask{} + subtask.SetModelManager(manager) err := manager.Query().Equals("task_id", ptaskId).Equals("subtask_id", subtaskId).First(&subtask) if err != nil { if err != sql.ErrNoRows { @@ -47,9 +48,12 @@ func (manager *SSubTaskmanager) GetSubTask(ptaskId string, subtaskId string) *SS return &subtask } -func (manager *SSubTaskmanager) GetInitSubtasks(taskId string, stage string) []SSubTask { +func (manager *SSubTaskmanager) GetTotalSubtasks(taskId string, stage string, status string) []SSubTask { subtasks := make([]SSubTask, 0) - q := manager.Query().Equals("task_id", taskId).Equals("stage", stage).Equals("status", SUBTASK_INIT) + q := manager.Query().Equals("task_id", taskId).Equals("stage", stage) + if len(status) > 0 { + q = q.Equals("status", status) + } err := db.FetchModelObjects(manager, q, &subtasks) if err != nil { log.Errorf("GetInitSubtasks fail %s", err) @@ -58,6 +62,10 @@ func (manager *SSubTaskmanager) GetInitSubtasks(taskId string, stage string) []S return subtasks } +func (manager *SSubTaskmanager) GetInitSubtasks(taskId string, stage string) []SSubTask { + return manager.GetTotalSubtasks(taskId, stage, SUBTASK_INIT) +} + func (self *SSubTask) SaveResults(failed bool, result jsonutils.JSONObject) error { _, err := self.GetModelManager().TableSpec().Update(self, func() error { if failed { diff --git a/pkg/cloudcommon/db/taskman/tasks.go b/pkg/cloudcommon/db/taskman/tasks.go index dbda4a7bd0..c95aa63c67 100644 --- a/pkg/cloudcommon/db/taskman/tasks.go +++ b/pkg/cloudcommon/db/taskman/tasks.go @@ -196,6 +196,15 @@ func (manager *STaskManager) NewTask(ctx context.Context, taskName string, obj d log.Errorf("Task insert error %s", err) return nil, err } + parentTask := task.GetParentTask() + if parentTask != nil { + st := SSubTask{TaskId: parentTask.Id, Stage: parentTask.Stage, SubtaskId: task.Id} + err := SubTaskManager.TableSpec().Insert(&st) + if err != nil { + log.Errorf("Subtask insert error %s", err) + return nil, err + } + } return &task, nil } @@ -516,14 +525,19 @@ func (self *STask) NotifyParentTaskComplete(ctx context.Context, body *jsonutils if subTask != nil { subTask.SaveResults(failed, body) } - pTask := TaskManager.fetchTask(parentTaskId) - if pTask == nil { - log.Errorf("Parent task %s not found", parentTaskId) - return - } - if pTask.IsCurrentStageComplete() { - pTask.ScheduleRun(body) - } + func() { + lockman.LockRawObject(ctx, "tasks", parentTaskId) + defer lockman.ReleaseRawObject(ctx, "tasks", parentTaskId) + + pTask := TaskManager.fetchTask(parentTaskId) + if pTask == nil { + log.Errorf("Parent task %s not found", parentTaskId) + return + } + if pTask.IsCurrentStageComplete() { + pTask.ScheduleRun(body) + } + }() } if len(parentTaskNotify) > 0 { notifyRemoteTask(ctx, parentTaskNotify, parentTaskId, body, 0) @@ -560,8 +574,9 @@ func (self *STask) NotifyParentTaskFailure(ctx context.Context, reason string) { } func (self *STask) IsCurrentStageComplete() bool { - subtasks := SubTaskManager.GetInitSubtasks(self.Id, self.Stage) - if len(subtasks) == 0 { + totalSubtasks := SubTaskManager.GetTotalSubtasks(self.Id, self.Stage, "") + initSubtasks := SubTaskManager.GetInitSubtasks(self.Id, self.Stage) + if len(totalSubtasks) > 0 && len(initSubtasks) == 0 { return true } else { return false diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go index 9cd3c5e66d..f2aff8ad1d 100644 --- a/pkg/compute/models/cloudaccounts.go +++ b/pkg/compute/models/cloudaccounts.go @@ -97,7 +97,7 @@ func (self *SCloudaccountManager) ValidateCreateData(ctx context.Context, userCr } if subAccount, err := GetSubAccounts(name, url, account, secret, provider); err != nil { - return nil, err + return nil, httperrors.NewInputParameterError("Invalidate account with secret") } else if accounts, err := subAccount.GetArray("data"); err != nil { return nil, err } else { @@ -110,6 +110,7 @@ func (self *SCloudaccount) PostCreate(ctx context.Context, userCred mcclient.Tok self.SEnabledStatusStandaloneResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data) self.savePassword(self.Secret) _import, _ := data.Bool("import") + enableProject, _ := data.Bool("enable_project") if subAccounts, err := data.GetArray("accounts"); err == nil && _import && len(subAccounts) > 0 { for _, subAccount := range subAccounts { name, _ := subAccount.GetString("name") @@ -123,6 +124,7 @@ func (self *SCloudaccount) PostCreate(ctx context.Context, userCred mcclient.Tok CloudaccountId: self.Id, Provider: self.Provider, } + newCloudprovider.SetModelManager(CloudproviderManager) if err := CloudproviderManager.TableSpec().Insert(&newCloudprovider); err != nil { log.Errorf("Create cloudprovider error: %v", err) } else if _, err := CloudproviderManager.TableSpec().Update(&newCloudprovider, func() error { @@ -131,6 +133,9 @@ func (self *SCloudaccount) PostCreate(ctx context.Context, userCred mcclient.Tok }); err != nil { log.Errorf("Update cloudprovider error: %v", err) } + if enableProject { + newCloudprovider.SyncProject() + } } } } @@ -221,9 +226,9 @@ func (self *SCloudaccount) PerformUpdateCredential(ctx context.Context, userCred changed = true } if (len(account) > 0 && account != self.Account) || (len(accessUrl) > 0 && accessUrl != self.AccessUrl) { - for _, cloudprovider := range self.GetCloudproviders() { - if cloudprovider.Account == self.Account { - if len(account) > 0 { + if len(account) > 0 && account != self.Account { + for _, cloudprovider := range self.GetCloudproviders() { + if cloudprovider.Account == self.Account { if _, err = cloudprovider.GetModelManager().TableSpec().Update(&cloudprovider, func() error { cloudprovider.Account = account return nil @@ -259,12 +264,21 @@ func (self *SCloudaccount) startSyncCloudProviderInfoTask(ctx context.Context, u if syncRange != nil { params.Add(jsonutils.Marshal(syncRange), "sync_range") } - task, err := taskman.TaskManager.NewTask(ctx, "CloudAccountSyncInfoTask", self, userCred, params, parentTaskId, "", nil) + cloudproviders := self.GetCloudproviders() + + taskItems := make([]db.IStandaloneModel, 0) + for i := 0; i < len(cloudproviders); i++ { + if cloudproviders[i].Enabled { + taskItems = append(taskItems, &cloudproviders[i]) + } + } + + task, err := taskman.TaskManager.NewParallelTask(ctx, "CloudAccountSyncInfoTask", taskItems, userCred, params, "", "", nil) if err != nil { - log.Errorf("startSyncCloudAccountInfoTask newTask error %s", err) - return err + log.Errorf("CloudAccountSyncInfoTask newTask error %s", err) + } else { + task.ScheduleRun(nil) } - task.ScheduleRun(nil) return nil } @@ -294,14 +308,6 @@ func (self *SCloudaccount) GetDriver() (cloudprovider.ICloudProvider, error) { return cloudprovider.GetProvider(self.Id, self.Name, self.AccessUrl, self.Account, secret, self.Provider) } -func (self *SCloudaccount) AllowPerformGetSubAccounts(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { - return userCred.IsSystemAdmin() -} - -func (self *SCloudaccount) AllowPerformImport(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { - return userCred.IsSystemAdmin() -} - func (self *SCloudaccount) GetSubAccounts() (jsonutils.JSONObject, error) { if secret, err := self.getPassword(); err != nil { return nil, err @@ -310,6 +316,10 @@ func (self *SCloudaccount) GetSubAccounts() (jsonutils.JSONObject, error) { } } +func (self *SCloudaccount) AllowPerformImport(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return userCred.IsSystemAdmin() +} + func (self *SCloudaccount) PerformImport(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { if subAccounts, err := self.GetSubAccounts(); err != nil { return nil, err @@ -317,6 +327,7 @@ func (self *SCloudaccount) PerformImport(ctx context.Context, userCred mcclient. return nil, err } else { enabled, _ := data.Bool("enabled") + enableProject, _ := data.Bool("enable_project") for _, _account := range accounts { name, _ := _account.GetString("name") account, _ := _account.GetString("account") @@ -329,6 +340,7 @@ func (self *SCloudaccount) PerformImport(ctx context.Context, userCred mcclient. CloudaccountId: self.Id, Provider: self.Provider, } + newCloudprovider.SetModelManager(CloudproviderManager) if err := CloudproviderManager.TableSpec().Insert(&newCloudprovider); err != nil { log.Errorf("Create cloudprovider error: %v", err) return nil, err @@ -340,6 +352,9 @@ func (self *SCloudaccount) PerformImport(ctx context.Context, userCred mcclient. log.Errorf("Update cloudprovider error: %v", err) return nil, err } + if enableProject { + newCloudprovider.SyncProject() + } if enabled { newCloudprovider.StartSyncCloudProviderInfoTask(ctx, userCred, &SSyncRange{FullSync: true}, "") } @@ -407,14 +422,11 @@ func (manager *SCloudaccountManager) InitializeData() error { if err := db.FetchModelObjects(CloudproviderManager, q, &cloudproviders); err != nil { return err } - newAccounts := map[string]string{} - for _, cloudprovider := range cloudproviders { + for i := 0; i < len(cloudproviders); i++ { + cloudprovider := cloudproviders[i] Account, providerAccount, providerName := cloudprovider.Account, "", cloudprovider.Name if cloudprovider.Provider == CLOUD_PROVIDER_AZURE { if accountInfo := strings.Split(cloudprovider.Account, "/"); len(accountInfo) == 2 { - if _, ok := newAccounts[accountInfo[0]]; ok { - continue - } Account, providerAccount = accountInfo[0], accountInfo[1] if len(cloudprovider.Description) > 0 { providerName = cloudprovider.Description @@ -424,18 +436,22 @@ func (manager *SCloudaccountManager) InitializeData() error { continue } } - account := SCloudaccount{ - AccessUrl: cloudprovider.AccessUrl, - Account: Account, - LastSync: cloudprovider.LastSync, - Sysinfo: cloudprovider.Sysinfo, - Provider: cloudprovider.Provider, - } + account := SCloudaccount{} account.SetModelManager(CloudaccountManager) - if err := CloudaccountManager.TableSpec().Insert(&account); err != nil { - log.Errorf("Insert Account error: %v", err) - } else { - newAccounts[Account] = account.Id + if err := CloudaccountManager.Query(). + Equals("access_url", cloudprovider.AccessUrl). + Equals("account", Account). + Equals("provider", cloudprovider.Provider).First(&account); err != nil { + account = SCloudaccount{ + AccessUrl: cloudprovider.AccessUrl, + Account: Account, + LastSync: cloudprovider.LastSync, + Sysinfo: cloudprovider.Sysinfo, + Provider: cloudprovider.Provider, + } + if err := CloudaccountManager.TableSpec().Insert(&account); err != nil { + log.Errorf("Insert Account error: %v", err) + } if _, err := CloudaccountManager.TableSpec().Update(&account, func() error { account.Name = Account account.Status = cloudprovider.Status @@ -447,15 +463,18 @@ func (manager *SCloudaccountManager) InitializeData() error { log.Errorf("Get password from provider %s error %v", cloudprovider.Name, err) } else if err := account.savePassword(secret); err != nil { log.Errorf("Set password for account %s error %v", account.Name, err) - } else if _, err := CloudproviderManager.TableSpec().Update(&cloudprovider, func() error { - cloudprovider.CloudaccountId = account.Id - cloudprovider.Account = providerAccount - cloudprovider.Name = providerName - return nil - }); err != nil { - log.Errorf("Update provider %s error: %v", cloudprovider.Name, err) } } + if _, err := CloudproviderManager.TableSpec().Update(&cloudprovider, func() error { + log.Errorf("update %s", cloudprovider.Name) + cloudprovider.CloudaccountId = account.Id + cloudprovider.Account = providerAccount + cloudprovider.Secret = "" + cloudprovider.Name = providerName + return nil + }); err != nil { + log.Errorf("Update provider %s error: %v", cloudprovider.Name, err) + } } return nil } diff --git a/pkg/compute/models/cloudproviders.go b/pkg/compute/models/cloudproviders.go index 6f7776c911..81774adc2e 100644 --- a/pkg/compute/models/cloudproviders.go +++ b/pkg/compute/models/cloudproviders.go @@ -4,7 +4,6 @@ import ( "context" "database/sql" "fmt" - "strings" "time" "yunion.io/x/jsonutils" @@ -130,39 +129,35 @@ func (self *SCloudprovider) CanSync() bool { func (self *SCloudprovider) SyncProject() (err error) { projectId := "" - if len(self.ProjectId) == 0 && len(self.Name) > 0 && self.Provider == CLOUD_PROVIDER_AZURE { - s := auth.GetAdminSession(options.Options.Region, "") - if project, err := modules.Projects.GetByName(s, self.Name, nil); err == nil { - if projectId, err = project.GetString("id"); err != nil { - return err - } - } else if strings.Index(err.Error(), "404 NotFoundError") > 0 { + if len(self.ProjectId) == 0 && len(self.Name) > 0 { + if tenant, err := db.TenantCacheManager.FetchTenantByIdOrName(context.Background(), self.Name); err != nil { + s := auth.GetAdminSession(options.Options.Region, "") if project, err := modules.Projects.Create(s, jsonutils.Marshal(map[string]string{"name": self.Name})); err != nil { return err } else if projectId, err = project.GetString("id"); err != nil { return err } } else { - return err - } - if len(projectId) > 0 { - if _, err := self.GetModelManager().TableSpec().Update(self, func() error { - self.ProjectId = projectId - return nil - }); err != nil { - return err - } + projectId = tenant.Id } } + if len(projectId) > 0 { + _, err := self.GetModelManager().TableSpec().Update(self, func() error { + self.ProjectId = projectId + return nil + }) + return err + } return nil } type SSyncRange struct { - Force bool - FullSync bool - Region []string - Zone []string - Host []string + Force bool + FullSync bool + ProjectSync bool + Region []string + Zone []string + Host []string } func (sr *SSyncRange) NeedSyncInfo() bool { @@ -268,9 +263,6 @@ func (self *SCloudprovider) PerformSync(ctx context.Context, userCred mcclient.T } func (self *SCloudprovider) StartSyncCloudProviderInfoTask(ctx context.Context, userCred mcclient.TokenCredential, syncRange *SSyncRange, parentTaskId string) error { - if err := self.SyncProject(); err != nil { - log.Errorf("Sync cloudprovider project error: %v", err) - } params := jsonutils.NewDict() if syncRange != nil { params.Add(jsonutils.Marshal(syncRange), "sync_range") @@ -284,6 +276,26 @@ func (self *SCloudprovider) StartSyncCloudProviderInfoTask(ctx context.Context, return nil } +func (self *SCloudprovider) AllowPerformChangeProject(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return userCred.IsSystemAdmin() +} + +func (self *SCloudprovider) PerformChangeProject(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if project, err := data.GetString("project"); err != nil { + return nil, httperrors.NewInputParameterError("Missing project parameter") + } else if tenant, err := db.TenantCacheManager.FetchTenantByIdOrName(ctx, project); err != nil { + return nil, httperrors.NewNotFoundError("project %s not found", project) + } else if _, err := self.GetModelManager().TableSpec().Update(self, func() error { + self.ProjectId = tenant.Id + return nil + }); err != nil { + log.Errorf("Update cloudprovider error: %v", err) + return nil, err + } else { + return nil, self.StartSyncCloudProviderInfoTask(ctx, userCred, &SSyncRange{FullSync: true, ProjectSync: true}, "") + } +} + func (self *SCloudprovider) MarkStartSync(userCred mcclient.TokenCredential) { _, err := self.GetModelManager().TableSpec().Update(self, func() error { self.LastSync = timeutils.UtcNow() @@ -296,6 +308,18 @@ func (self *SCloudprovider) MarkStartSync(userCred mcclient.TokenCredential) { self.SetStatus(userCred, CLOUD_PROVIDER_START_SYNC, "") } +func (self *SCloudprovider) GetDriver() (cloudprovider.ICloudProvider, error) { + if !self.Enabled { + return nil, fmt.Errorf("Cloud provider is not enabled") + } + + account, err := self.getAccount() + if err != nil { + return nil, err + } + return cloudprovider.GetProvider(self.Id, self.Name, account.AccessUrl, account.Account, account.Secret, self.Provider) +} + type SAccount struct { AccessUrl string Account string @@ -327,18 +351,6 @@ func (self *SCloudprovider) getAccount() (*SAccount, error) { } -func (self *SCloudprovider) GetDriver() (cloudprovider.ICloudProvider, error) { - if !self.Enabled { - return nil, fmt.Errorf("Cloud provider is not enabled") - } - - account, err := self.getAccount() - if err != nil { - return nil, err - } - return cloudprovider.GetProvider(self.Id, self.Name, account.AccessUrl, account.Account, account.Secret, self.Provider) -} - func (self *SCloudprovider) SaveSysInfo(info jsonutils.JSONObject) { self.GetModelManager().TableSpec().Update(self, func() error { self.Sysinfo = info diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 34abdd376c..7ddc02d301 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -637,7 +637,7 @@ func (manager *SDiskManager) getDisksByStorage(storage *SStorage) ([]SDisk, erro return disks, nil } -func (manager *SDiskManager) syncCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, vdisk cloudprovider.ICloudDisk) (*SDisk, error) { +func (manager *SDiskManager) syncCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, vdisk cloudprovider.ICloudDisk, projectId string, projectSync bool) (*SDisk, error) { diskObj, err := manager.FetchByExternalId(vdisk.GetGlobalId()) if err != nil { if err == sql.ErrNoRows { @@ -648,13 +648,13 @@ func (manager *SDiskManager) syncCloudDisk(ctx context.Context, userCred mcclien return nil, err } storage := storageObj.(*SStorage) - return manager.newFromCloudDisk(ctx, userCred, vdisk, storage) + return manager.newFromCloudDisk(ctx, userCred, vdisk, storage, projectId) } else { return nil, err } } else { disk := diskObj.(*SDisk) - err = disk.syncWithCloudDisk(ctx, userCred, vdisk) + err = disk.syncWithCloudDisk(ctx, userCred, vdisk, projectId, projectSync) if err != nil { return nil, err } @@ -662,7 +662,7 @@ func (manager *SDiskManager) syncCloudDisk(ctx context.Context, userCred mcclien } } -func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.TokenCredential, storage *SStorage, disks []cloudprovider.ICloudDisk) ([]SDisk, []cloudprovider.ICloudDisk, compare.SyncResult) { +func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.TokenCredential, storage *SStorage, disks []cloudprovider.ICloudDisk, projectId string, projectSync bool) ([]SDisk, []cloudprovider.ICloudDisk, compare.SyncResult) { localDisks := make([]SDisk, 0) remoteDisks := make([]cloudprovider.ICloudDisk, 0) syncResult := compare.SyncResult{} @@ -694,7 +694,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].syncWithCloudDisk(ctx, userCred, commonext[i]) + err = commondb[i].syncWithCloudDisk(ctx, userCred, commonext[i], projectId, projectSync) if err != nil { syncResult.UpdateError(err) } else { @@ -705,7 +705,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To } for i := 0; i < len(added); i += 1 { - new, err := manager.newFromCloudDisk(ctx, userCred, added[i], storage) + new, err := manager.newFromCloudDisk(ctx, userCred, added[i], storage, projectId) if err != nil { syncResult.AddError(err) } else { @@ -718,7 +718,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To return localDisks, remoteDisks, syncResult } -func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk) error { +func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk, projectId string, projectSync bool) error { _, err := self.GetModelManager().TableSpec().Update(self, func() error { extDisk.Refresh() self.Name = extDisk.GetName() @@ -737,10 +737,8 @@ func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.Toke self.ExpiredAt = extDisk.GetExpiredAt() self.ProjectId = userCred.GetProjectId() - if manageId := extDisk.GetIStorge().GetManagerId(); len(manageId) > 0 { - if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil { - self.ProjectId = provider.ProjectId - } + if projectSync && len(projectId) > 0 { + self.ProjectId = projectId } return nil }) @@ -765,7 +763,7 @@ func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.Toke return nil } -func (manager *SDiskManager) newFromCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk, storage *SStorage) (*SDisk, error) { +func (manager *SDiskManager) newFromCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk, storage *SStorage, projectId string) (*SDisk, error) { disk := SDisk{} disk.SetModelManager(manager) @@ -774,10 +772,8 @@ func (manager *SDiskManager) newFromCloudDisk(ctx context.Context, userCred mccl disk.ExternalId = extDisk.GetGlobalId() disk.StorageId = storage.Id disk.ProjectId = userCred.GetProjectId() - if manageId := extDisk.GetIStorge().GetManagerId(); len(manageId) > 0 { - if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil { - disk.ProjectId = provider.ProjectId - } + if len(projectId) > 0 { + disk.ProjectId = projectId } disk.DiskFormat = extDisk.GetDiskFormat() disk.DiskSize = extDisk.GetDiskSizeMB() diff --git a/pkg/compute/models/elasticips.go b/pkg/compute/models/elasticips.go index c01b30df1f..5d32ed6707 100644 --- a/pkg/compute/models/elasticips.go +++ b/pkg/compute/models/elasticips.go @@ -136,7 +136,7 @@ func (self *SElasticip) GetRegion() *SCloudregion { return CloudregionManager.FetchRegionById(self.CloudregionId) } -func (manager *SElasticipManager) SyncEips(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, eips []cloudprovider.ICloudEIP) compare.SyncResult { +func (manager *SElasticipManager) SyncEips(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, eips []cloudprovider.ICloudEIP, projectId string, projectSync bool) compare.SyncResult { // localEips := make([]SElasticip, 0) // remoteEips := make([]cloudprovider.ICloudEIP, 0) syncResult := compare.SyncResult{} @@ -167,7 +167,7 @@ func (manager *SElasticipManager) SyncEips(ctx context.Context, userCred mcclien } } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithCloudEip(userCred, commonext[i]) + err = commondb[i].SyncWithCloudEip(userCred, commonext[i], projectId, projectSync) if err != nil { syncResult.UpdateError(err) } else { @@ -175,7 +175,7 @@ func (manager *SElasticipManager) SyncEips(ctx context.Context, userCred mcclien } } for i := 0; i < len(added); i += 1 { - _, err := manager.newFromCloudEip(userCred, added[i], region) + _, err := manager.newFromCloudEip(userCred, added[i], region, projectId) if err != nil { syncResult.AddError(err) } else { @@ -221,7 +221,7 @@ func (self *SElasticip) SyncInstanceWithCloudEip(ctx context.Context, userCred m return nil } -func (self *SElasticip) SyncWithCloudEip(userCred mcclient.TokenCredential, ext cloudprovider.ICloudEIP) error { +func (self *SElasticip) SyncWithCloudEip(userCred mcclient.TokenCredential, ext cloudprovider.ICloudEIP, projectId string, projectSync bool) error { _, err := self.GetModelManager().TableSpec().Update(self, func() error { // self.Name = ext.GetName() @@ -233,10 +233,8 @@ func (self *SElasticip) SyncWithCloudEip(userCred mcclient.TokenCredential, ext // self.ManagerId = ext.GetManagerId() self.IsEmulated = ext.IsEmulated() self.ProjectId = userCred.GetProjectId() - if manageId := ext.GetManagerId(); len(manageId) > 0 { - if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil { - self.ProjectId = provider.ProjectId - } + if projectSync && len(projectId) > 0 { + self.ProjectId = projectId } self.ChargeType = ext.GetInternetChargeType() @@ -248,7 +246,7 @@ func (self *SElasticip) SyncWithCloudEip(userCred mcclient.TokenCredential, ext return err } -func (manager *SElasticipManager) newFromCloudEip(userCred mcclient.TokenCredential, extEip cloudprovider.ICloudEIP, region *SCloudregion) (*SElasticip, error) { +func (manager *SElasticipManager) newFromCloudEip(userCred mcclient.TokenCredential, extEip cloudprovider.ICloudEIP, region *SCloudregion, projectId string) (*SElasticip, error) { eip := SElasticip{} eip.SetModelManager(manager) @@ -263,11 +261,10 @@ func (manager *SElasticipManager) newFromCloudEip(userCred mcclient.TokenCredent eip.ChargeType = extEip.GetInternetChargeType() eip.ProjectId = userCred.GetProjectId() - if manageId := extEip.GetManagerId(); len(manageId) > 0 { - if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil { - eip.ProjectId = provider.ProjectId - } + if len(projectId) > 0 { + eip.ProjectId = projectId } + err := manager.TableSpec().Insert(&eip) if err != nil { log.Errorf("newFromCloudEip fail %s", err) @@ -353,7 +350,7 @@ func (self *SElasticip) AssociateVM(userCred mcclient.TokenCredential, vm *SGues return nil } -func (manager *SElasticipManager) getEipByExtEip(userCred mcclient.TokenCredential, extEip cloudprovider.ICloudEIP, region *SCloudregion) (*SElasticip, error) { +func (manager *SElasticipManager) getEipByExtEip(userCred mcclient.TokenCredential, extEip cloudprovider.ICloudEIP, region *SCloudregion, projectId string) (*SElasticip, error) { eipObj, err := manager.FetchByExternalId(extEip.GetGlobalId()) if err == nil { return eipObj.(*SElasticip), nil @@ -363,7 +360,7 @@ func (manager *SElasticipManager) getEipByExtEip(userCred mcclient.TokenCredenti return nil, err } - return manager.newFromCloudEip(userCred, extEip, region) + return manager.newFromCloudEip(userCred, extEip, region, projectId) } func (manager *SElasticipManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 4a4a823959..c51a855e13 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -1337,7 +1337,7 @@ func (self *SGuest) GetIsolatedDevices() []SIsolatedDevice { return IsolatedDeviceManager.findAttachedDevicesOfGuest(self) } -func (self *SGuest) syncWithCloudVM(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, extVM cloudprovider.ICloudVM) error { +func (self *SGuest) syncWithCloudVM(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, extVM cloudprovider.ICloudVM, projectId string, projectSync bool) error { metaData := extVM.GetMetadata() diff, err := GuestManager.TableSpec().Update(self, func() error { extVM.Refresh() @@ -1353,10 +1353,9 @@ func (self *SGuest) syncWithCloudVM(ctx context.Context, userCred mcclient.Token self.Machine = extVM.GetMachine() self.HostId = host.Id self.ProjectId = userCred.GetProjectId() - if manageId := extVM.GetIHost().GetManagerId(); len(manageId) > 0 { - if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil { - self.ProjectId = provider.ProjectId - } + + if projectSync && len(projectId) > 0 { + self.ProjectId = projectId } self.Hypervisor = extVM.GetHypervisor() @@ -1402,7 +1401,7 @@ func (self *SGuest) syncWithCloudVM(ctx context.Context, userCred mcclient.Token return nil } -func (manager *SGuestManager) newCloudVM(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, extVM cloudprovider.ICloudVM) (*SGuest, error) { +func (manager *SGuestManager) newCloudVM(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, extVM cloudprovider.ICloudVM, projectId string) (*SGuest, error) { guest := SGuest{} guest.SetModelManager(manager) @@ -1428,10 +1427,8 @@ func (manager *SGuestManager) newCloudVM(ctx context.Context, userCred mcclient. guest.HostId = host.Id guest.ProjectId = userCred.GetProjectId() - if manageId := extVM.GetIHost().GetManagerId(); len(manageId) > 0 { - if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil { - guest.ProjectId = provider.ProjectId - } + if len(projectId) > 0 { + guest.ProjectId = projectId } metaData := extVM.GetMetadata() @@ -1905,7 +1902,7 @@ type sSyncDiskPair struct { vdisk cloudprovider.ICloudDisk } -func (self *SGuest) SyncVMDisks(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, vdisks []cloudprovider.ICloudDisk) compare.SyncResult { +func (self *SGuest) SyncVMDisks(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, vdisks []cloudprovider.ICloudDisk, projectId string, projectSync bool) compare.SyncResult { result := compare.SyncResult{} newdisks := make([]sSyncDiskPair, 0) @@ -1913,7 +1910,7 @@ func (self *SGuest) SyncVMDisks(ctx context.Context, userCred mcclient.TokenCred if len(vdisks[i].GetGlobalId()) == 0 { continue } - disk, err := DiskManager.syncCloudDisk(ctx, userCred, vdisks[i]) + disk, err := DiskManager.syncCloudDisk(ctx, userCred, vdisks[i], projectId, projectSync) if err != nil { result.Error(err) return result @@ -4282,7 +4279,7 @@ func (self *SGuest) GetEip() (*SElasticip, error) { return ElasticipManager.getEipForInstance("server", self.Id) } -func (self *SGuest) SyncVMEip(ctx context.Context, userCred mcclient.TokenCredential, extEip cloudprovider.ICloudEIP) compare.SyncResult { +func (self *SGuest) SyncVMEip(ctx context.Context, userCred mcclient.TokenCredential, extEip cloudprovider.ICloudEIP, projectId string) compare.SyncResult { result := compare.SyncResult{} eip, err := self.GetEip() @@ -4295,7 +4292,7 @@ func (self *SGuest) SyncVMEip(ctx context.Context, userCred mcclient.TokenCreden // do nothing } else if eip == nil && extEip != nil { // add - neip, err := ElasticipManager.getEipByExtEip(userCred, extEip, self.getRegion()) + neip, err := ElasticipManager.getEipByExtEip(userCred, extEip, self.getRegion(), projectId) if err != nil { result.AddError(err) } else { @@ -4324,7 +4321,7 @@ func (self *SGuest) SyncVMEip(ctx context.Context, userCred mcclient.TokenCreden result.DeleteError(err) } else { result.Delete() - neip, err := ElasticipManager.getEipByExtEip(userCred, extEip, self.getRegion()) + neip, err := ElasticipManager.getEipByExtEip(userCred, extEip, self.getRegion(), projectId) if err != nil { result.AddError(err) } else { @@ -4338,7 +4335,7 @@ func (self *SGuest) SyncVMEip(ctx context.Context, userCred mcclient.TokenCreden } } else { // do nothing - err := eip.SyncWithCloudEip(userCred, extEip) + err := eip.SyncWithCloudEip(userCred, extEip, projectId, false) if err != nil { result.UpdateError(err) } else { diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 80c153b325..4ce8d8e9ab 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -1210,7 +1210,7 @@ func (self *SHost) newCloudHostWire(ctx context.Context, userCred mcclient.Token return err } -func (self *SHost) SyncHostVMs(ctx context.Context, userCred mcclient.TokenCredential, vms []cloudprovider.ICloudVM) ([]SGuest, []cloudprovider.ICloudVM, compare.SyncResult) { +func (self *SHost) SyncHostVMs(ctx context.Context, userCred mcclient.TokenCredential, vms []cloudprovider.ICloudVM, projectId string, projectSync bool) ([]SGuest, []cloudprovider.ICloudVM, compare.SyncResult) { localVMs := make([]SGuest, 0) remoteVMs := make([]cloudprovider.ICloudVM, 0) syncResult := compare.SyncResult{} @@ -1238,7 +1238,7 @@ func (self *SHost) SyncHostVMs(ctx context.Context, userCred mcclient.TokenCrede } for i := 0; i < len(commondb); i += 1 { - err := commondb[i].syncWithCloudVM(ctx, userCred, self, commonext[i]) + err := commondb[i].syncWithCloudVM(ctx, userCred, self, commonext[i], projectId, projectSync) if err != nil { syncResult.UpdateError(err) } else { @@ -1249,7 +1249,7 @@ func (self *SHost) SyncHostVMs(ctx context.Context, userCred mcclient.TokenCrede } for i := 0; i < len(added); i += 1 { - new, err := GuestManager.newCloudVM(ctx, userCred, self, added[i]) + new, err := GuestManager.newCloudVM(ctx, userCred, self, added[i], projectId) if err != nil { syncResult.AddError(err) } else { diff --git a/pkg/compute/models/networks.go b/pkg/compute/models/networks.go index ce539e5431..3b718978b6 100644 --- a/pkg/compute/models/networks.go +++ b/pkg/compute/models/networks.go @@ -413,7 +413,7 @@ func (manager *SNetworkManager) getNetworksByWire(wire *SWire) ([]SNetwork, erro return nets, nil } -func (manager *SNetworkManager) SyncNetworks(ctx context.Context, userCred mcclient.TokenCredential, wire *SWire, nets []cloudprovider.ICloudNetwork) ([]SNetwork, []cloudprovider.ICloudNetwork, compare.SyncResult) { +func (manager *SNetworkManager) SyncNetworks(ctx context.Context, userCred mcclient.TokenCredential, wire *SWire, nets []cloudprovider.ICloudNetwork, projectId string, projectSync bool) ([]SNetwork, []cloudprovider.ICloudNetwork, compare.SyncResult) { localNets := make([]SNetwork, 0) remoteNets := make([]cloudprovider.ICloudNetwork, 0) syncResult := compare.SyncResult{} @@ -455,7 +455,7 @@ func (manager *SNetworkManager) SyncNetworks(ctx context.Context, userCred mccli } } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithCloudNetwork(userCred, commonext[i]) + err = commondb[i].SyncWithCloudNetwork(userCred, commonext[i], projectId, projectSync) if err != nil { syncResult.UpdateError(err) } else { @@ -465,7 +465,7 @@ func (manager *SNetworkManager) SyncNetworks(ctx context.Context, userCred mccli } } for i := 0; i < len(added); i += 1 { - new, err := manager.newFromCloudNetwork(userCred, added[i], wire) + new, err := manager.newFromCloudNetwork(userCred, added[i], wire, projectId) if err != nil { syncResult.AddError(err) } else { @@ -478,7 +478,7 @@ func (manager *SNetworkManager) SyncNetworks(ctx context.Context, userCred mccli return localNets, remoteNets, syncResult } -func (self *SNetwork) SyncWithCloudNetwork(userCred mcclient.TokenCredential, extNet cloudprovider.ICloudNetwork) error { +func (self *SNetwork) SyncWithCloudNetwork(userCred mcclient.TokenCredential, extNet cloudprovider.ICloudNetwork, projectId string, projectSync bool) error { _, err := self.GetModelManager().TableSpec().Update(self, func() error { extNet.Refresh() self.Name = extNet.GetName() @@ -493,10 +493,8 @@ func (self *SNetwork) SyncWithCloudNetwork(userCred mcclient.TokenCredential, ex self.AllocTimoutSeconds = extNet.GetAllocTimeoutSeconds() self.ProjectId = userCred.GetProjectId() - if manageId := extNet.GetIWire().GetIVpc().GetManagerId(); len(manageId) > 0 { - if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil { - self.ProjectId = provider.ProjectId - } + if projectSync && len(projectId) > 0 { + self.ProjectId = projectId } return nil }) @@ -506,7 +504,7 @@ func (self *SNetwork) SyncWithCloudNetwork(userCred mcclient.TokenCredential, ex return err } -func (manager *SNetworkManager) newFromCloudNetwork(userCred mcclient.TokenCredential, extNet cloudprovider.ICloudNetwork, wire *SWire) (*SNetwork, error) { +func (manager *SNetworkManager) newFromCloudNetwork(userCred mcclient.TokenCredential, extNet cloudprovider.ICloudNetwork, wire *SWire, projectId string) (*SNetwork, error) { net := SNetwork{} net.SetModelManager(manager) @@ -524,10 +522,8 @@ func (manager *SNetworkManager) newFromCloudNetwork(userCred mcclient.TokenCrede net.AllocTimoutSeconds = extNet.GetAllocTimeoutSeconds() net.ProjectId = userCred.GetProjectId() - if manageId := extNet.GetIWire().GetIVpc().GetManagerId(); len(manageId) > 0 { - if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil { - net.ProjectId = provider.ProjectId - } + if len(projectId) > 0 { + net.ProjectId = projectId } err := manager.TableSpec().Insert(&net) if err != nil { diff --git a/pkg/compute/models/snapshots.go b/pkg/compute/models/snapshots.go index bef171376e..12224b8947 100644 --- a/pkg/compute/models/snapshots.go +++ b/pkg/compute/models/snapshots.go @@ -225,11 +225,6 @@ func (self *SSnapshotManager) CreateSnapshot(ctx context.Context, userCred mccli snapshot := &SSnapshot{} snapshot.SetModelManager(self) snapshot.ProjectId = userCred.GetProjectId() - if manageId := disk.GetStorage().ManagerId; len(manageId) > 0 { - if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil { - snapshot.ProjectId = provider.ProjectId - } - } snapshot.DiskId = disk.Id snapshot.StorageId = disk.StorageId snapshot.Size = disk.DiskSize @@ -391,9 +386,12 @@ func totalSnapshotCount(projectId string) int { } // Only sync snapshot status -func (self *SSnapshot) SyncWithCloudSnapshot(userCred mcclient.TokenCredential, ext cloudprovider.ICloudSnapshot) error { +func (self *SSnapshot) SyncWithCloudSnapshot(userCred mcclient.TokenCredential, ext cloudprovider.ICloudSnapshot, projectId string, projectSync bool) error { _, err := self.GetModelManager().TableSpec().Update(self, func() error { self.Status = ext.GetStatus() + if projectSync && len(projectId) > 0 { + self.ProjectId = projectId + } return nil }) if err != nil { @@ -402,7 +400,7 @@ func (self *SSnapshot) SyncWithCloudSnapshot(userCred mcclient.TokenCredential, return err } -func (manager *SSnapshotManager) newFromCloudSnapshot(userCred mcclient.TokenCredential, extSnapshot cloudprovider.ICloudSnapshot, region *SCloudregion) (*SSnapshot, error) { +func (manager *SSnapshotManager) newFromCloudSnapshot(userCred mcclient.TokenCredential, extSnapshot cloudprovider.ICloudSnapshot, region *SCloudregion, projectId string) (*SSnapshot, error) { snapshot := SSnapshot{} snapshot.SetModelManager(manager) @@ -423,10 +421,8 @@ func (manager *SSnapshotManager) newFromCloudSnapshot(userCred mcclient.TokenCre snapshot.CloudregionId = region.Id snapshot.ProjectId = userCred.GetProjectId() - if len(snapshot.ManagerId) > 0 { - if provider := CloudproviderManager.FetchCloudproviderById(snapshot.ManagerId); provider != nil { - snapshot.ProjectId = provider.ProjectId - } + if len(projectId) > 0 { + snapshot.ProjectId = projectId } err := manager.TableSpec().Insert(&snapshot) if err != nil { @@ -449,7 +445,7 @@ func (manager *SSnapshotManager) getProviderSnapshotsByRegion(region *SCloudregi return snapshots, nil } -func (manager *SSnapshotManager) SyncSnapshots(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, snapshots []cloudprovider.ICloudSnapshot) compare.SyncResult { +func (manager *SSnapshotManager) SyncSnapshots(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, snapshots []cloudprovider.ICloudSnapshot, projectId string, projectSync bool) compare.SyncResult { syncResult := compare.SyncResult{} dbSnapshots, err := manager.getProviderSnapshotsByRegion(region, provider) if err != nil { @@ -475,7 +471,7 @@ func (manager *SSnapshotManager) SyncSnapshots(ctx context.Context, userCred mcc } } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithCloudSnapshot(userCred, commonext[i]) + err = commondb[i].SyncWithCloudSnapshot(userCred, commonext[i], projectId, projectSync) if err != nil { syncResult.UpdateError(err) } else { @@ -483,7 +479,7 @@ func (manager *SSnapshotManager) SyncSnapshots(ctx context.Context, userCred mcc } } for i := 0; i < len(added); i += 1 { - _, err := manager.newFromCloudSnapshot(userCred, added[i], region) + _, err := manager.newFromCloudSnapshot(userCred, added[i], region, projectId) if err != nil { syncResult.AddError(err) } else { diff --git a/pkg/compute/tasks/cloud_account_sync_task.go b/pkg/compute/tasks/cloud_account_sync_task.go index 434b6e5b3c..9b3062d12a 100644 --- a/pkg/compute/tasks/cloud_account_sync_task.go +++ b/pkg/compute/tasks/cloud_account_sync_task.go @@ -2,13 +2,12 @@ package tasks import ( "context" - "time" "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/pkg/utils" ) type CloudAccountSyncInfoTask struct { @@ -19,11 +18,29 @@ func init() { taskman.RegisterTask(CloudAccountSyncInfoTask{}) } -func (self *CloudAccountSyncInfoTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { - account := obj.(*models.SCloudaccount) - account.MarkStartSync(self.UserCred) +func (self *CloudAccountSyncInfoTask) OnInit(ctx context.Context, objs []db.IStandaloneModel, body jsonutils.JSONObject) { + cloudproviders := make([]*models.SCloudprovider, 0) + for _, obj := range objs { + cloudprovider := obj.(*models.SCloudprovider) + if cloudprovider.Enabled { + cloudproviders = append(cloudproviders, cloudprovider) + } + } + + var account *models.SCloudaccount + if len(cloudproviders) > 0 { + if account = models.CloudaccountManager.FetchCloudaccountById(cloudproviders[0].CloudaccountId); account == nil { + account.SetStatus(self.UserCred, models.CLOUD_PROVIDER_CONNECTED, "") + self.SetStageComplete(ctx, nil) + return + } + } else { + account.SetStatus(self.UserCred, models.CLOUD_PROVIDER_CONNECTED, "") + self.SetStageComplete(ctx, nil) + return + } if _, err := account.GetSubAccounts(); err != nil { - account.SetStatus(self.UserCred, models.CLOUD_PROVIDER_DISCONNECTED, err.Error()) + account.SetStatus(self.UserCred, models.CLOUD_PROVIDER_DISCONNECTED, "") self.SetStageFailed(ctx, err.Error()) return } @@ -33,28 +50,36 @@ func (self *CloudAccountSyncInfoTask) OnInit(ctx context.Context, obj db.IStanda if syncRangeJson != nil { syncRangeJson.Unmarshal(&syncRange) } - // do sync - exsitSubTask := false - self.SetStage("on_cloudaccount_sync_complete", nil) - for _, cloudprovider := range account.GetCloudproviders() { - if cloudprovider.Enabled { - exsitSubTask = true - cloudprovider.StartSyncCloudProviderInfoTask(ctx, self.UserCred, &syncRange, self.GetTaskId()) - } - } - if !exsitSubTask { + + if len(syncRange.Host) == 0 && !syncRange.FullSync { account.SetStatus(self.UserCred, models.CLOUD_PROVIDER_CONNECTED, "") self.SetStageComplete(ctx, nil) + return + } + // do sync + self.SetStage("on_cloudaccount_sync_complete", nil) + self.SyncCloudaccount(ctx, account, cloudproviders, &syncRange) +} + +func (self *CloudAccountSyncInfoTask) SyncCloudaccount(ctx context.Context, account *models.SCloudaccount, cloudproviders []*models.SCloudprovider, syncRange *models.SSyncRange) { + for _, cloudprovider := range cloudproviders { + self.SyncCloudprovider(ctx, cloudprovider, syncRange) } } -func (self *CloudAccountSyncInfoTask) OnCloudaccountSyncComplete(ctx context.Context, account *models.SCloudaccount, data jsonutils.JSONObject) { - for _, cloudprovider := range account.GetCloudproviders() { - if cloudprovider.Enabled && - utils.IsInStringArray(account.Status, []string{models.CLOUD_PROVIDER_START_SYNC, models.CLOUD_PROVIDER_SYNCING}) && time.Now().Sub(cloudprovider.LastSync) < time.Minute*20 { - return +func (self *CloudAccountSyncInfoTask) SyncCloudprovider(ctx context.Context, cloudprovider *models.SCloudprovider, syncRange *models.SSyncRange) { + lockman.LockObject(ctx, cloudprovider) + defer lockman.ReleaseObject(ctx, cloudprovider) + + cloudprovider.StartSyncCloudProviderInfoTask(ctx, self.UserCred, syncRange, self.GetId()) +} + +func (self *CloudAccountSyncInfoTask) OnCloudaccountSyncComplete(ctx context.Context, items []db.IStandaloneModel, data jsonutils.JSONObject) { + if len(items) > 0 { + cloudprovider := items[0].(*models.SCloudprovider) + if account := models.CloudaccountManager.FetchCloudaccountById(cloudprovider.CloudaccountId); account != nil { + account.SetStatus(self.UserCred, models.CLOUD_PROVIDER_CONNECTED, "") } } - account.SetStatus(self.UserCred, models.CLOUD_PROVIDER_CONNECTED, "") self.SetStageComplete(ctx, nil) } diff --git a/pkg/compute/tasks/cloud_provider_sync_info_task.go b/pkg/compute/tasks/cloud_provider_sync_info_task.go index 90bb917edd..2a53ccc4f3 100644 --- a/pkg/compute/tasks/cloud_provider_sync_info_task.go +++ b/pkg/compute/tasks/cloud_provider_sync_info_task.go @@ -108,11 +108,11 @@ func syncCloudProviderInfo(ctx context.Context, provider *models.SCloudprovider, if len(syncRange.Region) > 0 && !utils.IsInStringArray(localRegions[i].Id, syncRange.Region) { continue } - syncRegionEips(ctx, provider, task, &localRegions[i], remoteRegions[i]) + syncRegionEips(ctx, provider, task, &localRegions[i], remoteRegions[i], syncRange) localZones, remoteZones := syncRegionZones(ctx, provider, task, &localRegions[i], remoteRegions[i]) - syncRegionVPCs(ctx, provider, task, &localRegions[i], remoteRegions[i]) + syncRegionVPCs(ctx, provider, task, &localRegions[i], remoteRegions[i], syncRange) if localZones != nil && remoteZones != nil { for j := 0; j < len(localZones); j += 1 { @@ -120,15 +120,15 @@ func syncCloudProviderInfo(ctx context.Context, provider *models.SCloudprovider, if len(syncRange.Zone) > 0 && !utils.IsInStringArray(localZones[j].Id, syncRange.Zone) { continue } - syncZoneStorages(ctx, provider, task, &localZones[j], remoteZones[j]) + syncZoneStorages(ctx, provider, task, &localZones[j], remoteZones[j], syncRange) syncZoneHosts(ctx, provider, task, &localZones[j], remoteZones[j], syncRange) } } - syncRegionSnapshots(ctx, provider, task, &localRegions[i], remoteRegions[i]) + syncRegionSnapshots(ctx, provider, task, &localRegions[i], remoteRegions[i], syncRange) } } -func syncRegionSnapshots(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, localRegion *models.SCloudregion, remoteRegion cloudprovider.ICloudRegion) { +func syncRegionSnapshots(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, localRegion *models.SCloudregion, remoteRegion cloudprovider.ICloudRegion, syncRange *models.SSyncRange) { snapshots, err := remoteRegion.GetISnapshots() if err != nil { msg := fmt.Sprintf("GetISnapshots for region %s failed %s", remoteRegion.GetName(), err) @@ -137,7 +137,7 @@ func syncRegionSnapshots(ctx context.Context, provider *models.SCloudprovider, t return } - result := models.SnapshotManager.SyncSnapshots(ctx, task.GetUserCred(), provider, localRegion, snapshots) + result := models.SnapshotManager.SyncSnapshots(ctx, task.GetUserCred(), provider, localRegion, snapshots, provider.ProjectId, syncRange.ProjectSync) msg := result.Result() log.Infof("SyncSnapshots for region %s result: %s", localRegion.Name, msg) if result.IsError() { @@ -147,7 +147,7 @@ func syncRegionSnapshots(ctx context.Context, provider *models.SCloudprovider, t db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, task.GetUserCred()) } -func syncRegionEips(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, localRegion *models.SCloudregion, remoteRegion cloudprovider.ICloudRegion) { +func syncRegionEips(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, localRegion *models.SCloudregion, remoteRegion cloudprovider.ICloudRegion, syncRange *models.SSyncRange) { eips, err := remoteRegion.GetIEips() if err != nil { msg := fmt.Sprintf("GetIEips for region %s failed %s", remoteRegion.GetName(), err) @@ -156,7 +156,7 @@ func syncRegionEips(ctx context.Context, provider *models.SCloudprovider, task * return } - result := models.ElasticipManager.SyncEips(ctx, task.UserCred, provider, localRegion, eips) + result := models.ElasticipManager.SyncEips(ctx, task.UserCred, provider, localRegion, eips, provider.ProjectId, syncRange.ProjectSync) msg := result.Result() log.Infof("SyncEips for region %s result: %s", localRegion.Name, msg) if result.IsError() { @@ -187,7 +187,7 @@ func syncRegionZones(ctx context.Context, provider *models.SCloudprovider, task return localZones, remoteZones } -func syncRegionVPCs(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, localRegion *models.SCloudregion, remoteRegion cloudprovider.ICloudRegion) { +func syncRegionVPCs(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, localRegion *models.SCloudregion, remoteRegion cloudprovider.ICloudRegion, syncRange *models.SSyncRange) { vpcs, err := remoteRegion.GetIVpcs() if err != nil { msg := fmt.Sprintf("GetVpcs for region %s failed %s", remoteRegion.GetName(), err) @@ -207,7 +207,7 @@ func syncRegionVPCs(ctx context.Context, provider *models.SCloudprovider, task * db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, task.UserCred) logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true) for j := 0; j < len(localVpcs); j += 1 { - syncVpcWires(ctx, provider, task, &localVpcs[j], remoteVpcs[j]) + syncVpcWires(ctx, provider, task, &localVpcs[j], remoteVpcs[j], syncRange) syncVpcSecGroup(ctx, provider, task, &localVpcs[j], remoteVpcs[j]) } } @@ -230,7 +230,7 @@ func syncVpcSecGroup(ctx context.Context, provider *models.SCloudprovider, task } } -func syncVpcWires(ctx context.Context, provider *models.SCloudprovider, task taskman.ITask, localVpc *models.SVpc, remoteVpc cloudprovider.ICloudVpc) { +func syncVpcWires(ctx context.Context, provider *models.SCloudprovider, task taskman.ITask, localVpc *models.SVpc, remoteVpc cloudprovider.ICloudVpc, syncRange *models.SSyncRange) { wires, err := remoteVpc.GetIWires() if err != nil { msg := fmt.Sprintf("GetIWires for vpc %s failed %s", remoteVpc.GetId(), err) @@ -249,11 +249,11 @@ func syncVpcWires(ctx context.Context, provider *models.SCloudprovider, task tas db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, task.GetUserCred()) logclient.AddActionLog(provider, getAction(task.GetParams()), notes, task.GetUserCred(), true) for i := 0; i < len(localWires); i += 1 { - syncWireNetworks(ctx, provider, task, &localWires[i], remoteWires[i]) + syncWireNetworks(ctx, provider, task, &localWires[i], remoteWires[i], syncRange) } } -func syncWireNetworks(ctx context.Context, provider *models.SCloudprovider, task taskman.ITask, localWire *models.SWire, remoteWire cloudprovider.ICloudWire) { +func syncWireNetworks(ctx context.Context, provider *models.SCloudprovider, task taskman.ITask, localWire *models.SWire, remoteWire cloudprovider.ICloudWire, syncRange *models.SSyncRange) { nets, err := remoteWire.GetINetworks() if err != nil { msg := fmt.Sprintf("GetINetworks for wire %s failed %s", remoteWire.GetId(), err) @@ -261,7 +261,7 @@ func syncWireNetworks(ctx context.Context, provider *models.SCloudprovider, task logSyncFailed(provider, task, msg) return } - _, _, result := models.NetworkManager.SyncNetworks(ctx, task.GetUserCred(), localWire, nets) + _, _, result := models.NetworkManager.SyncNetworks(ctx, task.GetUserCred(), localWire, nets, provider.ProjectId, syncRange.ProjectSync) msg := result.Result() notes := fmt.Sprintf("SyncNetworks for wire %s result: %s", localWire.Name, msg) log.Infof(notes) @@ -273,7 +273,7 @@ func syncWireNetworks(ctx context.Context, provider *models.SCloudprovider, task logclient.AddActionLog(provider, getAction(task.GetParams()), notes, task.GetUserCred(), true) } -func syncZoneStorages(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, localZone *models.SZone, remoteZone cloudprovider.ICloudZone) { +func syncZoneStorages(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, localZone *models.SZone, remoteZone cloudprovider.ICloudZone, syncRange *models.SSyncRange) { storages, err := remoteZone.GetIStorages() if err != nil { msg := fmt.Sprintf("GetIStorages for zone %s failed %s", remoteZone.GetName(), err) @@ -294,7 +294,7 @@ func syncZoneStorages(ctx context.Context, provider *models.SCloudprovider, task for i := 0; i < len(localStorages); i += 1 { syncStorageCaches(ctx, provider, task, &localStorages[i], remoteStorages[i]) - syncStorageDisks(ctx, provider, task, &localStorages[i], remoteStorages[i]) + syncStorageDisks(ctx, provider, task, &localStorages[i], remoteStorages[i], syncRange) } } @@ -315,7 +315,7 @@ func syncStorageCaches(ctx context.Context, provider *models.SCloudprovider, tas } } -func syncStorageDisks(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, localStorage *models.SStorage, remoteStorage cloudprovider.ICloudStorage) { +func syncStorageDisks(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, localStorage *models.SStorage, remoteStorage cloudprovider.ICloudStorage, syncRange *models.SSyncRange) { disks, err := remoteStorage.GetIDisks() if err != nil { msg := fmt.Sprintf("GetIDisks for storage %s failed %s", remoteStorage.GetName(), err) @@ -323,7 +323,7 @@ func syncStorageDisks(ctx context.Context, provider *models.SCloudprovider, task logSyncFailed(provider, task, msg) return } - _, _, result := models.DiskManager.SyncDisks(ctx, task.UserCred, localStorage, disks) + _, _, result := models.DiskManager.SyncDisks(ctx, task.UserCred, localStorage, disks, provider.ProjectId, syncRange.ProjectSync) msg := result.Result() notes := fmt.Sprintf("SyncDisks for storage %s result: %s", localStorage.Name, msg) log.Infof(notes) @@ -359,7 +359,7 @@ func syncZoneHosts(ctx context.Context, provider *models.SCloudprovider, task *C } syncHostStorages(ctx, provider, task, &localHosts[i], remoteHosts[i]) syncHostWires(ctx, provider, task, &localHosts[i], remoteHosts[i]) - syncHostVMs(ctx, provider, task, &localHosts[i], remoteHosts[i]) + syncHostVMs(ctx, provider, task, &localHosts[i], remoteHosts[i], syncRange) } } @@ -403,7 +403,7 @@ func syncHostWires(ctx context.Context, provider *models.SCloudprovider, task ta logclient.AddActionLog(provider, getAction(task.GetParams()), notes, task.GetUserCred(), true) } -func syncHostVMs(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, localHost *models.SHost, remoteHost cloudprovider.ICloudHost) { +func syncHostVMs(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, localHost *models.SHost, remoteHost cloudprovider.ICloudHost, syncRange *models.SSyncRange) { vms, err := remoteHost.GetIVMs() if err != nil { msg := fmt.Sprintf("GetIVMs for host %s failed %s", remoteHost.GetName(), err) @@ -411,7 +411,7 @@ func syncHostVMs(ctx context.Context, provider *models.SCloudprovider, task *Clo logSyncFailed(provider, task, msg) return } - localVMs, remoteVMs, result := localHost.SyncHostVMs(ctx, task.UserCred, vms) + localVMs, remoteVMs, result := localHost.SyncHostVMs(ctx, task.UserCred, vms, provider.ProjectId, syncRange.ProjectSync) msg := result.Result() notes := fmt.Sprintf("SyncHostVMs for host %s result: %s", localHost.Name, msg) log.Infof(notes) @@ -423,7 +423,7 @@ func syncHostVMs(ctx context.Context, provider *models.SCloudprovider, task *Clo logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true) for i := 0; i < len(localVMs); i += 1 { syncVMNics(ctx, provider, task, localHost, &localVMs[i], remoteVMs[i]) - syncVMDisks(ctx, provider, task, localHost, &localVMs[i], remoteVMs[i]) + syncVMDisks(ctx, provider, task, localHost, &localVMs[i], remoteVMs[i], syncRange) syncVMEip(ctx, provider, task, &localVMs[i], remoteVMs[i]) } } @@ -448,7 +448,7 @@ func syncVMNics(ctx context.Context, provider *models.SCloudprovider, task *Clou logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true) } -func syncVMDisks(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, host *models.SHost, localVM *models.SGuest, remoteVM cloudprovider.ICloudVM) { +func syncVMDisks(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, host *models.SHost, localVM *models.SGuest, remoteVM cloudprovider.ICloudVM, syncRange *models.SSyncRange) { disks, err := remoteVM.GetIDisks() if err != nil { msg := fmt.Sprintf("GetIDisks for VM %s failed %s", remoteVM.GetName(), err) @@ -456,7 +456,7 @@ func syncVMDisks(ctx context.Context, provider *models.SCloudprovider, task *Clo logSyncFailed(provider, task, msg) return } - result := localVM.SyncVMDisks(ctx, task.UserCred, host, disks) + result := localVM.SyncVMDisks(ctx, task.UserCred, host, disks, provider.ProjectId, syncRange.ProjectSync) msg := result.Result() notes := fmt.Sprintf("syncVMNics for VM %s result: %s", localVM.Name, msg) log.Infof(notes) @@ -476,7 +476,7 @@ func syncVMEip(ctx context.Context, provider *models.SCloudprovider, task *Cloud logSyncFailed(provider, task, msg) return } - result := localVM.SyncVMEip(ctx, task.UserCred, eip) + result := localVM.SyncVMEip(ctx, task.UserCred, eip, provider.ProjectId) msg := result.Result() log.Infof("syncVMEip for VM %s result: %s", localVM.Name, msg) if result.IsError() { diff --git a/pkg/compute/tasks/eip_allocate_task.go b/pkg/compute/tasks/eip_allocate_task.go index 0ea90fa603..da5a8ff982 100644 --- a/pkg/compute/tasks/eip_allocate_task.go +++ b/pkg/compute/tasks/eip_allocate_task.go @@ -39,7 +39,7 @@ func (self *EipAllocateTask) OnInit(ctx context.Context, obj db.IStandaloneModel return } - err = eip.SyncWithCloudEip(self.UserCred, extEip) + err = eip.SyncWithCloudEip(self.UserCred, extEip, "", false) if err != nil { msg := fmt.Sprintf("sync eip fail %s", err) diff --git a/pkg/compute/tasks/eip_syncstatus_task.go b/pkg/compute/tasks/eip_syncstatus_task.go index 6df2bb0fa8..799d316bed 100644 --- a/pkg/compute/tasks/eip_syncstatus_task.go +++ b/pkg/compute/tasks/eip_syncstatus_task.go @@ -38,7 +38,7 @@ func (self *EipSyncstatusTask) OnInit(ctx context.Context, obj db.IStandaloneMod return } - err = eip.SyncWithCloudEip(self.UserCred, extEip) + err = eip.SyncWithCloudEip(self.UserCred, extEip, "", false) if err != nil { msg := fmt.Sprintf("fail to sync eip status %s", err) eip.SetStatus(self.UserCred, models.EIP_STATUS_UNKNOWN, msg) diff --git a/pkg/compute/tasks/network_create_task.go b/pkg/compute/tasks/network_create_task.go index 76919660bf..f22fec944c 100644 --- a/pkg/compute/tasks/network_create_task.go +++ b/pkg/compute/tasks/network_create_task.go @@ -64,7 +64,7 @@ func (self *NetworkCreateTask) OnInit(ctx context.Context, obj db.IStandaloneMod return } - err = network.SyncWithCloudNetwork(self.UserCred, inet) + err = network.SyncWithCloudNetwork(self.UserCred, inet, "", false) if err != nil { self.taskFailed(ctx, network, "SyncWithCloudNetwork", err) diff --git a/pkg/compute/tasks/vpc_create_task.go b/pkg/compute/tasks/vpc_create_task.go index 7708bb04f4..f4342dcaf6 100644 --- a/pkg/compute/tasks/vpc_create_task.go +++ b/pkg/compute/tasks/vpc_create_task.go @@ -57,7 +57,7 @@ func (self *VpcCreateTask) OnInit(ctx context.Context, obj db.IStandaloneModel, } provider := models.CloudproviderManager.FetchCloudproviderById(vpc.ManagerId) - syncVpcWires(ctx, provider, self, vpc, ivpc) + syncVpcWires(ctx, provider, self, vpc, ivpc, &models.SSyncRange{}) hosts := models.HostManager.GetHostsByManagerAndRegion(provider.Id, vpc.CloudregionId) if hosts != nil { diff --git a/pkg/util/esxi/manager.go b/pkg/util/esxi/manager.go index ed1dd5d76d..5c1f8c229c 100644 --- a/pkg/util/esxi/manager.go +++ b/pkg/util/esxi/manager.go @@ -88,6 +88,20 @@ func (cli *SESXiClient) disconnect() error { return nil } +func (cli *SESXiClient) GetSubAccounts() (jsonutils.JSONObject, error) { + if err := cli.connect(); err != nil { + return nil, err + } else { + result := jsonutils.NewDict() + data := jsonutils.NewArray() + account := jsonutils.Marshal(map[string]string{"account": cli.account, "name": cli.providerName}) + data.Add(account) + result.Add(data, "data") + result.Add(jsonutils.NewInt(1), "total") + return result, nil + } +} + func (cli *SESXiClient) About() jsonutils.JSONObject { return jsonutils.Marshal(&cli.client.ServiceContent.About) } diff --git a/pkg/util/esxi/provider/provider.go b/pkg/util/esxi/provider/provider.go index 02c5434485..f7d5362740 100644 --- a/pkg/util/esxi/provider/provider.go +++ b/pkg/util/esxi/provider/provider.go @@ -88,7 +88,7 @@ func (self *SESXiProvider) GetSysInfo() (jsonutils.JSONObject, error) { } func (self *SESXiProvider) GetSubAccounts() (jsonutils.JSONObject, error) { - return nil, nil + return self.client.GetSubAccounts() } func (self *SESXiProvider) GetIRegions() []cloudprovider.ICloudRegion {