修正：cloudprovider/cloudacount的代码recode

2026-09-19 02:37:24 +08:00 · 2018-10-18 12:22:50 +08:00
parent b319a5722b
commit dfa5a634ab
14 changed files with 483 additions and 273 deletions
@@ -31,15 +31,16 @@ func init() {
 	})

 	type CloudaccountCreateOptions struct {
-		NAME          string `help:"Name of cloud account"`
-		ACCOUNT       string `help:"Account to access the cloud account"`
-		SECRET        string `help:"Secret to access the cloud account, clientId/clientScret for Azure"`
-		PROVIDER      string `help:"Driver for cloud account" choices:"VMware|Aliyun|Azure|Qcloud"`
-		AccessURL     string `helo:"hello" metavar:"Azure choices: <AzureGermanCloud、AzureChinaCloud、AzureUSGovernmentCloud、AzurePublicCloud>"`
-		Desc          string `help:"Description"`
-		Enabled       bool   `help:"Enabled the account automatically"`
-		EnableProject bool   `help:"Enable the account with same name project"`
-		Import        bool   `help:"Import all sub account automatically"`
+		NAME      string `help:"Name of cloud account"`
+		ACCOUNT   string `help:"Account to access the cloud account"`
+		SECRET    string `help:"Secret to access the cloud account, clientId/clientScret for Azure"`
+		PROVIDER  string `help:"Driver for cloud account" choices:"VMware|Aliyun|Azure|Qcloud"`
+		AccessURL string `helo:"hello" metavar:"Azure choices: <AzureGermanCloud、AzureChinaCloud、AzureUSGovernmentCloud、AzurePublicCloud>"`
+		Desc      string `help:"Description"`
+
+		Import            bool `help:"Import all sub account automatically"`
+		AutoSync          bool `help:"Enabled the account automatically"`
+		AutoCreateProject bool `help:"Enable the account with same name project"`
 	}
 	R(&CloudaccountCreateOptions{}, "cloud-account-create", "Create a cloud account", func(s *mcclient.ClientSession, args *CloudaccountCreateOptions) error {
 		params := jsonutils.NewDict()
@@ -47,15 +48,17 @@ func init() {
 		params.Add(jsonutils.NewString(args.ACCOUNT), "account")
 		params.Add(jsonutils.NewString(args.SECRET), "secret")
 		params.Add(jsonutils.NewString(args.PROVIDER), "provider")
-		if args.Enabled {
-			params.Add(jsonutils.JSONTrue, "enabled")
-		}
-		if args.EnableProject {
-			params.Add(jsonutils.JSONTrue, "enable_project")
-		}
+
 		if args.Import {
 			params.Add(jsonutils.JSONTrue, "import")
+			if args.AutoSync {
+				params.Add(jsonutils.JSONTrue, "auto_sync")
+			}
+			if args.AutoCreateProject {
+				params.Add(jsonutils.JSONTrue, "auto_create_project")
+			}
 		}
+
 		if len(args.AccessURL) > 0 {
 			params.Add(jsonutils.NewString(args.AccessURL), "access_url")
 		}
@@ -147,12 +150,16 @@ func init() {
 	})

 	type CloudaccountImportOptions struct {
-		ID            string `help:"ID or Name of cloud account"`
-		Enabled       bool   `help:"Import sub accounts with enabled status"`
-		EnableProject bool   `help:"Import sub account with project"`
+		ID                string `help:"ID or Name of cloud account" json:"-"`
+		AutoSync          bool   `help:"Import sub accounts with enabled status"`
+		AutoCreateProject bool   `help:"Import sub account with project"`
 	}
 	R(&CloudaccountImportOptions{}, "cloud-account-import", "Import sub cloud account", func(s *mcclient.ClientSession, args *CloudaccountImportOptions) error {
-		result, err := modules.Cloudaccounts.PerformAction(s, args.ID, "import", jsonutils.Marshal(map[string]bool{"enabled": args.Enabled, "enable_project": args.EnableProject}))
+		params, err := options.StructToParams(args)
+		if err != nil {
+			return err
+		}
+		result, err := modules.Cloudaccounts.PerformAction(s, args.ID, "import", params)
 		if err != nil {
 			return err
 		}
@@ -13,7 +13,6 @@ type ICloudProviderFactory interface {
 }

 type ICloudProvider interface {
-	GetSubAccounts() (jsonutils.JSONObject, error)
 	GetId() string
 	GetName() string
 	GetIRegions() []ICloudRegion
@@ -28,6 +27,8 @@ type ICloudProvider interface {
 	GetIStoragecacheById(id string) (ICloudStoragecache, error)

 	GetBalance() (float64, error)
+
+	GetSubAccounts() ([]SSubAccount, error)
 }

 var providerTable map[string]ICloudProviderFactory
@@ -0,0 +1,7 @@
+package cloudprovider
+
+type SSubAccount struct {
+	Name string
+	State string
+	Account string
+}
@@ -18,6 +18,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
 )

 type SCloudaccountManager struct {
@@ -53,8 +54,10 @@ type SCloudaccount struct {
 func (self *SCloudaccount) GetCloudproviders() []SCloudprovider {
 	cloudproviders := []SCloudprovider{}
 	q := CloudproviderManager.Query().Equals("cloudaccount_id", self.Id)
-	if err := db.FetchModelObjects(CloudproviderManager, q, &cloudproviders); err != nil {
+	err := db.FetchModelObjects(CloudproviderManager, q, &cloudproviders)
+	if err != nil {
 		log.Errorf("getCloudproviders error: %v", err)
+		return nil
 	}
 	return cloudproviders
 }
@@ -73,9 +76,9 @@ func (self *SCloudaccount) ValidateUpdateData(ctx context.Context, userCred mccl
 	return self.SEnabledStatusStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, data)
 }

-func (self *SCloudaccountManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+func (manager *SCloudaccountManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	// check provider
-	name, _ := data.GetString("name")
+	// name, _ := data.GetString("name")
 	provider, _ := data.GetString("provider")
 	if !cloudprovider.IsSupported(provider) {
 		return nil, httperrors.NewInputParameterError("Unsupported provider %s", provider)
@@ -83,66 +86,33 @@ func (self *SCloudaccountManager) ValidateCreateData(ctx context.Context, userCr
 	// check duplication
 	// url, account, provider must be unique
 	account, _ := data.GetString("account")
-	secret, _ := data.GetString("secret")
+	// secret, _ := data.GetString("secret")
 	url, _ := data.GetString("access_url")
-	q := self.Query().Equals("provider", provider)
+
+	q := manager.Query().Equals("provider", provider)
 	if len(account) > 0 {
 		q = q.Equals("account", account)
 	}
 	if len(url) > 0 {
 		q = q.Equals("access_url", url)
 	}
+
 	if q.Count() > 0 {
 		return nil, httperrors.NewConflictError("The account has been registered")
 	}

-	if subAccount, err := GetSubAccounts(name, url, account, secret, provider); err != nil {
-		return nil, httperrors.NewInputParameterError("Invalidate account with secret")
-	} else if accounts, err := subAccount.GetArray("data"); err != nil {
-		return nil, err
-	} else {
-		data.Add(jsonutils.NewArray(accounts...), "accounts")
-	}
-	return self.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)
+	return manager.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)
 }

 func (self *SCloudaccount) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) {
 	self.SEnabledStatusStandaloneResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data)
 	self.savePassword(self.Secret)
-	_import, _ := data.Bool("import")
-	enableProject, _ := data.Bool("enable_project")
-	if subAccounts, err := data.GetArray("accounts"); err == nil && _import && len(subAccounts) > 0 {
-		for _, subAccount := range subAccounts {
-			name, _ := subAccount.GetString("name")
-			account, _ := subAccount.GetString("account")
-			if len(name) > 0 && len(account) > 0 {
-				if q := CloudproviderManager.Query().Equals("cloudaccount_id", self.Id).Equals("account", account); q.Count() > 0 {
-					log.Errorf("sub account conflict")
-				} else {
-					newCloudprovider := SCloudprovider{
-						Account:        account,
-						CloudaccountId: self.Id,
-						Provider:       self.Provider,
-					}
-					newCloudprovider.SetModelManager(CloudproviderManager)
-					if err := CloudproviderManager.TableSpec().Insert(&newCloudprovider); err != nil {
-						log.Errorf("Create cloudprovider error: %v", err)
-					} else if _, err := CloudproviderManager.TableSpec().Update(&newCloudprovider, func() error {
-						newCloudprovider.Name = name
-						return nil
-					}); err != nil {
-						log.Errorf("Update cloudprovider error: %v", err)
-					}
-					if enableProject {
-						newCloudprovider.SyncProject()
-					}
-				}
-			}
-		}
-	}

-	if self.Enabled {
-		self.startSyncCloudProviderInfoTask(ctx, userCred, nil, "")
+	doImport := jsonutils.QueryBoolean(data, "import", false)
+	if doImport {
+		autoCreateProject := jsonutils.QueryBoolean(data, "auto_create_project", false)
+		autoSync := jsonutils.QueryBoolean(data, "auto_sync", false)
+		self.startImportSubAccountTask(ctx, userCred, autoCreateProject, autoSync, "")
 	}
 }

@@ -189,7 +159,7 @@ func (self *SCloudaccount) PerformSync(ctx context.Context, userCred mcclient.To
 		return nil, httperrors.NewInputParameterError("invalid input %s", err)
 	}
 	if self.CanSync() || syncRange.Force {
-		err = self.startSyncCloudProviderInfoTask(ctx, userCred, &syncRange, "")
+		err = self.StartSyncCloudProviderInfoTask(ctx, userCred, nil, &syncRange, "")
 	}
 	return nil, err
 }
@@ -218,6 +188,7 @@ func (self *SCloudaccount) PerformUpdateCredential(ctx context.Context, userCred
 			return nil, httperrors.NewConflictError("Access url and account conflict")
 		}
 	}
+
 	if len(secret) > 0 {
 		err = self.savePassword(secret)
 		if err != nil {
@@ -229,10 +200,11 @@ func (self *SCloudaccount) PerformUpdateCredential(ctx context.Context, userCred
 		if len(account) > 0 && account != self.Account {
 			for _, cloudprovider := range self.GetCloudproviders() {
 				if cloudprovider.Account == self.Account {
-					if _, err = cloudprovider.GetModelManager().TableSpec().Update(&cloudprovider, func() error {
+					_, err = cloudprovider.GetModelManager().TableSpec().Update(&cloudprovider, func() error {
 						cloudprovider.Account = account
 						return nil
-					}); err != nil {
+					})
+					if err != nil {
 						return nil, err
 					}
 				}
@@ -254,22 +226,25 @@ func (self *SCloudaccount) PerformUpdateCredential(ctx context.Context, userCred
 	}
 	if changed {
 		self.SetStatus(userCred, CLOUD_PROVIDER_INIT, "Change credential")
-		self.startSyncCloudProviderInfoTask(ctx, userCred, nil, "")
+		self.StartSyncCloudProviderInfoTask(ctx, userCred, nil, nil, "")
 	}
 	return nil, nil
 }

-func (self *SCloudaccount) startSyncCloudProviderInfoTask(ctx context.Context, userCred mcclient.TokenCredential, syncRange *SSyncRange, parentTaskId string) error {
+func (self *SCloudaccount) StartSyncCloudProviderInfoTask(ctx context.Context, userCred mcclient.TokenCredential, cloudProviders []SCloudprovider, syncRange *SSyncRange, parentTaskId string) error {
 	params := jsonutils.NewDict()
 	if syncRange != nil {
 		params.Add(jsonutils.Marshal(syncRange), "sync_range")
 	}
-	cloudproviders := self.GetCloudproviders()
+
+	if cloudProviders == nil {
+		cloudProviders = self.GetCloudproviders()
+	}

 	taskItems := make([]db.IStandaloneModel, 0)
-	for i := 0; i < len(cloudproviders); i++ {
-		if cloudproviders[i].Enabled {
-			taskItems = append(taskItems, &cloudproviders[i])
+	for i := 0; i < len(cloudProviders); i++ {
+		if cloudProviders[i].Enabled {
+			taskItems = append(taskItems, &cloudProviders[i])
 		}
 	}

@@ -308,12 +283,76 @@ func (self *SCloudaccount) GetDriver() (cloudprovider.ICloudProvider, error) {
 	return cloudprovider.GetProvider(self.Id, self.Name, self.AccessUrl, self.Account, secret, self.Provider)
 }

-func (self *SCloudaccount) GetSubAccounts() (jsonutils.JSONObject, error) {
-	if secret, err := self.getPassword(); err != nil {
+func (self *SCloudaccount) GetSubAccounts() ([]cloudprovider.SSubAccount, error) {
+	secret, err := self.getPassword()
+	if err != nil {
 		return nil, err
-	} else {
-		return GetSubAccounts(self.Name, self.AccessUrl, self.Account, secret, self.Provider)
 	}
+	return getSubAccounts(self.Name, self.AccessUrl, self.Account, secret, self.Provider)
+}
+
+func (self *SCloudaccount) ImportSubAccount(ctx context.Context, userCred mcclient.TokenCredential, subAccount cloudprovider.SSubAccount, autoCreateProject bool) (*SCloudprovider, bool, error) {
+	isNew := false
+	q := CloudproviderManager.Query().Equals("cloudaccount_id", self.Id).Equals("account", subAccount.Account)
+	providerCount := q.Count()
+	if providerCount > 1 {
+		log.Errorf("cloudaccount %s has duplicate subaccount with name %s", self.Name, subAccount.Account)
+		return nil, isNew, cloudprovider.ErrDuplicateId
+	}
+	if providerCount == 1 {
+		providerObj, err := db.NewModelObject(CloudproviderManager)
+		if err != nil {
+			return nil, isNew, err
+		}
+		provider := providerObj.(*SCloudprovider)
+		err = q.First(provider)
+		if err != nil {
+			return nil, isNew, err
+		}
+
+		_, err = CloudproviderManager.TableSpec().Update(provider, func() error {
+			provider.Name = subAccount.Name
+			provider.Enabled = true
+			return nil
+		})
+
+		if err != nil {
+			log.Errorf("Update cloudprovider error: %v", err)
+			return nil, isNew, err
+		}
+
+		return provider, isNew, nil
+	}
+	// not found, create a new cloudprovider
+	isNew = true
+
+	newCloudprovider := SCloudprovider{}
+	newCloudprovider.Account = subAccount.Account
+	newCloudprovider.CloudaccountId = self.Id
+	newCloudprovider.Provider =     self.Provider
+	newCloudprovider.Enabled = true
+	newCloudprovider.Name = subAccount.Name
+	if ! autoCreateProject {
+		newCloudprovider.ProjectId = auth.AdminCredential().GetProjectId()
+	}
+
+	newCloudprovider.SetModelManager(CloudproviderManager)
+
+	err := CloudproviderManager.TableSpec().Insert(&newCloudprovider)
+	if err != nil {
+		log.Errorf("insert new cloudprovider fail %s", err)
+		return nil, isNew, err
+	}
+
+	if autoCreateProject {
+		err = newCloudprovider.syncProject(ctx)
+		if err != nil {
+			log.Errorf("syncproject fail %s", err)
+			return nil, isNew, err
+		}
+	}
+
+	return &newCloudprovider, isNew, nil
 }

 func (self *SCloudaccount) AllowPerformImport(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
@@ -321,56 +360,37 @@ func (self *SCloudaccount) AllowPerformImport(ctx context.Context, userCred mccl
 }

 func (self *SCloudaccount) PerformImport(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	if subAccounts, err := self.GetSubAccounts(); err != nil {
-		return nil, err
-	} else if accounts, err := subAccounts.GetArray("data"); err != nil {
-		return nil, err
-	} else {
-		enabled, _ := data.Bool("enabled")
-		enableProject, _ := data.Bool("enable_project")
-		for _, _account := range accounts {
-			name, _ := _account.GetString("name")
-			account, _ := _account.GetString("account")
-			if len(name) > 0 && len(account) > 0 {
-				if q := CloudproviderManager.Query().Equals("cloudaccount_id", self.Id).Equals("account", account); q.Count() > 0 {
-					log.Debugf("account %s has imported", account)
-				} else {
-					newCloudprovider := SCloudprovider{
-						Account:        account,
-						CloudaccountId: self.Id,
-						Provider:       self.Provider,
-					}
-					newCloudprovider.SetModelManager(CloudproviderManager)
-					if err := CloudproviderManager.TableSpec().Insert(&newCloudprovider); err != nil {
-						log.Errorf("Create cloudprovider error: %v", err)
-						return nil, err
-					} else if _, err := CloudproviderManager.TableSpec().Update(&newCloudprovider, func() error {
-						newCloudprovider.Name = name
-						newCloudprovider.Enabled = true
-						return nil
-					}); err != nil {
-						log.Errorf("Update cloudprovider error: %v", err)
-						return nil, err
-					}
-					if enableProject {
-						newCloudprovider.SyncProject()
-					}
-					if enabled {
-						newCloudprovider.StartSyncCloudProviderInfoTask(ctx, userCred, &SSyncRange{FullSync: true}, "")
-					}
-				}
-			}
-		}
-	}
-	return nil, nil
+	autoCreateProject := jsonutils.QueryBoolean(data, "auto_create_project", false)
+	autoSync := jsonutils.QueryBoolean(data, "auto_sync", false)
+
+	err := self.startImportSubAccountTask(ctx, userCred, autoCreateProject, autoSync, "")
+
+	return nil, err
 }

-func GetSubAccounts(name, accessUrl, account, secret, provider string) (jsonutils.JSONObject, error) {
-	if provider, err := cloudprovider.GetProvider("", name, accessUrl, account, secret, provider); err != nil {
-		return nil, err
-	} else {
-		return provider.GetSubAccounts()
+func (self *SCloudaccount) startImportSubAccountTask(ctx context.Context, userCred mcclient.TokenCredential, autoCreateProject bool, autoSync bool, parentTaskId string) error {
+	params := jsonutils.NewDict()
+	if autoCreateProject {
+		params.Add(jsonutils.JSONTrue, "auto_create_project")
 	}
+	if autoSync {
+		params.Add(jsonutils.JSONTrue, "auto_sync")
+	}
+	task, err := taskman.TaskManager.NewTask(ctx, "CloudAccountImportTask", self, userCred, params, parentTaskId, "", nil)
+	if err != nil {
+		log.Errorf("create task fail %s", err)
+		return err
+	}
+	task.ScheduleRun(nil)
+	return nil
+}
+
+func getSubAccounts(name, accessUrl, account, secret, provider string) ([]cloudprovider.SSubAccount, error) {
+	iprovider, err := cloudprovider.GetProvider("", name, accessUrl, account, secret, provider)
+	if err != nil {
+		return nil, err
+	}
+	return iprovider.GetSubAccounts()
 }

 func (self *SCloudaccount) SaveSysInfo(info jsonutils.JSONObject) {
@@ -415,65 +435,90 @@ func (self *SCloudaccount) GetExtraDetails(ctx context.Context, userCred mcclien
 	return self.getMoreDetails(extra)
 }

+func migrateCloudprovider(cloudprovider *SCloudprovider) error {
+	mainAccount, providerAccount, providerName := cloudprovider.Account, cloudprovider.Account, cloudprovider.Name
+
+	if cloudprovider.Provider == CLOUD_PROVIDER_AZURE {
+		accountInfo := strings.Split(cloudprovider.Account, "/");
+		if len(accountInfo) == 2 {
+			mainAccount, providerAccount = accountInfo[0], accountInfo[1]
+			if len(cloudprovider.Description) > 0 {
+				providerName = cloudprovider.Description
+			}
+		} else {
+			msg := fmt.Sprintf("error azure provider account format %s", cloudprovider.Account)
+			log.Errorf(msg)
+			return fmt.Errorf(msg)
+		}
+	}
+
+	account := SCloudaccount{}
+	account.SetModelManager(CloudaccountManager)
+	q := CloudaccountManager.Query().Equals("access_url", cloudprovider.AccessUrl).
+		Equals("account", mainAccount).
+		Equals("provider", cloudprovider.Provider)
+	err := q.First(&account)
+
+	if err != nil && err != sql.ErrNoRows {
+		return err
+	}
+
+	if err == sql.ErrNoRows {
+		account.AccessUrl = cloudprovider.AccessUrl
+		account.Account = mainAccount
+		account.LastSync = cloudprovider.LastSync
+		account.Sysinfo = cloudprovider.Sysinfo
+		account.Provider = cloudprovider.Provider
+		account.Name = providerName
+		account.Status = cloudprovider.Status
+
+		err := CloudaccountManager.TableSpec().Insert(&account)
+		if err != nil {
+			log.Errorf("Insert Account error: %v", err)
+			return err
+		}
+
+		secret, err := cloudprovider.getPassword()
+		if err != nil {
+			account.SetStatus(auth.AdminCredential(), CLOUD_PROVIDER_DISCONNECTED, "invalid secret")
+			log.Errorf("Get password from provider %s error %v", cloudprovider.Name, err)
+		} else {
+			err = account.savePassword(secret)
+			if err != nil {
+				log.Errorf("Set password for account %s error %v", account.Name, err)
+				return err
+			}
+		}
+	}
+
+	_, err = CloudproviderManager.TableSpec().Update(cloudprovider, func() error {
+		cloudprovider.CloudaccountId = account.Id
+		cloudprovider.Account = providerAccount
+		cloudprovider.Secret = ""
+		cloudprovider.Name = providerName
+		return nil
+	})
+	if err != nil {
+		log.Errorf("Update provider %s error: %v", cloudprovider.Name, err)
+		return err
+	}
+
+	return nil
+}
+
 func (manager *SCloudaccountManager) InitializeData() error {
 	cloudproviders := []SCloudprovider{}
 	q := CloudproviderManager.Query()
 	q = q.Filter(sqlchemy.OR(sqlchemy.IsEmpty(q.Field("cloudaccount_id")), sqlchemy.IsNull(q.Field("cloudaccount_id"))))
-	if err := db.FetchModelObjects(CloudproviderManager, q, &cloudproviders); err != nil {
+	err := db.FetchModelObjects(CloudproviderManager, q, &cloudproviders)
+	if err != nil {
+		log.Errorf("fetch all clound provider fail %s", err)
 		return err
 	}
 	for i := 0; i < len(cloudproviders); i++ {
-		cloudprovider := cloudproviders[i]
-		Account, providerAccount, providerName := cloudprovider.Account, "", cloudprovider.Name
-		if cloudprovider.Provider == CLOUD_PROVIDER_AZURE {
-			if accountInfo := strings.Split(cloudprovider.Account, "/"); len(accountInfo) == 2 {
-				Account, providerAccount = accountInfo[0], accountInfo[1]
-				if len(cloudprovider.Description) > 0 {
-					providerName = cloudprovider.Description
-				}
-			} else {
-				log.Errorf("Error provider format %s", cloudprovider.Account)
-				continue
-			}
-		}
-		account := SCloudaccount{}
-		account.SetModelManager(CloudaccountManager)
-		if err := CloudaccountManager.Query().
-			Equals("access_url", cloudprovider.AccessUrl).
-			Equals("account", Account).
-			Equals("provider", cloudprovider.Provider).First(&account); err != nil {
-			account = SCloudaccount{
-				AccessUrl: cloudprovider.AccessUrl,
-				Account:   Account,
-				LastSync:  cloudprovider.LastSync,
-				Sysinfo:   cloudprovider.Sysinfo,
-				Provider:  cloudprovider.Provider,
-			}
-			if err := CloudaccountManager.TableSpec().Insert(&account); err != nil {
-				log.Errorf("Insert Account error: %v", err)
-			}
-			if _, err := CloudaccountManager.TableSpec().Update(&account, func() error {
-				account.Name = Account
-				account.Status = cloudprovider.Status
-				return nil
-			}); err != nil {
-				log.Errorf("Update Account %s error: %v", account.Id, err)
-			}
-			if secret, err := cloudprovider.getPassword(); err != nil {
-				log.Errorf("Get password from provider %s error %v", cloudprovider.Name, err)
-			} else if err := account.savePassword(secret); err != nil {
-				log.Errorf("Set password for account %s error %v", account.Name, err)
-			}
-		}
-		if _, err := CloudproviderManager.TableSpec().Update(&cloudprovider, func() error {
-			log.Errorf("update %s", cloudprovider.Name)
-			cloudprovider.CloudaccountId = account.Id
-			cloudprovider.Account = providerAccount
-			cloudprovider.Secret = ""
-			cloudprovider.Name = providerName
-			return nil
-		}); err != nil {
-			log.Errorf("Update provider %s error: %v", cloudprovider.Name, err)
+		err = migrateCloudprovider(&cloudproviders[i])
+		if err != nil {
+			return err
 		}
 	}
 	return nil
@@ -8,14 +8,15 @@ import (

 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/sqlchemy"
+	"yunion.io/x/pkg/util/timeutils"
+	"yunion.io/x/pkg/utils"
+
 	"yunion.io/x/onecloud/pkg/compute/options"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
-	"yunion.io/x/pkg/util/timeutils"
-	"yunion.io/x/pkg/utils"
-
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -103,6 +104,10 @@ func (self *SCloudprovider) getEipCount() int {
 	return ElasticipManager.Query().Equals("manager_id", self.Id).Count()
 }

+func (self *SCloudprovider) getSnapshotCount() int {
+	return SnapshotManager.Query().Equals("manager_id", self.Id).Count()
+}
+
 func (self *SCloudprovider) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	return self.SEnabledStatusStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, data)
 }
@@ -127,27 +132,58 @@ func (self *SCloudprovider) CanSync() bool {
 	}
 }

-func (self *SCloudprovider) SyncProject() (err error) {
-	projectId := ""
-	if len(self.ProjectId) == 0 && len(self.Name) > 0 {
-		if tenant, err := db.TenantCacheManager.FetchTenantByIdOrName(context.Background(), self.Name); err != nil {
-			s := auth.GetAdminSession(options.Options.Region, "")
-			if project, err := modules.Projects.Create(s, jsonutils.Marshal(map[string]string{"name": self.Name})); err != nil {
-				return err
-			} else if projectId, err = project.GetString("id"); err != nil {
-				return err
-			}
-		} else {
-			projectId = tenant.Id
+func (self *SCloudprovider) syncProject(ctx context.Context) error {
+	if len(self.ProjectId) > 0 {
+		_, err := db.TenantCacheManager.FetchTenantById(ctx, self.ProjectId)
+		if err != nil && err != sql.ErrNoRows {
+			log.Errorf("fetch existing tenant by id fail %s", err)
+		} else if err == nil {
+			return nil // find the project, skip sync
 		}
 	}
-	if len(projectId) > 0 {
-		_, err := self.GetModelManager().TableSpec().Update(self, func() error {
-			self.ProjectId = projectId
-			return nil
-		})
+
+	if len(self.Name) == 0 {
+		log.Errorf("syncProject: provider name is empty???")
+		return fmt.Errorf("cannot syncProject for empty name")
+	}
+
+	tenant, err := db.TenantCacheManager.FetchTenantByIdOrName(ctx, self.Name)
+	if err != nil && err != sql.ErrNoRows {
+		log.Errorf("fetchTenantByIdorName error %s: %s", self.Name, err)
 		return err
 	}
+
+	var projectId string
+	if err == sql.ErrNoRows { // create one
+		s := auth.GetAdminSession(options.Options.Region, "")
+		params := jsonutils.NewDict()
+		params.Add(jsonutils.NewString(self.Name), "name")
+		params.Add(jsonutils.NewString(fmt.Sprintf("auto create from cloud provider %s", self.Name, self.Id)), "description")
+
+		project, err := modules.Projects.Create(s, params)
+
+		if err != nil {
+			log.Errorf("create project fail %s", err)
+			return err
+		}
+		projectId, err = project.GetString("id")
+		if err != nil {
+			return err
+		}
+	} else {
+		projectId = tenant.Id
+	}
+
+	_, err = self.GetModelManager().TableSpec().Update(self, func() error {
+		self.ProjectId = projectId
+		return nil
+	})
+
+	if err != nil {
+		log.Errorf("update projectId fail: %s", err)
+		return err
+	}
+
 	return nil
 }

@@ -281,19 +317,31 @@ func (self *SCloudprovider) AllowPerformChangeProject(ctx context.Context, userC
 }

 func (self *SCloudprovider) PerformChangeProject(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	if project, err := data.GetString("project"); err != nil {
+	project, err := data.GetString("project")
+	if err != nil {
 		return nil, httperrors.NewInputParameterError("Missing project parameter")
-	} else if tenant, err := db.TenantCacheManager.FetchTenantByIdOrName(ctx, project); err != nil {
+	}
+
+	tenant, err := db.TenantCacheManager.FetchTenantByIdOrName(ctx, project)
+	if err != nil {
 		return nil, httperrors.NewNotFoundError("project %s not found", project)
-	} else if _, err := self.GetModelManager().TableSpec().Update(self, func() error {
+	}
+
+	if self.ProjectId == tenant.Id {
+		return nil, nil
+	}
+
+	_, err = self.GetModelManager().TableSpec().Update(self, func() error {
 		self.ProjectId = tenant.Id
 		return nil
-	}); err != nil {
+	})
+
+	if err != nil {
 		log.Errorf("Update cloudprovider error: %v", err)
-		return nil, err
-	} else {
-		return nil, self.StartSyncCloudProviderInfoTask(ctx, userCred, &SSyncRange{FullSync: true, ProjectSync: true}, "")
+		return nil, httperrors.NewGeneralError(err)
 	}
+
+	return nil, self.StartSyncCloudProviderInfoTask(ctx, userCred, &SSyncRange{FullSync: true, ProjectSync: true}, "")
 }

 func (self *SCloudprovider) MarkStartSync(userCred mcclient.TokenCredential) {
@@ -326,29 +374,32 @@ type SAccount struct {
 	Secret    string
 }

-func (self *SCloudprovider) getCloudaccount() (*SCloudaccount, error) {
-	if cloudaccount := CloudaccountManager.FetchCloudaccountById(self.CloudaccountId); cloudaccount != nil {
-		return cloudaccount, nil
-	}
-	return nil, fmt.Errorf("Failed to find cloud account for cloud provider %s", self.Name)
+func (self *SCloudprovider) GetCloudaccount() (*SCloudaccount) {
+	return CloudaccountManager.FetchCloudaccountById(self.CloudaccountId)
 }

-func (self *SCloudprovider) getAccount() (*SAccount, error) {
-	if cloudaccount, err := self.getCloudaccount(); err != nil {
-		return nil, err
-	} else {
-		account := SAccount{AccessUrl: cloudaccount.AccessUrl, Account: cloudaccount.Account}
-		if passwd, err := cloudaccount.getPassword(); err != nil {
-			return nil, err
-		} else {
-			account.Secret = passwd
-		}
-		if len(self.Account) > 0 && self.Account != cloudaccount.Account {
-			account.Account = fmt.Sprintf("%s/%s", account.Account, self.Account)
-		}
-		return &account, nil
+func (self *SCloudprovider) getAccount() (SAccount, error) {
+	account := SAccount{}
+
+	cloudaccount := self.GetCloudaccount()
+	if cloudaccount == nil {
+		return account, fmt.Errorf("fail to find cloudaccount???")
 	}

+	passwd, err := cloudaccount.getPassword()
+	if err != nil {
+		return account, err
+	}
+
+	account.Account = cloudaccount.Account
+	account.AccessUrl = cloudaccount.AccessUrl
+	account.Secret = passwd
+
+	if len(self.Account) > 0 && self.Account != account.Account {
+		account.Account = fmt.Sprintf("%s/%s", account.Account, self.Account)
+	}
+
+	return account, nil
 }

 func (self *SCloudprovider) SaveSysInfo(info jsonutils.JSONObject) {
@@ -385,6 +436,7 @@ type SCloudproviderUsage struct {
 	StorageCount      int
 	StorageCacheCount int
 	EipCount          int
+	SnapshotCount     int
 }

 func (usage *SCloudproviderUsage) isEmpty() bool {
@@ -403,34 +455,48 @@ func (usage *SCloudproviderUsage) isEmpty() bool {
 	if usage.EipCount > 0 {
 		return false
 	}
+	if usage.SnapshotCount > 0 {
+		return false
+	}
 	return true
 }

 func (self *SCloudprovider) getUsage() *SCloudproviderUsage {
 	usage := SCloudproviderUsage{}
+
 	usage.GuestCount = self.GetGuestCount()
 	usage.HostCount = self.GetHostCount()
 	usage.VpcCount = self.getVpcCount()
 	usage.StorageCount = self.getStorageCount()
 	usage.StorageCacheCount = self.getStoragecacheCount()
 	usage.EipCount = self.getEipCount()
+	usage.SnapshotCount = self.getSnapshotCount()

 	return &usage
 }

-func (self *SCloudprovider) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict {
+func (self *SCloudprovider) getProject(ctx context.Context) *db.STenant {
+	proj, _ := db.TenantCacheManager.FetchTenantById(ctx, self.ProjectId)
+	return proj
+}
+
+func (self *SCloudprovider) getMoreDetails(ctx context.Context, extra *jsonutils.JSONDict) *jsonutils.JSONDict {
 	extra.Update(jsonutils.Marshal(self.getUsage()))
+	project := self.getProject(ctx)
+	if project != nil {
+		extra.Add(jsonutils.NewString(project.Name), "tenant")
+	}
 	return extra
 }

 func (self *SCloudprovider) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
 	extra := self.SEnabledStatusStandaloneResourceBase.GetCustomizeColumns(ctx, userCred, query)
-	return self.getMoreDetails(extra)
+	return self.getMoreDetails(ctx, extra)
 }

 func (self *SCloudprovider) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
 	extra := self.SEnabledStatusStandaloneResourceBase.GetExtraDetails(ctx, userCred, query)
-	return self.getMoreDetails(extra)
+	return self.getMoreDetails(ctx, extra)
 }

 func (manager *SCloudproviderManager) InitializeData() error {
@@ -465,6 +531,27 @@ func (manager *SCloudproviderManager) InitializeData() error {
 			log.Debugf("vcenter info has been migrate into cloudprovider")
 		}
 	}
+
+	// fill empty projectId with system project ID
+	providers := make([]SCloudprovider, 0)
+	q = CloudproviderManager.Query()
+	q = q.Filter(sqlchemy.OR(sqlchemy.IsEmpty(q.Field("tenant_id")), sqlchemy.IsNull(q.Field("tenant_id"))))
+	err = db.FetchModelObjects(CloudproviderManager, q, &providers)
+	if err != nil {
+		log.Errorf("query cloudproviders with empty tenant_id fail %s", err)
+		return err
+	}
+	for i := 0; i < len(providers); i += 1 {
+		_, err := CloudproviderManager.TableSpec().Update(&providers[i], func() error {
+			providers[i].ProjectId = auth.AdminCredential().GetProjectId()
+			return nil
+		})
+		if err != nil {
+			log.Errorf("update cloudprovider project fail %s", err)
+			return err
+		}
+	}
+
 	return nil
 }

@@ -20,7 +20,7 @@ func InitDB() error {
 		err := manager.InitializeData()
 		if err != nil {
 			log.Errorf("Manager %s initializeData fail %s", manager.Keyword(), err)
-			return err
+			// return err skip error table
 		}
 	}
 	return nil
@@ -0,0 +1,65 @@
+package tasks
+
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/compute/models"
+)
+
+
+type CloudAccountImportTask struct {
+	taskman.STask
+}
+
+func init() {
+	taskman.RegisterTask(CloudAccountImportTask{})
+}
+
+func (self *CloudAccountImportTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	cloudAccount := obj.(*models.SCloudaccount)
+
+	cloudAccount.MarkStartSync(self.UserCred)
+
+	autoCreateProject := jsonutils.QueryBoolean(self.Params, "auto_create_project", false)
+	autoSync := jsonutils.QueryBoolean(self.Params, "auto_sync", false)
+
+	newProviders := make([]models.SCloudprovider, 0)
+	subAccounts, err := cloudAccount.GetSubAccounts()
+	if err != nil {
+		cloudAccount.SetStatus(self.UserCred, models.CLOUD_PROVIDER_DISCONNECTED, "")
+		self.SetStageFailed(ctx, err.Error())
+		return
+	}
+	for i := 0; i < len(subAccounts); i += 1 {
+		provider, isNew, err := cloudAccount.ImportSubAccount(ctx, self.UserCred, subAccounts[i], autoCreateProject)
+		if err != nil {
+			cloudAccount.SetStatus(self.UserCred, models.CLOUD_PROVIDER_DISCONNECTED, "")
+			self.SetStageFailed(ctx, err.Error())
+			return
+		}
+		if isNew {
+			newProviders = append(newProviders, *provider)
+		}
+	}
+
+	if autoSync {
+		self.SetStage("SyncNewCloudProviderComplete", nil)
+		cloudAccount.StartSyncCloudProviderInfoTask(ctx, self.UserCred, newProviders, nil, self.GetTaskId())
+	} else {
+		self.complete(ctx, cloudAccount)
+	}
+}
+
+func (self *CloudAccountImportTask) SyncNewCloudProviderComplete(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	cloudAccount := obj.(*models.SCloudaccount)
+	self.complete(ctx, cloudAccount)
+}
+
+func (self *CloudAccountImportTask) complete(ctx context.Context, cloudAccount *models.SCloudaccount) {
+	cloudAccount.SetStatus(self.UserCred, models.CLOUD_PROVIDER_CONNECTED, "import complete")
+	self.SetStageComplete(ctx, nil)
+}
@@ -5,7 +5,6 @@ import (

 	"yunion.io/x/jsonutils"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
-	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/compute/models"
 )
@@ -29,16 +28,19 @@ func (self *CloudAccountSyncInfoTask) OnInit(ctx context.Context, objs []db.ISta

 	var account *models.SCloudaccount
 	if len(cloudproviders) > 0 {
-		if account = models.CloudaccountManager.FetchCloudaccountById(cloudproviders[0].CloudaccountId); account == nil {
+		account = models.CloudaccountManager.FetchCloudaccountById(cloudproviders[0].CloudaccountId)
+		if account == nil { // ???
 			account.SetStatus(self.UserCred, models.CLOUD_PROVIDER_CONNECTED, "")
 			self.SetStageComplete(ctx, nil)
 			return
 		}
+		account.MarkStartSync(self.UserCred)
 	} else {
 		account.SetStatus(self.UserCred, models.CLOUD_PROVIDER_CONNECTED, "")
 		self.SetStageComplete(ctx, nil)
 		return
 	}
+
 	if _, err := account.GetSubAccounts(); err != nil {
 		account.SetStatus(self.UserCred, models.CLOUD_PROVIDER_DISCONNECTED, "")
 		self.SetStageFailed(ctx, err.Error())
@@ -62,14 +64,14 @@ func (self *CloudAccountSyncInfoTask) OnInit(ctx context.Context, objs []db.ISta
 }

 func (self *CloudAccountSyncInfoTask) SyncCloudaccount(ctx context.Context, account *models.SCloudaccount, cloudproviders []*models.SCloudprovider, syncRange *models.SSyncRange) {
-	for _, cloudprovider := range cloudproviders {
-		self.SyncCloudprovider(ctx, cloudprovider, syncRange)
+	for i := 0; i < len(cloudproviders); i += 1 {
+		self.SyncCloudprovider(ctx, cloudproviders[i], syncRange)
 	}
 }

 func (self *CloudAccountSyncInfoTask) SyncCloudprovider(ctx context.Context, cloudprovider *models.SCloudprovider, syncRange *models.SSyncRange) {
-	lockman.LockObject(ctx, cloudprovider)
-	defer lockman.ReleaseObject(ctx, cloudprovider)
+	// lockman.LockObject(ctx, cloudprovider)
+	// defer lockman.ReleaseObject(ctx, cloudprovider)

 	cloudprovider.StartSyncCloudProviderInfoTask(ctx, self.UserCred, syncRange, self.GetId())
 }
@@ -77,7 +79,8 @@ func (self *CloudAccountSyncInfoTask) SyncCloudprovider(ctx context.Context, clo
 func (self *CloudAccountSyncInfoTask) OnCloudaccountSyncComplete(ctx context.Context, items []db.IStandaloneModel, data jsonutils.JSONObject) {
 	if len(items) > 0 {
 		cloudprovider := items[0].(*models.SCloudprovider)
-		if account := models.CloudaccountManager.FetchCloudaccountById(cloudprovider.CloudaccountId); account != nil {
+		account := cloudprovider.GetCloudaccount()
+		if account != nil {
 			account.SetStatus(self.UserCred, models.CLOUD_PROVIDER_CONNECTED, "")
 		}
 	}
@@ -121,18 +121,15 @@ func (self *SAliyunClient) GetRegions() []SRegion {
 	return regions
 }

-func (self *SAliyunClient) GetSubAccounts() (jsonutils.JSONObject, error) {
-	if err := self.fetchRegions(); err != nil {
+func (self *SAliyunClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error) {
+	err := self.fetchRegions()
+	if err != nil {
 		return nil, err
-	} else {
-		result := jsonutils.NewDict()
-		data := jsonutils.NewArray()
-		account := jsonutils.Marshal(map[string]string{"account": self.accessKey, "name": self.providerName})
-		data.Add(account)
-		result.Add(data, "data")
-		result.Add(jsonutils.NewInt(1), "total")
-		return result, nil
 	}
+	subAccount := cloudprovider.SSubAccount{}
+	subAccount.Name = self.providerName
+	subAccount.Account = self.accessKey
+	return []cloudprovider.SSubAccount{subAccount}, nil
 }

 func (self *SAliyunClient) GetIRegions() []cloudprovider.ICloudRegion {
@@ -71,7 +71,7 @@ func (self *SAliyunProvider) GetSysInfo() (jsonutils.JSONObject, error) {
 	return info, nil
 }

-func (self *SAliyunProvider) GetSubAccounts() (jsonutils.JSONObject, error) {
+func (self *SAliyunProvider) GetSubAccounts() ([]cloudprovider.SSubAccount, error) {
 	return self.client.GetSubAccounts()
 }

@@ -153,25 +153,25 @@ func (self *SAzureClient) GetRegions() []SRegion {
 	return regions
 }

-func (self *SAzureClient) GetSubAccounts() (jsonutils.JSONObject, error) {
+func (self *SAzureClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error) {
 	subClient := subscription.NewSubscriptionsClientWithBaseURI(self.baseUrl)
 	subClient.Authorizer = self.authorizer
-	result := jsonutils.NewDict()
-	accounts := jsonutils.NewArray()
-	if resp, err := subClient.List(context.Background()); err != nil {
+	resp, err := subClient.List(context.Background())
+	if err != nil {
 		return nil, err
-	} else {
-		for _, value := range resp.Values() {
-			data := jsonutils.NewDict()
-			data.Add(jsonutils.NewString(*value.SubscriptionID), "account")
-			data.Add(jsonutils.NewString(string(value.State)), "state")
-			data.Add(jsonutils.NewString(*value.DisplayName), "name")
-			accounts.Add(data)
-		}
-		result.Add(accounts, "data")
-		result.Add(jsonutils.NewInt(int64(accounts.Length())), "total")
 	}
-	return result, nil
+
+	subAccounts := make([]cloudprovider.SSubAccount, len(resp.Values()))
+
+	for i, value := range resp.Values() {
+		subAccounts[i] = cloudprovider.SSubAccount{
+			Account: *value.SubscriptionID,
+			State: string(value.State),
+			Name: *value.DisplayName,
+		}
+	}
+
+	return subAccounts, nil
 }

 func (self *SAzureClient) GetIRegions() []cloudprovider.ICloudRegion {
@@ -51,7 +51,7 @@ func (self *SAzureProvider) GetSysInfo() (jsonutils.JSONObject, error) {
 	return info, nil
 }

-func (self *SAzureProvider) GetSubAccounts() (jsonutils.JSONObject, error) {
+func (self *SAzureProvider) GetSubAccounts() ([]cloudprovider.SSubAccount, error) {
 	return self.client.GetSubAccounts()
 }

@@ -88,18 +88,16 @@ func (cli *SESXiClient) disconnect() error {
 	return nil
 }

-func (cli *SESXiClient) GetSubAccounts() (jsonutils.JSONObject, error) {
-	if err := cli.connect(); err != nil {
+func (cli *SESXiClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error) {
+	err := cli.connect()
+	if err != nil {
 		return nil, err
-	} else {
-		result := jsonutils.NewDict()
-		data := jsonutils.NewArray()
-		account := jsonutils.Marshal(map[string]string{"account": cli.account, "name": cli.providerName})
-		data.Add(account)
-		result.Add(data, "data")
-		result.Add(jsonutils.NewInt(1), "total")
-		return result, nil
 	}
+	subAccount := cloudprovider.SSubAccount{
+		Account: cli.account,
+		Name: cli.providerName,
+	}
+	return []cloudprovider.SSubAccount{subAccount}, nil
 }

 func (cli *SESXiClient) About() jsonutils.JSONObject {
@@ -87,7 +87,7 @@ func (self *SESXiProvider) GetSysInfo() (jsonutils.JSONObject, error) {
 	return self.client.About(), nil
 }

-func (self *SESXiProvider) GetSubAccounts() (jsonutils.JSONObject, error) {
+func (self *SESXiProvider) GetSubAccounts() ([]cloudprovider.SSubAccount, error) {
 	return self.client.GetSubAccounts()
 }