Merge pull request #6969 from ioito/automated-cherry-pick-of-#6903-upstream-release-3.2

Automated cherry pick of #6903: fix: 修正项目映射同步

pkg/apis/compute/externalproject.go

@@ -16,8 +16,16 @@ package compute
 
 import "yunion.io/x/onecloud/pkg/apis"
 
+const (
+	EXTERNAL_PROJECT_STATUS_AVAILABLE   = "available"   // 可用
+	EXTERNAL_PROJECT_STATUS_UNAVAILABLE = "unavailable" // 不可用
+	EXTERNAL_PROJECT_STATUS_CREATING    = "creating"    // 创建中
+	EXTERNAL_PROJECT_STATUS_DELETING    = "deleting"    // 删除中
+	EXTERNAL_PROJECT_STATUS_UNKNOWN     = "unknown"     // 未知
+)
+
 type ExternalProjectDetails struct {
-	apis.StandaloneResourceDetails
+	apis.StatusStandaloneResourceDetails
 	apis.ProjectizedResourceInfo
 	ManagedResourceInfo
 

pkg/apis/compute/input.go

@@ -66,7 +66,7 @@ type CachedimageListInput struct {
 }
 
 type ExternalProjectListInput struct {
-	apis.StandaloneResourceListInput
+	apis.StatusStandaloneResourceListInput
 	apis.ProjectizedResourceListInput
 	apis.ExternalizedResourceBaseListInput
 

pkg/compute/guestdrivers/esxi.go

@@ -234,10 +234,26 @@ func (self *SESXiGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gues
 	}
 	config.Add(jsonutils.NewString(extId), "guest_ext_id")
 
-	accessInfo, err := host.GetCloudaccount().GetVCenterAccessInfo(storage.ExternalId)
+	account := host.GetCloudaccount()
+	accessInfo, err := account.GetVCenterAccessInfo(storage.ExternalId)
 	if err != nil {
 		return err
 	}
+
+	action, _ := config.GetString("action")
+	if action == "create" {
+		extProj, name, err := account.GetExternalProject(ctx, task.GetUserCred(), guest.ProjectId)
+		if err != nil {
+			log.Errorf("failed to get external project %s from account %s(%s) error: %v", guest.ProjectId, account.Name, account.Id, err)
+		}
+		if extProj != nil {
+			config.Add(jsonutils.NewString(extProj.ExternalId), "desc", "group_id")
+		}
+		if len(name) > 0 {
+			config.Add(jsonutils.NewString(name), "desc", "resource_pool")
+		}
+	}
+
 	config.Add(jsonutils.Marshal(accessInfo), "datastore")
 
 	url := "/disks/agent/deploy"

pkg/compute/hostdrivers/managedvirtual.go

@@ -275,7 +275,7 @@ func (self *SManagedVirtualizationHostDriver) RequestAllocateDiskOnStorage(ctx c
 
 		cloudprovider.WaitStatus(iDisk, api.DISK_READY, time.Second*5, time.Minute*5)
 
-		models.SyncMetadata(ctx, task.GetUserCred(), disk, iDisk)
+		models.SyncVirtualResourceMetadata(ctx, task.GetUserCred(), disk, iDisk)
 
 		data := jsonutils.NewDict()
 		data.Add(jsonutils.NewInt(int64(iDisk.GetDiskSizeMB())), "disk_size")

pkg/compute/models/cloudaccounts.go

@@ -1954,6 +1954,16 @@ func (self *SCloudaccount) Delete(ctx context.Context, userCred mcclient.TokenCr
 
 func (self *SCloudaccount) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
 	self.SetStatus(userCred, api.CLOUD_PROVIDER_DELETED, "real delete")
+	projects, err := self.GetExternalProjects()
+	if err != nil {
+		return errors.Wrap(err, "GetExternalProjects")
+	}
+	for i := range projects {
+		err = projects[i].Delete(ctx, userCred)
+		if err != nil {
+			return errors.Wrapf(err, "project %s Delete", projects[i].Id)
+		}
+	}
 	return self.SEnabledStatusInfrasResourceBase.Delete(ctx, userCred)
 }
 
@@ -2284,6 +2294,16 @@ func (account *SCloudaccount) PerformSyncSkus(ctx context.Context, userCred mccl
 	return nil, nil
 }
 
+func (self *SCloudaccount) GetExternalProjects() ([]SExternalProject, error) {
+	projects := []SExternalProject{}
+	q := ExternalProjectManager.Query().Equals("cloudaccount_id", self.Id)
+	err := db.FetchModelObjects(ExternalProjectManager, q, &projects)
+	if err != nil {
+		return nil, errors.Wrap(err, "db.FetchModelObjects")
+	}
+	return projects, nil
+}
+
 func (manager *SCloudaccountManager) queryCloudAccountByCapability(region *SCloudregion, zone *SZone, domainId string, enabled tristate.TriState, capability string) *sqlchemy.SQuery {
 	providers := CloudproviderManager.Query().SubQuery()
 	q := manager.Query()
@@ -2375,3 +2395,61 @@ func (account *SCloudaccount) GetUsages() []db.IUsage {
 		&usage,
 	}
 }
+
+func (self *SCloudaccount) GetExternalProject(ctx context.Context, userCred mcclient.TokenCredential, id string) (*SExternalProject, string, error) {
+	projects, err := self.GetExternalProjects()
+	if err != nil {
+		return nil, "", errors.Wrap(err, "GetExternalProjects")
+	}
+	for i := range projects {
+		if projects[i].ProjectId == id && projects[i].Status == api.EXTERNAL_PROJECT_STATUS_AVAILABLE {
+			return &projects[i], projects[i].Name, nil
+		}
+	}
+
+	project, err := db.TenantCacheManager.FetchById(id)
+	if err != nil {
+		return nil, "", errors.Wrap(err, "TenantCacheManager.FetchById")
+	}
+
+	for i := range projects {
+		if projects[i].Name == project.GetName() {
+			if projects[i].Status != api.EXTERNAL_PROJECT_STATUS_AVAILABLE {
+				return nil, "", fmt.Errorf("external project %s not available", projects[i].Name)
+			}
+			return &projects[i], project.GetName(), nil
+		}
+	}
+	return nil, project.GetName(), cloudprovider.ErrNotFound
+}
+
+func (self *SCloudaccount) SyncProject(ctx context.Context, userCred mcclient.TokenCredential, id string) (string, error) {
+	lockman.LockRawObject(ctx, self.Id, id)
+	defer lockman.ReleaseRawObject(ctx, self.Id, id)
+
+	project, projectName, err := self.GetExternalProject(ctx, userCred, id)
+	if err == nil {
+		return project.ExternalId, nil
+	}
+	if err != cloudprovider.ErrNotFound {
+		return "", err
+	}
+
+	if len(projectName) == 0 {
+		return "", fmt.Errorf("empty project name")
+	}
+
+	provider, err := self.GetProvider()
+	if err != nil {
+		return "", errors.Wrap(err, "GetProvider")
+	}
+	iProject, err := provider.CreateIProject(projectName)
+	if err != nil {
+		return "", errors.Wrap(err, "CreateIProject")
+	}
+	extProj, err := ExternalProjectManager.newFromCloudProject(ctx, userCred, self, iProject)
+	if err != nil {
+		return "", errors.Wrap(err, "newFromCloudProject")
+	}
+	return extProj.ExternalId, nil
+}

pkg/compute/models/cloudproviders.go

@@ -1372,7 +1372,6 @@ func (self *SCloudprovider) RealDelete(ctx context.Context, userCred mcclient.To
 		ElasticipManager,
 		NetworkInterfaceManager,
 		CloudproviderRegionManager,
-		ExternalProjectManager,
 		CloudregionManager,
 		CloudproviderQuotaManager,
 	} {
@@ -1628,51 +1627,10 @@ func (provider *SCloudprovider) GetChangeOwnerCandidateDomainIds() []string {
 	return []string{}
 }
 
-func (self *SCloudprovider) GetExternalProjects() ([]SExternalProject, error) {
-	q := ExternalProjectManager.Query().Equals("manager_id", self.Id)
-	projects := []SExternalProject{}
-	err := db.FetchModelObjects(ExternalProjectManager, q, &projects)
-	if err != nil {
-		return nil, errors.Wrap(err, "FetchModelObjects")
-	}
-	return projects, nil
-}
-
 func (self *SCloudprovider) SyncProject(ctx context.Context, userCred mcclient.TokenCredential, id string) (string, error) {
-	lockman.LockRawObject(ctx, self.Id, id)
-	defer lockman.ReleaseRawObject(ctx, self.Id, id)
-
-	projects, err := self.GetExternalProjects()
-	if err != nil {
-		return "", errors.Wrap(err, "GetExternalProjects")
+	account := self.GetCloudaccount()
+	if account == nil {
+		return "", fmt.Errorf("failed to get cloudprovider %s account", self.Name)
 	}
-	for _, project := range projects {
-		if project.ProjectId == id {
-			return project.ExternalId, nil
-		}
-	}
-
-	project, err := db.TenantCacheManager.FetchById(id)
-	if err != nil {
-		return "", errors.Wrap(err, "TenantCacheManager.FetchById")
-	}
-
-	for _, extProj := range projects {
-		if extProj.Name == project.GetName() {
-			return extProj.ExternalId, nil
-		}
-	}
-	provider, err := self.GetProvider()
-	if err != nil {
-		return "", errors.Wrap(err, "GetProvider")
-	}
-	iProject, err := provider.CreateIProject(project.GetName())
-	if err != nil {
-		return "", errors.Wrap(err, "CreateIProject")
-	}
-	extProj, err := ExternalProjectManager.newFromCloudProject(ctx, userCred, self, iProject)
-	if err != nil {
-		return "", errors.Wrap(err, "newFromCloudProject")
-	}
-	return extProj.ExternalId, nil
+	return account.SyncProject(ctx, userCred, id)
 }

pkg/compute/models/cloudsync.go

@@ -172,28 +172,6 @@ func syncRegionSkus(ctx context.Context, userCred mcclient.TokenCredential, loca
 	}
 }
 
-func syncProjects(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, driver cloudprovider.ICloudProvider, provider *SCloudprovider) {
-	projects, err := driver.GetIProjects()
-	if err != nil {
-		msg := fmt.Sprintf("GetIProjects for provider %s failed %s", provider.GetName(), err)
-		log.Errorf(msg)
-		// logSyncFailed(provider, task, msg)
-		return
-	}
-
-	result := ExternalProjectManager.SyncProjects(ctx, userCred, provider, projects)
-
-	syncResults.Add(ExternalProjectManager, result)
-
-	msg := result.Result()
-	log.Infof("SyncProjects for provider %s result: %s", provider.Name, msg)
-	if result.IsError() {
-		// logSyncFailed(provider, task, msg)
-		return
-	}
-	// db.OpsLog.LogEvent(provider, db.ACT_SYNC_PROJECT_COMPLETE, msg, task.UserCred)
-}
-
 func syncRegionEips(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion, syncRange *SSyncRange) {
 	eips, err := remoteRegion.GetIEips()
 	if err != nil {
@@ -658,7 +636,7 @@ func syncHostVMs(ctx context.Context, userCred mcclient.TokenCredential, syncRes
 }
 
 func syncVMPeripherals(ctx context.Context, userCred mcclient.TokenCredential, local *SGuest, remote cloudprovider.ICloudVM, host *SHost, provider *SCloudprovider, driver cloudprovider.ICloudProvider) {
-	syncMetadata(ctx, userCred, local, remote)
+	syncVirtualResourceMetadata(ctx, userCred, local, remote)
 	err := syncVMNics(ctx, userCred, provider, host, local, remote)
 	if err != nil {
 		log.Errorf("syncVMNics error %s", err)
@@ -1027,10 +1005,6 @@ func syncPublicCloudProviderInfo(
 
 	storageCachePairs := make([]sStoragecacheSyncPair, 0)
 
-	if cloudprovider.IsSupportProject(driver) {
-		syncProjects(ctx, userCred, syncResults, driver, provider)
-	}
-
 	syncRegionQuotas(ctx, userCred, syncResults, driver, provider, localRegion, remoteRegion)
 
 	localZones, remoteZones, _ := syncRegionZones(ctx, userCred, syncResults, provider, localRegion, remoteRegion)
@@ -1124,10 +1098,6 @@ func syncOnPremiseCloudProviderInfo(
 ) error {
 	log.Debugf("Start sync on-premise provider %s(%s)", provider.Name, provider.Provider)
 
-	if cloudprovider.IsSupportProject(driver) {
-		syncProjects(ctx, userCred, syncResults, driver, provider)
-	}
-
 	iregion, err := driver.GetOnPremiseIRegion()
 	if err != nil {
 		msg := fmt.Sprintf("GetOnPremiseIRegion for provider %s failed %s", provider.GetName(), err)
@@ -1221,7 +1191,7 @@ func SyncCloudProject(userCred mcclient.TokenCredential, model db.IVirtualModel,
 	if extProjectId := extModel.GetProjectId(); len(extProjectId) > 0 {
 		extProject, err := ExternalProjectManager.GetProject(extProjectId, managerId)
 		if err != nil {
-			log.Errorln(err)
+			log.Errorf("sync project for %s %s error: %v", model.Keyword(), model.GetName(), err)
 		} else {
 			newOwnerId = extProject.GetOwnerId()
 		}

pkg/compute/models/dbinstances.go

@@ -1296,7 +1296,7 @@ func (manager *SDBInstanceManager) SyncDBInstances(ctx context.Context, userCred
 			syncResult.UpdateError(err)
 			continue
 		}
-		syncMetadata(ctx, userCred, &commondb[i], commonext[i])
+		syncVirtualResourceMetadata(ctx, userCred, &commondb[i], commonext[i])
 		localDBInstances = append(localDBInstances, commondb[i])
 		remoteDBInstances = append(remoteDBInstances, commonext[i])
 		syncResult.Update()
@@ -1308,7 +1308,7 @@ func (manager *SDBInstanceManager) SyncDBInstances(ctx context.Context, userCred
 			syncResult.AddError(err)
 			continue
 		}
-		syncMetadata(ctx, userCred, instance, added[i])
+		syncVirtualResourceMetadata(ctx, userCred, instance, added[i])
 		localDBInstances = append(localDBInstances, *instance)
 		remoteDBInstances = append(remoteDBInstances, added[i])
 		syncResult.Add()

pkg/compute/models/disks.go

@@ -1138,6 +1138,14 @@ func (self *SDisk) IsLocal() bool {
 	return false
 }
 
+func (self *SDisk) GetCloudproviderId() string {
+	storage := self.GetStorage()
+	if storage != nil {
+		return storage.GetCloudproviderId()
+	}
+	return ""
+}
+
 func (self *SDisk) GetStorage() *SStorage {
 	store, _ := StorageManager.FetchById(self.StorageId)
 	if store != nil {
@@ -1284,7 +1292,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To
 		if err != nil {
 			syncResult.UpdateError(err)
 		} else {
-			syncMetadata(ctx, userCred, &commondb[i], commonext[i])
+			syncVirtualResourceMetadata(ctx, userCred, &commondb[i], commonext[i])
 			localDisks = append(localDisks, commondb[i])
 			remoteDisks = append(remoteDisks, commonext[i])
 			syncResult.Update()
@@ -1314,7 +1322,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To
 		if err != nil {
 			syncResult.AddError(err)
 		} else {
-			syncMetadata(ctx, userCred, new, added[i])
+			syncVirtualResourceMetadata(ctx, userCred, new, added[i])
 			localDisks = append(localDisks, *new)
 			remoteDisks = append(remoteDisks, added[i])
 			syncResult.Add()

pkg/compute/models/elasticcache_instances.go

@@ -472,7 +472,7 @@ func (manager *SElasticcacheManager) SyncElasticcaches(ctx context.Context, user
 			syncResult.UpdateError(err)
 			continue
 		}
-		syncMetadata(ctx, userCred, &commondb[i], commonext[i])
+		syncVirtualResourceMetadata(ctx, userCred, &commondb[i], commonext[i])
 		localElasticcaches = append(localElasticcaches, commondb[i])
 		remoteElasticcaches = append(remoteElasticcaches, commonext[i])
 		syncResult.Update()
@@ -484,7 +484,7 @@ func (manager *SElasticcacheManager) SyncElasticcaches(ctx context.Context, user
 			syncResult.AddError(err)
 			continue
 		}
-		syncMetadata(ctx, userCred, instance, added[i])
+		syncVirtualResourceMetadata(ctx, userCred, instance, added[i])
 		localElasticcaches = append(localElasticcaches, *instance)
 		remoteElasticcaches = append(remoteElasticcaches, added[i])
 		syncResult.Add()

pkg/compute/models/elasticips.go

@@ -358,7 +358,7 @@ func (manager *SElasticipManager) SyncEips(ctx context.Context, userCred mcclien
 		if err != nil {
 			syncResult.UpdateError(err)
 		} else {
-			syncMetadata(ctx, userCred, &commondb[i], commonext[i])
+			syncVirtualResourceMetadata(ctx, userCred, &commondb[i], commonext[i])
 			syncResult.Update()
 		}
 	}
@@ -367,7 +367,7 @@ func (manager *SElasticipManager) SyncEips(ctx context.Context, userCred mcclien
 		if err != nil {
 			syncResult.AddError(err)
 		} else {
-			syncMetadata(ctx, userCred, new, added[i])
+			syncVirtualResourceMetadata(ctx, userCred, new, added[i])
 			syncResult.Add()
 		}
 	}

pkg/compute/models/external_projects.go

@@ -16,6 +16,7 @@ package models
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 
 	"yunion.io/x/jsonutils"
@@ -35,7 +36,7 @@ import (
 )
 
 type SExternalProjectManager struct {
-	db.SStandaloneResourceBaseManager
+	db.SStatusStandaloneResourceBaseManager
 	db.SProjectizedResourceBaseManager
 	db.SExternalizedResourceBaseManager
 	SManagedResourceBaseManager
@@ -45,7 +46,7 @@ var ExternalProjectManager *SExternalProjectManager
 
 func init() {
 	ExternalProjectManager = &SExternalProjectManager{
-		SStandaloneResourceBaseManager: db.NewStandaloneResourceBaseManager(
+		SStatusStandaloneResourceBaseManager: db.NewStatusStandaloneResourceBaseManager(
 			SExternalProject{},
 			"externalprojects_tbl",
 			"externalproject",
@@ -56,10 +57,13 @@ func init() {
 }
 
 type SExternalProject struct {
-	db.SStandaloneResourceBase
+	db.SStatusStandaloneResourceBase
 	db.SProjectizedResourceBase
 	db.SExternalizedResourceBase
 	SManagedResourceBase
+
+	// 归属云账号ID
+	CloudaccountId string `width:"36" charset:"ascii" nullable:"false" list:"user"`
 }
 
 func (manager *SExternalProjectManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
@@ -70,15 +74,6 @@ func (self *SExternalProject) AllowUpdateItem(ctx context.Context, userCred mccl
 	return false
 }
 
-func (manager *SExternalProjectManager) getProjectsByProviderId(providerId string) ([]SExternalProject, error) {
-	projects := []SExternalProject{}
-	err := fetchByManagerId(manager, providerId, &projects)
-	if err != nil {
-		return nil, err
-	}
-	return projects, nil
-}
-
 func (self *SExternalProject) getCloudProviderInfo() SCloudProviderInfo {
 	provider := self.GetCloudprovider()
 	return MakeCloudProviderInfo(nil, nil, provider)
@@ -103,15 +98,15 @@ func (manager *SExternalProjectManager) FetchCustomizeColumns(
 ) []api.ExternalProjectDetails {
 	rows := make([]api.ExternalProjectDetails, len(objs))
 
-	stdRows := manager.SStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
+	stdRows := manager.SStatusStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
 	manRows := manager.SManagedResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
 	projRows := manager.SProjectizedResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
 
 	for i := range rows {
 		rows[i] = api.ExternalProjectDetails{
-			StandaloneResourceDetails: stdRows[i],
-			ManagedResourceInfo:       manRows[i],
-			ProjectizedResourceInfo:   projRows[i],
+			StatusStandaloneResourceDetails: stdRows[i],
+			ManagedResourceInfo:             manRows[i],
+			ProjectizedResourceInfo:         projRows[i],
 		}
 	}
 
@@ -121,7 +116,8 @@ func (manager *SExternalProjectManager) FetchCustomizeColumns(
 func (manager *SExternalProjectManager) GetProject(externalId string, providerId string) (*SExternalProject, error) {
 	project := &SExternalProject{}
 	project.SetModelManager(manager, project)
-	q := manager.Query().Equals("external_id", externalId).Equals("manager_id", providerId)
+	sq := CloudproviderManager.Query("cloudaccount_id").Equals("id", providerId)
+	q := manager.Query().Equals("external_id", externalId).Equals("cloudaccount_id", sq.SubQuery())
 	count, err := q.CountWithError()
 	if err != nil {
 		return nil, err
@@ -135,13 +131,13 @@ func (manager *SExternalProjectManager) GetProject(externalId string, providerId
 	return project, q.First(project)
 }
 
-func (manager *SExternalProjectManager) SyncProjects(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, projects []cloudprovider.ICloudProject) compare.SyncResult {
+func (manager *SExternalProjectManager) SyncProjects(ctx context.Context, userCred mcclient.TokenCredential, account *SCloudaccount, projects []cloudprovider.ICloudProject) compare.SyncResult {
 	lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 	defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 
 	syncResult := compare.SyncResult{}
 
-	dbProjects, err := manager.getProjectsByProviderId(provider.Id)
+	dbProjects, err := account.GetExternalProjects()
 	if err != nil {
 		syncResult.Error(err)
 		return syncResult
@@ -167,7 +163,7 @@ func (manager *SExternalProjectManager) SyncProjects(ctx context.Context, userCr
 		}
 	}
 	for i := 0; i < len(commondb); i++ {
-		err = commondb[i].SyncWithCloudProject(ctx, userCred, provider, commonext[i])
+		err = commondb[i].SyncWithCloudProject(ctx, userCred, account, commonext[i])
 		if err != nil {
 			syncResult.UpdateError(err)
 		} else {
@@ -175,7 +171,7 @@ func (manager *SExternalProjectManager) SyncProjects(ctx context.Context, userCr
 		}
 	}
 	for i := 0; i < len(added); i++ {
-		_, err := manager.newFromCloudProject(ctx, userCred, provider, added[i])
+		_, err := manager.newFromCloudProject(ctx, userCred, account, added[i])
 		if err != nil {
 			syncResult.AddError(err)
 		} else {
@@ -192,13 +188,14 @@ func (self *SExternalProject) syncRemoveCloudProject(ctx context.Context, userCr
 	return self.Delete(ctx, userCred)
 }
 
-func (self *SExternalProject) SyncWithCloudProject(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, ext cloudprovider.ICloudProject) error {
+func (self *SExternalProject) SyncWithCloudProject(ctx context.Context, userCred mcclient.TokenCredential, account *SCloudaccount, ext cloudprovider.ICloudProject) error {
 	diff, err := db.UpdateWithLock(ctx, self, func() error {
 		self.Name = ext.GetName()
 		self.IsEmulated = ext.IsEmulated()
-		if self.DomainId != provider.DomainId {
-			self.ProjectId = provider.ProjectId
-			self.DomainId = provider.DomainId
+		self.Status = ext.GetStatus()
+		if self.DomainId != account.DomainId {
+			self.ProjectId = account.ProjectId
+			self.DomainId = account.DomainId
 		}
 		return nil
 	})
@@ -210,20 +207,20 @@ func (self *SExternalProject) SyncWithCloudProject(ctx context.Context, userCred
 	return nil
 }
 
-func (manager *SExternalProjectManager) newFromCloudProject(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, extProject cloudprovider.ICloudProject) (*SExternalProject, error) {
+func (manager *SExternalProjectManager) newFromCloudProject(ctx context.Context, userCred mcclient.TokenCredential, account *SCloudaccount, extProject cloudprovider.ICloudProject) (*SExternalProject, error) {
 	project := SExternalProject{}
 	project.SetModelManager(manager, &project)
 
 	project.Name = extProject.GetName()
+	project.Status = extProject.GetStatus()
 	project.ExternalId = extProject.GetGlobalId()
 	project.IsEmulated = extProject.IsEmulated()
-	project.ManagerId = provider.Id
-	project.DomainId = provider.DomainId
-	project.ProjectId = provider.ProjectId
-	account := provider.GetCloudaccount()
-	if account != nil && account.AutoCreateProject {
+	project.CloudaccountId = account.Id
+	project.DomainId = account.DomainId
+	project.ProjectId = account.ProjectId
+	if account.AutoCreateProject {
 		desc := fmt.Sprintf("auto create from cloud project %s (%s)", project.Name, project.ExternalId)
-		domainId, projectId, err := getOrCreateTenant(ctx, project.Name, provider.DomainId, "", desc)
+		domainId, projectId, err := getOrCreateTenant(ctx, project.Name, account.DomainId, "", desc)
 		if err != nil {
 			log.Errorf("failed to get or create tenant %s(%s) %v", project.Name, project.ExternalId, err)
 		} else {
@@ -303,23 +300,42 @@ func (manager *SExternalProjectManager) ListItemFilter(
 ) (*sqlchemy.SQuery, error) {
 	var err error
 
-	q, err = manager.SStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query.StandaloneResourceListInput)
+	q, err = manager.SStatusStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query.StatusStandaloneResourceListInput)
 	if err != nil {
-		return nil, errors.Wrap(err, "SStandaloneResourceBaseManager.ListItemFilter")
+		return nil, errors.Wrap(err, "SStatusStandaloneResourceBaseManager.ListItemFilter")
 	}
 	q, err = manager.SExternalizedResourceBaseManager.ListItemFilter(ctx, q, userCred, query.ExternalizedResourceBaseListInput)
 	if err != nil {
 		return nil, errors.Wrap(err, "SExternalizedResourceBaseManager.ListItemFilter")
 	}
-	q, err = manager.SManagedResourceBaseManager.ListItemFilter(ctx, q, userCred, query.ManagedResourceListInput)
-	if err != nil {
-		return nil, errors.Wrap(err, "SManagedResourceBaseManager.ListItemFilter")
-	}
 	q, err = manager.SProjectizedResourceBaseManager.ListItemFilter(ctx, q, userCred, query.ProjectizedResourceListInput)
 	if err != nil {
 		return nil, errors.Wrap(err, "SProjectizedResourceBaseManager.ListItemFilter")
 	}
 
+	if len(query.Cloudprovider) > 0 {
+		p, err := CloudproviderManager.FetchByIdOrName(userCred, query.Cloudprovider)
+		if err != nil {
+			if errors.Cause(err) == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2("cloudprovider", query.Cloudprovider)
+			}
+			return nil, httperrors.NewGeneralError(err)
+		}
+		provider := p.(*SCloudprovider)
+		query.Cloudaccount = provider.CloudaccountId
+	}
+
+	if len(query.Cloudaccount) > 0 {
+		account, err := CloudaccountManager.FetchByIdOrName(userCred, query.Cloudaccount)
+		if err != nil {
+			if errors.Cause(err) == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2("cloudaccount", query.Cloudaccount)
+			}
+			return nil, httperrors.NewGeneralError(err)
+		}
+		q = q.Equals("cloudaccount_id", account.GetId())
+	}
+
 	return q, nil
 }
 
@@ -331,9 +347,9 @@ func (manager *SExternalProjectManager) OrderByExtraFields(
 ) (*sqlchemy.SQuery, error) {
 	var err error
 
-	q, err = manager.SStandaloneResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.StandaloneResourceListInput)
+	q, err = manager.SStatusStandaloneResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.StatusStandaloneResourceListInput)
 	if err != nil {
-		return nil, errors.Wrap(err, "SStandaloneResourceBaseManager.OrderByExtraFields")
+		return nil, errors.Wrap(err, "SStatusStandaloneResourceBaseManager.OrderByExtraFields")
 	}
 	q, err = manager.SManagedResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.ManagedResourceListInput)
 	if err != nil {
@@ -350,7 +366,7 @@ func (manager *SExternalProjectManager) OrderByExtraFields(
 func (manager *SExternalProjectManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) {
 	var err error
 
-	q, err = manager.SStandaloneResourceBaseManager.QueryDistinctExtraField(q, field)
+	q, err = manager.SStatusStandaloneResourceBaseManager.QueryDistinctExtraField(q, field)
 	if err == nil {
 		return q, nil
 	}
@@ -368,9 +384,9 @@ func (manager *SExternalProjectManager) QueryDistinctExtraField(q *sqlchemy.SQue
 
 func (manager *SExternalProjectManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential,
 	keys stringutils2.SSortedStrings) (*sqlchemy.SQuery, error) {
-	q, err := manager.SStandaloneResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys)
+	q, err := manager.SStatusStandaloneResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys)
 	if err != nil {
-		return nil, errors.Wrap(err, "SStandaloneResourceBaseManager.ListItemExportKeys")
+		return nil, errors.Wrap(err, "SStatusStandaloneResourceBaseManager.ListItemExportKeys")
 	}
 	q, err = manager.SProjectizedResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys)
 	if err != nil {
@@ -378,3 +394,23 @@ func (manager *SExternalProjectManager) ListItemExportKeys(ctx context.Context,
 	}
 	return q, nil
 }
+
+func (manager *SExternalProjectManager) InitializeData() error {
+	eps := []SExternalProject{}
+	q := manager.Query().IsNullOrEmpty("cloudaccount_id")
+	err := db.FetchModelObjects(manager, q, &eps)
+	if err != nil {
+		return err
+	}
+	for i := range eps {
+		_, err = db.Update(&eps[i], func() error {
+			provider := eps[i].GetCloudprovider()
+			if provider == nil {
+				return fmt.Errorf("failed to get external project %s cloudprovider", eps[i].Id)
+			}
+			eps[i].CloudaccountId = provider.CloudaccountId
+			return nil
+		})
+	}
+	return nil
+}

pkg/compute/models/guests.go

@@ -771,6 +771,14 @@ func (guest *SGuest) CustomizeCreate(ctx context.Context, userCred mcclient.Toke
 	return guest.SVirtualResourceBase.CustomizeCreate(ctx, userCred, ownerId, query, data)
 }
 
+func (guest *SGuest) GetCloudproviderId() string {
+	host := guest.GetHost()
+	if host != nil {
+		return host.GetCloudproviderId()
+	}
+	return ""
+}
+
 func (guest *SGuest) GetHost() *SHost {
 	if len(guest.HostId) > 0 && regutils.MatchUUID(guest.HostId) {
 		host, _ := HostManager.FetchById(guest.HostId)

pkg/compute/models/initdb.go

@@ -56,6 +56,7 @@ func InitDB() error {
 		DynamicschedtagManager,
 		ServerSkuManager,
 		ElasticcacheSkuManager,
+		ExternalProjectManager,
 	} {
 		err := manager.InitializeData()
 		if err != nil {

pkg/compute/models/loadbalancerbackends.go

@@ -264,6 +264,14 @@ func (lbb *SLoadbalancerBackend) AllowPerformStatus(ctx context.Context, userCre
 	return false
 }
 
+func (lbb *SLoadbalancerBackend) GetCloudproviderId() string {
+	lbbg := lbb.GetLoadbalancerBackendGroup()
+	if lbbg != nil {
+		return lbbg.GetCloudproviderId()
+	}
+	return ""
+}
+
 func (lbb *SLoadbalancerBackend) GetLoadbalancerBackendGroup() *SLoadbalancerBackendGroup {
 	backendgroup, err := LoadbalancerBackendGroupManager.FetchById(lbb.BackendGroupId)
 	if err != nil {

pkg/compute/models/loadbalancerlistenerresource.go

@@ -62,6 +62,14 @@ func (self *SLoadbalancerListenerResourceBase) GetLoadbalancerListener() *SLoadb
 	return listener.(*SLoadbalancerListener)
 }
 
+func (self *SLoadbalancerListenerResourceBase) GetCloudproviderId() string {
+	cloudprovider := self.GetCloudprovider()
+	if cloudprovider != nil {
+		return cloudprovider.Id
+	}
+	return ""
+}
+
 func (self *SLoadbalancerListenerResourceBase) GetCloudprovider() *SCloudprovider {
 	listener := self.GetLoadbalancerListener()
 	if listener != nil {

pkg/compute/models/loadbalancers.go

@@ -385,6 +385,10 @@ func (lb *SLoadbalancer) GetRegion() *SCloudregion {
 	return lb.SCloudregionResourceBase.GetRegion()
 }
 
+func (lb *SLoadbalancer) GetCloudproviderId() string {
+	return lb.SManagedResourceBase.GetCloudproviderId()
+}
+
 func (lb *SLoadbalancer) GetZone() *SZone {
 	return lb.SZoneResourceBase.GetZone()
 }
@@ -726,7 +730,7 @@ func (man *SLoadbalancerManager) SyncLoadbalancers(ctx context.Context, userCred
 		if err != nil {
 			syncResult.UpdateError(err)
 		} else {
-			syncMetadata(ctx, userCred, &commondb[i], commonext[i])
+			syncVirtualResourceMetadata(ctx, userCred, &commondb[i], commonext[i])
 			localLbs = append(localLbs, commondb[i])
 			remoteLbs = append(remoteLbs, commonext[i])
 			syncResult.Update()
@@ -737,7 +741,7 @@ func (man *SLoadbalancerManager) SyncLoadbalancers(ctx context.Context, userCred
 		if err != nil {
 			syncResult.AddError(err)
 		} else {
-			syncMetadata(ctx, userCred, new, added[i])
+			syncVirtualResourceMetadata(ctx, userCred, new, added[i])
 			localLbs = append(localLbs, *new)
 			remoteLbs = append(remoteLbs, added[i])
 			syncResult.Add()

pkg/compute/models/managedresource.go

@@ -46,6 +46,10 @@ type SManagedResourceBaseManager struct {
 	managerIdFieldName string
 }
 
+func (self *SManagedResourceBase) GetCloudproviderId() string {
+	return self.ManagerId
+}
+
 func ValidateCloudproviderResourceInput(userCred mcclient.TokenCredential, query api.CloudproviderResourceInput) (*SCloudprovider, api.CloudproviderResourceInput, error) {
 	managerObj, err := CloudproviderManager.FetchByIdOrName(userCred, query.Cloudprovider)
 	if err != nil {

pkg/compute/models/natdtable.go

@@ -237,6 +237,14 @@ func (manager *SNatDEntryManager) SyncNatDTable(ctx context.Context, userCred mc
 	return result
 }
 
+func (self *SNatDEntry) GetCloudproviderId() string {
+	nat, _ := self.GetNatgateway()
+	if nat != nil {
+		return nat.GetCloudproviderId()
+	}
+	return ""
+}
+
 func (self *SNatDEntry) syncRemoveCloudNatDTable(ctx context.Context, userCred mcclient.TokenCredential) error {
 	lockman.LockObject(ctx, self)
 	defer lockman.ReleaseObject(ctx, self)

pkg/compute/models/natstable.go

@@ -63,6 +63,14 @@ type SNatSEntry struct {
 	SourceCIDR string `width:"22" charset:"ascii" list:"user" create:"required"`
 }
 
+func (self *SNatSEntry) GetCloudproviderId() string {
+	network, err := self.GetNetwork()
+	if err == nil {
+		return network.GetCloudproviderId()
+	}
+	return ""
+}
+
 func (self *SNatSEntry) GetNetwork() (*SNetwork, error) {
 	if len(self.NetworkId) == 0 {
 		return nil, nil

pkg/compute/models/networks.go

@@ -631,7 +631,7 @@ func (manager *SNetworkManager) SyncNetworks(ctx context.Context, userCred mccli
 		if err != nil {
 			syncResult.UpdateError(err)
 		} else {
-			syncMetadata(ctx, userCred, &commondb[i], commonext[i])
+			syncVirtualResourceMetadata(ctx, userCred, &commondb[i], commonext[i])
 			localNets = append(localNets, commondb[i])
 			remoteNets = append(remoteNets, commonext[i])
 			syncResult.Update()
@@ -642,7 +642,7 @@ func (manager *SNetworkManager) SyncNetworks(ctx context.Context, userCred mccli
 		if err != nil {
 			syncResult.AddError(err)
 		} else {
-			syncMetadata(ctx, userCred, new, added[i])
+			syncVirtualResourceMetadata(ctx, userCred, new, added[i])
 			localNets = append(localNets, *new)
 			remoteNets = append(remoteNets, added[i])
 			syncResult.Add()

pkg/compute/models/purge.go

@@ -1039,20 +1039,6 @@ func (manager *SCloudproviderregionManager) purgeAll(ctx context.Context, userCr
 	return nil
 }
 
-func (manager *SExternalProjectManager) purgeAll(ctx context.Context, userCred mcclient.TokenCredential, providerId string) error {
-	projs, err := manager.getProjectsByProviderId(providerId)
-	if err != nil {
-		return err
-	}
-	for i := range projs {
-		err = projs[i].Delete(ctx, userCred)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 func (zone *SZone) Purge(ctx context.Context, userCred mcclient.TokenCredential) error {
 	lockman.LockObject(ctx, zone)
 	defer lockman.ReleaseObject(ctx, zone)

pkg/compute/models/snapshotpolicy.go

@@ -371,6 +371,10 @@ func (sp *SSnapshotPolicy) getMoreDetails(out api.SnapshotPolicyDetails) api.Sna
 	return out
 }
 
+func (sp *SSnapshotPolicy) GetCloudproviderId() string {
+	return ""
+}
+
 // ==================================================== sync ===========================================================
 func (manager *SSnapshotPolicyManager) SyncSnapshotPolicies(ctx context.Context, userCred mcclient.TokenCredential,
 	provider *SCloudprovider, region *SCloudregion, cloudSPs []cloudprovider.ICloudSnapshotPolicy,
@@ -507,7 +511,7 @@ func (manager *SSnapshotPolicyManager) allNewFromCloudSnapshotPolicy(
 		if err != nil {
 			syncResult.AddError(err)
 		} else {
-			syncMetadata(ctx, userCred, local, added[i])
+			syncVirtualResourceMetadata(ctx, userCred, local, added[i])
 			syncResult.Add()
 		}
 	}

pkg/compute/models/snapshots.go

@@ -955,7 +955,7 @@ func (manager *SSnapshotManager) SyncSnapshots(ctx context.Context, userCred mcc
 		if err != nil {
 			syncResult.UpdateError(err)
 		} else {
-			syncMetadata(ctx, userCred, &commondb[i], commonext[i])
+			syncVirtualResourceMetadata(ctx, userCred, &commondb[i], commonext[i])
 			syncResult.Update()
 		}
 	}
@@ -964,7 +964,7 @@ func (manager *SSnapshotManager) SyncSnapshots(ctx context.Context, userCred mcc
 		if err != nil {
 			syncResult.AddError(err)
 		} else {
-			syncMetadata(ctx, userCred, local, added[i])
+			syncVirtualResourceMetadata(ctx, userCred, local, added[i])
 			syncResult.Add()
 		}
 	}

pkg/compute/models/syncutils.go

@@ -25,9 +25,10 @@ import (
 )
 
 type IMetadataSetter interface {
-	// SetAllMetadata(ctx context.Context, meta map[string]interface{}, userCred mcclient.TokenCredential) error
-	// SetMetadata(ctx context.Context, key string, value interface{}, userCred mcclient.TokenCredential) error
 	SetCloudMetadataAll(ctx context.Context, meta map[string]interface{}, userCred mcclient.TokenCredential) error
+	Keyword() string
+	GetName() string
+	GetCloudproviderId() string
 }
 
 func syncMetadata(ctx context.Context, userCred mcclient.TokenCredential, model IMetadataSetter, remote cloudprovider.ICloudResource) error {
@@ -50,6 +51,42 @@ func syncMetadata(ctx context.Context, userCred mcclient.TokenCredential, model
 	return nil
 }
 
+func syncVirtualResourceMetadata(ctx context.Context, userCred mcclient.TokenCredential, model IMetadataSetter, remote cloudprovider.IVirtualResource) error {
+	metaData := remote.GetMetadata()
+	store := make(map[string]interface{}, 0)
+	if metaData != nil {
+		meta := make(map[string]interface{}, 0)
+		err := metaData.Unmarshal(meta)
+		if err != nil {
+			log.Errorf("Get VM Metadata error: %v", err)
+			return err
+		}
+		for key, value := range meta {
+			store[db.CLOUD_TAG_PREFIX+key] = value
+		}
+
+	}
+
+	extProjectId := remote.GetProjectId()
+	if len(extProjectId) > 0 {
+		extProject, err := ExternalProjectManager.GetProject(extProjectId, model.GetCloudproviderId())
+		if err != nil {
+			log.Errorf("sync project metadata for %s %s error: %v", model.Keyword(), model.GetName(), err)
+		} else {
+			store[db.CLOUD_TAG_PREFIX+"project"] = extProject.Name
+		}
+	}
+
+	if len(store) > 0 {
+		model.SetCloudMetadataAll(ctx, store, userCred)
+	}
+	return nil
+}
+
 func SyncMetadata(ctx context.Context, userCred mcclient.TokenCredential, model IMetadataSetter, remote cloudprovider.ICloudResource) error {
 	return syncMetadata(ctx, userCred, model, remote)
 }
+
+func SyncVirtualResourceMetadata(ctx context.Context, userCred mcclient.TokenCredential, model IMetadataSetter, remote cloudprovider.IVirtualResource) error {
+	return syncVirtualResourceMetadata(ctx, userCred, model, remote)
+}

pkg/compute/models/wireresource.go

@@ -63,6 +63,14 @@ func (self *SWireResourceBase) GetWire() *SWire {
 	return nil
 }
 
+func (self *SWireResourceBase) GetCloudproviderId() string {
+	vpc := self.GetVpc()
+	if vpc != nil {
+		return vpc.ManagerId
+	}
+	return ""
+}
+
 func (self *SWireResourceBase) GetVpc() *SVpc {
 	wire := self.GetWire()
 	if wire != nil {

pkg/compute/models/zones.go

@@ -171,6 +171,10 @@ func (zone *SZone) GetExtraDetails(ctx context.Context, userCred mcclient.TokenC
 	return api.ZoneDetails{}, nil
 }
 
+func (zone *SZone) GetCloudproviderId() string {
+	return ""
+}
+
 func (zone *SZone) GetCloudRegionId() string {
 	if len(zone.CloudregionId) == 0 {
 		return "default"

pkg/compute/tasks/cloud_account_sync_task.go

@@ -16,6 +16,7 @@ package tasks
 
 import (
 	"context"
+	"fmt"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -23,6 +24,7 @@ import (
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/util/logclient"
@@ -56,6 +58,26 @@ func (self *CloudAccountSyncInfoTask) OnInit(ctx context.Context, obj db.IStanda
 		return
 	}
 
+	driver, err := cloudaccount.GetProvider()
+	if err != nil {
+		cloudaccount.MarkEndSyncWithLock(ctx, self.UserCred)
+		db.OpsLog.LogEvent(cloudaccount, db.ACT_SYNC_HOST_FAILED, err, self.UserCred)
+		self.SetStageFailed(ctx, err.Error())
+		logclient.AddActionLogWithStartable(self, cloudaccount, logclient.ACT_CLOUD_SYNC, err, self.UserCred, false)
+		return
+	}
+
+	if cloudprovider.IsSupportProject(driver) {
+		projects, err := driver.GetIProjects()
+		if err != nil {
+			msg := fmt.Sprintf("GetIProjects for cloudaccount %s failed %s", cloudaccount.GetName(), err)
+			log.Errorf(msg)
+		} else {
+			result := models.ExternalProjectManager.SyncProjects(ctx, self.GetUserCred(), cloudaccount, projects)
+			log.Infof("Sync project for cloudaccount %s result: %s", cloudaccount.GetName(), result.Result())
+		}
+	}
+
 	syncRange := models.SSyncRange{}
 	syncRangeJson, _ := self.Params.Get("sync_range")
 	if syncRangeJson != nil {

pkg/multicloud/aliyun/project.go

@@ -20,6 +20,7 @@ import (
 
 	"yunion.io/x/pkg/errors"
 
+	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud"
 )
@@ -51,7 +52,16 @@ func (self *SResourceGroup) GetName() string {
 }
 
 func (self *SResourceGroup) GetStatus() string {
-	return ""
+	switch self.Status {
+	case "Creating":
+		return api.EXTERNAL_PROJECT_STATUS_CREATING
+	case "OK":
+		return api.EXTERNAL_PROJECT_STATUS_AVAILABLE
+	case "Deleted", "Deleting", "PendingDelete":
+		return api.EXTERNAL_PROJECT_STATUS_DELETING
+	default:
+		return api.EXTERNAL_PROJECT_STATUS_UNKNOWN
+	}
 }
 
 func (self *SAliyunClient) GetResourceGroups(pageNumber int, pageSize int) ([]SResourceGroup, int, error) {

pkg/multicloud/azure/resourcegroup.go

@@ -19,6 +19,8 @@ import (
 	"strings"
 
 	"yunion.io/x/jsonutils"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
 )
 
 type GroupProperties struct {
@@ -75,7 +77,7 @@ func (r *SResourceGroup) GetGlobalId() string {
 }
 
 func (r *SResourceGroup) GetStatus() string {
-	return r.Properties.ProvisioningState
+	return api.EXTERNAL_PROJECT_STATUS_AVAILABLE
 }
 
 func (r *SResourceGroup) GetMetadata() *jsonutils.JSONDict {

pkg/multicloud/esxi/cluster.go

@@ -0,0 +1,106 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package esxi
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/vmware/govmomi/object"
+	"github.com/vmware/govmomi/vim25/mo"
+	"github.com/vmware/govmomi/vim25/types"
+
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+type SCluster struct {
+	SManagedObject
+}
+
+func NewCluster(manager *SESXiClient, cluster *mo.ClusterComputeResource, dc *SDatacenter) *SCluster {
+	return &SCluster{SManagedObject: newManagedObject(manager, cluster, dc)}
+}
+
+func (cluster *SCluster) ListResourcePools() ([]mo.ResourcePool, error) {
+	var pools, result []mo.ResourcePool
+	err := cluster.manager.scanMObjects(cluster.object.Entity().Self, RESOURCEPOOL_PROPS, &pools)
+	if err != nil {
+		return nil, errors.Wrap(err, "scanMObjects")
+	}
+	for i := range pools {
+		if pools[i].Parent.Type == "ClusterComputeResource" {
+			continue
+		}
+		result = append(result, pools[i])
+	}
+	return result, nil
+}
+
+func (cluster *SCluster) getDefaultResourcePool() (mo.ResourcePool, error) {
+	pools := []mo.ResourcePool{}
+	err := cluster.manager.scanMObjects(cluster.object.Entity().Self, RESOURCEPOOL_PROPS, &pools)
+	if err != nil {
+		return mo.ResourcePool{}, errors.Wrap(err, "scanMObjects")
+	}
+	for i := range pools {
+		if pools[i].Parent.Type == "ClusterComputeResource" {
+			return pools[i], nil
+		}
+	}
+	return mo.ResourcePool{}, cloudprovider.ErrNotFound
+}
+
+func (cluster *SCluster) CreateResourcePool(name string) (*mo.ResourcePool, error) {
+	if len(name) == 0 {
+		return nil, errors.Error("empty name str")
+	}
+	root, err := cluster.getDefaultResourcePool()
+	if err != nil {
+		return nil, errors.Wrap(err, "getDefaultResourcePool")
+	}
+	pool := object.NewResourcePool(cluster.manager.client.Client, root.Reference())
+	pool.InventoryPath = fmt.Sprintf("/%s/host/%s/Resources", cluster.datacenter.GetName(), cluster.GetName())
+	_, err = pool.Create(context.Background(), name, types.DefaultResourceConfigSpec())
+	if err != nil {
+		return nil, errors.Wrap(err, "pool.Create")
+	}
+
+	pools, err := cluster.ListResourcePools()
+	if err != nil {
+		return nil, errors.Wrap(err, "listResourcePools")
+	}
+	for i := range pools {
+		p := NewResourcePool(cluster.manager, &pools[i], cluster.datacenter)
+		if p.GetName() == name {
+			return &pools[i], nil
+		}
+	}
+	return nil, errors.Wrap(cloudprovider.ErrNotFound, "AfterCreate")
+}
+
+func (cluster *SCluster) SyncResourcePool(groupId string, name string) (*mo.ResourcePool, error) {
+	pools, err := cluster.ListResourcePools()
+	if err != nil {
+		return nil, errors.Wrap(err, "ListResourcePools")
+	}
+	for i := range pools {
+		if pools[i].Self.Value == groupId || pools[i].Entity().Name == name {
+			return &pools[i], nil
+		}
+	}
+	return cluster.CreateResourcePool(name)
+}

pkg/multicloud/esxi/datacenter.go

@@ -22,7 +22,6 @@ import (
 	"github.com/vmware/govmomi/vim25/types"
 
 	"yunion.io/x/pkg/errors"
-	"yunion.io/x/pkg/utils"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -67,18 +66,14 @@ func (dc *SDatacenter) getObjectDatacenter() *object.Datacenter {
 
 func (dc *SDatacenter) scanResourcePool() error {
 	if dc.iresoucePool == nil {
-		var pools []mo.ResourcePool
-		err := dc.manager.scanMObjects(dc.object.Entity().Self, RESOURCEPOOL_PROPS, &pools)
+		pools, err := dc.listResourcePools()
 		if err != nil {
-			return errors.Wrap(err, "scanMObjects")
+			return errors.Wrap(err, "listResourcePools")
 		}
 		dc.iresoucePool = []cloudprovider.ICloudProject{}
 		for i := 0; i < len(pools); i++ {
 			p := NewResourcePool(dc.manager, &pools[i], dc)
-			rpPath := p.GetPath()
-			if utils.IsInStringArray("Resources", rpPath) && rpPath[len(rpPath)-1] != "Resources" {
-				dc.iresoucePool = append(dc.iresoucePool, p)
-			}
+			dc.iresoucePool = append(dc.iresoucePool, p)
 		}
 	}
 	return nil
@@ -117,6 +112,53 @@ func (dc *SDatacenter) GetResourcePools() ([]cloudprovider.ICloudProject, error)
 	return dc.iresoucePool, nil
 }
 
+func (dc *SDatacenter) listResourcePools() ([]mo.ResourcePool, error) {
+	var pools, result []mo.ResourcePool
+	err := dc.manager.scanMObjects(dc.object.Entity().Self, RESOURCEPOOL_PROPS, &pools)
+	if err != nil {
+		return nil, errors.Wrap(err, "scanMObjects")
+	}
+	for i := range pools {
+		if pools[i].Parent.Type == "ClusterComputeResource" {
+			continue
+		}
+		result = append(result, pools[i])
+	}
+	return result, nil
+}
+
+func (dc *SDatacenter) ListClusters() ([]*SCluster, error) {
+	return dc.listClusters()
+}
+
+func (dc *SDatacenter) GetCluster(cluster string) (*SCluster, error) {
+	clusters, err := dc.ListClusters()
+	if err != nil {
+		return nil, errors.Wrap(err, "ListClusters")
+	}
+	for i := range clusters {
+		if clusters[i].GetName() == cluster {
+			return clusters[i], nil
+		}
+
+	}
+	return nil, cloudprovider.ErrNotFound
+}
+
+func (dc *SDatacenter) listClusters() ([]*SCluster, error) {
+	clusters := []mo.ClusterComputeResource{}
+	err := dc.manager.scanMObjects(dc.object.Entity().Self, RESOURCEPOOL_PROPS, &clusters)
+	if err != nil {
+		return nil, errors.Wrap(err, "scanMObjects")
+	}
+	ret := []*SCluster{}
+	for i := range clusters {
+		c := NewCluster(dc.manager, &clusters[i], dc)
+		ret = append(ret, c)
+	}
+	return ret, nil
+}
+
 func (dc *SDatacenter) GetIHosts() ([]cloudprovider.ICloudHost, error) {
 	err := dc.scanHosts()
 	if err != nil {

pkg/multicloud/esxi/host.go

@@ -645,15 +645,17 @@ func (self *SHost) CreateVM(desc *cloudprovider.SManagedVMCreateConfig) (cloudpr
 }
 
 type SCreateVMParam struct {
-	Name   string
-	Uuid   string
-	OsName string
-	Cpu    int
-	Mem    int
-	Bios   string
-	Cdrom  jsonutils.JSONObject
-	Disks  []SDiskInfo
-	Nics   []jsonutils.JSONObject
+	Name         string
+	Uuid         string
+	OsName       string
+	Cpu          int
+	Mem          int
+	Bios         string
+	Cdrom        jsonutils.JSONObject
+	Disks        []SDiskInfo
+	Nics         []jsonutils.JSONObject
+	GroupId      string // resourcePoolId
+	ResourcePool string
 }
 
 type SDiskInfo struct {
@@ -840,9 +842,9 @@ func (self *SHost) DoCreateVM(ctx context.Context, ds *SDatastore, params SCreat
 		return nil, errors.Wrap(err, "object.DataCenter.Folders")
 	}
 	vmFolder := folders.VmFolder
-	resourcePool, err := self.GetResourcePool()
+	resourcePool, err := self.SyncResourcePool(params.GroupId, params.ResourcePool)
 	if err != nil {
-		return nil, errors.Wrap(err, "SHost.GetResourcePool")
+		return nil, errors.Wrap(err, "SyncResourcePool")
 	}
 	task, err := vmFolder.CreateVM(ctx, spec, resourcePool, self.GetoHostSystem())
 	if err != nil {
@@ -1357,7 +1359,7 @@ func (host *SHost) GetoHostSystem() *object.HostSystem {
 func (host *SHost) GetResourcePool() (*object.ResourcePool, error) {
 	var err error
 	if host.parent == nil {
-		host.parent, err = host.getResourcePool()
+		host.parent, err = host.getParent()
 		if err != nil {
 			return nil, err
 		}
@@ -1365,7 +1367,7 @@ func (host *SHost) GetResourcePool() (*object.ResourcePool, error) {
 	return object.NewResourcePool(host.manager.client.Client, *host.parent.ResourcePool), nil
 }
 
-func (host *SHost) getResourcePool() (*mo.ComputeResource, error) {
+func (host *SHost) getParent() (*mo.ComputeResource, error) {
 	var mcr *mo.ComputeResource
 	var parent interface{}
 
@@ -1383,19 +1385,58 @@ func (host *SHost) getResourcePool() (*mo.ComputeResource, error) {
 		return nil, errors.Error(fmt.Sprintf("unknown host parent type: %s", moHost.Parent.Type))
 	}
 
-	err := host.manager.reference2Object(*moHost.Parent, []string{"resourcePool"}, parent)
+	err := host.manager.reference2Object(*moHost.Parent, []string{"name", "resourcePool"}, parent)
 	if err != nil {
 		return nil, errors.Wrap(err, "SESXiClient.reference2Object")
 	}
 	return mcr, nil
 }
 
-func (host *SHost) GetCluster() (*mo.ComputeResource, error) {
-	return host.getResourcePool()
+func (host *SHost) GetResourcePools() ([]mo.ResourcePool, error) {
+	cluster, err := host.GetCluster()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetCluster")
+	}
+	return cluster.ListResourcePools()
+}
+
+func (host *SHost) GetCluster() (*SCluster, error) {
+	cluster, err := host.getCluster()
+	if err != nil {
+		return nil, errors.Wrap(err, "getCluster")
+	}
+	return NewCluster(host.manager, cluster, host.datacenter), nil
+}
+
+func (host *SHost) SyncResourcePool(groupId, name string) (*object.ResourcePool, error) {
+	cluster, err := host.GetCluster()
+	if err != nil {
+		log.Errorf("failed to get host %s cluster info: %v", host.GetName(), err)
+		return host.GetResourcePool()
+	}
+	pool, err := cluster.SyncResourcePool(groupId, name)
+	if err != nil {
+		log.Errorf("failed to sync resourcePool(%s, %s) for cluster %s error: %v", groupId, name, cluster.GetName(), err)
+		return host.GetResourcePool()
+	}
+	return object.NewResourcePool(host.manager.client.Client, pool.Reference()), nil
+}
+
+func (host *SHost) getCluster() (*mo.ClusterComputeResource, error) {
+	moHost := host.getHostSystem()
+	if moHost.Parent.Type != "ClusterComputeResource" {
+		return nil, fmt.Errorf("host %s parent is not the cluster resource", host.GetName())
+	}
+	cluster := &mo.ClusterComputeResource{}
+	err := host.manager.reference2Object(*moHost.Parent, []string{"name", "resourcePool"}, cluster)
+	if err != nil {
+		return nil, errors.Wrap(err, "SESXiClient.reference2Object")
+	}
+	return cluster, nil
 }
 
 func (host *SHost) GetSiblingHosts() ([]*SHost, error) {
-	rp, err := host.GetCluster()
+	rp, err := host.getParent()
 	if err != nil {
 		return nil, err
 	}

pkg/multicloud/esxi/resourcepool.go

@@ -15,8 +15,12 @@
 package esxi
 
 import (
+	"net/url"
+	"strings"
+
 	"github.com/vmware/govmomi/vim25/mo"
 
+	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/multicloud"
 )
 
@@ -32,12 +36,20 @@ func (pool *SResourcePool) GetGlobalId() string {
 }
 
 func (pool *SResourcePool) GetStatus() string {
-	return ""
+	return api.EXTERNAL_PROJECT_STATUS_AVAILABLE
 }
 
 func (pool *SResourcePool) GetName() string {
 	path := pool.GetPath()
-	return path[len(path)-1]
+	if len(path) > 5 {
+		path = path[5:]
+	}
+	name := []string{}
+	for _, _name := range path {
+		p, _ := url.PathUnescape(_name)
+		name = append([]string{p}, name...)
+	}
+	return strings.Join(name, "/")
 }
 
 func NewResourcePool(manager *SESXiClient, rp *mo.ResourcePool, dc *SDatacenter) *SResourcePool {

pkg/multicloud/esxi/shell/cluster.go

@@ -0,0 +1,87 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shell
+
+import (
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/multicloud/esxi"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	type ClusterListOptions struct {
+		DATACENTER string `help:"List clusters in datacenter"`
+	}
+	shellutils.R(&ClusterListOptions{}, "cluster-list", "List all clusters", func(cli *esxi.SESXiClient, args *ClusterListOptions) error {
+		dc, err := cli.FindDatacenterByMoId(args.DATACENTER)
+		if err != nil {
+			return err
+		}
+		clusters, err := dc.ListClusters()
+		if err != nil {
+			return err
+		}
+		printList(clusters, nil)
+		return nil
+	})
+
+	type ClusterPoolListOptions struct {
+		DATACENTER string `help:"List clusters in datacenter"`
+		CLUSTER    string `help:"List cluster resource pool"`
+	}
+
+	shellutils.R(&ClusterPoolListOptions{}, "cluster-pool-list", "List all cluster resource pool", func(cli *esxi.SESXiClient, args *ClusterPoolListOptions) error {
+		dc, err := cli.FindDatacenterByMoId(args.DATACENTER)
+		if err != nil {
+			return err
+		}
+		cluster, err := dc.GetCluster(args.CLUSTER)
+		if err != nil {
+			return err
+		}
+		pools, err := cluster.ListResourcePools()
+		if err != nil {
+			return errors.Wrap(err, "ListResourcePools")
+		}
+		printList(pools, nil)
+		return nil
+	})
+
+	type ClusterPoolSyncOptions struct {
+		DATACENTER string `help:"List clusters in datacenter"`
+		CLUSTER    string `help:"List cluster resource pool"`
+		GroupId    string `help:"Resource pool Id"`
+		Name       string `help:"Resource pool name"`
+	}
+
+	shellutils.R(&ClusterPoolSyncOptions{}, "cluster-pool-sync", "Sync cluster resource pool", func(cli *esxi.SESXiClient, args *ClusterPoolSyncOptions) error {
+		dc, err := cli.FindDatacenterByMoId(args.DATACENTER)
+		if err != nil {
+			return err
+		}
+		cluster, err := dc.GetCluster(args.CLUSTER)
+		if err != nil {
+			return err
+		}
+		pool, err := cluster.SyncResourcePool(args.GroupId, args.Name)
+		if err != nil {
+			return errors.Wrap(err, "SyncResourcePool")
+		}
+		printObject(pool)
+		return nil
+	})
+
+}

pkg/multicloud/esxi/shell/host.go

@@ -89,4 +89,32 @@ func init() {
 		printList(networks, nil)
 		return nil
 	})
+
+	shellutils.R(&HostShowOptions{}, "host-cluster", "Show host cluster", func(cli *esxi.SESXiClient,
+		args *HostShowOptions) error {
+		host, err := cli.FindHostByIp(args.IP)
+		if err != nil {
+			return err
+		}
+		cluster, err := host.GetCluster()
+		if err != nil {
+			return err
+		}
+		printObject(cluster)
+		return nil
+	})
+
+	shellutils.R(&HostShowOptions{}, "host-pool-list", "List host pools", func(cli *esxi.SESXiClient,
+		args *HostShowOptions) error {
+		host, err := cli.FindHostByIp(args.IP)
+		if err != nil {
+			return err
+		}
+		pool, err := host.GetResourcePool()
+		if err != nil {
+			return err
+		}
+		printObject(pool)
+		return nil
+	})
 }

pkg/multicloud/esxi/shell/resourcepool.go

@@ -15,6 +15,8 @@
 package shell
 
 import (
+	"yunion.io/x/pkg/errors"
+
 	"yunion.io/x/onecloud/pkg/multicloud/esxi"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
 )
@@ -35,4 +37,27 @@ func init() {
 		printList(pools, nil)
 		return nil
 	})
+
+	type ResourcePoolCreateOptions struct {
+		DATACENTER string `help:"Create resource pool in datacenter"`
+		CLUSTER    string `help:"Cluster name"`
+		NAME       string `help:"Resource pool name"`
+	}
+
+	shellutils.R(&ResourcePoolCreateOptions{}, "resource-pool-create", "Create resource pool", func(cli *esxi.SESXiClient, args *ResourcePoolCreateOptions) error {
+		dc, err := cli.FindDatacenterByMoId(args.DATACENTER)
+		if err != nil {
+			return err
+		}
+		cluster, err := dc.GetCluster(args.CLUSTER)
+		if err != nil {
+			return errors.Wrap(err, "GetCluster")
+		}
+		pool, err := cluster.CreateResourcePool(args.NAME)
+		if err != nil {
+			return err
+		}
+		printObject(pool)
+		return nil
+	})
 }

pkg/multicloud/google/project.go

@@ -17,9 +17,10 @@ package google
 import (
 	"time"
 
-	"github.com/pkg/errors"
-
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
 )
 
 type SProject struct {
@@ -84,7 +85,7 @@ func (p *SProject) GetGlobalId() string {
 }
 
 func (p *SProject) GetStatus() string {
-	return ""
+	return api.EXTERNAL_PROJECT_STATUS_AVAILABLE
 }
 
 func (p *SProject) Refresh() error {

pkg/multicloud/huawei/enterpriceprojects.go

@@ -21,6 +21,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
 
+	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud"
 )
@@ -60,9 +61,9 @@ func (ep *SEnterpriseProject) GetGlobalId() string {
 
 func (ep *SEnterpriseProject) GetStatus() string {
 	if ep.Status == 1 {
-		return "available"
+		return api.EXTERNAL_PROJECT_STATUS_AVAILABLE
 	}
-	return "unavailable"
+	return api.EXTERNAL_PROJECT_STATUS_UNAVAILABLE
 }
 
 func (ep *SEnterpriseProject) GetName() string {

pkg/multicloud/openstack/project.go

@@ -14,7 +14,11 @@
 
 package openstack
 
-import "yunion.io/x/jsonutils"
+import (
+	"yunion.io/x/jsonutils"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+)
 
 type SProject struct {
 	Description string
@@ -40,7 +44,7 @@ func (p *SProject) GetName() string {
 }
 
 func (p *SProject) GetStatus() string {
-	return ""
+	return api.EXTERNAL_PROJECT_STATUS_AVAILABLE
 }
 
 func (p *SProject) IsEmulated() bool {

pkg/multicloud/qcloud/project.go

@@ -22,6 +22,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
 
+	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 )
 
@@ -59,7 +60,7 @@ func (p *SProject) GetName() string {
 }
 
 func (p *SProject) GetStatus() string {
-	return ""
+	return api.EXTERNAL_PROJECT_STATUS_AVAILABLE
 }
 
 func (p *SProject) IsEmulated() bool {