Merge branch 'release/2.3.0' of ssh://git.yunion.io/~quxuan/onecloud into hotfix/qx-cloud-account

pkg/compute/models/billingresource.go

@@ -4,12 +4,12 @@ import "time"
 
 const (
 	BILLING_TYPE_POSTPAID = "postpaid"
-	BILLING_TYPE_PREPAID = "prepaid"
+	BILLING_TYPE_PREPAID  = "prepaid"
 )
 
 type SBillingResourceBase struct {
-	BillingType string `width:"36" charset:"ascii" nullable:"true" default:"postpaid" list:"user" create:"optional"`
-	ExpiredAt time.Time `nullable:"true" list:"user" create:"optional"`
+	BillingType string    `width:"36" charset:"ascii" nullable:"true" default:"postpaid" list:"user" create:"optional"`
+	ExpiredAt   time.Time `nullable:"true" list:"user" create:"optional"`
 }
 
 func (self *SBillingResourceBase) GetChargeType() string {

pkg/compute/models/cloudaccounts.go

@@ -14,6 +14,7 @@ import (
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -49,7 +50,7 @@ type SCloudaccount struct {
 	Provider string `width:"64" charset:"ascii" list:"admin" create:"admin_required"`
 }
 
-func (self *SCloudaccount) getCloudproviders() []SCloudprovider {
+func (self *SCloudaccount) GetCloudproviders() []SCloudprovider {
 	cloudproviders := []SCloudprovider{}
 	q := CloudproviderManager.Query().Equals("cloudaccount_id", self.Id)
 	if err := db.FetchModelObjects(CloudproviderManager, q, &cloudproviders); err != nil {
@@ -62,7 +63,7 @@ func (self *SCloudaccount) ValidateDeleteCondition(ctx context.Context) error {
 	if self.Enabled {
 		return httperrors.NewInvalidStatusError("account is enabled")
 	}
-	if len(self.getCloudproviders()) > 0 {
+	if len(self.GetCloudproviders()) > 0 {
 		return httperrors.NewNotEmptyError("Not an empty cloud account")
 	}
 	return self.SEnabledStatusStandaloneResourceBase.ValidateDeleteCondition(ctx)
@@ -95,7 +96,7 @@ func (self *SCloudaccountManager) ValidateCreateData(ctx context.Context, userCr
 		return nil, httperrors.NewConflictError("The account has been registered")
 	}
 
-	if subAccount, err := getSubAccounts(name, url, account, secret, provider); err != nil {
+	if subAccount, err := GetSubAccounts(name, url, account, secret, provider); err != nil {
 		return nil, err
 	} else if accounts, err := subAccount.GetArray("data"); err != nil {
 		return nil, err
@@ -108,8 +109,8 @@ func (self *SCloudaccountManager) ValidateCreateData(ctx context.Context, userCr
 func (self *SCloudaccount) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) {
 	self.SEnabledStatusStandaloneResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data)
 	self.savePassword(self.Secret)
-
-	if subAccounts, err := data.GetArray("accounts"); err == nil && len(subAccounts) > 0 {
+	_import, _ := data.Bool("import")
+	if subAccounts, err := data.GetArray("accounts"); err == nil && _import && len(subAccounts) > 0 {
 		for _, subAccount := range subAccounts {
 			name, _ := subAccount.GetString("name")
 			account, _ := subAccount.GetString("account")
@@ -183,7 +184,7 @@ func (self *SCloudaccount) PerformSync(ctx context.Context, userCred mcclient.To
 		return nil, httperrors.NewInputParameterError("invalid input %s", err)
 	}
 	if self.CanSync() || syncRange.Force {
-		err = self.startSyncCloudProviderInfoTask(ctx, userCred, nil, "")
+		err = self.startSyncCloudProviderInfoTask(ctx, userCred, &syncRange, "")
 	}
 	return nil, err
 }
@@ -220,6 +221,18 @@ func (self *SCloudaccount) PerformUpdateCredential(ctx context.Context, userCred
 		changed = true
 	}
 	if (len(account) > 0 && account != self.Account) || (len(accessUrl) > 0 && accessUrl != self.AccessUrl) {
+		for _, cloudprovider := range self.GetCloudproviders() {
+			if cloudprovider.Account == self.Account {
+				if len(account) > 0 {
+					if _, err = cloudprovider.GetModelManager().TableSpec().Update(&cloudprovider, func() error {
+						cloudprovider.Account = account
+						return nil
+					}); err != nil {
+						return nil, err
+					}
+				}
+			}
+		}
 		_, err = self.GetModelManager().TableSpec().Update(self, func() error {
 			if len(account) > 0 {
 				self.Account = account
@@ -242,15 +255,16 @@ func (self *SCloudaccount) PerformUpdateCredential(ctx context.Context, userCred
 }
 
 func (self *SCloudaccount) startSyncCloudProviderInfoTask(ctx context.Context, userCred mcclient.TokenCredential, syncRange *SSyncRange, parentTaskId string) error {
-	for _, cloudprovider := range self.getCloudproviders() {
-		params := jsonutils.NewDict()
-		if syncRange != nil {
-			params.Add(jsonutils.Marshal(syncRange), "sync_range")
-		}
-		if cloudprovider.Enabled {
-			cloudprovider.startSyncCloudProviderInfoTask(ctx, userCred, nil, "")
-		}
+	params := jsonutils.NewDict()
+	if syncRange != nil {
+		params.Add(jsonutils.Marshal(syncRange), "sync_range")
 	}
+	task, err := taskman.TaskManager.NewTask(ctx, "CloudAccountSyncInfoTask", self, userCred, params, parentTaskId, "", nil)
+	if err != nil {
+		log.Errorf("startSyncCloudAccountInfoTask newTask error %s", err)
+		return err
+	}
+	task.ScheduleRun(nil)
 	return nil
 }
 
@@ -288,10 +302,16 @@ func (self *SCloudaccount) AllowPerformImport(ctx context.Context, userCred mccl
 	return userCred.IsSystemAdmin()
 }
 
-func (self *SCloudaccount) PerformImport(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+func (self *SCloudaccount) GetSubAccounts() (jsonutils.JSONObject, error) {
 	if secret, err := self.getPassword(); err != nil {
 		return nil, err
-	} else if subAccounts, err := getSubAccounts(self.Name, self.AccessUrl, self.Account, secret, self.Provider); err != nil {
+	} else {
+		return GetSubAccounts(self.Name, self.AccessUrl, self.Account, secret, self.Provider)
+	}
+}
+
+func (self *SCloudaccount) PerformImport(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	if subAccounts, err := self.GetSubAccounts(); err != nil {
 		return nil, err
 	} else if accounts, err := subAccounts.GetArray("data"); err != nil {
 		return nil, err
@@ -321,74 +341,16 @@ func (self *SCloudaccount) PerformImport(ctx context.Context, userCred mcclient.
 						return nil, err
 					}
 					if enabled {
-						newCloudprovider.startSyncCloudProviderInfoTask(ctx, userCred, &SSyncRange{FullSync: true}, "")
+						newCloudprovider.StartSyncCloudProviderInfoTask(ctx, userCred, &SSyncRange{FullSync: true}, "")
 					}
 				}
 			}
 		}
 	}
-	return jsonutils.NewDict(), nil
+	return nil, nil
 }
 
-// func (self *SCloudaccount) PerformGetSubAccounts(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-// 	if !self.Enabled {
-// 		return nil, httperrors.NewInvalidStatusError("Account disabled")
-// 	}
-// 	if provider, err := self.GetDriver(); err != nil {
-// 		return nil, err
-// 	} else if _subAccounts, err := provider.GetSubAccounts(); err != nil {
-// 		return nil, err
-// 	} else {
-// 		result := jsonutils.NewDict()
-// 		data := jsonutils.NewArray()
-// 		accounts := []string{}
-// 		for _, account := range self.getCloudproviders() {
-// 			accounts = append(accounts, account.Account)
-// 			_account := jsonutils.NewDict()
-// 			_account.Add(jsonutils.NewString(account.Account), "account")
-// 			_account.Add(jsonutils.NewString(account.Name), "name")
-// 			_account.Add(jsonutils.JSONTrue, "exist")
-// 			data.Add(_account)
-// 		}
-// 		if _subAccounts != nil {
-// 			if subAccounts, err := _subAccounts.GetArray("data"); err != nil {
-// 				return nil, err
-// 			} else {
-// 				for _, subAccount := range subAccounts {
-// 					if account, err := subAccount.GetString("account"); err != nil {
-// 						log.Errorf("Get subAccount error %v", err)
-// 					} else if !utils.IsInStringArray(account, accounts) {
-// 						_account := subAccount.(*jsonutils.JSONDict)
-// 						_account.Add(jsonutils.JSONFalse, "exist")
-// 						data.Add(_account)
-// 					}
-// 				}
-// 			}
-// 		}
-// 		result.Add(data, "data")
-// 		result.Add(jsonutils.NewInt(int64(data.Length())), "total")
-// 		return result, nil
-// 	}
-// }
-
-// func (manager *SCloudaccountManager) AllowPerformGetSubAccounts(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
-// 	return userCred.IsSystemAdmin()
-// }
-
-// func (manager *SCloudaccountManager) PerformGetSubAccounts(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-// 	name, _ := data.GetString("name")
-// 	accessUrl, _ := data.GetString("access_url")
-// 	account, _ := data.GetString("account")
-// 	secret, _ := data.GetString("secret")
-// 	_provider, _ := data.GetString("provider")
-// 	if provider, err := cloudprovider.GetProvider("", name, accessUrl, account, secret, _provider); err != nil {
-// 		return nil, err
-// 	} else {
-// 		return provider.GetSubAccounts()
-// 	}
-// }
-
-func getSubAccounts(name, accessUrl, account, secret, provider string) (jsonutils.JSONObject, error) {
+func GetSubAccounts(name, accessUrl, account, secret, provider string) (jsonutils.JSONObject, error) {
 	if provider, err := cloudprovider.GetProvider("", name, accessUrl, account, secret, provider); err != nil {
 		return nil, err
 	} else {
@@ -403,27 +365,28 @@ func (self *SCloudaccount) SaveSysInfo(info jsonutils.JSONObject) {
 	})
 }
 
-func (manager *SCloudaccountManager) FetchCloudproviderById(providerId string) *SCloudprovider {
-	providerObj, err := manager.FetchById(providerId)
+func (manager *SCloudaccountManager) FetchCloudaccountById(accountId string) *SCloudaccount {
+	providerObj, err := manager.FetchById(accountId)
 	if err != nil {
 		log.Errorf("%s", err)
 		return nil
 	}
-	return providerObj.(*SCloudprovider)
+	return providerObj.(*SCloudaccount)
 }
 
-func (manager *SCloudaccountManager) FetchCloudproviderByIdOrName(providerId string) *SCloudprovider {
-	providerObj, err := manager.FetchByIdOrName("", providerId)
+func (manager *SCloudaccountManager) FetchCloudaccountByIdOrName(accountId string) *SCloudaccount {
+	providerObj, err := manager.FetchByIdOrName(nil, accountId)
 	if err != nil {
 		if err != sql.ErrNoRows {
 			log.Errorf("%s", err)
 		}
 		return nil
 	}
-	return providerObj.(*SCloudprovider)
+	return providerObj.(*SCloudaccount)
 }
 
 func (self *SCloudaccount) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict {
+	extra.Add(jsonutils.Marshal(self.GetCloudproviders()), "accounts")
 	return extra
 }
 
@@ -446,7 +409,7 @@ func (manager *SCloudaccountManager) InitializeData() error {
 	}
 	newAccounts := map[string]string{}
 	for _, cloudprovider := range cloudproviders {
-		Account, providerAccount, providerName := cloudprovider.Account, cloudprovider.Account, cloudprovider.Name
+		Account, providerAccount, providerName := cloudprovider.Account, "", cloudprovider.Name
 		if cloudprovider.Provider == CLOUD_PROVIDER_AZURE {
 			if accountInfo := strings.Split(cloudprovider.Account, "/"); len(accountInfo) == 2 {
 				if _, ok := newAccounts[accountInfo[0]]; ok {

pkg/compute/models/cloudproviders.go

@@ -4,12 +4,16 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"strings"
 	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/onecloud/pkg/compute/options"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/pkg/util/timeutils"
 	"yunion.io/x/pkg/utils"
 
@@ -51,7 +55,9 @@ type SCloudprovider struct {
 	Account string `width:"128" charset:"ascii" nullable:"false" list:"admin" create:"admin_required"` // Column(VARCHAR(64, charset='ascii'), nullable=False)
 	Secret  string `width:"256" charset:"ascii" nullable:"false" list:"admin" create:"admin_required"` // Column(VARCHAR(256, charset='ascii'), nullable=False)
 
-	CloudaccountId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required" key_index:"true""`
+	CloudaccountId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required" key_index:"true"`
+
+	ProjectId string `name:"tenant_id" width:"128" charset:"ascii" nullable:"true" list:"admin"`
 
 	LastSync time.Time `get:"admin" list:"admin"` // = Column(DateTime, nullable=True)
 
@@ -122,6 +128,35 @@ func (self *SCloudprovider) CanSync() bool {
 	}
 }
 
+func (self *SCloudprovider) SyncProject() (err error) {
+	projectId := ""
+	if len(self.ProjectId) == 0 && len(self.Name) > 0 && self.Provider == CLOUD_PROVIDER_AZURE {
+		s := auth.GetAdminSession(options.Options.Region, "")
+		if project, err := modules.Projects.GetByName(s, self.Name, nil); err == nil {
+			if projectId, err = project.GetString("id"); err != nil {
+				return err
+			}
+		} else if strings.Index(err.Error(), "404 NotFoundError") > 0 {
+			if project, err := modules.Projects.Create(s, jsonutils.Marshal(map[string]string{"name": self.Name})); err != nil {
+				return err
+			} else if projectId, err = project.GetString("id"); err != nil {
+				return err
+			}
+		} else {
+			return err
+		}
+		if len(projectId) > 0 {
+			if _, err := self.GetModelManager().TableSpec().Update(self, func() error {
+				self.ProjectId = projectId
+				return nil
+			}); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 type SSyncRange struct {
 	Force    bool
 	FullSync bool
@@ -148,7 +183,7 @@ func (sr *SSyncRange) NeedSyncInfo() bool {
 
 func (sr *SSyncRange) normalizeRegionIds() error {
 	for i := 0; i < len(sr.Region); i += 1 {
-		obj, err := CloudregionManager.FetchByIdOrName("", sr.Region[i])
+		obj, err := CloudregionManager.FetchByIdOrName(nil, sr.Region[i])
 		if err != nil {
 			if err == sql.ErrNoRows {
 				return httperrors.NewResourceNotFoundError("Region %s not found", sr.Region[i])
@@ -163,7 +198,7 @@ func (sr *SSyncRange) normalizeRegionIds() error {
 
 func (sr *SSyncRange) normalizeZoneIds() error {
 	for i := 0; i < len(sr.Zone); i += 1 {
-		obj, err := ZoneManager.FetchByIdOrName("", sr.Zone[i])
+		obj, err := ZoneManager.FetchByIdOrName(nil, sr.Zone[i])
 		if err != nil {
 			if err == sql.ErrNoRows {
 				return httperrors.NewResourceNotFoundError("Zone %s not found", sr.Zone[i])
@@ -178,7 +213,7 @@ func (sr *SSyncRange) normalizeZoneIds() error {
 
 func (sr *SSyncRange) normalizeHostIds() error {
 	for i := 0; i < len(sr.Host); i += 1 {
-		obj, err := HostManager.FetchByIdOrName("", sr.Host[i])
+		obj, err := HostManager.FetchByIdOrName(nil, sr.Host[i])
 		if err != nil {
 			if err == sql.ErrNoRows {
 				return httperrors.NewResourceNotFoundError("Host %s not found", sr.Host[i])
@@ -227,12 +262,15 @@ func (self *SCloudprovider) PerformSync(ctx context.Context, userCred mcclient.T
 		return nil, httperrors.NewInputParameterError("invalid input %s", err)
 	}
 	if self.CanSync() || syncRange.Force {
-		err = self.startSyncCloudProviderInfoTask(ctx, userCred, &syncRange, "")
+		err = self.StartSyncCloudProviderInfoTask(ctx, userCred, &syncRange, "")
 	}
 	return nil, err
 }
 
-func (self *SCloudprovider) startSyncCloudProviderInfoTask(ctx context.Context, userCred mcclient.TokenCredential, syncRange *SSyncRange, parentTaskId string) error {
+func (self *SCloudprovider) StartSyncCloudProviderInfoTask(ctx context.Context, userCred mcclient.TokenCredential, syncRange *SSyncRange, parentTaskId string) error {
+	if err := self.SyncProject(); err != nil {
+		log.Errorf("Sync cloudprovider project error: %v", err)
+	}
 	params := jsonutils.NewDict()
 	if syncRange != nil {
 		params.Add(jsonutils.Marshal(syncRange), "sync_range")
@@ -265,12 +303,10 @@ type SAccount struct {
 }
 
 func (self *SCloudprovider) getCloudaccount() (*SCloudaccount, error) {
-	cloudaccount := &SCloudaccount{}
-	q := CloudaccountManager.Query().Equals("id", self.CloudaccountId)
-	if err := db.FetchModelObjects(CloudaccountManager, q, cloudaccount); err != nil {
-		return nil, err
+	if cloudaccount := CloudaccountManager.FetchCloudaccountById(self.CloudaccountId); cloudaccount != nil {
+		return cloudaccount, nil
 	}
-	return cloudaccount, nil
+	return nil, fmt.Errorf("Failed to find cloud account for cloud provider %s", self.Name)
 }
 
 func (self *SCloudprovider) getAccount() (*SAccount, error) {
@@ -283,7 +319,7 @@ func (self *SCloudprovider) getAccount() (*SAccount, error) {
 		} else {
 			account.Secret = passwd
 		}
-		if account.Account != self.Account {
+		if len(self.Account) > 0 && self.Account != cloudaccount.Account {
 			account.Account = fmt.Sprintf("%s/%s", account.Account, self.Account)
 		}
 		return &account, nil
@@ -320,7 +356,7 @@ func (manager *SCloudproviderManager) FetchCloudproviderById(providerId string)
 }
 
 func (manager *SCloudproviderManager) FetchCloudproviderByIdOrName(providerId string) *SCloudprovider {
-	providerObj, err := manager.FetchByIdOrName("", providerId)
+	providerObj, err := manager.FetchByIdOrName(nil, providerId)
 	if err != nil {
 		if err != sql.ErrNoRows {
 			log.Errorf("%s", err)

pkg/compute/models/disks.go

@@ -38,6 +38,7 @@ const (
 	DISK_STARTALLOC     = "start_alloc"
 	DISK_ALLOCATING     = "allocating"
 	DISK_READY          = "ready"
+	DISK_RESET          = "reset"
 	DISK_DEALLOC        = "deallocating"
 	DISK_DEALLOC_FAILED = "dealloc_failed"
 	DISK_UNKNOWN        = "unknown"
@@ -155,7 +156,7 @@ func (manager *SDiskManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu
 
 	storageStr := jsonutils.GetAnyString(queryDict, []string{"storage", "storage_id"})
 	if len(storageStr) > 0 {
-		storageObj, err := StorageManager.FetchByIdOrName(userCred.GetProjectId(), storageStr)
+		storageObj, err := StorageManager.FetchByIdOrName(userCred, storageStr)
 		if err != nil {
 			return nil, httperrors.NewResourceNotFoundError("storage %s not found: %s", storageStr, err)
 		}
@@ -391,10 +392,8 @@ func (self *SDisk) CleanUpDiskSnapshots(ctx context.Context, userCred mcclient.T
 	convertSnapshots := jsonutils.NewArray()
 	deleteSnapshots := jsonutils.NewArray()
 	for i := 0; i < len(dest); i++ {
-		if dest[i].CreatedBy == MANUAL && !dest[i].FakeDeleted {
-			if !dest[i].OutOfChain {
-				convertSnapshots.Add(jsonutils.NewString(dest[i].Id))
-			}
+		if !dest[i].FakeDeleted && !dest[i].OutOfChain {
+			convertSnapshots.Add(jsonutils.NewString(dest[i].Id))
 		} else {
 			deleteSnapshots.Add(jsonutils.NewString(dest[i].Id))
 		}
@@ -416,6 +415,9 @@ func (self *SDisk) AllowPerformDiskReset(ctx context.Context, userCred mcclient.
 }
 
 func (self *SDisk) PerformDiskReset(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	if self.Status != DISK_READY {
+		return nil, httperrors.NewInvalidStatusError("Cannot reset disk in status %s", self.Status)
+	}
 	snapshotId, err := data.GetString("snapshot_id")
 	if err != nil {
 		return nil, err
@@ -436,13 +438,16 @@ func (self *SDisk) PerformDiskReset(ctx context.Context, userCred mcclient.Token
 	if snapshot.Status != SNAPSHOT_READY {
 		return nil, httperrors.NewBadRequestError("Cannot reset disk with snapshot in status %s", snapshot.Status)
 	}
-	self.StartResetDisk(ctx, userCred, snapshotId)
+	autoStart := jsonutils.QueryBoolean(data, "auto_start", false)
+	self.StartResetDisk(ctx, userCred, snapshotId, autoStart)
 	return nil, nil
 }
 
-func (self *SDisk) StartResetDisk(ctx context.Context, userCred mcclient.TokenCredential, snapshotId string) error {
+func (self *SDisk) StartResetDisk(ctx context.Context, userCred mcclient.TokenCredential, snapshotId string, autoStart bool) error {
+	self.SetStatus(userCred, DISK_RESET, "")
 	params := jsonutils.NewDict()
 	params.Set("snapshot_id", jsonutils.NewString(snapshotId))
+	params.Set("auto_start", jsonutils.NewBool(autoStart))
 	task, err := taskman.TaskManager.NewTask(ctx, "DiskResetTask", self, userCred, params, "", "", nil)
 	if err != nil {
 		return err
@@ -732,7 +737,11 @@ func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.Toke
 		self.ExpiredAt = extDisk.GetExpiredAt()
 
 		self.ProjectId = userCred.GetProjectId()
-
+		if manageId := extDisk.GetIStorge().GetManagerId(); len(manageId) > 0 {
+			if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil {
+				self.ProjectId = provider.ProjectId
+			}
+		}
 		return nil
 	})
 	if err != nil {
@@ -765,7 +774,11 @@ func (manager *SDiskManager) newFromCloudDisk(ctx context.Context, userCred mccl
 	disk.ExternalId = extDisk.GetGlobalId()
 	disk.StorageId = storage.Id
 	disk.ProjectId = userCred.GetProjectId()
-
+	if manageId := extDisk.GetIStorge().GetManagerId(); len(manageId) > 0 {
+		if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil {
+			disk.ProjectId = provider.ProjectId
+		}
+	}
 	disk.DiskFormat = extDisk.GetDiskFormat()
 	disk.DiskSize = extDisk.GetDiskSizeMB()
 	disk.AutoDelete = extDisk.GetIsAutoDelete()
@@ -1249,7 +1262,7 @@ func (manager *SDiskManager) AutoDiskSnapshot(ctx context.Context, userCred mccl
 			continue
 		}
 		// name
-		name := guests[0].Name + time.Now().Format("2006-01-02#15:04:05")
+		name := "Auto-" + guests[0].Name + time.Now().Format("2006-01-02#15:04:05")
 		snap, err := SnapshotManager.CreateSnapshot(ctx, userCred, AUTO, disk.Id, guests[0].Id, "", name)
 		if err != nil {
 			log.Errorln(err)

pkg/compute/models/elasticips.go

@@ -84,7 +84,7 @@ func (manager *SElasticipManager) ListItemFilter(ctx context.Context, q *sqlchem
 
 	managerFilter, _ := query.GetString("manager")
 	if len(managerFilter) > 0 {
-		managerI, err := CloudproviderManager.FetchByIdOrName(userCred.GetProjectId(), managerFilter)
+		managerI, err := CloudproviderManager.FetchByIdOrName(userCred, managerFilter)
 		if err != nil {
 			if err == sql.ErrNoRows {
 				return nil, httperrors.NewResourceNotFoundError("cloud provider %s not found", managerFilter)
@@ -97,7 +97,7 @@ func (manager *SElasticipManager) ListItemFilter(ctx context.Context, q *sqlchem
 
 	regionFilter, _ := query.GetString("region")
 	if len(regionFilter) > 0 {
-		regionObj, err := CloudregionManager.FetchByIdOrName(userCred.GetProjectId(), regionFilter)
+		regionObj, err := CloudregionManager.FetchByIdOrName(userCred, regionFilter)
 		if err != nil {
 			if err == sql.ErrNoRows {
 				return nil, httperrors.NewResourceNotFoundError("cloud region %s not found", regionFilter)
@@ -232,7 +232,12 @@ func (self *SElasticip) SyncWithCloudEip(userCred mcclient.TokenCredential, ext
 		self.ExternalId = ext.GetGlobalId()
 		// self.ManagerId = ext.GetManagerId()
 		self.IsEmulated = ext.IsEmulated()
-		// self.ProjectId = userCred.GetProjectId()
+		self.ProjectId = userCred.GetProjectId()
+		if manageId := ext.GetManagerId(); len(manageId) > 0 {
+			if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil {
+				self.ProjectId = provider.ProjectId
+			}
+		}
 		self.ChargeType = ext.GetInternetChargeType()
 
 		return nil
@@ -258,7 +263,11 @@ func (manager *SElasticipManager) newFromCloudEip(userCred mcclient.TokenCredent
 	eip.ChargeType = extEip.GetInternetChargeType()
 
 	eip.ProjectId = userCred.GetProjectId()
-
+	if manageId := extEip.GetManagerId(); len(manageId) > 0 {
+		if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil {
+			eip.ProjectId = provider.ProjectId
+		}
+	}
 	err := manager.TableSpec().Insert(&eip)
 	if err != nil {
 		log.Errorf("newFromCloudEip fail %s", err)
@@ -362,7 +371,7 @@ func (manager *SElasticipManager) ValidateCreateData(ctx context.Context, userCr
 	if len(regionStr) == 0 {
 		return nil, httperrors.NewInputParameterError("Missing region/region_id")
 	}
-	region, err := CloudregionManager.FetchByIdOrName("", regionStr)
+	region, err := CloudregionManager.FetchByIdOrName(nil, regionStr)
 	if err != nil {
 		if err != sql.ErrNoRows {
 			return nil, httperrors.NewGeneralError(err)
@@ -377,7 +386,7 @@ func (manager *SElasticipManager) ValidateCreateData(ctx context.Context, userCr
 		return nil, httperrors.NewInputParameterError("Missing manager/manager_id")
 	}
 
-	provider, err := CloudproviderManager.FetchByIdOrName("", managerStr)
+	provider, err := CloudproviderManager.FetchByIdOrName(nil, managerStr)
 	if err != nil {
 		if err != sql.ErrNoRows {
 			return nil, httperrors.NewGeneralError(err)
@@ -490,7 +499,7 @@ func (self *SElasticip) PerformAssociate(ctx context.Context, userCred mcclient.
 		return nil, httperrors.NewInputParameterError("Unsupported %s", instanceType)
 	}
 
-	vmObj, err := GuestManager.FetchByIdOrName(userCred.GetProjectId(), instanceId)
+	vmObj, err := GuestManager.FetchByIdOrName(userCred, instanceId)
 	if err != nil {
 		if err == sql.ErrNoRows {
 			return nil, httperrors.NewResourceNotFoundError("server %s not found", instanceId)

pkg/compute/models/guestnetworks.go

@@ -578,4 +578,4 @@ func (manager *SGuestnetworkManager) getRecentlyReleasedIPAddresses(networkId st
 		}
 	}
 	return ret
-}
+}

pkg/compute/models/guests.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"database/sql"
+	"encoding/base64"
 	"fmt"
 	"net/http"
 	"strconv"
@@ -29,14 +30,17 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
+
 	"yunion.io/x/onecloud/pkg/cloudprovider"
-	"yunion.io/x/onecloud/pkg/compute/options"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/onecloud/pkg/util/logclient"
 	"yunion.io/x/onecloud/pkg/util/seclib2"
+
+	"yunion.io/x/onecloud/pkg/compute/options"
+	"yunion.io/x/onecloud/pkg/compute/sshkeys"
 )
 
 const (
@@ -222,7 +226,7 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ
 
 	hostFilter, _ := queryDict.GetString("host")
 	if len(hostFilter) > 0 {
-		host, _ := HostManager.FetchByIdOrName("", hostFilter)
+		host, _ := HostManager.FetchByIdOrName(nil, hostFilter)
 		if host == nil {
 			return nil, httperrors.NewResourceNotFoundError("host %s not found", hostFilter)
 		}
@@ -231,7 +235,7 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ
 
 	secgrpFilter, _ := queryDict.GetString("secgroup")
 	if len(secgrpFilter) > 0 {
-		secgrp, _ := SecurityGroupManager.FetchByIdOrName("", secgrpFilter)
+		secgrp, _ := SecurityGroupManager.FetchByIdOrName(nil, secgrpFilter)
 		if secgrp == nil {
 			return nil, httperrors.NewResourceNotFoundError("secgroup %s not found", secgrpFilter)
 		}
@@ -240,7 +244,7 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ
 
 	zoneFilter, _ := queryDict.GetString("zone")
 	if len(zoneFilter) > 0 {
-		zone, _ := ZoneManager.FetchByIdOrName("", zoneFilter)
+		zone, _ := ZoneManager.FetchByIdOrName(nil, zoneFilter)
 		if zone == nil {
 			return nil, httperrors.NewResourceNotFoundError("zone %s not found", zoneFilter)
 		}
@@ -253,7 +257,7 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ
 
 	wireFilter, _ := queryDict.GetString("wire")
 	if len(wireFilter) > 0 {
-		wire, _ := WireManager.FetchByIdOrName("", wireFilter)
+		wire, _ := WireManager.FetchByIdOrName(nil, wireFilter)
 		if wire == nil {
 			return nil, httperrors.NewResourceNotFoundError("wire %s not found", wireFilter)
 		}
@@ -265,7 +269,7 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ
 
 	networkFilter, _ := queryDict.GetString("network")
 	if len(networkFilter) > 0 {
-		netI, _ := NetworkManager.FetchByIdOrName(userCred.GetProjectId(), networkFilter)
+		netI, _ := NetworkManager.FetchByIdOrName(userCred, networkFilter)
 		if netI == nil {
 			return nil, httperrors.NewResourceNotFoundError("network %s not found", networkFilter)
 		}
@@ -279,7 +283,7 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ
 
 	diskFilter, _ := queryDict.GetString("disk")
 	if len(diskFilter) > 0 {
-		diskI, _ := DiskManager.FetchByIdOrName(userCred.GetProjectId(), diskFilter)
+		diskI, _ := DiskManager.FetchByIdOrName(userCred, diskFilter)
 		if diskI == nil {
 			return nil, httperrors.NewResourceNotFoundError("disk %s not found", diskFilter)
 		}
@@ -312,7 +316,7 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ
 
 	managerFilter, _ := queryDict.GetString("manager")
 	if len(managerFilter) > 0 {
-		managerI, _ := CloudproviderManager.FetchByIdOrName(userCred.GetProjectId(), managerFilter)
+		managerI, _ := CloudproviderManager.FetchByIdOrName(userCred, managerFilter)
 		if managerI == nil {
 			return nil, httperrors.NewResourceNotFoundError("cloud provider %s not found", managerFilter)
 		}
@@ -323,7 +327,7 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ
 
 	regionFilter, _ := queryDict.GetString("region")
 	if len(regionFilter) > 0 {
-		regionObj, err := CloudregionManager.FetchByIdOrName(userCred.GetProjectId(), regionFilter)
+		regionObj, err := CloudregionManager.FetchByIdOrName(userCred, regionFilter)
 		if err != nil {
 			if err == sql.ErrNoRows {
 				return nil, httperrors.NewResourceNotFoundError("cloud region %s not found", regionFilter)
@@ -555,7 +559,7 @@ func (self *SGuest) ValidateUpdateData(ctx context.Context, userCred mcclient.To
 
 	err = self.checkUpdateQuota(ctx, userCred, vcpuCount, vmemSize)
 	if err != nil {
-		return nil, err
+		return nil, httperrors.NewOutOfQuotaError(err.Error())
 	}
 
 	if data.Contains("name") {
@@ -640,7 +644,7 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m
 		if len(bmName) == 0 {
 			bmName, _ = data.GetString("prefer_baremetal")
 		}
-		bmObj, err := HostManager.FetchByIdOrName("", bmName)
+		bmObj, err := HostManager.FetchByIdOrName(nil, bmName)
 		if err != nil {
 			if err == sql.ErrNoRows {
 				return nil, httperrors.NewResourceNotFoundError("Host %s not found", bmName)
@@ -804,7 +808,7 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m
 		keypairId, _ = data.GetString("keypair_id")
 	}
 	if len(keypairId) > 0 {
-		keypairObj, err := KeypairManager.FetchByIdOrName(userCred.GetUserId(), keypairId)
+		keypairObj, err := KeypairManager.FetchByIdOrName(userCred, keypairId)
 		if err != nil {
 			return nil, httperrors.NewResourceNotFoundError("Keypair %s not found", keypairId)
 		}
@@ -815,7 +819,7 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m
 
 	if data.Contains("secgroup") {
 		secGrpId, _ := data.GetString("secgroup")
-		secGrpObj, err := SecurityGroupManager.FetchByIdOrName(userCred.GetProjectId(), secGrpId)
+		secGrpObj, err := SecurityGroupManager.FetchByIdOrName(userCred, secGrpId)
 		if err != nil {
 			return nil, httperrors.NewResourceNotFoundError("Secgroup %s not found", secGrpId)
 		}
@@ -937,6 +941,11 @@ func (guest *SGuest) PostCreate(ctx context.Context, userCred mcclient.TokenCred
 	if osProfileJson != nil {
 		guest.setOSProfile(ctx, userCred, osProfileJson)
 	}
+
+	userData, _ := data.GetString("user_data")
+	if len(userData) > 0 {
+		guest.setUserData(ctx, userCred, userData)
+	}
 }
 
 func (guest *SGuest) setApptags(ctx context.Context, appTags []string, userCred mcclient.TokenCredential) {
@@ -1344,6 +1353,12 @@ func (self *SGuest) syncWithCloudVM(ctx context.Context, userCred mcclient.Token
 		self.Machine = extVM.GetMachine()
 		self.HostId = host.Id
 		self.ProjectId = userCred.GetProjectId()
+		if manageId := extVM.GetIHost().GetManagerId(); len(manageId) > 0 {
+			if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil {
+				self.ProjectId = provider.ProjectId
+			}
+		}
+
 		self.Hypervisor = extVM.GetHypervisor()
 
 		self.IsEmulated = extVM.IsEmulated()
@@ -1411,7 +1426,13 @@ func (manager *SGuestManager) newCloudVM(ctx context.Context, userCred mcclient.
 	guest.ExpiredAt = extVM.GetExpiredAt()
 
 	guest.HostId = host.Id
+
 	guest.ProjectId = userCred.GetProjectId()
+	if manageId := extVM.GetIHost().GetManagerId(); len(manageId) > 0 {
+		if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil {
+			guest.ProjectId = provider.ProjectId
+		}
+	}
 
 	metaData := extVM.GetMetadata()
 
@@ -1764,9 +1785,10 @@ func (self *SGuest) PerformDeploy(ctx context.Context, userCred mcclient.TokenCr
 	if kwargs.Contains("__delete_keypair__") || kwargs.Contains("keypair") {
 		doRestart = true
 		var kpId string
-		if !jsonutils.QueryBoolean(kwargs, "__delete_keypair__", false) {
+
+		if kwargs.Contains("keypair") {
 			keypair, _ := kwargs.GetString("keypair")
-			iKp, err := KeypairManager.FetchByIdOrName(userCred.GetProjectId(), keypair)
+			iKp, err := KeypairManager.FetchByIdOrName(userCred, keypair)
 			if err != nil {
 				return nil, err
 			}
@@ -1776,11 +1798,18 @@ func (self *SGuest) PerformDeploy(ctx context.Context, userCred mcclient.TokenCr
 			kp := iKp.(*SKeypair)
 			kpId = kp.Id
 		}
+
 		if self.KeypairId != kpId {
+			okey := self.getKeypair()
+			if okey != nil {
+				kwargs.Set("delete_public_key", jsonutils.NewString(okey.PublicKey))
+			}
+
 			self.GetModelManager().TableSpec().Update(self, func() error {
 				self.KeypairId = kpId
 				return nil
 			})
+
 			kwargs.Set("reset_password", jsonutils.JSONTrue)
 		}
 	}
@@ -1795,6 +1824,7 @@ func (self *SGuest) PerformDeploy(ctx context.Context, userCred mcclient.TokenCr
 		}
 		return nil, nil
 	}
+
 	return nil, httperrors.NewServerStatusError("Cannot deploy in status %s", self.Status)
 }
 
@@ -1830,7 +1860,7 @@ func (self *SGuest) PerformAttachdisk(ctx context.Context, userCred mcclient.Tok
 	if diskId, err := data.GetString("disk_id"); err != nil {
 		return nil, err
 	} else {
-		if disk, err := DiskManager.FetchByIdOrName(userCred.GetProjectId(), diskId); err != nil {
+		if disk, err := DiskManager.FetchByIdOrName(userCred, diskId); err != nil {
 			return nil, err
 		} else if disk == nil {
 			return nil, httperrors.NewResourceNotFoundError("Disk %s not found", diskId)
@@ -2451,7 +2481,7 @@ func (self *SGuest) PerformAssignSecgroup(ctx context.Context, userCred mcclient
 	} else {
 		if secgrp, err := data.GetString("secgrp"); err != nil {
 			return nil, err
-		} else if sg, err := SecurityGroupManager.FetchByIdOrName(userCred.GetProjectId(), secgrp); err != nil {
+		} else if sg, err := SecurityGroupManager.FetchByIdOrName(userCred, secgrp); err != nil {
 			return nil, httperrors.NewNotFoundError("SecurityGroup %s not found", secgrp)
 		} else {
 			if _, err := self.GetModelManager().TableSpec().Update(self, func() error {
@@ -2538,7 +2568,7 @@ func (self *SGuest) PerformRebuildRoot(ctx context.Context, userCred mcclient.To
 
 	keypairStr := jsonutils.GetAnyString(data, []string{"keypair", "keypair_id"})
 	if len(keypairStr) > 0 {
-		keypairObj, err := KeypairManager.FetchByIdOrName(userCred.GetUserId(), keypairStr)
+		keypairObj, err := KeypairManager.FetchByIdOrName(userCred, keypairStr)
 		if err != nil {
 			if err == sql.ErrNoRows {
 				return nil, httperrors.NewResourceNotFoundError("keypair %s not found", keypairStr)
@@ -2680,7 +2710,7 @@ func (self *SGuest) PerformDetachdisk(ctx context.Context, userCred mcclient.Tok
 		return nil, err
 	}
 	keepDisk := jsonutils.QueryBoolean(data, "keep_disk", false)
-	iDisk, err := DiskManager.FetchByIdOrName(userCred.GetProjectId(), diskId)
+	iDisk, err := DiskManager.FetchByIdOrName(userCred, diskId)
 	if err != nil {
 		return nil, err
 	}
@@ -2735,7 +2765,7 @@ func (self *SGuest) PerformDetachIsolatedDevice(ctx context.Context, userCred mc
 		logclient.AddActionLog(self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false)
 		return nil, httperrors.NewBadRequestError(msg)
 	}
-	iDev, err := IsolatedDeviceManager.FetchByIdOrName(userCred.GetProjectId(), device)
+	iDev, err := IsolatedDeviceManager.FetchByIdOrName(userCred, device)
 	if err != nil {
 		msg := fmt.Sprintf("Isolated device %s not found", device)
 		logclient.AddActionLog(self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false)
@@ -2785,7 +2815,7 @@ func (self *SGuest) PerformAttachIsolatedDevice(ctx context.Context, userCred mc
 		logclient.AddActionLog(self, logclient.ACT_GUEST_ATTACH_ISOLATED_DEVICE, msg, userCred, false)
 		return nil, httperrors.NewBadRequestError(msg)
 	}
-	iDev, err := IsolatedDeviceManager.FetchByIdOrName(userCred.GetProjectId(), device)
+	iDev, err := IsolatedDeviceManager.FetchByIdOrName(userCred, device)
 	if err != nil {
 		msg := fmt.Sprintf("Isolated device %s not found", device)
 		logclient.AddActionLog(self, logclient.ACT_GUEST_ATTACH_ISOLATED_DEVICE, msg, userCred, false)
@@ -3289,10 +3319,29 @@ func (self *SGuest) GetDeployConfigOnHost(ctx context.Context, host *SHost, para
 		if keypair != nil {
 			config.Add(jsonutils.NewString(keypair.PublicKey), "public_key")
 		}
+		deletePubKey, _ := params.GetString("delete_public_key")
+		if len(deletePubKey) > 0 {
+			config.Add(jsonutils.NewString(deletePubKey), "delete_public_key")
+		}
 	} else {
 		config.Add(jsonutils.JSONFalse, "reset_password")
 	}
 
+	// add default public keys
+	_, adminPubKey, err := sshkeys.GetSshAdminKeypair(ctx)
+	if err != nil {
+		log.Errorf("fail to get ssh admin public key %s", err)
+	}
+
+	_, projPubKey, err := sshkeys.GetSshProjectKeypair(ctx, self.ProjectId)
+
+	if err != nil {
+		log.Errorf("fail to get ssh project public key %s", err)
+	}
+
+	config.Add(jsonutils.NewString(adminPubKey), "admin_public_key")
+	config.Add(jsonutils.NewString(projPubKey), "project_public_key")
+
 	config.Add(jsonutils.NewString(deployAction), "action")
 
 	onFinish := "shutdown"
@@ -4343,7 +4392,7 @@ func (self *SGuest) PerformAssociateEip(ctx context.Context, userCred mcclient.T
 	if len(eipStr) == 0 {
 		return nil, httperrors.NewInputParameterError("missing eip or eip_id")
 	}
-	eipObj, err := ElasticipManager.FetchByIdOrName(userCred.GetProjectId(), eipStr)
+	eipObj, err := ElasticipManager.FetchByIdOrName(userCred, eipStr)
 	if err != nil {
 		if err == sql.ErrNoRows {
 			return nil, httperrors.NewResourceNotFoundError("eip %s not found", eipStr)
@@ -4496,3 +4545,37 @@ func (self *SGuest) getDefaultStorageType() string {
 	}
 	return STORAGE_LOCAL
 }
+
+func (self *SGuest) setUserData(ctx context.Context, userCred mcclient.TokenCredential, data string) error {
+	data = base64.StdEncoding.EncodeToString([]byte(data))
+	if len(data) > 16*1024 {
+		return fmt.Errorf("User data is limited to 16 KB.")
+	}
+	err := self.SetMetadata(ctx, "user_data", data, userCred)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (self *SGuest) AllowPerformUserData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return self.IsOwner(userCred)
+}
+
+func (self *SGuest) PerformUserData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	userData, err := data.GetString("user_data")
+	if err != nil {
+		return nil, httperrors.NewInputParameterError("missing user_data %s", err)
+	}
+	err = self.setUserData(ctx, userCred, userData)
+	if err != nil {
+		return nil, httperrors.NewGeneralError(err)
+	}
+	if len(self.HostId) > 0 {
+		err = self.StartSyncTask(ctx, userCred, false, "")
+		if err != nil {
+			return nil, httperrors.NewGeneralError(err)
+		}
+	}
+	return nil, nil
+}

pkg/compute/models/hosts.go

@@ -153,7 +153,7 @@ func (manager *SHostManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu
 
 	schedTagStr := jsonutils.GetAnyString(query, []string{"schedtag", "schedtag_id"})
 	if len(schedTagStr) > 0 {
-		schedTag, _ := SchedtagManager.FetchByIdOrName("", schedTagStr)
+		schedTag, _ := SchedtagManager.FetchByIdOrName(nil, schedTagStr)
 		if schedTag == nil {
 			return nil, httperrors.NewResourceNotFoundError("Schedtag %s not found", schedTagStr)
 		}
@@ -163,7 +163,7 @@ func (manager *SHostManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu
 
 	wireStr := jsonutils.GetAnyString(query, []string{"wire", "wire_id"})
 	if len(wireStr) > 0 {
-		wire, _ := WireManager.FetchByIdOrName("", wireStr)
+		wire, _ := WireManager.FetchByIdOrName(nil, wireStr)
 		if wire == nil {
 			return nil, httperrors.NewResourceNotFoundError("Wire %s not found", wireStr)
 		}
@@ -173,7 +173,7 @@ func (manager *SHostManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu
 
 	storageStr := jsonutils.GetAnyString(query, []string{"storage", "storage_id"})
 	if len(storageStr) > 0 {
-		storage, _ := StorageManager.FetchByIdOrName("", storageStr)
+		storage, _ := StorageManager.FetchByIdOrName(nil, storageStr)
 		if storage == nil {
 			return nil, httperrors.NewResourceNotFoundError("Storage %s not found", storageStr)
 		}
@@ -183,7 +183,7 @@ func (manager *SHostManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu
 
 	zoneStr := jsonutils.GetAnyString(query, []string{"zone", "zone_id"})
 	if len(zoneStr) > 0 {
-		zone, _ := ZoneManager.FetchByIdOrName("", zoneStr)
+		zone, _ := ZoneManager.FetchByIdOrName(nil, zoneStr)
 		if zone == nil {
 			return nil, httperrors.NewResourceNotFoundError("Zone %s not found", zoneStr)
 		}

pkg/compute/models/isolated_devices.go

@@ -115,7 +115,7 @@ func (manager *SIsolatedDeviceManager) ListItemFilter(ctx context.Context, q *sq
 	}
 	zoneStr := jsonutils.GetAnyString(query, []string{"zone", "zone_id"})
 	if len(zoneStr) > 0 {
-		zone, _ := ZoneManager.FetchByIdOrName("", zoneStr)
+		zone, _ := ZoneManager.FetchByIdOrName(nil, zoneStr)
 		if zone == nil {
 			return nil, httperrors.NewResourceNotFoundError("Zone %s not found", zoneStr)
 		}

pkg/compute/models/keypairs.go

@@ -4,10 +4,15 @@ import (
 	"context"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/sqlchemy"
+
+	"golang.org/x/crypto/ssh"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
-	"yunion.io/x/sqlchemy"
+	"yunion.io/x/onecloud/pkg/util/seclib2"
+	"yunion.io/x/pkg/utils"
 )
 
 type SKeypairManager struct {
@@ -23,11 +28,11 @@ func init() {
 type SKeypair struct {
 	db.SStandaloneResourceBase
 
-	Scheme      string `width:"12" charset:"ascii" nullable:"true" default:"RSA" list:"user" create:"optional"` // Column(VARCHAR(length=12, charset='ascii'), nullable=True, default='RSA')
-	Fingerprint string `width:"48" charset:"ascii" nullable:"false" list:"user"`                                // Column(VARCHAR(length=48, charset='ascii'), nullable=False)
-	PrivateKey  string `width:"2048" charset:"ascii" nullable:"false"`                                          // Column(VARCHAR(length=2048, charset='ascii'), nullable=False)
-	PublicKey   string `width:"1024" charset:"ascii" nullable:"false" list:"user"`                              // Column(VARCHAR(length=1024, charset='ascii'), nullable=False)
-	OwnerId     string `width:"128" charset:"ascii" index:"true" nullable:"false"`                              // Column(VARCHAR(length=36, charset='ascii'), index=True, nullable=False)
+	Scheme      string `width:"12" charset:"ascii" nullable:"true" default:"RSA" list:"user" create:"required"` // Column(VARCHAR(length=12, charset='ascii'), nullable=True, default='RSA')
+	Fingerprint string `width:"48" charset:"ascii" nullable:"false" list:"user" create:"required"`              // Column(VARCHAR(length=48, charset='ascii'), nullable=False)
+	PrivateKey  string `width:"2048" charset:"ascii" nullable:"false" create:"optional"`                        // Column(VARCHAR(length=2048, charset='ascii'), nullable=False)
+	PublicKey   string `width:"1024" charset:"ascii" nullable:"false" list:"user" create:"required"`            // Column(VARCHAR(length=1024, charset='ascii'), nullable=False)
+	OwnerId     string `width:"128" charset:"ascii" index:"true" nullable:"false" create:"required"`            // Column(VARCHAR(length=36, charset='ascii'), index=True, nullable=False)
 }
 
 func (manager *SKeypairManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
@@ -100,7 +105,40 @@ func (self *SKeypair) GetLinkedGuestsCount() int {
 }
 
 func (manager *SKeypairManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
-	// XXX: TODO
+	publicKey, _ := data.GetString("public_key")
+	if len(publicKey) == 0 {
+		scheme, _ := data.GetString("scheme")
+		if len(scheme) > 0 {
+			if !utils.IsInStringArray(scheme, []string{"RSA", "DSA"}) {
+				return nil, httperrors.NewInputParameterError("Unsupported scheme %s", scheme)
+			}
+		} else {
+			scheme = "RSA"
+		}
+		var privKey, pubKey string
+		var err error
+		if scheme == "RSA" {
+			privKey, pubKey, err = seclib2.GenerateRSASSHKeypair()
+		} else {
+			privKey, pubKey, err = seclib2.GenerateDSASSHKeypair()
+		}
+		if err != nil {
+			log.Errorf("fail to generate ssh keypair %s", err)
+			return nil, httperrors.NewGeneralError(err)
+		}
+		publicKey = pubKey
+		data.Set("public_key", jsonutils.NewString(pubKey))
+		data.Set("private_key", jsonutils.NewString(privKey))
+	}
+	pubKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(publicKey))
+	if err != nil {
+		log.Errorf("invalid public key %s", err)
+		return nil, httperrors.NewInputParameterError("invalid public")
+	}
+	data.Set("fingerprint", jsonutils.NewString(ssh.FingerprintLegacyMD5(pubKey)))
+	data.Set("scheme", jsonutils.NewString(seclib2.GetPublicKeyScheme(pubKey)))
+	data.Set("owner_id", jsonutils.NewString(userCred.GetUserId()))
+
 	return manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)
 }
 
@@ -124,6 +162,37 @@ func (self *SKeypair) GetOwnerProjectId() string {
 	return self.OwnerId
 }
 
-func (manager *SKeypairManager) GetOwnerId(userCred mcclient.TokenCredential) string {
+func (manager *SKeypairManager) GetOwnerId(userCred mcclient.IIdentityProvider) string {
 	return userCred.GetUserId()
 }
+
+func (manager *SKeypairManager) FetchByName(userCred mcclient.IIdentityProvider, idStr string) (db.IModel, error) {
+	return db.FetchByName(manager, userCred, idStr)
+}
+
+func (manager *SKeypairManager) FetchByIdOrName(userCred mcclient.IIdentityProvider, idStr string) (db.IModel, error) {
+	return db.FetchByIdOrName(manager, userCred, idStr)
+}
+
+func (keypair *SKeypair) AllowGetDetailsPrivatekey(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+	return keypair.OwnerId == userCred.GetUserId()
+}
+
+func (keypair *SKeypair) GetDetailsPrivatekey(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	retval := jsonutils.NewDict()
+	if len(keypair.PrivateKey) > 0 {
+		retval.Add(jsonutils.NewString(keypair.PrivateKey), "private_key")
+		retval.Add(jsonutils.NewString(keypair.Name), "name")
+		retval.Add(jsonutils.NewString(keypair.Scheme), "scheme")
+		_, err := keypair.GetModelManager().TableSpec().Update(keypair, func() error {
+			keypair.PrivateKey = ""
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		db.OpsLog.LogEvent(keypair, db.ACT_FETCH, nil, userCred)
+	}
+	return retval, nil
+}

pkg/compute/models/networks.go

@@ -493,6 +493,11 @@ func (self *SNetwork) SyncWithCloudNetwork(userCred mcclient.TokenCredential, ex
 		self.AllocTimoutSeconds = extNet.GetAllocTimeoutSeconds()
 
 		self.ProjectId = userCred.GetProjectId()
+		if manageId := extNet.GetIWire().GetIVpc().GetManagerId(); len(manageId) > 0 {
+			if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil {
+				self.ProjectId = provider.ProjectId
+			}
+		}
 		return nil
 	})
 	if err != nil {
@@ -519,7 +524,11 @@ func (manager *SNetworkManager) newFromCloudNetwork(userCred mcclient.TokenCrede
 	net.AllocTimoutSeconds = extNet.GetAllocTimeoutSeconds()
 
 	net.ProjectId = userCred.GetProjectId()
-
+	if manageId := extNet.GetIWire().GetIVpc().GetManagerId(); len(manageId) > 0 {
+		if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil {
+			net.ProjectId = provider.ProjectId
+		}
+	}
 	err := manager.TableSpec().Insert(&net)
 	if err != nil {
 		log.Errorf("newFromCloudZone fail %s", err)
@@ -691,7 +700,7 @@ func parseNetworkInfo(userCred mcclient.TokenCredential, info jsonutils.JSONObje
 		} else if p == "[vip]" {
 			netConfig.Vip = true
 		} else {
-			netObj, err := NetworkManager.FetchByIdOrName(userCred.GetProjectId(), p)
+			netObj, err := NetworkManager.FetchByIdOrName(userCred, p)
 			if err != nil {
 				return nil, err
 			}
@@ -710,7 +719,7 @@ func (self *SNetwork) getFreeAddressCount() int {
 
 func isValidNetworkInfo(userCred mcclient.TokenCredential, netConfig *SNetworkConfig) error {
 	if len(netConfig.Network) > 0 {
-		netObj, err := NetworkManager.FetchByIdOrName(userCred.GetProjectId(), netConfig.Network)
+		netObj, err := NetworkManager.FetchByIdOrName(userCred, netConfig.Network)
 		if err != nil {
 			return httperrors.NewResourceNotFoundError("Network %s not found %s", err)
 		}
@@ -971,7 +980,7 @@ func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred
 
 	wireStr := jsonutils.GetAnyString(data, []string{"wire", "wire_id"})
 	if len(wireStr) > 0 {
-		wireObj, err := WireManager.FetchByIdOrName(userCred.GetProjectId(), wireStr)
+		wireObj, err := WireManager.FetchByIdOrName(userCred, wireStr)
 		if err != nil {
 			if err == sql.ErrNoRows {
 				return nil, httperrors.NewNotFoundError("wire %s not found", wireStr)
@@ -985,7 +994,7 @@ func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred
 		if len(zoneStr) > 0 {
 			vpcStr := jsonutils.GetAnyString(data, []string{"vpc", "vpc_id"})
 			if len(vpcStr) > 0 {
-				zoneObj, err := ZoneManager.FetchByIdOrName(userCred.GetProjectId(), zoneStr)
+				zoneObj, err := ZoneManager.FetchByIdOrName(userCred, zoneStr)
 				if err != nil {
 					if err == sql.ErrNoRows {
 						return nil, httperrors.NewNotFoundError("zone %s not found", zoneStr)
@@ -993,7 +1002,7 @@ func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred
 						return nil, httperrors.NewInternalServerError("query zone %s error %s", zoneStr, err)
 					}
 				}
-				vpcObj, err := VpcManager.FetchByIdOrName(userCred.GetProjectId(), vpcStr)
+				vpcObj, err := VpcManager.FetchByIdOrName(userCred, vpcStr)
 				if err != nil {
 					if err == sql.ErrNoRows {
 						return nil, httperrors.NewNotFoundError("vpc %s not found", vpcStr)
@@ -1277,7 +1286,7 @@ func (manager *SNetworkManager) ListItemFilter(ctx context.Context, q *sqlchemy.
 	}
 	zoneStr, _ := query.GetString("zone")
 	if len(zoneStr) > 0 {
-		zoneObj, err := ZoneManager.FetchByIdOrName(userCred.GetProjectId(), zoneStr)
+		zoneObj, err := ZoneManager.FetchByIdOrName(userCred, zoneStr)
 		if err != nil {
 			return nil, httperrors.NewNotFoundError("Zone %s not found", zoneStr)
 		}
@@ -1286,7 +1295,7 @@ func (manager *SNetworkManager) ListItemFilter(ctx context.Context, q *sqlchemy.
 	}
 	vpcStr, _ := query.GetString("vpc")
 	if len(vpcStr) > 0 {
-		vpcObj, err := VpcManager.FetchByIdOrName(userCred.GetProjectId(), vpcStr)
+		vpcObj, err := VpcManager.FetchByIdOrName(userCred, vpcStr)
 		if err != nil {
 			return nil, httperrors.NewNotFoundError("VPC %s not found", vpcStr)
 		}
@@ -1295,7 +1304,7 @@ func (manager *SNetworkManager) ListItemFilter(ctx context.Context, q *sqlchemy.
 	}
 	regionStr := jsonutils.GetAnyString(query, []string{"region_id", "region", "cloudregion_id", "cloudregion"})
 	if len(regionStr) > 0 {
-		region, err := CloudregionManager.FetchByIdOrName(userCred.GetProjectId(), regionStr)
+		region, err := CloudregionManager.FetchByIdOrName(userCred, regionStr)
 		if err != nil {
 			if err == sql.ErrNoRows {
 				return nil, httperrors.NewResourceNotFoundError("cloud region %s not found", regionStr)

pkg/compute/models/quotas.go

@@ -7,9 +7,9 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
 	"yunion.io/x/onecloud/pkg/compute/options"
-	"yunion.io/x/pkg/tristate"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/pkg/tristate"
 )
 
 var QuotaManager *quotas.SQuotaManager
@@ -278,7 +278,7 @@ func (self *SQuota) Exceed(request quotas.IQuota, quota quotas.IQuota) error {
 	if sreq.IsolatedDevice > 0 && self.IsolatedDevice > squota.IsolatedDevice {
 		return ErrOutOfIsolatedDevice
 	}
-	if self.Snapshot > squota.Snapshot {
+	if sreq.Snapshot > 0 && self.Snapshot > squota.Snapshot {
 		return ErrOutOfSnapshot
 	}
 	return nil

pkg/compute/models/reservedips.go

@@ -122,7 +122,7 @@ func (manager *SReservedipManager) ListItemFilter(ctx context.Context, q *sqlche
 	}
 	network, _ := query.GetString("network")
 	if len(network) > 0 {
-		netObj, _ := NetworkManager.FetchByIdOrName(userCred.GetProjectId(), network)
+		netObj, _ := NetworkManager.FetchByIdOrName(userCred, network)
 		if netObj == nil {
 			return nil, httperrors.NewResourceNotFoundError(fmt.Sprintf("network %s not found", network))
 		}

pkg/compute/models/schedtags.go

@@ -59,7 +59,7 @@ func (manager *SSchedtagManager) AllowCreateItem(ctx context.Context, userCred m
 func (manager *SSchedtagManager) ValidateSchedtags(userCred mcclient.TokenCredential, schedtags map[string]string) (map[string]string, error) {
 	ret := make(map[string]string)
 	for tag, act := range schedtags {
-		schedtagObj, err := manager.FetchByIdOrName("", tag)
+		schedtagObj, err := manager.FetchByIdOrName(nil, tag)
 		if err != nil {
 			if err == sql.ErrNoRows {
 				return nil, httperrors.NewResourceNotFoundError("Invalid schedtag %s", tag)

pkg/compute/models/secgrouprules.go

@@ -98,7 +98,7 @@ func (manager *SSecurityGroupRuleManager) ListItemFilter(ctx context.Context, q
 		return nil, err
 	}
 	if defsecgroup, _ := query.GetString("secgroup"); len(defsecgroup) > 0 {
-		if secgroup, _ := SecurityGroupManager.FetchByIdOrName(userCred.GetProjectId(), defsecgroup); secgroup != nil {
+		if secgroup, _ := SecurityGroupManager.FetchByIdOrName(userCred, defsecgroup); secgroup != nil {
 			sql = sql.Equals("secgroup_id", secgroup.GetId())
 		} else {
 			return nil, httperrors.NewNotFoundError(fmt.Sprintf("Security Group %s not found", defsecgroup))
@@ -130,7 +130,7 @@ func (manager *SSecurityGroupRuleManager) ValidateCreateData(
 	data *jsonutils.JSONDict,
 ) (*jsonutils.JSONDict, error) {
 	if defsecgroup, _ := data.GetString("secgroup"); len(defsecgroup) > 0 {
-		if secgroup, _ := SecurityGroupManager.FetchByIdOrName(userCred.GetProjectId(), defsecgroup); secgroup != nil {
+		if secgroup, _ := SecurityGroupManager.FetchByIdOrName(userCred, defsecgroup); secgroup != nil {
 			data.Set("secgroup_id", jsonutils.NewString(secgroup.GetId()))
 		} else {
 			return nil, httperrors.NewNotFoundError(fmt.Sprintf("Security Group %s not found", defsecgroup))

pkg/compute/models/snapshots.go

@@ -89,6 +89,32 @@ func (manager *SSnapshotManager) ListItemFilter(ctx context.Context, q *sqlchemy
 	return q, nil
 }
 
+func (self *SSnapshot) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
+	extra := self.SVirtualResourceBase.GetCustomizeColumns(ctx, userCred, query)
+	return self.getMoreDetails(extra)
+}
+
+func (self *SSnapshot) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
+	extra := self.SVirtualResourceBase.GetExtraDetails(ctx, userCred, query)
+	return self.getMoreDetails(extra)
+}
+
+func (self *SSnapshot) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict {
+	disk, _ := self.GetDisk()
+	if disk != nil {
+		extra.Add(jsonutils.NewString(disk.DiskType), "disk_type")
+		guests := disk.GetGuests()
+		if len(guests) == 1 {
+			extra.Add(jsonutils.NewString(guests[0].Id), "guest")
+			extra.Add(jsonutils.NewString(guests[0].Status), "guest_status")
+		}
+	}
+	if cloudprovider := self.GetCloudprovider(); cloudprovider != nil {
+		extra.Add(jsonutils.NewString(cloudprovider.Provider), "provider")
+	}
+	return extra
+}
+
 func (self *SSnapshot) AllowCreateItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
 	return false
 }
@@ -199,6 +225,11 @@ func (self *SSnapshotManager) CreateSnapshot(ctx context.Context, userCred mccli
 	snapshot := &SSnapshot{}
 	snapshot.SetModelManager(self)
 	snapshot.ProjectId = userCred.GetProjectId()
+	if manageId := disk.GetStorage().ManagerId; len(manageId) > 0 {
+		if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil {
+			snapshot.ProjectId = provider.ProjectId
+		}
+	}
 	snapshot.DiskId = disk.Id
 	snapshot.StorageId = disk.StorageId
 	snapshot.Size = disk.DiskSize
@@ -392,6 +423,11 @@ func (manager *SSnapshotManager) newFromCloudSnapshot(userCred mcclient.TokenCre
 	snapshot.CloudregionId = region.Id
 
 	snapshot.ProjectId = userCred.GetProjectId()
+	if len(snapshot.ManagerId) > 0 {
+		if provider := CloudproviderManager.FetchCloudproviderById(snapshot.ManagerId); provider != nil {
+			snapshot.ProjectId = provider.ProjectId
+		}
+	}
 	err := manager.TableSpec().Insert(&snapshot)
 	if err != nil {
 		log.Errorf("newFromCloudEip fail %s", err)

pkg/compute/models/storages.go

@@ -675,7 +675,7 @@ func (manager *SStorageManager) ListItemFilter(ctx context.Context, q *sqlchemy.
 
 	regionStr, _ := query.GetString("region")
 	if len(regionStr) > 0 {
-		regionObj, err := CloudregionManager.FetchByIdOrName(userCred.GetProjectId(), regionStr)
+		regionObj, err := CloudregionManager.FetchByIdOrName(userCred, regionStr)
 		if err != nil {
 			return nil, httperrors.NewNotFoundError("Region %s not found: %s", regionStr, err)
 		}

pkg/compute/models/wires.go

@@ -54,7 +54,7 @@ func (manager *SWireManager) ValidateCreateData(ctx context.Context, userCred mc
 	}
 
 	if len(vpcStr) > 0 {
-		vpcObj, err := VpcManager.FetchByIdOrName(userCred.GetProjectId(), vpcStr)
+		vpcObj, err := VpcManager.FetchByIdOrName(userCred, vpcStr)
 		if err != nil {
 			if err == sql.ErrNoRows {
 				return nil, httperrors.NewNotFoundError("Vpc %s not found", vpcStr)
@@ -534,7 +534,7 @@ func (manager *SWireManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu
 
 	vpcStr := jsonutils.GetAnyString(query, []string{"vpc_id", "vpc"})
 	if len(vpcStr) > 0 {
-		vpc, err := VpcManager.FetchByIdOrName(userCred.GetProjectId(), vpcStr)
+		vpc, err := VpcManager.FetchByIdOrName(userCred, vpcStr)
 		if err != nil {
 			if err == sql.ErrNoRows {
 				return nil, httperrors.NewNotFoundError("vpc %s not found", vpcStr)
@@ -547,7 +547,7 @@ func (manager *SWireManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu
 
 	regionStr := jsonutils.GetAnyString(query, []string{"region_id", "region", "cloudregion_id", "cloudregion"})
 	if len(regionStr) > 0 {
-		region, err := CloudregionManager.FetchByIdOrName(userCred.GetProjectId(), regionStr)
+		region, err := CloudregionManager.FetchByIdOrName(userCred, regionStr)
 		if err != nil {
 			if err == sql.ErrNoRows {
 				return nil, httperrors.NewNotFoundError("region %s not found", regionStr)
@@ -605,4 +605,4 @@ func (self *SWire) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict
 		}
 	}
 	return extra
-}
+}

pkg/compute/models/zones.go

@@ -523,7 +523,7 @@ func (manager *SZoneManager) ValidateCreateData(ctx context.Context, userCred mc
 	regionStr := jsonutils.GetAnyString(query, []string{"region", "region_id", "cloudregion", "cloudregion_id"})
 	var regionId string
 	if len(regionStr) > 0 {
-		regionObj, err := CloudregionManager.FetchByIdOrName("", regionStr)
+		regionObj, err := CloudregionManager.FetchByIdOrName(nil, regionStr)
 		if err != nil {
 			if err == sql.ErrNoRows {
 				return nil, httperrors.NewResourceNotFoundError("Region %s not found", regionStr)