Merge branch 'release/2.7.0' of ssh://git.yunion.io/~wanyaoqi/onecloud into release/2.7.0

cmd/climc/shell/baremetalagents.go

@@ -28,4 +28,25 @@ func init() {
 		printList(result, modules.Baremetalagents.GetColumns(s))
 		return nil
 	})
+
+	type BaremetalAgentOpsOperations struct {
+		ID string `help:"ID or name of agent"`
+	}
+	R(&BaremetalAgentOpsOperations{}, "baremetal-agent-enable", "Enable baremetal agent", func(s *mcclient.ClientSession, args *BaremetalAgentOpsOperations) error {
+		result, err := modules.Baremetalagents.PerformAction(s, args.ID, "enable", nil)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	R(&BaremetalAgentOpsOperations{}, "baremetal-agent-disable", "Disable baremetal agent", func(s *mcclient.ClientSession, args *BaremetalAgentOpsOperations) error {
+		result, err := modules.Baremetalagents.PerformAction(s, args.ID, "disable", nil)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
 }

cmd/climc/shell/cloudaccounts.go

@@ -33,7 +33,6 @@ func init() {
 	})
 
 	type CloudaccountCreateOptions struct {
-		NAME            string `help:"Name of cloud account"`
 		AccessKeyID     string `help:"Aiyun|HuaWei|Aws access_key_id"`
 		AccessKeySecret string `help:"Aiyun|HuaWei|Aws access_key_secret"`
 		AppID           string `help:"Qcloud appid"`
@@ -49,60 +48,85 @@ func init() {
 		ClientID        string `help:"Azure client_id"`
 		ClientSecret    string `help:"Azure clinet_secret"`
 		Environment     string `help:"Azure|Huawei|Aws environment" choices:"AzureGermanCloud|AzureChinaCloud|AzureUSGovernmentCloud|AzurePublicCloud|InternationalCloud|ChinaCloud|"`
-		PROVIDER        string `help:"Driver for cloud account" choices:"VMware|Aliyun|Azure|Qcloud|OpenStack|Huawei|Aws"`
-		Desc            string `help:"Description"`
-		Enabled         bool   `help:"Enabled the account automatically"`
 
-		Import            bool `help:"Import all sub account automatically"`
-		AutoSync          bool `help:"Enabled the account automatically"`
-		AutoCreateProject bool `help:"Enable the account with same name project"`
+		Enabled bool `help:"Enabled the account automatically"`
+
+		Import   bool `help:"Import all sub account automatically"`
+		AutoSync bool `help:"Enabled the account automatically"`
 	}
 	R(&CloudaccountCreateOptions{}, "cloud-account-create", "Create a cloud account", func(s *mcclient.ClientSession, args *CloudaccountCreateOptions) error {
-		params := jsonutils.NewDict()
-		params.Add(jsonutils.NewString(args.NAME), "name")
-		params.Add(jsonutils.NewString(args.PROVIDER), "provider")
-		data := jsonutils.Marshal(args)
+		return fmt.Errorf("obsolete, please try cloud-account-create-xxx, where xxx is vmware, aliyun, azure, qcloud, aws, openstack, huawei etc.")
+	})
 
-		requireParamsMap := map[string][]string{
-			"VMware":    {"username", "password", "host", "port"},
-			"Aliyun":    {"access_key_id", "access_key_secret"},
-			"Azure":     {"directory_id", "client_id", "client_secret", "environment"},
-			"Qcloud":    {"app_id", "secret_id", "secret_key"},
-			"OpenStack": {"project_name", "username", "password", "auth_url", "endpoint_type"},
-			"Huawei":    {"access_key_id", "access_key_secret", "environment"},
-			"Aws":       {"access_key_id", "access_key_secret", "environment"},
+	R(&options.SVMwareCloudAccountCreateOptions{}, "cloud-account-create-vmware", "Create a VMware cloud account", func(s *mcclient.ClientSession, args *options.SVMwareCloudAccountCreateOptions) error {
+		params := jsonutils.Marshal(args)
+		params.(*jsonutils.JSONDict).Add(jsonutils.NewString("VMware"), "provider")
+		result, err := modules.Cloudaccounts.Create(s, params)
+		if err != nil {
+			return err
 		}
+		printObject(result)
+		return nil
+	})
 
-		requireParams, ok := requireParamsMap[args.PROVIDER]
-		if !ok {
-			return fmt.Errorf("Unsupport provider %s", args.PROVIDER)
+	R(&options.SAliyunCloudAccountCreateOptions{}, "cloud-account-create-aliyun", "Create an Aliyun cloud account", func(s *mcclient.ClientSession, args *options.SAliyunCloudAccountCreateOptions) error {
+		params := jsonutils.Marshal(args)
+		params.(*jsonutils.JSONDict).Add(jsonutils.NewString("Aliyun"), "provider")
+		result, err := modules.Cloudaccounts.Create(s, params)
+		if err != nil {
+			return err
 		}
+		printObject(result)
+		return nil
+	})
 
-		for _, key := range requireParams {
-			v, _ := data.GetString(key)
-			if len(v) == 0 {
-				return fmt.Errorf("Missing %s", key)
-			}
-			params.Add(jsonutils.NewString(v), key)
+	R(&options.SAzureCloudAccountCreateOptions{}, "cloud-account-create-azure", "Create an Azure cloud account", func(s *mcclient.ClientSession, args *options.SAzureCloudAccountCreateOptions) error {
+		params := jsonutils.Marshal(args)
+		params.(*jsonutils.JSONDict).Add(jsonutils.NewString("Azure"), "provider")
+		result, err := modules.Cloudaccounts.Create(s, params)
+		if err != nil {
+			return err
 		}
+		printObject(result)
+		return nil
+	})
 
-		if args.Enabled {
-			params.Add(jsonutils.JSONTrue, "enabled")
+	R(&options.SQcloudCloudAccountCreateOptions{}, "cloud-account-create-qcloud", "Create a Qcloud cloud account", func(s *mcclient.ClientSession, args *options.SQcloudCloudAccountCreateOptions) error {
+		params := jsonutils.Marshal(args)
+		params.(*jsonutils.JSONDict).Add(jsonutils.NewString("Qcloud"), "provider")
+		result, err := modules.Cloudaccounts.Create(s, params)
+		if err != nil {
+			return err
 		}
+		printObject(result)
+		return nil
+	})
 
-		if args.Import {
-			params.Add(jsonutils.JSONTrue, "import")
-			if args.AutoSync {
-				params.Add(jsonutils.JSONTrue, "auto_sync")
-			}
-			if args.AutoCreateProject {
-				params.Add(jsonutils.JSONTrue, "auto_create_project")
-			}
+	R(&options.SAWSCloudAccountCreateOptions{}, "cloud-account-create-aws", "Create an AWS cloud account", func(s *mcclient.ClientSession, args *options.SAWSCloudAccountCreateOptions) error {
+		params := jsonutils.Marshal(args)
+		params.(*jsonutils.JSONDict).Add(jsonutils.NewString("Aws"), "provider")
+		result, err := modules.Cloudaccounts.Create(s, params)
+		if err != nil {
+			return err
 		}
+		printObject(result)
+		return nil
+	})
 
-		if len(args.Desc) > 0 {
-			params.Add(jsonutils.NewString(args.Desc), "description")
+	R(&options.SOpenStackCloudAccountCreateOptions{}, "cloud-account-create-openstack", "Create an OpenStack cloud account", func(s *mcclient.ClientSession, args *options.SOpenStackCloudAccountCreateOptions) error {
+		params := jsonutils.Marshal(args)
+		params.(*jsonutils.JSONDict).Add(jsonutils.NewString("OpenStack"), "provider")
+		result, err := modules.Cloudaccounts.Create(s, params)
+		if err != nil {
+			return err
 		}
+		printObject(result)
+		return nil
+	})
+
+	R(&options.SHuaweiCloudAccountCreateOptions{}, "cloud-account-create-huawei", "Create a Huawei cloud account", func(s *mcclient.ClientSession, args *options.SHuaweiCloudAccountCreateOptions) error {
+		params := jsonutils.Marshal(args)
+		params.(*jsonutils.JSONDict).Add(jsonutils.NewString("Huawei"), "provider")
 		result, err := modules.Cloudaccounts.Create(s, params)
 		if err != nil {
 			return err
@@ -233,6 +257,70 @@ func init() {
 		ClientSecret    string `help:"Azure clinet_secret"`
 	}
 	R(&CloudaccountUpdateCredentialOptions{}, "cloud-account-update-credential", "Update credential of a cloud account", func(s *mcclient.ClientSession, args *CloudaccountUpdateCredentialOptions) error {
+		return fmt.Errorf("obsolete command, please try cloud-account-update-credential-xxx, where xxx is vmware, aliyun, azure, qcloud, aws, openstack, huawei, etc.")
+	})
+
+	R(&options.SVMwareCloudAccountUpdateCredentialOptions{}, "cloud-account-update-credential-vmware", "Update credential of a VMware cloud account", func(s *mcclient.ClientSession, args *options.SVMwareCloudAccountUpdateCredentialOptions) error {
+		params := jsonutils.Marshal(args)
+		result, err := modules.Cloudaccounts.PerformAction(s, args.ID, "update-credential", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	R(&options.SAliyunCloudAccountUpdateCredentialOptions{}, "cloud-account-update-credential-aliyun", "Update credential of an Aliyun cloud account", func(s *mcclient.ClientSession, args *options.SAliyunCloudAccountUpdateCredentialOptions) error {
+		params := jsonutils.Marshal(args)
+		result, err := modules.Cloudaccounts.PerformAction(s, args.ID, "update-credential", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	R(&options.SAzureCloudAccountUpdateCredentialOptions{}, "cloud-account-update-credential-azure", "Update credential of an Azure cloud account", func(s *mcclient.ClientSession, args *options.SAzureCloudAccountUpdateCredentialOptions) error {
+		params := jsonutils.Marshal(args)
+		result, err := modules.Cloudaccounts.PerformAction(s, args.ID, "update-credential", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	R(&options.SQcloudCloudAccountUpdateCredentialOptions{}, "cloud-account-update-credential-qcloud", "Update credential of a Qcloud cloud account", func(s *mcclient.ClientSession, args *options.SQcloudCloudAccountUpdateCredentialOptions) error {
+		params := jsonutils.Marshal(args)
+		result, err := modules.Cloudaccounts.PerformAction(s, args.ID, "update-credential", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	R(&options.SAWSCloudAccountUpdateCredentialOptions{}, "cloud-account-update-credential-aws", "Update credential of an AWS cloud account", func(s *mcclient.ClientSession, args *options.SAWSCloudAccountUpdateCredentialOptions) error {
+		params := jsonutils.Marshal(args)
+		result, err := modules.Cloudaccounts.PerformAction(s, args.ID, "update-credential", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	R(&options.SOpenStackCloudAccountUpdateCredentialOptions{}, "cloud-account-update-credential-openstack", "Update credential of an OpenStack cloud account", func(s *mcclient.ClientSession, args *options.SOpenStackCloudAccountUpdateCredentialOptions) error {
+		params := jsonutils.Marshal(args)
+		result, err := modules.Cloudaccounts.PerformAction(s, args.ID, "update-credential", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	R(&options.SHuaweiCloudAccountUpdateCredentialOptions{}, "cloud-account-update-credential-huawei", "Update credential of an Huawei cloud account", func(s *mcclient.ClientSession, args *options.SHuaweiCloudAccountUpdateCredentialOptions) error {
 		params := jsonutils.Marshal(args)
 		result, err := modules.Cloudaccounts.PerformAction(s, args.ID, "update-credential", params)
 		if err != nil {

pkg/cloudcommon/db/taskman/tasks.go

@@ -2,6 +2,7 @@ package taskman
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"net/http"
 	"reflect"
@@ -10,19 +11,18 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
-	"yunion.io/x/onecloud/pkg/appctx"
-	"yunion.io/x/onecloud/pkg/mcclient"
-	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/pkg/util/reflectutils"
 	"yunion.io/x/pkg/util/stringutils"
+	"yunion.io/x/pkg/util/timeutils"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
-	"database/sql"
+	"yunion.io/x/onecloud/pkg/appctx"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
-	"yunion.io/x/pkg/util/timeutils"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 )
 
 const (
@@ -451,6 +451,14 @@ func (task *STask) ScheduleRun(data jsonutils.JSONObject) {
 	runTask(task.Id, data)
 }
 
+func (self *STask) HasParentTask() bool {
+	parentTaskId, _ := self.Params.GetString(PARENT_TASK_ID_KEY)
+	if len(parentTaskId) > 0 {
+		return true
+	}
+	return false
+}
+
 func (self *STask) GetParentTask() *STask {
 	parentTaskId, _ := self.Params.GetString(PARENT_TASK_ID_KEY)
 	if len(parentTaskId) > 0 {

pkg/compute/models/baremetalagents.go

@@ -9,6 +9,8 @@ import (
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/pkg/util/regutils"
+	"yunion.io/x/log"
+	"database/sql"
 )
 
 const (
@@ -17,9 +19,11 @@ const (
 	BAREMETAL_AGENT_OFFLINE  = "offline"
 )
 
+type TAgentType string
+
 const (
-	AgentTypeBaremetal = "baremetal"
-	AgentTypeEsxi      = "esxiagent"
+	AgentTypeBaremetal = TAgentType("baremetal")
+	AgentTypeEsxi      = TAgentType("esxiagent")
 	AgentTypeDefault   = AgentTypeBaremetal
 )
 
@@ -175,3 +179,24 @@ func (self *SBaremetalagent) GetExtraDetails(ctx context.Context, userCred mccli
 	}
 	return extra, nil
 }
+
+func (manager *SBaremetalagentManager) GetAgent(agentType TAgentType, zoneId string) *SBaremetalagent {
+	q := manager.Query().Equals("agent_type", agentType).Equals("zone_id", zoneId).Asc("created_at")
+	agents := make([]SBaremetalagent, 0)
+	err := db.FetchModelObjects(manager, q, &agents)
+	if err != nil {
+		if err != sql.ErrNoRows {
+			log.Errorf("GetAgent query fail %s", err)
+		}
+		return nil
+	}
+	if len(agents) == 0 {
+		return nil
+	}
+	for i := range agents {
+		if agents[i].Status == BAREMETAL_AGENT_ENABLED {
+			return &agents[i]
+		}
+	}
+	return &agents[0]
+}

pkg/compute/models/cachedimages.go

@@ -336,14 +336,17 @@ func (self *SCachedimage) ChooseSourceStoragecacheInRange(hostType string, exclu
 
 	scimgs := make([]SStoragecachedimage, 0)
 	q := storageCachedImage.Query().
-		Join(storage, sqlchemy.AND(sqlchemy.Equals(storage.Field("storagecache_id"), storageCachedImage.Field("storagecache_id")))).
-		Join(hostStorage, sqlchemy.AND(sqlchemy.Equals(hostStorage.Field("storage_id"), storage.Field("id")))).
-		Join(host, sqlchemy.AND(sqlchemy.Equals(hostStorage.Field("host_id"), host.Field("id")))).
+		Join(storage, sqlchemy.Equals(storage.Field("storagecache_id"), storageCachedImage.Field("storagecache_id"))).
+		Join(hostStorage, sqlchemy.Equals(hostStorage.Field("storage_id"), storage.Field("id"))).
+		Join(host, sqlchemy.Equals(hostStorage.Field("host_id"), host.Field("id"))).
 		Filter(sqlchemy.Equals(storageCachedImage.Field("cachedimage_id"), self.Id)).
 		Filter(sqlchemy.Equals(storageCachedImage.Field("status"), CACHED_IMAGE_STATUS_READY)).
 		Filter(sqlchemy.Equals(host.Field("status"), HOST_STATUS_RUNNING)).
 		Filter(sqlchemy.IsTrue(host.Field("enabled"))).
-		Filter(sqlchemy.Equals(host.Field("host_status"), HOST_ONLINE))
+		Filter(sqlchemy.Equals(host.Field("host_status"), HOST_ONLINE)).
+		Filter(sqlchemy.IsTrue(storage.Field("enabled"))).
+		Filter(sqlchemy.In(storage.Field("status"), []string{STORAGE_ENABLED, STORAGE_ONLINE})).
+		Filter(sqlchemy.Equals(storage.Field("storage_type"), STORAGE_LOCAL))
 
 	if len(excludes) > 0 {
 		q = q.Filter(sqlchemy.NotIn(host.Field("id"), excludes))

pkg/compute/models/cloudaccounts.go

@@ -132,6 +132,13 @@ func (self *SCloudaccount) ValidateDeleteCondition(ctx context.Context) error {
 	return self.SEnabledStatusStandaloneResourceBase.ValidateDeleteCondition(ctx)
 }
 
+func (self *SCloudaccount) PreDelete(ctx context.Context, userCred mcclient.TokenCredential) {
+	cloudproviders := self.GetCloudproviders()
+	for i := 0; i < len(cloudproviders); i++ {
+		cloudproviders[i].Delete(ctx, userCred)
+	}
+}
+
 func (self *SCloudaccount) PerformEnable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	_, err := self.SEnabledStatusStandaloneResourceBase.PerformEnable(ctx, userCred, query, data)
 	if err != nil {
@@ -226,16 +233,9 @@ func (manager *SCloudaccountManager) ValidateCreateData(ctx context.Context, use
 	return manager.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)
 }
 
-func (self *SCloudaccount) PreDelete(ctx context.Context, userCred mcclient.TokenCredential) {
-	cloudproviders := self.GetCloudproviders()
-	for i := 0; i < len(cloudproviders); i++ {
-		cloudproviders[i].Delete(ctx, userCred)
-	}
-}
-
 func (self *SCloudaccount) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
 	self.Enabled = true
-	self.EnableAutoSync = false
+	// self.EnableAutoSync = false
 	return self.SEnabledStatusStandaloneResourceBase.CustomizeCreate(ctx, userCred, ownerProjId, query, data)
 }
 
@@ -243,7 +243,9 @@ func (self *SCloudaccount) PostCreate(ctx context.Context, userCred mcclient.Tok
 	self.SEnabledStatusStandaloneResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data)
 	self.savePassword(self.Secret)
 
-	self.StartSyncCloudProviderInfoTask(ctx, userCred, nil, "")
+	if !self.EnableAutoSync {
+		self.StartSyncCloudProviderInfoTask(ctx, userCred, &SSyncRange{}, "")
+	}
 }
 
 func (self *SCloudaccount) savePassword(secret string) error {
@@ -1013,15 +1015,6 @@ func (manager *SCloudaccountManager) AutoSyncCloudaccountTask(ctx context.Contex
 	}
 
 	q := manager.Query()
-	/*
-		q = q.Filter(
-			sqlchemy.OR(
-				sqlchemy.NotEquals(q.Field("status"), CLOUD_PROVIDER_DISCONNECTED),
-				sqlchemy.LT(q.Field("error_count"), options.Options.MaxCloudAccountErrorCount),
-			),
-		)
-	*/
-	// q = q.Equals("sync_status", CLOUD_PROVIDER_SYNC_STATUS_IDLE)
 
 	accounts := make([]SCloudaccount, 0)
 	err := db.FetchModelObjects(manager, q, &accounts)
@@ -1051,7 +1044,7 @@ func (account *SCloudaccount) probeAccountStatus(ctx context.Context, userCred m
 		return nil, err
 	}
 	balance, err := manager.GetBalance()
-	if err != nil {
+	if err != nil && err != cloudprovider.ErrNotSupported {
 		log.Errorf("manager.GetBalance fail %s", err)
 		return nil, err
 	}
@@ -1129,7 +1122,7 @@ func (account *SCloudaccount) SubmitSyncAccountTask(ctx context.Context, userCre
 		if waitChan != nil {
 			waitChan <- err
 		} else {
-			if err == nil && autoSync && account.EnableAutoSync {
+			if err == nil && autoSync && account.Enabled && account.EnableAutoSync {
 				syncRange := SSyncRange{FullSync: true}
 				providers := account.GetEnabledCloudproviders()
 				for i := range providers {

pkg/compute/models/cloudproviderregions.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"context"
+	"database/sql"
 	"time"
 
 	"yunion.io/x/jsonutils"
@@ -143,7 +144,9 @@ func (manager *SCloudproviderregionManager) FetchByIds(providerId string, region
 	}
 	err = q.First(obj)
 	if err != nil {
-		log.Errorf("q.First fail %s", err)
+		if err != sql.ErrNoRows {
+			log.Errorf("q.First fail %s", err)
+		}
 		return nil
 	}
 	return obj.(*SCloudproviderregion)

pkg/compute/models/cloudsync.go

@@ -519,7 +519,7 @@ func syncVMEip(ctx context.Context, userCred mcclient.TokenCredential, provider
 	db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred)
 }
 
-func syncRegionSkusFromCloud(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localZone *SZone, remoteRegion cloudprovider.ICloudRegion, remoteZone cloudprovider.ICloudZone) {
+func syncZoneSkusFromCloud(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localZone *SZone, remoteRegion cloudprovider.ICloudRegion, remoteZone cloudprovider.ICloudZone) {
 	skus, err := remoteRegion.GetSkus(remoteZone.GetId())
 	if err != nil {
 		msg := fmt.Sprintf("GetSkus for zone %s failed %v", localZone.Name, err)
@@ -527,7 +527,7 @@ func syncRegionSkusFromCloud(ctx context.Context, userCred mcclient.TokenCredent
 		return
 	}
 
-	result := ServerSkuManager.SyncCloudSkusByRegion(ctx, userCred, provider, localZone, skus)
+	result := ServerSkuManager.SyncCloudSkusByZone(ctx, userCred, provider, localZone, skus)
 
 	syncResults.Add(ServerSkuManager, result)
 
@@ -763,7 +763,7 @@ func syncPublicCloudProviderInfo(
 		}
 
 		if driver.GetFactory().NeedSyncSkuFromCloud() {
-			syncRegionSkusFromCloud(ctx, userCred, syncResults, provider, &localZones[j], remoteRegion, remoteZones[j])
+			syncZoneSkusFromCloud(ctx, userCred, syncResults, provider, &localZones[j], remoteRegion, remoteZones[j])
 		}
 	}
 

pkg/compute/models/disks.go

@@ -726,10 +726,13 @@ func (self *SDisk) PerformResize(ctx context.Context, userCred mcclient.TokenCre
 	}
 
 	guests := self.GetGuests()
-	if len(guests) != 1 {
-		return nil, httperrors.NewBadRequestError("Cann't resize disk when attach to mutil guest")
+
+	var guest *SGuest
+	if len(guests) == 1 {
+		guest = &guests[0]
 	}
-	return nil, self.StartDiskResizeTask(ctx, userCred, int64(sizeMb), "", &pendingUsage, &guests[0])
+
+	return nil, self.StartDiskResizeTask(ctx, userCred, int64(sizeMb), "", &pendingUsage, guest)
 }
 
 func (self *SDisk) GetIStorage() (cloudprovider.ICloudStorage, error) {

pkg/compute/models/hosts.go

@@ -2689,26 +2689,6 @@ func (self *SHost) StartBaremetalUnmaintenanceTask(ctx context.Context, userCred
 	return nil
 }
 
-func (self *SHost) IsBaremetalAgentReady() bool {
-	url, err := auth.GetServiceURL("baremetal", options.Options.Region, self.GetZone().GetName(), "")
-	if err != nil {
-		log.Errorln("is baremetal agent ready: false")
-		return false
-	}
-	log.Infof("baremetal url:%s", url)
-	return true
-}
-
-func (self *SHost) BaremetalSyncRequest(ctx context.Context, method httputils.THttpMethod, url string, headers http.Header, body *jsonutils.JSONDict) (jsonutils.JSONObject, error) {
-	serviceUrl, err := auth.GetServiceURL("baremetal", options.Options.Region, self.GetZone().GetName(), "")
-	if err != nil {
-		return nil, err
-	}
-	url = serviceUrl + url
-	_, data, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, method, url, headers, body, false)
-	return data, err
-}
-
 func (self *SHost) StartSyncstatus(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error {
 	guest := self.GetBaremetalServer()
 	if guest != nil {
@@ -3659,20 +3639,39 @@ func (host *SHost) SyncHostExternalNics(ctx context.Context, userCred mcclient.T
 // 	return HostManager.GetEsxiAgentHost(host.Id)
 // }
 
+func (self *SHost) IsBaremetalAgentReady() bool {
+	return self.isAgentReady(AgentTypeBaremetal)
+}
+
+func (self *SHost) BaremetalSyncRequest(ctx context.Context, method httputils.THttpMethod, url string, headers http.Header, body *jsonutils.JSONDict) (jsonutils.JSONObject, error) {
+	return self.doAgentRequest(AgentTypeBaremetal, ctx, method, url, headers, body)
+}
+
 func (self *SHost) IsEsxiAgentReady() bool {
-	url, err := auth.GetServiceURL("esxiagent", options.Options.Region, self.GetZone().GetName(), "")
-	if err != nil {
-		log.Errorln("is esxi agent ready: false")
-		return false
-	}
-	log.Infof("esxi agent url:%s", url)
-	return true
+	return self.isAgentReady(AgentTypeEsxi)
 }
 
 func (self *SHost) EsxiRequest(ctx context.Context, method httputils.THttpMethod, url string, headers http.Header, body *jsonutils.JSONDict) (jsonutils.JSONObject, error) {
-	serviceUrl, err := auth.GetServiceURL("esxiagent", options.Options.Region, self.GetZone().GetName(), "")
-	if err != nil {
-		return nil, err
+	return self.doAgentRequest(AgentTypeEsxi, ctx, method, url, headers, body)
+}
+
+func (self *SHost) isAgentReady(agentType TAgentType) bool {
+	agent := BaremetalagentManager.GetAgent(agentType, self.ZoneId)
+	if agent == nil {
+		log.Errorf("%s ready: false", agentType)
+		return false
+	}
+	return true
+}
+
+func (self *SHost) doAgentRequest(agentType TAgentType, ctx context.Context, method httputils.THttpMethod, url string, headers http.Header, body *jsonutils.JSONDict) (jsonutils.JSONObject, error) {
+	agent := BaremetalagentManager.GetAgent(agentType, self.ZoneId)
+	if agent == nil {
+		return nil, fmt.Errorf("no valid %s", agentType)
+	}
+	serviceUrl := agent.ManagerUri
+	if url[0] != '/' && serviceUrl[len(serviceUrl)-1] != '/' {
+		serviceUrl += "/"
 	}
 	url = serviceUrl + url
 	_, data, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, method, url, headers, body, false)

pkg/compute/models/networks.go

@@ -674,7 +674,7 @@ func (manager *SNetworkManager) allNetworksQ(providers []string, rangeObj db.ISt
 	networks := manager.Query().SubQuery()
 	hostwires := HostwireManager.Query().SubQuery()
 	hosts := HostManager.Query().SubQuery()
-	q := networks.Query()
+	q := networks.Query(networks.Field("id"))
 	q = q.Join(hostwires, sqlchemy.Equals(hostwires.Field("wire_id"), networks.Field("wire_id")))
 	q = q.Join(hosts, sqlchemy.Equals(hosts.Field("id"), hostwires.Field("host_id")))
 	q = q.Filter(sqlchemy.IsTrue(hosts.Field("enabled")))
@@ -686,13 +686,12 @@ func (manager *SNetworkManager) allNetworksQ(providers []string, rangeObj db.ISt
 
 func (manager *SNetworkManager) totalPortCountQ(userCred mcclient.TokenCredential, providers []string, rangeObj db.IStandaloneModel) *sqlchemy.SQuery {
 	q := manager.allNetworksQ(providers, rangeObj)
-	networks := manager.Query().SubQuery()
 	if userCred != nil && !db.IsAdminAllowList(userCred, manager) {
 		q = q.Filter(sqlchemy.OR(
-			sqlchemy.Equals(networks.Field("tenant_id"), userCred.GetProjectId()),
-			sqlchemy.IsTrue(networks.Field("is_public"))))
+			sqlchemy.Equals(q.Field("tenant_id"), userCred.GetProjectId()),
+			sqlchemy.IsTrue(q.Field("is_public"))))
 	}
-	return q
+	return manager.Query().In("id", q.Distinct().SubQuery())
 }
 
 type NetworkPortStat struct {
@@ -700,7 +699,11 @@ type NetworkPortStat struct {
 	CountExt int
 }
 
-func (manager *SNetworkManager) TotalPortCount(userCred mcclient.TokenCredential, providers []string, rangeObj db.IStandaloneModel) NetworkPortStat {
+func (manager *SNetworkManager) TotalPortCount(
+	userCred mcclient.TokenCredential,
+	providers []string,
+	rangeObj db.IStandaloneModel,
+) NetworkPortStat {
 	nets := make([]SNetwork, 0)
 	err := manager.totalPortCountQ(userCred, providers, rangeObj).All(&nets)
 	if err != nil {

pkg/compute/models/skus.go

@@ -799,7 +799,7 @@ func (manager *SServerSkuManager) PendingDeleteInvalidSku() error {
 	return nil
 }
 
-func (manager *SServerSkuManager) SyncCloudSkusByRegion(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, zone *SZone, skus []cloudprovider.ICloudSku) compare.SyncResult {
+func (manager *SServerSkuManager) SyncCloudSkusByZone(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, zone *SZone, skus []cloudprovider.ICloudSku) compare.SyncResult {
 	lockman.LockClass(ctx, manager, manager.GetOwnerId(userCred))
 	defer lockman.ReleaseClass(ctx, manager, manager.GetOwnerId(userCred))
 

pkg/compute/tasks/disk_resize_task.go

@@ -47,7 +47,7 @@ func (self *DiskResizeTask) OnInit(ctx context.Context, obj db.IStandaloneModel,
 	}
 
 	disk.SetStatus(self.GetUserCred(), models.DISK_START_RESIZE, "")
-	if masterGuest == nil {
+	if masterGuest != nil {
 		masterGuest.SetStatus(self.GetUserCred(), models.VM_RESIZE_DISK, "")
 	}
 	self.StartResizeDisk(ctx, host, storage, disk, masterGuest)

pkg/compute/tasks/guest_sync_task.go

@@ -47,8 +47,12 @@ func (self *GuestSyncConfTask) OnSyncComplete(ctx context.Context, obj db.IStand
 }
 
 func (self *GuestSyncConfTask) OnDiskSyncComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
-	self.SetStage("on_sync_status_complete", nil)
-	guest.StartSyncstatus(ctx, self.GetUserCred(), self.GetTaskId())
+	if self.HasParentTask() {
+		self.OnSyncStatusComplete(ctx, guest, nil)
+	} else {
+		self.SetStage("on_sync_status_complete", nil)
+		guest.StartSyncstatus(ctx, self.GetUserCred(), self.GetTaskId())
+	}
 }
 
 func (self *GuestSyncConfTask) OnDiskSyncCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {

pkg/compute/usages/handler.go

@@ -509,7 +509,9 @@ func guestHypervisorsUsage(
 	if userCred != nil {
 		projectId = userCred.GetProjectId()
 	}
-	guest := models.GuestManager.TotalCount(projectId, rangeObj, status, hypervisors, true, pendingDelete, hostTypes, resourceTypes, providers)
+	// temporarily hide system resources
+	// XXX needs more work later
+	guest := models.GuestManager.TotalCount(projectId, rangeObj, status, hypervisors, false, pendingDelete, hostTypes, resourceTypes, providers)
 	count := make(map[string]interface{})
 	count[prefix] = guest.TotalGuestCount
 	count[fmt.Sprintf("%s.cpu", prefix)] = guest.TotalCpuCount

pkg/image/models/images.go

@@ -1028,6 +1028,24 @@ func (self *SImage) DoCheckStatus(ctx context.Context, userCred mcclient.TokenCr
 				}
 			}
 		}
+		img, err := qemuimg.NewQemuImage(self.getLocalLocation())
+		if err == nil {
+			format := string(img.Format)
+			virtualSizeMB := int32(img.SizeBytes / 1024 / 1024)
+			if (len(format) > 0 && self.DiskFormat != format) || (virtualSizeMB > 0 && self.MinDiskMB != virtualSizeMB) {
+				db.Update(self, func() error {
+					if len(format) > 0 {
+						self.DiskFormat = format
+					}
+					if virtualSizeMB > 0 {
+						self.MinDiskMB = virtualSizeMB
+					}
+					return nil
+				})
+			}
+		} else {
+			log.Warningf("fail to check image size of %s(%s)", self.Id, self.Name)
+		}
 	} else {
 		if self.Status != IMAGE_STATUS_QUEUED {
 			self.SetStatus(userCred, IMAGE_STATUS_QUEUED, "check inactive")

pkg/mcclient/options/cloudaccounts.go

@@ -0,0 +1,145 @@
+package options
+
+type SUserPasswordCredential struct {
+	Username string `help:"Username" positional:"true"`
+	Password string `help:"Password" positional:"true"`
+}
+
+type SVMwareCredentialWithEnvironment struct {
+	SUserPasswordCredential
+
+	Host string `help:"VMware VCenter/ESXi host" positional:"true"`
+	Port string `help:"VMware VCenter/ESXi host port" default:"443"`
+}
+
+type SAzureCredential struct {
+	ClientID     string `help:"Azure client_id" positional:"true"`
+	ClientSecret string `help:"Azure clinet_secret" positional:"true"`
+}
+
+type SAzureCredentialWithEnvironment struct {
+	DirectoryID string `help:"Azure directory_id" positional:"true"`
+
+	SAzureCredential
+
+	Environment string `help:"Cloud environment" choices:"AzureGermanCloud|AzureChinaCloud|AzurePublicCloud" default:"AzureChinaCloud"`
+}
+
+type SQcloudCredential struct {
+	AppID     string `help:"Qcloud appid" positional:"true"`
+	SecretID  string `help:"Qcloud secret_id" positional:"true"`
+	SecretKey string `help:"Qcloud secret_key" positional:"true"`
+}
+
+type SOpenStackCredential struct {
+	ProjectName string `help:"OpenStack project_name" positional:"true"`
+
+	SUserPasswordCredential
+
+	EndpointType string `help:"OpenStack endpoint type" choices:"internal|public|admin"`
+}
+
+type SOpenStackCredentialWithAuthURL struct {
+	SOpenStackCredential
+
+	AuthURL string `help:"OpenStack auth_url" positional:"true" json:"auth_url"`
+}
+
+type SAccessKeyCredential struct {
+	AccessKeyID     string `help:"Access_key_id" positional:"true"`
+	AccessKeySecret string `help:"Access_key_secret" positional:"true"`
+}
+
+type SAccessKeyCredentialWithEnvironment struct {
+	SAccessKeyCredential
+	Environment string `help:"Cloud environment" choices:"InternationalCloud|ChinaCloud" default:"ChinaCloud"`
+}
+
+/// create options
+
+type SCloudAccountCreateBaseOptions struct {
+	Name string `help:"Name of cloud account" positional:"true"`
+	// PROVIDER string `help:"Driver for cloud account" choices:"VMware|Aliyun|Azure|Qcloud|OpenStack|Huawei|Aws"`
+	Desc string `help:"Description" token:"desc" json:"description"`
+
+	AutoCreateProject bool `help:"Enable the account with same name project"`
+	EnableAutoSync    bool `help:"Enable automatically synchronize resources of this account"`
+
+	SyncIntervalSeconds int `help:"Interval to synchronize if auto sync is enable" metavar:"SECONDS"`
+}
+
+type SVMwareCloudAccountCreateOptions struct {
+	SCloudAccountCreateBaseOptions
+	SVMwareCredentialWithEnvironment
+}
+
+type SAliyunCloudAccountCreateOptions struct {
+	SCloudAccountCreateBaseOptions
+	SAccessKeyCredential
+}
+
+type SAzureCloudAccountCreateOptions struct {
+	SCloudAccountCreateBaseOptions
+	SAzureCredentialWithEnvironment
+}
+
+type SQcloudCloudAccountCreateOptions struct {
+	SCloudAccountCreateBaseOptions
+	SQcloudCredential
+}
+
+type SAWSCloudAccountCreateOptions struct {
+	SCloudAccountCreateBaseOptions
+	SAccessKeyCredentialWithEnvironment
+}
+
+type SOpenStackCloudAccountCreateOptions struct {
+	SCloudAccountCreateBaseOptions
+	SOpenStackCredentialWithAuthURL
+}
+
+type SHuaweiCloudAccountCreateOptions struct {
+	SCloudAccountCreateBaseOptions
+	SAccessKeyCredentialWithEnvironment
+}
+
+// update credential options
+
+type SCloudAccountUpdateCredentialBaseOptions struct {
+	ID string `help:"ID or Name of cloud account" json:"-"`
+}
+
+type SVMwareCloudAccountUpdateCredentialOptions struct {
+	SCloudAccountUpdateCredentialBaseOptions
+	SUserPasswordCredential
+}
+
+type SAliyunCloudAccountUpdateCredentialOptions struct {
+	SCloudAccountUpdateCredentialBaseOptions
+	SAccessKeyCredential
+}
+
+type SAzureCloudAccountUpdateCredentialOptions struct {
+	SCloudAccountUpdateCredentialBaseOptions
+	SAzureCredential
+}
+
+type SQcloudCloudAccountUpdateCredentialOptions struct {
+	SCloudAccountUpdateCredentialBaseOptions
+	SQcloudCredential
+}
+
+type SAWSCloudAccountUpdateCredentialOptions struct {
+	SCloudAccountUpdateCredentialBaseOptions
+	SAccessKeyCredential
+}
+
+type SOpenStackCloudAccountUpdateCredentialOptions struct {
+	SCloudAccountUpdateCredentialBaseOptions
+	SOpenStackCredential
+}
+
+type SHuaweiCloudAccountUpdateCredentialOptions struct {
+	SCloudAccountUpdateCredentialBaseOptions
+	SAccessKeyCredential
+}

pkg/util/aws/aws.go

@@ -202,7 +202,7 @@ type SAccountBalance struct {
 
 func (self *SAwsClient) QueryAccountBalance() (*SAccountBalance, error) {
 	// todo: aws 貌似没有余额？
-	panic("implement me")
+	return nil, cloudprovider.ErrNotSupported
 }
 
 func (self *SAwsClient) GetIProjects() ([]cloudprovider.ICloudProject, error) {

pkg/util/aws/provider/provider.go

@@ -122,11 +122,7 @@ func (self *SAwsProvider) GetIRegionById(id string) (cloudprovider.ICloudRegion,
 }
 
 func (self *SAwsProvider) GetBalance() (float64, error) {
-	balance, err := self.client.QueryAccountBalance()
-	if err != nil {
-		return 0.0, err
-	}
-	return balance.AvailableAmount, nil
+	return 0.0, cloudprovider.ErrNotSupported
 }
 
 func (self *SAwsProvider) GetIProjects() ([]cloudprovider.ICloudProject, error) {

pkg/util/azure/azure.go

@@ -766,7 +766,7 @@ type SAccountBalance struct {
 }
 
 func (self *SAzureClient) QueryAccountBalance() (*SAccountBalance, error) {
-	return nil, cloudprovider.ErrNotImplemented
+	return nil, cloudprovider.ErrNotSupported
 }
 
 func getResourceGroup(id string) string {

pkg/util/azure/provider/provider.go

@@ -128,11 +128,7 @@ func (self *SAzureProvider) GetIRegionById(id string) (cloudprovider.ICloudRegio
 }
 
 func (self *SAzureProvider) GetBalance() (float64, error) {
-	balance, err := self.client.QueryAccountBalance()
-	if err != nil {
-		return 0.0, err
-	}
-	return balance.AvailableAmount, nil
+	return 0.0, cloudprovider.ErrNotSupported
 }
 
 func (self *SAzureProvider) GetIProjects() ([]cloudprovider.ICloudProject, error) {

pkg/util/esxi/provider/provider.go

@@ -156,7 +156,7 @@ func (self *SESXiProvider) GetIRegionById(id string) (cloudprovider.ICloudRegion
 }
 
 func (self *SESXiProvider) GetBalance() (float64, error) {
-	return 0.0, nil
+	return 0.0, cloudprovider.ErrNotSupported
 }
 
 func (self *SESXiProvider) GetOnPremiseIRegion() (cloudprovider.ICloudRegion, error) {

pkg/util/netutils2/macutils.go

@@ -6,7 +6,6 @@ import (
 	"strconv"
 	"strings"
 
-	"yunion.io/x/log"
 	"yunion.io/x/pkg/util/netutils"
 )
 
@@ -36,14 +35,12 @@ func ParseMac(macStr string) (SMacAddr, error) {
 func (mac SMacAddr) Add(step int) SMacAddr {
 	mac2 := SMacAddr{}
 	leftOver := step
-	for i := 5; i >= 0; i -= 1 { // skip first 2 vendor bytes
+	for i := 5; i >= 0; i -= 1 {
 		newByte := int(mac[i]) + leftOver
 		res := 0
 		if newByte < 0 {
-			log.Debugf("%d %d", mac[i], newByte)
 			res = ((-newByte) / 0x100) + 1
 			newByte = newByte + res*0x100
-			log.Debugf("%d %d", newByte, res)
 		}
 		mac2[i] = byte(newByte % 0x100)
 		leftOver = newByte/0x100 - res

pkg/util/openstack/provider/provider.go

@@ -169,7 +169,7 @@ func (self *SOpenStackProvider) GetIRegionById(extId string) (cloudprovider.IClo
 }
 
 func (self *SOpenStackProvider) GetBalance() (float64, error) {
-	return 0.0, nil
+	return 0.0, cloudprovider.ErrNotSupported
 }
 
 func (self *SOpenStackProvider) GetIProjects() ([]cloudprovider.ICloudProject, error) {