Merge branch 'master' into hotfix/qx-external-project-change-project

pkg/appsrv/appsrv.go

@@ -21,6 +21,7 @@ import (
 	"math/rand"
 	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"strings"
 	"sync"
@@ -234,6 +235,11 @@ func (app *Application) handleCORS(w http.ResponseWriter, r *http.Request) bool
 
 func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, rid string) (*SHandlerInfo, *SAppParams) {
 	segs := SplitPath(r.URL.EscapedPath())
+	for i := range segs {
+		if p, err := url.PathUnescape(segs[i]); err == nil {
+			segs[i] = p
+		}
+	}
 	params := make(map[string]string)
 	w.Header().Set("Server", "Yunion AppServer/Go/2018.4")
 	w.Header().Set("X-Frame-Options", "SAMEORIGIN")

pkg/cloudcommon/db/scoperesource.go

@@ -0,0 +1,188 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package db
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
+)
+
+type SScopedResourceBaseManager struct{}
+
+type SScopedResourceBase struct {
+	DomainId  string `width:"64" charset:"ascii" nullable:"true" index:"true" list:"user"`
+	ProjectId string `name:"tenant_id" width:"128" charset:"ascii" nullable:"true" index:"true" list:"user"`
+}
+
+func (m *SScopedResourceBaseManager) ResourceScope() rbacutils.TRbacScope {
+	return rbacutils.ScopeProject
+}
+
+func (m *SScopedResourceBaseManager) FilterByOwner(q *sqlchemy.SQuery, userCred mcclient.IIdentityProvider, scope rbacutils.TRbacScope) *sqlchemy.SQuery {
+	if userCred == nil {
+		return q
+	}
+	switch scope {
+	case rbacutils.ScopeDomain:
+		q = q.Filter(sqlchemy.Equals(q.Field("domain_id"), userCred.GetDomainId()))
+	case rbacutils.ScopeProject:
+		q = q.Filter(sqlchemy.Equals(q.Field("tenant_id"), userCred.GetProjectId()))
+	}
+	return q
+}
+
+func getScopedResourceScope(domainId, projectId string) rbacutils.TRbacScope {
+	if domainId == "" && projectId == "" {
+		return rbacutils.ScopeSystem
+	}
+	if domainId != "" && projectId == "" {
+		return rbacutils.ScopeDomain
+	}
+	if domainId != "" && projectId != "" {
+		return rbacutils.ScopeProject
+	}
+	return rbacutils.ScopeNone
+}
+
+func (s *SScopedResourceBase) GetOwnerId() mcclient.IIdentityProvider {
+	return &SOwnerId{
+		DomainId:  s.DomainId,
+		ProjectId: s.ProjectId,
+	}
+}
+
+func (s *SScopedResourceBase) GetResourceScope() rbacutils.TRbacScope {
+	return getScopedResourceScope(s.DomainId, s.ProjectId)
+}
+
+func (s *SScopedResourceBase) GetDomainId() string {
+	return s.DomainId
+}
+
+func (s *SScopedResourceBase) GetProjectId() string {
+	return s.ProjectId
+}
+
+func (s *SScopedResourceBase) SetResourceScope(domainId, projectId string) error {
+	s.DomainId = domainId
+	s.ProjectId = projectId
+	return nil
+}
+
+func (s *SScopedResourceBase) GetMoreColumns(extra *jsonutils.JSONDict) *jsonutils.JSONDict {
+	if s.ProjectId != "" {
+		extra.Add(jsonutils.NewString(s.ProjectId), "project_id")
+	}
+	return extra
+}
+
+type IScopedResourceModel interface {
+	IModel
+
+	GetDomainId() string
+	GetProjectId() string
+	SetResourceScope(domainId, projectId string) error
+}
+
+func (m *SScopedResourceBaseManager) PerformSetScope(
+	ctx context.Context,
+	obj IScopedResourceModel,
+	userCred mcclient.TokenCredential,
+	data jsonutils.JSONObject,
+) (jsonutils.JSONObject, error) {
+	domainId := jsonutils.GetAnyString(data, []string{"domain_id", "domain", "project_domain_id", "project_domain"})
+	projectId := jsonutils.GetAnyString(data, []string{"project_id", "project"})
+	if projectId != "" {
+		project, err := TenantCacheManager.FetchTenantByIdOrName(ctx, projectId)
+		if err != nil {
+			return nil, err
+		}
+		projectId = project.GetId()
+		domainId = project.GetDomainId()
+	}
+	if domainId != "" {
+		domain, err := TenantCacheManager.FetchDomainByIdOrName(ctx, domainId)
+		if err != nil {
+			return nil, err
+		}
+		domainId = domain.GetId()
+	}
+	scopeToSet := getScopedResourceScope(domainId, projectId)
+	var err error
+	switch scopeToSet {
+	case rbacutils.ScopeSystem:
+		err = m.SetScopedResourceToSystem(obj, userCred)
+	case rbacutils.ScopeDomain:
+		err = m.SetScopedResourceToDomain(obj, userCred, domainId)
+	case rbacutils.ScopeProject:
+		err = m.SetScopedResourceToProject(obj, userCred, projectId)
+	}
+	return nil, err
+}
+
+func setScopedResourceIds(model IScopedResourceModel, userCred mcclient.TokenCredential, domainId, projectId string) error {
+	diff, err := Update(model, func() error {
+		model.SetResourceScope(domainId, projectId)
+		return nil
+	})
+	if err == nil {
+		OpsLog.LogEvent(model, ACT_UPDATE, diff, userCred)
+	}
+	return err
+}
+
+func (m *SScopedResourceBaseManager) SetScopedResourceToSystem(model IScopedResourceModel, userCred mcclient.TokenCredential) error {
+	if !IsAdminAllowPerform(userCred, model, "set-scope") {
+		return httperrors.NewForbiddenError("Not allow set scope to system")
+	}
+	if model.GetProjectId() == "" && model.GetDomainId() == "" {
+		return nil
+	}
+	return setScopedResourceIds(model, userCred, "", "")
+}
+
+func (m *SScopedResourceBaseManager) SetScopedResourceToDomain(model IScopedResourceModel, userCred mcclient.TokenCredential, domainId string) error {
+	if !IsDomainAllowPerform(userCred, model, "set-scope") {
+		return httperrors.NewForbiddenError("Not allow set scope to domain %s", domainId)
+	}
+	if model.GetDomainId() == domainId && model.GetProjectId() == "" {
+		return nil
+	}
+	domain, err := TenantCacheManager.FetchDomainById(context.TODO(), domainId)
+	if err != nil {
+		return err
+	}
+	return setScopedResourceIds(model, userCred, domain.GetId(), "")
+}
+
+func (m *SScopedResourceBaseManager) SetScopedResourceToProject(model IScopedResourceModel, userCred mcclient.TokenCredential, projectId string) error {
+	if !IsProjectAllowPerform(userCred, model, "set-scope") {
+		return httperrors.NewForbiddenError("Not allow set scope to project %s", projectId)
+	}
+	if model.GetProjectId() == projectId {
+		return nil
+	}
+	project, err := TenantCacheManager.FetchTenantById(context.TODO(), projectId)
+	if err != nil {
+		return err
+	}
+	return setScopedResourceIds(model, userCred, project.GetDomainId(), projectId)
+}

pkg/cloudcommon/policy/defaults.go

@@ -90,18 +90,6 @@ var (
 					Action:   PolicyActionGet,
 					Result:   rbacutils.Allow,
 				},
-				{
-					Service:  "compute",
-					Resource: "schedtags",
-					Action:   PolicyActionList,
-					Result:   rbacutils.Allow,
-				},
-				{
-					Service:  "compute",
-					Resource: "schedtags",
-					Action:   PolicyActionGet,
-					Result:   rbacutils.Allow,
-				},
 				{
 					Service:  "compute",
 					Resource: "cloudregions",

pkg/cloudcommon/policy/resources.go

@@ -25,7 +25,6 @@ var (
 		"vpcs",
 		"route_tables",
 		"cloudregions",
-		"schedtags",
 		"serverskus",
 		"cachedimages",
 		"dynamicschedtags",

pkg/compute/guestdrivers/base.go

@@ -313,12 +313,3 @@ func (self *SBaseGuestDriver) GetGuestSecgroupVpcid(guest *models.SGuest) (strin
 	}
 	return vpcId, nil
 }
-
-func (self *SBaseGuestDriver) OnGuestDeployTaskComplete(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {
-	if jsonutils.QueryBoolean(task.GetParams(), "restart", false) {
-		task.SetStage("OnDeployStartGuestComplete", nil)
-		return guest.StartGueststartTask(ctx, task.GetUserCred(), nil, task.GetTaskId())
-	}
-	task.SetStage("OnDeployGuestSyncstatusComplete", nil)
-	return guest.StartSyncstatus(ctx, task.GetUserCred(), task.GetTaskId())
-}

pkg/compute/guestdrivers/container.go

@@ -143,12 +143,6 @@ func (self *SContainerDriver) RequestUndeployGuestOnHost(ctx context.Context, gu
 	return err
 }
 
-func (self *SContainerDriver) OnGuestDeployTaskComplete(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {
-	guest.SetStatus(task.GetUserCred(), api.VM_RUNNING, "on deploy complete")
-	task.SetStageComplete(ctx, nil)
-	return nil
-}
-
 func (self *SContainerDriver) GetJsonDescAtHost(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, host *models.SHost) jsonutils.JSONObject {
 	return guest.GetJsonDescAtHypervisor(ctx, host)
 }

pkg/compute/guestdrivers/managedvirtual.go

@@ -722,7 +722,8 @@ func (self *SManagedVirtualizedGuestDriver) RequestChangeVmConfig(ctx context.Co
 						return true
 					}
 				} else {
-					if iVM.GetVcpuCount() == int(vcpuCount) && iVM.GetVmemSizeMB() == int(vmemSize) {
+					// aws 目前取不到内存。返回值永远为0
+					if iVM.GetVcpuCount() == int(vcpuCount) && (iVM.GetVmemSizeMB() == int(vmemSize) || iVM.GetVmemSizeMB() == 0) {
 						return true
 					}
 				}

pkg/compute/models/guest_actions.go

@@ -1726,7 +1726,7 @@ func (self *SGuest) PerformDetachnetwork(ctx context.Context, userCred mcclient.
 
 	netStr, _ := data.GetString("net_id")
 	if len(netStr) > 0 {
-		netObj, err := NetworkManager.FetchById(netStr)
+		netObj, err := NetworkManager.FetchByIdOrName(userCred, netStr)
 		if err != nil {
 			if err == sql.ErrNoRows {
 				return nil, httperrors.NewResourceNotFoundError2(NetworkManager.Keyword(), netStr)
@@ -3474,3 +3474,47 @@ func (guest *SGuest) StartGuestDiskResizeTask(ctx context.Context, userCred mccl
 	task.ScheduleRun(nil)
 	return nil
 }
+
+func (self *SGuest) AllowPerformIoThrottle(ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject) bool {
+	return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "io-throttle")
+}
+
+func (self *SGuest) PerformIoThrottle(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	if self.Hypervisor != api.HYPERVISOR_KVM {
+		return nil, httperrors.NewBadRequestError("Hypervisor %s can't do io throttle", self.Hypervisor)
+	}
+	if self.Status != api.VM_RUNNING {
+		return nil, httperrors.NewServerStatusError("Cannot do io throttle in status %s", self.Status)
+	}
+	bpsMb, err := data.Int("bps")
+	if err != nil {
+		return nil, httperrors.NewMissingParameterError("bps")
+	}
+	if bpsMb < 0 {
+		return nil, httperrors.NewInputParameterError("bps must > 0")
+	}
+	iops, err := data.Int("iops")
+	if err != nil {
+		return nil, httperrors.NewMissingParameterError("iops")
+	}
+	if iops < 0 {
+		return nil, httperrors.NewInputParameterError("iops must > 0")
+	}
+	return nil, self.StartBlockIoThrottleTask(ctx, userCred, bpsMb, iops)
+}
+
+func (self *SGuest) StartBlockIoThrottleTask(ctx context.Context, userCred mcclient.TokenCredential, bpsMb, iops int64) error {
+	params := jsonutils.NewDict()
+	params.Set("bps", jsonutils.NewInt(bpsMb))
+	params.Set("iops", jsonutils.NewInt(iops))
+	task, err := taskman.TaskManager.NewTask(ctx, "GuestBlockIoThrottleTask", self, userCred, params, "", "", nil)
+	if err != nil {
+		log.Errorf("%s", err)
+		return err
+	}
+	task.ScheduleRun(nil)
+	return nil
+}

pkg/compute/models/guestdrivers.go

@@ -96,8 +96,6 @@ type IGuestDriver interface {
 
 	OnGuestDeployTaskDataReceived(ctx context.Context, guest *SGuest, task taskman.ITask, data jsonutils.JSONObject) error
 
-	OnGuestDeployTaskComplete(ctx context.Context, guest *SGuest, task taskman.ITask) error
-
 	StartGuestSyncstatusTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error
 
 	RequestSyncConfigOnHost(ctx context.Context, guest *SGuest, host *SHost, task taskman.ITask) error

pkg/compute/models/guestnetworks.go

@@ -224,7 +224,7 @@ func (self *SGuestnetwork) getVirtualRand(width int, randomized bool) string {
 
 func (self *SGuestnetwork) generateIfname(network *SNetwork, virtual bool, randomized bool) string {
 	pattern := regexp.MustCompile(`\W+`)
-	nName := pattern.ReplaceAllString(network.Name, "")
+	nName := pattern.ReplaceAllString(network.IfnameHint, "")
 	if len(nName) > MAX_IFNAME_SIZE-4 {
 		nName = nName[:(MAX_IFNAME_SIZE - 4)]
 	}

pkg/compute/models/networks.go

@@ -44,6 +44,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/util/logclient"
+	"yunion.io/x/onecloud/pkg/util/rand"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
@@ -76,8 +77,6 @@ func init() {
 			"networks",
 		),
 	}
-	NetworkManager.NameLength = 9
-	NetworkManager.NameRequireAscii = true
 	NetworkManager.SetVirtualObject(NetworkManager)
 }
 
@@ -85,6 +84,8 @@ type SNetwork struct {
 	db.SSharableVirtualResourceBase
 	db.SExternalizedResourceBase
 
+	IfnameHint string `width:"9" charset:"ascii" nullable:"false" list:"user" create:"optional"`
+
 	GuestIpStart string `width:"16" charset:"ascii" nullable:"false" list:"user" update:"user" create:"required"` // Column(VARCHAR(16, charset='ascii'), nullable=False)
 	GuestIpEnd   string `width:"16" charset:"ascii" nullable:"false" list:"user" update:"user" create:"required"` // Column(VARCHAR(16, charset='ascii'), nullable=False)
 	GuestIpMask  int8   `nullable:"false" list:"user" update:"user" create:"required"`                            // Column(TINYINT, nullable=False)
@@ -1107,6 +1108,39 @@ func isValidMaskLen(maskLen int64) bool {
 	}
 }
 
+func (manager *SNetworkManager) newIfnameHint(hint string) (string, error) {
+	r := ""
+	// sanitize hint
+	for _, c := range hint {
+		if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-' || c == '_' {
+			r += string(c)
+		}
+	}
+	newHint := func() (string, error) {
+		for i := 0; i < 3; i++ {
+			r := rand.String(7)
+			cnt, err := manager.Query().Equals("ifname_hint", r).CountWithError()
+			if err == nil && cnt == 0 {
+				return r, nil
+			}
+		}
+		return "", fmt.Errorf("failed finding ifname hint after 3 tries")
+	}
+	if len(r) < 3 {
+		return newHint()
+	}
+	if len(r) > 9 {
+		r = r[:9]
+	}
+	if cnt, err := manager.Query().Equals("ifname_hint", r).CountWithError(); err != nil {
+		return "", err
+	} else if cnt > 0 {
+		r, err := newHint()
+		return r, err
+	}
+	return r, nil
+}
+
 func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	prefixStr, _ := data.GetString("guest_ip_prefix")
 	var maskLen64 int64
@@ -1146,6 +1180,18 @@ func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred
 	data.Add(jsonutils.NewString(startIp.String()), "guest_ip_start")
 	data.Add(jsonutils.NewString(endIp.String()), "guest_ip_end")
 
+	{
+		hint, _ := data.GetString("ifname_hint")
+		if hint == "" {
+			hint, _ = data.GetString("name")
+		}
+		hint, err = manager.newIfnameHint(hint)
+		if err != nil {
+			return nil, httperrors.NewBadRequestError("cannot derive valid ifname hint: %v", err)
+		}
+		data.Set("ifname_hint", jsonutils.NewString(hint))
+	}
+
 	for _, key := range []string{"guest_gateway", "guest_dns", "guest_dhcp"} {
 		ipStr, _ := data.GetString(key)
 		if len(ipStr) > 0 && !regutils.MatchIPAddr(ipStr) {
@@ -1703,9 +1749,26 @@ func (manager *SNetworkManager) InitializeData() error {
 		return err
 	}
 	for _, n := range networks {
-		if len(n.ExternalId) == 0 && len(n.WireId) > 0 && n.Status == api.NETWORK_STATUS_INIT {
+		if n.ExternalId != "" {
+			var statusNew string
+			if n.WireId != "" && n.Status == api.NETWORK_STATUS_INIT {
+				statusNew = api.NETWORK_STATUS_AVAILABLE
+			}
 			db.Update(&n, func() error {
-				n.Status = api.NETWORK_STATUS_AVAILABLE
+				if statusNew != "" {
+					n.Status = statusNew
+				}
+				return nil
+			})
+		} else {
+			var ifnameHintNew string
+			if n.IfnameHint == "" {
+				ifnameHintNew = n.Name
+			}
+			db.Update(&n, func() error {
+				if ifnameHintNew != "" {
+					n.IfnameHint = ifnameHintNew
+				}
 				return nil
 			})
 		}

pkg/compute/models/schedtags.go

@@ -63,6 +63,7 @@ type IModelWithSchedtag interface {
 
 type SSchedtagManager struct {
 	db.SStandaloneResourceBaseManager
+	db.SScopedResourceBaseManager
 
 	jointsManager map[string]ISchedtagJointManager
 }
@@ -121,6 +122,7 @@ func (manager *SSchedtagManager) GetResourceTypes() []string {
 
 type SSchedtag struct {
 	db.SStandaloneResourceBase
+	db.SScopedResourceBase
 
 	DefaultStrategy string `width:"16" charset:"ascii" nullable:"true" default:"" list:"user" update:"admin" create:"admin_optional"` // Column(VARCHAR(16, charset='ascii'), nullable=True, default='')
 	ResourceType    string `width:"16" charset:"ascii" nullable:"true" list:"user" create:"required"`                                 // Column(VARCHAR(16, charset='ascii'), nullable=True, default='')
@@ -301,6 +303,7 @@ func (self *SSchedtag) getDynamicSchedtagCount() (int, error) {
 }
 
 func (self *SSchedtag) getMoreColumns(extra *jsonutils.JSONDict) *jsonutils.JSONDict {
+	extra = self.SScopedResourceBase.GetMoreColumns(extra)
 	cnt, _ := self.GetObjectCount()
 	extra.Add(jsonutils.NewInt(int64(cnt)), fmt.Sprintf("%s_count", self.GetJointManager().GetMasterManager().Keyword()))
 	cnt, _ = self.getDynamicSchedtagCount()
@@ -458,3 +461,11 @@ func GetSchedtagsDetailsToResource(obj IModelWithSchedtag, ctx context.Context,
 	}
 	return extra
 }
+
+func (s *SSchedtag) AllowPerformSetScope(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return true
+}
+
+func (s *SSchedtag) PerformSetScope(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	return SchedtagManager.PerformSetScope(ctx, s, userCred, data)
+}

pkg/compute/models/snapshotpolicy.go

@@ -20,6 +20,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/tristate"
 	"yunion.io/x/pkg/util/compare"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
@@ -51,8 +52,8 @@ type SSnapshotPolicy struct {
 	// 0~6, 0 is Monday
 	RepeatWeekdays uint8 `charset:"utf8" create:"required"`
 	// 0~23
-	TimePoints  uint32 `charset:"utf8" create:"required"`
-	IsActivated bool   `list:"user" get:"user" create:"optional" default:"true"`
+	TimePoints  uint32            `charset:"utf8" create:"required"`
+	IsActivated tristate.TriState `list:"user" get:"user" create:"optional" default:"true"`
 }
 
 var SnapshotPolicyManager *SSnapshotPolicyManager

pkg/compute/tasks/guest_block_io_throttle_task.go

@@ -0,0 +1,40 @@
+package tasks
+
+import (
+	"context"
+	"fmt"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/compute/models"
+)
+
+type GuestBlockIoThrottleTask struct {
+	SGuestBaseTask
+}
+
+func init() {
+	taskman.RegisterTask(GuestBlockIoThrottleTask{})
+}
+
+func (self *GuestBlockIoThrottleTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	guest := obj.(*models.SGuest)
+	url := fmt.Sprintf("/servers/%s/io-throttle", guest.Id)
+	headers := self.GetTaskRequestHeader()
+	host := guest.GetHost()
+	self.SetStage("OnIoThrottle", nil)
+	_, err := host.Request(ctx, self.UserCred, "POST", url, headers, self.Params)
+	if err != nil {
+		self.SetStageFailed(ctx, err.Error())
+	}
+}
+
+func (self *GuestBlockIoThrottleTask) OnIoThrottle(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
+	guest.SetMetadata(ctx, "io-throttle", self.Params.String(), self.UserCred)
+	self.SetStageComplete(ctx, nil)
+}
+
+func (self *GuestBlockIoThrottleTask) OnIoThrottleFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
+	self.SetStageFailed(ctx, data.String())
+}

pkg/compute/tasks/guest_change_config_task.go

@@ -239,7 +239,7 @@ func (self *GuestChangeConfigTask) OnGuestChangeCpuMemSpecCompleteFailed(ctx con
 func (self *GuestChangeConfigTask) OnGuestChangeCpuMemSpecFinish(ctx context.Context, guest *models.SGuest) {
 	models.HostManager.ClearSchedDescCache(guest.HostId)
 	self.SetStage("OnSyncConfigComplete", nil)
-	err := guest.StartSyncTask(ctx, self.UserCred, false, self.GetTaskId())
+	err := guest.StartSyncTaskWithoutSyncstatus(ctx, self.UserCred, false, self.GetTaskId())
 	if err != nil {
 		self.markStageFailed(ctx, guest, fmt.Sprintf("StartSyncstatus fail %s", err))
 		return

pkg/compute/tasks/guest_deploy_task.go

@@ -108,7 +108,6 @@ func (self *GuestDeployTask) OnDeployGuestComplete(ctx context.Context, obj db.I
 	log.Infof("on_guest_deploy_task_data_received %s", data)
 	guest := obj.(*models.SGuest)
 	guest.GetDriver().OnGuestDeployTaskDataReceived(ctx, guest, self, data)
-	guest.GetDriver().OnGuestDeployTaskComplete(ctx, guest, self)
 	action, _ := self.Params.GetString("deploy_action")
 	keypair, _ := self.Params.GetString("keypair")
 	reset_password := jsonutils.QueryBoolean(self.Params, "reset_password", false)
@@ -133,6 +132,19 @@ func (self *GuestDeployTask) OnDeployGuestComplete(ctx context.Context, obj db.I
 		// 如果 deploy 有其他事件，统一记在这里。
 		logclient.AddActionLogWithStartable(self, guest, "misc部署", "", self.UserCred, true)
 	}
+
+	if self.HasParentTask() {
+		self.SetStageComplete(ctx, nil)
+		return
+	}
+
+	if jsonutils.QueryBoolean(self.GetParams(), "restart", false) {
+		self.SetStage("OnDeployStartGuestComplete", nil)
+		guest.StartGueststartTask(ctx, self.GetUserCred(), nil, self.GetTaskId())
+	} else {
+		self.SetStage("OnDeployGuestSyncstatusComplete", nil)
+		guest.StartSyncstatus(ctx, self.GetUserCred(), self.GetTaskId())
+	}
 }
 
 func (self *GuestDeployTask) OnDeployGuestCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {

pkg/hostman/guestman/guesthandlers/guesthandler.go

@@ -40,20 +40,19 @@ type actionFunc func(context.Context, string, jsonutils.JSONObject) (interface{}
 var (
 	keyWords    = []string{"servers"}
 	actionFuncs = map[string]actionFunc{
-		"create":  guestCreate,
-		"deploy":  guestDeploy,
-		"start":   guestStart,
-		"stop":    guestStop,
-		"monitor": guestMonitor,
-		"sync":    guestSync,
-		"suspend": guestSuspend,
+		"create":      guestCreate,
+		"deploy":      guestDeploy,
+		"start":       guestStart,
+		"stop":        guestStop,
+		"monitor":     guestMonitor,
+		"sync":        guestSync,
+		"suspend":     guestSuspend,
+		"io-throttle": guestIoThrottle,
 
 		"snapshot":             guestSnapshot,
 		"delete-snapshot":      guestDeleteSnapshot,
 		"reload-disk-snapshot": guestReloadDiskSnapshot,
 		// "remove-statefile":     guestRemoveStatefile,
-		// "io-throttle":          guestIoThrottle,
-
 		"src-prepare-migrate":  guestSrcPrepareMigrate,
 		"dest-prepare-migrate": guestDestPrepareMigrate,
 		"live-migrate":         guestLiveMigrate,
@@ -244,6 +243,26 @@ func guestSuspend(ctx context.Context, sid string, body jsonutils.JSONObject) (i
 	return nil, nil
 }
 
+func guestIoThrottle(ctx context.Context, sid string, body jsonutils.JSONObject) (interface{}, error) {
+	guest, ok := guestman.GetGuestManager().Servers[sid]
+	if !ok {
+		return nil, httperrors.NewNotFoundError("Guest %s not found", sid)
+	}
+	if !guest.IsRunning() {
+		return nil, httperrors.NewInvalidStatusError("Not running")
+	}
+	bps, err := body.Int("bps")
+	if err != nil {
+		return nil, httperrors.NewMissingParameterError("bps")
+	}
+	iops, err := body.Int("iops")
+	if err != nil {
+		return nil, httperrors.NewMissingParameterError("iops")
+	}
+	hostutils.DelayTaskWithoutReqctx(ctx, guestman.GetGuestManager().GuestIoThrottle, &guestman.SGuestIoThrottle{sid, bps, iops})
+	return nil, nil
+}
+
 func guestSrcPrepareMigrate(ctx context.Context, sid string, body jsonutils.JSONObject) (interface{}, error) {
 	if !guestman.GetGuestManager().IsGuestExist(sid) {
 		return nil, httperrors.NewNotFoundError("Guest %s not found", sid)

pkg/hostman/guestman/guesthelper.go

@@ -105,3 +105,9 @@ type SGuestCreateFromLibvirt struct {
 	GuestDesc   *jsonutils.JSONDict
 	DisksPath   *jsonutils.JSONDict
 }
+
+type SGuestIoThrottle struct {
+	Sid  string
+	BPS  int64
+	IOPS int64
+}

pkg/hostman/guestman/guestman.go

@@ -454,6 +454,18 @@ func (m *SGuestManager) GuestSuspend(ctx context.Context, params interface{}) (j
 	return nil, nil
 }
 
+func (m *SGuestManager) GuestIoThrottle(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) {
+	guestIoThrottle, ok := params.(*SGuestIoThrottle)
+	if !ok {
+		return nil, hostutils.ParamsError
+	}
+	guest := m.Servers[guestIoThrottle.Sid]
+	if guest.IsRunning() {
+		return nil, guest.BlockIoThrottle(ctx, guestIoThrottle.BPS, guestIoThrottle.IOPS)
+	}
+	return nil, httperrors.NewInvalidStatusError("Guest not running")
+}
+
 func (m *SGuestManager) SrcPrepareMigrate(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) {
 	migParams, ok := params.(*SSrcPrepareMigrate)
 	if !ok {

pkg/hostman/guestman/guesttasks.go

@@ -1155,3 +1155,54 @@ func (task *SGuestHotplugCpuMemTask) onFail(reason string) {
 func (task *SGuestHotplugCpuMemTask) onSucc() {
 	hostutils.TaskComplete(task.ctx, nil)
 }
+
+type SGuestBlockIoThrottleTask struct {
+	*SKVMGuestInstance
+
+	ctx  context.Context
+	bps  int64
+	iops int64
+}
+
+func (task *SGuestBlockIoThrottleTask) Start() error {
+	task.findBlockDevices()
+	return nil
+}
+
+func (task *SGuestBlockIoThrottleTask) findBlockDevices() {
+	task.Monitor.GetBlocks(task.onBlockDriversSucc)
+}
+
+func (task *SGuestBlockIoThrottleTask) onBlockDriversSucc(res *jsonutils.JSONArray) {
+	drivers := make([]string, 0)
+	for i := 0; i < res.Length(); i++ {
+		device, err := res.GetAt(i)
+		if err == nil {
+			driver, err := device.GetString("device")
+			if err == nil {
+				if strings.HasPrefix(driver, "drive_") {
+					drivers = append(drivers, driver)
+				}
+			}
+		}
+	}
+	log.Infof("Drivers %s do io throttle bps %d iops %d", drivers, task.bps, task.iops)
+	task.doIoThrottle(drivers)
+}
+
+func (task *SGuestBlockIoThrottleTask) doIoThrottle(drivers []string) {
+	if len(drivers) == 0 {
+		hostutils.TaskComplete(task.ctx, nil)
+	} else {
+		driver := drivers[0]
+		drivers = drivers[1:]
+		_cb := func(res string) {
+			if len(res) > 0 {
+				hostutils.TaskFailed(task.ctx, res)
+			} else {
+				task.doIoThrottle(drivers)
+			}
+		}
+		task.Monitor.BlockIoThrottle(driver, task.bps, task.iops, _cb)
+	}
+}

pkg/hostman/guestman/qemu-kvm.go

@@ -1422,3 +1422,8 @@ func (s *SKVMGuestInstance) onlineResizeDisk(ctx context.Context, diskId string,
 	task := NewGuestOnlineResizeDiskTask(ctx, s, diskId, sizeMB)
 	task.Start()
 }
+
+func (s *SKVMGuestInstance) BlockIoThrottle(ctx context.Context, bps, iops int64) error {
+	task := SGuestBlockIoThrottleTask{s, ctx, bps, iops}
+	return task.Start()
+}

pkg/hostman/monitor/hmp.go

@@ -453,3 +453,8 @@ func (m *HmpMonitor) ObjectAdd(objectType string, params map[string]string, call
 	cmd := fmt.Sprintf("object_add %s,%s", objectType, strings.Join(paramsKvs, ","))
 	m.Query(cmd, callback)
 }
+
+func (m *HmpMonitor) BlockIoThrottle(driveName string, bps, iops int64, callback StringCallback) {
+	cmd := fmt.Sprintf("block_set_io_throttle %s %d 0 0 %d 0 0", driveName, bps, iops)
+	m.Query(cmd, callback)
+}

pkg/hostman/monitor/monitor.go

@@ -71,6 +71,7 @@ type Monitor interface {
 	StartNbdServer(port int, exportAllDevice, writable bool, callback StringCallback)
 
 	ResizeDisk(driveName string, sizeMB int64, callback StringCallback)
+	BlockIoThrottle(driveName string, bps, iops int64, callback StringCallback)
 }
 
 type MonitorErrorFunc func(error)

pkg/hostman/monitor/qmp.go

@@ -807,3 +807,8 @@ func (m *QmpMonitor) GeMemtSlotIndex(callback func(index int)) {
 	}
 	m.HumanMonitorCommand("info memory-devices", cb)
 }
+
+func (m *QmpMonitor) BlockIoThrottle(driveName string, bps, iops int64, callback StringCallback) {
+	cmd := fmt.Sprintf("block_set_io_throttle %s %d 0 0 %d 0 0", driveName, bps, iops)
+	m.HumanMonitorCommand(cmd, callback)
+}

pkg/mcclient/modules/mod_schedtags.go

@@ -69,7 +69,7 @@ func (this *SchedtagManager) DoBatchSchedtagHostAddRemove(s *mcclient.ClientSess
 
 func init() {
 	Schedtags = SchedtagManager{NewComputeManager("schedtag", "schedtags",
-		[]string{"ID", "Name", "Default_strategy", "Resource_type"},
+		[]string{"ID", "Name", "Default_strategy", "Resource_type", "Domain_id", "Project_id"},
 		[]string{})}
 
 	registerCompute(&Schedtags)

pkg/multicloud/huawei/instance.go

@@ -787,20 +787,26 @@ func (self *SRegion) CreateInstance(name string, imageId string, instanceType st
 	}
 
 	// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212668.html#ZH-CN_TOPIC_0020212668__table761103195216
-	params.KeyName = keypair
 	if len(keypair) > 0 {
-		udata, err := updateUserData(userData, "root", "", publicKey)
-		if err != nil {
-			return "", errors.Wrap(err, "region.CreateInstance.UpdateUserData")
-		}
-		params.AdminPass = passwd
-		params.UserData = udata
+		params.KeyName = keypair
 	} else {
-		udata, err := updateUserData(userData, "root", passwd, "")
+		params.AdminPass = passwd
+	}
+
+	if len(userData) > 0 {
+		pwd := ""
+		k := ""
+		if len(keypair) > 0 {
+			k = keypair
+		} else {
+			pwd = passwd
+		}
+
+		udata, err := updateUserData(userData, "root", pwd, k)
 		if err != nil {
 			return "", errors.Wrap(err, "region.CreateInstance.UpdateUserData")
 		}
-		params.AdminPass = passwd
+
 		params.UserData = udata
 	}
 

pkg/multicloud/qcloud/localstorage.go

@@ -27,6 +27,7 @@ import (
 type SLocalStorage struct {
 	zone        *SZone
 	storageType string
+	available   bool
 }
 
 func (self *SLocalStorage) GetMetadata() *jsonutils.JSONDict {
@@ -84,6 +85,9 @@ func (self *SLocalStorage) GetStorageConf() jsonutils.JSONObject {
 }
 
 func (self *SLocalStorage) GetStatus() string {
+	if !self.available {
+		return api.STORAGE_OFFLINE
+	}
 	return api.STORAGE_ONLINE
 }
 
@@ -93,7 +97,7 @@ func (self *SLocalStorage) Refresh() error {
 }
 
 func (self *SLocalStorage) GetEnabled() bool {
-	return true
+	return self.available == true
 }
 
 func (self *SLocalStorage) GetIStoragecache() cloudprovider.ICloudStoragecache {

pkg/multicloud/qcloud/storage.go

@@ -29,6 +29,7 @@ import (
 type SStorage struct {
 	zone        *SZone
 	storageType string
+	available   bool
 }
 
 func (self *SStorage) GetMetadata() *jsonutils.JSONDict {
@@ -97,6 +98,9 @@ func (self *SStorage) GetStorageConf() jsonutils.JSONObject {
 }
 
 func (self *SStorage) GetStatus() string {
+	if !self.available {
+		return api.STORAGE_OFFLINE
+	}
 	return api.STORAGE_ONLINE
 }
 
@@ -106,7 +110,7 @@ func (self *SStorage) Refresh() error {
 }
 
 func (self *SStorage) GetEnabled() bool {
-	return true
+	return self.available == true
 }
 
 func (self *SStorage) GetIStoragecache() cloudprovider.ICloudStoragecache {

pkg/multicloud/qcloud/zone.go

@@ -141,14 +141,20 @@ func (self *SZone) fetchStorages() error {
 	}
 	storageTypes := []string{}
 	for _, diskConfig := range diskConfigSet {
-		if diskConfig.Available && !utils.IsInStringArray(diskConfig.DiskType, storageTypes) {
-			storageTypes = append(storageTypes, diskConfig.DiskType)
-			storage := SStorage{zone: self, storageType: diskConfig.DiskType}
+		if !utils.IsInStringArray(strings.ToUpper(diskConfig.DiskType), storageTypes) {
+			storageTypes = append(storageTypes, strings.ToUpper(diskConfig.DiskType))
+			storage := SStorage{zone: self, storageType: diskConfig.DiskType, available: diskConfig.Available}
+			self.istorages = append(self.istorages, &storage)
+		}
+	}
+	for _, storageType := range []string{"CLOUD_PREMIUM", "CLOUD_SSD", "CLOUD_BASIC"} {
+		if !utils.IsInStringArray(storageType, storageTypes) {
+			storage := SStorage{zone: self, storageType: storageType, available: false}
 			self.istorages = append(self.istorages, &storage)
 		}
 	}
 	for _, localstorageType := range []string{"LOCAL_BASIC", "LOCAL_SSD"} {
-		storage := SLocalStorage{zone: self, storageType: localstorageType}
+		storage := SLocalStorage{zone: self, storageType: localstorageType, available: self.region.GetId() == "ap-hongkong"} //仅有香港本地存储可用
 		self.istorages = append(self.istorages, &storage)
 	}
 	return nil
@@ -186,7 +192,7 @@ func (self *SZone) getStorageByCategory(category string) (*SStorage, error) {
 			//return &SStorage{zone: self, storageType: strings.ToUpper(storages[i].GetStorageType())}, nil
 		}
 		storage := storages[i].(*SStorage)
-		if storage.storageType == category {
+		if strings.ToLower(storage.storageType) == strings.ToLower(category) {
 			return storage, nil
 		}
 	}

pkg/scheduler/algorithm/predicates/disk_schedtag_predicate.go

@@ -56,6 +56,10 @@ func (d diskW) ResourceKeyword() string {
 	return "storage"
 }
 
+func (d diskW) IsSpecifyResource() bool {
+	return d.Storage != ""
+}
+
 func (d diskW) GetSchedtags() []*computeapi.SchedtagConfig {
 	return d.DiskConfig.Schedtags
 }

pkg/scheduler/algorithm/predicates/network_schedtag_predicate.go

@@ -59,6 +59,10 @@ func (n netW) ResourceKeyword() string {
 	return "network"
 }
 
+func (n netW) IsSpecifyResource() bool {
+	return n.Network != ""
+}
+
 func (n netW) GetSchedtags() []*computeapi.SchedtagConfig {
 	return n.NetworkConfig.Schedtags
 }

pkg/scheduler/algorithm/predicates/predicates.go

@@ -282,6 +282,7 @@ func (p *PredicatedSchedtagResource) hasAvoidTags() bool {
 type ISchedtagCustomer interface {
 	JSON(interface{}) *jsonutils.JSONDict
 	Keyword() string
+	IsSpecifyResource() bool
 	GetSchedtags() []*computeapi.SchedtagConfig
 	ResourceKeyword() string
 }
@@ -330,7 +331,7 @@ func (p *BaseSchedtagPredicate) check(input ISchedtagCustomer, candidate ISchedt
 	res := &PredicatedSchedtagResource{
 		ISchedtagCandidateResource: candidate,
 	}
-	if shouldExec {
+	if shouldExec && !input.IsSpecifyResource() {
 		if err := tagPredicate.Check(
 			SchedtagResourceW{
 				candidater: candidate,