修复io1卷主机不能正常创建的问题

Gopkg.lock

@@ -1368,11 +1368,11 @@
 
 [[projects]]
   branch = "master"
-  digest = "1:2fdf064ae928c1b311e67ec51e856ce655abc39e8eacf5e6b3f7bd0417fac0a6"
+  digest = "1:04973e1902449b00dd7f3a9ad0b2b892c637cc8bc2e0dc569a82026fe1c4a4b3"
   name = "yunion.io/x/sqlchemy"
   packages = ["."]
   pruneopts = "UT"
-  revision = "e22221d5efcc667e68b0fdeed19958e788c3ad63"
+  revision = "998e91b54b0b9441f21dc9c09036f875a02ef8c5"
 
 [[projects]]
   branch = "master"
@@ -1450,6 +1450,7 @@
     "github.com/miekg/dns",
     "github.com/moul/http2curl",
     "github.com/nelsonken/cos-go-sdk-v5/cos",
+    "github.com/pierrec/lz4",
     "github.com/serialx/hashring",
     "github.com/stretchr/testify/assert",
     "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common",

Gopkg.toml

@@ -24,6 +24,9 @@
 #   go-tests = true
 #   unused-packages = true
 
+[[constraint]]
+  name = "github.com/pierrec/lz4"
+  version = "2.0.7"
 
 [[constraint]]
   name = "github.com/360EntSecGroup-Skylar/excelize"

cmd/climc/climc.go

@@ -96,7 +96,7 @@ func getSubcommandsParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	fmt.Printf("Error: %s\n", e)
+	fmt.Fprintf(os.Stderr, "Error: %s\n", e)
 	os.Exit(1)
 }
 

cmd/climc/shell/cloudregions.go

@@ -11,10 +11,11 @@ import (
 func init() {
 	type CloudregionListOptions struct {
 		options.BaseListOptions
-		Private bool   `help:"show private cloud regions only"`
-		Public  bool   `help:"show public cloud regions only"`
-		Manager string `help:"Show regions belongs to the cloud provider"`
-		Usable  bool   `help:"List regions that are usable"`
+		Private  bool   `help:"show private cloud regions only"`
+		Public   bool   `help:"show public cloud regions only"`
+		Manager  string `help:"Show regions belongs to the cloud provider"`
+		Provider string `help:"List regions of the public cloud provider" choices:"Aliyun|Qcloud|Azure|Aws|Huawei"`
+		Usable   bool   `help:"List regions that are usable"`
 	}
 	R(&CloudregionListOptions{}, "cloud-region-list", "List cloud regions", func(s *mcclient.ClientSession, args *CloudregionListOptions) error {
 		var params *jsonutils.JSONDict
@@ -38,6 +39,9 @@ func init() {
 		if len(args.Manager) > 0 {
 			params.Add(jsonutils.NewString(args.Manager), "manager")
 		}
+		if len(args.Provider) > 0 {
+			params.Add(jsonutils.NewString(args.Provider), "provider")
+		}
 		result, err := modules.Cloudregions.List(s, params)
 		if err != nil {
 			return err

cmd/climc/shell/disks.go

@@ -18,7 +18,9 @@ func init() {
 		Guest   string `help:"Guest ID or name"`
 		Storage string `help:"Storage ID or name"`
 
-		Provider string `help:"Provider for disk" choices:"Aliyun|VMware|Azure"`
+		Manager  string `help:"List disks belongs to the cloud provider"`
+		Account  string `help:"List disks belongs to the cloud account"`
+		Provider string `help:"List disks belongs to the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei"`
 
 		BillingType string `help:"billing type" choices:"postpaid|prepaid"`
 	}

cmd/climc/shell/elasticips.go

@@ -10,9 +10,12 @@ import (
 
 func init() {
 	type ElasticipListOptions struct {
-		Manager string `help:"Show servers imported from manager"`
-		Region  string `help:"Show servers in cloudregion"`
-		Usable  bool   `help:"List all zones that is usable"`
+		Manager  string `help:"Show servers imported from manager"`
+		Region   string `help:"Show servers in cloudregion"`
+		Account  string `help:"List hosts belongs to the cloud account"`
+		Provider string `help:"List hosts belongs to the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei"`
+
+		Usable bool `help:"List all zones that is usable"`
 
 		options.BaseListOptions
 	}
@@ -28,6 +31,12 @@ func init() {
 		if len(args.Manager) > 0 {
 			params.Add(jsonutils.NewString(args.Manager), "manager")
 		}
+		if len(args.Account) > 0 {
+			params.Add(jsonutils.NewString(args.Account), "account")
+		}
+		if len(args.Provider) > 0 {
+			params.Add(jsonutils.NewString(args.Provider), "provider")
+		}
 		if len(args.Region) > 0 {
 			params.Add(jsonutils.NewString(args.Region), "region")
 		}

cmd/climc/shell/hosts.go

@@ -13,8 +13,8 @@ func init() {
 	type HostListOptions struct {
 		Schedtag  string `help:"List hosts in schedtag"`
 		Zone      string `help:"List hosts in zone"`
+		Region    string `help:"List hosts in region"`
 		Wire      string `help:"List hosts in wire"`
-		VCenter   string `help:"List hosts in vcenter"`
 		Image     string `help:"List hosts cached images"`
 		Storage   string `help:"List hosts attached to storages"`
 		Baremetal string `help:"List hosts that is managed by baremetal system" choices:"true|false"`
@@ -27,8 +27,11 @@ func init() {
 
 		ResourceType string `help:"Resource type" choices:"shared|prepaid|dedicated"`
 
-		Manager string `help:"Show regions belongs to the cloud provider"`
-		Usable  bool   `help:"List all zones that is usable"`
+		Manager  string `help:"List hosts belongs to the cloud provider"`
+		Account  string `help:"List hosts belongs to the cloud account"`
+		Provider string `help:"List hosts belongs to the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei"`
+
+		Usable bool `help:"List all zones that is usable"`
 
 		options.BaseListOptions
 	}
@@ -48,12 +51,12 @@ func init() {
 		if len(args.Zone) > 0 {
 			params.Add(jsonutils.NewString(args.Zone), "zone")
 		}
+		if len(args.Region) > 0 {
+			params.Add(jsonutils.NewString(args.Region), "region")
+		}
 		if len(args.Wire) > 0 {
 			params.Add(jsonutils.NewString(args.Wire), "wire")
 		}
-		if len(args.VCenter) > 0 {
-			params.Add(jsonutils.NewString(args.VCenter), "vcenter")
-		}
 		if len(args.Image) > 0 {
 			params.Add(jsonutils.NewString(args.Image), "cachedimage")
 		}
@@ -73,6 +76,12 @@ func init() {
 		if len(args.Manager) > 0 {
 			params.Add(jsonutils.NewString(args.Manager), "manager")
 		}
+		if len(args.Account) > 0 {
+			params.Add(jsonutils.NewString(args.Account), "account")
+		}
+		if len(args.Provider) > 0 {
+			params.Add(jsonutils.NewString(args.Provider), "provider")
+		}
 
 		if args.Usable {
 			params.Add(jsonutils.JSONTrue, "usable")

cmd/climc/shell/hoststorages.go

@@ -25,9 +25,9 @@ func init() {
 		}
 		var result *modules.ListResult
 		var err error
-		if len(args.Storage) > 0 {
-			params.Add(jsonutils.NewString(args.Storage), "storage")
-		}
+		// if len(args.Storage) > 0 {
+		// 	params.Add(jsonutils.NewString(args.Storage), "storage")
+		// }
 		if len(args.Host) > 0 {
 			result, err = modules.Hoststorages.ListDescendent(s, args.Host, params)
 		} else if len(args.Storage) > 0 {

cmd/climc/shell/networks.go

@@ -18,6 +18,10 @@ func init() {
 		Vpc        string `help:"search networks belongs to a VPC"`
 		Region     string `help:"search networks belongs to a CloudRegion"`
 		ServerType string `help:"search networks belongs to a ServerType"`
+
+		Manager  string `help:"List networks belongs to the cloud provider"`
+		Account  string `help:"List networks belongs to the cloud account"`
+		Provider string `help:"List networks belongs to the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei"`
 	}
 	R(&NetworkListOptions{}, "network-list", "List networks", func(s *mcclient.ClientSession, args *NetworkListOptions) error {
 		var params *jsonutils.JSONDict
@@ -44,6 +48,15 @@ func init() {
 		if len(args.ServerType) > 0 {
 			params.Add(jsonutils.NewString(args.ServerType), "server_type")
 		}
+		if len(args.Manager) > 0 {
+			params.Add(jsonutils.NewString(args.Manager), "manager")
+		}
+		if len(args.Account) > 0 {
+			params.Add(jsonutils.NewString(args.Account), "account")
+		}
+		if len(args.Provider) > 0 {
+			params.Add(jsonutils.NewString(args.Provider), "provider")
+		}
 		var result *modules.ListResult
 		var err error
 		if len(args.Wire) > 0 {

cmd/climc/shell/routetables.go

@@ -114,4 +114,12 @@ func init() {
 		printObjectRecursive(routetable)
 		return nil
 	})
+	R(&options.RouteTablePurgeOptions{}, "routetable-purge", "Purge routetable", func(s *mcclient.ClientSession, opts *options.RouteTablePurgeOptions) error {
+		routetable, err := modules.RouteTables.PerformAction(s, opts.ID, "purge", nil)
+		if err != nil {
+			return err
+		}
+		printObjectRecursive(routetable)
+		return nil
+	})
 }

cmd/climc/shell/schedulers.go

@@ -66,7 +66,15 @@ func init() {
 				}
 			}
 			if len(args.Project) > 0 {
-				data.Add(jsonutils.NewString(args.Project), "tenant")
+				ret, err := modules.Projects.Get(s, args.Project, nil)
+				if err != nil {
+					return err
+				}
+				projectId, err := ret.GetString("id")
+				if err != nil {
+					return err
+				}
+				data.Add(jsonutils.NewString(projectId), "owner_tenant_id")
 			}
 			if len(args.Hypervisor) > 0 {
 				data.Add(jsonutils.NewString(args.Hypervisor), "hypervisor")

cmd/climc/shell/servers.go

@@ -313,7 +313,7 @@ func init() {
 		return nil
 	})
 
-	R(&options.ServerSecGroupOptions{}, "server-add-secgroup", "Add security group to a VM", func(s *mcclient.ClientSession, opts *options.ServerSecGroupOptions) error {
+	R(&options.ServerSecGroupsOptions{}, "server-add-secgroup", "Add security group to a VM", func(s *mcclient.ClientSession, opts *options.ServerSecGroupsOptions) error {
 		params, err := options.StructToParams(opts)
 		if err != nil {
 			return err

cmd/climc/shell/skus.go

@@ -11,7 +11,7 @@ import (
 func init() {
 	type ServerSkusListOptions struct {
 		options.BaseListOptions
-		Provider string `help:"provider" choices:"all|kvm|esxi|xen|hyperv|aliyun|azure|aws|qcloud|huawei"`
+		Provider string `help:"provider" choices:"all|aliyun|azure|aws|qcloud|huawei" default:""`
 		Region   string `help:"region Id or name"`
 		Zone     string `help:"zone Id or name"`
 		Cpu      int    `help:"Cpu core count"`
@@ -50,14 +50,11 @@ func init() {
 	})
 
 	type ServerSkusCreateOptions struct {
-		Name         string `help:"Name ID of SKU" required:"true" positional:"true"`
-		CpuCoreCount int    `help:"Cpu Count" required:"true" positional:"true"`
-		MemorySizeMB int    `help:"Memory MB" required:"true" positional:"true"`
-		Provider     string `help:"Provider name" choices:"kvm|esxi"`
+		CpuCoreCount int `help:"Cpu Count" required:"true" positional:"true"`
+		MemorySizeMB int `help:"Memory MB" required:"true" positional:"true"`
 
-		OsName      *string `help:"OS name/type" choices:"Linux|Windows|Any" default:"Any"`
-		SkuFamily   *string `help:"sku family"`
-		SkuCategory *string `help:"sku category" choices:"general_purpose|compute_optimized|memory_optimized|storage_optimized|hardware_accelerated|high_memory|high_storage"`
+		OsName               *string `help:"OS name/type" choices:"Linux|Windows|Any" default:"Any"`
+		InstanceTypeCategory *string `help:"instance type category" choices:"general_purpose|compute_optimized|memory_optimized|storage_optimized|hardware_accelerated|high_memory|high_storage"`
 
 		SysDiskResizable *bool   `help:"system disk is resizable"`
 		SysDiskType      *string `help:"system disk type" default:"local" choices:"local"`
@@ -95,12 +92,10 @@ func init() {
 	type ServerSkusUpdateOptions struct {
 		ID string `help:"Name or ID of SKU" json:"-"`
 
-		Name         *string `help:"new name of SKU"`
-		CpuCoreCount *int    `help:"Cpu Count"`
-		MemorySizeMB *int    `help:"Memory MB"`
+		CpuCoreCount *int `help:"Cpu Count"`
+		MemorySizeMB *int `help:"Memory MB"`
 
-		SkuFamily   *string `help:"sku family"`
-		SkuCategory *string `help:"sku category" choices:"general_purpose|compute_optimized|memory_optimized|storage_optimized|hardware_accelerated|high_memory|high_storage"`
+		InstanceTypeCategory *string `help:"instance type category" choices:"general_purpose|compute_optimized|memory_optimized|storage_optimized|hardware_accelerated|high_memory|high_storage"`
 
 		SysDiskResizable *bool `help:"system disk is resizable"`
 		SysDiskMaxSizeGB *int  `help:"system disk maximal size in gb"`

cmd/climc/shell/snapshots.go

@@ -15,9 +15,10 @@ func init() {
 		Local       bool   `help:"Show local snapshots"`
 		Share       bool   `help:"Show shared snapshots"`
 		DiskType    string `help:"Filter by disk type" choices:"sys|data"`
-		Provider    string `help:"Cloud provider" choices:"Aliyun|VMware|Azure"`
 
-		Manager string `help:"Show snapshots belongs to a specific cloud provider"`
+		Manager  string `help:"Show snapshots belongs to a specific cloud provider"`
+		Account  string `help:"List hosts belongs to the cloud account"`
+		Provider string `help:"List hosts belongs to the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei"`
 	}
 	R(&SnapshotsListOptions{}, "snapshot-list", "Show snapshots", func(s *mcclient.ClientSession, args *SnapshotsListOptions) error {
 		params, err := args.BaseListOptions.Params()
@@ -46,6 +47,9 @@ func init() {
 		if len(args.Manager) > 0 {
 			params.Add(jsonutils.NewString(args.Manager), "manager")
 		}
+		if len(args.Account) > 0 {
+			params.Add(jsonutils.NewString(args.Account), "account")
+		}
 		result, err := modules.Snapshots.List(s, params)
 		if err != nil {
 			return err

cmd/climc/shell/storages.go

@@ -18,7 +18,9 @@ func init() {
 		Zone   string `help:"List storages in zone"`
 		Region string `help:"List storages in region"`
 
-		Manager string `help:"Show regions belongs to the cloud provider"`
+		Manager  string `help:"List storages belongs to the cloud provider"`
+		Account  string `help:"List storages belongs to the cloud account"`
+		Provider string `help:"List storages belongs to the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei"`
 	}
 	R(&StorageListOptions{}, "storage-list", "List storages", func(s *mcclient.ClientSession, args *StorageListOptions) error {
 		var params *jsonutils.JSONDict
@@ -46,6 +48,12 @@ func init() {
 		if len(args.Manager) > 0 {
 			params.Add(jsonutils.NewString(args.Manager), "manager")
 		}
+		if len(args.Account) > 0 {
+			params.Add(jsonutils.NewString(args.Account), "account")
+		}
+		if len(args.Provider) > 0 {
+			params.Add(jsonutils.NewString(args.Provider), "provider")
+		}
 
 		var result *modules.ListResult
 		var err error

cmd/climc/shell/vpcs.go

@@ -10,8 +10,12 @@ import (
 func init() {
 	type VpcListOptions struct {
 		options.BaseListOptions
-		Region  string `help:"ID or Name of region"`
-		Manager string `help:"Show vpcs belongs to the cloud provider"`
+
+		Region string `help:"ID or Name of region"`
+
+		Manager  string `help:"List vpcs belongs to the cloud provider"`
+		Account  string `help:"List vpcs belongs to the cloud account"`
+		Provider string `help:"List vpcs belongs to the public cloud" choices:"Aliyun|Qcloud|Azure|Aws|Huawei"`
 	}
 	R(&VpcListOptions{}, "vpc-list", "List VPCs", func(s *mcclient.ClientSession, args *VpcListOptions) error {
 		var params *jsonutils.JSONDict
@@ -27,6 +31,12 @@ func init() {
 		if len(args.Manager) > 0 {
 			params.Add(jsonutils.NewString(args.Manager), "manager")
 		}
+		if len(args.Account) > 0 {
+			params.Add(jsonutils.NewString(args.Account), "account")
+		}
+		if len(args.Provider) > 0 {
+			params.Add(jsonutils.NewString(args.Provider), "provider")
+		}
 
 		var result *modules.ListResult
 		var err error

cmd/climc/shell/wires.go

@@ -10,8 +10,14 @@ import (
 func init() {
 	type WireListOptions struct {
 		options.BaseListOptions
-		Zone string `help:"list wires in zone"`
-		Vpc  string `help:"List wires in vpc"`
+
+		Region string `help:"List hosts in region"`
+		Zone   string `help:"list wires in zone"`
+		Vpc    string `help:"List wires in vpc"`
+
+		Manager  string `help:"List hosts belongs to the cloud provider"`
+		Account  string `help:"List hosts belongs to the cloud account"`
+		Provider string `help:"List hosts belongs to the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei"`
 	}
 	R(&WireListOptions{}, "wire-list", "List wires", func(s *mcclient.ClientSession, args *WireListOptions) error {
 		var params *jsonutils.JSONDict
@@ -26,6 +32,19 @@ func init() {
 		if len(args.Vpc) > 0 {
 			params.Add(jsonutils.NewString(args.Vpc), "vpc")
 		}
+		if len(args.Region) > 0 {
+			params.Add(jsonutils.NewString(args.Region), "region")
+		}
+		if len(args.Manager) > 0 {
+			params.Add(jsonutils.NewString(args.Manager), "manager")
+		}
+		if len(args.Account) > 0 {
+			params.Add(jsonutils.NewString(args.Account), "account")
+		}
+		if len(args.Provider) > 0 {
+			params.Add(jsonutils.NewString(args.Provider), "provider")
+		}
+
 		var result *modules.ListResult
 		var err error
 		if len(args.Zone) > 0 {

cmd/host-image/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "yunion.io/x/onecloud/pkg/hostimage"
+
+func main() {
+	hostimage.StartService()
+}

pkg/appsrv/response.go

@@ -2,9 +2,9 @@ package appsrv
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 
-	"fmt"
 	"yunion.io/x/onecloud/pkg/httperrors"
 )
 
@@ -56,6 +56,16 @@ func (w *responseWriterChannel) WriteHeader(status int) {
 	<-w.statusResp
 }
 
+// implent http.Flusher
+func (w *responseWriterChannel) Flush() {
+	if w.isClosed {
+		return
+	}
+	if f, ok := w.backend.(http.Flusher); ok {
+		f.Flush()
+	}
+}
+
 func (w *responseWriterChannel) wait(ctx context.Context, workerChan chan *SWorker) interface{} {
 	var err error
 	var worker *SWorker

pkg/cloudcommon/validators/misc.go

@@ -0,0 +1,29 @@
+package validators
+
+import (
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/sqlchemy"
+)
+
+type ModelFilterOptions struct {
+	Key          string
+	ModelKeyword string
+	ProjectId    string
+}
+
+func ApplyModelFilters(q *sqlchemy.SQuery, data *jsonutils.JSONDict, opts []*ModelFilterOptions) (*sqlchemy.SQuery, error) {
+	var err error
+	for _, opt := range opts {
+		v := NewModelIdOrNameValidator(
+			opt.Key,
+			opt.ModelKeyword,
+			opt.ProjectId,
+		)
+		v.Optional(true)
+		q, err = v.QueryFilter(q, data)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return q, nil
+}

pkg/cloudprovider/consts.go

@@ -17,3 +17,4 @@ var ErrInvalidStatus = errors.New("invalid status")
 var ErrTimeout = errors.New("timeout")
 var ErrNotImplemented = errors.New("Not implemented")
 var ErrNotSupported = errors.New("Not supported")
+var ErrInvalidProvider = errors.New("Invalid provider")

pkg/cloudprovider/fakeregion.go

@@ -36,12 +36,8 @@ func (region *SFakeOnPremiseRegion) GetMetadata() *jsonutils.JSONDict {
 	return nil
 }
 
-func (region *SFakeOnPremiseRegion) GetLatitude() float32 {
-	return 0.0
-}
-
-func (region *SFakeOnPremiseRegion) GetLongitude() float32 {
-	return 0.0
+func (region *SFakeOnPremiseRegion) GetGeographicInfo() SGeographicInfo {
+	return SGeographicInfo{}
 }
 
 func (region *SFakeOnPremiseRegion) GetIZones() ([]ICloudZone, error) {

pkg/cloudprovider/geoinfo.go

@@ -0,0 +1,9 @@
+package cloudprovider
+
+type SGeographicInfo struct {
+	Latitude  float32 `list:"user" update:"admin" create:"admin_optional"`
+	Longitude float32 `list:"user" update:"admin" create:"admin_optional"`
+
+	City        string `list:"user" width:"32" update:"admin" create:"admin_optional"`
+	CountryCode string `list:"user" width:"4" update:"admin" create:"admin_optional"`
+}

pkg/cloudprovider/resources.go

@@ -33,8 +33,9 @@ type IBillingResource interface {
 type ICloudRegion interface {
 	ICloudResource
 
-	GetLatitude() float32
-	GetLongitude() float32
+	// GetLatitude() float32
+	// GetLongitude() float32
+	GetGeographicInfo() SGeographicInfo
 
 	GetIZones() ([]ICloudZone, error)
 	GetIVpcs() ([]ICloudVpc, error)

pkg/compute/guestdrivers/aliyun.go

@@ -7,13 +7,13 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
-	"yunion.io/x/onecloud/pkg/httperrors"
-	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/billing"
 	"yunion.io/x/onecloud/pkg/util/seclib2"
 )

pkg/compute/guestdrivers/aws.go

@@ -5,16 +5,15 @@ import (
 	"fmt"
 	"time"
 
-	"yunion.io/x/onecloud/pkg/util/ansible"
-	"yunion.io/x/pkg/utils"
-
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/ansible"
 	"yunion.io/x/onecloud/pkg/util/billing"
+	"yunion.io/x/pkg/utils"
 )
 
 type SAwsGuestDriver struct {

pkg/compute/guestdrivers/azure.go

@@ -6,20 +6,20 @@ import (
 	"strings"
 	"time"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
-	"yunion.io/x/onecloud/pkg/cloudprovider"
-	"yunion.io/x/onecloud/pkg/httperrors"
-	"yunion.io/x/onecloud/pkg/mcclient"
-	"yunion.io/x/onecloud/pkg/util/ansible"
-	"yunion.io/x/onecloud/pkg/util/seclib2"
 	"yunion.io/x/pkg/util/compare"
 	"yunion.io/x/pkg/utils"
 
-	"yunion.io/x/jsonutils"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/ansible"
 	"yunion.io/x/onecloud/pkg/util/billing"
+	"yunion.io/x/onecloud/pkg/util/seclib2"
 )
 
 type SAzureGuestDriver struct {

pkg/compute/guestdrivers/base.go

@@ -4,11 +4,11 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 
-	"time"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/compute/models"

pkg/compute/guestdrivers/esxi.go

@@ -4,12 +4,12 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/utils"
 
-	"time"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/util/billing"

pkg/compute/guestdrivers/kvm.go

@@ -277,7 +277,7 @@ func (self *SKVMGuestDriver) GetChangeConfigStatus() ([]string, error) {
 }
 
 func (self *SKVMGuestDriver) GetDeployStatus() ([]string, error) {
-	return []string{models.VM_READY}, nil
+	return []string{models.VM_READY, models.VM_RUNNING, models.VM_ADMIN}, nil
 }
 
 func (self *SKVMGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk *models.SDisk, storage *models.SStorage) error {

pkg/compute/guestdrivers/managedvirtual.go

@@ -3,19 +3,19 @@ package guestdrivers
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
-	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/pkg/util/compare"
 	"yunion.io/x/pkg/util/secrules"
 
-	"time"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/billing"
 )
 

pkg/compute/guestdrivers/qcloud.go

@@ -7,8 +7,6 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
-	"yunion.io/x/onecloud/pkg/httperrors"
-	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/pkg/util/compare"
 	"yunion.io/x/pkg/utils"
 
@@ -16,6 +14,8 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/billing"
 	"yunion.io/x/onecloud/pkg/util/seclib2"
 )

pkg/compute/guestdrivers/virtualization.go

@@ -7,12 +7,12 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
-	"yunion.io/x/onecloud/pkg/httperrors"
-	"yunion.io/x/onecloud/pkg/mcclient"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
 type SVirtualizedGuestDriver struct {

pkg/compute/handlers.go

@@ -34,6 +34,7 @@ func InitHandlers(app *appsrv.Application) {
 		db.Metadata,
 		models.GuestcdromManager,
 		models.NetInterfaceManager,
+		models.VCenterManager,
 	} {
 		db.RegisterModelManager(manager)
 	}

pkg/compute/hostdrivers/base.go

@@ -6,11 +6,12 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/utils"
+
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/compute/baremetal"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/mcclient"
-	"yunion.io/x/pkg/utils"
 )
 
 type SBaseHostDriver struct {

pkg/compute/hostdrivers/esxi.go

@@ -3,9 +3,11 @@ package hostdrivers
 import (
 	"context"
 	"fmt"
+
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 
+	"github.com/golang-plus/errors"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/util/httputils"
@@ -62,6 +64,12 @@ func (self *SESXiHostDriver) CheckAndSetCacheImage(ctx context.Context, host *mo
 	content.Format = cacheImage.GetFormat()
 
 	storage := host.GetStorageByFilePath(storageCache.Path)
+	if storage == nil {
+		msg := fmt.Sprintf("fail to find storage for storageCache %s", storageCache.Path)
+		log.Errorf(msg)
+		return errors.New(msg)
+	}
+
 	accessInfo, err := host.GetCloudaccount().GetVCenterAccessInfo(storage.ExternalId)
 	if err != nil {
 		return err

pkg/compute/hostdrivers/kvm.go

@@ -7,6 +7,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/compute/baremetal"
 	"yunion.io/x/onecloud/pkg/compute/models"
@@ -120,6 +121,22 @@ func (self *SKVMHostDriver) RequestUncacheImage(ctx context.Context, host *model
 
 func (self *SKVMHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error {
 	header := task.GetTaskRequestHeader()
+	if snapshotId, err := content.GetString("snapshot"); err == nil {
+		iSnapshot, _ := models.SnapshotManager.FetchById(snapshotId)
+		snapshot := iSnapshot.(*models.SSnapshot)
+		snapshotStorage := models.StorageManager.FetchStorageById(snapshot.StorageId)
+		snapshotHost := snapshotStorage.GetMasterHost()
+		if options.Options.SnapshotCreateDiskProtocol == "url" {
+			content.Set("snapshot_url",
+				jsonutils.NewString(fmt.Sprintf("%s/download/snapshots/%s/%s/%s",
+					snapshotHost.ManagerUri, snapshotStorage.Id, snapshot.DiskId, snapshot.Id)))
+			content.Set("snapshot_out_of_chain", jsonutils.NewBool(snapshot.OutOfChain))
+		} else if options.Options.SnapshotCreateDiskProtocol == "fuse" {
+			content.Set("snapshot_url", jsonutils.NewString(fmt.Sprintf("%s/snapshots/%s/%s",
+				snapshotHost.GetFetchUrl(), snapshot.DiskId, snapshot.Id)))
+		}
+		content.Set("protocol", jsonutils.NewString(options.Options.SnapshotCreateDiskProtocol))
+	}
 
 	url := fmt.Sprintf("/disks/%s/create/%s", storage.Id, disk.Id)
 	body := jsonutils.NewDict()

pkg/compute/hostdrivers/managedvirtual.go

@@ -8,14 +8,12 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 
-	"yunion.io/x/onecloud/pkg/cloudprovider"
-	"yunion.io/x/onecloud/pkg/httperrors"
-
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
-	"yunion.io/x/onecloud/pkg/compute/options"
-
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/compute/options"
+	"yunion.io/x/onecloud/pkg/httperrors"
 )
 
 type SManagedVirtualizationHostDriver struct {

pkg/compute/misc/handler.go

@@ -6,13 +6,14 @@ import (
 	"net/http"
 
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/tristate"
+
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/compute/options"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
-	"yunion.io/x/pkg/tristate"
 )
 
 func AddMiscHandler(prefix string, app *appsrv.Application) {

pkg/compute/models/billingresource.go

@@ -2,9 +2,6 @@ package models
 
 import (
 	"time"
-
-	"yunion.io/x/onecloud/pkg/appctx"
-	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 )
 
 const (
@@ -26,12 +23,14 @@ func (self *SBillingResourceBase) GetChargeType() string {
 	}
 }
 
-func (self *SBillingResourceBase) FetchCloudBillingInfo(info *SCloudBillingInfo) {
+func (self *SBillingResourceBase) getBillingBaseInfo() SBillingBaseInfo {
+	info := SBillingBaseInfo{}
 	info.ChargeType = self.GetChargeType()
 	if self.GetChargeType() == BILLING_TYPE_PREPAID {
 		info.ExpiredAt = self.ExpiredAt
 		info.BillingCycle = self.BillingCycle
 	}
+	return info
 }
 
 func (self *SBillingResourceBase) IsValidPrePaid() bool {
@@ -44,75 +43,17 @@ func (self *SBillingResourceBase) IsValidPrePaid() bool {
 	return false
 }
 
+type SBillingBaseInfo struct {
+	ChargeType   string    `json:",omitempty"`
+	ExpiredAt    time.Time `json:",omitempty"`
+	BillingCycle string    `json:",omitempty"`
+}
+
 type SCloudBillingInfo struct {
-	Provider            string
-	Account             string
-	AccountId           string
-	SubAccount          string
-	SubAccountId        string
-	SubAccountProject   string
-	SubAccountProjectId string
-	Region              string
-	RegionId            string
-	RegionExtId         string
-	Zone                string
-	ZoneId              string
-	ZoneExtId           string
-	PriceKey            string
-	ChargeType          string
-	InternetChargeType  string
-	ExpiredAt           time.Time
-	BillingCycle        string
-}
+	SCloudProviderInfo
 
-func MakeCloudBillingInfo(region *SCloudregion, zone *SZone, provider *SCloudprovider) SCloudBillingInfo {
-	info := SCloudBillingInfo{}
+	SBillingBaseInfo
 
-	if zone != nil {
-		info.Zone = zone.GetName()
-		info.ZoneId = zone.GetId()
-	}
-
-	if region != nil {
-		info.Region = region.GetName()
-		info.RegionId = region.GetId()
-	}
-
-	if provider != nil {
-		info.SubAccount = provider.GetName()
-		info.SubAccountId = provider.GetId()
-
-		if len(provider.ProjectId) > 0 {
-			info.SubAccountProjectId = provider.ProjectId
-			tc, err := db.TenantCacheManager.FetchTenantById(appctx.Background, provider.ProjectId)
-			if err == nil {
-				info.SubAccountProject = tc.GetName()
-			}
-		}
-
-		account := provider.GetCloudaccount()
-		info.Account = account.GetName()
-		info.AccountId = account.GetId()
-
-		driver, err := provider.GetDriver()
-
-		if err == nil {
-			info.Provider = driver.GetId()
-
-			if region != nil {
-				iregion, err := driver.GetIRegionById(region.ExternalId)
-				if err == nil {
-					info.RegionExtId = iregion.GetId()
-					if zone != nil {
-						izone, err := iregion.GetIZoneById(zone.ExternalId)
-						if err == nil {
-							info.ZoneExtId = izone.GetId()
-						}
-					}
-				}
-			}
-		}
-	}
-
-	return info
+	PriceKey           string `json:",omitempty"`
+	InternetChargeType string `json:",omitempty"`
 }

pkg/compute/models/capabilities.go

@@ -132,11 +132,7 @@ func getNetworkCount(zone *SZone) int {
 }
 
 func isSchedPolicySupported(zone *SZone) bool {
-	if zone != nil {
-		return !zone.isManaged()
-	} else {
-		return true
-	}
+	return true
 }
 
 func getMinNicCount(zone *SZone) int {

pkg/compute/models/cloudregions.go

@@ -43,9 +43,9 @@ func init() {
 type SCloudregion struct {
 	db.SEnabledStatusStandaloneResourceBase
 
-	Latitude  float32 `list:"user"`
-	Longitude float32 `list:"user"`
-	Provider  string  `width:"64" charset:"ascii" list:"user"`
+	cloudprovider.SGeographicInfo
+
+	Provider string `width:"64" charset:"ascii" list:"user"`
 }
 
 func (manager *SCloudregionManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
@@ -227,8 +227,7 @@ func (self *SCloudregion) syncWithCloudRegion(cloudRegion cloudprovider.ICloudRe
 	_, err := self.GetModelManager().TableSpec().Update(self, func() error {
 		self.Name = cloudRegion.GetName()
 		self.Status = cloudRegion.GetStatus()
-		self.Latitude = cloudRegion.GetLatitude()
-		self.Longitude = cloudRegion.GetLongitude()
+		self.SGeographicInfo = cloudRegion.GetGeographicInfo()
 		self.Provider = cloudRegion.GetProvider()
 
 		self.IsEmulated = cloudRegion.IsEmulated()
@@ -247,8 +246,7 @@ func (manager *SCloudregionManager) newFromCloudRegion(cloudRegion cloudprovider
 
 	region.ExternalId = cloudRegion.GetGlobalId()
 	region.Name = cloudRegion.GetName()
-	region.Latitude = cloudRegion.GetLatitude()
-	region.Longitude = cloudRegion.GetLongitude()
+	region.SGeographicInfo = cloudRegion.GetGeographicInfo()
 	region.Status = cloudRegion.GetStatus()
 	region.Enabled = true
 	region.Provider = cloudRegion.GetProvider()
@@ -352,6 +350,7 @@ func (manager *SCloudregionManager) ListItemFilter(ctx context.Context, q *sqlch
 		}
 		q = q.Equals("provider", manager.Provider)
 	}
+
 	if jsonutils.QueryBoolean(query, "usable", false) {
 		networks := NetworkManager.Query().SubQuery()
 		wires := WireManager.Query().SubQuery()

pkg/compute/models/disks.go

@@ -100,6 +100,9 @@ type SDisk struct {
 
 	// # backing template id and type
 	TemplateId string `width:"256" charset:"ascii" nullable:"true" list:"user"` // Column(VARCHAR(ID_LENGTH, charset='ascii'), nullable=True)
+	// backing snapshot id
+	SnapshotId string `width:"256" charset:"ascii" nullable:"true" list:"user"`
+
 	// # file system
 	FsFormat string `width:"32" charset:"ascii" nullable:"true" list:"user"` // Column(VARCHAR(32, charset='ascii'), nullable=True)
 	// # disk type, OS, SWAP, DAT
@@ -113,6 +116,15 @@ func (manager *SDiskManager) GetContextManager() []db.IModelManager {
 	return []db.IModelManager{StorageManager}
 }
 
+func (manager *SDiskManager) FetchDiskById(diskId string) *SDisk {
+	disk, err := manager.FetchById(diskId)
+	if err != nil {
+		log.Errorf("FetchById fail %s", err)
+		return nil
+	}
+	return disk.(*SDisk)
+}
+
 func (manager *SDiskManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
 	queryDict, ok := query.(*jsonutils.JSONDict)
 	if !ok {
@@ -158,14 +170,7 @@ func (manager *SDiskManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu
 		sq := storages.Query(storages.Field("id")).Filter(sqlchemy.In(storages.Field("storage_type"), STORAGE_LOCAL_TYPES))
 		q = q.Filter(sqlchemy.In(q.Field("storage_id"), sq))
 	}
-	if provier, _ := queryDict.GetString("provider"); len(provier) > 0 {
-		cloudprovider := CloudproviderManager.Query().SubQuery()
-		sq := storages.Query(storages.Field("id")).Join(cloudprovider,
-			sqlchemy.AND(
-				sqlchemy.Equals(cloudprovider.Field("id"), storages.Field("manager_id")),
-				sqlchemy.Equals(cloudprovider.Field("provider"), provier)))
-		q = q.Filter(sqlchemy.In(q.Field("storage_id"), sq))
-	}
+
 	guestId, _ := queryDict.GetString("guest")
 	if len(guestId) != 0 {
 		guest := GuestManager.FetchGuestById(guestId)
@@ -190,6 +195,47 @@ func (manager *SDiskManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu
 		}
 		q = q.Filter(sqlchemy.Equals(q.Field("storage_id"), storageObj.GetId()))
 	}
+
+	managerStr := jsonutils.GetAnyString(query, []string{"manager", "cloudprovider", "cloudprovider_id", "manager_id"})
+	if len(managerStr) > 0 {
+		provider, err := CloudproviderManager.FetchByIdOrName(nil, managerStr)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(CloudproviderManager.Keyword(), managerStr)
+			}
+			return nil, httperrors.NewGeneralError(err)
+		}
+		subq := storages.Query(storages.Field("id")).Equals("manager_id", provider.GetId())
+		q = q.Filter(sqlchemy.In(q.Field("storage_id"), subq.SubQuery()))
+	}
+
+	accountStr := jsonutils.GetAnyString(query, []string{"account", "account_id", "cloudaccount", "cloudaccount_id"})
+	if len(accountStr) > 0 {
+		account, err := CloudaccountManager.FetchByIdOrName(nil, accountStr)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr)
+			}
+			return nil, httperrors.NewGeneralError(err)
+		}
+
+		cloudproviders := CloudproviderManager.Query().SubQuery()
+		subq := storages.Query(storages.Field("id"))
+		subq = subq.Join(cloudproviders, sqlchemy.Equals(cloudproviders.Field("id"), storages.Field("manager_id")))
+		subq = subq.Filter(sqlchemy.Equals(cloudproviders.Field("cloudaccount_id"), account.GetId()))
+
+		q = q.Filter(sqlchemy.In(q.Field("storage_id"), subq.SubQuery()))
+	}
+
+	if provier, _ := queryDict.GetString("provider"); len(provier) > 0 {
+		cloudproviders := CloudproviderManager.Query().SubQuery()
+		sq := storages.Query(storages.Field("id"))
+		sq = sq.Join(cloudproviders, sqlchemy.Equals(cloudproviders.Field("id"), storages.Field("manager_id")))
+		sq = sq.Filter(sqlchemy.Equals(cloudproviders.Field("provider"), provier))
+
+		q = q.Filter(sqlchemy.In(q.Field("storage_id"), sq.SubQuery()))
+	}
+
 	return q, nil
 }
 
@@ -437,6 +483,8 @@ func (self *SDisk) StartAllocate(ctx context.Context, host *SHost, storage *SSto
 	content.Add(jsonutils.NewInt(int64(self.DiskSize)), "size")
 	if len(snapshot) > 0 {
 		content.Add(jsonutils.NewString(snapshot), "snapshot")
+		SnapshotManager.AddRefCount(self.SnapshotId, 1)
+		self.SetMetadata(ctx, "merge_snapshot", jsonutils.JSONTrue, userCred)
 	} else if len(templateId) > 0 {
 		content.Add(jsonutils.NewString(templateId), "image_id")
 	}
@@ -1006,6 +1054,10 @@ func totalDiskSize(projectId string, active tristate.TriState, ready tristate.Tr
 
 type SDiskConfig struct {
 	ImageId string
+
+	SnapshotId string
+	DiskType   string // sys, data, swap
+
 	// ImageDiskFormat string
 	SizeMb          int    // MB
 	Fs              string // file system
@@ -1062,28 +1114,15 @@ func parseDiskInfo(ctx context.Context, userCred mcclient.TokenCredential, info
 			diskConfig.SizeMb = -1
 		} else if utils.IsInStringArray(p, STORAGE_TYPES) {
 			diskConfig.Backend = p
+		} else if strings.HasPrefix(p, "snapshot-") {
+			// HACK: use snapshot creat disk format snapshot-id
+			// example: snapshot-3140cecb-ccc4-4865-abae-3a5ba8c69d9b
+			if err := fillDiskConfigBySnapshot(userCred, &diskConfig, p[len("snapshot-"):]); err != nil {
+				return nil, err
+			}
 		} else if len(p) > 0 {
-			if userCred == nil {
-				diskConfig.ImageId = p
-			} else {
-				image, err := CachedimageManager.getImageInfo(ctx, userCred, p, false)
-				if err != nil {
-					log.Errorf("getImageInfo fail %s", err)
-					return nil, err
-				}
-				if image.Status != IMAGE_STATUS_ACTIVE {
-					return nil, httperrors.NewInvalidStatusError("Image status is not active")
-				}
-				diskConfig.ImageId = image.Id
-				diskConfig.ImageProperties = image.Properties
-				if len(diskConfig.Format) == 0 {
-					diskConfig.Format = image.DiskFormat
-				}
-				// diskConfig.ImageDiskFormat = image.DiskFormat
-				CachedimageManager.ImageAddRefCount(image.Id)
-				if diskConfig.SizeMb == 0 {
-					diskConfig.SizeMb = image.MinDisk // MB
-				}
+			if err := fillDiskConfigByImage(ctx, userCred, &diskConfig, p); err != nil {
+				return nil, err
 			}
 		}
 	}
@@ -1095,6 +1134,60 @@ func parseDiskInfo(ctx context.Context, userCred mcclient.TokenCredential, info
 	return &diskConfig, nil
 }
 
+func fillDiskConfigBySnapshot(userCred mcclient.TokenCredential, diskConfig *SDiskConfig, snapshotId string) error {
+	iSnapshot, err := SnapshotManager.FetchByIdOrName(userCred, snapshotId)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return httperrors.NewNotFoundError("Snapshot %s not found", snapshotId)
+		}
+		return err
+	}
+	var snapshot = iSnapshot.(*SSnapshot)
+	if storage := StorageManager.FetchStorageById(snapshot.StorageId); storage == nil {
+		return httperrors.NewBadRequestError("Snapshot %s storage %s not found, is public cloud?",
+			snapshotId, snapshot.StorageId)
+	} else {
+		if disk := DiskManager.FetchDiskById(snapshot.DiskId); disk != nil {
+			diskConfig.Fs = disk.FsFormat
+			if len(diskConfig.Format) == 0 {
+				diskConfig.Format = disk.DiskFormat
+			}
+		}
+		diskConfig.SnapshotId = snapshot.Id
+		diskConfig.DiskType = snapshot.DiskType
+		diskConfig.SizeMb = snapshot.Size
+		diskConfig.Backend = storage.StorageType
+	}
+	return nil
+}
+
+func fillDiskConfigByImage(ctx context.Context, userCred mcclient.TokenCredential,
+	diskConfig *SDiskConfig, imageId string) error {
+	if userCred == nil {
+		diskConfig.ImageId = imageId
+	} else {
+		image, err := CachedimageManager.getImageInfo(ctx, userCred, imageId, false)
+		if err != nil {
+			log.Errorf("getImageInfo fail %s", err)
+			return err
+		}
+		if image.Status != IMAGE_STATUS_ACTIVE {
+			return httperrors.NewInvalidStatusError("Image status is not active")
+		}
+		diskConfig.ImageId = image.Id
+		diskConfig.ImageProperties = image.Properties
+		if len(diskConfig.Format) == 0 {
+			diskConfig.Format = image.DiskFormat
+		}
+		// diskConfig.ImageDiskFormat = image.DiskFormat
+		CachedimageManager.ImageAddRefCount(image.Id)
+		if diskConfig.SizeMb == 0 {
+			diskConfig.SizeMb = image.MinDisk // MB
+		}
+	}
+	return nil
+}
+
 func parseIsoInfo(ctx context.Context, userCred mcclient.TokenCredential, info string) (string, error) {
 	image, err := CachedimageManager.getImageInfo(ctx, userCred, info, false)
 	if err != nil {
@@ -1111,6 +1204,9 @@ func (self *SDisk) fetchDiskInfo(diskConfig *SDiskConfig) {
 	if len(diskConfig.ImageId) > 0 {
 		self.TemplateId = diskConfig.ImageId
 		self.DiskType = DISK_TYPE_SYS
+	} else if len(diskConfig.SnapshotId) > 0 {
+		self.SnapshotId = diskConfig.SnapshotId
+		self.DiskType = diskConfig.DiskType
 	}
 	if len(diskConfig.Fs) > 0 {
 		self.FsFormat = diskConfig.Fs
@@ -1202,15 +1298,19 @@ func (self *SDisk) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict
 		extra.Add(jsonutils.NewString(storage.GetName()), "storage")
 		extra.Add(jsonutils.NewString(storage.StorageType), "storage_type")
 		extra.Add(jsonutils.NewString(storage.MediumType), "medium_type")
-		extra.Add(jsonutils.NewString(storage.ZoneId), "zone_id")
+		/*extra.Add(jsonutils.NewString(storage.ZoneId), "zone_id")
 		if zone := storage.getZone(); zone != nil {
 			extra.Add(jsonutils.NewString(zone.Name), "zone")
 			extra.Add(jsonutils.NewString(zone.CloudregionId), "region_id")
 			if region := zone.GetRegion(); region != nil {
 				extra.Add(jsonutils.NewString(region.Name), "region")
 			}
-		}
+		}*/
+
+		info := storage.getCloudProviderInfo()
+		extra.Update(jsonutils.Marshal(&info))
 	}
+
 	guests, guest_status := []string{}, []string{}
 	for _, guest := range self.GetGuests() {
 		guests = append(guests, guest.Name)
@@ -1344,14 +1444,14 @@ func (self *SDisk) GetShortDesc() *jsonutils.JSONDict {
 	var billingInfo SCloudBillingInfo
 
 	if storage != nil {
-		billingInfo = storage.getCloudBillingInfo()
+		billingInfo.SCloudProviderInfo = storage.getCloudProviderInfo()
 	}
 
 	if priceKey := self.GetMetadata("price_key", nil); len(priceKey) > 0 {
 		billingInfo.PriceKey = priceKey
 	}
 
-	self.FetchCloudBillingInfo(&billingInfo)
+	billingInfo.SBillingBaseInfo = self.getBillingBaseInfo()
 
 	desc.Update(jsonutils.Marshal(billingInfo))
 

pkg/compute/models/elasticips.go

@@ -114,6 +114,25 @@ func (manager *SElasticipManager) ListItemFilter(ctx context.Context, q *sqlchem
 		q = q.Equals("cloudregion_id", regionObj.GetId())
 	}
 
+	accountStr := jsonutils.GetAnyString(query, []string{"account", "account_id", "cloudaccount", "cloudaccount_id"})
+	if len(accountStr) > 0 {
+		account, err := CloudaccountManager.FetchByIdOrName(nil, accountStr)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr)
+			}
+			return nil, httperrors.NewGeneralError(err)
+		}
+		subq := CloudproviderManager.Query("id").Equals("cloudaccount_id", account.GetId()).SubQuery()
+		q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq))
+	}
+
+	providerStr := jsonutils.GetAnyString(query, []string{"provider"})
+	if len(providerStr) > 0 {
+		subq := CloudproviderManager.Query("id").Equals("provider", providerStr).SubQuery()
+		q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq))
+	}
+
 	if query.Contains("usable") {
 		usable := jsonutils.QueryBoolean(query, "usable", false)
 		if usable {
@@ -159,7 +178,9 @@ func (self *SElasticip) GetShortDesc() *jsonutils.JSONDict {
 	// }
 	//}
 
-	billingInfo := self.getCloudBillingInfo()
+	billingInfo := SCloudBillingInfo{}
+
+	billingInfo.SCloudProviderInfo = self.getCloudProviderInfo()
 
 	billingInfo.InternetChargeType = self.ChargeType
 
@@ -921,8 +942,8 @@ func (self *SElasticip) DoPendingDelete(ctx context.Context, userCred mcclient.T
 	self.Dissociate(ctx, userCred)
 }
 
-func (self *SElasticip) getCloudBillingInfo() SCloudBillingInfo {
+func (self *SElasticip) getCloudProviderInfo() SCloudProviderInfo {
 	region := self.GetRegion()
 	provider := self.GetCloudprovider()
-	return MakeCloudBillingInfo(region, nil, provider)
+	return MakeCloudProviderInfo(region, nil, provider)
 }

pkg/compute/models/guest_actions.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
+	"time"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
@@ -22,7 +23,6 @@ import (
 	"yunion.io/x/onecloud/pkg/util/logclient"
 	"yunion.io/x/onecloud/pkg/util/seclib2"
 
-	"time"
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/util/billing"
@@ -98,7 +98,7 @@ func (self *SGuest) PerformSaveImage(ctx context.Context, userCred mcclient.Toke
 		return nil, httperrors.NewInputParameterError("No root image")
 	} else {
 		kwargs := data.(*jsonutils.JSONDict)
-		restart := self.Status == VM_RUNNING
+		restart := (self.Status == VM_RUNNING) || jsonutils.QueryBoolean(data, "auto_start", false)
 		properties := jsonutils.NewDict()
 		if notes, err := data.GetString("notes"); err != nil && len(notes) > 0 {
 			properties.Add(jsonutils.NewString(notes), "notes")
@@ -347,7 +347,8 @@ func (self *SGuest) PerformDeploy(ctx context.Context, userCred mcclient.TokenCr
 	}
 
 	if utils.IsInStringArray(self.Status, deployStatus) {
-		if doRestart && self.Status == VM_RUNNING {
+		if (doRestart && self.Status == VM_RUNNING) ||
+			jsonutils.QueryBoolean(kwargs, "auto_start", false) {
 			kwargs.Set("restart", jsonutils.JSONTrue)
 		}
 		err := self.StartGuestDeployTask(ctx, userCred, kwargs, "deploy", "")
@@ -643,25 +644,46 @@ func (self *SGuest) PerformAddSecgroup(ctx context.Context, userCred mcclient.To
 		return nil, httperrors.NewInputParameterError("Cannot assign security rules in status %s", self.Status)
 	}
 
-	secgrpV := validators.NewModelIdOrNameValidator("secgrp", "secgroup", userCred.GetProjectId())
-	if err := secgrpV.Validate(data.(*jsonutils.JSONDict)); err != nil {
-		return nil, err
-	}
-
 	maxCount := self.GetDriver().GetMaxSecurityGroupCount()
 	if maxCount == 0 {
 		return nil, httperrors.NewUnsupportOperationError("Cannot assign security group for this guest %s", self.Name)
 	}
 
+	secgrps := []string{}
+	if err := data.Unmarshal(&secgrps, "secgrps"); err != nil {
+		return nil, httperrors.NewInputParameterError(err.Error())
+	}
+
 	secgroups := self.GetSecgroups()
-	if len(secgroups) >= maxCount {
+	if len(secgroups)+len(secgrps) >= maxCount {
 		return nil, httperrors.NewUnsupportOperationError("guest %s band to up to %d security groups", self.Name, maxCount)
 	}
 
-	secgroup := secgrpV.Model.(*SSecurityGroup)
-	if _, err := GuestsecgroupManager.newGuestSecgroup(ctx, userCred, self, secgroup); err != nil {
-		return nil, httperrors.NewInputParameterError(err.Error())
+	secgroupIds := []string{}
+	for _, secgroup := range secgroups {
+		secgroupIds = append(secgroupIds, secgroup.Id)
 	}
+
+	addSecgroups := []*SSecurityGroup{}
+
+	for _, _secgrp := range secgrps {
+		secgrp, err := SecurityGroupManager.FetchByIdOrName(userCred, _secgrp)
+		if err != nil {
+			return nil, httperrors.NewInputParameterError(err.Error())
+		}
+
+		if utils.IsInStringArray(secgrp.GetId(), secgroupIds) {
+			return nil, httperrors.NewInputParameterError("security group %s has already been assigned to guest %s", secgrp.GetName(), self.Name)
+		}
+		addSecgroups = append(addSecgroups, secgrp.(*SSecurityGroup))
+	}
+
+	for _, secgroup := range addSecgroups {
+		if _, err := GuestsecgroupManager.newGuestSecgroup(ctx, userCred, self, secgroup); err != nil {
+			return nil, httperrors.NewInputParameterError(err.Error())
+		}
+	}
+
 	return nil, self.StartSyncTask(ctx, userCred, true, "")
 }
 
@@ -749,7 +771,7 @@ func (self *SGuest) AllowPerformPurge(ctx context.Context, userCred mcclient.Tok
 }
 
 func (self *SGuest) PerformPurge(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	err := self.ValidateDeleteCondition(ctx)
+	err := self.ValidatePurgeCondition(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -2112,3 +2134,18 @@ func (self *SGuest) doSaveRenewInfo(userCred mcclient.TokenCredential, bc *billi
 	db.OpsLog.LogEvent(self, db.ACT_RENEW, self.GetShortDesc(), userCred)
 	return nil
 }
+
+func (self *SGuest) AllowPerformStreamDisksComplete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return db.IsAdminAllowPerform(userCred, self, "stream-disks-complete")
+}
+
+func (self *SGuest) PerformStreamDisksComplete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	for _, disk := range self.GetDisks() {
+		d := disk.GetDisk()
+		if len(d.SnapshotId) > 0 {
+			SnapshotManager.AddRefCount(d.SnapshotId, -1)
+			d.SetMetadata(ctx, "merge_snapshot", jsonutils.JSONFalse, userCred)
+		}
+	}
+	return nil, nil
+}

pkg/compute/models/guestdisks.go

@@ -166,6 +166,12 @@ func (self *SGuestdisk) GetJsonDescAtHost(host *SHost) jsonutils.JSONObject {
 	if len(tid) > 0 {
 		desc.Add(jsonutils.NewString(tid), "template_id")
 	}
+	if len(disk.SnapshotId) > 0 {
+		needMerge := disk.GetMetadata("merge_snapshot", nil)
+		if needMerge == "true" {
+			desc.Set("merge_snapshot", jsonutils.JSONTrue)
+		}
+	}
 	fs := disk.GetFsFormat()
 	if len(fs) > 0 {
 		desc.Add(jsonutils.NewString(fs), "fs")

pkg/compute/models/guests.go

@@ -399,6 +399,37 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ
 		q = q.In("host_id", sq)
 	}
 
+	accountStr := jsonutils.GetAnyString(query, []string{"account", "account_id", "cloudaccount", "cloudaccount_id"})
+	if len(accountStr) > 0 {
+		account, err := CloudaccountManager.FetchByIdOrName(nil, accountStr)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr)
+			}
+			return nil, httperrors.NewGeneralError(err)
+		}
+		hosts := HostManager.Query().SubQuery()
+		cloudproviders := CloudproviderManager.Query().SubQuery()
+
+		subq := hosts.Query(hosts.Field("id"))
+		subq = subq.Join(cloudproviders, sqlchemy.Equals(cloudproviders.Field("id"), hosts.Field("manager_id")))
+		subq = subq.Filter(sqlchemy.Equals(cloudproviders.Field("cloudaccount_id"), account.GetId()))
+
+		q = q.Filter(sqlchemy.In(q.Field("host_id"), subq.SubQuery()))
+	}
+
+	providerStr := jsonutils.GetAnyString(query, []string{"provider"})
+	if len(providerStr) > 0 {
+		hosts := HostManager.Query().SubQuery()
+		cloudproviders := CloudproviderManager.Query().SubQuery()
+
+		subq := hosts.Query(hosts.Field("id"))
+		subq = subq.Join(cloudproviders, sqlchemy.Equals(cloudproviders.Field("id"), hosts.Field("manager_id")))
+		subq = subq.Filter(sqlchemy.Equals(cloudproviders.Field("provider"), providerStr))
+
+		q = q.Filter(sqlchemy.In(q.Field("host_id"), subq.SubQuery()))
+	}
+
 	regionFilter, _ := queryDict.GetString("region")
 	if len(regionFilter) > 0 {
 		regionObj, err := CloudregionManager.FetchByIdOrName(userCred, regionFilter)
@@ -481,16 +512,24 @@ func (guest *SGuest) GetDriver() IGuestDriver {
 	return GetDriver(hypervisor)
 }
 
-func (guest *SGuest) ValidateDeleteCondition(ctx context.Context) error {
+func (guest *SGuest) validateDeleteCondition(ctx context.Context, isPurge bool) error {
 	if guest.DisableDelete.IsTrue() {
 		return httperrors.NewInvalidStatusError("Virtual server is locked, cannot delete")
 	}
-	if guest.IsValidPrePaid() {
+	if !isPurge && guest.IsValidPrePaid() {
 		return httperrors.NewForbiddenError("not allow to delete prepaid server in valid status")
 	}
 	return guest.SVirtualResourceBase.ValidateDeleteCondition(ctx)
 }
 
+func (guest *SGuest) ValidatePurgeCondition(ctx context.Context) error {
+	return guest.validateDeleteCondition(ctx, true)
+}
+
+func (guest *SGuest) ValidateDeleteCondition(ctx context.Context) error {
+	return guest.validateDeleteCondition(ctx, false)
+}
+
 func (guest *SGuest) GetDisksQuery() *sqlchemy.SQuery {
 	return GuestdiskManager.Query().Equals("guest_id", guest.Id)
 }
@@ -689,6 +728,9 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m
 		if err != nil {
 			return nil, httperrors.NewInputParameterError("Invalid root image: %s", err)
 		}
+		if len(diskConfig.SnapshotId) > 0 && diskConfig.DiskType != DISK_TYPE_SYS {
+			return nil, httperrors.NewBadRequestError("Snapshot error: disk index 0 but disk type is %s", diskConfig.DiskType)
+		}
 
 		if len(diskConfig.Backend) == 0 {
 			diskConfig.Backend = STORAGE_LOCAL
@@ -777,6 +819,9 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m
 			if err != nil {
 				return nil, httperrors.NewInputParameterError("parse disk description error %s", err)
 			}
+			if diskConfig.DiskType == DISK_TYPE_SYS {
+				return nil, httperrors.NewBadRequestError("Snapshot error: disk index %d > 0 but disk type is %s", i+1, DISK_TYPE_SYS)
+			}
 			if len(diskConfig.Backend) == 0 {
 				diskConfig.Backend = rootStorageType
 			}
@@ -1113,7 +1158,7 @@ func (self *SGuest) GetCustomizeColumns(ctx context.Context, userCred mcclient.T
 }
 
 func (self *SGuest) moreExtraInfo(extra *jsonutils.JSONDict) *jsonutils.JSONDict {
-	zone := self.getZone()
+	/*zone := self.getZone()
 	if zone != nil {
 		extra.Add(jsonutils.NewString(zone.GetId()), "zone_id")
 		extra.Add(jsonutils.NewString(zone.GetName()), "zone")
@@ -1139,6 +1184,12 @@ func (self *SGuest) moreExtraInfo(extra *jsonutils.JSONDict) *jsonutils.JSONDict
 				extra.Add(jsonutils.NewString(provider.GetName()), "manager")
 			}
 		}
+	}*/
+
+	host := self.GetHost()
+	if host != nil {
+		info := host.getCloudProviderInfo()
+		extra.Update(jsonutils.Marshal(&info))
 	}
 
 	err := self.CanPerformPrepaidRecycle()
@@ -2344,6 +2395,9 @@ func (self *SGuest) CreateDisksOnHost(ctx context.Context, userCred mcclient.Tok
 			return err
 		}
 		data.Add(jsonutils.NewString(disk.Id), fmt.Sprintf("disk.%d.id", idx))
+		if len(diskConfig.SnapshotId) > 0 {
+			data.Add(jsonutils.NewString(diskConfig.SnapshotId), fmt.Sprintf("disk.%d.snapshot", idx))
+		}
 	}
 	return nil
 }
@@ -3171,14 +3225,14 @@ func (self *SGuest) GetShortDesc() *jsonutils.JSONDict {
 	var billingInfo SCloudBillingInfo
 
 	if host != nil {
-		billingInfo = host.getCloudBillingInfo()
+		billingInfo.SCloudProviderInfo = host.getCloudProviderInfo()
 	}
 
 	if priceKey := self.GetMetadata("price_key", nil); len(priceKey) > 0 {
 		billingInfo.PriceKey = priceKey
 	}
 
-	self.FetchCloudBillingInfo(&billingInfo)
+	billingInfo.SBillingBaseInfo = self.getBillingBaseInfo()
 
 	desc.Update(jsonutils.Marshal(billingInfo))
 

pkg/compute/models/host_recycle.go

@@ -11,6 +11,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/tristate"
+	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
@@ -107,7 +108,7 @@ func (self *SGuest) doPrepaidRecycleNoLock(ctx context.Context, userCred mcclien
 		info.Size = int64(disk.DiskSize)
 		info.Index = int64(i)
 		info.Slot = i
-		info.Driver = storage.StorageType
+		info.Driver = baremetal.DISK_DRIVER_LINUX
 		info.Rotate = (storage.MediumType != DISK_TYPE_SSD)
 
 		storageInfo = append(storageInfo, info)
@@ -320,6 +321,10 @@ func (self *SHost) PerformUndoPrepaidRecycle(ctx context.Context, userCred mccli
 		return nil, httperrors.NewInvalidStatusError("a recycle host shoud not allocate more than 1 guest")
 	}
 
+	if guests[0].PendingDeleted {
+		return nil, httperrors.NewInvalidStatusError("cannot undo a recycle host with pending_deleted guest")
+	}
+
 	err := doUndoPrepaidRecycle(ctx, userCred, self, &guests[0])
 	if err != nil {
 		logclient.AddActionLog(self, logclient.ACT_UNDO_RECYCLE_PREPAID, self.GetShortDesc(), userCred, false)
@@ -351,7 +356,10 @@ func doUndoPrepaidRecycle(ctx context.Context, userCred mcclient.TokenCredential
 	q := HostManager.Query()
 	q = q.Equals("external_id", host.ExternalId)
 	q = q.Equals("host_type", host.HostType)
-	q = q.IsNullOrEmpty("resource_type")
+	q = q.Filter(sqlchemy.OR(
+		sqlchemy.IsNullOrEmpty(q.Field("resource_type")),
+		sqlchemy.Equals(q.Field("resource_type"), HostResourceTypeShared),
+	))
 
 	oHostCnt := q.Count()
 
@@ -428,13 +436,14 @@ func doUndoPrepaidRecycle(ctx context.Context, userCred mcclient.TokenCredential
 			return err
 		}
 
-		oStorageObj, err := StorageManager.FetchByExternalId(istorage.GetGlobalId())
-		if err != nil {
-			log.Errorf("StorageManager.FetchByExternalId fail %s", err)
-			return err
+		oHostStorage := oHost.GetHoststorageByExternalId(istorage.GetGlobalId())
+		if oHostStorage == nil {
+			msg := fmt.Sprintf("oHost.GetHoststorageByExternalId not found %s", istorage.GetGlobalId())
+			log.Errorf(msg)
+			return errors.New(msg)
 		}
 
-		oStorage := oStorageObj.(*SStorage)
+		oStorage := oHostStorage.GetStorage()
 
 		if storage.StorageType == STORAGE_LOCAL {
 			_, err = disk.GetModelManager().TableSpec().Update(disk, func() error {
@@ -579,7 +588,11 @@ func (host *SHost) RebuildRecycledGuest(ctx context.Context, userCred mcclient.T
 
 	q := HostManager.Query()
 	q = q.Equals("external_id", host.ExternalId)
-	q = q.IsNullOrEmpty("resource_type")
+	q = q.Filter(sqlchemy.OR(
+		sqlchemy.IsNullOrEmpty(q.Field("resource_type")),
+		sqlchemy.Equals(q.Field("resource_type"), HostResourceTypeShared),
+	))
+
 	err := q.First(&oHost)
 	if err != nil {
 		log.Errorf("query oHost fail %s", err)

pkg/compute/models/hosts.go

@@ -196,9 +196,10 @@ func (self *SHost) AllowDeleteItem(ctx context.Context, userCred mcclient.TokenC
 }
 
 func (manager *SHostManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
+	queryDict := query.(*jsonutils.JSONDict)
+
 	resType, _ := query.GetString("resource_type")
 	if len(resType) > 0 {
-		queryDict := query.(*jsonutils.JSONDict)
 		queryDict.Remove("resource_type")
 
 		switch resType {
@@ -268,23 +269,65 @@ func (manager *SHostManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu
 
 	zoneStr := jsonutils.GetAnyString(query, []string{"zone", "zone_id"})
 	if len(zoneStr) > 0 {
-		zone, _ := ZoneManager.FetchByIdOrName(nil, zoneStr)
-		if zone == nil {
-			return nil, httperrors.NewResourceNotFoundError("Zone %s not found", zoneStr)
+		zone, err := ZoneManager.FetchByIdOrName(nil, zoneStr)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(ZoneManager.Keyword(), zoneStr)
+			}
+			return nil, httperrors.NewGeneralError(err)
 		}
 		q = q.Filter(sqlchemy.Equals(q.Field("zone_id"), zone.GetId()))
+
+		queryDict.Remove("zone_id")
 	}
+
+	regionStr := jsonutils.GetAnyString(query, []string{"region", "region_id"})
+	if len(regionStr) > 0 {
+		region, err := CloudregionManager.FetchByIdOrName(nil, regionStr)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(CloudregionManager.Keyword(), regionStr)
+			}
+			return nil, httperrors.NewGeneralError(err)
+		}
+		subq := ZoneManager.Query("id").Equals("cloudregion_id", region.GetId()).SubQuery()
+		q = q.Filter(sqlchemy.In(q.Field("zone_id"), subq))
+	}
+
 	// vcenter
 	// zone
 	// cachedimage
 
-	managerStr := jsonutils.GetAnyString(query, []string{"manager", "provider", "manager_id", "provider_id"})
+	managerStr := jsonutils.GetAnyString(query, []string{"manager", "cloudprovider", "cloudprovider_id", "manager_id"})
 	if len(managerStr) > 0 {
-		provider := CloudproviderManager.FetchCloudproviderByIdOrName(managerStr)
-		if provider == nil {
-			return nil, httperrors.NewResourceNotFoundError("provider %s not found", managerStr)
+		provider, err := CloudproviderManager.FetchByIdOrName(nil, managerStr)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(CloudproviderManager.Keyword(), managerStr)
+			}
+			return nil, httperrors.NewGeneralError(err)
 		}
 		q = q.Filter(sqlchemy.Equals(q.Field("manager_id"), provider.GetId()))
+		queryDict.Remove("manager_id")
+	}
+
+	accountStr := jsonutils.GetAnyString(query, []string{"account", "account_id", "cloudaccount", "cloudaccount_id"})
+	if len(accountStr) > 0 {
+		account, err := CloudaccountManager.FetchByIdOrName(nil, accountStr)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr)
+			}
+			return nil, httperrors.NewGeneralError(err)
+		}
+		subq := CloudproviderManager.Query("id").Equals("cloudaccount_id", account.GetId()).SubQuery()
+		q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq))
+	}
+
+	providerStr := jsonutils.GetAnyString(query, []string{"provider"})
+	if len(providerStr) > 0 {
+		subq := CloudproviderManager.Query("id").Equals("provider", providerStr).SubQuery()
+		q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq))
 	}
 
 	usable := jsonutils.QueryBoolean(query, "usable", false)
@@ -502,6 +545,26 @@ func (self *SHost) GetHoststorageOfId(storageId string) *SHoststorage {
 	return &hoststorage
 }
 
+func (self *SHost) GetHoststorageByExternalId(extId string) *SHoststorage {
+	hoststorage := SHoststorage{}
+	hoststorage.SetModelManager(HoststorageManager)
+
+	hoststorages := HoststorageManager.Query().SubQuery()
+	storages := StorageManager.Query().SubQuery()
+	q := hoststorages.Query()
+	q = q.Join(storages, sqlchemy.Equals(hoststorages.Field("storage_id"), storages.Field("id")))
+	q = q.Filter(sqlchemy.Equals(hoststorages.Field("host_id"), self.Id))
+	q = q.Filter(sqlchemy.Equals(storages.Field("external_id"), extId))
+
+	err := q.First(&hoststorage)
+	if err != nil {
+		log.Errorf("GetHoststorageByExternalId fail %s", err)
+		return nil
+	}
+
+	return &hoststorage
+}
+
 func (self *SHost) GetStorageByFilePath(path string) *SStorage {
 	hoststorages := self.GetHoststorages()
 	if hoststorages == nil {
@@ -629,7 +692,7 @@ func (self *SHost) GetFetchUrl() string {
 			port = 80
 		}
 	}
-	return fmt.Sprintf("%s://%s:%d", managerUrl.Scheme, managerUrl.Host, port+40000)
+	return fmt.Sprintf("%s://%s:%d", managerUrl.Scheme, strings.Split(managerUrl.Host, ":")[0], port+40000)
 }
 
 func (self *SHost) GetAttachedStorages(storageType string) []SStorage {
@@ -733,16 +796,21 @@ func (self *SHost) GetSpec(statusCheck bool) *jsonutils.JSONDict {
 		}
 	}
 	spec.Set("nic_count", jsonutils.NewInt(nicCount))
-	manufacture, err := self.SysInfo.Get("manufacture")
-	if err != nil {
-		manufacture = jsonutils.NewString("Unknown")
+
+	var manufacture string
+	var model string
+	if self.SysInfo != nil {
+		manufacture, _ = self.SysInfo.GetString("manufacture")
+		model, _ = self.SysInfo.GetString("model")
 	}
-	spec.Set("manufacture", manufacture)
-	model, err := self.SysInfo.Get("model")
-	if err != nil {
-		model = jsonutils.NewString("Unknown")
+	if manufacture == "" {
+		manufacture = "Unknown"
 	}
-	spec.Set("model", model)
+	if model == "" {
+		model = "Unknown"
+	}
+	spec.Set("manufacture", jsonutils.NewString(manufacture))
+	spec.Set("model", jsonutils.NewString(model))
 	return spec
 }
 
@@ -1272,6 +1340,8 @@ func (manager *SHostManager) newFromCloudHost(extHost cloudprovider.ICloudHost,
 	host.MemSize = extHost.GetMemSizeMB()
 	host.StorageSize = extHost.GetStorageSizeMB()
 	host.StorageType = extHost.GetStorageType()
+	host.CpuCmtbound = 8.0
+	host.MemCmtbound = 1.0
 
 	host.ManagerId = extHost.GetManagerId()
 	host.IsEmulated = extHost.IsEmulated()
@@ -1942,13 +2012,23 @@ func (self *SHost) getGuestsResource(status string) *SHostGuestResourceUsage {
 }
 
 func (self *SHost) getMoreDetails(ctx context.Context, extra *jsonutils.JSONDict) *jsonutils.JSONDict {
-	zone := self.GetZone()
+	/*zone := self.GetZone()
 	if zone != nil {
 		extra.Add(jsonutils.NewString(zone.Id), "zone_id")
 		extra.Add(jsonutils.NewString(zone.Name), "zone")
-		extra.Add(jsonutils.NewString(zone.GetRegion().GetName()), "region")
-		extra.Add(jsonutils.NewString(zone.GetRegion().GetId()), "region_id")
-	}
+		if len(zone.ExternalId) > 0 {
+			extra.Add(jsonutils.NewString(zone.ExternalId), "")
+		}
+		region := zone.GetRegion()
+		if region != nil {
+			extra.Add(jsonutils.NewString(zone.GetRegion().GetName()), "region")
+			extra.Add(jsonutils.NewString(zone.GetRegion().GetId()), "region_id")
+		}
+	}*/
+
+	info := self.getCloudProviderInfo()
+	extra.Update(jsonutils.Marshal(&info))
+
 	server := self.GetBaremetalServer()
 	if server != nil {
 		extra.Add(jsonutils.NewString(server.Id), "server_id")
@@ -2001,7 +2081,8 @@ func (self *SHost) getMoreDetails(ctx context.Context, extra *jsonutils.JSONDict
 	}
 	extra.Add(jsonutils.NewFloat(memCommitRate), "mem_commit_rate")
 	extra.Add(self.GetHardwareSpecification(), "spec")
-	extra = self.SManagedResourceBase.getExtraDetails(ctx, extra)
+
+	// extra = self.SManagedResourceBase.getExtraDetails(ctx, extra)
 
 	if self.IsPrepaidRecycle() {
 		extra.Add(jsonutils.JSONTrue, "is_prepaid_recycle")
@@ -3444,17 +3525,17 @@ func (manager *SHostManager) GetHostByIp(hostIp string) (*SHost, error) {
 	return host.(*SHost), nil
 }
 
-func (self *SHost) getCloudBillingInfo() SCloudBillingInfo {
+func (self *SHost) getCloudProviderInfo() SCloudProviderInfo {
 	var region *SCloudregion
 	zone := self.GetZone()
 	if zone != nil {
 		region = zone.GetRegion()
 	}
 	provider := self.GetCloudprovider()
-	return MakeCloudBillingInfo(region, zone, provider)
+	return MakeCloudProviderInfo(region, zone, provider)
 }
 
 func (self *SHost) GetShortDesc() *jsonutils.JSONDict {
-	info := self.getCloudBillingInfo()
+	info := self.getCloudProviderInfo()
 	return jsonutils.Marshal(&info).(*jsonutils.JSONDict)
 }

pkg/compute/models/loadbalancerbackendgroups.go

@@ -52,13 +52,11 @@ func (man *SLoadbalancerBackendGroupManager) ListItemFilter(ctx context.Context,
 	}
 	userProjId := userCred.GetProjectId()
 	data := query.(*jsonutils.JSONDict)
-	{
-		lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", userProjId)
-		lbV.Optional(true)
-		q, err = lbV.QueryFilter(q, data)
-		if err != nil {
-			return nil, err
-		}
+	q, err = validators.ApplyModelFilters(q, data, []*validators.ModelFilterOptions{
+		{Key: "loadbalancer", ModelKeyword: "loadbalancer", ProjectId: userProjId},
+	})
+	if err != nil {
+		return nil, err
 	}
 	return q, nil
 }

pkg/compute/models/loadbalancerbackends.go

@@ -55,22 +55,12 @@ func (man *SLoadbalancerBackendManager) ListItemFilter(ctx context.Context, q *s
 	}
 	userProjId := userCred.GetProjectId()
 	data := query.(*jsonutils.JSONDict)
-	{
-		backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", userProjId)
-		backendGroupV.Optional(true)
-		q, err = backendGroupV.QueryFilter(q, data)
-		if err != nil {
-			return nil, err
-		}
-	}
-	{
-		// NOTE extend this when new backend_type was added
-		backendV := validators.NewModelIdOrNameValidator("backend", "server", userProjId)
-		backendV.Optional(true)
-		q, err = backendV.QueryFilter(q, data)
-		if err != nil {
-			return nil, err
-		}
+	q, err = validators.ApplyModelFilters(q, data, []*validators.ModelFilterOptions{
+		{Key: "backend_group", ModelKeyword: "loadbalancerbackendgroup", ProjectId: userProjId},
+		{Key: "backend", ModelKeyword: "server", ProjectId: userProjId}, // NOTE extend this when new backend_type was added
+	})
+	if err != nil {
+		return nil, err
 	}
 	return q, nil
 }

pkg/compute/models/loadbalancerlistenerrules.go

@@ -72,21 +72,12 @@ func (man *SLoadbalancerListenerRuleManager) ListItemFilter(ctx context.Context,
 	}
 	userProjId := userCred.GetProjectId()
 	data := query.(*jsonutils.JSONDict)
-	{
-		listenerV := validators.NewModelIdOrNameValidator("listener", "loadbalancerlistener", userProjId)
-		listenerV.Optional(true)
-		q, err = listenerV.QueryFilter(q, data)
-		if err != nil {
-			return nil, err
-		}
-	}
-	{
-		backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", userProjId)
-		backendGroupV.Optional(true)
-		q, err = backendGroupV.QueryFilter(q, data)
-		if err != nil {
-			return nil, err
-		}
+	q, err = validators.ApplyModelFilters(q, data, []*validators.ModelFilterOptions{
+		{Key: "listener", ModelKeyword: "loadbalancerlistener", ProjectId: userProjId},
+		{Key: "backend_group", ModelKeyword: "loadbalancerbackendgroup", ProjectId: userProjId},
+	})
+	if err != nil {
+		return nil, err
 	}
 	return q, nil
 }

pkg/compute/models/loadbalancerlisteners.go

@@ -143,29 +143,13 @@ func (man *SLoadbalancerListenerManager) ListItemFilter(ctx context.Context, q *
 	}
 	userProjId := userCred.GetProjectId()
 	data := query.(*jsonutils.JSONDict)
-	{
-		lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", userProjId)
-		lbV.Optional(true)
-		q, err = lbV.QueryFilter(q, data)
-		if err != nil {
-			return nil, err
-		}
-	}
-	{
-		backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", userProjId)
-		backendGroupV.Optional(true)
-		q, err = backendGroupV.QueryFilter(q, data)
-		if err != nil {
-			return nil, err
-		}
-	}
-	{
-		aclV := validators.NewModelIdOrNameValidator("acl", "loadbalanceracl", userProjId)
-		aclV.Optional(true)
-		q, err = aclV.QueryFilter(q, data)
-		if err != nil {
-			return nil, err
-		}
+	q, err = validators.ApplyModelFilters(q, data, []*validators.ModelFilterOptions{
+		{Key: "loadbalancer", ModelKeyword: "loadbalancer", ProjectId: userProjId},
+		{Key: "backend_group", ModelKeyword: "loadbalancerbackendgroup", ProjectId: userProjId},
+		{Key: "acl", ModelKeyword: "loadbalanceracl", ProjectId: userProjId},
+	})
+	if err != nil {
+		return nil, err
 	}
 	return q, nil
 }
@@ -357,7 +341,7 @@ func (lblis *SLoadbalancerListener) ValidateUpdateData(ctx context.Context, user
 
 		"certificate":       certV,
 		"tls_cipher_policy": tlsCipherPolicyV,
-		"enable_http2":      validators.NewBoolValidator("enable_http2").Default(true),
+		"enable_http2":      validators.NewBoolValidator("enable_http2"),
 	}
 	for _, v := range keyV {
 		v.Optional(true)

pkg/compute/models/loadbalancers.go

@@ -62,21 +62,12 @@ func (man *SLoadbalancerManager) ListItemFilter(ctx context.Context, q *sqlchemy
 	}
 	userProjId := userCred.GetProjectId()
 	data := query.(*jsonutils.JSONDict)
-	{
-		networkV := validators.NewModelIdOrNameValidator("network", "network", userProjId)
-		networkV.Optional(true)
-		q, err = networkV.QueryFilter(q, data)
-		if err != nil {
-			return nil, err
-		}
-	}
-	{
-		zoneV := validators.NewModelIdOrNameValidator("zone", "zone", userProjId)
-		zoneV.Optional(true)
-		q, err = zoneV.QueryFilter(q, data)
-		if err != nil {
-			return nil, err
-		}
+	q, err = validators.ApplyModelFilters(q, data, []*validators.ModelFilterOptions{
+		{Key: "network", ModelKeyword: "network", ProjectId: userProjId},
+		{Key: "zone", ModelKeyword: "zone", ProjectId: userProjId},
+	})
+	if err != nil {
+		return nil, err
 	}
 	return q, nil
 }

pkg/compute/models/managedresource.go

@@ -1,10 +1,10 @@
 package models
 
 import (
-	"context"
 	"fmt"
 
-	"yunion.io/x/jsonutils"
+	"yunion.io/x/onecloud/pkg/appctx"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 )
 
@@ -30,6 +30,9 @@ func (self *SManagedResourceBase) GetCloudaccount() *SCloudaccount {
 func (self *SManagedResourceBase) GetDriver() (cloudprovider.ICloudProvider, error) {
 	provider := self.GetCloudprovider()
 	if provider == nil {
+		if len(self.ManagerId) > 0 {
+			return nil, cloudprovider.ErrInvalidProvider
+		}
 		return nil, fmt.Errorf("Resource is self managed")
 	}
 	return provider.GetDriver()
@@ -47,7 +50,7 @@ func (self *SManagedResourceBase) IsManaged() bool {
 	return len(self.ManagerId) > 0
 }
 
-func (self *SManagedResourceBase) getExtraDetails(ctx context.Context, extra *jsonutils.JSONDict) *jsonutils.JSONDict {
+/*func (self *SManagedResourceBase) getExtraDetails(ctx context.Context, extra *jsonutils.JSONDict) *jsonutils.JSONDict {
 	manager := self.GetCloudprovider()
 	if manager != nil {
 		extra.Add(jsonutils.NewString(manager.Name), "manager")
@@ -61,3 +64,72 @@ func (self *SManagedResourceBase) getExtraDetails(ctx context.Context, extra *js
 	}
 	return extra
 }
+*/
+
+type SCloudProviderInfo struct {
+	Provider         string `json:",omitempty"`
+	Account          string `json:",omitempty"`
+	AccountId        string `json:",omitempty"`
+	Manager          string `json:",omitempty"`
+	ManagerId        string `json:",omitempty"`
+	ManagerProject   string `json:",omitempty"`
+	ManagerProjectId string `json:",omitempty"`
+	Region           string `json:",omitempty"`
+	RegionId         string `json:",omitempty"`
+	RegionExtId      string `json:",omitempty"`
+	Zone             string `json:",omitempty"`
+	ZoneId           string `json:",omitempty"`
+	ZoneExtId        string `json:",omitempty"`
+}
+
+func MakeCloudProviderInfo(region *SCloudregion, zone *SZone, provider *SCloudprovider) SCloudProviderInfo {
+	info := SCloudProviderInfo{}
+
+	if zone != nil {
+		info.Zone = zone.GetName()
+		info.ZoneId = zone.GetId()
+	}
+
+	if region != nil {
+		info.Region = region.GetName()
+		info.RegionId = region.GetId()
+	}
+
+	if provider != nil {
+		info.Manager = provider.GetName()
+		info.ManagerId = provider.GetId()
+
+		if len(provider.ProjectId) > 0 {
+			info.ManagerProjectId = provider.ProjectId
+			tc, err := db.TenantCacheManager.FetchTenantById(appctx.Background, provider.ProjectId)
+			if err == nil {
+				info.ManagerProject = tc.GetName()
+			}
+		}
+
+		account := provider.GetCloudaccount()
+		info.Account = account.GetName()
+		info.AccountId = account.GetId()
+
+		driver, err := provider.GetDriver()
+
+		if err == nil {
+			info.Provider = driver.GetId()
+
+			if region != nil {
+				iregion, err := driver.GetIRegionById(region.ExternalId)
+				if err == nil {
+					info.RegionExtId = iregion.GetId()
+					if zone != nil {
+						izone, err := iregion.GetIZoneById(zone.ExternalId)
+						if err == nil {
+							info.ZoneExtId = izone.GetId()
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return info
+}

pkg/compute/models/networks.go

@@ -830,7 +830,7 @@ func (self *SNetwork) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSOND
 	extra.Add(jsonutils.NewInt(int64(self.GetGroupNicsCount())), "group_vnics")
 	extra.Add(jsonutils.NewInt(int64(self.GetReservedNicsCount())), "reserve_vnics")
 
-	zone := self.getZone()
+	/*zone := self.getZone()
 	if zone != nil {
 		extra.Add(jsonutils.NewString(zone.GetId()), "zone_id")
 		extra.Add(jsonutils.NewString(zone.GetName()), "zone")
@@ -846,14 +846,14 @@ func (self *SNetwork) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSOND
 		if len(region.GetExternalId()) > 0 {
 			extra.Add(jsonutils.NewString(region.GetExternalId()), "region_external_id")
 		}
-	}
+	}*/
 
 	vpc := self.getVpc()
 	if vpc != nil {
 		extra.Add(jsonutils.NewString(vpc.GetId()), "vpc_id")
 		extra.Add(jsonutils.NewString(vpc.GetName()), "vpc")
 		if len(vpc.GetExternalId()) > 0 {
-			extra.Add(jsonutils.NewString(vpc.GetExternalId()), "vpc_external_id")
+			extra.Add(jsonutils.NewString(vpc.GetExternalId()), "vpc_ext_id")
 		}
 	}
 	routes := self.GetRoutes()
@@ -861,6 +861,9 @@ func (self *SNetwork) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSOND
 		extra.Add(jsonutils.Marshal(routes), "routes")
 	}
 
+	info := vpc.getCloudProviderInfo()
+	extra.Update(jsonutils.Marshal(&info))
+
 	return extra
 }
 
@@ -1331,6 +1334,7 @@ func (manager *SNetworkManager) ListItemFilter(ctx context.Context, q *sqlchemy.
 	if err != nil {
 		return nil, err
 	}
+
 	zoneStr, _ := query.GetString("zone")
 	if len(zoneStr) > 0 {
 		zoneObj, err := ZoneManager.FetchByIdOrName(userCred, zoneStr)
@@ -1340,6 +1344,7 @@ func (manager *SNetworkManager) ListItemFilter(ctx context.Context, q *sqlchemy.
 		sq := WireManager.Query("id").Equals("zone_id", zoneObj.GetId())
 		q = q.Filter(sqlchemy.In(q.Field("wire_id"), sq.SubQuery()))
 	}
+
 	vpcStr, _ := query.GetString("vpc")
 	if len(vpcStr) > 0 {
 		vpcObj, err := VpcManager.FetchByIdOrName(userCred, vpcStr)
@@ -1349,6 +1354,7 @@ func (manager *SNetworkManager) ListItemFilter(ctx context.Context, q *sqlchemy.
 		sq := WireManager.Query("id").Equals("vpc_id", vpcObj.GetId())
 		q = q.Filter(sqlchemy.In(q.Field("wire_id"), sq.SubQuery()))
 	}
+
 	regionStr := jsonutils.GetAnyString(query, []string{"region_id", "region", "cloudregion_id", "cloudregion"})
 	if len(regionStr) > 0 {
 		region, err := CloudregionManager.FetchByIdOrName(userCred, regionStr)
@@ -1367,6 +1373,63 @@ func (manager *SNetworkManager) ListItemFilter(ctx context.Context, q *sqlchemy.
 				sqlchemy.Equals(wires.Field("vpc_id"), vpcs.Field("id"))))
 		q = q.Filter(sqlchemy.In(q.Field("wire_id"), sq.SubQuery()))
 	}
+
+	managerStr := jsonutils.GetAnyString(query, []string{"manager", "cloudprovider", "cloudprovider_id", "manager_id"})
+	if len(managerStr) > 0 {
+		provider, err := CloudproviderManager.FetchByIdOrName(nil, managerStr)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(CloudproviderManager.Keyword(), managerStr)
+			}
+			return nil, httperrors.NewGeneralError(err)
+		}
+
+		wires := WireManager.Query().SubQuery()
+		vpcs := VpcManager.Query().SubQuery()
+
+		subq := wires.Query(wires.Field("id"))
+		subq = subq.Join(vpcs, sqlchemy.Equals(vpcs.Field("id"), wires.Field("vpc_id")))
+		subq = subq.Filter(sqlchemy.Equals(vpcs.Field("manager_id"), provider.GetId()))
+
+		q = q.Filter(sqlchemy.In(q.Field("wire_id"), subq.SubQuery()))
+	}
+
+	accountStr := jsonutils.GetAnyString(query, []string{"account", "account_id", "cloudaccount", "cloudaccount_id"})
+	if len(accountStr) > 0 {
+		account, err := CloudaccountManager.FetchByIdOrName(nil, accountStr)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr)
+			}
+			return nil, httperrors.NewGeneralError(err)
+		}
+
+		wires := WireManager.Query().SubQuery()
+		vpcs := VpcManager.Query().SubQuery()
+		cloudproviders := CloudproviderManager.Query().SubQuery()
+
+		subq := wires.Query(wires.Field("id"))
+		subq = subq.Join(vpcs, sqlchemy.Equals(vpcs.Field("id"), wires.Field("vpc_id")))
+		subq = subq.Join(cloudproviders, sqlchemy.Equals(cloudproviders.Field("id"), vpcs.Field("manager_id")))
+		subq = subq.Filter(sqlchemy.Equals(cloudproviders.Field("cloudaccount_id"), account.GetId()))
+
+		q = q.Filter(sqlchemy.In(q.Field("wire_id"), subq.SubQuery()))
+	}
+
+	providerStr := jsonutils.GetAnyString(query, []string{"provider"})
+	if len(providerStr) > 0 {
+		wires := WireManager.Query().SubQuery()
+		vpcs := VpcManager.Query().SubQuery()
+		cloudproviders := CloudproviderManager.Query().SubQuery()
+
+		subq := wires.Query(wires.Field("id"))
+		subq = subq.Join(vpcs, sqlchemy.Equals(vpcs.Field("id"), wires.Field("vpc_id")))
+		subq = subq.Join(cloudproviders, sqlchemy.Equals(cloudproviders.Field("id"), vpcs.Field("manager_id")))
+		subq = subq.Filter(sqlchemy.Equals(cloudproviders.Field("provider"), providerStr))
+
+		q = q.Filter(sqlchemy.In(q.Field("wire_id"), subq.SubQuery()))
+	}
+
 	return q, nil
 }
 

pkg/compute/models/routetables.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"context"
+	"database/sql"
 	"net"
 	"reflect"
 	"strings"
@@ -108,14 +109,58 @@ func (man *SRouteTableManager) ListItemFilter(ctx context.Context, q *sqlchemy.S
 	}
 	userProjId := userCred.GetProjectId()
 	data := query.(*jsonutils.JSONDict)
-	for _, key := range []string{"vpc", "cloudregion"} {
-		v := validators.NewModelIdOrNameValidator(key, key, userProjId)
-		v.Optional(true)
-		q, err = v.QueryFilter(q, data)
-		if err != nil {
-			return nil, err
-		}
+	q, err = validators.ApplyModelFilters(q, data, []*validators.ModelFilterOptions{
+		{Key: "vpc", ModelKeyword: "vpc", ProjectId: userProjId},
+		{Key: "cloudregion", ModelKeyword: "cloudregion", ProjectId: userProjId},
+		{Key: "manager", ModelKeyword: "cloudprovider", ProjectId: userProjId},
+	})
+	if err != nil {
+		return nil, err
 	}
+
+	managerStr := jsonutils.GetAnyString(query, []string{"manager", "cloudprovider", "cloudprovider_id", "manager_id"})
+	if len(managerStr) > 0 {
+		provider, err := CloudproviderManager.FetchByIdOrName(nil, managerStr)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(CloudproviderManager.Keyword(), managerStr)
+			}
+			return nil, httperrors.NewGeneralError(err)
+		}
+		sq := VpcManager.Query("id").Equals("manager_id", provider.GetId())
+		q = q.In("vpc_id", sq.SubQuery())
+	}
+
+	accountStr := jsonutils.GetAnyString(query, []string{"account", "account_id", "cloudaccount", "cloudaccount_id"})
+	if len(accountStr) > 0 {
+		account, err := CloudaccountManager.FetchByIdOrName(nil, accountStr)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr)
+			}
+			return nil, httperrors.NewGeneralError(err)
+		}
+		vpcs := VpcManager.Query().SubQuery()
+		cloudproviders := CloudproviderManager.Query().SubQuery()
+
+		subq := vpcs.Query(vpcs.Field("id"))
+		subq = subq.Join(cloudproviders, sqlchemy.Equals(cloudproviders.Field("id"), vpcs.Field("manager_id")))
+		subq = subq.Filter(sqlchemy.Equals(cloudproviders.Field("cloudaccount_id"), account.GetId()))
+		q = q.Filter(sqlchemy.In(q.Field("vpc_id"), subq.SubQuery()))
+	}
+
+	providerStr := jsonutils.GetAnyString(query, []string{"provider"})
+	if len(providerStr) > 0 {
+		vpcs := VpcManager.Query().SubQuery()
+		cloudproviders := CloudproviderManager.Query().SubQuery()
+
+		subq := vpcs.Query(vpcs.Field("id"))
+		subq = subq.Join(cloudproviders, sqlchemy.Equals(cloudproviders.Field("id"), vpcs.Field("manager_id")))
+		subq = subq.Filter(sqlchemy.Equals(cloudproviders.Field("provider"), providerStr))
+
+		q = q.Filter(sqlchemy.In(q.Field("vpc_id"), subq.SubQuery()))
+	}
+
 	return q, nil
 }
 
@@ -150,6 +195,29 @@ func (man *SRouteTableManager) ValidateCreateData(ctx context.Context, userCred
 	return man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)
 }
 
+func (rt *SRouteTable) AllowPerformPurge(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return db.IsAdminAllowPerform(userCred, rt, "purge")
+}
+
+func (rt *SRouteTable) PerformPurge(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	err := rt.ValidateDeleteCondition(ctx)
+	if err != nil {
+		return nil, err
+	}
+	provider := rt.GetCloudprovider()
+	if provider != nil {
+		if provider.Enabled {
+			return nil, httperrors.NewInvalidStatusError("Cannot purge route_table on enabled cloud provider")
+		}
+	}
+	err = rt.RealDelete(ctx, userCred)
+	return nil, err
+}
+
+func (rt *SRouteTable) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	return rt.SVirtualResourceBase.Delete(ctx, userCred)
+}
+
 func (rt *SRouteTable) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	data, err := RouteTableManager.validateRoutes(data, true)
 	if err != nil {
@@ -232,6 +300,12 @@ func (rt *SRouteTable) PerformDelRoutes(ctx context.Context, userCred mcclient.T
 	return nil, nil
 }
 
+func (rt *SRouteTable) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict {
+	info := rt.getCloudProviderInfo()
+	extra.Update(jsonutils.Marshal(&info))
+	return extra
+}
+
 func (rt *SRouteTable) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
 	extra := rt.SVirtualResourceBase.GetCustomizeColumns(ctx, userCred, query)
 	vpcM, err := VpcManager.FetchById(rt.VpcId)
@@ -248,12 +322,14 @@ func (rt *SRouteTable) GetCustomizeColumns(ctx context.Context, userCred mcclien
 	}
 	extra.Set("vpc", jsonutils.NewString(vpcM.GetName()))
 	extra.Set("cloudregion", jsonutils.NewString(cloudregionM.GetName()))
+
+	extra = rt.getMoreDetails(extra)
 	return extra
 }
 
 func (rt *SRouteTable) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
 	extra := rt.GetCustomizeColumns(ctx, userCred, query)
-	extra = rt.SManagedResourceBase.getExtraDetails(ctx, extra)
+	extra = rt.getMoreDetails(extra)
 	return extra
 }
 
@@ -361,3 +437,26 @@ func (self *SRouteTable) SyncWithCloudRouteTable(userCred mcclient.TokenCredenti
 	}
 	return nil
 }
+
+func (self *SRouteTable) getVpc() (*SVpc, error) {
+	val, err := VpcManager.FetchById(self.VpcId)
+	if err != nil {
+		log.Errorf("VpcManager.FetchById fail %s", err)
+		return nil, err
+	}
+	return val.(*SVpc), nil
+}
+
+func (self *SRouteTable) getRegion() (*SCloudregion, error) {
+	vpc, err := self.getVpc()
+	if err != nil {
+		return nil, err
+	}
+	return vpc.GetRegion()
+}
+
+func (self *SRouteTable) getCloudProviderInfo() SCloudProviderInfo {
+	region, _ := self.getRegion()
+	provider := self.GetCloudprovider()
+	return MakeCloudProviderInfo(region, nil, provider)
+}

pkg/compute/models/schedpolicies.go

@@ -2,16 +2,16 @@ package models
 
 import (
 	"context"
+	"database/sql"
 
 	"yunion.io/x/jsonutils"
-
-	"database/sql"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/utils"
+
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/conditionparser"
-	"yunion.io/x/pkg/utils"
 )
 
 type SSchedpolicyManager struct {

pkg/compute/models/schedtags.go

@@ -7,11 +7,12 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/utils"
+	"yunion.io/x/sqlchemy"
+
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
-	"yunion.io/x/pkg/utils"
-	"yunion.io/x/sqlchemy"
 )
 
 type SchedStrategyType string

pkg/compute/models/secgroupcache.go

@@ -6,13 +6,14 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/util/stringutils"
+	"yunion.io/x/sqlchemy"
+
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
-	"yunion.io/x/pkg/util/stringutils"
-	"yunion.io/x/sqlchemy"
 )
 
 type SSecurityGroupCacheManager struct {
@@ -123,6 +124,23 @@ func (manager *SSecurityGroupCacheManager) GetSecgroupCache(ctx context.Context,
 	return &secgroupCache
 }
 
+func (manager *SSecurityGroupCacheManager) CheckExist(ctx context.Context, userCred mcclient.TokenCredential, externalId, vpcId, regionId string, providerId string) (*SSecurityGroup, bool) {
+	secgroupCaches := []SSecurityGroupCache{}
+	query := manager.Query()
+	cond := sqlchemy.AND(sqlchemy.Equals(query.Field("external_id"), externalId), sqlchemy.Equals(query.Field("vpc_id"), vpcId), sqlchemy.Equals(query.Field("cloudregion_id"), regionId), sqlchemy.Equals(query.Field("manager_id"), providerId))
+	query = query.Filter(cond)
+
+	if err := query.All(&secgroupCaches); err != nil {
+		return nil, false
+	}
+	for _, secgroupCache := range secgroupCaches {
+		if secgroup, err := SecurityGroupManager.FetchById(secgroupCache.SecgroupId); err == nil {
+			return secgroup.(*SSecurityGroup), true
+		}
+	}
+	return nil, false
+}
+
 func (manager *SSecurityGroupCacheManager) Register(ctx context.Context, userCred mcclient.TokenCredential, secgroupId, vpcId, regionId string, providerId string) *SSecurityGroupCache {
 	lockman.LockClass(ctx, manager, userCred.GetProjectId())
 	defer lockman.ReleaseClass(ctx, manager, userCred.GetProjectId())

pkg/compute/models/secgrouprules.go

@@ -9,14 +9,15 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
-	"yunion.io/x/onecloud/pkg/cloudcommon/db"
-	"yunion.io/x/onecloud/pkg/httperrors"
-	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/pkg/util/compare"
 	"yunion.io/x/pkg/util/regutils"
 	"yunion.io/x/pkg/util/secrules"
 	"yunion.io/x/pkg/util/stringutils"
 	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
 type SSecurityGroupRuleManager struct {

pkg/compute/models/secgroups.go

@@ -241,7 +241,12 @@ func (self *SSecurityGroup) SyncWithCloudSecurityGroup(userCred mcclient.TokenCr
 	return nil
 }
 
-func (manager *SSecurityGroupManager) newFromCloudVpc(userCred mcclient.TokenCredential, extSec cloudprovider.ICloudSecurityGroup, vpc *SVpc, projectId string) (*SSecurityGroup, error) {
+func (manager *SSecurityGroupManager) newFromCloudVpc(userCred mcclient.TokenCredential, extSec cloudprovider.ICloudSecurityGroup, vpc *SVpc, projectId string) (*SSecurityGroup, bool, error) {
+	if secgroup, exist := SecurityGroupCacheManager.CheckExist(context.Background(), userCred, extSec.GetGlobalId(), extSec.GetVpcId(), vpc.CloudregionId, vpc.ManagerId); exist {
+		//避免重复同步
+		return secgroup, true, nil
+	}
+
 	secgroup := SSecurityGroup{}
 	secgroup.SetModelManager(manager)
 	secgroup.Name = extSec.GetName()
@@ -253,7 +258,7 @@ func (manager *SSecurityGroupManager) newFromCloudVpc(userCred mcclient.TokenCre
 	}
 
 	if err := manager.TableSpec().Insert(&secgroup); err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
 	if secgroupcache := SecurityGroupCacheManager.Register(context.Background(), userCred, secgroup.Id, extSec.GetVpcId(), vpc.CloudregionId, vpc.ManagerId); secgroupcache != nil {
@@ -262,7 +267,7 @@ func (manager *SSecurityGroupManager) newFromCloudVpc(userCred mcclient.TokenCre
 		}
 	}
 
-	return &secgroup, nil
+	return &secgroup, false, nil
 }
 
 func (manager *SSecurityGroupManager) SyncSecgroups(ctx context.Context, userCred mcclient.TokenCredential, secgroups []cloudprovider.ICloudSecurityGroup, vpc *SVpc, projectId string, projectSync bool) ([]SSecurityGroup, []cloudprovider.ICloudSecurityGroup, compare.SyncResult) {
@@ -306,11 +311,14 @@ func (manager *SSecurityGroupManager) SyncSecgroups(ctx context.Context, userCre
 			syncResult.AddError(err)
 			continue
 		}
-		new, err := manager.newFromCloudVpc(userCred, added[i], vpc, projectId)
+		new, exist, err := manager.newFromCloudVpc(userCred, added[i], vpc, projectId)
 		if err != nil {
 			syncResult.AddError(err)
 			continue
 		}
+		if exist {
+			continue
+		}
 		localSecgroups = append(localSecgroups, *new)
 		remoteSecgroups = append(remoteSecgroups, added[i])
 		SecurityGroupRuleManager.SyncRules(ctx, userCred, new, rules)

pkg/compute/models/skus.go

@@ -3,7 +3,7 @@ package models
 import (
 	"context"
 	"database/sql"
-	"encoding/json"
+	"fmt"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -25,6 +25,17 @@ const (
 	SkuCategoryHighMemory          = "high_memory"          // 高内存型
 )
 
+var InstanceFamilies = map[string]string{
+	SkuCategoryGeneralPurpose:      "g1",
+	SkuCategoryBurstable:           "t1",
+	SkuCategoryComputeOptimized:    "c1",
+	SkuCategoryMemoryOptimized:     "r1",
+	SkuCategoryStorageIOOptimized:  "i1",
+	SkuCategoryHardwareAccelerated: "",
+	SkuCategoryHighStorage:         "hc1",
+	SkuCategoryHighMemory:          "hr1",
+}
+
 type SServerSkuManager struct {
 	db.SStandaloneResourceBaseManager
 }
@@ -54,10 +65,10 @@ type SServerSku struct {
 	CpuCoreCount int `nullable:"false" list:"user" create:"admin_required" update:"admin"`
 	MemorySizeMB int `nullable:"false" list:"user" create:"admin_required" update:"admin"`
 
-	OsName string `width:"32" charset:"ascii" nullable:"false" list:"user" create:"admin_required" update:"admin" default:"Any"` // Windows|Linux|Any
+	OsName string `width:"32" charset:"ascii" nullable:"false" list:"user" create:"admin_optional" update:"admin" default:"Any"` // Windows|Linux|Any
 
 	SysDiskResizable bool   `default:"true" nullable:"false" list:"user" create:"admin_optional" update:"admin"`
-	SysDiskType      string `width:"32" charset:"ascii" nullable:"false" list:"user" create:"admin_required" update:"admin"`
+	SysDiskType      string `width:"32" charset:"ascii" nullable:"false" list:"user" create:"admin_optional" update:"admin"`
 	SysDiskMinSizeGB int    `nullable:"false" list:"user" create:"admin_optional" update:"admin"` // not required。 windows比较新的版本都是50G左右。
 	SysDiskMaxSizeGB int    `nullable:"false" list:"user" create:"admin_optional" update:"admin"` // not required
 
@@ -82,18 +93,26 @@ type SServerSku struct {
 }
 
 func inWhiteList(provider string) bool {
-	// 只有为true的hypervisor才进行创建和更新操作
+	// provider 字段为空时表示私有云套餐
 	if len(provider) == 0 {
 		return true
-	}
-	switch provider {
-	case HYPERVISOR_ESXI, HYPERVISOR_KVM:
-		return true
-	default:
+	} else {
 		return false
 	}
 }
 
+func genInstanceType(family string, cpu, mem_mb int64) (string, error) {
+	if cpu <= 0 {
+		return "", fmt.Errorf("cpu_core_count should great than zero")
+	}
+
+	if mem_mb <= 0 || mem_mb%1024 != 0 {
+		return "", fmt.Errorf("memory_size_mb should great than zero. and should be integral multiple of 1024")
+	}
+
+	return fmt.Sprintf("ecs.%s.c%dm%d", family, cpu, mem_mb/1024), nil
+}
+
 func (self *SServerSkuManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
 	return true
 }
@@ -119,6 +138,8 @@ func (self *SServerSkuManager) ValidateCreateData(ctx context.Context,
 		return nil, httperrors.NewForbiddenError("can not create instance_type for public cloud %s", provider)
 	}
 
+	data.Remove("provider")
+
 	regionStr := jsonutils.GetAnyString(data, []string{"region", "region_id", "cloudregion", "cloudregion_id"})
 	if len(regionStr) > 0 {
 		regionObj, err := CloudregionManager.FetchByIdOrName(userCred, regionStr)
@@ -143,6 +164,47 @@ func (self *SServerSkuManager) ValidateCreateData(ctx context.Context,
 		}
 		data.Add(jsonutils.NewString(zoneObj.GetId()), "zone_id")
 	}
+
+	// name 由服务器端生成
+	cpu, err := data.Int("cpu_core_count")
+	if err != nil {
+		return nil, httperrors.NewInputParameterError("cpu_core_count should not be empty")
+	} else {
+		data.Set("cpu_core_count", jsonutils.NewInt(cpu))
+	}
+
+	mem, err := data.Int("memory_size_mb")
+	if err != nil {
+		return nil, httperrors.NewInputParameterError("memory_size_mb should not be empty")
+	} else {
+		data.Set("memory_size_mb", jsonutils.NewInt(mem))
+	}
+
+	category, _ := data.GetString("instance_type_category")
+	family, exists := InstanceFamilies[category]
+	if !exists {
+		return nil, httperrors.NewInputParameterError("instance_type_category %s is invalid", category)
+	}
+
+	data.Set("instance_type_family", jsonutils.NewString(family))
+	// 格式 ecs.g1.c1m1
+	name, err := genInstanceType(family, cpu, mem)
+	if err != nil {
+		return nil, httperrors.NewInputParameterError(err.Error())
+	}
+
+	data.Set("name", jsonutils.NewString(name))
+
+	q := self.Query()
+	q = q.Equals("name", name).Filter(sqlchemy.OR(
+		sqlchemy.IsNull(q.Field("provider")),
+		sqlchemy.IsEmpty(q.Field("provider")),
+	))
+
+	if q.Count() > 0 {
+		return nil, httperrors.NewDuplicateResourceError("Duplicate sku %s", name)
+	}
+
 	return self.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)
 }
 
@@ -182,17 +244,38 @@ func (self *SServerSkuManager) AllowGetPropertyInstanceSpecs(ctx context.Context
 
 func (self *SServerSkuManager) GetPropertyInstanceSpecs(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	q := self.Query()
-	zone, err := query.GetString("zone")
-	if err == nil && len(zone) > 0 {
-		q = q.Equals("zone_id", zone)
+	provider, _ := query.GetString("provider")
+	if inWhiteList(provider) {
+		q = q.Filter(sqlchemy.OR(
+			sqlchemy.IsNull(q.Field("provider")),
+			sqlchemy.IsEmpty(q.Field("provider")),
+		))
 	} else {
-		return nil, httperrors.NewMissingParameterError("zone")
+		q = q.Equals("provider", provider)
+	}
+
+	// 如果是查询私有云需要忽略zone参数
+	zone := jsonutils.GetAnyString(query, []string{"zone", "zone_id"})
+	if !inWhiteList(provider) {
+		if len(zone) > 0 {
+			zoneObj, err := ZoneManager.FetchByIdOrName(userCred, zone)
+			if err != nil {
+				if err == sql.ErrNoRows {
+					return nil, httperrors.NewResourceNotFoundError2(ZoneManager.Keyword(), zone)
+				}
+				return nil, httperrors.NewGeneralError(err)
+			}
+
+			q = q.Equals("zone_id", zoneObj.GetId())
+		} else {
+			return nil, httperrors.NewMissingParameterError("zone")
+		}
 	}
 
 	skus := make([]SServerSku, 0)
 	q = q.GroupBy(q.Field("cpu_core_count"), q.Field("memory_size_mb"))
 	q = q.Asc(q.Field("cpu_core_count"), q.Field("memory_size_mb"))
-	err = q.All(&skus)
+	err := q.All(&skus)
 	if err != nil {
 		log.Errorf("%s", err)
 		return nil, httperrors.NewBadRequestError("instance specs list query error")
@@ -228,17 +311,17 @@ func (self *SServerSkuManager) GetPropertyInstanceSpecs(ctx context.Context, use
 	ret.Add(cpus, "cpus")
 	ret.Add(mems_mb, "mems_mb")
 
-	r, err := json.Marshal(&cpu_mems_mb)
+	/* r, err := json.Marshal(&cpu_mems_mb)
 	if err != nil {
 		log.Errorf("%s", err)
 		return nil, httperrors.NewInternalServerError("instance specs list marshal failed")
-	}
+	}*/
 
-	r_obj, err := jsonutils.Parse(r)
-	if err != nil {
+	r_obj := jsonutils.Marshal(&cpu_mems_mb)
+	/*if err != nil {
 		log.Errorf("%s", err)
 		return nil, httperrors.NewInternalServerError("instance specs list parse failed")
-	}
+	}*/
 
 	ret.Add(r_obj, "cpu_mems_mb")
 	return ret, nil
@@ -255,13 +338,14 @@ func (self *SServerSku) ValidateUpdateData(
 	data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 
 	if !inWhiteList(self.Provider) {
-		return nil, httperrors.NewForbiddenError("can not create instance_type for public cloud %s", self.Provider)
+		return nil, httperrors.NewForbiddenError("can not update instance_type for public cloud %s", self.Provider)
 	}
 
 	provider, err := data.GetString("provider")
 	if err == nil && !inWhiteList(provider) {
-		return nil, httperrors.NewForbiddenError("can not create instance_type for public cloud %s", provider)
+		return nil, httperrors.NewForbiddenError("can not update instance_type for public cloud %s", provider)
 	}
+	data.Remove("provider")
 
 	zoneStr := jsonutils.GetAnyString(data, []string{"zone", "zone_id"})
 	if len(zoneStr) > 0 {
@@ -274,6 +358,52 @@ func (self *SServerSku) ValidateUpdateData(
 		}
 		data.Add(jsonutils.NewString(zoneObj.GetId()), "zone_id")
 	}
+
+	// name 由服务器端生成
+	cpu, err := data.Int("cpu_core_count")
+	if err != nil {
+		cpu = int64(self.CpuCoreCount)
+	}
+	data.Set("cpu_core_count", jsonutils.NewInt(cpu))
+
+	mem, err := data.Int("memory_size_mb")
+	if err != nil {
+		mem = int64(self.MemorySizeMB)
+	}
+	data.Set("memory_size_mb", jsonutils.NewInt(mem))
+
+	category, err := data.GetString("instance_type_category")
+	family := ""
+	if err != nil {
+		family = self.InstanceTypeFamily
+	} else {
+		f, exists := InstanceFamilies[category]
+		if !exists {
+			return nil, httperrors.NewInputParameterError("instance_type_category %s is invalid", category)
+		}
+
+		family = f
+	}
+
+	data.Set("instance_type_family", jsonutils.NewString(family))
+	// 格式 ecs.g1.c1m1
+	name, err := genInstanceType(family, cpu, mem)
+	if err != nil {
+		return nil, httperrors.NewInputParameterError(err.Error())
+	}
+
+	data.Set("name", jsonutils.NewString(name))
+
+	q := self.GetModelManager().Query()
+	q = q.Equals("name", name).Filter(sqlchemy.OR(
+		sqlchemy.IsNull(q.Field("provider")),
+		sqlchemy.IsEmpty(q.Field("provider")),
+	))
+
+	if q.Count() > 0 {
+		return nil, httperrors.NewDuplicateResourceError("sku cpu %d mem %d(Mb) already exists", cpu, mem)
+	}
+
 	return self.SStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, data)
 }
 
@@ -303,20 +433,18 @@ func (self *SServerSku) GetZoneExternalId() (string, error) {
 }
 
 func (manager *SServerSkuManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
-	queryDict := query.(*jsonutils.JSONDict)
-
 	provider := jsonutils.GetAnyString(query, []string{"provider"})
-	if len(provider) > 0 {
-		if provider != "all" {
-			q = q.Equals("provider", provider)
-		}
-
-		queryDict.Remove("provider")
-	} else {
+	queryDict := query.(*jsonutils.JSONDict)
+	if provider == "" {
 		q = q.Filter(sqlchemy.OR(
 			sqlchemy.IsNull(q.Field("provider")),
 			sqlchemy.IsEmpty(q.Field("provider")),
 		))
+	} else if provider == "all" {
+		// provider 参数为all时。表示查询所有instance type.
+		queryDict.Remove("provider")
+	} else {
+		q = q.Equals("provider", provider)
 	}
 
 	q, err := manager.SStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query)
@@ -336,8 +464,9 @@ func (manager *SServerSkuManager) ListItemFilter(ctx context.Context, q *sqlchem
 		q = q.Equals("cloudregion_id", regionObj.GetId())
 	}
 
+	// 当查询私有云时，需要忽略zone参数
 	zoneStr := jsonutils.GetAnyString(query, []string{"zone", "zone_id"})
-	if len(zoneStr) > 0 {
+	if !inWhiteList(provider) && len(zoneStr) > 0 {
 		zoneObj, err := ZoneManager.FetchByIdOrName(nil, zoneStr)
 		if err != nil {
 			if err == sql.ErrNoRows {
@@ -346,6 +475,9 @@ func (manager *SServerSkuManager) ListItemFilter(ctx context.Context, q *sqlchem
 			return nil, httperrors.NewGeneralError(err)
 		}
 		q = q.Equals("zone_id", zoneObj.GetId())
+	} else {
+		queryDict.Remove("zone")
+		queryDict.Remove("zone_id")
 	}
 
 	return q, err

pkg/compute/models/snapshots.go

@@ -49,6 +49,9 @@ type SSnapshot struct {
 	FakeDeleted bool   `nullable:"false" default:"false" index:"true"`
 	DiskType    string `width:"32" charset:"ascii" nullable:"true" list:"user"`
 
+	// create disk from snapshot, snapshot as disk backing file
+	RefCount int `nullable:"false" default:"0" list:"user"`
+
 	CloudregionId string `width:"36" charset:"ascii" nullable:"true" list:"user"`
 }
 
@@ -138,6 +141,19 @@ func (manager *SSnapshotManager) ListItemFilter(ctx context.Context, q *sqlchemy
 		q = q.Equals("manager_id", managerObj.GetId())
 	}
 
+	accountStr := jsonutils.GetAnyString(query, []string{"account", "account_id", "cloudaccount", "cloudaccount_id"})
+	if len(accountStr) > 0 {
+		account, err := CloudaccountManager.FetchByIdOrName(nil, accountStr)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr)
+			}
+			return nil, httperrors.NewGeneralError(err)
+		}
+		subq := CloudproviderManager.Query("id").Equals("cloudaccount_id", account.GetId()).SubQuery()
+		q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq))
+	}
+
 	return q, nil
 }
 
@@ -155,13 +171,13 @@ func (self *SSnapshot) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSON
 	if IStorage, _ := StorageManager.FetchById(self.StorageId); IStorage != nil {
 		storage := IStorage.(*SStorage)
 		extra.Add(jsonutils.NewString(storage.StorageType), "storage_type")
-		if provider := storage.GetCloudprovider(); provider != nil {
-			extra.Add(jsonutils.NewString(provider.Name), "provider")
-		}
+		// if provider := storage.GetCloudprovider(); provider != nil {
+		// 	extra.Add(jsonutils.NewString(provider.Name), "provider")
+		// }
 	} else {
-		if cloudprovider := self.GetCloudprovider(); cloudprovider != nil {
-			extra.Add(jsonutils.NewString(cloudprovider.Provider), "provider")
-		}
+		// if cloudprovider := self.GetCloudprovider(); cloudprovider != nil {
+		// 	extra.Add(jsonutils.NewString(cloudprovider.Provider), "provider")
+		// }
 	}
 	disk, _ := self.GetDisk()
 	if disk != nil {
@@ -174,6 +190,10 @@ func (self *SSnapshot) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSON
 		}
 		extra.Add(jsonutils.NewString(disk.Name), "disk_name")
 	}
+
+	info := self.getCloudProviderInfo()
+	extra.Update(jsonutils.Marshal(&info))
+
 	return extra
 }
 
@@ -189,9 +209,8 @@ func (self *SSnapshot) GetShortDesc() *jsonutils.JSONDict {
 			res.Add(jsonutils.NewString(cloudRegion.ExternalId), "region")
 		}
 	}*/
-	info := self.getCloudBillingInfo()
+	info := self.getCloudProviderInfo()
 	res.Update(jsonutils.Marshal(&info))
-
 	return res
 }
 
@@ -246,6 +265,20 @@ func (self *SSnapshot) GetHost() *SHost {
 	return storage.GetMasterHost()
 }
 
+func (self *SSnapshotManager) AddRefCount(snapshotId string, count int) {
+	iSnapshot, _ := self.FetchById(snapshotId)
+	if iSnapshot != nil {
+		snapshot := iSnapshot.(*SSnapshot)
+		_, err := self.TableSpec().Update(snapshot, func() error {
+			snapshot.RefCount += count
+			return nil
+		})
+		if err != nil {
+			log.Errorf("Snapshot add refence count error: %s", err)
+		}
+	}
+}
+
 func (self *SSnapshotManager) GetDiskSnapshotsByCreate(diskId, createdBy string) []SSnapshot {
 	dest := make([]SSnapshot, 0)
 	q := self.Query().SubQuery()
@@ -343,6 +376,9 @@ func (self *SSnapshot) StartSnapshotDeleteTask(ctx context.Context, userCred mcc
 }
 
 func (self *SSnapshot) ValidateDeleteCondition(ctx context.Context) error {
+	if self.RefCount > 0 {
+		return fmt.Errorf("Snapshot reference(by disk) count > 0, can not delete")
+	}
 	return nil
 }
 
@@ -628,8 +664,8 @@ func (self *SSnapshot) PerformPurge(ctx context.Context, userCred mcclient.Token
 	return nil, err
 }
 
-func (self *SSnapshot) getCloudBillingInfo() SCloudBillingInfo {
+func (self *SSnapshot) getCloudProviderInfo() SCloudProviderInfo {
 	region := self.GetRegion()
 	provider := self.GetCloudprovider()
-	return MakeCloudBillingInfo(region, nil, provider)
+	return MakeCloudProviderInfo(region, nil, provider)
 }

pkg/compute/models/storagecachedimages.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/serialx/hashring"
+
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/utils"

pkg/compute/models/storagecaches.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 
 	"github.com/serialx/hashring"
+
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/sqlchemy"

pkg/compute/models/storages.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"path"
 	"strings"
@@ -446,6 +447,10 @@ func (self *SStorage) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSOND
 		extra.Add(jsonutils.NewFloat(0.0), "commit_rate")
 	}
 	extra.Add(jsonutils.NewFloat(float64(self.GetOvercommitBound())), "commit_bound")
+
+	info := self.getCloudProviderInfo()
+	extra.Update(jsonutils.Marshal(&info))
+
 	return extra
 }
 
@@ -454,6 +459,11 @@ func (self *SStorage) GetCustomizeColumns(ctx context.Context, userCred mcclient
 	return self.getMoreDetails(extra)
 }
 
+func (self *SStorage) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
+	extra := self.SStandaloneResourceBase.GetExtraDetails(ctx, userCred, query)
+	return self.getMoreDetails(extra)
+}
+
 func (self *SStorage) GetUsedCapacity(isReady tristate.TriState) int {
 	disks := DiskManager.Query().SubQuery()
 	q := disks.Query(sqlchemy.SUM("sum", disks.Field("disk_size"))).Equals("storage_id", self.Id)
@@ -736,6 +746,7 @@ func (manager *SStorageManager) newFromCloudStorage(extStorage cloudprovider.ICl
 	storage.MediumType = extStorage.GetMediumType()
 	storage.StorageConf = extStorage.GetStorageConf()
 	storage.Capacity = extStorage.GetCapacityMB()
+	storage.Cmtbound = 1.0
 
 	storage.Enabled = extStorage.GetEnabled()
 
@@ -1112,15 +1123,37 @@ func (manager *SStorageManager) ListItemFilter(ctx context.Context, q *sqlchemy.
 			Filter(sqlchemy.IsTrue(q.Field("enabled")))
 	}
 
-	managerStr := jsonutils.GetAnyString(query, []string{"manager", "provider", "manager_id", "provider_id"})
+	managerStr := jsonutils.GetAnyString(query, []string{"manager", "cloudprovider", "cloudprovider_id", "manager_id"})
 	if len(managerStr) > 0 {
-		provider := CloudproviderManager.FetchCloudproviderByIdOrName(managerStr)
-		if provider == nil {
-			return nil, httperrors.NewResourceNotFoundError("provider %s not found", managerStr)
+		provider, err := CloudproviderManager.FetchByIdOrName(nil, managerStr)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(CloudproviderManager.Keyword(), managerStr)
+			}
+			return nil, httperrors.NewGeneralError(err)
 		}
 		q = q.Filter(sqlchemy.Equals(q.Field("manager_id"), provider.GetId()))
 	}
 
+	accountStr := jsonutils.GetAnyString(query, []string{"account", "account_id", "cloudaccount", "cloudaccount_id"})
+	if len(accountStr) > 0 {
+		account, err := CloudaccountManager.FetchByIdOrName(nil, accountStr)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr)
+			}
+			return nil, httperrors.NewGeneralError(err)
+		}
+		subq := CloudproviderManager.Query("id").Equals("cloudaccount_id", account.GetId()).SubQuery()
+		q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq))
+	}
+
+	providerStr := jsonutils.GetAnyString(query, []string{"provider"})
+	if len(providerStr) > 0 {
+		subq := CloudproviderManager.Query("id").Equals("provider", providerStr).SubQuery()
+		q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq))
+	}
+
 	return q, err
 }
 
@@ -1141,17 +1174,17 @@ func (self *SStorage) ClearSchedDescCache() error {
 	return nil
 }
 
-func (self *SStorage) getCloudBillingInfo() SCloudBillingInfo {
+func (self *SStorage) getCloudProviderInfo() SCloudProviderInfo {
 	var region *SCloudregion
 	zone := self.getZone()
 	if zone != nil {
 		region = zone.GetRegion()
 	}
 	provider := self.GetCloudprovider()
-	return MakeCloudBillingInfo(region, zone, provider)
+	return MakeCloudProviderInfo(region, zone, provider)
 }
 
 func (self *SStorage) GetShortDesc() *jsonutils.JSONDict {
-	info := self.getCloudBillingInfo()
+	info := self.getCloudProviderInfo()
 	return jsonutils.Marshal(&info).(*jsonutils.JSONDict)
 }

pkg/compute/models/usage.go

@@ -1,10 +1,10 @@
 package models
 
 import (
+	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
-	"yunion.io/x/pkg/utils"
 )
 
 func AttachUsageQuery(

pkg/compute/models/vcenters.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"yunion.io/x/jsonutils"
+
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )

pkg/compute/models/vpcs.go

@@ -102,7 +102,7 @@ func (self *SVpc) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCr
 }
 
 func (self *SVpc) ValidateDeleteCondition(ctx context.Context) error {
-	if self.GetNetworkCount() > 0 {
+	if self.GetNetworkCount() > 0 || self.GetRouteTableCount() > 0 {
 		return httperrors.NewNotEmptyError("VPC not empty")
 	}
 	if self.Id == DEFAULT_VPC_ID {
@@ -150,10 +150,15 @@ func (self *SVpc) GetNetworkCount() int {
 	return q.Count()
 }
 
+func (self *SVpc) GetRouteTableCount() int {
+	return RouteTableManager.Query().Equals("vpc_id", self.Id).Count()
+}
+
 func (self *SVpc) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict {
 	extra.Add(jsonutils.NewInt(int64(self.GetWireCount())), "wire_count")
 	extra.Add(jsonutils.NewInt(int64(self.GetNetworkCount())), "network_count")
-	region, err := self.GetRegion()
+	extra.Add(jsonutils.NewInt(int64(self.GetRouteTableCount())), "routetable_count")
+	/* region, err := self.GetRegion()
 	if err != nil {
 		log.Errorf("failed getting region for vpc %s(%s)", self.Name, self.Id)
 		return extra
@@ -161,10 +166,20 @@ func (self *SVpc) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict
 	extra.Add(jsonutils.NewString(region.GetName()), "region")
 	if len(region.GetExternalId()) > 0 {
 		extra.Add(jsonutils.NewString(region.GetExternalId()), "region_external_id")
-	}
+	}*/
+
+	info := self.getCloudProviderInfo()
+	extra.Update(jsonutils.Marshal(&info))
+
 	return extra
 }
 
+func (self *SVpc) getCloudProviderInfo() SCloudProviderInfo {
+	region, _ := self.GetRegion()
+	provider := self.GetCloudprovider()
+	return MakeCloudProviderInfo(region, nil, provider)
+}
+
 func (self *SVpc) GetRegion() (*SCloudregion, error) {
 	region, err := CloudregionManager.FetchById(self.CloudregionId)
 	if err != nil {
@@ -510,18 +525,43 @@ func (self *SVpc) PerformPurge(ctx context.Context, userCred mcclient.TokenCrede
 }
 
 func (manager *SVpcManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
+	queryDict := query.(*jsonutils.JSONDict)
+
+	managerStr := jsonutils.GetAnyString(query, []string{"manager", "cloudprovider", "cloudprovider_id", "manager_id"})
+	if len(managerStr) > 0 {
+		provider, err := CloudproviderManager.FetchByIdOrName(nil, managerStr)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(CloudproviderManager.Keyword(), managerStr)
+			}
+			return nil, httperrors.NewGeneralError(err)
+		}
+		q = q.Filter(sqlchemy.Equals(q.Field("manager_id"), provider.GetId()))
+		queryDict.Remove("manager_id")
+	}
+
 	q, err := manager.SStatusStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query)
 	if err != nil {
 		return nil, err
 	}
 
-	managerStr := jsonutils.GetAnyString(query, []string{"manager", "provider", "manager_id", "provider_id"})
-	if len(managerStr) > 0 {
-		provider := CloudproviderManager.FetchCloudproviderByIdOrName(managerStr)
-		if provider == nil {
-			return nil, httperrors.NewResourceNotFoundError("provider %s not found", managerStr)
+	accountStr := jsonutils.GetAnyString(query, []string{"account", "account_id", "cloudaccount", "cloudaccount_id"})
+	if len(accountStr) > 0 {
+		account, err := CloudaccountManager.FetchByIdOrName(nil, accountStr)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr)
+			}
+			return nil, httperrors.NewGeneralError(err)
 		}
-		q = q.Filter(sqlchemy.Equals(q.Field("manager_id"), provider.GetId()))
+		subq := CloudproviderManager.Query("id").Equals("cloudaccount_id", account.GetId()).SubQuery()
+		q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq))
+	}
+
+	providerStr := jsonutils.GetAnyString(query, []string{"provider"})
+	if len(providerStr) > 0 {
+		subq := CloudproviderManager.Query("id").Equals("provider", providerStr).SubQuery()
+		q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq))
 	}
 
 	return q, nil

pkg/compute/models/wires.go

@@ -7,14 +7,15 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
-	"yunion.io/x/onecloud/pkg/cloudcommon/db"
-	"yunion.io/x/onecloud/pkg/cloudprovider"
-	"yunion.io/x/onecloud/pkg/httperrors"
-	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/pkg/tristate"
 	"yunion.io/x/pkg/util/compare"
 	"yunion.io/x/pkg/util/netutils"
 	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
 type SWireManager struct {
@@ -632,6 +633,49 @@ func (manager *SWireManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu
 		q = q.In("vpc_id", sq.SubQuery())
 	}
 
+	managerStr := jsonutils.GetAnyString(query, []string{"manager", "cloudprovider", "cloudprovider_id", "manager_id"})
+	if len(managerStr) > 0 {
+		provider, err := CloudproviderManager.FetchByIdOrName(nil, managerStr)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(CloudproviderManager.Keyword(), managerStr)
+			}
+			return nil, httperrors.NewGeneralError(err)
+		}
+		sq := VpcManager.Query("id").Equals("manager_id", provider.GetId())
+		q = q.In("vpc_id", sq.SubQuery())
+	}
+
+	accountStr := jsonutils.GetAnyString(query, []string{"account", "account_id", "cloudaccount", "cloudaccount_id"})
+	if len(accountStr) > 0 {
+		account, err := CloudaccountManager.FetchByIdOrName(nil, accountStr)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr)
+			}
+			return nil, httperrors.NewGeneralError(err)
+		}
+		vpcs := VpcManager.Query().SubQuery()
+		cloudproviders := CloudproviderManager.Query().SubQuery()
+
+		subq := vpcs.Query(vpcs.Field("id"))
+		subq = subq.Join(cloudproviders, sqlchemy.Equals(cloudproviders.Field("id"), vpcs.Field("manager_id")))
+		subq = subq.Filter(sqlchemy.Equals(cloudproviders.Field("cloudaccount_id"), account.GetId()))
+		q = q.Filter(sqlchemy.In(q.Field("vpc_id"), subq.SubQuery()))
+	}
+
+	providerStr := jsonutils.GetAnyString(query, []string{"provider"})
+	if len(providerStr) > 0 {
+		vpcs := VpcManager.Query().SubQuery()
+		cloudproviders := CloudproviderManager.Query().SubQuery()
+
+		subq := vpcs.Query(vpcs.Field("id"))
+		subq = subq.Join(cloudproviders, sqlchemy.Equals(cloudproviders.Field("id"), vpcs.Field("manager_id")))
+		subq = subq.Filter(sqlchemy.Equals(cloudproviders.Field("provider"), providerStr))
+
+		q = q.Filter(sqlchemy.In(q.Field("vpc_id"), subq.SubQuery()))
+	}
+
 	return q, err
 }
 
@@ -655,7 +699,7 @@ func (self *SWire) GetExtraDetails(ctx context.Context, userCred mcclient.TokenC
 
 func (self *SWire) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict {
 	extra.Add(jsonutils.NewInt(int64(self.NetworkCount())), "networks")
-	zone := self.GetZone()
+	/*zone := self.GetZone()
 	if zone != nil {
 		extra.Add(jsonutils.NewString(zone.GetName()), "zone")
 		if len(zone.GetExternalId()) > 0 {
@@ -669,13 +713,17 @@ func (self *SWire) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict
 		if len(region.GetExternalId()) > 0 {
 			extra.Add(jsonutils.NewString(region.GetExternalId()), "region_external_id")
 		}
-	}
+	}*/
 	vpc := self.getVpc()
 	if vpc != nil {
 		extra.Add(jsonutils.NewString(vpc.GetName()), "vpc")
 		if len(vpc.GetExternalId()) > 0 {
-			extra.Add(jsonutils.NewString(vpc.GetExternalId()), "vpc_external_id")
+			extra.Add(jsonutils.NewString(vpc.GetExternalId()), "vpc_ext_id")
 		}
 	}
+
+	info := vpc.getCloudProviderInfo()
+	extra.Update(jsonutils.Marshal(&info))
+
 	return extra
 }

pkg/compute/models/zones.go

@@ -6,13 +6,14 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/tristate"
+	"yunion.io/x/pkg/util/compare"
+	"yunion.io/x/sqlchemy"
+
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
-	"yunion.io/x/pkg/tristate"
-	"yunion.io/x/pkg/util/compare"
-	"yunion.io/x/sqlchemy"
 )
 
 const (

pkg/compute/options/options.go

@@ -74,6 +74,8 @@ type ComputeOptions struct {
 
 	NfsDefaultImageCacheDir string `default:"image_cache"`
 
+	SnapshotCreateDiskProtocol string `help:"Snapshot create disk protocol" choices:"url|fuse" default:"fuse"`
+
 	cloudcommon.DBOptions
 }
 

pkg/compute/tasks/cloud_account_sync_task.go

@@ -91,6 +91,7 @@ func (self *CloudAccountSyncInfoTask) OnCloudaccountSyncComplete(ctx context.Con
 				err := skus.SyncSkusByProviderIds([]string{cloudprovider.Provider})
 				return nil, err
 			})
+			return
 		}
 	}
 	self.SetStageComplete(ctx, nil)

pkg/compute/tasks/disk_delete_task.go

@@ -93,6 +93,9 @@ func (self *DiskDeleteTask) OnGuestDiskDeleteComplete(ctx context.Context, obj d
 	disk := obj.(*models.SDisk)
 	self.CleanHostSchedCache(disk)
 	db.OpsLog.LogEvent(disk, db.ACT_DELOCATE, disk.GetShortDesc(), self.UserCred)
+	if len(disk.SnapshotId) > 0 && disk.GetMetadata("merge_snapshot", nil) == "true" {
+		models.SnapshotManager.AddRefCount(disk.SnapshotId, -1)
+	}
 	disk.RealDelete(ctx, self.UserCred)
 	self.SetStageComplete(ctx, nil)
 }

pkg/compute/tasks/eip_deallocate_task.go

@@ -27,7 +27,7 @@ func (self *EipDeallocateTask) OnInit(ctx context.Context, obj db.IStandaloneMod
 	if len(eip.ExternalId) > 0 {
 		expEip, err := eip.GetIEip()
 		if err != nil {
-			if err != cloudprovider.ErrNotFound {
+			if err != cloudprovider.ErrNotFound && err != cloudprovider.ErrInvalidProvider {
 				msg := fmt.Sprintf("fail to find iEIP for eip %s", err)
 				eip.SetStatus(self.UserCred, models.EIP_STATUS_DEALLOCATE_FAIL, msg)
 				self.SetStageFailed(ctx, msg)

pkg/compute/tasks/guest_create_disk_task.go

@@ -3,10 +3,10 @@ package tasks
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"yunion.io/x/jsonutils"
 
-	"time"
 	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
@@ -66,11 +66,8 @@ func (self *KVMGuestCreateDiskTask) OnKvmDiskPrepared(ctx context.Context, obj d
 		}
 		disk := iDisk.(*models.SDisk)
 		if disk.Status == models.DISK_INIT {
-			snapInfo, err := self.Params.GetString(fmt.Sprintf("disk.%d.snapshot", diskIndex))
-			if err != nil {
-				snapInfo = ""
-			}
-			err = disk.StartDiskCreateTask(ctx, self.UserCred, false, snapInfo, self.GetTaskId())
+			snapshotId, _ := self.Params.GetString(fmt.Sprintf("disk.%d.snapshot", diskIndex))
+			err = disk.StartDiskCreateTask(ctx, self.UserCred, false, snapshotId, self.GetTaskId())
 			if err != nil {
 				self.SetStageFailed(ctx, err.Error())
 				return

pkg/hostimage/host_image_service.go

@@ -0,0 +1,233 @@
+package hostimage
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"yunion.io/x/log"
+	"yunion.io/x/onecloud/pkg/appctx"
+	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/cloudcommon"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+
+	"github.com/pierrec/lz4"
+)
+
+type SHostImageOptions struct {
+	cloudcommon.Options
+	LocalImagePath    []string `help:"Local Image Paths"`
+	SnapshotDirSuffix string   `help:"Snapshot dir name equal diskId concat snapshot dir suffix" default:"_snap"`
+}
+
+var HostImageOptions SHostImageOptions
+
+func StartService() {
+	consts.SetServiceType("host-image")
+	cloudcommon.ParseOptions(&HostImageOptions, &HostImageOptions.Options, os.Args, "host.conf")
+	HostImageOptions.Port += 40000
+	cloudcommon.InitAuth(&HostImageOptions.Options, func() {
+		log.Infof("Auth complete!!")
+	})
+	app := cloudcommon.InitApp(&HostImageOptions.Options)
+	initHandlers(app, "")
+	cloudcommon.ServeForever(app, &HostImageOptions.Options)
+}
+
+func initHandlers(app *appsrv.Application, prefix string) {
+	app.AddHandler("GET", fmt.Sprintf("%s/disks/<sid>", prefix), auth.Authenticate(getImage))
+	app.AddHandler("GET", fmt.Sprintf("%s/snapshots/<diskId>/<sid>", prefix), auth.Authenticate(getImage))
+	app.AddHandler("HEAD", fmt.Sprintf("%s/disks/<sid>", prefix), auth.Authenticate(getImageMeta))
+	app.AddHandler("HEAD", fmt.Sprintf("%s/snapshots/<diskId>/<sid>", prefix), auth.Authenticate(getImageMeta))
+}
+
+func getDiskPath(diskId string) string {
+	for _, imagePath := range HostImageOptions.LocalImagePath {
+		diskPath := path.Join(imagePath, diskId)
+		if _, err := os.Stat(diskPath); !os.IsNotExist(err) {
+			return diskPath
+		}
+	}
+	return ""
+}
+
+func getSnapshotPath(diskId, snapshotId string) string {
+	for _, imagePath := range HostImageOptions.LocalImagePath {
+		diskPath := path.Join(imagePath, "snapshots",
+			diskId+HostImageOptions.SnapshotDirSuffix, snapshotId)
+		if _, err := os.Stat(diskPath); !os.IsNotExist(err) {
+			return diskPath
+		}
+	}
+	return ""
+}
+
+func inputCheck(ctx context.Context) (string, error) {
+	var userCred = auth.FetchUserCredential(ctx, nil)
+	if !userCred.HasSystemAdminPrivelege() {
+		return "", httperrors.NewForbiddenError("System admin only")
+	}
+
+	var params = appctx.AppContextParams(ctx)
+	var sid = params["<sid>"]
+	var imagePath string
+	if diskId, ok := params["<diskId>"]; ok {
+		imagePath = getSnapshotPath(diskId, sid)
+	} else {
+		imagePath = getDiskPath(sid)
+	}
+	if len(imagePath) == 0 {
+		return "", httperrors.NewNotFoundError("Disk not found")
+	}
+	return imagePath, nil
+}
+
+func parseRange(reqRange string) (int64, int64, error) {
+	if !strings.HasPrefix(reqRange, "bytes=") {
+		return 0, 0, httperrors.NewInputParameterError("Invalid range header")
+	}
+	reqRange = reqRange[len("bytes="):]
+	ranges := strings.Split(reqRange, "-")
+	if len(ranges) != 2 {
+		return 0, 0, httperrors.NewInputParameterError("Invalid range header")
+	}
+	startPos, err := strconv.ParseInt(ranges[0], 10, 0)
+	if err != nil {
+		return 0, 0, httperrors.NewInputParameterError("Invalid range header")
+	}
+	endPos, err := strconv.ParseInt(ranges[1], 10, 0)
+	if err != nil {
+		return 0, 0, httperrors.NewInputParameterError("Invalid range header")
+	}
+	return startPos, endPos, nil
+}
+
+func getImage(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	imagePath, err := inputCheck(ctx)
+	if err != nil {
+		httperrors.GeneralServerError(w, err)
+		return
+	}
+
+	var f IImage
+	var startPos, endPos int64
+	var rateLimit int64 = -1
+
+	if r.Header.Get("X-Read-File") == "true" {
+		f = &SFile{}
+	} else {
+		f = &SQcow2Image{}
+	}
+	if err = f.Open(imagePath, true); err != nil {
+		log.Errorf("Open image error: %s", err)
+		httperrors.GeneralServerError(w, err)
+		return
+	}
+	defer f.Close() // Remenber close fd
+
+	endPos = f.Length() - 1
+	reqRange := r.Header.Get("Range")
+	if len(reqRange) > 0 {
+		startPos, endPos, err = parseRange(reqRange)
+		if err != nil {
+			log.Errorf("Parse range error: %s", err)
+			httperrors.GeneralServerError(w, err)
+			return
+		}
+	}
+
+	strRateLimit := r.Header.Get("X-Rate-Limit-Mbps")
+	if len(strRateLimit) > 0 {
+		rateLimit, err = strconv.ParseInt(strRateLimit, 10, 0)
+		if err != nil {
+			log.Errorf("Parse ratelimit error: %s", err)
+			httperrors.InvalidInputError(w, "Invaild rate limit header")
+			return
+		}
+	}
+
+	streamHeader(w, f, startPos, endPos)
+	startStream(w, f, startPos, endPos, rateLimit)
+}
+
+func streamHeader(w http.ResponseWriter, f IImage, startPos, endPos int64) {
+	var statusCode = http.StatusOK
+	w.Header().Set("Content-Type", "application/octet-stream")
+	if startPos > 0 || endPos < f.Length()-1 {
+		statusCode = http.StatusPartialContent
+		w.Header().Set("Content-Range",
+			fmt.Sprintf("bytes %d-%d/%d", startPos, endPos, f.Length()))
+	}
+	w.WriteHeader(statusCode)
+}
+
+func startStream(w http.ResponseWriter, f IImage, startPos, endPos, rateLimit int64) {
+	var CHUNK_SIZE int64 = 4 * 1024
+	var readSize int64 = CHUNK_SIZE
+	var sendBytes int64
+	var lz4Writer = lz4.NewWriter(w)
+	var startTime = time.Now()
+
+	for startPos < endPos {
+		if endPos-startPos < CHUNK_SIZE {
+			readSize = endPos - startPos + 1
+		}
+		buf, total := f.Read(startPos, readSize)
+		if total < 0 {
+			log.Errorf("Read image error: %d", total)
+			goto fail
+		}
+		startPos += readSize
+		wSize, err := lz4Writer.Write(buf)
+		if err != nil {
+			log.Errorf("lz4Write error: %s", err)
+			goto fail
+		}
+		sendBytes += int64(wSize)
+		if rateLimit > 0 {
+			tmDelta := time.Now().Sub(startTime)
+			tms := tmDelta.Seconds()
+			vtmDelta := float64(sendBytes*8) / float64(1024.0*1024.0*rateLimit)
+			if vtmDelta > tms {
+				time.Sleep(time.Duration(vtmDelta - tms))
+			}
+		}
+	}
+
+fail:
+	if err := lz4Writer.Close(); err != nil {
+		log.Errorf("lz4 Close error: %s", err)
+	}
+}
+
+func getImageMeta(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	imagePath, err := inputCheck(ctx)
+	if err != nil {
+		httperrors.GeneralServerError(w, err)
+		return
+	}
+
+	var f IImage
+	if r.Header.Get("X-Read-File") == "true" {
+		f = &SFile{}
+	} else {
+		f = &SQcow2Image{}
+	}
+	if err = f.Open(imagePath, true); err != nil {
+		httperrors.GeneralServerError(w, err)
+		return
+	}
+	defer f.Close() // Remenber close fd
+
+	w.Header().Set("Content-Length", fmt.Sprintf("%d", f.Length()))
+	w.Header().Set("Content-Type", "application/octet-stream")
+	w.Header().Set("Accept-Ranges", "bytes")
+	w.WriteHeader(200)
+}

pkg/hostimage/image.go

@@ -0,0 +1,134 @@
+package hostimage
+
+/*
+#cgo pkg-config: glib-2.0 zlib
+#cgo CFLAGS: -I/home/yunion/rpmbuild/SOURCES/qemu/src -I/home/yunion/rpmbuild/SOURCES/qemu/src/include
+#cgo LDFLAGS: -laio -lqemuio -lpthread  -L /home/yunion/rpmbuild/SOURCES/qemu/src
+
+#include "libqemuio.h"
+#include "qemu/osdep.h"
+*/
+import "C"
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"unsafe"
+)
+
+func init() {
+	C.qemuio_init()
+}
+
+func ReadQcow2(qemuioBlk *C.struct_QemuioBlk, offset int64, count int64) ([]byte, int64) {
+	if qemuioBlk == nil || offset < 0 || count < 0 {
+		return nil, -1
+	}
+	b := make([]byte, count)
+	var total = C.int64_t(0)
+	ret := C.read_qcow2(qemuioBlk, unsafe.Pointer(&b[0]), C.int64_t(offset), C.int64_t(count), &total)
+	if ret < 0 {
+		return nil, int64(ret)
+	} else {
+		return b, int64(total)
+	}
+}
+
+func OpenQcow2(imagePath string, readonly bool) *C.struct_QemuioBlk {
+	return C.open_qcow2(C.CString(imagePath), C.bool(readonly))
+}
+
+func Qcow2GetLenth(qemuioBlk *C.struct_QemuioBlk) int64 {
+	return int64(C.qcow2_get_length(qemuioBlk))
+}
+
+func CloseQcow2(qemuioBlk *C.struct_QemuioBlk) {
+	C.close_qcow2(qemuioBlk)
+}
+
+type IImage interface {
+	// Open image file and its backing file (if have)
+	Open(imagePath string, readonly bool) error
+
+	// Close may not really close image file handle, just reudce ref count
+	Close()
+
+	// If return number < 0 indicate read failed
+	Read(offset, count int64) ([]byte, int64)
+
+	// Get image file length, not file actual length, it's image virtual size
+	Length() int64
+}
+
+type SQcow2Image struct {
+	fd *C.struct_QemuioBlk
+}
+
+func (img *SQcow2Image) Open(imagePath string, readonly bool) error {
+	fd := OpenQcow2(imagePath, readonly)
+	if fd == nil {
+		return fmt.Errorf("Open image %s failed", imagePath)
+	} else {
+		img.fd = fd
+		return nil
+	}
+}
+
+func (img *SQcow2Image) Read(offset, count int64) ([]byte, int64) {
+	return ReadQcow2(img.fd, offset, count)
+}
+
+func (img *SQcow2Image) Close() {
+	CloseQcow2(img.fd)
+}
+
+func (img *SQcow2Image) Length() int64 {
+	return Qcow2GetLenth(img.fd)
+}
+
+type SFile struct {
+	fd *os.File
+}
+
+func (f *SFile) Open(imagePath string, readonly bool) error {
+	var mode = os.O_RDWR
+	if readonly {
+		mode = os.O_RDONLY
+	}
+	fd, err := os.OpenFile(imagePath, mode, 0644)
+	if err != nil {
+		return err
+	} else {
+		f.fd = fd
+		return nil
+	}
+}
+
+func (f *SFile) Read(offset, count int64) ([]byte, int64) {
+	buf := make([]byte, count)
+	var readCount int64 = 0
+	for readCount < count {
+		cnt, err := f.fd.Read(buf[readCount:])
+		readCount += int64(cnt)
+		if err == io.EOF {
+			return buf[0:readCount], readCount
+		}
+		if err != nil {
+			return nil, -1
+		}
+	}
+	return buf, readCount
+}
+
+func (f *SFile) Close() {
+	f.fd.Close()
+}
+
+func (f *SFile) Length() int64 {
+	stat, e := f.fd.Stat()
+	if e != nil {
+		return -1
+	}
+	return stat.Size()
+}

pkg/lbagent/models/reflect.go

@@ -88,7 +88,10 @@ func GetModels(opts *GetModelsOptions) error {
 	listOptions := options.BaseListOptions{
 		Admin:   options.Bool(true),
 		Details: options.Bool(true),
-		Filter:  []string{minUpdatedAtFilter(minUpdatedAt)},
+		Filter: []string{
+			minUpdatedAtFilter(minUpdatedAt), // order matters, filter.0
+			"isempty(manager_id)",            // len(manager_id) > 0 is for pubcloud objects
+		},
 		OrderBy: []string{"updated_at", "id"},
 		Order:   "asc",
 		Limit:   options.Int(opts.BatchListSize),

pkg/mcclient/modules/mod_cloudregions.go

@@ -6,8 +6,10 @@ var (
 
 func init() {
 	Cloudregions = NewComputeManager("cloudregion", "cloudregions",
-		[]string{"ID", "Name", "Enabled", "Status", "Provider", "Latitude", "Longitude",
-			"vpc_count", "zone_count", "guest_count", "guest_increment_count"},
+		[]string{"ID", "Name", "Enabled", "Status", "Provider",
+			"Latitude", "Longitude", "City", "Country_Code",
+			"vpc_count", "zone_count", "guest_count", "guest_increment_count",
+			"External_Id"},
 		[]string{})
 
 	registerCompute(&Cloudregions)

pkg/mcclient/options/routetables.go

@@ -125,9 +125,18 @@ type RouteTableDeleteOptions struct {
 	ID string
 }
 
+type RouteTablePurgeOptions struct {
+	ID string
+}
+
 type RouteTableListOptions struct {
+	Manager     string
 	Vpc         string
 	Cloudregion string
 
+	Manager  string `help:"List hosts belongs to the cloud provider"`
+	Account  string `help:"List hosts belongs to the cloud account"`
+	Provider string `help:"List hosts belongs to the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei"`
+
 	BaseListOptions
 }

pkg/mcclient/options/servers.go

@@ -19,11 +19,14 @@ type ServerListOptions struct {
 	Secgroup      string `help:"Secgroup ID or Name"`
 	AdminSecgroup string `help:"AdminSecgroup ID or Name"`
 	Hypervisor    string `help:"Show server of hypervisor" choices:"kvm|esxi|container|baremetal|aliyun|azure|aws"`
-	Manager       string `help:"Show servers imported from manager"`
 	Region        string `help:"Show servers in cloudregion"`
 	WithEip       *bool  `help:"Show Servers with EIP"`
 	WithoutEip    *bool  `help:"Show Servers without EIP"`
 
+	Manager  string `help:"Show servers imported from manager"`
+	Account  string `help:"List hosts belongs to the cloud account"`
+	Provider string `help:"List hosts belongs to the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei"`
+
 	ResourceType string `help:"Resource type" choices:"shared|prepaid|dedicated"`
 
 	BillingType string `help:"billing type" choices:"postpaid|prepaid"`
@@ -263,6 +266,11 @@ type ServerSecGroupOptions struct {
 	Secgrp string `help:"ID of Security Group" metavar:"Security Group" positional:"true"`
 }
 
+type ServerSecGroupsOptions struct {
+	ID      string   `help:"ID or Name of server" metavar:"Guest" json:"-"`
+	Secgrps []string `help:"Ids of Security Groups" metavar:"Security Groups" positional:"true"`
+}
+
 type ServerSendKeyOptions struct {
 	ID   string `help:"ID or Name of server" metavar:"Guest" json:"-"`
 	KEYS string `help:"Special keys to send, eg. ctrl, alt, f12, shift, etc, separated by \"-\""`

pkg/scheduler/algorithm/plugin/plugin.go

@@ -0,0 +1,12 @@
+package plugin
+
+import (
+	"yunion.io/x/onecloud/pkg/scheduler/core"
+)
+
+type BasePlugin struct{}
+
+// Customize priority
+func (p BasePlugin) OnPriorityEnd(u *core.Unit, c core.Candidater) {}
+
+func (p BasePlugin) OnSelectEnd(u *core.Unit, c core.Candidater, count int64) {}

pkg/scheduler/algorithm/predicates/aggregate_predicate.go

@@ -6,8 +6,10 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 
+	"yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin"
 	"yunion.io/x/onecloud/pkg/scheduler/api"
 	"yunion.io/x/onecloud/pkg/scheduler/core"
+	"yunion.io/x/onecloud/pkg/scheduler/core/score"
 	"yunion.io/x/onecloud/pkg/scheduler/db/models"
 	"yunion.io/x/onecloud/pkg/util/conditionparser"
 )
@@ -23,6 +25,7 @@ import (
 // the host is available.
 type AggregatePredicate struct {
 	BasePredicate
+	plugin.BasePlugin
 	AggregateHosts    hostsAggregatesMap
 	RequireAggregates []api.Aggregate
 	ExcludeAggregates []api.Aggregate
@@ -193,33 +196,6 @@ func getHostAggregateCount(inAggs []api.Aggregate, hAggs []*models.Aggregate, st
 	return
 }
 
-func (p *AggregatePredicate) OnSelect(u *core.Unit, c core.Candidater) bool {
-	hostAggs, ok := p.AggregateHosts[c.IndexKey()]
-	if !ok {
-		return true
-	}
-
-	avoidCountMap := getHostAggregateCount(p.AvoidAggregates, hostAggs, api.AggregateStrategyAvoid)
-	preferCountMap := getHostAggregateCount(p.PreferAggregates, hostAggs, api.AggregateStrategyPrefer)
-
-	setScore := func(aggCountMap map[string]int, postiveScore bool) {
-		stepScore := core.PriorityStep
-		if !postiveScore {
-			stepScore = -stepScore
-		}
-		for n, count := range aggCountMap {
-			u.IncreaseScore(c.IndexKey(), n, count*stepScore)
-		}
-	}
-
-	setScore(avoidCountMap, false)
-	setScore(preferCountMap, true)
-
-	return true
-}
-
-func (p *AggregatePredicate) OnSelectEnd(u *core.Unit, c core.Candidater, count int64) {}
-
 func (p *AggregatePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) {
 	h := NewPredicateHelper(p, u, c)
 
@@ -281,3 +257,29 @@ func (p *AggregatePredicate) exec(h *PredicateHelper) string {
 
 	return ""
 }
+
+func (p *AggregatePredicate) OnPriorityEnd(u *core.Unit, c core.Candidater) {
+	hostAggs, ok := p.AggregateHosts[c.IndexKey()]
+	if !ok {
+		return
+	}
+
+	avoidCountMap := getHostAggregateCount(p.AvoidAggregates, hostAggs, api.AggregateStrategyAvoid)
+	preferCountMap := getHostAggregateCount(p.PreferAggregates, hostAggs, api.AggregateStrategyPrefer)
+
+	setScore := func(aggCountMap map[string]int, postiveScore bool) {
+		stepScore := core.PriorityStep
+		if !postiveScore {
+			stepScore = -stepScore
+		}
+		for n, count := range aggCountMap {
+			u.SetFrontScore(
+				c.IndexKey(),
+				score.NewScore(score.TScore(count*stepScore), n),
+			)
+		}
+	}
+
+	setScore(preferCountMap, true)
+	setScore(avoidCountMap, false)
+}

pkg/scheduler/algorithm/predicates/baremetal/network_predicate.go

@@ -7,6 +7,7 @@ import (
 
 	"yunion.io/x/pkg/utils"
 
+	"yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin"
 	"yunion.io/x/onecloud/pkg/scheduler/algorithm/predicates"
 	"yunion.io/x/onecloud/pkg/scheduler/api"
 	"yunion.io/x/onecloud/pkg/scheduler/core"
@@ -14,6 +15,7 @@ import (
 
 type NetworkPredicate struct {
 	BasePredicate
+	plugin.BasePlugin
 	SelectedNetworks sync.Map
 }
 
@@ -141,11 +143,3 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor
 
 	return h.GetResult()
 }
-
-func (p *NetworkPredicate) OnSelect(u *core.Unit, c core.Candidater) bool {
-	u.SetFiltedData(c.IndexKey(), "networks", &p.SelectedNetworks)
-	return true
-}
-
-func (p *NetworkPredicate) OnSelectEnd(u *core.Unit, c core.Candidater, count int64) {
-}

pkg/scheduler/algorithm/predicates/guest/group_predicate.go

@@ -3,14 +3,17 @@ package guest
 import (
 	"fmt"
 
+	"yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin"
 	"yunion.io/x/onecloud/pkg/scheduler/algorithm/predicates"
 	"yunion.io/x/onecloud/pkg/scheduler/core"
+	"yunion.io/x/onecloud/pkg/scheduler/core/score"
 )
 
 // GroupPredicate filter the packet based on the label information,
 // the same group of guests should avoid schedule on same host.
 type GroupPredicate struct {
 	predicates.BasePredicate
+	plugin.BasePlugin
 
 	ExcludeGroups []string
 	RequireGroups []string
@@ -47,33 +50,6 @@ func (p *GroupPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, e
 	return true, nil
 }
 
-func (p *GroupPredicate) OnSelect(u *core.Unit, c core.Candidater) bool {
-	if len(p.ExcludeGroups) > 0 {
-		return false
-	}
-
-	if len(p.RequireGroups) > 0 {
-		// TODO: what?
-	}
-
-	if len(p.AvoidGroups) > 0 {
-		u.IncreaseScore(c.IndexKey(),
-			p.Name()+":avoid", -core.PriorityStep*len(p.AvoidGroups),
-		)
-	}
-
-	if len(p.PreferGroups) > 0 {
-		u.IncreaseScore(c.IndexKey(),
-			p.Name()+":prefer", core.PriorityStep*len(p.PreferGroups),
-		)
-	}
-
-	return true
-}
-
-func (p *GroupPredicate) OnSelectEnd(u *core.Unit, c core.Candidater, count int64) {
-}
-
 func (p *GroupPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) {
 
 	h := predicates.NewPredicateHelper(p, u, c)
@@ -101,3 +77,23 @@ func (p *GroupPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.
 
 	return h.GetResult()
 }
+
+func (p *GroupPredicate) OnPriorityEnd(u *core.Unit, c core.Candidater) {
+	if len(p.AvoidGroups) > 0 {
+		u.SetFrontScore(
+			c.IndexKey(),
+			score.NewScore(
+				score.TScore(-core.PriorityStep*len(p.AvoidGroups)),
+				p.Name()+":avoid",
+			))
+	}
+
+	if len(p.PreferGroups) > 0 {
+		u.SetFrontScore(
+			c.IndexKey(),
+			score.NewScore(
+				score.TScore(core.PriorityStep*len(p.PreferGroups)),
+				p.Name()+":prefer",
+			))
+	}
+}

pkg/scheduler/algorithm/predicates/guest/network_predicate.go

@@ -5,12 +5,14 @@ import (
 	"strings"
 	"sync"
 
+	"yunion.io/x/pkg/util/sets"
+	"yunion.io/x/pkg/utils"
+
+	"yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin"
 	"yunion.io/x/onecloud/pkg/scheduler/algorithm/predicates"
 	"yunion.io/x/onecloud/pkg/scheduler/api"
 	"yunion.io/x/onecloud/pkg/scheduler/core"
 	networks "yunion.io/x/onecloud/pkg/scheduler/db/models"
-	"yunion.io/x/pkg/util/sets"
-	"yunion.io/x/pkg/utils"
 )
 
 // NetworkPredicate will filter the current network information with
@@ -18,6 +20,7 @@ import (
 // randomly match the available network resources.
 type NetworkPredicate struct {
 	predicates.BasePredicate
+	plugin.BasePlugin
 	SelectedNetworks sync.Map
 }
 
@@ -105,10 +108,6 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor
 				p.SelectedNetworks.Store(n.ID, counter.GetCount())
 				counters.Add(counter)
 				found = true
-
-				if counters.GetCount() >= d.Count {
-					break
-				}
 			} else {
 				fullErrMsgs = append(fullErrMsgs,
 					fmt.Sprintf("%s: %s", n.ID, strings.Join(errMsgs, ",")),
@@ -202,11 +201,3 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor
 
 	return h.GetResult()
 }
-
-func (p *NetworkPredicate) OnSelect(u *core.Unit, c core.Candidater) bool {
-	u.SetFiltedData(c.IndexKey(), "networks", &p.SelectedNetworks)
-	return true
-}
-
-func (p *NetworkPredicate) OnSelectEnd(u *core.Unit, c core.Candidater, count int64) {
-}

pkg/scheduler/algorithm/priorities/guest/avoid_same_host.go

@@ -27,7 +27,7 @@ func (p *AvoidSameHostPriority) Map(u *core.Unit, c core.Candidater) (core.HostP
 
 	ownerTenantID := u.SchedData().OwnerTenantID
 	if count, ok := hc.Tenants[ownerTenantID]; ok && count > 0 {
-		h.SetScore(-50 * int(count))
+		h.SetFrontRawScore(-1 * int(count))
 	}
 
 	return h.GetResult()

pkg/scheduler/algorithm/priorities/guest/capacity.go

@@ -21,7 +21,7 @@ func (p *CapacityPriority) Map(u *core.Unit, c core.Candidater) (core.HostPriori
 	h := priorities.NewPriorityHelper(p, u, c)
 
 	capacity := u.GetCapacity(c.IndexKey())
-	h.SetScore(50 * int(capacity))
+	h.SetRawScore(int(capacity))
 
 	return h.GetResult()
 }

pkg/scheduler/algorithm/priorities/guest/creating.go

@@ -27,8 +27,8 @@ func (p *CreatingPriority) Map(u *core.Unit, c core.Candidater) (core.HostPriori
 	}
 
 	if hc.CreatingGuestCount > 0 {
-		score := -int(hc.CreatingGuestCount) * 20
-		h.SetScore(score)
+		score := -int(hc.CreatingGuestCount)
+		h.SetFrontScore(score)
 	}
 
 	return h.GetResult()

pkg/scheduler/algorithm/priorities/guest/lowload.go

@@ -3,6 +3,7 @@ package guest
 import (
 	"yunion.io/x/onecloud/pkg/scheduler/algorithm/priorities"
 	"yunion.io/x/onecloud/pkg/scheduler/core"
+	"yunion.io/x/onecloud/pkg/scheduler/core/score"
 )
 
 type LowLoadPriority struct {
@@ -28,8 +29,12 @@ func (p *LowLoadPriority) Map(u *core.Unit, c core.Candidater) (core.HostPriorit
 	cpuCommitRate := float64(hc.RunningCPUCount) / float64(hc.TotalCPUCount)
 	memCommitRate := float64(hc.RunningMemSize) / float64(hc.TotalMemSize)
 	if cpuCommitRate < 0.5 && memCommitRate < 0.5 {
-		score := 20 * (1 - cpuCommitRate - memCommitRate)
+		score := 10 * (1 - cpuCommitRate - memCommitRate)
 		h.SetScore(int(score))
 	}
 	return h.GetResult()
 }
+
+func (p *LowLoadPriority) ScoreIntervals() score.Intervals {
+	return score.NewIntervals(0, 1, 5)
+}

pkg/scheduler/algorithm/priorities/priorities.go

@@ -1,23 +1,18 @@
 package priorities
 
 import (
-	"math"
-
 	"yunion.io/x/onecloud/pkg/scheduler/algorithm"
 	"yunion.io/x/onecloud/pkg/scheduler/cache/candidate"
 	"yunion.io/x/onecloud/pkg/scheduler/core"
+	"yunion.io/x/onecloud/pkg/scheduler/core/score"
 )
 
-func aggPriority(x float64) float64 {
-	return math.Log(x + math.Sqrt(x*x+1))
-}
-
 // PriorityHelper is a struct that as a base interface for all priorities.
 type PriorityHelper struct {
 	priority  core.Priority
 	unit      *core.Unit
 	Candidate core.Candidater
-	score     int
+	score     score.SScore
 	err       error
 }
 
@@ -29,9 +24,38 @@ func NewPriorityHelper(p core.Priority, u *core.Unit, c core.Candidater) *Priori
 	}
 }
 
-func (h *PriorityHelper) SetScore(score int) {
-	h.score = score
-	h.unit.SetScore(h.Candidate.IndexKey(), h.priority.Name(), score)
+func (h *PriorityHelper) setIntervalScore(val int) score.SScore {
+	h.score = score.NewScore(
+		h.priority.ScoreIntervals().ToScore(int64(val)),
+		h.priority.Name())
+	return h.score
+}
+
+func (h *PriorityHelper) setRawScore(val int) score.SScore {
+	h.score = score.NewScore(
+		score.TScore(val),
+		h.priority.Name())
+	return h.score
+}
+
+func (h *PriorityHelper) SetScore(val int) {
+	h.setIntervalScore(val)
+	h.unit.SetScore(h.Candidate.IndexKey(), h.score)
+}
+
+func (h *PriorityHelper) SetFrontScore(val int) {
+	h.setIntervalScore(val)
+	h.unit.SetFrontScore(h.Candidate.IndexKey(), h.score)
+}
+
+func (h *PriorityHelper) SetRawScore(val int) {
+	h.setRawScore(val)
+	h.unit.SetScore(h.Candidate.IndexKey(), h.score)
+}
+
+func (h *PriorityHelper) SetFrontRawScore(val int) {
+	h.setRawScore(val)
+	h.unit.SetFrontScore(h.Candidate.IndexKey(), h.score)
 }
 
 func (h *PriorityHelper) SetError(err error) {
@@ -41,7 +65,6 @@ func (h *PriorityHelper) SetError(err error) {
 func (h *PriorityHelper) GetResult() (core.HostPriority, error) {
 	return core.HostPriority{
 		Host:      h.Candidate.IndexKey(),
-		Score:     h.score,
 		Candidate: h.Candidate,
 	}, h.err
 }
@@ -71,3 +94,7 @@ func (b *BasePriority) Name() string {
 func (b *BasePriority) HostCandidate(c core.Candidater) (*candidate.HostDesc, error) {
 	return algorithm.ToHostCandidate(c)
 }
+
+func (b *BasePriority) ScoreIntervals() score.Intervals {
+	return score.NewIntervals(0, 1, 2)
+}

pkg/scheduler/algorithmprovider/defaults.go

@@ -31,7 +31,6 @@ func defaultPredicates() sets.String {
 
 func defaultPriorities() sets.String {
 	return sets.NewString(
-		factory.RegisterPriority("guest-avoid-same-cluster", &priorityguest.AvoidSameClusterPriority{}, 1),
 		factory.RegisterPriority("guest-avoid-same-host", &priorityguest.AvoidSameHostPriority{}, 1),
 		factory.RegisterPriority("guest-lowload", &priorityguest.LowLoadPriority{}, 1),
 		factory.RegisterPriority("guest-creating", &priorityguest.CreatingPriority{}, 1),

pkg/scheduler/core/context.go

@@ -9,17 +9,15 @@ import (
 	"yunion.io/x/log"
 
 	"yunion.io/x/onecloud/pkg/scheduler/api"
+	"yunion.io/x/onecloud/pkg/scheduler/core/score"
 )
 
 const (
-	EmptyScore    int   = 0x7FFFFFFFFFFFFFFF
-	BaseScore     int   = 10000
 	EmptyCapacity int64 = -1
 	MaxCapacity   int64 = 0x7FFFFFFFFFFFFFFF
 )
 
 var (
-	EmptyScores     = make(map[string]int)
 	EmptyCapacities = make(map[string]Counter)
 )
 
@@ -186,8 +184,19 @@ type Capacity struct {
 }
 
 type Score struct {
-	Values map[string]int
-	Sum    int
+	*score.ScoreBucket
+}
+
+func newScore() *Score {
+	return &Score{
+		ScoreBucket: score.NewScoreBuckets(),
+	}
+}
+
+func newZeroScore() Score {
+	s := newScore()
+	s.Append(score.NewZeroScore())
+	return *s
 }
 
 type SchedContextDataItem struct {
@@ -484,7 +493,11 @@ func validateCapacityInput(c Counter) bool {
 	return false
 }
 
-func (u *Unit) SetScore(id, name string, score int) error {
+type ScoreValue struct {
+	value score.TScore
+}
+
+func (u *Unit) setScore(id string, val score.SScore, tofront bool) {
 	u.scoreLock.Lock()
 	defer u.scoreLock.Unlock()
 
@@ -494,75 +507,45 @@ func (u *Unit) SetScore(id, name string, score int) error {
 	)
 
 	if scoreObj, ok = u.ScoreMap[id]; !ok {
-		scoreObj = Score{Values: make(map[string]int), Sum: EmptyScore}
+		scoreObj = *newScore()
 		u.ScoreMap[id] = scoreObj
 	}
 
-	scoreObj.Values[name] = score
-	scoreObj.Sum = EmptyScore
-
-	log.V(10).Infof("%q SetScore: %q -> %d", name, id, score)
-	return nil
-}
-
-func (u *Unit) IncreaseScore(id string, name string, increase int) error {
-
-	u.scoreLock.Lock()
-	defer u.scoreLock.Unlock()
-
-	var (
-		scoreObj Score
-		ok       bool
-	)
-
-	score := int(0)
-	if scoreObj, ok = u.ScoreMap[id]; !ok {
-		scoreObj = Score{Values: make(map[string]int), Sum: EmptyScore}
-		u.ScoreMap[id] = scoreObj
-		score = increase
+	if tofront {
+		scoreObj.AddToFirst(val)
 	} else {
-		if value, ok := scoreObj.Values[name]; ok {
-			score = value + increase
-		} else {
-			score = increase
-		}
+		scoreObj.SetScore(val)
 	}
 
-	scoreObj.Values[name] = score
-	scoreObj.Sum = EmptyScore
-
-	log.V(10).Infof("%q IncreaseScore: %q -> %d", name, id, score)
-	return nil
+	log.V(10).Infof("SetScore: %q -> %s", id, val.String())
 }
 
-func (u *Unit) GetScore(id string) int {
+func (u *Unit) SetScore(id string, val score.SScore) {
+	u.setScore(id, val, false)
+}
+
+func (u *Unit) SetFrontScore(id string, val score.SScore) {
+	u.setScore(id, val, true)
+}
+
+func (u *Unit) GetScore(id string) Score {
 	var (
 		scoreObj Score
 		ok       bool
 	)
 
 	if scoreObj, ok = u.ScoreMap[id]; !ok {
-		return BaseScore
+		return *newScore()
 	}
-
-	if scoreObj.Sum == EmptyScore {
-		sum := int(0)
-		for _, value := range scoreObj.Values {
-			sum += value
-		}
-
-		scoreObj.Sum = sum
-	}
-
-	return scoreObj.Sum + BaseScore
+	return scoreObj
 }
 
-func (u *Unit) GetScores(id string) map[string]int {
-	if scores, ok := u.ScoreMap[id]; ok {
-		return scores.Values
+func (u *Unit) GetScoreDetails(id string) string {
+	if score, ok := u.ScoreMap[id]; ok {
+		return score.String()
 	}
 
-	return EmptyScores
+	return "EmptyScore"
 }
 func (u *Unit) SetFiltedData(id string, name string, data interface{}) error {
 	u.scoreLock.Lock()

pkg/scheduler/core/generic_scheduler.go

@@ -169,13 +169,13 @@ func newSchedResultByCtx(u *Unit, count int64, c Candidater) *SchedResultItem {
 		Count:    count,
 		Capacity: u.GetCapacity(id),
 		Name:     fmt.Sprintf("%v", c.Get("Name")),
-		Score:    u.GetScore(id),
+		Score:    u.GetScore(id).DigitString(),
 		Data:     u.GetFiltedData(id, count),
 	}
 
 	if showDetails {
 		r.CapacityDetails = GetCapacities(u, id)
-		r.ScoreDetails = u.GetScores(id)
+		r.ScoreDetails = u.GetScoreDetails(id)
 	}
 	return r
 }
@@ -229,10 +229,10 @@ type SchedResultItem struct {
 	Count    int64                  `json:"count"`
 	Data     map[string]interface{} `json:"data"`
 	Capacity int64                  `json:"capacity"`
-	Score    int                    `json:"score"`
+	Score    string                 `json:"score"`
 
 	CapacityDetails map[string]int64 `json:"capacity_details"`
-	ScoreDetails    map[string]int   `json:"score_details"`
+	ScoreDetails    string           `json:"score_details"`
 }
 
 func GetCapacities(u *Unit, id string) (res map[string]int64) {
@@ -246,22 +246,6 @@ func GetCapacities(u *Unit, id string) (res map[string]int64) {
 	return
 }
 
-func GetScore(u *Unit, id string, details bool) string {
-	score := u.GetScore(id)
-	s := fmt.Sprintf("%v", score)
-	if details {
-		scores := u.GetScores(id)
-		if len(scores) > 0 {
-			ss := []string{}
-			for name, score := range scores {
-				ss = append(ss, fmt.Sprintf("%v:%v", name, score))
-			}
-			s += " (" + strings.Join(ss, ", ") + ")"
-		}
-	}
-	return s
-}
-
 type SchedResultItemList struct {
 	Unit *Unit
 	Data []*SchedResultItem
@@ -325,16 +309,17 @@ func SelectHosts(unit *Unit, priorityList HostPriorityList) ([]*SelectedCandidat
 		return nil, fmt.Errorf("SelectHosts get empty priorityList.")
 	}
 
-	sort.Sort(sort.Reverse(priorityList))
-
 	selectedMap := make(map[string]*SelectedCandidate)
 	schedData := unit.SchedData()
 	count := schedData.Count
 	isSuggestion := unit.SchedInfo.IsSuggestion
 	bestEffort := unit.SchedInfo.BestEffort
 	selectedCandidates := []*SelectedCandidate{}
+
 	plugins := unit.AllSelectPlugins()
 
+	sort.Sort(sort.Reverse(priorityList))
+
 completed:
 	for len(priorityList) > 0 {
 		log.V(10).Debugf("PriorityList: %#v", priorityList)
@@ -357,18 +342,8 @@ completed:
 			}
 			selectedItem.Count++
 			count--
-			doPlugins := func() bool {
-				r := true
-				for _, plugin := range plugins {
-					if !plugin.OnSelect(unit, selectedItem.Candidate) {
-						r = false
-					}
-				}
-				return r
-			}
-			// if no one of plugins return false or capacity of the host large than
-			// selected count, this host can be added to priorityList.
-			if doPlugins() && unit.GetCapacity(hostID) > selectedItem.Count {
+			// if capacity of the host large than selected count, this host can be added to priorityList.
+			if unit.GetCapacity(hostID) > selectedItem.Count {
 				priorityList0 = append(priorityList0, it)
 			}
 		}
@@ -635,15 +610,23 @@ func PrioritizeCandidates(
 	result := make(HostPriorityList, 0, len(candidates))
 	// TODO: Consider parallelizing it
 
-	for i := range candidates {
-		result = append(result, HostPriority{Host: candidates[i].IndexKey(), Score: 0, Candidate: candidates[i]})
-		for j := range newPriorities {
-			result[i].Score += results[j][i].Score * newPriorities[j].Weight
+	// Do plugin priorities step
+	for _, candidate := range candidates {
+		for _, plugin := range unit.AllSelectPlugins() {
+			plugin.OnPriorityEnd(unit, candidate)
 		}
 	}
+
+	for i, candidate := range candidates {
+		result = append(result, HostPriority{Host: candidates[i].IndexKey(), Score: *newScore(), Candidate: candidates[i]})
+		//for j := range newPriorities {
+		//result[i].Score += results[j][i].Score * newPriorities[j].Weight
+		//}
+		result[i].Score = unit.GetScore(candidate.IndexKey())
+	}
 	if log.V(10) {
 		for i := range result {
-			log.Infof("Host %s => Score %d", result[i].Host, result[i].Score)
+			log.Infof("Host %s => Score %s", result[i].Host, result[i].Score.DigitString())
 		}
 	}
 	return result, nil
@@ -671,7 +654,7 @@ func EqualPriority(_ *Unit, candidate Candidater) (HostPriority, error) {
 	}
 	return HostPriority{
 		Host:      indexKey,
-		Score:     1,
+		Score:     newZeroScore(),
 		Candidate: candidate,
 	}, nil
 }

pkg/scheduler/core/score/score.go

@@ -0,0 +1,294 @@
+package score
+
+import (
+	"container/list"
+	"fmt"
+	"math"
+	//"yunion.io/x/log"
+)
+
+type TScore int
+
+const (
+	MinScore  TScore = -1
+	ZeroScore TScore = 0
+	MidScore  TScore = 1
+	MaxScore  TScore = 2
+
+	ZeroScoreName = "zero"
+)
+
+type SScore struct {
+	Score TScore
+	Name  string
+}
+
+func NewScore(score TScore, name string) SScore {
+	return SScore{
+		Score: score,
+		Name:  name,
+	}
+}
+
+func NewMinScore(name string) SScore {
+	return NewScore(MinScore, name)
+}
+
+func NewZeroScore() SScore {
+	return NewScore(ZeroScore, ZeroScoreName)
+}
+
+func NewMidScore(name string) SScore {
+	return NewScore(MidScore, name)
+}
+
+func NewMaxScore(name string) SScore {
+	return NewScore(MaxScore, name)
+}
+
+func (v SScore) GetScore() TScore {
+	return v.Score
+}
+
+func (v SScore) String() string {
+	return fmt.Sprintf("%s: %d", v.Name, v.Score)
+}
+
+type Scores struct {
+	scores *list.List
+}
+
+func newScores() *Scores {
+	return &Scores{
+		scores: list.New(),
+	}
+}
+
+func (s *Scores) Append(scores ...SScore) *Scores {
+	for _, score := range scores {
+		s.scores.PushBack(score)
+	}
+	return s
+}
+
+func (s *Scores) AddToFirst(score SScore) *Scores {
+	s.scores.PushFront(score)
+	return s
+}
+
+func (s *Scores) Range(iterFunc func(ele *list.Element, score SScore) bool) {
+	for ele := s.scores.Front(); ele != nil; ele = ele.Next() {
+		cont := iterFunc(ele, ele.Value.(SScore))
+		if !cont {
+			break
+		}
+	}
+}
+
+func (s *Scores) SetScore(score SScore) *Scores {
+	exists := false
+	rf := func(ele *list.Element, oscore SScore) bool {
+		if oscore.Name == score.Name {
+			exists = true
+			oscore.Score = score.Score
+			ele.Value = oscore
+			return false
+		}
+		return true
+	}
+	s.Range(rf)
+	if !exists {
+		s.Append(score)
+	}
+	return s
+}
+
+func (s *Scores) AddScore(score SScore) *Scores {
+	exists := false
+	rf := func(ele *list.Element, oscore SScore) bool {
+		if oscore.Name == score.Name {
+			exists = true
+			oscore.Score += score.Score
+			ele.Value = oscore
+			return false
+		}
+		return true
+	}
+	s.Range(rf)
+	if !exists {
+		s.Append(score)
+	}
+	return s
+}
+
+func (s *Scores) Len() int {
+	return s.scores.Len()
+}
+
+func (s *Scores) GetScores() []SScore {
+	ret := make([]SScore, 0)
+	rf := func(_ *list.Element, score SScore) bool {
+		ret = append(ret, score)
+		return true
+	}
+	s.Range(rf)
+	return ret
+}
+
+type ScoreBucket struct {
+	scores *Scores
+}
+
+func NewScoreBuckets() *ScoreBucket {
+	return &ScoreBucket{
+		scores: newScores(),
+	}
+}
+
+func (b *ScoreBucket) AddToFirst(score SScore) *ScoreBucket {
+	b.scores.AddToFirst(score)
+	return b
+}
+
+func (b *ScoreBucket) Append(scores ...SScore) *ScoreBucket {
+	b.scores.Append(scores...)
+	return b
+}
+
+func (b *ScoreBucket) GetScores() []SScore {
+	return b.scores.GetScores()
+}
+
+func (b *ScoreBucket) SetScore(score SScore) *ScoreBucket {
+	b.scores.SetScore(score)
+	return b
+}
+
+func (b *ScoreBucket) AddScore(score SScore) *ScoreBucket {
+	b.scores.AddScore(score)
+	return b
+}
+
+func (b *ScoreBucket) GetScore(scoreName string) (int, SScore) {
+	for i, oscore := range b.scores.GetScores() {
+		if oscore.Name == scoreName {
+			return i, oscore
+		}
+	}
+	return -1, SScore{}
+}
+
+func (b *ScoreBucket) Len() int {
+	return b.scores.Len()
+}
+
+func (b *ScoreBucket) DigitString() string {
+	s := ""
+	rf := func(_ *list.Element, score SScore) bool {
+		s = fmt.Sprintf("%s%d", s, score.Score)
+		return true
+	}
+	b.scores.Range(rf)
+	return s
+}
+
+func extend(scores []SScore, length int) []SScore {
+	olen := len(scores)
+	if olen >= length {
+		return scores
+	}
+	ret := make([]SScore, 0)
+	zeroDigits := length - olen
+	for i := 0; i < zeroDigits; i++ {
+		ret = append(ret, NewZeroScore())
+	}
+	ret = append(ret, scores...)
+	return ret
+}
+
+func Equal(b1, b2 *ScoreBucket) bool {
+	return compare(b1, b2, func(s1, s2 TScore) bool { return s1 == s2 })
+}
+
+func Less(b1, b2 *ScoreBucket) bool {
+	return compare(b1, b2, func(s1, s2 TScore) bool { return s1 < s2 })
+}
+
+func compare(b1, b2 *ScoreBucket, cf func(s1, s2 TScore) bool) bool {
+	maxLen := int(math.Max(float64(b1.Len()), float64(b2.Len())))
+	s1 := b1.GetScores()
+	s2 := b2.GetScores()
+	s1 = extend(s1, maxLen)
+	s2 = extend(s2, maxLen)
+	for i := range s1 {
+		v1 := s1[i].GetScore()
+		v2 := s2[i].GetScore()
+		ok := cf(v1, v2)
+		if ok {
+			return true
+		} else if !ok {
+			return false
+		}
+	}
+	return false
+}
+
+func (b *ScoreBucket) debugString(vals []SScore, ret string) string {
+	if len(vals) == 0 {
+		return ret
+	}
+	restVal := vals[1:]
+	if len(restVal) == 0 {
+		return vals[0].String()
+	}
+	str := b.debugString(restVal, ret)
+	str = fmt.Sprintf("%s, %s", vals[0].String(), str)
+	return str
+}
+
+func (b *ScoreBucket) String() string {
+	return b.debugString(b.GetScores(), "")
+}
+
+type Interval struct {
+	start int64
+	end   int64
+}
+
+func NewInterval(start, end int64) *Interval {
+	return &Interval{start: start, end: end}
+}
+
+func (i Interval) IsContain(val int64) bool {
+	return val >= i.start && val < i.end
+}
+
+type Intervals struct {
+	MinInterval  *Interval
+	ZeroInterval *Interval
+	MidInterval  *Interval
+	MaxInterval  *Interval
+}
+
+func NewIntervals(min, zero, mid int64) Intervals {
+	return Intervals{
+		MinInterval:  NewInterval(math.MinInt64, min),
+		ZeroInterval: NewInterval(min, zero),
+		MidInterval:  NewInterval(zero, mid),
+		MaxInterval:  NewInterval(mid, math.MaxInt64),
+	}
+}
+
+func (is Intervals) ToScore(val int64) TScore {
+	for score, interval := range map[TScore]*Interval{
+		MinScore:  is.MinInterval,
+		ZeroScore: is.ZeroInterval,
+		MidScore:  is.MidInterval,
+		MaxScore:  is.MaxInterval,
+	} {
+		if interval != nil && interval.IsContain(val) {
+			return score
+		}
+	}
+	return ZeroScore
+}

pkg/scheduler/core/score/score_test.go

@@ -0,0 +1,156 @@
+package score
+
+import (
+	"testing"
+)
+
+func TestScoreBucket_String(t *testing.T) {
+	type fields struct {
+		scores *Scores
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		want   string
+	}{
+		{
+			name:   "EmptyScores",
+			fields: fields{newScores()},
+			want:   "",
+		},
+		{
+			name: "Scores100",
+			fields: fields{newScores().Append(
+				NewMidScore("mid"),
+				NewZeroScore(),
+				NewZeroScore(),
+			)},
+			want: "mid: 1, zero: 0, zero: 0",
+		},
+		{
+			name: "Scores201-1",
+			fields: fields{newScores().Append(
+				NewMaxScore("max"),
+				NewZeroScore(),
+				NewMidScore("mid"),
+				NewMinScore("min"),
+			)},
+			want: "max: 2, zero: 0, mid: 1, min: -1",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			b := &ScoreBucket{
+				scores: tt.fields.scores,
+			}
+			if got := b.String(); got != tt.want {
+				t.Errorf("ScoreBucket.String() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestLess(t *testing.T) {
+	type args struct {
+		b1 *ScoreBucket
+		b2 *ScoreBucket
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "equal",
+			args: args{
+				b1: NewScoreBuckets(),
+				b2: NewScoreBuckets(),
+			},
+			want: false,
+		},
+		{
+			name: "extendEqual",
+			args: args{
+				b1: NewScoreBuckets().Append(
+					NewZeroScore(), NewMidScore("1"),
+				),
+				b2: NewScoreBuckets().Append(NewMidScore("1")),
+			},
+			want: false,
+		},
+		{
+			name: "10<100",
+			args: args{
+				b1: NewScoreBuckets().Append(
+					NewMidScore("1"),
+					NewZeroScore(),
+				),
+				b2: NewScoreBuckets().Append(
+					NewMidScore("1"),
+					NewZeroScore(),
+					NewZeroScore(),
+				),
+			},
+			want: true,
+		},
+		{
+			name: "101>10",
+			args: args{
+				b1: NewScoreBuckets().Append(
+					NewMidScore("1"),
+					NewZeroScore(),
+					NewMidScore("1"),
+				),
+				b2: NewScoreBuckets().Append(
+					NewMidScore("1"),
+					NewZeroScore(),
+				),
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := Less(tt.args.b1, tt.args.b2); got != tt.want {
+				t.Errorf("Less() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestScoreBucket_DigitString(t *testing.T) {
+	type fields struct {
+		scores *Scores
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		want   string
+	}{
+		{
+			name: "2-101",
+			fields: fields{newScores().Append(
+				NewMaxScore(""),
+				NewMinScore(""),
+				NewZeroScore(),
+				NewMidScore(""),
+			)},
+			want: "2-101",
+		},
+		{
+			name:   "empty",
+			fields: fields{newScores()},
+			want:   "",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			b := &ScoreBucket{
+				scores: tt.fields.scores,
+			}
+			if got := b.DigitString(); got != tt.want {
+				t.Errorf("ScoreBucket.DigitString() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/scheduler/core/types.go

@@ -1,15 +1,14 @@
 package core
 
 import (
-	//"sync"
-	//"yunion.io/x/onecloud/pkg/scheduler/cache/candidate"
 	"yunion.io/x/jsonutils"
 
+	"yunion.io/x/onecloud/pkg/scheduler/core/score"
 	"yunion.io/x/onecloud/pkg/scheduler/db/models"
 )
 
 const (
-	PriorityStep int = 100
+	PriorityStep int = 1
 )
 
 type FailedCandidate struct {
@@ -23,7 +22,8 @@ type FailedCandidates struct {
 }
 
 type SelectPlugin interface {
-	OnSelect(*Unit, Candidater) bool
+	Name() string
+	OnPriorityEnd(*Unit, Candidater)
 	OnSelectEnd(u *Unit, c Candidater, count int64)
 }
 
@@ -58,7 +58,7 @@ type HostPriority struct {
 	// Name of the host
 	Host string
 	// Score associated with the host
-	Score int
+	Score Score
 	// Resource wraps Candidate host info
 	Candidate Candidater
 }
@@ -70,10 +70,10 @@ func (h HostPriorityList) Len() int {
 }
 
 func (h HostPriorityList) Less(i, j int) bool {
-	if h[i].Score == h[j].Score {
+	if score.Equal(h[i].Score.ScoreBucket, h[j].Score.ScoreBucket) {
 		return h[i].Host < h[j].Host
 	}
-	return h[i].Score < h[j].Score
+	return score.Less(h[i].Score.ScoreBucket, h[j].Score.ScoreBucket)
 }
 
 func (h HostPriorityList) Swap(i, j int) {
@@ -115,4 +115,7 @@ type Priority interface {
 	Map(*Unit, Candidater) (HostPriority, error)
 	Reduce(*Unit, []Candidater, HostPriorityList) error
 	PreExecute(*Unit, []Candidater) (bool, []PredicateFailureReason, error)
+
+	// Score intervals
+	ScoreIntervals() score.Intervals
 }

pkg/util/aliyun/aliyun.go

@@ -8,9 +8,10 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/utils"
+
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
-	"yunion.io/x/pkg/utils"
 )
 
 const (