Merge branch 'release/2.1.0' of ssh://git.yunion.io/~quxuan/onecloud into feature/server-disk

Gopkg.toml

@@ -34,7 +34,6 @@
   name = "yunion.io/x/jsonutils"
 
 [[constraint]]
-  #revision = "a14641a90b9e5c72868b7a8358d62ea1d83eab53"
   branch = "master"
   name = "yunion.io/x/sqlchemy"
 
@@ -43,8 +42,7 @@
   name = "yunion.io/x/log"
 
 [[constraint]]
-  revision = "07cad0cb1df5155b3bfa15079594d01050931554"
-  #branch = "master"
+  branch = "master"
   name = "yunion.io/x/pkg"
 
 [[constraint]]

Makefile

@@ -3,7 +3,7 @@
 REPO_PREFIX := yunion.io/x/onecloud
 VENDOR_PATH := $(REPO_PREFIX)/vendor
 VERSION_PKG := $(VENDOR_PATH)/yunion.io/x/pkg/util/version
-ROOT_DIR := $(shell readlink -f `pwd`)
+ROOT_DIR := $(CURDIR)
 BUILD_DIR := $(ROOT_DIR)/_output
 BIN_DIR := $(BUILD_DIR)/bin
 BUILD_SCRIPT := $(ROOT_DIR)/build/build.sh
@@ -83,10 +83,6 @@ bin_dir: output_dir
 output_dir:
 	@mkdir -p $(BUILD_DIR)
 
-
-dep:
-	cd $(ROOT_DIR) && dep ensure -v
-
 dep_clean:
 	rm -fr $(GOPATH)/pkg/dep/sources/*
 
@@ -105,5 +101,8 @@ fmt:
 	find . -type f -name "*.go" -not -path "./_output/*" \
 		-not -path "./vendor/*" | xargs gofmt -s -w
 
+dep:
+	cd $(ROOT_DIR) && dep ensure -v -update $(shell for p in $$(ls vendor/yunion.io/x/); do echo "yunion.io/x/$$p"; done | xargs)
+
 %:
 	@:

build/region-dns/vars

@@ -0,0 +1,2 @@
+DESCRIPTION="Yunion Cloud Region DNS Service"
+# SERVICE="yes"

cmd/climc/climc.go

@@ -5,28 +5,32 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
 	prompt "github.com/c-bata/go-prompt"
+
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/util/version"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/cmd/climc/promputils"
 	"yunion.io/x/onecloud/cmd/climc/shell"
-	_ "yunion.io/x/onecloud/cmd/climc/shell/k8s"
 	"yunion.io/x/onecloud/pkg/mcclient"
+
+	_ "yunion.io/x/onecloud/cmd/climc/shell/k8s"
 )
 
 type BaseOptions struct {
-	Help       bool   `help:"Show help" short-token:"h"`
-	Debug      bool   `help:"Show debug information"`
-	Version    bool   `help:"Show version"`
-	Timeout    int    `default:"600" help:"Number of seconds to wait for a response"`
-	Secure     bool   `default:"False" help:"do server cert verification if URL is https"`
-	OsUsername string `default:"$OS_USERNAME" help:"Username, defaults to env[OS_USERNAME]"`
-	OsPassword string `default:"$OS_PASSWORD" help:"Password, defaults to env[OS_PASSWORD]"`
+	Help          bool   `help:"Show help" short-token:"h"`
+	Debug         bool   `help:"Show debug information"`
+	Version       bool   `help:"Show version"`
+	Timeout       int    `default:"600" help:"Number of seconds to wait for a response"`
+	Insecure      bool   `default:"false" help:"Allow skip server cert verification if URL is https" short-token:"k"`
+	NoCachedToken bool   `default:"false" help:"Force not use cached token"`
+	OsUsername    string `default:"$OS_USERNAME" help:"Username, defaults to env[OS_USERNAME]"`
+	OsPassword    string `default:"$OS_PASSWORD" help:"Password, defaults to env[OS_PASSWORD]"`
 	// OsProjectId string `default:"$OS_PROJECT_ID" help:"Proejct ID, defaults to env[OS_PROJECT_ID]"`
 	OsProjectName  string `default:"$OS_PROJECT_NAME" help:"Project name, defaults to env[OS_PROJECT_NAME]"`
 	OsDomainName   string `default:"$OS_DOMAIN_NAME" help:"Domain name, defaults to env[OS_DOMAIN_NAME]"`
@@ -119,11 +123,12 @@ func newClientSession(options *BaseOptions) (*mcclient.ClientSession, error) {
 	client := mcclient.NewClient(options.OsAuthURL,
 		options.Timeout,
 		options.Debug,
-		options.Secure)
+		options.Insecure)
 
 	var cacheToken mcclient.TokenCredential
-	cacheFile, err := os.Open("/tmp/OS_AUTH_CACHE_TOKEN")
-	if err == nil && cacheFile != nil {
+	tokenCachePath := filepath.Join(os.TempDir(), "OS_AUTH_CACHE_TOKEN")
+	cacheFile, err := os.Open(tokenCachePath)
+	if err == nil && cacheFile != nil && !options.NoCachedToken {
 		fileInfo, _ := cacheFile.Stat()
 		dur, err := time.ParseDuration("-24h")
 		if fileInfo != nil && err == nil && fileInfo.ModTime().After(time.Now().Add(dur)) {
@@ -153,12 +158,12 @@ func newClientSession(options *BaseOptions) (*mcclient.ClientSession, error) {
 		if err != nil {
 			fmt.Printf("Marshal token error:%s", err)
 		} else {
-			fo, _ := os.Create("/tmp/OS_AUTH_CACHE_TOKEN")
+			fo, _ := os.Create(tokenCachePath)
 			fo.Write(bytesCacheToken)
 			fo.Close()
 		}
 	} else {
-		fmt.Println("******** Use Token Cache At /tmp/OS_AUTH_CACHE_TOKEN ********")
+		fmt.Printf("******** Use Token Cache At %s ********\n", tokenCachePath)
 	}
 
 	session := client.NewSession(options.OsRegionName,
@@ -169,6 +174,40 @@ func newClientSession(options *BaseOptions) (*mcclient.ClientSession, error) {
 	return session, nil
 }
 
+func enterInteractiveMode(
+	parser *structarg.ArgumentParser,
+	sessionFactory func() *mcclient.ClientSession,
+) {
+	promputils.InitEnv(parser, sessionFactory())
+	defer fmt.Println("Bye!")
+	p := prompt.New(
+		promputils.Executor,
+		promputils.Completer,
+		prompt.OptionPrefix("climc> "),
+		prompt.OptionTitle("Climc, a Command Line Interface to Manage Clouds"),
+		prompt.OptionMaxSuggestion(16),
+	)
+	p.Run()
+}
+
+func executeSubcommand(
+	subcmd *structarg.SubcommandArgument,
+	subparser *structarg.ArgumentParser,
+	options *BaseOptions,
+	sessionFactory func() *mcclient.ClientSession,
+) {
+	var e error
+	suboptions := subparser.Options()
+	if options.SUBCOMMAND == "help" {
+		e = subcmd.Invoke(suboptions)
+	} else {
+		e = subcmd.Invoke(sessionFactory(), suboptions)
+	}
+	if e != nil {
+		showErrorAndExit(e)
+	}
+}
+
 func main() {
 	parser, e := getSubcommandsParser()
 	if e != nil {
@@ -179,47 +218,39 @@ func main() {
 
 	if options.Help {
 		fmt.Print(parser.HelpString())
-	} else if options.Version {
-		fmt.Printf("Yunion API client version:\n %s\n", version.GetJsonString())
-	} else if len(options.SUBCOMMAND) == 0 {
-		session, e := newClientSession(options)
-		if e != nil {
-			showErrorAndExit(e)
-		}
-		promputils.InitEnv(parser, session)
-		defer fmt.Println("Bye!")
-		p := prompt.New(
-			promputils.Executor,
-			promputils.Completer,
-			prompt.OptionPrefix("climc> "),
-			prompt.OptionTitle("Climc, a Command Line Interface to Manage Clouds"),
-			prompt.OptionMaxSuggestion(16),
-		)
-		p.Run()
-	} else {
-		subcmd := parser.GetSubcommand()
-		subparser := subcmd.GetSubParser()
-		if e != nil {
-			if subparser != nil {
-				fmt.Print(subparser.Usage())
-			} else {
-				fmt.Print(parser.Usage())
-			}
-			showErrorAndExit(e)
-		} else {
-			session, e := newClientSession(options)
-			if e != nil {
-				showErrorAndExit(e)
-			}
-			suboptions := subparser.Options()
-			if options.SUBCOMMAND == "help" {
-				e = subcmd.Invoke(suboptions)
-			} else {
-				e = subcmd.Invoke(session, suboptions)
-			}
-			if e != nil {
-				showErrorAndExit(e)
-			}
-		}
+		return
 	}
+
+	if options.Version {
+		fmt.Printf("Yunion API client version:\n %s\n", version.GetJsonString())
+		return
+	}
+
+	ensureSessionFactory := func() *mcclient.ClientSession {
+		session, err := newClientSession(options)
+		if err != nil {
+			showErrorAndExit(err)
+		}
+		return session
+	}
+
+	// enter interactive mode when not enough argument and SUBCOMMAND is empty
+	if _, ok := e.(*structarg.NotEnoughArgumentsError); ok && options.SUBCOMMAND == "" {
+		enterInteractiveMode(parser, ensureSessionFactory)
+		return
+	}
+
+	subcmd := parser.GetSubcommand()
+	subparser := subcmd.GetSubParser()
+	if e != nil {
+		if subparser != nil {
+			fmt.Print(subparser.Usage())
+		} else {
+			fmt.Print(parser.Usage())
+		}
+		showErrorAndExit(e)
+	}
+
+	// execute subcommand in non-interactive mode
+	executeSubcommand(subcmd, subparser, options, ensureSessionFactory)
 }

cmd/climc/promputils/executor.go

@@ -9,6 +9,7 @@ import (
 	"regexp"
 	"strings"
 
+	"yunion.io/x/pkg/utils"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -51,8 +52,8 @@ func Executor(s string) {
 		return
 	}
 	s = escaper(s)
-	args := strings.Split(s, " ")
-	e := parser.ParseArgs(strings.Split(s, " "), false)
+	args := utils.ArgsStringToArray(s)
+	e := parser.ParseArgs(utils.ArgsStringToArray(s), false)
 	subcmd := parser.GetSubcommand()
 	subparser := subcmd.GetSubParser()
 	if args[0] == "--debug" {

cmd/climc/shell/cloudproviders.go

@@ -140,12 +140,12 @@ func init() {
 	})
 
 	type CloudproviderSyncOptions struct {
-		ID     string   `help:"ID or Name of cloud provider"`
-		Force  bool     `help:"Force sync no matter what"`
-		FullSync bool   `help:"Synchronize everything"`
-		Region []string `help:"region to sync"`
-		Zone   []string `help:"region to sync"`
-		Host   []string `help:"region to sync"`
+		ID       string   `help:"ID or Name of cloud provider"`
+		Force    bool     `help:"Force sync no matter what"`
+		FullSync bool     `help:"Synchronize everything"`
+		Region   []string `help:"region to sync"`
+		Zone     []string `help:"region to sync"`
+		Host     []string `help:"region to sync"`
 	}
 	R(&CloudproviderSyncOptions{}, "cloud-provider-sync", "Sync of a cloud provider account", func(s *mcclient.ClientSession, args *CloudproviderSyncOptions) error {
 		params := jsonutils.NewDict()

cmd/climc/shell/dnsrecords.go

@@ -10,12 +10,12 @@ import (
 )
 
 type DNSRecordOptions struct {
-	A       []string `help:"DNS A record" metavar:"A_RECORD" optional:"true"`
-	AAAA    []string `help:"DNS AAAA record" metavar:"AAAA_RECORD" optional:"true"`
-	CNAME   string   `help:"DNS CNAME record" metavar:"CNAME_RECORD" optional:"true"`
-	SRVHost string   `help:"DNS SRV record, server of service" metavar:"SRV_RECORD_HOST" optional:"true"`
-	SRVPort int64    `help:"DNS SRV record, port of service" metavar:"SRV_RECORD_PORT" optional:"true"`
-	PTR     string   `help:"DNS PTR record" metavar:"PTR_RECORD" optional:"true"`
+	A       []string `help:"DNS A record" metavar:"A_RECORD" positional:"false"`
+	AAAA    []string `help:"DNS AAAA record" metavar:"AAAA_RECORD" positional:"false"`
+	CNAME   string   `help:"DNS CNAME record" metavar:"CNAME_RECORD" positional:"false"`
+	SRVHost string   `help:"DNS SRV record, server of service" metavar:"SRV_RECORD_HOST" positional:"false"`
+	SRVPort int64    `help:"DNS SRV record, port of service" metavar:"SRV_RECORD_PORT" positional:"false"`
+	PTR     string   `help:"DNS PTR record" metavar:"PTR_RECORD" positional:"false"`
 }
 
 func parseDNSRecords(args *DNSRecordOptions, params *jsonutils.JSONDict) {
@@ -52,7 +52,7 @@ func init() {
 
 	type DNSCreateOptions struct {
 		NAME string `help:"DNS name to create"`
-		TTL  int64  `help:"TTL in seconds" optional:"true"`
+		TTL  int64  `help:"TTL in seconds" positional:"false"`
 		Desc string `help:"Description" metavar:"DESCRIPTION"`
 		DNSRecordOptions
 	}
@@ -92,7 +92,7 @@ func init() {
 	type DNSUpdateOptions struct {
 		ID   string `help:"ID of DNS record to update"`
 		Name string `help:"Domain name"`
-		TTL  int64  `help:"TTL in seconds" optional:"true"`
+		TTL  int64  `help:"TTL in seconds" positional:"false"`
 		Desc string `help:"Description"`
 	}
 	R(&DNSUpdateOptions{}, "dns-update", "Update details of a dns records", func(s *mcclient.ClientSession, args *DNSUpdateOptions) error {

cmd/climc/shell/k8s/charts.go

@@ -0,0 +1,63 @@
+package k8s
+
+import (
+	"fmt"
+
+	json "yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/k8s"
+)
+
+func initChart() {
+	cmdN := func(suffix string) string {
+		return resourceCmdN("chart", suffix)
+	}
+
+	type listOpt struct {
+		baseListOptions
+		Name       string `help:"Chart name"`
+		Repo       string `help:"Repository name"`
+		RepoUrl    string `help:"Repository url"`
+		AllVersion bool   `json:"Get Chart all history versions"`
+		Keyword    string `json:"Chart keyword"`
+	}
+	R(&listOpt{}, cmdN("list"), "List k8s helm global charts", func(s *mcclient.ClientSession, args *listOpt) error {
+		params := fetchPagingParams(args.baseListOptions)
+		if len(args.Name) != 0 {
+			params.Add(json.NewString(args.Name), "name")
+		}
+		if len(args.Repo) != 0 {
+			params.Add(json.NewString(args.Repo), "repo")
+		}
+		if len(args.RepoUrl) != 0 {
+			params.Add(json.NewString(args.RepoUrl), "repo_url")
+		}
+		if args.AllVersion {
+			params.Add(json.JSONTrue, "all_version")
+		}
+		if len(args.Keyword) != 0 {
+			params.Add(json.NewString(args.Keyword), "keyword")
+		}
+		charts, err := k8s.Charts.List(s, params)
+		if err != nil {
+			return err
+		}
+		PrintHelmListResult(charts, ChartPrinter{})
+		return nil
+	})
+}
+
+type ChartPrinter struct{}
+
+func (p ChartPrinter) Columns() []interface{} {
+	return []interface{}{"REPO/NAME", "VERSION", "DESCRIPTION"}
+}
+
+func (p ChartPrinter) RowKeys(obj *json.JSONDict) []interface{} {
+	repo, _ := obj.GetString("repo")
+	name, _ := obj.GetString("chart", "metadata", "name")
+	version, _ := obj.GetString("chart", "metadata", "version")
+	desc, _ := obj.GetString("chart", "metadata", "description")
+	return []interface{}{fmt.Sprintf("%s/%s", repo, name), version, desc}
+}

cmd/climc/shell/k8s/configmap.go

@@ -0,0 +1,27 @@
+package k8s
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/k8s"
+)
+
+func initConfigMap() {
+	cmdN := func(suffix string) string {
+		return resourceCmdN("configmap", suffix)
+	}
+
+	type listOpt struct {
+		namespaceListOptions
+		baseListOptions
+	}
+	R(&listOpt{}, cmdN("list"), "List k8s configmap", func(s *mcclient.ClientSession, args *listOpt) error {
+		params := fetchNamespaceParams(args.namespaceListOptions)
+		params.Update(fetchPagingParams(args.baseListOptions))
+		ret, err := k8s.ConfigMaps.ListInContexts(s, params, args.ClusterContext())
+		if err != nil {
+			return err
+		}
+		printList(ret, k8s.ConfigMaps.GetColumns(s))
+		return nil
+	})
+}

cmd/climc/shell/k8s/deployment.go

@@ -0,0 +1,145 @@
+package k8s
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/util/sets"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/k8s"
+)
+
+func initDeployment() {
+	cmdN := func(suffix string) string {
+		return resourceCmdN("deployment", suffix)
+	}
+
+	type listOpt struct {
+		namespaceListOptions
+		baseListOptions
+	}
+	R(&listOpt{}, cmdN("list"), "List k8s deployment", func(s *mcclient.ClientSession, args *listOpt) error {
+		params := fetchNamespaceParams(args.namespaceListOptions)
+		params.Update(fetchPagingParams(args.baseListOptions))
+		ret, err := k8s.Deployments.ListInContexts(s, params, args.ClusterContext())
+		if err != nil {
+			return err
+		}
+		printList(ret, k8s.Deployments.GetColumns(s))
+		return nil
+	})
+
+	type createOpt struct {
+		namespaceOptions
+		NAME            string   `help:"Name of deployment"`
+		Image           string   `help:"The image for the container to run"`
+		Replicas        int64    `help:"Number of replicas for pods in this deployment"`
+		RunAsPrivileged bool     `help:"Whether to run the container as privileged user"`
+		Labels          string   `help:"Comma separated labels to apply to the pod(s)"`
+		Env             []string `help:"Environment variables to set in container"`
+		Port            []string `help:"Port for the service that is created, format is <protocol>:<service_port>:<container_port> e.g. tcp:80:3000"`
+	}
+	R(&createOpt{}, cmdN("create"), "Create deployment resource", func(s *mcclient.ClientSession, args *createOpt) error {
+		params := jsonutils.NewDict()
+		if len(args.Image) == 0 {
+			return fmt.Errorf("Image must provided")
+		}
+		params.Add(jsonutils.NewString(args.NAME), "name")
+		params.Add(jsonutils.NewString(args.Image), "containerImage")
+		if args.Namespace != "" {
+			params.Add(jsonutils.NewString(args.Namespace), "namespace")
+		}
+		if args.Replicas > 1 {
+			params.Add(jsonutils.NewInt(args.Replicas), "replicas")
+		}
+		if args.RunAsPrivileged {
+			params.Add(jsonutils.JSONTrue, "runAsPrivileged")
+		}
+		if len(args.Port) != 0 {
+			portMappings, err := parsePortMappings(args.Port)
+			if err != nil {
+				return err
+			}
+			params.Add(portMappings, "portMappings")
+		}
+		ret, err := k8s.Deployments.CreateInContexts(s, params, args.ClusterContext())
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
+
+	type getOpt struct {
+		resourceGetOptions
+	}
+	R(&getOpt{}, cmdN("show"), "Get deployment details", func(s *mcclient.ClientSession, args *getOpt) error {
+		id := args.NAME
+		params := jsonutils.NewDict()
+		if args.Namespace != "" {
+			params.Add(jsonutils.NewString(args.Namespace), "namespace")
+		}
+		ret, err := k8s.Deployments.GetInContexts(s, id, params, args.ClusterContext())
+		if err != nil {
+			return err
+		}
+		printObjectYAML(ret)
+		return nil
+	})
+}
+
+type portMapping struct {
+	Port       int32  `json:"port"`
+	TargetPort int32  `json:"targetPort"`
+	Protocol   string `json:"protocol"`
+}
+
+func parsePortMapping(port string) (*portMapping, error) {
+	if len(port) == 0 {
+		return nil, fmt.Errorf("empty port mapping desc string")
+	}
+	parts := strings.Split(port, ":")
+	mapping := &portMapping{}
+	for _, part := range parts {
+		if sets.NewString("tcp", "udp").Has(strings.ToLower(part)) {
+			mapping.Protocol = strings.ToUpper(part)
+		}
+		if port, err := strconv.Atoi(part); err != nil {
+			continue
+		} else {
+			if mapping.Port == 0 {
+				mapping.Port = int32(port)
+			} else {
+				mapping.TargetPort = int32(port)
+			}
+		}
+	}
+	if mapping.Protocol == "" {
+		mapping.Protocol = "TCP"
+	}
+	if mapping.Port <= 0 {
+		return nil, fmt.Errorf("Service port not provided")
+	}
+	if mapping.TargetPort < 0 {
+		return nil, fmt.Errorf("Container invalid targetPort %d", mapping.TargetPort)
+	}
+	if mapping.TargetPort == 0 {
+		mapping.TargetPort = mapping.Port
+	}
+	return mapping, nil
+}
+
+func parsePortMappings(ports []string) (*jsonutils.JSONArray, error) {
+	ret := jsonutils.NewArray()
+	for _, port := range ports {
+		mapping, err := parsePortMapping(port)
+		if err != nil {
+			return nil, fmt.Errorf("Port %q error: %v", port, err)
+		}
+		ret.Add(jsonutils.Marshal(mapping))
+	}
+	return ret, nil
+}

cmd/climc/shell/k8s/helm_list.go

@@ -0,0 +1,51 @@
+package k8s
+
+import (
+	"fmt"
+
+	"github.com/gosuri/uitable"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+type ListPrinter interface {
+	Columns() []interface{}
+	RowKeys(obj *jsonutils.JSONDict) []interface{}
+}
+
+func ListerTable(res *modules.ListResult, printer ListPrinter) *uitable.Table {
+	min := func(x, y int) int {
+		if x < y {
+			return x
+		}
+		return y
+	}
+	table := uitable.New()
+	table.MaxColWidth = 80
+	table.AddRow(printer.Columns()...)
+	var idx int
+	for ; idx < min(res.Limit, res.Total); idx++ {
+		table.AddRow(printer.RowKeys(res.Data[idx].(*jsonutils.JSONDict))...)
+	}
+	return table
+}
+
+func PrintHelmListResult(res *modules.ListResult, printer ListPrinter) {
+	fmt.Println(ListerTable(res, printer))
+
+	table := uitable.New()
+	total := res.Total
+	offset := res.Offset
+	limit := res.Limit
+	page := (offset / limit) + 1
+	pages := total / limit
+	if pages*limit < total {
+		pages += 1
+	}
+	table.AddRow("")
+	table.AddRow("Total", "Pages", "Limit", "Offset", "Page")
+	table.AddRow(total, pages, limit, offset, page)
+	fmt.Println(table)
+}

cmd/climc/shell/k8s/k8s.go

@@ -12,24 +12,84 @@ import (
 )
 
 func init() {
+	// cluster resources
 	initCluster()
 	initNode()
+
+	// helm resources
+	initTiller()
+	initRepo()
+	initChart()
+	initRelease()
+
+	// kubernetes original resources
+	initRaw()
+	initConfigMap()
+	initDeployment()
 	initPod()
+	initService()
 }
 
 type BaseListOptions shell.BaseListOptions
 
 type clusterBaseOptions struct {
-	Cluster string `default:"$K8S_CLUSTER" help:"Kubernetes cluster name"`
+	Cluster string `default:"$K8S_CLUSTER|default" help:"Kubernetes cluster name"`
 }
 
 func (o clusterBaseOptions) ClusterContext() []modules.ManagerContext {
 	return []modules.ManagerContext{clusterContext(o.Cluster)}
 }
 
-type k8sBaseListOptions struct {
+type baseListOptions struct {
+	Limit  int `default:"20" help:"Page limit"`
+	Offset int `default:"0" help:"page offset"`
+}
+
+func fetchPagingParams(opt baseListOptions) *jsonutils.JSONDict {
+	params := jsonutils.NewDict()
+	if opt.Limit > 0 {
+		params.Add(jsonutils.NewInt(int64(opt.Limit)), "limit")
+	}
+	if opt.Offset > 0 {
+		params.Add(jsonutils.NewInt(int64(opt.Offset)), "offset")
+	}
+	return params
+}
+
+type namespaceListOptions struct {
+	namespaceOptions
+	AllNamespace bool `help:"Show resource in all namespace"`
+}
+
+type namespaceOptions struct {
 	clusterBaseOptions
-	Limit int `default:"20" help:"Page limit"`
+	Namespace string `help:"Namespace of this resource"`
+}
+
+type resourceGetOptions struct {
+	clusterBaseOptions
+	Namespace string `help:"Namespace of this resource"`
+	NAME      string `help:"Name ident of the resource"`
+}
+
+func (o resourceGetOptions) ToJSON() *jsonutils.JSONDict {
+	params := jsonutils.NewDict()
+	if o.Namespace != "" {
+		params.Add(jsonutils.NewString(o.Namespace), "namespace")
+	}
+	return params
+}
+
+func fetchNamespaceParams(opt namespaceListOptions) *jsonutils.JSONDict {
+	params := jsonutils.NewDict()
+	if opt.AllNamespace {
+		params.Add(jsonutils.JSONTrue, "all_namespace")
+		return params
+	}
+	if opt.Namespace != "" {
+		params.Add(jsonutils.NewString(opt.Namespace), "namespace")
+	}
+	return params
 }
 
 var (
@@ -53,3 +113,7 @@ func clusterContext(clusterId string) modules.ManagerContext {
 		InstanceId:      clusterId,
 	}
 }
+
+func printObjectYAML(obj jsonutils.JSONObject) {
+	fmt.Println(obj.YAMLString())
+}

cmd/climc/shell/k8s/pods.go

@@ -13,10 +13,26 @@ func initPod() {
 	}
 
 	type listOpt struct {
-		k8sBaseListOptions
+		namespaceListOptions
+		baseListOptions
 	}
 	R(&listOpt{}, cmdN("list"), "List k8s pod", func(s *mcclient.ClientSession, args *listOpt) error {
-		ret, err := k8s.Pods.ListInContexts(s, nil, args.ClusterContext())
+		params := fetchNamespaceParams(args.namespaceListOptions)
+		params.Update(fetchPagingParams(args.baseListOptions))
+		ret, err := k8s.Pods.ListInContexts(s, params, args.ClusterContext())
+		if err != nil {
+			return err
+		}
+		printList(ret, k8s.Pods.GetColumns(s))
+		return nil
+	})
+
+	type deleteOpt struct {
+		resourceGetOptions
+	}
+	R(&deleteOpt{}, cmdN("delete"), "Delete pod", func(s *mcclient.ClientSession, args *deleteOpt) error {
+		id := args.NAME
+		ret, err := k8s.Pods.DeleteInContexts(s, id, args.ToJSON(), args.ClusterContext())
 		if err != nil {
 			return err
 		}

cmd/climc/shell/k8s/raw.go

@@ -0,0 +1,35 @@
+package k8s
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/k8s"
+)
+
+type rawGetOpt struct {
+	namespaceOptions
+	KIND string `help:"resource kind"`
+	NAME string `help:"instance name"`
+}
+
+type rawDeleteOpt struct {
+	rawGetOpt
+}
+
+func initRaw() {
+	R(&rawGetOpt{}, "k8s-get", "Get k8s resource instance raw info", func(s *mcclient.ClientSession, args *rawGetOpt) error {
+		obj, err := k8s.RawResource.Get(s, args.KIND, args.Namespace, args.NAME, nil, args.ClusterContext())
+		if err != nil {
+			return err
+		}
+		printObjectYAML(obj)
+		return nil
+	})
+
+	R(&rawDeleteOpt{}, "k8s-delete", "Delete k8s resource instance", func(s *mcclient.ClientSession, args *rawDeleteOpt) error {
+		err := k8s.RawResource.Delete(s, args.KIND, args.Namespace, args.NAME, nil, args.ClusterContext())
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+}

cmd/climc/shell/k8s/releases.go

@@ -0,0 +1,177 @@
+package k8s
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	json "yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/k8s"
+)
+
+func initRelease() {
+	cmdN := func(suffix string) string {
+		return resourceCmdN("release", suffix)
+	}
+	type listOpt struct {
+		namespaceListOptions
+		baseListOptions
+		Filter     string `help:"Filter, split by space"`
+		Admin      bool   `help:"Admin to show all namespace releases"`
+		Deployed   bool   `help:"Show deployed status releases"`
+		Deleted    bool   `help:"Show deleted status releases"`
+		Deleting   bool   `help:"Show deleting status releases"`
+		Failed     bool   `help:"Show failed status releases"`
+		Superseded bool   `help:"Show superseded status releases"`
+		Pending    bool   `help:"Show pending status releases"`
+	}
+	R(&listOpt{}, cmdN("list"), "List k8s cluster helm releases", func(s *mcclient.ClientSession, args *listOpt) error {
+		params := fetchNamespaceParams(args.namespaceListOptions)
+		params.Update(fetchPagingParams(args.baseListOptions))
+		if args.Filter != "" {
+			params.Add(json.NewString(args.Filter), "filter")
+		}
+		if args.Namespace != "" {
+			params.Add(json.NewString(args.Namespace), "namespace")
+		}
+		params.Add(json.JSONTrue, "all")
+		if args.Admin {
+			params.Add(json.JSONTrue, "admin")
+		}
+		if args.Deployed {
+			params.Add(json.JSONTrue, "deployed")
+		}
+		if args.Deleted {
+			params.Add(json.JSONTrue, "deleted")
+		}
+		if args.Deleting {
+			params.Add(json.JSONTrue, "deleting")
+		}
+		if args.Failed {
+			params.Add(json.JSONTrue, "failed")
+		}
+		if args.Superseded {
+			params.Add(json.JSONTrue, "superseded")
+		}
+		if args.Pending {
+			params.Add(json.JSONTrue, "pending")
+		}
+		ret, err := k8s.Releases.ListInContexts(s, params, args.ClusterContext())
+		if err != nil {
+			return err
+		}
+		printList(ret, k8s.Releases.GetColumns(s))
+		return nil
+	})
+
+	type showOpt struct {
+		clusterBaseOptions
+		NAME string `help:"Release instance name"`
+	}
+	R(&showOpt{}, cmdN("show"), "Get helm release details", func(s *mcclient.ClientSession, args *showOpt) error {
+		ret, err := k8s.Releases.GetInContexts(s, args.NAME, nil, args.ClusterContext())
+		if err != nil {
+			return err
+		}
+		resources, err := ret.GetString("info", "status", "resources")
+		if err != nil {
+			return err
+		}
+		fmt.Println(resources)
+		return nil
+	})
+
+	type releaseCUOpts struct {
+		Values  string `help:"Specify values in a YAML file (can specify multiple)" short-token:"f"`
+		Version string `help:"Specify the exact chart version to install. If not specified, latest version installed"`
+		//Set     []string `help:"set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)"`
+		DryRun  bool  `help:"Simulate an install"`
+		Details bool  `help:"Show release deploy details, include kubernetes created resources"`
+		Timeout int64 `help:"Time in seconds to wait for any individual kubernetes operation (like Jobs for hooks)" default:"600"`
+	}
+	releaseCUDict := func(args releaseCUOpts) (*json.JSONDict, error) {
+		params := json.NewDict()
+		if args.Version != "" {
+			params.Add(json.NewString(args.Version), "version")
+		}
+		if args.DryRun {
+			params.Add(json.JSONTrue, "dry_run")
+		}
+		params.Add(json.NewInt(args.Timeout), "timeout")
+		if args.Values != "" {
+			//vals, err := helm.MergeValuesF(args.Values, args.Set, []string{})
+			vals, err := ioutil.ReadFile(args.Values)
+			if err != nil {
+				return nil, err
+			}
+			params.Add(json.NewString(string(vals)), "values")
+		}
+		return params, nil
+	}
+
+	type releaseCreateOpts struct {
+		namespaceOptions
+		releaseCUOpts
+		Name      string `help:"Release name, If unspecified, it will autogenerate one for you"`
+		CHARTNAME string `help:"Helm chart name, e.g stable/etcd"`
+	}
+	R(&releaseCreateOpts{}, cmdN("create"), "Create release with specified helm chart", func(s *mcclient.ClientSession, args *releaseCreateOpts) error {
+		params, err := releaseCUDict(args.releaseCUOpts)
+		if err != nil {
+			return err
+		}
+		params.Add(json.NewString(args.CHARTNAME), "chart_name")
+		if args.Namespace != "" {
+			params.Add(json.NewString(args.Namespace), "namespace")
+		}
+		if args.Name != "" {
+			params.Add(json.NewString(args.Name), "release_name")
+		}
+		ret, err := k8s.Releases.CreateInContexts(s, params, args.ClusterContext())
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
+
+	type releaseUpgradeOpts struct {
+		clusterBaseOptions
+		releaseCUOpts
+		NAME        string `help:"Release instance name"`
+		CHARTNAME   string `help:"Helm chart name, e.g stable/etcd"`
+		ReuseValues bool   `help:"When upgrading, reuse the last release's values, and merge in any new values. If '--reset-values' is specified, this is ignored"`
+		ResetValues bool   `help:"When upgrading, reset the values to the ones built into the chart"`
+	}
+	R(&releaseUpgradeOpts{}, cmdN("upgrade"), "Upgrade release", func(s *mcclient.ClientSession, args *releaseUpgradeOpts) error {
+		params, err := releaseCUDict(args.releaseCUOpts)
+		if err != nil {
+			return err
+		}
+		params.Add(json.NewString(args.CHARTNAME), "chart_name")
+		params.Add(json.NewString(args.NAME), "release_name")
+		if args.ReuseValues {
+			params.Add(json.JSONTrue, "reuse_values")
+		}
+		if args.ResetValues {
+			params.Add(json.JSONTrue, "reset_values")
+		}
+
+		res, err := k8s.Releases.PutInContexts(s, args.NAME, params, args.ClusterContext())
+		if err != nil {
+			return err
+		}
+		printObject(res)
+		return nil
+	})
+
+	type deleteOpt struct {
+		clusterBaseOptions
+		NAME string `help:"Release instance name"`
+	}
+	R(&deleteOpt{}, cmdN("delete"), "Delete release", func(s *mcclient.ClientSession, args *deleteOpt) error {
+		_, err := k8s.Releases.DeleteInContexts(s, args.NAME, nil, args.ClusterContext())
+		return err
+	})
+}

cmd/climc/shell/k8s/repos.go

@@ -0,0 +1,106 @@
+package k8s
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/k8s"
+)
+
+func initRepo() {
+	cmdN := func(suffix string) string {
+		return resourceCmdN("repo", suffix)
+	}
+	type listOpt struct {
+		BaseListOptions
+	}
+	R(&listOpt{}, cmdN("list"), "List k8s global helm repos", func(s *mcclient.ClientSession, args *listOpt) error {
+		params := FetchPagingParams(args.BaseListOptions)
+		result, err := k8s.Repos.List(s, params)
+		if err != nil {
+			return err
+		}
+		printList(result, k8s.Repos.GetColumns(s))
+		return nil
+	})
+
+	type getOpt struct {
+		NAME string `help:"ID or name of the repo"`
+	}
+	R(&getOpt{}, cmdN("show"), "Show details fo a repo", func(s *mcclient.ClientSession, args *getOpt) error {
+		repo, err := k8s.Repos.Get(s, args.NAME, nil)
+		if err != nil {
+			return err
+		}
+		printObject(repo)
+		return nil
+	})
+
+	type createOpt struct {
+		getOpt
+		URL    string `help:"Repository url"`
+		Public bool   `help:"Make repository public"`
+	}
+	R(&createOpt{}, cmdN("create"), "Add repository", func(s *mcclient.ClientSession, args *createOpt) error {
+		params := jsonutils.NewDict()
+		params.Add(jsonutils.NewString(args.NAME), "name")
+		params.Add(jsonutils.NewString(args.URL), "url")
+		if args.Public {
+			params.Add(jsonutils.JSONTrue, "is_public")
+		}
+		repo, err := k8s.Repos.Create(s, params)
+		if err != nil {
+			return err
+		}
+		printObject(repo)
+		return nil
+	})
+
+	type updateOpt struct {
+		getOpt
+		Name string `help:"Repository name to change"`
+		Url  string `help:"Repository url to change"`
+	}
+	R(&updateOpt{}, cmdN("update"), "Update helm repository", func(s *mcclient.ClientSession, args *updateOpt) error {
+		params := jsonutils.NewDict()
+		if args.Name != "" {
+			params.Add(jsonutils.NewString(args.Name), "name")
+		}
+		if args.Url != "" {
+			params.Add(jsonutils.NewString(args.Url), "url")
+		}
+		repo, err := k8s.Repos.Update(s, args.NAME, params)
+		if err != nil {
+			return err
+		}
+		printObject(repo)
+		return nil
+	})
+
+	R(&getOpt{}, cmdN("private"), "Make repository private", func(s *mcclient.ClientSession, args *getOpt) error {
+		repo, err := k8s.Repos.PerformAction(s, args.NAME, "private", nil)
+		if err != nil {
+			return err
+		}
+		printObject(repo)
+		return nil
+	})
+
+	R(&getOpt{}, cmdN("public"), "Make repository public", func(s *mcclient.ClientSession, args *getOpt) error {
+		repo, err := k8s.Repos.PerformAction(s, args.NAME, "public", nil)
+		if err != nil {
+			return err
+		}
+		printObject(repo)
+		return nil
+	})
+
+	R(&getOpt{}, cmdN("delete"), "Delete a repository", func(s *mcclient.ClientSession, args *getOpt) error {
+		repo, err := k8s.Repos.Delete(s, args.NAME, nil)
+		if err != nil {
+			return err
+		}
+		printObject(repo)
+		return nil
+	})
+}

cmd/climc/shell/k8s/service.go

@@ -0,0 +1,27 @@
+package k8s
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/k8s"
+)
+
+func initService() {
+	cmdN := func(suffix string) string {
+		return resourceCmdN("service", suffix)
+	}
+
+	type listOpt struct {
+		namespaceListOptions
+		baseListOptions
+	}
+	R(&listOpt{}, cmdN("list"), "List k8s service", func(s *mcclient.ClientSession, args *listOpt) error {
+		params := fetchNamespaceParams(args.namespaceListOptions)
+		params.Update(fetchPagingParams(args.baseListOptions))
+		ret, err := k8s.Services.ListInContexts(s, params, args.ClusterContext())
+		if err != nil {
+			return err
+		}
+		printList(ret, k8s.Services.GetColumns(s))
+		return nil
+	})
+}

cmd/climc/shell/k8s/tiller.go

@@ -0,0 +1,56 @@
+package k8s
+
+import (
+	json "yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/k8s"
+)
+
+func initTiller() {
+	cmdN := func(suffix string) string {
+		return resourceCmdN("tiller", suffix)
+	}
+	type createOpt struct {
+		clusterBaseOptions
+		KubeContext string `json:"kube_context"`
+		Namespace   string `json:"namespace" default:"kube-system"`
+		// Upgrade if Tiller is already installed
+		Upgrade bool `json:"upgrade"`
+		// Name of service account
+		ServiceAccount string `json:"service_account" default:"tiller"`
+		// Use the canary Tiller image
+		Canary bool `json:"canary_image"`
+
+		// Override Tiller image
+		Image string `json:"tiller_image" default:"yunion/tiller:v2.9.0"`
+		// Limit the maximum number of revisions saved per release. Use 0 for no limit.
+		MaxHistory int `json:"history_max"`
+	}
+	R(&createOpt{}, cmdN("create"), "Install helm tiller server to Kubernetes cluster", func(s *mcclient.ClientSession, args *createOpt) error {
+		params := json.NewDict()
+		if len(args.KubeContext) > 0 {
+			params.Add(json.NewString(args.KubeContext), "kube_context")
+		}
+		params.Add(json.NewString(args.Namespace), "namespace")
+		params.Add(json.NewString(args.ServiceAccount), "service_account")
+		if args.Canary {
+			params.Add(json.JSONTrue, "canary_image")
+		}
+		if args.Upgrade {
+			params.Add(json.JSONTrue, "upgrade")
+		}
+		if len(args.Image) > 0 {
+			params.Add(json.NewString(args.Image), "tiller_image")
+		}
+		if args.MaxHistory > 0 {
+			params.Add(json.NewInt(int64(args.MaxHistory)), "history_max")
+		}
+		ret, err := k8s.Tiller.CreateInContexts(s, params, args.ClusterContext())
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
+}

cmd/climc/shell/networks.go

@@ -45,7 +45,7 @@ func init() {
 		ID          string `help:"ID or Name of zone to update"`
 		Name        string `help:"Name of zone"`
 		Desc        string `metavar:"<DESCRIPTION>" help:"Description"`
-		ServerType  string `help:"server type," choices:"baremetal|guest"`
+		ServerType  string `help:"server type," choices:"baremetal|guest|container"`
 		StartIp     string `help:"Start ip"`
 		EndIp       string `help:"end ip"`
 		NetMask     int64  `help:"Netmask"`
@@ -178,7 +178,7 @@ func init() {
 		Gateway     string `help:"Default gateway"`
 		VlanId      int64  `help:"Vlan ID" default:"1"`
 		AllocPolicy string `help:"Address allocation policy" choices:"none|stepdown|stepup|random"`
-		ServerType  string `help:"Server type" choices:"baremetal|guest"`
+		ServerType  string `help:"Server type" choices:"baremetal|guest|container"`
 		Desc        string `help:"Description" metavar:"DESCRIPTION"`
 	}
 	R(&NetworkCreateOptions{}, "network-create", "Create a virtual network", func(s *mcclient.ClientSession, args *NetworkCreateOptions) error {

cmd/climc/shell/servers.go

@@ -64,6 +64,7 @@ func init() {
 		Secgroup      string `help:"Secgroup ID or Name"`
 		AdminSecgroup string `help:"AdminSecgroup ID or Name"`
 		Hypervisor    string `help:"Show server of hypervisor" choices:"kvm|esxi|container|baremetal|aliyun"`
+		Manager       string `help:"Show servers imported from manager"`
 		BaseListOptions
 	}
 	R(&ServerListOptions{}, "server-list", "List virtual servers", func(s *mcclient.ClientSession, args *ServerListOptions) error {
@@ -98,6 +99,9 @@ func init() {
 		if len(args.Hypervisor) > 0 {
 			params.Add(jsonutils.NewString(args.Hypervisor), "hypervisor")
 		}
+		if len(args.Manager) > 0 {
+			params.Add(jsonutils.NewString(args.Manager), "manager")
+		}
 		result, err := modules.Servers.List(s, params)
 		if err != nil {
 			return err
@@ -139,6 +143,7 @@ func init() {
 		Net              []string `help:"Network descriptions" metavar:"NETWORK"`
 		IsolatedDevice   []string `help:"Isolated device model or ID" metavar:"ISOLATED_DEVICE"`
 		Keypair          string   `help:"SSH Keypair"`
+		Password         string   `help:"Default user password"`
 		Iso              string   `help:"ISO image ID" metavar:"IMAGE_ID"`
 		Ncpu             int64    `help:"#CPU cores of VM server, default 1" default:"1" metavar:"<SERVER_CPU_COUNT>"`
 		Vga              string   `help:"VGA driver" choices:"std|vmware|cirrus|qxl"`
@@ -192,6 +197,9 @@ func init() {
 		if args.NoAccountInit {
 			params.Add(jsonutils.JSONFalse, "reset_password")
 		}
+		if args.Password != "" {
+			params.Add(jsonutils.NewString(args.Password), "password")
+		}
 		if len(args.Vdi) > 0 {
 			params.Add(jsonutils.NewString(args.Vdi), "vdi")
 		}
@@ -339,8 +347,16 @@ func init() {
 		return nil
 	})
 
-	R(&ServerOpsOptions{}, "server-reset", "Reset servers", func(s *mcclient.ClientSession, args *ServerOpsOptions) error {
-		ret := modules.Servers.BatchPerformAction(s, args.ID, "reset", nil)
+	type ServerResetOptions struct {
+		ServerOpsOptions
+		Hard bool `help:"Hard reset or not; default soft"`
+	}
+	R(&ServerResetOptions{}, "server-reset", "Reset servers", func(s *mcclient.ClientSession, args *ServerResetOptions) error {
+		params := jsonutils.NewDict()
+		if args.Hard {
+			params.Add(jsonutils.JSONTrue, "is_hard")
+		}
+		ret := modules.Servers.BatchPerformAction(s, args.ID, "reset", params)
 		printBatchResults(ret, modules.Servers.GetColumns(s))
 		return nil
 	})
@@ -484,6 +500,7 @@ func init() {
 		DeleteKeypair bool     `help:"Remove ssh Keypairs"`
 		Deploy        []string `help:"Specify deploy files in virtual server file system"`
 		ResetPassword bool     `help:"Force reset password"`
+		Password      string   `help:"Default user password"`
 	}
 	R(&ServerDeployOptions{}, "server-deploy", "Deploy hostname and keypair to a stopped virtual server", func(s *mcclient.ClientSession, args *ServerDeployOptions) error {
 		params := jsonutils.NewDict()
@@ -501,6 +518,9 @@ func init() {
 		if args.ResetPassword {
 			params.Add(jsonutils.JSONTrue, "reset_password")
 		}
+		if args.Password != "" {
+			params.Add(jsonutils.NewString(args.Password), "password")
+		}
 		srv, e := modules.Servers.PerformAction(s, args.ID, "deploy", params)
 		if e != nil {
 			return e
@@ -511,7 +531,7 @@ func init() {
 
 	type ServerSecGroupOptions struct {
 		ID     string `help:"ID or Name of server" metavar:"Guest"`
-		SecGrp string `help:"ID of Security Group" metavar:"Security Group" optional:"false"`
+		SecGrp string `help:"ID of Security Group" metavar:"Security Group" positional:"true"`
 	}
 
 	R(&ServerSecGroupOptions{}, "server-assign-secgroup", "Assign security group to a VM", func(s *mcclient.ClientSession, args *ServerSecGroupOptions) error {

cmd/scheduler/app/server.go

@@ -8,10 +8,10 @@ import (
 	"net/http"
 	"strconv"
 
+	"gopkg.in/gin-gonic/gin.v1"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/util/prometheus"
 	"yunion.io/x/pkg/utils"
-	"gopkg.in/gin-gonic/gin.v1"
 
 	o "yunion.io/x/onecloud/cmd/scheduler/options"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"

pkg/appsrv/dispatcher/dispatcher.go

@@ -209,8 +209,7 @@ func performClassActionHandler(ctx context.Context, w http.ResponseWriter, r *ht
 		if data == nil {
 			data = body.(*jsonutils.JSONDict)
 		}
-	}
-	if data == nil {
+	} else {
 		data = jsonutils.NewDict()
 	}
 	results, err := manager.PerformClassAction(ctx, params["<action>"], query, data)

pkg/cloudcommon/cronman/cronman.go

@@ -1,33 +1,34 @@
 package cronman
 
 import (
-	"time"
-	"runtime/debug"
 	"context"
+	"reflect"
+	"runtime"
+	"runtime/debug"
+	"time"
 
 	"yunion.io/x/log"
 
+	"yunion.io/x/onecloud/pkg/appctx"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
-	"reflect"
-	"runtime"
 )
 
 const (
-	DEFAULT_CRON_INTERVAL = 60*time.Second // default resolution is 1 monutes
+	DEFAULT_CRON_INTERVAL = 60 * time.Second // default resolution is 1 monutes
 )
 
 type SCronJobManager struct {
 	checkInterval time.Duration
-	timer *time.Timer
-	jobs []SCronJob
+	timer         *time.Timer
+	jobs          []SCronJob
 }
 
 type SCronJob struct {
-	name string
+	name        string
 	runInterval time.Duration
-	job func(ctx context.Context, userCred mcclient.TokenCredential)
-	lastRun time.Time
+	job         func(ctx context.Context, userCred mcclient.TokenCredential)
+	lastRun     time.Time
 }
 
 func NewCronJobManager(interval time.Duration) *SCronJobManager {
@@ -89,7 +90,7 @@ func runJob(name string, job func(ctx context.Context, userCred mcclient.TokenCr
 	}()
 
 	ctx := context.Background()
+	ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_APPNAME, "Region-Corn-Service")
 	userCred := auth.AdminCredential()
 	job(ctx, userCred)
 }
-

pkg/cloudcommon/db/db_dispatcher.go

@@ -251,6 +251,30 @@ func applyListItemsGeneralFilters(manager IModelManager, q *sqlchemy.SQuery,
 	return q, nil
 }
 
+func applyListItemsGeneralJointFilters(manager IModelManager, q *sqlchemy.SQuery,
+	userCred mcclient.TokenCredential, jointFilters []string, filterAny bool) (*sqlchemy.SQuery, error) {
+	for _, f := range jointFilters {
+		jfc := filterclause.ParseJointFilterClause(f)
+		if jfc != nil {
+			jointModelManager := GetModelManager(jfc.GetJointModelName())
+			schFields := searchFields(jointModelManager, userCred)
+			if ok, _ := utils.InStringArray(jfc.GetField(), schFields); ok {
+				sq := jointModelManager.Query(jfc.ReleatedKey)
+				cond := jfc.GetJointFilter(sq)
+				if cond != nil {
+					sq = sq.Filter(cond)
+					if filterAny {
+						q = q.Filter(sqlchemy.OR(sqlchemy.In(q.Field("id"), sq)))
+					} else {
+						q = q.Filter(sqlchemy.AND(sqlchemy.In(q.Field("id"), sq)))
+					}
+				}
+			}
+		}
+	}
+	return q, nil
+}
+
 func listItemQueryFilters(manager IModelManager, ctx context.Context, q *sqlchemy.SQuery,
 	userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
 
@@ -272,11 +296,15 @@ func listItemQueryFilters(manager IModelManager, ctx context.Context, q *sqlchem
 			return nil, err
 		}
 	}
+	filterAny, _ := query.Bool("filter_any")
 	filters := jsonutils.GetQueryStringArray(query, "filter")
 	if len(filters) > 0 {
-		filterAny, _ := query.Bool("filter_any")
 		q, err = applyListItemsGeneralFilters(manager, q, userCred, filters, filterAny)
 	}
+	jointFilter := jsonutils.GetQueryStringArray(query, "joint_filter")
+	if len(jointFilter) > 0 {
+		q, _ = applyListItemsGeneralJointFilters(manager, q, userCred, jointFilter, filterAny)
+	}
 	return q, nil
 }
 
@@ -524,7 +552,7 @@ func (dispatcher *DBModelDispatcher) tryGetModelProperty(ctx context.Context, pr
 }
 
 func (dispatcher *DBModelDispatcher) Get(ctx context.Context, idStr string, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	log.Debugf("Get %s", idStr)
+	// log.Debugf("Get %s", idStr)
 	userCred := fetchUserCredential(ctx)
 
 	data, err := dispatcher.tryGetModelProperty(ctx, idStr, query)
@@ -541,7 +569,7 @@ func (dispatcher *DBModelDispatcher) Get(ctx context.Context, idStr string, quer
 	} else if err != nil {
 		return nil, err
 	}
-	log.Debugf("Get found %s", model)
+	// log.Debugf("Get found %s", model)
 	if !model.AllowGetDetails(ctx, userCred, query) {
 		return nil, httperrors.NewForbiddenError("Not allow to get details")
 	}
@@ -841,7 +869,12 @@ func (dispatcher *DBModelDispatcher) PerformClassAction(ctx context.Context, act
 
 	managerValue := reflect.ValueOf(dispatcher.modelManager)
 	if action == "check-create-data" {
-		return dispatcher.modelManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data.(*jsonutils.JSONDict))
+		manager := dispatcher.modelManager
+		if body, err := data.(*jsonutils.JSONDict).Get(manager.Keyword()); err != nil {
+			return nil, httperrors.NewGeneralError(err)
+		} else {
+			return manager.ValidateCreateData(ctx, userCred, ownerProjId, query, body.(*jsonutils.JSONDict))
+		}
 	}
 	return objectPerformAction(dispatcher, managerValue, ctx, userCred, action, query, data)
 }
@@ -975,7 +1008,6 @@ func updateItem(manager IModelManager, item IModel, ctx context.Context, userCre
 		return nil, httperrors.NewGeneralError(err)
 	}
 	item.PreUpdate(ctx, userCred, query, dataDict)
-
 	diff, err := manager.TableSpec().Update(item, func() error {
 		filterData := dataDict.CopyIncludes(updateFields(manager, userCred)...)
 		err = filterData.Unmarshal(item)

pkg/cloudcommon/db/taskman/tasks.go

@@ -246,17 +246,21 @@ func (manager *STaskManager) NewParallelTask(ctx context.Context, taskName strin
 }
 
 func (manager *STaskManager) fetchTask(idStr string) *STask {
-	task, err := db.NewModelObject(manager)
+	iTask, err := db.NewModelObject(manager)
 	if err != nil {
 		log.Errorf("New task object fail: %s", err)
 		return nil
 	}
-	err = manager.Query().Equals("id", idStr).First(task)
+	err = manager.Query().Equals("id", idStr).First(iTask)
 	if err != nil {
 		log.Errorf("GetTask %s fail: %s", idStr, err)
 		return nil
 	}
-	return task.(*STask)
+	task := iTask.(*STask)
+	if task.Params == nil {
+		task.Params = jsonutils.NewDict()
+	}
+	return task
 }
 
 func (manager *STaskManager) execTask(taskId string, data jsonutils.JSONObject) {
@@ -285,20 +289,21 @@ func (manager *STaskManager) execTask(taskId string, data jsonutils.JSONObject)
 	}
 }
 
-func execITask(taskValue reflect.Value, task *STask, data jsonutils.JSONObject, isMulti bool) {
+func execITask(taskValue reflect.Value, task *STask, odata jsonutils.JSONObject, isMulti bool) {
 	var err error
 	ctxData := task.GetRequestContext()
 	ctx := ctxData.GetContext()
 
 	taskFailed := false
 
+	data := odata
 	if data != nil {
 		taskStatus, _ := data.GetString("__status__")
 		if len(taskStatus) > 0 && taskStatus != "OK" {
 			taskFailed = true
-			data, err = data.Get("reason")
+			data, err = data.Get("__reason__")
 			if err != nil {
-				data = jsonutils.NewString("Task failed due to unknown remote errors!")
+				data = jsonutils.NewString(fmt.Sprintf("Task failed due to unknown remote errors! %s", odata))
 			}
 		}
 	} else {
@@ -324,7 +329,13 @@ func execITask(taskValue reflect.Value, task *STask, data jsonutils.JSONObject,
 
 		if !funcValue.IsValid() || funcValue.IsNil() {
 			msg := fmt.Sprintf("Stage %s not found", stageName)
-			log.Errorf(msg)
+			if taskFailed {
+				// failed handler is optional, ignore the error
+				log.Warningf(msg)
+				msg, _ = data.GetString()
+			} else {
+				log.Errorf(msg)
+			}
 			task.SetStageFailed(ctx, msg)
 			task.SaveRequestContext(&ctxData)
 			return
@@ -412,9 +423,11 @@ func (self *STask) GetParentTask() *STask {
 
 func (self *STask) GetRequestContext() appctx.AppContextData {
 	ctxData := appctx.AppContextData{}
-	ctxJson, _ := self.Params.Get(REQUEST_CONTEXT_KEY)
-	if ctxJson != nil {
-		ctxJson.Unmarshal(&ctxData)
+	if self.Params != nil {
+		ctxJson, _ := self.Params.Get(REQUEST_CONTEXT_KEY)
+		if ctxJson != nil {
+			ctxJson.Unmarshal(&ctxData)
+		}
 	}
 	return ctxData
 }
@@ -542,7 +555,7 @@ func (self *STask) NotifyParentTaskFailure(ctx context.Context, reason string) {
 	if len(reason) > 100 {
 		reason = reason[:100] + "..."
 	}
-	body.Add(jsonutils.NewString(fmt.Sprintf("Subtask %s failed: %s", self.TaskName, reason)))
+	body.Add(jsonutils.NewString(fmt.Sprintf("Subtask %s failed: %s", self.TaskName, reason)), "__reason__")
 	self.NotifyParentTaskComplete(ctx, body, true)
 }
 

pkg/cloudcommon/db/virtualresource.go

@@ -161,16 +161,16 @@ func (model *SVirtualResourceBase) AllowPerformMetadata(ctx context.Context, use
 }
 
 func (model *SVirtualResourceBase) GetTenantCache(ctx context.Context) (*STenant, error) {
-	log.Debugf("Get tenant by Id %s", model.ProjectId)
+	// log.Debugf("Get tenant by Id %s", model.ProjectId)
 	return TenantCacheManager.FetchTenantById(ctx, model.ProjectId)
 }
 
 func (model *SVirtualResourceBase) getMoreDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, extra *jsonutils.JSONDict) *jsonutils.JSONDict {
 	if userCred.IsSystemAdmin() {
-		log.Debugf("GetCustomizeColumns")
+		// log.Debugf("GetCustomizeColumns")
 		tobj, err := model.GetTenantCache(ctx)
 		if err == nil {
-			log.Debugf("GetTenantFromCache %s", jsonutils.Marshal(tobj))
+			// log.Debugf("GetTenantFromCache %s", jsonutils.Marshal(tobj))
 			extra.Add(jsonutils.NewString(tobj.GetName()), "tenant")
 		} else {
 			log.Errorf("GetTenantCache fail %s", err)

pkg/cloudprovider/resources.go

@@ -18,6 +18,7 @@ type ICloudResource interface {
 	Refresh() error
 
 	IsEmulated() bool
+	GetMetadata() *jsonutils.JSONDict
 }
 
 type ICloudRegion interface {
@@ -153,7 +154,13 @@ type ICloudVM interface {
 	StopVM(isForce bool) error
 	DeleteVM() error
 
+	UpdateVM(name string) error
+	RebuildRoot(imageId string) error
+	DeployVM(name string, password string, publicKey string, resetPassword bool, deleteKeypair bool, description string) error
+	ChangeConfig(instanceId string, ncpu int, vmem int) error
 	GetVNCInfo() (jsonutils.JSONObject, error)
+	AttachDisk(diskId string) error
+	DetachDisk(diskId string) error
 }
 
 type ICloudNic interface {
@@ -193,7 +200,8 @@ type ICloudDisk interface {
 	GetCacheMode() string
 	GetMountpoint() string
 	Delete() error
-	Resize(int64) error
+
+	Resize(newSize int64) error
 }
 
 type ICloudVpc interface {

pkg/compute/guestdrivers/aliyun.go

@@ -104,7 +104,6 @@ func (self *SAliyunGuestDriver) GetJsonDescAtHost(ctx context.Context, guest *mo
 			imageId := disk.GetTemplateId()
 			scimg := models.StoragecachedimageManager.GetStoragecachedimage(cache.Id, imageId)
 			config.ExternalImageId = scimg.ExternalId
-
 			img := scimg.GetCachedimage()
 			config.OsDistribution, _ = img.Info.GetString("properties", "os_distribution")
 			config.OsVersion, _ = img.Info.GetString("properties", "os_version")
@@ -119,108 +118,159 @@ func (self *SAliyunGuestDriver) GetJsonDescAtHost(ctx context.Context, guest *mo
 }
 
 type SDiskInfo struct {
-	Size int
-	Uuid string
+	Size     int
+	Uuid     string
+	Metadata map[string]string
 }
 
 func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error {
 	config := guest.GetDeployConfigOnHost(ctx, host, task.GetParams())
 
-	onfinish, err := config.GetString("on_finish")
+	/* onfinish, err := config.GetString("on_finish")
 	if err != nil {
 		return err
-	}
+	} */
+
 	action, err := config.GetString("action")
 	if err != nil {
 		return err
 	}
 
-	if action != "create" {
-		return fmt.Errorf("Action %s not supported", action)
-	}
-
 	ihost, err := host.GetIHost()
 	if err != nil {
 		return err
 	}
 
-	desc := SAliyunVMCreateConfig{}
-	err = config.Unmarshal(&desc, "desc")
-	if err != nil {
-		return err
-	}
-
-	taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
-		passwd := seclib2.RandomPassword2(12)
-
-		iVM, err := ihost.CreateVM(desc.Name, desc.ExternalImageId, desc.SysDiskSize, desc.Cpu, desc.Memory, desc.ExternalNetworkId,
-			desc.IpAddr, desc.Description, passwd, desc.StorageType, desc.DataDisks, desc.PublicKey)
+	if action == "create" {
+		desc := SAliyunVMCreateConfig{}
+		err = config.Unmarshal(&desc, "desc")
 		if err != nil {
-			return nil, err
-		}
-		log.Debugf("VMcreated %s, wait status ready ...", iVM.GetGlobalId())
-		err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*1800)
-		if err != nil {
-			return nil, err
-		}
-		log.Debugf("VMcreated %s, and status is ready", iVM.GetGlobalId())
-
-		iVM, err = ihost.GetIVMById(iVM.GetGlobalId())
-		if err != nil {
-			log.Errorf("cannot find vm %s", err)
-			return nil, err
+			return err
 		}
 
-		if len(guest.SecgrpId) > 0 {
-			if err := iVM.SyncSecurityGroup(guest.SecgrpId, guest.GetSecgroupName(), guest.GetSecRules()); err != nil {
-				log.Errorf("SyncSecurityGroup error: %v", err)
-				return nil, err
-			}
-		}
+		taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
+			passwd := seclib2.RandomPassword2(12)
 
-		if onfinish == "none" {
-			err = iVM.StartVM()
+			iVM, err := ihost.CreateVM(desc.Name, desc.ExternalImageId, desc.SysDiskSize, desc.Cpu, desc.Memory, desc.ExternalNetworkId,
+				desc.IpAddr, desc.Description, passwd, desc.StorageType, desc.DataDisks, desc.PublicKey)
 			if err != nil {
 				return nil, err
 			}
-		}
-
-		encpasswd, err := utils.EncryptAESBase64(guest.Id, passwd)
-		if err != nil {
-			log.Errorf("encrypt password failed %s", err)
-		}
-
-		data := jsonutils.NewDict()
-		data.Add(jsonutils.NewString(iVM.GetOSType()), "os")
-		data.Add(jsonutils.NewString("root"), "account")
-		data.Add(jsonutils.NewString(encpasswd), "key")
-
-		if len(desc.OsDistribution) > 0 {
-			data.Add(jsonutils.NewString(desc.OsDistribution), "distro")
-		}
-		if len(desc.OsVersion) > 0 {
-			data.Add(jsonutils.NewString(desc.OsVersion), "version")
-		}
-
-		idisks, err := iVM.GetIDisks()
-
-		if err != nil {
-			log.Errorf("GetiDisks error %s", err)
-		} else {
-			diskInfo := make([]SDiskInfo, len(idisks))
-			for i := 0; i < len(idisks); i += 1 {
-				dinfo := SDiskInfo{}
-				dinfo.Uuid = idisks[i].GetGlobalId()
-				dinfo.Size = idisks[i].GetDiskSizeMB()
-				diskInfo[i] = dinfo
+			log.Debugf("VMcreated %s, wait status ready ...", iVM.GetGlobalId())
+			err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*1800)
+			if err != nil {
+				return nil, err
 			}
-			data.Add(jsonutils.Marshal(&diskInfo), "disks")
+			log.Debugf("VMcreated %s, and status is ready", iVM.GetGlobalId())
+
+			iVM, err = ihost.GetIVMById(iVM.GetGlobalId())
+			if err != nil {
+				log.Errorf("cannot find vm %s", err)
+				return nil, err
+			}
+
+			if len(guest.SecgrpId) > 0 {
+				if err := iVM.SyncSecurityGroup(guest.SecgrpId, guest.GetSecgroupName(), guest.GetSecRules()); err != nil {
+					log.Errorf("SyncSecurityGroup error: %v", err)
+					return nil, err
+				}
+			}
+
+			/*if onfinish == "none" {
+				err = iVM.StartVM()
+				if err != nil {
+					return nil, err
+				}
+			}*/
+
+			encpasswd, err := utils.EncryptAESBase64(guest.Id, passwd)
+			if err != nil {
+				log.Errorf("encrypt password failed %s", err)
+			}
+
+			data := jsonutils.NewDict()
+			data.Add(jsonutils.NewString(iVM.GetOSType()), "os")
+			data.Add(jsonutils.NewString("root"), "account")
+			data.Add(jsonutils.NewString(encpasswd), "key")
+
+			if len(desc.OsDistribution) > 0 {
+				data.Add(jsonutils.NewString(desc.OsDistribution), "distro")
+			}
+			if len(desc.OsVersion) > 0 {
+				data.Add(jsonutils.NewString(desc.OsVersion), "version")
+			}
+
+			idisks, err := iVM.GetIDisks()
+
+			if err != nil {
+				log.Errorf("GetiDisks error %s", err)
+			} else {
+				diskInfo := make([]SDiskInfo, len(idisks))
+				for i := 0; i < len(idisks); i += 1 {
+					dinfo := SDiskInfo{}
+					dinfo.Uuid = idisks[i].GetGlobalId()
+					dinfo.Size = idisks[i].GetDiskSizeMB()
+					if metaData := idisks[i].GetMetadata(); metaData != nil {
+						dinfo.Metadata = make(map[string]string, 0)
+						if err := metaData.Unmarshal(dinfo.Metadata); err != nil {
+							log.Errorf("Get disk %s metadata info error: %v", idisks[i].GetName(), err)
+						}
+					}
+					diskInfo[i] = dinfo
+				}
+				data.Add(jsonutils.Marshal(&diskInfo), "disks")
+			}
+
+			data.Add(jsonutils.NewString(iVM.GetGlobalId()), "uuid")
+			data.Add(iVM.GetMetadata(), "metadata")
+
+			return data, nil
+		})
+	} else if action == "deploy" {
+		iVM, err := ihost.GetIVMById(guest.GetExternalId())
+		if err != nil || iVM == nil {
+			log.Errorf("cannot find vm %s", err)
+			return fmt.Errorf("cannot find vm")
 		}
 
-		data.Add(jsonutils.NewString(iVM.GetGlobalId()), "uuid")
+		params := task.GetParams()
+		log.Debugf("Deploy VM params %s", params.String())
+		var name string
+		if v, e := params.GetString("name"); e != nil {
+			name = v
+		}
+		var description string
+		if v, e := params.GetString("description"); e != nil {
+			description = v
+		}
+		resetPassword := jsonutils.QueryBoolean(params, "reset_password", false)
+		deleteKeypair := jsonutils.QueryBoolean(params, "__delete_keypair__", false)
+		password, _ := params.GetString("password")
+		if resetPassword && len(password) == 0 {
+			password = seclib2.RandomPassword2(12)
+		}
+
+		publicKey := ""
+		if k, e := config.GetString("public_key"); e != nil {
+			publicKey = k
+		}
+
+		taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
+			encpasswd, err := utils.EncryptAESBase64(guest.Id, password)
+			if err != nil {
+				log.Errorf("encrypt password failed %s", err)
+			}
+
+			data := jsonutils.NewDict()
+			data.Add(jsonutils.NewString("root"), "account") // 用户名
+			data.Add(jsonutils.NewString(encpasswd), "key")  // 密码
+			e := iVM.DeployVM(name, password, publicKey, resetPassword, deleteKeypair, description)
+			return data, e
+		})
+	} else {
+		return fmt.Errorf("Action %s not supported", action)
+	}
 
-		return data, nil
-	})
 	return nil
 }
 
@@ -244,6 +294,13 @@ func (self *SAliyunGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Contex
 				disk.DiskSize = diskInfo[i].Size
 				disk.ExternalId = diskInfo[i].Uuid
 				disk.Status = models.DISK_READY
+				if len(diskInfo[i].Metadata) > 0 {
+					for key, value := range diskInfo[i].Metadata {
+						if err := disk.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil {
+							log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err)
+						}
+					}
+				}
 				return nil
 			})
 			if err != nil {
@@ -259,6 +316,20 @@ func (self *SAliyunGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Contex
 	if len(uuid) > 0 {
 		guest.SetExternalId(uuid)
 	}
+
+	if metaData, _ := data.Get("metadata"); metaData != nil {
+		meta := make(map[string]string, 0)
+		if err := metaData.Unmarshal(meta); err != nil {
+			log.Errorf("Get guest %s metadata error: %v", guest.Name, err)
+		} else {
+			for key, value := range meta {
+				if err := guest.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil {
+					log.Errorf("set guest %s mata %s => %s error: %v", guest.Name, key, value, err)
+				}
+			}
+		}
+	}
+
 	guest.SaveDeployInfo(ctx, task.GetUserCred(), data)
 	return nil
 }
@@ -275,7 +346,6 @@ func (self *SAliyunGuestDriver) RequestSyncConfigOnHost(ctx context.Context, gue
 					return nil, err
 				}
 			} else {
-				iVM.Refresh()
 				if iDisks, err := iVM.GetIDisks(); err != nil {
 					return nil, err
 				} else {
@@ -293,18 +363,16 @@ func (self *SAliyunGuestDriver) RequestSyncConfigOnHost(ctx context.Context, gue
 					if err := compare.CompareSets(disks, iDisks, &added, &commondb, &commonext, &removed); err != nil {
 						return nil, err
 					}
-					// for _, disk := range removed {
-					// 	log.Errorf("need remove %v", disk)
-					// 	if err := iVM.DetachDisk(disk.GetId()); err != nil {
-					// 		return nil, err
-					// 	}
-					// }
-					// for _, disk := range added {
-					// 	log.Errorf("need add %v", disk)
-					// 	if err := iVM.AttachDisk(disk.ExternalId); err != nil {
-					// 		return nil, err
-					// 	}
-					// }
+					for _, disk := range removed {
+						if err := iVM.DetachDisk(disk.GetId()); err != nil {
+							return nil, err
+						}
+					}
+					for _, disk := range added {
+						if err := iVM.AttachDisk(disk.ExternalId); err != nil {
+							return nil, err
+						}
+					}
 				}
 			}
 		}
@@ -312,3 +380,118 @@ func (self *SAliyunGuestDriver) RequestSyncConfigOnHost(ctx context.Context, gue
 	})
 	return nil
 }
+
+type SAliyunVMChangeConfig struct {
+	InstanceId string
+	Cpu        int
+	Memory     int
+}
+
+func (self *SAliyunGuestDriver) DoGuestCreateDisksTask(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {
+	subtask, err := taskman.TaskManager.NewTask(ctx, "AliyunGuestCreateDiskTask", guest, task.GetUserCred(), task.GetParams(), task.GetTaskId(), "", nil)
+	if err != nil {
+		return err
+	}
+	subtask.ScheduleRun(nil)
+	return nil
+}
+
+func (self *SAliyunGuestDriver) AllowReconfigGuest() bool {
+	return true
+}
+
+func (self *SAliyunGuestDriver) RequestChangeVmConfig(ctx context.Context, guest *models.SGuest, task taskman.ITask, vcpuCount, vmemSize int64) error {
+	config := SAliyunVMChangeConfig{}
+	config.InstanceId = guest.GetExternalId()
+	config.Cpu = int(vcpuCount)
+	config.Memory = int(vmemSize)
+	ihost, err := guest.GetHost().GetIHost()
+	if err != nil {
+		return err
+	}
+
+	iVM, err := ihost.GetIVMById(config.InstanceId)
+	if err != nil {
+		return err
+	}
+
+	if int(guest.VcpuCount) != config.Cpu || guest.VmemSize != config.Memory {
+		err = iVM.ChangeConfig(config.InstanceId, config.Cpu, config.Memory)
+		if err != nil {
+			return err
+		}
+	}
+
+	log.Debugf("VMchangeConfig %s, wait status ready ...", iVM.GetGlobalId())
+	err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*300)
+	if err != nil {
+		return err
+	}
+	log.Debugf("VMchangeConfig %s, and status is ready", iVM.GetGlobalId())
+	return nil
+}
+
+func (self *SAliyunGuestDriver) RequestStartOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) {
+	ihost, e := host.GetIHost()
+	if e != nil {
+		return nil, e
+	}
+
+	ivm, e := ihost.GetIVMById(guest.GetExternalId())
+	if e != nil {
+		return nil, e
+	}
+
+	err := ivm.StartVM()
+	if err != nil {
+		return nil, e
+	}
+
+	result := jsonutils.NewDict()
+	result.Add(jsonutils.NewBool(true), "is_running")
+	return result, e
+}
+
+func (self *SAliyunGuestDriver) RequestRebuildRootDisk(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {
+	ihost, e := guest.GetHost().GetIHost()
+	if e != nil {
+		return e
+	}
+
+	externalId := guest.GetExternalId()
+	if len(externalId) <= 0 {
+		return fmt.Errorf("external id not found")
+	}
+
+	disks := guest.GetDisks()
+	if len(disks) <= 0 {
+		return fmt.Errorf("guest has no disk")
+	}
+
+	imageId := guest.CategorizeDisks().Root.TemplateId
+	cacheId := disks[0].GetDisk().GetStorage().GetStoragecache().Id
+	externalImageId := models.StoragecachedimageManager.GetStoragecachedimage(cacheId, imageId).ExternalId
+	if len(externalImageId) <= 0 {
+		return fmt.Errorf("external image (%s) id is not found", imageId)
+	}
+
+	iVM, err := ihost.GetIVMById(externalId)
+	if err != nil {
+		return err
+	}
+
+	err = iVM.RebuildRoot(externalImageId)
+	if err != nil {
+		return err
+	}
+
+	log.Debugf("VMrebuildRoot %s, wait status ready ...", iVM.GetGlobalId())
+	err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*1800)
+	if err != nil {
+		return err
+	}
+	log.Debugf("VMrebuildRoot %s, and status is ready", iVM.GetGlobalId())
+
+	task.ScheduleRun(nil)
+	return nil
+}

pkg/compute/guestdrivers/base.go

@@ -102,6 +102,14 @@ func (self *SBaseGuestDriver) RqeuestSuspendOnHost(ctx context.Context, guest *m
 	return fmt.Errorf("Not Implement")
 }
 
+func (self *SBaseGuestDriver) StartGuestResetTask(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, isHard bool, parentTaskId string) error {
+	return fmt.Errorf("Not Implement")
+}
+
+func (self *SBaseGuestDriver) RequestSoftReset(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {
+	return fmt.Errorf("Not Implement")
+}
+
 func (self *SBaseGuestDriver) AllowReconfigGuest() bool {
 	return true
 }

pkg/compute/guestdrivers/kvm.go

@@ -206,6 +206,11 @@ func (self *SKVMGuestDriver) RequestChangeVmConfig(ctx context.Context, guest *m
 	return nil
 }
 
+func (self *SKVMGuestDriver) RequestSoftReset(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {
+	_, err := guest.SendMonitorCommand(ctx, task.GetUserCred(), "system_reset")
+	return err
+}
+
 func (self *SKVMGuestDriver) RequestDetachDisk(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {
 	return guest.StartSyncTask(ctx, task.GetUserCred(), false, task.GetTaskId())
 }
@@ -219,18 +224,21 @@ func (self *SKVMGuestDriver) RequestDeleteDetachedDisk(ctx context.Context, disk
 }
 
 func (self *SKVMGuestDriver) RequestSyncConfigOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error {
-	desc := guest.GetDriver().GetJsonDescAtHost(ctx, guest, host)
-	body := jsonutils.NewDict()
-	body.Add(desc, "desc")
-	if fw_only, _ := task.GetParams().Bool("fw_only"); fw_only {
-		body.Add(jsonutils.JSONTrue, "fw_only")
+	if guest.Status == models.VM_RUNNING {
+		desc := guest.GetDriver().GetJsonDescAtHost(ctx, guest, host)
+		body := jsonutils.NewDict()
+		body.Add(desc, "desc")
+		if fw_only, _ := task.GetParams().Bool("fw_only"); fw_only {
+			body.Add(jsonutils.JSONTrue, "fw_only")
+		}
+		url := fmt.Sprintf("/servers/%s/sync", guest.Id)
+		header := http.Header{}
+		header.Add("X-Task-Id", task.GetTaskId())
+		header.Add("X-Region-Version", "v2")
+		_, err := host.Request(task.GetUserCred(), "POST", url, header, body)
+		return err
 	}
-	url := fmt.Sprintf("/servers/%s/sync", guest.Id)
-	header := http.Header{}
-	header.Add("X-Task-Id", task.GetTaskId())
-	header.Add("X-Region-Version", "v2")
-	_, err := host.Request(task.GetUserCred(), "POST", url, header, body)
-	return err
+	return nil
 }
 
 func (self *SKVMGuestDriver) RqeuestSuspendOnHost(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {

pkg/compute/guestdrivers/virtualization.go

@@ -95,6 +95,19 @@ func (self *SVirtualizedGuestDriver) StartGuestStopTask(guest *models.SGuest, ct
 	return nil
 }
 
+func (self *SVirtualizedGuestDriver) StartGuestResetTask(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, isHard bool, parentTaskId string) error {
+	var taskName = "GuestSoftResetTask"
+	if isHard {
+		taskName = "GuestHardResetTask"
+	}
+	task, err := taskman.TaskManager.NewTask(ctx, taskName, guest, userCred, nil, parentTaskId, "", nil)
+	if err != nil {
+		return err
+	}
+	task.ScheduleRun(nil)
+	return nil
+}
+
 func (self *SVirtualizedGuestDriver) OnGuestDeployTaskComplete(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {
 	if jsonutils.QueryBoolean(task.GetParams(), "restart", false) {
 		task.SetStage("OnDeployStartGuestComplete", nil)

pkg/compute/hostdrivers/aliyun.go

@@ -48,7 +48,7 @@ func (self *SAliyunHostDriver) CheckAndSetCacheImage(ctx context.Context, host *
 	return nil
 }
 
-func (self *SAliyunHostDriver) RequestAllocateDiskOnStorage(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error {
+func (self *SAliyunHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error {
 	if iCloudStorage, err := storage.GetIStorage(); err != nil {
 		return err
 	} else {
@@ -61,6 +61,20 @@ func (self *SAliyunHostDriver) RequestAllocateDiskOnStorage(host *models.SHost,
 			} else {
 				if _, err := disk.GetModelManager().TableSpec().Update(disk, func() error {
 					disk.ExternalId = iDisk.GetGlobalId()
+
+					if metaData := iDisk.GetMetadata(); metaData != nil {
+						meta := make(map[string]string)
+						if err := metaData.Unmarshal(meta); err != nil {
+							log.Errorf("Get disk %s Metadata error: %v", disk.Name, err)
+						} else {
+							for key, value := range meta {
+								if err := disk.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil {
+									log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err)
+								}
+							}
+						}
+					}
+
 					return nil
 				}); err != nil {
 					log.Errorf("Update disk externalId err: %v", err)
@@ -81,9 +95,13 @@ func (self *SAliyunHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, s
 		return err
 	} else if iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()); err != nil {
 		return err
+	} else if err := iDisk.Delete(); err != nil {
+		return err
 	} else {
-		return iDisk.Delete()
+		data := jsonutils.NewDict()
+		task.ScheduleRun(data)
 	}
+	return nil
 }
 
 func (self *SAliyunHostDriver) RequestResizeDiskOnHostOnline(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error {

pkg/compute/hostdrivers/kvm.go

@@ -74,7 +74,7 @@ func (self *SKVMHostDriver) CheckAndSetCacheImage(ctx context.Context, host *mod
 	return nil
 }
 
-func (self *SKVMHostDriver) RequestAllocateDiskOnStorage(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error {
+func (self *SKVMHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error {
 	header := http.Header{}
 	header.Add("X-Task-Id", task.GetTaskId())
 	header.Add("X-Region-Version", "v2")

pkg/compute/models/baremetalstatus.go

@@ -0,0 +1,22 @@
+package models
+
+const (
+	BAREMETAL_INIT           = "init"
+	BAREMETAL_PREPARE        = "prepare"
+	BAREMETAL_PREPARE_FAIL   = "prepare_fail"
+	BAREMETAL_READY          = "ready"
+	BAREMETAL_RUNNING        = "running"
+	BAREMETAL_MAINTAINING    = "maintaining"
+	BAREMETAL_START_MAINTAIN = "start_maintain"
+	BAREMETAL_DELETING       = "deleting"
+	BAREMETAL_DELETE         = "delete"
+	BAREMETAL_DELETE_FAIL    = "delete_fail"
+	BAREMETAL_UNKNOWN        = "unknown"
+	BAREMETAL_SYNCING_STATUS = "syncing_status"
+	BAREMETAL_SYNC           = "sync"
+	BAREMETAL_SYNC_FAIL      = "sync_fail"
+	BAREMETAL_START_CONVERT  = "start_convert"
+	BAREMETAL_CONVERTING     = "converting"
+	BAREMETAL_START_FAIL     = "start_fail"
+	BAREMETAL_STOP_FAIL      = "stop_fail"
+)

pkg/compute/models/cloudregions.go

@@ -145,9 +145,9 @@ func (manager *SCloudregionManager) SyncRegions(ctx context.Context, userCred mc
 				_, err = removed[i].PerformDisable(ctx, userCred, nil, nil)
 			}
 			if err != nil {
-				syncResult.Delete()
-			} else {
 				syncResult.DeleteError(err)
+			} else {
+				syncResult.Delete()
 			}
 		}
 	}

pkg/compute/models/disks.go

@@ -16,6 +16,7 @@ import (
 	"yunion.io/x/pkg/util/osprofile"
 	"yunion.io/x/pkg/util/regutils"
 	"yunion.io/x/pkg/util/sysutils"
+	"yunion.io/x/pkg/util/timeutils"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
@@ -296,7 +297,7 @@ func (self *SDisk) StartDiskCreateTask(ctx context.Context, userCred mcclient.To
 	return nil
 }
 
-func (self *SDisk) StartAllocate(host *SHost, storage *SStorage, taskId string, userCred mcclient.TokenCredential, rebuild bool, snapshot string, task taskman.ITask) error {
+func (self *SDisk) StartAllocate(ctx context.Context, host *SHost, storage *SStorage, taskId string, userCred mcclient.TokenCredential, rebuild bool, snapshot string, task taskman.ITask) error {
 	log.Infof("Allocating disk on host %s ...", host.GetName())
 
 	templateId := self.GetTemplateId()
@@ -325,7 +326,7 @@ func (self *SDisk) StartAllocate(host *SHost, storage *SStorage, taskId string,
 	if rebuild {
 		content.Add(jsonutils.JSONTrue, "rebuild")
 	}
-	return host.GetHostDriver().RequestAllocateDiskOnStorage(host, storage, self, task, content)
+	return host.GetHostDriver().RequestAllocateDiskOnStorage(ctx, host, storage, self, task, content)
 }
 
 func (self *SDisk) AllowPerformResize(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
@@ -384,6 +385,13 @@ func (self *SDisk) GetStorage() *SStorage {
 	return nil
 }
 
+func (self *SDisk) GetCloudprovider() *SCloudprovider {
+	if storage := self.GetStorage(); storage != nil {
+		return storage.GetCloudprovider()
+	}
+	return nil
+}
+
 func (self *SDisk) GetPathAtHost(host *SHost) string {
 	storage := self.GetStorage()
 	if storage.StorageType == STORAGE_RBD {
@@ -425,7 +433,7 @@ func (manager *SDiskManager) getDisksByStorage(storage *SStorage) ([]SDisk, erro
 	return disks, nil
 }
 
-func (manager *SDiskManager) syncCloudDisk(userCred mcclient.TokenCredential, vdisk cloudprovider.ICloudDisk) (*SDisk, error) {
+func (manager *SDiskManager) syncCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, vdisk cloudprovider.ICloudDisk) (*SDisk, error) {
 	diskObj, err := manager.FetchByExternalId(vdisk.GetGlobalId())
 	if err != nil {
 		if err == sql.ErrNoRows {
@@ -436,13 +444,13 @@ func (manager *SDiskManager) syncCloudDisk(userCred mcclient.TokenCredential, vd
 				return nil, err
 			}
 			storage := storageObj.(*SStorage)
-			return manager.newFromCloudDisk(userCred, vdisk, storage)
+			return manager.newFromCloudDisk(ctx, userCred, vdisk, storage)
 		} else {
 			return nil, err
 		}
 	} else {
 		disk := diskObj.(*SDisk)
-		err = disk.syncWithCloudDisk(userCred, vdisk)
+		err = disk.syncWithCloudDisk(ctx, userCred, vdisk)
 		if err != nil {
 			return nil, err
 		}
@@ -482,7 +490,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To
 	}
 
 	for i := 0; i < len(commondb); i += 1 {
-		err = commondb[i].syncWithCloudDisk(userCred, commonext[i])
+		err = commondb[i].syncWithCloudDisk(ctx, userCred, commonext[i])
 		if err != nil {
 			syncResult.UpdateError(err)
 		} else {
@@ -493,7 +501,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To
 	}
 
 	for i := 0; i < len(added); i += 1 {
-		new, err := manager.newFromCloudDisk(userCred, added[i], storage)
+		new, err := manager.newFromCloudDisk(ctx, userCred, added[i], storage)
 		if err != nil {
 			syncResult.AddError(err)
 		} else {
@@ -506,7 +514,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To
 	return localDisks, remoteDisks, syncResult
 }
 
-func (self *SDisk) syncWithCloudDisk(userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk) error {
+func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk) error {
 	_, err := self.GetModelManager().TableSpec().Update(self, func() error {
 		extDisk.Refresh()
 		self.Name = extDisk.GetName()
@@ -527,11 +535,26 @@ func (self *SDisk) syncWithCloudDisk(userCred mcclient.TokenCredential, extDisk
 	})
 	if err != nil {
 		log.Errorf("syncWithCloudDisk error %s", err)
+		return err
 	}
-	return err
+
+	if metaData := extDisk.GetMetadata(); metaData != nil {
+		meta := make(map[string]string, 0)
+		if err := metaData.Unmarshal(meta); err != nil {
+			log.Errorf("Get VM Metadata error: %v", err)
+		} else {
+			for key, value := range meta {
+				if err := self.SetMetadata(ctx, key, value, userCred); err != nil {
+					log.Errorf("set disk %s mata %s => %s error: %v", self.Name, key, value, err)
+				}
+			}
+		}
+	}
+
+	return nil
 }
 
-func (manager *SDiskManager) newFromCloudDisk(userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk, storage *SStorage) (*SDisk, error) {
+func (manager *SDiskManager) newFromCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk, storage *SStorage) (*SDisk, error) {
 	disk := SDisk{}
 	disk.SetModelManager(manager)
 
@@ -554,6 +577,20 @@ func (manager *SDiskManager) newFromCloudDisk(userCred mcclient.TokenCredential,
 		log.Errorf("newFromCloudZone fail %s", err)
 		return nil, err
 	}
+
+	if metaData := extDisk.GetMetadata(); metaData != nil {
+		meta := make(map[string]string)
+		if err := metaData.Unmarshal(meta); err != nil {
+			log.Errorf("Get VM Metadata error: %v", err)
+		} else {
+			for key, value := range meta {
+				if err := disk.SetMetadata(ctx, key, value, userCred); err != nil {
+					log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err)
+				}
+			}
+		}
+	}
+
 	return &disk, nil
 }
 
@@ -693,6 +730,27 @@ func parseIsoInfo(ctx context.Context, userCred mcclient.TokenCredential, info s
 	return image.Id, nil
 }
 
+// def get_disk_spec_v2(conf):
+//     def _get_spec(storages):
+//         spec = {}
+//         for adapter, ss in group_by_adapter(storages).items():
+//             if len(ss) == 0:
+//                 continue
+//             spec[adapter] = get_disk_spec(ss)
+//         return spec
+
+//     spec = {}
+//     for driver in DISK_DRIVERS:
+//         storages = [s for s in conf if s['driver'] == driver]
+//         if len(storages) != 0:
+//             spec[driver] = _get_spec(storages)
+//     return spec
+
+func GetDiskSpecV2(storageInfo jsonutils.JSONObject) *jsonutils.JSONDict {
+	// ToDo
+	return nil
+}
+
 func (self *SDisk) fetchDiskInfo(diskConfig *SDiskConfig) {
 	if len(diskConfig.ImageId) > 0 {
 		self.TemplateId = diskConfig.ImageId
@@ -746,6 +804,49 @@ func (self *SDisk) CustomizeDelete(ctx context.Context, userCred mcclient.TokenC
 	return self.StartDiskDeleteTask(ctx, userCred, "", false)
 }
 
+func (self *SDisk) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict {
+	if cloudprovider := self.GetCloudprovider(); cloudprovider != nil {
+		extra.Add(jsonutils.NewString(cloudprovider.Provider), "provider")
+	}
+	if storage := self.GetStorage(); storage != nil {
+		extra.Add(jsonutils.NewString(storage.GetName()), "storage")
+		extra.Add(jsonutils.NewString(storage.StorageType), "storage_type")
+		extra.Add(jsonutils.NewString(storage.MediumType), "medium_type")
+		extra.Add(jsonutils.NewString(storage.ZoneId), "zone_id")
+		if zone := storage.getZone(); zone != nil {
+			extra.Add(jsonutils.NewString(zone.Name), "zone")
+			extra.Add(jsonutils.NewString(zone.CloudregionId), "region_id")
+			if region := zone.GetRegion(); region != nil {
+				extra.Add(jsonutils.NewString(region.Name), "region")
+			}
+		}
+	}
+	guests, guest_status := []string{}, []string{}
+	for _, guest := range self.GetGuests() {
+		guests = append(guests, guest.Name)
+		guest_status = append(guest_status, guest.Status)
+	}
+	extra.Add(jsonutils.NewString(strings.Join(guests, ",")), "guest")
+	extra.Add(jsonutils.NewInt(int64(len(guests))), "guest_count")
+	extra.Add(jsonutils.NewString(strings.Join(guest_status, ",")), "guest_status")
+
+	if self.PendingDeleted {
+		pendingDeletedAt := self.PendingDeletedAt.Add(time.Second * time.Duration(options.Options.PendingDeleteExpireSeconds))
+		extra.Add(jsonutils.NewString(timeutils.FullIsoTime(pendingDeletedAt)), "auto_delete_at")
+	}
+	return extra
+}
+
+func (self *SDisk) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
+	extra := self.SSharableVirtualResourceBase.GetExtraDetails(ctx, userCred, query)
+	return self.getMoreDetails(extra)
+}
+
+func (self *SDisk) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
+	extra := self.SSharableVirtualResourceBase.GetCustomizeColumns(ctx, userCred, query)
+	return self.getMoreDetails(extra)
+}
+
 func (self *SDisk) StartDiskResizeTask(ctx context.Context, userCred mcclient.TokenCredential, size int64, parentTaskId string, pendingUsage quotas.IQuota) error {
 	params := jsonutils.NewDict()
 	params.Add(jsonutils.NewInt(size), "size")
@@ -781,9 +882,8 @@ func (self *SDisk) GetAttachedGuests() []SGuest {
 	q = q.Filter(sqlchemy.Equals(guestdisks.Field("disk_id"), self.Id))
 
 	ret := make([]SGuest, 0)
-	err := q.All(&ret)
-	if err != nil {
-		log.Errorf("%s", err)
+	if err := db.FetchModelObjects(GuestManager, q, &ret); err != nil {
+		log.Errorf("Fetch Geusts Objects %v", err)
 		return nil
 	}
 	return ret
@@ -820,6 +920,11 @@ func (self *SDisk) GetShortDesc() *jsonutils.JSONDict {
 	storage := self.GetStorage()
 	desc.Add(jsonutils.NewString(storage.StorageType), "storage_type")
 	desc.Add(jsonutils.NewString(storage.MediumType), "medium_type")
+
+	if priceKey := self.GetMetadata("price_key", nil); len(priceKey) > 0 {
+		desc.Add(jsonutils.NewString(priceKey), "price_key")
+	}
+
 	fs := self.GetFsFormat()
 	if len(fs) > 0 {
 		desc.Add(jsonutils.NewString(fs), "fs_format")

pkg/compute/models/dnsrecords.go

@@ -6,7 +6,6 @@ import (
 	"strings"
 
 	"yunion.io/x/jsonutils"
-	"yunion.io/x/log"
 	"yunion.io/x/pkg/util/regutils"
 	"yunion.io/x/sqlchemy"
 
@@ -205,7 +204,6 @@ func (man *SDnsRecordManager) QueryDns(projectId, name string) *SDnsRecord {
 
 	err := q.First(rec)
 	if err != nil {
-		log.Errorf("QueryDns %q fail: %s", name, err)
 		return nil
 	}
 	return rec

pkg/compute/models/guestdrivers.go

@@ -39,6 +39,9 @@ type IGuestDriver interface {
 	RequestGuestCreateInsertIso(ctx context.Context, imageId string, guest *SGuest, task taskman.ITask) error
 
 	StartGuestStopTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, parentTaskId string) error
+	StartGuestResetTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, isHard bool, parentTaskId string) error
+
+	RequestSoftReset(ctx context.Context, guest *SGuest, task taskman.ITask) error
 
 	RequestDeployGuestOnHost(ctx context.Context, guest *SGuest, host *SHost, task taskman.ITask) error
 

pkg/compute/models/guestnetworks.go

@@ -19,6 +19,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/compute/options"
+	"database/sql"
 )
 
 const (
@@ -263,17 +264,18 @@ func (self *SGuestnetwork) GetJsonDescAtHost(host *SHost) jsonutils.JSONObject {
 }
 
 func (manager *SGuestnetworkManager) GetGuestByAddress(address string) *SGuest {
-	gn := SGuestnetwork{}
-	gn.SetModelManager(GuestnetworkManager)
-
-	err := manager.Query().Equals("ip_addr", address).First(&gn)
-	if err != nil {
-		log.Errorf("GetGuestByAddress query fail: %s", err)
-		return nil
-	}
-	guest, _ := GuestManager.FetchById(gn.GuestId)
-	if guest != nil {
-		return guest.(*SGuest)
+	networks := manager.TableSpec().Instance()
+	guests := GuestManager.Query()
+	q := guests.Join(networks, sqlchemy.AND(
+		sqlchemy.IsFalse(networks.Field("deleted")),
+		sqlchemy.Equals(networks.Field("ip_addr"), address),
+		sqlchemy.Equals(networks.Field("guest_id"), guests.Field("id")),
+	))
+	guest := &SGuest{}
+	guest.SetModelManager(GuestManager)
+	err := q.First(guest)
+	if err == nil {
+		return guest
 	}
 	return nil
 }
@@ -333,6 +335,22 @@ func (manager *SGuestnetworkManager) DeleteGuestNics(ctx context.Context, guest
 	return nil
 }
 
+func (manager *SGuestnetworkManager) getGuestNicByIP(ip string) (*SGuestnetwork, error) {
+	gn := SGuestnetwork{}
+	q := manager.Query()
+	q = q.Equals("ip_addr", ip)
+	err := q.First(&gn)
+	if err != nil {
+		if err != sql.ErrNoRows {
+			log.Errorf("getGuestNicByIP fail %s", err)
+			return nil, err
+		}
+		return nil, nil
+	}
+	gn.SetModelManager(manager)
+	return &gn, nil
+}
+
 func (self *SGuestnetwork) LogDetachEvent(userCred mcclient.TokenCredential, guest *SGuest, network *SNetwork) {
 	if network == nil {
 		netTmp, _ := NetworkManager.FetchById(self.NetworkId)

pkg/compute/models/guests.go

@@ -205,49 +205,60 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ
 	if len(hypervisor) > 0 {
 		q = q.Equals("hypervisor", hypervisor)
 	}
+
 	hostFilter, _ := queryDict.GetString("host")
-	zoneFilter, _ := queryDict.GetString("zone")
-	wireFilter, _ := queryDict.GetString("wire")
-	networkFilter, _ := queryDict.GetString("network")
-	diskFilter, _ := queryDict.GetString("disk")
-	var sq *sqlchemy.SSubQuery
 	if len(hostFilter) > 0 {
 		host, _ := HostManager.FetchByIdOrName("", hostFilter)
 		if host == nil {
-			return nil, httperrors.NewResourceNotFoundError(fmt.Sprintf("host %s not found", hostFilter))
+			return nil, httperrors.NewResourceNotFoundError("host %s not found", hostFilter)
 		}
-		sq = HostManager.Query("id").Equals("id", host.GetId()).SubQuery()
-	} else if len(zoneFilter) > 0 {
+		q = q.Equals("host_id", host.GetId())
+	}
+
+	zoneFilter, _ := queryDict.GetString("zone")
+	if len(zoneFilter) > 0 {
 		zone, _ := ZoneManager.FetchByIdOrName("", zoneFilter)
 		if zone == nil {
-			return nil, httperrors.NewResourceNotFoundError(fmt.Sprintf("zone %s not found", zoneFilter))
+			return nil, httperrors.NewResourceNotFoundError("zone %s not found", zoneFilter)
 		}
 		hostTable := HostManager.Query().SubQuery()
 		zoneTable := ZoneManager.Query().SubQuery()
-		sq = hostTable.Query(hostTable.Field("id")).Join(zoneTable,
+		sq := hostTable.Query(hostTable.Field("id")).Join(zoneTable,
 			sqlchemy.Equals(zoneTable.Field("id"), hostTable.Field("zone_id"))).Filter(sqlchemy.Equals(zoneTable.Field("id"), zone.GetId())).SubQuery()
-	} else if len(wireFilter) > 0 {
+		q = q.In("host_id", sq)
+	}
+
+	wireFilter, _ := queryDict.GetString("wire")
+	if len(wireFilter) > 0 {
 		wire, _ := WireManager.FetchByIdOrName("", wireFilter)
 		if wire == nil {
-			return nil, httperrors.NewResourceNotFoundError(fmt.Sprintf("wire %s not found", wireFilter))
+			return nil, httperrors.NewResourceNotFoundError("wire %s not found", wireFilter)
 		}
 		hostTable := HostManager.Query().SubQuery()
 		hostWire := HostwireManager.Query().SubQuery()
-		sq = hostTable.Query(hostTable.Field("id")).Join(hostWire, sqlchemy.Equals(hostWire.Field("host_id"), hostTable.Field("id"))).Filter(sqlchemy.Equals(hostWire.Field("wire_id"), wire.GetId())).SubQuery()
-	} else if len(networkFilter) > 0 {
+		sq := hostTable.Query(hostTable.Field("id")).Join(hostWire, sqlchemy.Equals(hostWire.Field("host_id"), hostTable.Field("id"))).Filter(sqlchemy.Equals(hostWire.Field("wire_id"), wire.GetId())).SubQuery()
+		q = q.In("host_id", sq)
+	}
+
+	networkFilter, _ := queryDict.GetString("network")
+	if len(networkFilter) > 0 {
 		netI, _ := NetworkManager.FetchByIdOrName(userCred.GetProjectId(), networkFilter)
 		if netI == nil {
-			return nil, httperrors.NewResourceNotFoundError(fmt.Sprintf("network %s not found", networkFilter))
+			return nil, httperrors.NewResourceNotFoundError("network %s not found", networkFilter)
 		}
 		net := netI.(*SNetwork)
 		hostTable := HostManager.Query().SubQuery()
 		hostWire := HostwireManager.Query().SubQuery()
-		sq = hostTable.Query(hostTable.Field("id")).Join(hostWire,
+		sq := hostTable.Query(hostTable.Field("id")).Join(hostWire,
 			sqlchemy.Equals(hostWire.Field("host_id"), hostTable.Field("id"))).Filter(sqlchemy.Equals(hostWire.Field("wire_id"), net.WireId)).SubQuery()
-	} else if len(diskFilter) > 0 {
+		q = q.In("host_id", sq)
+	}
+
+	diskFilter, _ := queryDict.GetString("disk")
+	if len(diskFilter) > 0 {
 		diskI, _ := DiskManager.FetchByIdOrName(userCred.GetProjectId(), diskFilter)
 		if diskI == nil {
-			return nil, httperrors.NewResourceNotFoundError(fmt.Sprintf("disk %s not found", diskFilter))
+			return nil, httperrors.NewResourceNotFoundError("disk %s not found", diskFilter)
 		}
 		disk := diskI.(*SDisk)
 		guestdisks := GuestdiskManager.Query().SubQuery()
@@ -264,7 +275,7 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ
 			hosts := HostManager.Query().SubQuery()
 			hoststorages := HoststorageManager.Query().SubQuery()
 			storages := StorageManager.Query().SubQuery()
-			sq = hosts.Query(hosts.Field("id")).
+			sq := hosts.Query(hosts.Field("id")).
 				Join(hoststorages, sqlchemy.AND(
 					sqlchemy.Equals(hoststorages.Field("host_id"), hosts.Field("id")),
 					sqlchemy.IsFalse(hoststorages.Field("deleted")))).
@@ -272,11 +283,21 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ
 					sqlchemy.Equals(storages.Field("id"), hoststorages.Field("storage_id")),
 					sqlchemy.IsFalse(storages.Field("deleted")))).
 				Filter(sqlchemy.Equals(storages.Field("id"), disk.StorageId)).SubQuery()
+			q = q.In("host_id", sq)
 		}
 	}
-	if sq != nil {
+
+	managerFilter, _ := queryDict.GetString("manager")
+	if len(managerFilter) > 0 {
+		managerI, _ := CloudproviderManager.FetchByIdOrName(userCred.GetProjectId(), managerFilter)
+		if managerI == nil {
+			return nil, httperrors.NewResourceNotFoundError("cloud provider %s not found", managerFilter)
+		}
+		hosts := HostManager.Query().SubQuery()
+		sq := hosts.Query(hosts.Field("id")).Equals("manager_id", managerI.GetId()).SubQuery()
 		q = q.In("host_id", sq)
 	}
+
 	gpu, _ := queryDict.GetString("gpu")
 	if len(gpu) != 0 {
 		isodev := IsolatedDeviceManager.Query().SubQuery()
@@ -472,9 +493,11 @@ func (self *SGuest) ValidateUpdateData(ctx context.Context, userCred mcclient.To
 		return nil, err
 	}
 
-	// if data.Contains("name") {
-	//	return nil, httperrors.NewInputParameterError("cannot update server name")
-	// }
+	if data.Contains("name") {
+		if name, _ := data.GetString("name"); len(name) < 2 {
+			return nil, httperrors.NewInputParameterError("name is to short")
+		}
+	}
 	/* if self.GetHypervisor() == HYPERVISOR_BAREMETAL {
 		return nil, httperrors.NewInputParameterError("Cannot modify memory for baremetal")
 	}
@@ -906,11 +929,7 @@ func (self *SGuest) GetCustomizeColumns(ctx context.Context, userCred mcclient.T
 	// extra.Add(jsonutils.NewString(self.getFlavorName()), "flavor")
 	extra.Add(jsonutils.NewString(self.getKeypairName()), "keypair")
 	extra.Add(jsonutils.NewInt(int64(self.getExtBandwidth())), "ext_bw")
-	zone := self.getZone()
-	if zone != nil {
-		extra.Add(jsonutils.NewString(zone.Id), "zone_id")
-		extra.Add(jsonutils.NewString(zone.Name), "zone")
-	}
+
 	extra.Add(jsonutils.NewString(self.GetSecgroupName()), "secgroup")
 
 	if self.PendingDeleted {
@@ -918,6 +937,28 @@ func (self *SGuest) GetCustomizeColumns(ctx context.Context, userCred mcclient.T
 		extra.Add(jsonutils.NewString(timeutils.FullIsoTime(pendingDeletedAt)), "auto_delete_at")
 	}
 
+	return self.moreExtraInfo(extra)
+}
+
+func (self *SGuest) moreExtraInfo(extra *jsonutils.JSONDict) *jsonutils.JSONDict {
+	zone := self.getZone()
+	if zone != nil {
+		extra.Add(jsonutils.NewString(zone.GetId()), "zone_id")
+		extra.Add(jsonutils.NewString(zone.GetName()), "zone")
+		if len(zone.ExternalId) > 0 {
+			extra.Add(jsonutils.NewString(zone.ExternalId), "zone_external_id")
+		}
+
+		region := zone.GetRegion()
+		if region != nil {
+			extra.Add(jsonutils.NewString(region.Id), "region_id")
+			extra.Add(jsonutils.NewString(region.Name), "region")
+
+			if len(region.ExternalId) > 0 {
+				extra.Add(jsonutils.NewString(region.ExternalId), "region_external_id")
+			}
+		}
+	}
 	return extra
 }
 
@@ -948,14 +989,7 @@ func (self *SGuest) GetExtraDetails(ctx context.Context, userCred mcclient.Token
 		}
 		extra.Add(jsonutils.NewString(self.getAdminSecurityRules()), "admin_security_rules")
 	}
-	zone := self.getZone()
-	if zone != nil {
-		extra.Add(jsonutils.NewString(zone.GetId()), "zone_id")
-		extra.Add(jsonutils.NewString(zone.GetName()), "zone")
-		extra.Add(jsonutils.NewString(zone.GetRegion().GetName()), "region")
-		extra.Add(jsonutils.NewString(zone.GetRegion().GetId()), "region_id")
-	}
-	return extra
+	return self.moreExtraInfo(extra)
 }
 
 func (self *SGuest) getNetworksDetails() string {
@@ -1215,6 +1249,18 @@ func (self *SGuest) syncWithCloudVM(ctx context.Context, userCred mcclient.Token
 			db.OpsLog.LogEvent(self, db.ACT_UPDATE, diffStr, userCred)
 		}
 	}
+	if metaData := extVM.GetMetadata(); metaData != nil {
+		meta := make(map[string]string, 0)
+		if metaData.Unmarshal(meta); err != nil {
+			log.Errorf("Get VM Metadata error: %v", err)
+		} else {
+			for key, value := range meta {
+				if err := self.SetMetadata(ctx, key, value, userCred); err != nil {
+					log.Errorf("set guest %s mata %s => %s error: %v", self.Name, key, value, err)
+				}
+			}
+		}
+	}
 	return nil
 }
 
@@ -1245,6 +1291,20 @@ func (manager *SGuestManager) newCloudVM(ctx context.Context, userCred mcclient.
 	if err != nil {
 		log.Errorf("Insert fail %s", err)
 	}
+
+	if metaData := extVM.GetMetadata(); metaData != nil {
+		meta := make(map[string]string, 0)
+		if err := metaData.Unmarshal(meta); err != nil {
+			log.Errorf("Get VM Metadata error: %v", err)
+		} else {
+			for key, value := range meta {
+				if err := guest.SetMetadata(ctx, key, value, userCred); err != nil {
+					log.Errorf("set guest %s mata %s => %s error: %v", guest.Name, key, value, err)
+				}
+			}
+		}
+	}
+
 	return &guest, nil
 }
 
@@ -1443,7 +1503,20 @@ func (self *SGuest) SyncVMNics(ctx context.Context, userCred mcclient.TokenCrede
 		if add.net == nil {
 			continue // cannot determine which network it attached to
 		}
-		err := self.Attach2Network(ctx, userCred, add.net, nil, add.nic.GetIP(),
+		// check if the IP has been occupied, if yes, release the IP
+		gn, err := GuestnetworkManager.getGuestNicByIP(add.nic.GetIP())
+		if err != nil {
+			result.AddError(err)
+			continue
+		}
+		if gn != nil {
+			err = gn.Detach(ctx, userCred)
+			if err != nil {
+				result.AddError(err)
+				continue
+			}
+		}
+		err = self.Attach2Network(ctx, userCred, add.net, nil, add.nic.GetIP(),
 			add.nic.GetMAC(), add.nic.GetDriver(), 0, false, -1, add.reserve, IPAllocationDefault, true)
 		if err != nil {
 			result.AddError(err)
@@ -1507,7 +1580,11 @@ func (self *SGuest) PerformDeploy(ctx context.Context, userCred mcclient.TokenCr
 	if !ok {
 		return nil, fmt.Errorf("Parse query body error")
 	}
+
+	// 变更密码/密钥时需要Restart才能生效。更新普通字段不需要Restart
+	doRestart := false
 	if kwargs.Contains("__delete_keypair__") || kwargs.Contains("keypair") {
+		doRestart = true
 		var kpId string
 		if !jsonutils.QueryBoolean(kwargs, "__delete_keypair__", false) {
 			keypair, _ := kwargs.GetString("keypair")
@@ -1529,8 +1606,9 @@ func (self *SGuest) PerformDeploy(ctx context.Context, userCred mcclient.TokenCr
 			kwargs.Set("reset_password", jsonutils.JSONTrue)
 		}
 	}
+
 	if utils.IsInStringArray(self.Status, []string{VM_RUNNING, VM_READY, VM_ADMIN}) {
-		if self.Status == VM_RUNNING {
+		if doRestart && self.Status == VM_RUNNING {
 			kwargs.Set("restart", jsonutils.JSONTrue)
 		}
 		err := self.StartGuestDeployTask(ctx, userCred, kwargs, "deploy", "")
@@ -1627,7 +1705,7 @@ func (self *SGuest) SyncVMDisks(ctx context.Context, userCred mcclient.TokenCred
 		if len(vdisks[i].GetGlobalId()) == 0 {
 			continue
 		}
-		disk, err := DiskManager.syncCloudDisk(userCred, vdisks[i])
+		disk, err := DiskManager.syncCloudDisk(ctx, userCred, vdisks[i])
 		if err != nil {
 			result.Error(err)
 			return result
@@ -1885,12 +1963,10 @@ func (self *SGuest) createDiskOnHost(ctx context.Context, userCred mcclient.Toke
 	if storage == nil {
 		return nil, fmt.Errorf("No storage to create disk")
 	}
-
 	disk, err := self.createDiskOnStorage(ctx, userCred, storage, diskConfig, pendingUsage)
 	if err != nil {
 		return nil, err
 	}
-
 	err = self.attach2Disk(disk, userCred, diskConfig.Driver, diskConfig.Cache, diskConfig.Mountpoint)
 	return disk, err
 }
@@ -2316,7 +2392,7 @@ func (self *SGuest) PerformDetachdisk(ctx context.Context, userCred mcclient.Tok
 					disk.SetStatus(userCred, DISK_DETACHING, "")
 				}
 				taskData := jsonutils.NewDict()
-				taskData.Add(jsonutils.NewString(diskId), "disk_id")
+				taskData.Add(jsonutils.NewString(disk.Id), "disk_id")
 				taskData.Add(jsonutils.NewBool(keepDisk), "keep_disk")
 				self.GetDriver().StartGuestDetachdiskTask(ctx, userCred, self, taskData, "")
 				return nil, nil
@@ -2423,16 +2499,27 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T
 		diskIdx += 1
 	}
 
-	for storageId, needSize := range diskSizes {
-		iStorage, err := StorageManager.FetchById(storageId)
-		if err != nil {
-			return nil, httperrors.NewBadRequestError("Fetch storage error: %s", err)
-		}
-		storage := iStorage.(*SStorage)
-		if storage.GetFreeCapacity() < needSize {
-			return nil, httperrors.NewInsufficientResourceError("Not enough free space")
-		}
+	provider, e := self.GetHost().GetDriver()
+	if e != nil {
+		log.Errorf("Get Provider Error: %s", e)
+		return nil, httperrors.NewInsufficientResourceError("Provider Not Found")
 	}
+
+	if !provider.IsPublicCloud() {
+		for storageId, needSize := range diskSizes {
+			iStorage, err := StorageManager.FetchById(storageId)
+			if err != nil {
+				return nil, httperrors.NewBadRequestError("Fetch storage error: %s", err)
+			}
+			storage := iStorage.(*SStorage)
+			if storage.GetFreeCapacity() < needSize {
+				return nil, httperrors.NewInsufficientResourceError("Not enough free space")
+			}
+		}
+	} else {
+		log.Debugf("Skip storage free capacity validating for public cloud: %s", provider.GetName())
+	}
+
 	if newDisks.Length() > 0 {
 		confs.Add(newDisks, "create")
 	}
@@ -2592,9 +2679,9 @@ func (self *SGuest) AllowDeleteItem(ctx context.Context, userCred mcclient.Token
 func (self *SGuest) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
 	overridePendingDelete := false
 	purge := false
-	if data != nil {
-		overridePendingDelete = jsonutils.QueryBoolean(data, "override_pending_delete", false)
-		purge = jsonutils.QueryBoolean(data, "purge", false)
+	if query != nil {
+		overridePendingDelete = jsonutils.QueryBoolean(query, "override_pending_delete", false)
+		purge = jsonutils.QueryBoolean(query, "purge", false)
 	}
 	return self.StartDeleteGuestTask(ctx, userCred, "", purge, overridePendingDelete)
 }
@@ -3006,6 +3093,111 @@ func (manager *SGuestManager) FetchGuestById(guestId string) *SGuest {
 	return guest.(*SGuest)
 }
 
+func (self *SGuest) GetSpec(checkStatus bool) *jsonutils.JSONDict {
+	if checkStatus {
+		if !utils.IsInStringArray(self.Status, []string{VM_SCHEDULE_FAILED}) {
+			return nil
+		}
+	}
+	spec := jsonutils.NewDict()
+	spec.Set("cpu", jsonutils.NewInt(int64(self.VcpuCount)))
+	spec.Set("mem", jsonutils.NewInt(int64(self.VmemSize)))
+
+	// get disk spec
+	guestdisks := self.GetDisks()
+	diskSpecs := jsonutils.NewArray()
+	for _, guestdisk := range guestdisks {
+		disk := guestdisk.GetDisk()
+		diskSpec := jsonutils.NewDict()
+		diskSpec.Set("size", jsonutils.NewInt(int64(disk.DiskSize)))
+		s := disk.GetStorage()
+		diskSpec.Set("backend", jsonutils.NewString(s.StorageType))
+		diskSpec.Set("medium_type", jsonutils.NewString(s.MediumType))
+		diskSpecs.Add(diskSpec)
+	}
+	spec.Set("disk", diskSpecs)
+
+	// get nic spec
+	guestnics := self.GetNetworks()
+	nicSpecs := jsonutils.NewArray()
+	for _, guestnic := range guestnics {
+		nicSpec := jsonutils.NewDict()
+		nicSpec.Set("bandwidth", jsonutils.NewInt(int64(guestnic.getBandwidth())))
+		t := "int"
+		if guestnic.IsExit() {
+			t = "ext"
+		}
+		nicSpec.Set("type", jsonutils.NewString(t))
+		nicSpecs.Add(nicSpec)
+	}
+	spec.Set("nic", nicSpecs)
+
+	// get isolate device spec
+	guestgpus := self.GetIsolatedDevices()
+	gpuSpecs := jsonutils.NewArray()
+	for _, guestgpu := range guestgpus {
+		if strings.HasPrefix(guestgpu.DevType, "GPU") {
+			gs := guestgpu.GetSpec(false)
+			if gs != nil {
+				gpuSpecs.Add(gs)
+			}
+		}
+	}
+	spec.Set("gpu", gpuSpecs)
+	return spec
+}
+
+func (self *SGuest) GetTemplateId() string {
+	guestdisks := self.GetDisks()
+	for _, guestdisk := range guestdisks {
+		disk := guestdisk.GetDisk()
+		if disk != nil {
+			templateId := disk.GetTemplateId()
+			if len(templateId) > 0 {
+				return templateId
+			}
+		}
+	}
+	return ""
+}
+
+func (self *SGuest) GetShortDesc() *jsonutils.JSONDict {
+	desc := self.SStandaloneResourceBase.GetShortDesc()
+	desc.Set("mem", jsonutils.NewInt(int64(self.VmemSize)))
+	desc.Set("cpu", jsonutils.NewInt(int64(self.VcpuCount)))
+	templateId := self.GetTemplateId()
+	if len(templateId) > 0 {
+		desc.Set("cpu", jsonutils.NewString(templateId))
+	}
+	extBw := self.getBandwidth(true)
+	intBw := self.getBandwidth(false)
+	if extBw > 0 {
+		desc.Set("ext_bandwidth", jsonutils.NewInt(int64(extBw)))
+	}
+	if intBw > 0 {
+		desc.Set("int_bandwidth", jsonutils.NewInt(int64(intBw)))
+	}
+
+	if priceKey := self.GetMetadata("price_key", nil); len(priceKey) > 0 {
+		desc.Add(jsonutils.NewString(priceKey), "price_key")
+	}
+
+	desc.Set("hypervisor", jsonutils.NewString(self.GetHypervisor()))
+	spec := self.GetSpec(false)
+	if self.GetHypervisor() == HYPERVISOR_BAREMETAL {
+		host := self.GetHost()
+		if host != nil {
+			hostSpec := host.GetSpec(false)
+			hostSpecIdent := host.GetSpecIdent(hostSpec)
+			spec.Set("host_spec", jsonutils.NewString(strings.Join(hostSpecIdent, "/")))
+		}
+	}
+	if spec != nil {
+		desc.Update(spec)
+	}
+	return desc
+}
+
 func (self *SGuest) saveOsType(osType string) error {
 	_, err := self.GetModelManager().TableSpec().Update(self, func() error {
 		self.OsType = osType
@@ -3114,6 +3306,23 @@ func (self *SGuest) PerformStart(ctx context.Context, userCred mcclient.TokenCre
 	}
 }
 
+func (self *SGuest) AllowPerformReset(ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject) bool {
+	return self.IsOwner(userCred)
+}
+
+func (self *SGuest) PerformReset(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject,
+	data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	isHard := jsonutils.QueryBoolean(data, "is_hard", false)
+	if self.Status == VM_RUNNING || self.Status == VM_STOP_FAILED {
+		self.GetDriver().StartGuestResetTask(self, ctx, userCred, isHard, "")
+		return nil, nil
+	}
+	return nil, httperrors.NewInvalidStatusError("Cannot reset VM in status %s", self.Status)
+}
+
 func (self *SGuest) AllowPerformStop(ctx context.Context,
 	userCred mcclient.TokenCredential,
 	query jsonutils.JSONObject,

pkg/compute/models/hostdrivers.go

@@ -12,7 +12,7 @@ import (
 type IHostDriver interface {
 	GetHostType() string
 	CheckAndSetCacheImage(ctx context.Context, host *SHost, storagecache *SStoragecache, scimg *SStoragecachedimage, task taskman.ITask) error
-	RequestAllocateDiskOnStorage(host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask, content *jsonutils.JSONDict) error
+	RequestAllocateDiskOnStorage(ctx context.Context, host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask, content *jsonutils.JSONDict) error
 	RequestDeallocateDiskOnHost(host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask) error
 	RequestResizeDiskOnHostOnline(host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error
 	RequestResizeDiskOnHost(host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error

pkg/compute/models/hostnetworks.go

@@ -92,7 +92,7 @@ func (man *SHostnetworkManager) QueryByAddress(addr string) *sqlchemy.SQuery {
 func (man *SHostnetworkManager) GetHostNetworkByAddress(addr string) *SHostnetwork {
 	network := SHostnetwork{}
 	err := man.QueryByAddress(addr).First(&network)
-	if err != nil {
+	if err == nil {
 		return &network
 	}
 	return nil
@@ -107,9 +107,18 @@ func (man *SHostnetworkManager) GetNetworkByAddress(addr string) *SNetwork {
 }
 
 func (man *SHostnetworkManager) GetHostByAddress(addr string) *SHost {
-	net := man.GetHostNetworkByAddress(addr)
-	if net == nil {
-		return nil
+	networks := man.TableSpec().Instance()
+	hosts := HostManager.Query()
+	q := hosts.Join(networks, sqlchemy.AND(
+		sqlchemy.IsFalse(networks.Field("deleted")),
+		sqlchemy.Equals(networks.Field("ip_addr"), addr),
+		sqlchemy.Equals(networks.Field("baremetal_id"), hosts.Field("id")),
+	))
+	host := &SHost{}
+	host.SetModelManager(HostManager)
+	err := q.First(host)
+	if err == nil {
+		return host
 	}
-	return net.GetHost()
+	return nil
 }

pkg/compute/models/hosts.go

@@ -506,6 +506,58 @@ func (self *SHost) ClearSchedDescCache() error {
 	return HostManager.ClearSchedDescCache(self.Id)
 }
 
+func (self *SHost) GetSpec(statusCheck bool) *jsonutils.JSONDict {
+	if statusCheck {
+		if utils.IsInStringArray(self.Status, []string{BAREMETAL_INIT, BAREMETAL_PREPARE_FAIL, BAREMETAL_PREPARE}) ||
+			self.getBaremetalServer() != nil {
+			return nil
+		}
+		if self.MemSize == 0 || self.CpuCount == 0 {
+			return nil
+		}
+	}
+	spec := self.GetHardwareSpecification()
+	spec.Remove("storage_info")
+	netInfo := jsonutils.NewArray()
+	nifs := self.GetNetInterfaces()
+	for _, nif := range nifs {
+		netDesc := nif.getBaremetalJsonDesc()
+		nicType, err := netDesc.GetString("nic_type")
+		if err != nil && nicType != NIC_TYPE_IPMI {
+			netInfo.Add(netDesc)
+		}
+	}
+	spec.Set("nic_count", jsonutils.NewInt(int64(netInfo.Length())))
+	manufacture, err := self.SysInfo.Get("manufacture")
+	if err != nil {
+		manufacture = jsonutils.NewString("Unknown")
+	}
+	spec.Set("manufacture", manufacture)
+	model, err := self.SysInfo.Get("model")
+	if err != nil {
+		model = jsonutils.NewString("Unknown")
+	}
+	spec.Set("model", model)
+	return nil
+}
+
+func (self *SHost) GetSpecIdent(spec *jsonutils.JSONDict) []string {
+	// Todo
+	return []string{}
+}
+
+func (self *SHost) GetHardwareSpecification() *jsonutils.JSONDict {
+	spec := jsonutils.NewDict()
+	spec.Set("cpu", jsonutils.NewInt(int64(self.CpuCount)))
+	spec.Set("mem", jsonutils.NewInt(int64(self.MemSize)))
+	if self.StorageInfo != nil {
+		spec.Set("disk", GetDiskSpecV2(self.StorageInfo))
+		spec.Set("driver", jsonutils.NewString(self.StorageDriver))
+		spec.Set("storage_info", self.StorageInfo)
+	}
+	return spec
+}
+
 type SStorageCapacity struct {
 	Capacity  int
 	Used      int

pkg/compute/models/isolated_devices.go

@@ -380,6 +380,24 @@ func (self *SIsolatedDevice) getDesc() *jsonutils.JSONDict {
 	return desc
 }
 
+func (self *SIsolatedDevice) GetSpec(checkStatus bool) *jsonutils.JSONDict {
+	if checkStatus {
+		if len(self.GuestId) > 0 {
+			return nil
+		}
+		host := self.getHost()
+		if host.Status != BAREMETAL_RUNNING || !host.Enabled {
+			return nil
+		}
+	}
+	spec := jsonutils.NewDict()
+	spec.Set("dev_type", jsonutils.NewString(self.DevType))
+	spec.Set("model", jsonutils.NewString(self.Model))
+	spec.Set("pci_id", jsonutils.NewString(self.VendorDeviceId))
+	spec.Set("vendor", jsonutils.NewString(self.getVendor()))
+	return spec
+}
+
 func (self *SIsolatedDevice) GetShortDesc() *jsonutils.JSONDict {
 	desc := self.getDesc()
 	desc.Add(jsonutils.NewString(self.Keyword()), "res_name")

pkg/compute/models/secgrouprules.go

@@ -328,7 +328,7 @@ func (manager *SSecurityGroupRuleManager) SyncRules(ctx context.Context, userCre
 				cmp := strings.Compare(dbStr, ruleStr)
 				if cmp == 0 {
 					if dbRules[j].Description != rules[i].Description {
-						if _, err := manager.TableSpec().Update(dbRules[j], func() error {
+						if _, err := manager.TableSpec().Update(&dbRules[j], func() error {
 							dbRules[j].Description = rules[i].Description
 							return nil
 						}); err != nil {

pkg/compute/models/storages.go

@@ -12,6 +12,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/pkg/tristate"
 	"yunion.io/x/pkg/util/compare"
+	"yunion.io/x/pkg/util/sysutils"
 	"yunion.io/x/sqlchemy"
 )
 
@@ -100,6 +101,10 @@ func (self *SStorage) IsLocal() bool {
 	return self.StorageType == STORAGE_LOCAL || self.StorageType == STORAGE_BAREMETAL
 }
 
+func (manager *SStorageManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+	return true
+}
+
 func (self *SStorage) GetUsedCapacity(isReady tristate.TriState) int {
 	disks := DiskManager.Query().SubQuery()
 	q := disks.Query(sqlchemy.SUM("sum", disks.Field("disk_size"))).Equals("storage_id", self.Id)
@@ -654,6 +659,26 @@ func (manager *SStorageManager) ListItemFilter(ctx context.Context, q *sqlchemy.
 		q = q.Filter(sqlchemy.In(q.Field("zone_id"), sq.SubQuery()))
 	}
 
+	if jsonutils.QueryBoolean(query, "share", false) {
+		q = q.Filter(sqlchemy.NotIn(q.Field("storage_type"), sysutils.LOCAL_STORAGE_TYPES))
+	}
+
+	if jsonutils.QueryBoolean(query, "local", false) {
+		q = q.Filter(sqlchemy.In(q.Field("storage_type"), sysutils.LOCAL_STORAGE_TYPES))
+	}
+
+	if jsonutils.QueryBoolean(query, "usable", false) {
+		hostStorageTable := HoststorageManager.Query().SubQuery()
+		hostTable := HostManager.Query().SubQuery()
+		sq := hostStorageTable.Query(hostStorageTable.Field("storage_id")).Join(hostTable,
+			sqlchemy.Equals(hostTable.Field("id"), hostStorageTable.Field("host_id"))).
+			Filter(sqlchemy.Equals(hostTable.Field("host_status"), HOST_ONLINE))
+
+		q = q.Filter(sqlchemy.In(q.Field("id"), sq)).
+			Filter(sqlchemy.In(q.Field("status"), []string{STORAGE_ENABLED, STORAGE_ONLINE})).
+			Filter(sqlchemy.IsTrue(q.Field("enabled")))
+	}
+
 	managerStr := jsonutils.GetAnyString(query, []string{"manager", "provider", "manager_id", "provider_id"})
 	if len(managerStr) > 0 {
 		provider := CloudproviderManager.FetchCloudproviderByIdOrName(managerStr)

pkg/compute/models/wires.go

@@ -347,6 +347,7 @@ func (self *SWire) getNetworks() ([]SNetwork, error) {
 func (self *SWire) getGatewayNetworkQuery() *sqlchemy.SQuery {
 	q := self.getNetworkQuery()
 	q = q.IsNotNull("guest_gateway").IsNotEmpty("guest_gateway")
+	q = q.Equals("status", NETWORK_STATUS_AVAILABLE)
 	return q
 }
 

pkg/compute/models/zones.go

@@ -515,7 +515,7 @@ func (self *SZone) GetDetailsCapabilities(ctx context.Context, userCred mcclient
 }
 
 func (self *SZone) getHypervisors() []string {
-	q := HostManager.Query("host_type").Equals("zone_id", self.Id)
+	q := HostManager.Query("host_type").Equals("zone_id", self.Id).Distinct()
 	rows, err := q.Rows()
 	if err != nil {
 		return nil

pkg/compute/options/options.go

@@ -21,10 +21,10 @@ type ComputeOptions struct {
 
 	DefaultDiskSize int `default:"30720" help:"Default disk size in MB if not specified, default to 30GiB"`
 
-	EnablePendingDelete            bool  `default:"true" help:"Turn on/off pending delete VM and disk, default is on"`
-	PendingDeleteCheckSeconds      int `default:"3600" help:"How long to wait to scan pending delete VM or disks, default is 1 hour"`
-	PendingDeleteExpireSeconds     int `default:"259200" help:"How long a pending delete VM/disks cleaned automatically, default 3 days"`
-	PendingDeleteMaxCleanBatchSize int `default:"50" help:"How many pending delete servers can be clean in a batch"`
+	EnablePendingDelete            bool `default:"true" help:"Turn on/off pending delete VM and disk, default is on"`
+	PendingDeleteCheckSeconds      int  `default:"3600" help:"How long to wait to scan pending delete VM or disks, default is 1 hour"`
+	PendingDeleteExpireSeconds     int  `default:"259200" help:"How long a pending delete VM/disks cleaned automatically, default 3 days"`
+	PendingDeleteMaxCleanBatchSize int  `default:"50" help:"How many pending delete servers can be clean in a batch"`
 
 	ImageCacheStoragePolicy string `default:"least_used" choices:"best_fit|least_used" help:"Policy to choose storage for image cache, best_fit or least_used"`
 	MetricsRetentionDays    int32  `default:"30" help:"Retention days for monitoring metrics in influxdb"`

pkg/compute/tasks/disk_create_task.go

@@ -40,10 +40,10 @@ func (self *DiskCreateTask) OnStorageCacheImageComplete(ctx context.Context, dis
 	}
 	storage := disk.GetStorage()
 	host := storage.GetMasterHost()
-	db.OpsLog.LogEvent(disk, db.ACT_ALLOCATE, disk.GetShortDesc(), self.GetUserCred())
+	db.OpsLog.LogEvent(disk, db.ACT_ALLOCATING, disk.GetShortDesc(), self.GetUserCred())
 	disk.SetStatus(self.GetUserCred(), models.DISK_STARTALLOC, "")
 	self.SetStage("on_disk_ready", nil)
-	if err := disk.StartAllocate(host, storage, self.GetTaskId(), self.GetUserCred(), rebuild, snapshot, self); err != nil {
+	if err := disk.StartAllocate(ctx, host, storage, self.GetTaskId(), self.GetUserCred(), rebuild, snapshot, self); err != nil {
 		self.OnStartAllocateFailed(ctx, disk, jsonutils.NewString(err.Error()))
 	}
 }

pkg/compute/tasks/disk_delete_task.go

@@ -8,6 +8,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/compute/options"
+	"yunion.io/x/onecloud/pkg/httperrors"
 )
 
 type DiskDeleteTask struct {
@@ -51,7 +52,9 @@ func (self *DiskDeleteTask) startDeleteDisk(ctx context.Context, disk *models.SD
 		self.OnGuestDiskDeleteSucc(ctx, disk, nil)
 	} else {
 		self.SetStage("on_guest_disk_delete_succ", nil)
-		if err := host.GetHostDriver().RequestDeallocateDiskOnHost(host, storage, disk, self); err != nil {
+		if host == nil {
+			self.OnGuestDiskDeleteFailed(ctx, disk, httperrors.NewNotFoundError("fail to find master host"))
+		} else if err := host.GetHostDriver().RequestDeallocateDiskOnHost(host, storage, disk, self); err != nil {
 			self.OnGuestDiskDeleteFailed(ctx, disk, err)
 		}
 	}

pkg/compute/tasks/disk_resize_task.go

@@ -32,7 +32,7 @@ func (self *DiskResizeTask) OnInit(ctx context.Context, obj db.IStandaloneModel,
 	}
 	resion := "Cannot find host for disk"
 	if host == nil || host.HostStatus != models.HOST_ONLINE {
-		disk.SetStatus(self.GetUserCred(), models.DISK_READY, resion)
+		disk.SetDiskReady(ctx, self.GetUserCred(), resion)
 		self.SetStageFailed(ctx, resion)
 		db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, resion, self.GetUserCred())
 	} else {
@@ -65,7 +65,7 @@ func (self *DiskResizeTask) OnStartResizeDiskSucc(ctx context.Context, disk *mod
 }
 
 func (self *DiskResizeTask) OnStartResizeDiskFailed(ctx context.Context, disk *models.SDisk, resion error) {
-	disk.SetStatus(self.GetUserCred(), models.DISK_READY, resion.Error())
+	disk.SetDiskReady(ctx, self.GetUserCred(), resion.Error())
 	self.SetStageFailed(ctx, resion.Error())
 	db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, resion.Error(), self.GetUserCred())
 }
@@ -94,6 +94,7 @@ func (self *DiskResizeTask) OnDiskResizeComplete(ctx context.Context, disk *mode
 		self.OnStartResizeDiskFailed(ctx, disk, err)
 		return
 	}
+	disk.SetDiskReady(ctx, self.GetUserCred(), "")
 	notes := fmt.Sprintf("%s=>%s", oldStatus, disk.Status)
 	db.OpsLog.LogEvent(disk, db.ACT_UPDATE_STATUS, notes, self.UserCred)
 	self.CleanHostSchedCache(disk)
@@ -103,6 +104,6 @@ func (self *DiskResizeTask) OnDiskResizeComplete(ctx context.Context, disk *mode
 }
 
 func (self *DiskResizeTask) OnDiskResizeCompleteFailed(ctx context.Context, disk *models.SDisk, resion error) {
-	disk.SetStatus(self.UserCred, models.DISK_READY, resion.Error())
+	disk.SetDiskReady(ctx, self.GetUserCred(), resion.Error())
 	db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, disk.GetShortDesc(), self.UserCred)
 }

pkg/compute/tasks/guest_create_disk_task.go

@@ -9,6 +9,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/log"
 )
 
 type GuestCreateDiskTask struct {
@@ -118,7 +119,115 @@ func (self *KVMGuestCreateDiskTask) OnConfigSyncComplete(ctx context.Context, ob
 	self.SetStageComplete(ctx, nil)
 }
 
+type AliyunGuestCreateDiskTask struct {
+	SGuestBaseTask
+}
+
+func (self *AliyunGuestCreateDiskTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	self.SetStage("on_aliyun_disk_prepared", nil)
+	self.OnAliyunDiskPrepared(ctx, obj, data)
+}
+
+func (self *AliyunGuestCreateDiskTask) OnAliyunDiskPrepared(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	var diskIndex = 0
+	var diskReady = true
+	for {
+		diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex))
+		if !diskReady || err != nil {
+			break
+		}
+		iDisk, err := models.DiskManager.FetchById(diskId)
+		if err != nil {
+			self.SetStageFailed(ctx, err.Error())
+			return
+		}
+		if iDisk == nil {
+			self.SetStageFailed(ctx, "Disk not found")
+			return
+		}
+		disk := iDisk.(*models.SDisk)
+		if disk.Status == models.DISK_INIT {
+			snapInfo, err := self.Params.GetString(fmt.Sprintf("disk.%d.snapshot", diskIndex))
+			if err != nil {
+				snapInfo = ""
+			}
+			err = disk.StartDiskCreateTask(ctx, self.UserCred, false, snapInfo, self.GetTaskId())
+			if err != nil {
+				self.SetStageFailed(ctx, err.Error())
+				return
+			}
+			diskReady = false
+			break
+		}
+		diskIndex += 1
+	}
+	diskIndex = 0
+	guest := obj.(*models.SGuest)
+
+	for {
+		diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex))
+		if !diskReady || err != nil {
+			break
+		}
+		iDisk, err := models.DiskManager.FetchById(diskId)
+		if err != nil {
+			self.SetStageFailed(ctx, err.Error())
+			return
+		}
+		if iDisk == nil {
+			self.SetStageFailed(ctx, "Disk not found")
+			return
+		}
+		disk := iDisk.(*models.SDisk)
+		if disk.Status != models.DISK_READY {
+			diskReady = false
+			break
+		}
+
+		ihost, err := guest.GetHost().GetIHost()
+		if err != nil {
+			self.SetStageFailed(ctx, "Host not found")
+			return
+		}
+
+		iVM, e := ihost.GetIVMById(guest.GetExternalId())
+		if e != nil {
+			self.SetStageFailed(ctx, "Aliyun VM not found")
+			return
+		}
+
+		err = iVM.AttachDisk(disk.GetExternalId())
+		if err != nil {
+			log.Debugf("Attach Disk %s to guest fail: %s", diskId, err)
+			self.SetStageFailed(ctx, "Attach Disk to guest fail")
+			return
+		}
+		diskIndex += 1
+	}
+
+	if diskReady {
+		if guest.Status == models.VM_RUNNING {
+			self.SetStage("on_config_sync_complete", nil)
+			err := guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId())
+			if err != nil {
+				self.SetStageFailed(ctx, err.Error())
+			}
+		} else {
+			self.SetStageComplete(ctx, nil)
+		}
+	}
+}
+
+func (self *AliyunGuestCreateDiskTask) OnConfigSyncComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	self.SetStageComplete(ctx, nil)
+}
+
+func (self *AliyunGuestCreateDiskTask) AttachAliyunDisks(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	self.SetStageComplete(ctx, nil)
+}
+
 func init() {
 	taskman.RegisterTask(GuestCreateDiskTask{})
 	taskman.RegisterTask(KVMGuestCreateDiskTask{})
+	taskman.RegisterTask(AliyunGuestCreateDiskTask{})
 }

pkg/compute/tasks/guest_delete_task.go

@@ -4,12 +4,14 @@ import (
 	"context"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/utils"
+
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/compute/options"
-	"yunion.io/x/pkg/utils"
 )
 
 type GuestDeleteTask struct {
@@ -28,15 +30,18 @@ func (self *GuestDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel
 
 func (self *GuestDeleteTask) OnGuestStopComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
 	guest := obj.(*models.SGuest)
-	guestStatus, _ := self.Params.GetString("guest_status")
 	if options.Options.EnablePendingDelete && !guest.PendingDeleted &&
 		!jsonutils.QueryBoolean(self.Params, "purge", false) &&
-		!jsonutils.QueryBoolean(self.Params, "override_pending_delete", false) &&
-		!utils.IsInStringArray(guestStatus, []string{models.VM_SCHEDULE_FAILED, models.VM_NETWORK_FAILED, models.VM_DISK_FAILED,
+		!jsonutils.QueryBoolean(self.Params, "override_pending_delete", false) {
+		log.Debugf("XXXXXXX Do guest pending delete... XXXXXXX")
+		guestStatus, _ := self.Params.GetString("guest_status")
+		if !utils.IsInStringArray(guestStatus, []string{models.VM_SCHEDULE_FAILED, models.VM_NETWORK_FAILED, models.VM_DISK_FAILED,
 			models.VM_CREATE_FAILED, models.VM_DEVICE_FAILED}) {
-		self.StartPendingDeleteGuest(ctx, guest)
-		return
+			self.StartPendingDeleteGuest(ctx, guest)
+			return
+		}
 	}
+	log.Debugf("XXXXXXX Do real delete on guest ... XXXXXXX")
 	self.OnGuestStopCompleteFailed(ctx, guest, data)
 }
 

pkg/compute/tasks/guest_deploy_task.go

@@ -54,14 +54,10 @@ func (self *GuestDeployTask) StartDeployGuestOnHost(ctx context.Context, guest *
 		log.Errorf("request_deploy_guest_on_host %s", err)
 		self.OnDeployGuestFail(ctx, guest, err)
 	} else {
-		self.OnDeployGuestSucc(guest)
+		guest.SetStatus(self.UserCred, models.VM_DEPLOYING, "")
 	}
 }
 
-func (self *GuestDeployTask) OnDeployGuestSucc(guest *models.SGuest) {
-	guest.SetStatus(self.UserCred, models.VM_DEPLOYING, "")
-}
-
 func (self *GuestDeployTask) OnDeployGuestFail(ctx context.Context, guest *models.SGuest, err error) {
 	guest.SetStatus(self.UserCred, models.VM_DEPLOY_FAILED, err.Error())
 	self.SetStageFailed(ctx, err.Error())

pkg/compute/tasks/guest_rebuild_root_task.go

@@ -32,6 +32,10 @@ func (self *GuestRebuildRootTask) OnInit(ctx context.Context, obj db.IStandalone
 	}
 }
 
+func (self *GuestRebuildRootTask) OnStopServerComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
+	self.StartRebuildRootDisk(ctx, guest)
+}
+
 func (self *GuestRebuildRootTask) StartRebuildRootDisk(ctx context.Context, guest *models.SGuest) {
 	db.OpsLog.LogEvent(guest, db.ACT_REBUILDING_ROOT, nil, self.UserCred)
 	gds := guest.CategorizeDisks()
@@ -137,4 +141,4 @@ func (self *KVMGuestRebuildRootTask) OnRebuildRootDiskCompleteFailed(ctx context
 
 func (self *KVMGuestRebuildRootTask) OnGuestDeployComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
 	self.SetStageComplete(ctx, nil)
-}
+}

pkg/compute/tasks/guest_reset_task.go

@@ -0,0 +1,58 @@
+package tasks
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/compute/models"
+)
+
+func init() {
+	taskman.RegisterTask(GuestSoftResetTask{})
+	taskman.RegisterTask(GuestHardResetTask{})
+}
+
+type GuestSoftResetTask struct {
+	SGuestBaseTask
+}
+
+func (self *GuestSoftResetTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	guest := obj.(*models.SGuest)
+	err := guest.GetDriver().RequestSoftReset(ctx, guest, self)
+	if err == nil {
+		self.SetStageComplete(ctx, nil)
+	} else {
+		self.SetStageFailed(ctx, err.Error())
+	}
+}
+
+type GuestHardResetTask struct {
+	SGuestBaseTask
+}
+
+func (self *GuestHardResetTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	guest := obj.(*models.SGuest)
+	self.StopServer(ctx, guest)
+}
+
+func (self *GuestHardResetTask) StopServer(ctx context.Context, guest *models.SGuest) {
+	guest.SetStatus(self.UserCred, models.VM_STOPPING, "")
+	self.SetStage("OnServerStopComplete", nil)
+	guest.StartGuestStopTask(ctx, self.UserCred, false, self.GetTaskId())
+}
+
+func (self *GuestHardResetTask) OnServerStopComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
+	self.StartServer(ctx, guest)
+}
+
+func (self *GuestHardResetTask) StartServer(ctx context.Context, guest *models.SGuest) {
+	self.SetStage("OnServerStartComplete", nil)
+	guest.StartGueststartTask(ctx, self.UserCred, nil, self.GetTaskId())
+}
+
+func (self *GuestHardResetTask) OnServerStartComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
+	self.SetStageComplete(ctx, nil)
+}

pkg/compute/tasks/guest_start_task.go

@@ -73,6 +73,7 @@ func (self *GuestStartTask) OnStartCompleteFailed(ctx context.Context, obj db.IS
 }
 
 func (self *GuestStartTask) onStartGuestFailed(ctx context.Context, guest *models.SGuest, err error) {
+	guest.SetStatus(self.UserCred, models.VM_START_FAILED, err.Error())
 	self.SetStageFailed(ctx, err.Error())
 	self.OnStartCompleteFailed(ctx, guest, jsonutils.NewString(err.Error()))
 }

pkg/dns/README.md

@@ -0,0 +1,56 @@
+# 行为约定
+
+A
+
+```sh
+names=''
+names="$names titan" #ok, guest PlainName
+names="$names titan.hq.cloud.yunionyun.com" #ok, guest CloudZoneFQDN
+names="$names kubenode" #ok, host PlainName
+names="$names kubenode.hq.cloud.yunionyun.com" #ok, host CloudZoneFQDN
+names="$names whoever-the-ether" #NXDOMAIN, nonexistent PlainName
+names="$names whoever-the-ether.titan.hq.cloud.yunionyun.com" #NXDOMAIN, nonexistent CloudZoneFQDN
+names="$names mail.google.com" #ok, dnsrecords
+names="$names www.douban.com" #ok, pub
+names="$names app" #ok, k8s svc PlainName in "default" ns
+names="$names app.default" #NXDOMAIN, k8s svc name.namespace
+names="$names app.default.hq.cloud.yunionyun.com" #ok, k8s name.ns CloudZoneFQDN
+names="$names mon-kafka.system" #NXDOMAIN, k8s svc name.namespace
+names="$names mon-kafka.system.hq.cloud.yunionyun.com" #ok, k8s name.ns CloudZoneFQDN
+
+# TODO source IP
+# TODO other TYPEs
+for name in $names; do
+	echo "############### $name"
+	#dig @192.168.222.171 $name
+	#dig -p 54 @10.168.222.136 $name
+	dig -p 54 @192.168.222.171 $name
+done
+```
+
+PTR
+
+```sh
+names=''
+names="$names " #ok, guest ip
+names="$names " #ok, host ip
+names="$names " #ok, ptr records in db
+names="$names " #NXDOMAIN, others
+
+for name in $names; do
+	echo "############### $name"
+	dig -p 54 @192.168.222.171 -x $name
+done
+```
+
+# 配置
+
+	log {
+		# note that apart from rcode like NXDOMAIN, SERVFAIL, coredns will also
+		# log NOERROR response when it's NoData as defined by coredns itself
+		#
+		# > NoData indicates name found, but not the type: NOERROR in header, SOA in auth.
+		#
+		class denial
+		class error
+	}

pkg/dns/dns.go

@@ -18,25 +18,27 @@ import (
 	"github.com/mholt/caddy"
 	"github.com/miekg/dns"
 	v1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/kubernetes"
 
 	ylog "yunion.io/x/log"
-	"yunion.io/x/pkg/utils"
-	"yunion.io/x/sqlchemy"
-
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/util/k8s"
+	"yunion.io/x/pkg/utils"
+	"yunion.io/x/sqlchemy"
 )
 
 const (
 	PluginName string = "yunion"
 
 	// defaultTTL to apply to all answers
-	defaultTTL = 10
+	defaultTTL           = 10
+	defaultDbMaxOpenConn = 32
+	defaultDbMaxIdleConn = 32
 )
 
 var (
@@ -74,15 +76,17 @@ func (r *SRegionDNS) initDB(c *caddy.Controller) error {
 	if err != nil {
 		return err
 	}
-	dbConn, err := sql.Open(dialect, sqlStr)
+	sqlDb, err := sql.Open(dialect, sqlStr)
 	if err != nil {
 		return err
 	}
-	sqlchemy.SetDB(dbConn)
+	sqlDb.SetMaxOpenConns(defaultDbMaxOpenConn)
+	sqlDb.SetMaxIdleConns(defaultDbMaxIdleConn)
+	sqlchemy.SetDB(sqlDb)
 	db.InitAllManagers()
 
 	c.OnShutdown(func() error {
-		r.CloseDB()
+		sqlchemy.CloseDB()
 		return nil
 	})
 	return nil
@@ -103,10 +107,6 @@ func (r *SRegionDNS) initK8s(c *caddy.Controller) {
 	ylog.Infof("Init k8s client success, %d pods in the cluster", len(pods.Items))
 }
 
-func (r *SRegionDNS) CloseDB() {
-	sqlchemy.CloseDB()
-}
-
 func (r *SRegionDNS) ServeDNS(ctx context.Context, w dns.ResponseWriter, rmsg *dns.Msg) (int, error) {
 	var (
 		records []dns.RR
@@ -116,36 +116,26 @@ func (r *SRegionDNS) ServeDNS(ctx context.Context, w dns.ResponseWriter, rmsg *d
 
 	opt := plugin.Options{}
 	state := request.Request{W: w, Req: rmsg, Context: ctx}
-
 	zone := plugin.Zones(r.Zones).Matches(state.Name())
-
 	switch state.QType() {
 	case dns.TypeA:
-		ylog.Debugf("A question: %#v", state)
 		records, err = plugin.A(r, zone, state, nil, opt)
 	case dns.TypeAAAA:
-		ylog.Debugf("AAAA question: %#v", state)
+		// TODO fallthrough to next
 		records, err = plugin.AAAA(r, zone, state, nil, opt)
 	case dns.TypeTXT:
-		ylog.Debugf("TXT question: %#v", state)
 		records, err = plugin.TXT(r, zone, state, opt)
 	case dns.TypeCNAME:
-		ylog.Debugf("CNAME question: %#v", state)
 		records, err = plugin.CNAME(r, zone, state, opt)
 	case dns.TypePTR:
-		ylog.Debugf("PTR question: %#v", state)
 		records, err = plugin.PTR(r, zone, state, opt)
 	case dns.TypeMX:
-		ylog.Debugf("MX question: %#v", state)
 		records, extra, err = plugin.MX(r, zone, state, opt)
 	case dns.TypeSRV:
-		ylog.Debugf("SRV question: %#v", state)
 		records, extra, err = plugin.SRV(r, zone, state, opt)
 	case dns.TypeSOA:
-		ylog.Debugf("SOA question: %#v", state)
 		records, err = plugin.SOA(r, zone, state, opt)
 	case dns.TypeNS:
-		ylog.Debugf("NS question: %#v", state)
 		if state.Name() == zone {
 			records, extra, err = plugin.NS(r, zone, state, opt)
 			break
@@ -157,18 +147,19 @@ func (r *SRegionDNS) ServeDNS(ctx context.Context, w dns.ResponseWriter, rmsg *d
 		_, err = plugin.A(r, zone, state, nil, opt)
 	}
 
-	if r.IsNameError(err) {
+	if err == errCallNext {
 		if r.Fall.Through(state.Name()) {
 			return plugin.NextOrFailure(r.Name(), r.Next, ctx, w, rmsg)
 		}
 		return plugin.BackendError(r, zone, dns.RcodeNameError, state, nil /* err */, opt)
-	}
-	if err != nil {
-		return plugin.BackendError(r, zone, dns.RcodeServerFailure, state, err, opt)
+	} else if err == errRefused {
+		return plugin.BackendError(r, zone, dns.RcodeRefused, state, err, opt)
+	} else if err == errNotFound {
+		return plugin.BackendError(r, zone, dns.RcodeNameError, state, err, opt)
 	}
 
 	if len(records) == 0 {
-		return plugin.BackendError(r, zone, dns.RcodeSuccess, state, err, opt)
+		return plugin.BackendError(r, zone, dns.RcodeNameError, state, err, opt)
 	}
 
 	m := new(dns.Msg)
@@ -184,13 +175,14 @@ func (r *SRegionDNS) ServeDNS(ctx context.Context, w dns.ResponseWriter, rmsg *d
 }
 
 var (
-	errNoItems = errors.New("no items found")
+	errRefused  = errors.New("refused the query")
+	errNotFound = errors.New("not found")
+	errCallNext = errors.New("continue to next")
 )
 
 // Services implements the ServiceBackend interface
 func (r *SRegionDNS) Services(state request.Request, exact bool, opt plugin.Options) (services []msg.Service, err error) {
 	switch state.QType() {
-
 	case dns.TypeTXT:
 		t, _ := dnsutil.TrimZone(state.Name(), state.Zone)
 
@@ -203,7 +195,6 @@ func (r *SRegionDNS) Services(state request.Request, exact bool, opt plugin.Opti
 		}
 		svc := msg.Service{Text: "0.0.1", TTL: 28800, Key: msg.Path(state.QName(), "coredns")}
 		return []msg.Service{svc}, nil
-
 	case dns.TypeNS:
 		ns := r.nsAddr()
 		svc := msg.Service{Host: ns.A.String(), Key: msg.Path(state.QName(), "coredns")}
@@ -219,7 +210,6 @@ func (r *SRegionDNS) Services(state request.Request, exact bool, opt plugin.Opti
 	}
 
 	services, err = r.Records(state, false)
-	ylog.Debugf("Get records: %#v, error: %v", services, err)
 	return
 }
 
@@ -230,7 +220,7 @@ func (r *SRegionDNS) Lookup(state request.Request, name string, typ uint16) (*dn
 
 // IsNameError implements the ServiceBackend interface
 func (r *SRegionDNS) IsNameError(err error) bool {
-	return err == errNoItems
+	return err == errCallNext
 }
 
 // Records looks up records in region mysql
@@ -242,25 +232,22 @@ func (r *SRegionDNS) Records(state request.Request, exact bool) ([]msg.Service,
 	return r.findRecords(req)
 }
 
-func (r *SRegionDNS) getHostIpWithName(req *recordRequest) []string {
+func (r *SRegionDNS) getHostIpWithName(req *recordRequest) string {
 	name := req.QueryName()
 	host, _ := models.HostManager.FetchByName("", name)
 	if host == nil {
-		return nil
+		return ""
 	}
 	ip := host.(*models.SHost).AccessIp
-	if len(ip) == 0 {
-		return nil
-	}
-	return []string{ip}
+	return ip
 }
 
 func (r *SRegionDNS) getGuestIpWithName(req *recordRequest) []string {
 	ips := []string{}
 	name := req.QueryName()
 	projectId := req.ProjectId()
-	isExitOnly := req.IsExitOnly()
-	ips = models.GuestManager.GetIpInProjectWithName(projectId, name, isExitOnly)
+	wantOnlyExit := false
+	ips = models.GuestManager.GetIpInProjectWithName(projectId, name, wantOnlyExit)
 	return ips
 }
 
@@ -268,6 +255,9 @@ func (r *SRegionDNS) getK8sServiceBackends(req *recordRequest) ([]string, error)
 	queryInfo := req.GetK8sQueryInfo()
 	pods, err := r.getK8sServicePods(queryInfo.Namespace, queryInfo.ServiceName)
 	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			err = nil
+		}
 		return nil, err
 	}
 	ips := make([]string, 0)
@@ -281,7 +271,11 @@ func (r *SRegionDNS) getK8sServiceBackends(req *recordRequest) ([]string, error)
 }
 
 func (r *SRegionDNS) getK8sServicePods(namespace, name string) ([]v1.Pod, error) {
-	cli := r.K8sClient
+	cli, err := k8s.NewClientByFile(r.K8sConfigFile, nil)
+	if err != nil {
+		ylog.Errorf("Init kubernetes client error: %v", err)
+		return nil, err
+	}
 	svc, err := cli.CoreV1().Services(namespace).Get(name, metav1.GetOptions{})
 	if err != nil {
 		return nil, err
@@ -301,10 +295,9 @@ func (r *SRegionDNS) Name() string {
 	return PluginName
 }
 
-func (r *SRegionDNS) queryLocalDnsRecords(req *recordRequest) (recs []msg.Service, err error) {
+func (r *SRegionDNS) queryLocalDnsRecords(req *recordRequest) (recs []msg.Service) {
 	ips := models.DnsRecordManager.QueryDnsIps(req.ProjectId(), req.Name(), req.Type())
 	if len(ips) == 0 {
-		err = errNoItems
 		return
 	}
 
@@ -317,12 +310,12 @@ func (r *SRegionDNS) queryLocalDnsRecords(req *recordRequest) (recs []msg.Servic
 		if req.IsSRV() {
 			parts := strings.SplitN(ip.Addr, ":", 2)
 			if len(parts) != 2 {
-				err = fmt.Errorf("Invalid SRV records: %q", ip.Addr)
+				ylog.Errorf("Invalid SRV records: %q", ip.Addr)
 				return
 			}
 			port, e := strconv.Atoi(parts[1])
 			if e != nil {
-				err = e
+				ylog.Errorf("Invalid SRV records: %q", ip.Addr)
 				return
 			}
 			s = msg.Service{Host: parts[0], Port: port, TTL: ttl}
@@ -334,17 +327,6 @@ func (r *SRegionDNS) queryLocalDnsRecords(req *recordRequest) (recs []msg.Servic
 	return
 }
 
-func (r *SRegionDNS) IsCloudNetworkIp(req *recordRequest) bool {
-	if req.network != nil {
-		return true
-	}
-	return false
-}
-
-func (r *SRegionDNS) IsK8sClientReady() bool {
-	return r.K8sClient != nil
-}
-
 func (r *SRegionDNS) isMyDomain(req *recordRequest) bool {
 	zones := []string{fmt.Sprintf("%s.", r.PrimaryZone)}
 	zone := plugin.Zones(zones).Matches(req.state.Name())
@@ -354,54 +336,65 @@ func (r *SRegionDNS) isMyDomain(req *recordRequest) bool {
 	return false
 }
 
-func (r *SRegionDNS) findRecords(req *recordRequest) (recs []msg.Service, err error) {
+func (r *SRegionDNS) findRecords(req *recordRequest) ([]msg.Service, error) {
 	// 1. try local dns records table
-	recs, err = r.queryLocalDnsRecords(req)
-	if len(recs) != 0 {
-		return
+	rrs := r.queryLocalDnsRecords(req)
+	if len(rrs) > 0 {
+		return rrs, nil
 	}
 
+	isPlainName := req.IsPlainName()
 	isMyDomain := r.isMyDomain(req)
-
-	isCloudIp := r.IsCloudNetworkIp(req)
-
-	// 2. not my domain and src ip not in cloud network table
-	//    query from upstream
-	if !isMyDomain && !isCloudIp {
-		err = errNoItems
-		return
+	if isPlainName {
+		isCloudIp := req.SrcInCloud()
+		if isCloudIp {
+			ips := r.findInternalRecordIps(req)
+			if len(ips) > 0 {
+				return ips2DnsRecords(ips), nil
+			} else {
+				return nil, errNotFound
+			}
+		} else {
+			return nil, errRefused
+		}
+	} else if isMyDomain {
+		ips := r.findInternalRecordIps(req)
+		if len(ips) > 0 {
+			return ips2DnsRecords(ips), nil
+		} else {
+			return nil, errNotFound
+		}
+	} else {
+		return nil, errCallNext
 	}
-
-	// 3. internal query
-	ips, err := r.findInternalRecordIps(req)
-	return ips2DnsRecords(ips), err
 }
 
-func (r *SRegionDNS) findInternalRecordIps(req *recordRequest) ([]string, error) {
-	// 1. try host table
-	ip := r.getHostIpWithName(req)
-	if len(ip) != 0 {
-		return ip, nil
+func (r *SRegionDNS) findInternalRecordIps(req *recordRequest) []string {
+	{
+		// 1. try host table
+		ip := r.getHostIpWithName(req)
+		if len(ip) > 0 {
+			return []string{ip}
+		}
 	}
-	// 2. try guest table
-	ip = r.getGuestIpWithName(req)
-	if len(ip) != 0 {
-		return ip, nil
+	{
+		// 2. try guest table
+		ips := r.getGuestIpWithName(req)
+		if len(ips) > 0 {
+			return ips
+		}
 	}
 
-	if !r.IsK8sClientReady() {
+	if r.K8sClient == nil {
 		ylog.Warningf("K8s client not ready, skip it.")
-		return nil, errNoItems
+		return nil
 	}
 	// 3. try k8s service backends
 	ips, err := r.getK8sServiceBackends(req)
-	if len(ips) != 0 {
-		return ips, nil
-	}
 	if err != nil {
 		ylog.Errorf("Get k8s service backends error: %v", err)
 	}
-	return nil, errNoItems
+	return ips
 }
 
 func ips2DnsRecords(ips []string) []msg.Service {

pkg/dns/parse.go

@@ -7,17 +7,16 @@ import (
 	"github.com/coredns/coredns/request"
 	"github.com/miekg/dns"
 
-	"yunion.io/x/pkg/tristate"
-
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/pkg/tristate"
 )
 
 type recordRequest struct {
-	state      request.Request
-	domainSegs []string
-	guest      *models.SGuest
-	host       *models.SHost
-	network    *models.SNetwork
+	state        request.Request
+	domainSegs   []string
+	srcProjectId string
+	srcInCloud   bool
+	network      *models.SNetwork
 }
 
 func parseRequest(state request.Request) (r *recordRequest, err error) {
@@ -28,9 +27,19 @@ func parseRequest(state request.Request) (r *recordRequest, err error) {
 		domainSegs: segs,
 	}
 	srcIP := r.SrcIP4()
-	r.guest = models.GuestnetworkManager.GetGuestByAddress(srcIP)
-	r.host = models.HostnetworkManager.GetHostByAddress(srcIP)
-	r.network, _ = models.NetworkManager.GetNetworkOfIP(srcIP, "", tristate.None)
+	// NOTE the check on networks_tbl is a hack, we should be more specific
+	// by querying only guest networks, host networks, and others like
+	// loadbalancer network the to come.
+	//
+	// Order matters here, we want to find the srcIP project as accurately
+	// as possible
+	if guest := models.GuestnetworkManager.GetGuestByAddress(srcIP); guest != nil {
+		r.srcProjectId = guest.ProjectId
+		r.srcInCloud = true
+	} else if network, _ := models.NetworkManager.GetNetworkOfIP(srcIP, "", tristate.None); network != nil {
+		r.srcProjectId = network.ProjectId
+		r.srcInCloud = true
+	}
 	return
 }
 
@@ -41,6 +50,11 @@ func (r recordRequest) Name() string {
 	return name
 }
 
+func (r recordRequest) IsPlainName() bool {
+	nl := dns.CountLabel(r.Name())
+	return nl == 1
+}
+
 func (r recordRequest) QueryName() string {
 	seps := strings.Split(r.Name(), ".")
 	if len(seps) == 0 {
@@ -63,20 +77,11 @@ func (r recordRequest) SrcIP4() string {
 }
 
 func (r recordRequest) ProjectId() string {
-	if r.guest != nil {
-		return r.guest.ProjectId
-	}
-	if r.network != nil {
-		return r.network.ProjectId
-	}
-	return ""
+	return r.srcProjectId
 }
 
-func (r recordRequest) IsExitOnly() bool {
-	if r.guest == nil {
-		return false
-	}
-	return r.guest.IsExitOnly()
+func (r recordRequest) SrcInCloud() bool {
+	return r.srcInCloud
 }
 
 type K8sQueryInfo struct {

pkg/dns/reverse.go

@@ -8,8 +8,6 @@ import (
 	"github.com/coredns/coredns/plugin/pkg/dnsutil"
 	"github.com/coredns/coredns/request"
 
-	"yunion.io/x/log"
-
 	"yunion.io/x/onecloud/pkg/compute/models"
 )
 
@@ -21,13 +19,7 @@ func (r *SRegionDNS) Reverse(state request.Request, exact bool, opt plugin.Optio
 		return nil, e
 	}
 	records, err := r.getNameForIp(ip, state)
-	if err != nil {
-		log.Errorf("Reverse get name for ip: %v", err)
-	}
-	if len(records) == 0 {
-		return records, errNoItems
-	}
-	return records, nil
+	return records, err
 }
 
 func (r *SRegionDNS) getNameForIp(ip string, state request.Request) ([]msg.Service, error) {
@@ -53,7 +45,7 @@ func (r *SRegionDNS) getNameForIp(ip string, state request.Request) ([]msg.Servi
 	if guest != nil {
 		return []msg.Service{{Host: r.joinDomain(guest.Name), TTL: defaultTTL}}, nil
 	}
-	return nil, errNoItems
+	return nil, errNotFound
 }
 
 func (r *SRegionDNS) joinDomain(name string) string {

pkg/dns/setup.go

@@ -2,7 +2,6 @@ package dns
 
 import (
 	"fmt"
-	"os"
 
 	"github.com/coredns/coredns/core/dnsserver"
 	"github.com/coredns/coredns/plugin"
@@ -20,18 +19,16 @@ func init() {
 }
 
 func setup(c *caddy.Controller) error {
-	os.Stderr = os.Stdout
-
 	rDNS, err := regionDNSParse(c)
 	if err != nil {
 		return plugin.Error(PluginName, err)
 	}
 
-	if rDNS.PrimaryZone == "" {
-		return fmt.Errorf("dns_domain must provided")
+	if len(rDNS.PrimaryZone) == 0 {
+		return fmt.Errorf("dns_domain missing")
 	}
 	if !regutils.MatchDomainName(rDNS.PrimaryZone) {
-		return fmt.Errorf("dns_domain %q not match domain format", rDNS.PrimaryZone)
+		return fmt.Errorf("dns_domain %q invalid", rDNS.PrimaryZone)
 	}
 
 	err = rDNS.initDB(c)

pkg/mcclient/auth/auth.go

@@ -103,7 +103,7 @@ func (c *TokenCacheVerify) Verify(cli *mcclient.Client, adminToken, token string
 	if err != nil {
 		return nil, fmt.Errorf("Add %s credential to cache: %#v", cred.GetTokenString(), err)
 	}
-	log.Infof("Add token: %s", cred)
+	// log.Debugf("Add token: %s", cred)
 	return cred, nil
 }
 

pkg/mcclient/catalog.go

@@ -0,0 +1,6 @@
+package mcclient
+
+type IServiceCatalog interface {
+	GetServiceURL(service, region, zone, endpointType string) (string, error)
+	GetServiceURLs(service, region, zone, endpointType string) ([]string, error)
+}

pkg/mcclient/mcclient.go

@@ -10,36 +10,16 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/onecloud/pkg/util/httputils"
+	"yunion.io/x/log"
 )
 
-/*const (
-	USER_AGENT = "yunioncloud-go/201708"
-)
-
-var (
-	red    = color.New(color.FgRed, color.Bold).PrintlnFunc()
-	green  = color.New(color.FgGreen, color.Bold).PrintlnFunc()
-	yellow = color.New(color.FgYellow, color.Bold).PrintlnFunc()
-	cyan   = color.New(color.FgHiCyan, color.Bold).PrintlnFunc()
-)*/
-
-/*type JSONClientError struct {
-	Code    int
-	Class   string
-	Details string
-}
-
-func (e *JSONClientError) Error() string {
-	return fmt.Sprintf("JSONClientError: %s %d %s", e.Details, e.Code, e.Class)
-}*/
-
 type Client struct {
 	authUrl string
 	timeout int
 	debug   bool
 
 	httpconn *http.Client
-	// serviceCatalog *KeystoneServiceCatalog
+	serviceCatalog IServiceCatalog
 }
 
 func NewClient(authUrl string, timeout int, debug bool, insecure bool) *Client {
@@ -222,7 +202,7 @@ func (this *Client) _authV3(domainName, uname, passwd, projectId, projectName, t
 	if len(tokenId) == 0 {
 		return nil, fmt.Errorf("No X-Subject-Token in header")
 	}
-	ret, err := UnmarshalV3Token(rbody, tokenId)
+	ret, err := this.unmarshalV3Token(rbody, tokenId)
 	return ret, err
 }
 
@@ -245,7 +225,7 @@ func (this *Client) _authV2(uname, passwd, tenantId, tenantName, token string) (
 	if err != nil {
 		return nil, err
 	}
-	return UnmarshalV2Token(rbody)
+	return this.unmarshalV2Token(rbody)
 }
 
 func (this *Client) Authenticate(uname, passwd, domainName, tenantName string) (TokenCredential, error) {
@@ -255,16 +235,21 @@ func (this *Client) Authenticate(uname, passwd, domainName, tenantName string) (
 	return this._authV2(uname, passwd, "", tenantName, "")
 }
 
-func UnmarshalV3Token(rbody jsonutils.JSONObject, tokenId string) (cred TokenCredential, err error) {
+func (this *Client) unmarshalV3Token(rbody jsonutils.JSONObject, tokenId string) (cred TokenCredential, err error) {
 	cred = &TokenCredentialV3{Id: tokenId}
 	err = rbody.Unmarshal(cred)
 	if err != nil {
 		err = fmt.Errorf("Invalid response when unmarshal V3 Token: %v", err)
 	}
+	cata := cred.GetServiceCatalog()
+	if cata == nil {
+		log.Fatalf("No srvice catalog avaiable")
+	}
+	this.serviceCatalog = cata
 	return
 }
 
-func UnmarshalV2Token(rbody jsonutils.JSONObject) (cred TokenCredential, err error) {
+func (this *Client) unmarshalV2Token(rbody jsonutils.JSONObject) (cred TokenCredential, err error) {
 	access, err := rbody.Get("access")
 	if err == nil {
 		cred = &TokenCredentialV2{}
@@ -272,6 +257,11 @@ func UnmarshalV2Token(rbody jsonutils.JSONObject) (cred TokenCredential, err err
 		if err != nil {
 			err = fmt.Errorf("Invalid response when unmarshal V2 Token: %s", err)
 		}
+		cata := cred.GetServiceCatalog()
+		if cata == nil {
+			log.Fatalf("No srvice catalog avaiable")
+		}
+		this.serviceCatalog = cata
 		return
 	}
 	err = fmt.Errorf("Invalid response: no access object")
@@ -286,7 +276,7 @@ func (this *Client) verifyV3(adminToken, token string) (TokenCredential, error)
 	if err != nil {
 		return nil, err
 	}
-	return UnmarshalV3Token(rbody, token)
+	return this.unmarshalV3Token(rbody, token)
 }
 
 func (this *Client) verifyV2(adminToken, token string) (TokenCredential, error) {
@@ -297,7 +287,7 @@ func (this *Client) verifyV2(adminToken, token string) (TokenCredential, error)
 	if err != nil {
 		return nil, err
 	}
-	return UnmarshalV2Token(rbody)
+	return this.unmarshalV2Token(rbody)
 }
 
 func (this *Client) Verify(adminToken, token string) (cred TokenCredential, err error) {
@@ -320,6 +310,13 @@ func (this *Client) SetProject(tenantId, tenantName string, token TokenCredentia
 }
 
 func (this *Client) NewSession(region, zone, endpointType string, token TokenCredential, apiVersion string) *ClientSession {
+	cata := token.GetServiceCatalog()
+	if this.serviceCatalog == nil {
+		if cata == nil {
+			log.Fatalf("Missing service catalog in token")
+		}
+		this.serviceCatalog = cata
+	}
 	return &ClientSession{client: this, region: region, zone: zone,
 		endpointType: endpointType, token: token,
 		apiVersion: apiVersion,

pkg/mcclient/modules/k8s/base.go

@@ -1,13 +1,56 @@
 package k8s
 
 import (
+	"yunion.io/x/pkg/util/sets"
+
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 )
 
-func NewManager(keyword, keywordPlural string, columns, adminColumns []string) *modules.ResourceManager {
+func NewManager(keyword, keywordPlural string, columns, adminColumns *Columns) *modules.ResourceManager {
 	return &modules.ResourceManager{
-		BaseManager:   *modules.NewBaseManager("k8s", "", "", columns, adminColumns),
+		BaseManager:   *modules.NewBaseManager("k8s", "", "", columns.Array(), adminColumns.Array()),
 		Keyword:       keyword,
 		KeywordPlural: keywordPlural,
 	}
 }
+
+type Columns struct {
+	cols []string
+}
+
+func NewColumns(cols ...string) *Columns {
+	c := &Columns{cols: make([]string, 0)}
+	return c.Add(cols...)
+}
+
+func (c *Columns) Add(cols ...string) *Columns {
+	for _, col := range cols {
+		c.add(col)
+	}
+	return c
+}
+
+func (c *Columns) add(col string) *Columns {
+	src := sets.NewString(c.Array()...)
+	if src.Has(col) {
+		return c
+	}
+	c.cols = append(c.cols, col)
+	return c
+}
+
+func (c Columns) Array() []string {
+	return c.cols
+}
+
+func NewNamespaceCols(col ...string) *Columns {
+	return NewColumns("name", "namespace").Add(col...)
+}
+
+func NewClusterCols(col ...string) *Columns {
+	return NewColumns("cluster").Add(col...)
+}
+
+func NewResourceCols(col ...string) *Columns {
+	return NewColumns("name", "id").Add(col...)
+}

pkg/mcclient/modules/k8s/cluster.go

@@ -9,8 +9,8 @@ var (
 )
 
 func init() {
-	Clusters = NewManager("cluster", "clusters",
-		[]string{"id", "name", "mode", "k8s_version", "status", "api_endpoint"},
-		[]string{})
+	Clusters = NewManager("kube_cluster", "kube_clusters",
+		NewResourceCols("mode", "k8s_version", "status", "api_endpoint"),
+		NewColumns())
 	modules.Register(Clusters)
 }

pkg/mcclient/modules/k8s/configmap.go

@@ -0,0 +1,20 @@
+package k8s
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+var ConfigMaps *ConfigMapManager
+
+type ConfigMapManager struct {
+	modules.ResourceManager
+}
+
+func init() {
+	ConfigMaps = &ConfigMapManager{
+		ResourceManager: *NewManager(
+			"configmap", "configmaps",
+			NewNamespaceCols(),
+			NewClusterCols())}
+	modules.Register(ConfigMaps)
+}

pkg/mcclient/modules/k8s/deployment.go

@@ -0,0 +1,20 @@
+package k8s
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+var Deployments *DeploymentManager
+
+type DeploymentManager struct {
+	modules.ResourceManager
+}
+
+func init() {
+	Deployments = &DeploymentManager{
+		ResourceManager: *NewManager(
+			"deployment", "deployments",
+			NewNamespaceCols("labels"),
+			NewClusterCols())}
+	modules.Register(Deployments)
+}

pkg/mcclient/modules/k8s/event.go

@@ -10,7 +10,7 @@ var (
 
 func init() {
 	Logs = NewManager("kube_event", "kube_events",
-		[]string{"id", "ops_time", "obj_id", "obj_type", "obj_name", "user", "user_id", "tenant", "tenant_id", "owner_tenant_id", "action", "notes"},
-		[]string{})
+		NewColumns("id", "ops_time", "obj_id", "obj_type", "obj_name", "user", "user_id", "tenant", "tenant_id", "owner_tenant_id", "action", "notes"),
+		NewColumns())
 	modules.Register(Logs)
 }

pkg/mcclient/modules/k8s/helm.go

@@ -0,0 +1,17 @@
+package k8s
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+var (
+	Repos *modules.ResourceManager
+)
+
+func init() {
+	Repos = NewManager("repo", "repos",
+		NewResourceCols("url", "is_public", "source"),
+		NewColumns(),
+	)
+	modules.Register(Repos)
+}

pkg/mcclient/modules/k8s/helm_chart.go

@@ -0,0 +1,16 @@
+package k8s
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+var (
+	Charts *modules.ResourceManager
+)
+
+func init() {
+	Charts = NewManager("chart", "charts",
+		NewResourceCols(),
+		NewColumns())
+	modules.Register(Charts)
+}

pkg/mcclient/modules/k8s/helm_release.go

@@ -0,0 +1,23 @@
+package k8s
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+var (
+	Releases *ReleaseManager
+)
+
+type ReleaseManager struct {
+	modules.ResourceManager
+}
+
+func init() {
+	Releases = &ReleaseManager{
+		ResourceManager: *NewManager(
+			"release", "releases",
+			NewNamespaceCols(""),
+			NewColumns(),
+		)}
+	modules.Register(Releases)
+}

pkg/mcclient/modules/k8s/helm_tiller.go

@@ -0,0 +1,23 @@
+package k8s
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+var (
+	Tiller *TillerManager
+)
+
+type TillerManager struct {
+	modules.ResourceManager
+}
+
+func init() {
+	Tiller = &TillerManager{
+		ResourceManager: *NewManager(
+			"tiller", "tiller",
+			NewColumns(),
+			NewColumns(),
+		)}
+	modules.Register(Tiller)
+}

pkg/mcclient/modules/k8s/node.go

@@ -9,6 +9,6 @@ var (
 )
 
 func init() {
-	Nodes = NewManager("node", "nodes", []string{"id", "name", "cluster", "roles", "address", "status"}, []string{})
+	Nodes = NewManager("kube_node", "kube_nodes", NewResourceCols("cluster", "roles", "address", "status"), NewColumns())
 	modules.Register(Nodes)
 }

pkg/mcclient/modules/k8s/pod.go

@@ -14,7 +14,7 @@ func init() {
 	Pods = &PodManager{
 		ResourceManager: *NewManager(
 			"pod", "pods",
-			[]string{"id", "name", "cluster", "roles", "address", "status"},
-			[]string{})}
+			NewNamespaceCols("podIP", "status", "restartCount", "labels"),
+			NewClusterCols("nodeName"))}
 	modules.Register(Pods)
 }

pkg/mcclient/modules/k8s/raw.go

@@ -0,0 +1,95 @@
+package k8s
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+var RawResource *RawResourceManager
+
+func init() {
+	RawResource = &RawResourceManager{
+		serviceType: "k8s",
+	}
+}
+
+type rawResourceContext struct {
+	kind  string
+	name  string
+	query jsonutils.JSONObject
+	ctxs  []modules.ManagerContext
+}
+
+func newRawResourceContext(kind, namespace, name string, query jsonutils.JSONObject, ctxs []modules.ManagerContext) *rawResourceContext {
+	nsQuery := getNamespaceQuery(namespace)
+	if query != nil {
+		nsQuery.Update(query)
+	}
+	ctx := &rawResourceContext{kind: kind, name: name, query: nsQuery, ctxs: ctxs}
+	return ctx
+}
+
+func (ctx rawResourceContext) contextPath() string {
+	segs := make([]string, 0)
+	ctxs := ctx.ctxs
+	if ctxs != nil && len(ctxs) > 0 {
+		for _, c := range ctxs {
+			segs = append(segs, c.InstanceManager.KeyString())
+			if len(c.InstanceId) > 0 {
+				segs = append(segs, url.PathEscape(c.InstanceId))
+			}
+		}
+	}
+	segs = append(segs, "_raw", ctx.kind, ctx.name)
+	path := fmt.Sprintf("/%s", strings.Join(segs, "/"))
+	if ctx.query != nil {
+		qs := ctx.query.QueryString()
+		if len(qs) > 0 {
+			path = fmt.Sprintf("%s?%s", path, qs)
+		}
+	}
+	return path
+}
+
+type RawResourceManager struct {
+	serviceType string
+}
+
+func (m *RawResourceManager) request(s *mcclient.ClientSession, method string, path string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	_, ret, err := s.JSONRequest(m.serviceType, "", method, path, nil, body)
+	return ret, err
+}
+
+func getNamespaceQuery(namespace string) *jsonutils.JSONDict {
+	query := jsonutils.NewDict()
+	if namespace != "" {
+		query.Set("namespace", jsonutils.NewString(namespace))
+	}
+	return query
+}
+
+func (m *RawResourceManager) Get(s *mcclient.ClientSession, kind string, namespace string, name string, query jsonutils.JSONObject, ctxs []modules.ManagerContext) (jsonutils.JSONObject, error) {
+	ctx := newRawResourceContext(kind, namespace, name, query, ctxs)
+	return m.request(s, "GET", ctx.contextPath(), nil)
+}
+
+func (m *RawResourceManager) Put(s *mcclient.ClientSession, kind string, namespace string, name string, body jsonutils.JSONObject, ctxs []modules.ManagerContext) error {
+	rawBytes := body.String()
+	newBody := jsonutils.NewDict()
+	newBody.Add(jsonutils.NewString(rawBytes), "raw")
+	ctx := newRawResourceContext(kind, namespace, name, nil, ctxs)
+	_, err := m.request(s, "PUT", ctx.contextPath(), newBody)
+	return err
+}
+
+func (m *RawResourceManager) Delete(s *mcclient.ClientSession, kind string, namespace string, name string, query jsonutils.JSONObject, ctxs []modules.ManagerContext) error {
+	ctx := newRawResourceContext(kind, namespace, name, query, ctxs)
+	_, err := m.request(s, "DELETE", ctx.contextPath(), nil)
+	return err
+}

pkg/mcclient/modules/k8s/service.go

@@ -0,0 +1,20 @@
+package k8s
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+var Services *ServiceManager
+
+type ServiceManager struct {
+	modules.ResourceManager
+}
+
+func init() {
+	Services = &ServiceManager{
+		ResourceManager: *NewManager(
+			"k8s_service", "k8s_services",
+			NewNamespaceCols("clusterIP", "selector", "internalEndpoint", "externalEndpoints"),
+			NewClusterCols())}
+	modules.Register(Services)
+}

pkg/mcclient/modules/mod_domains.go

@@ -240,12 +240,7 @@ func (this *DomainManager) DoDomainConfigDelete(s *mcclient.ClientSession, param
 			return ret, httperrors.NewResourceNotFoundError("找不到该认证域")
 		}
 
-		driver, err := detail.GetString("driver")
-		if err != nil {
-			log.Errorf("got driver from domain detail error: %v", err)
-			return ret, httperrors.NewInternalServerError("服务器错误,获取认证协议失败,不允许删除")
-		}
-
+		driver, _ := detail.GetString("driver")
 		if driver != "ldap" {
 			if result, err := UsersV3.List(s, params); err != nil {
 				log.Errorf("user list got error: %v", err)

pkg/mcclient/modules/modules.go

@@ -2,10 +2,10 @@ package modules
 
 import (
 	"fmt"
-	"log"
 	"strings"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
@@ -142,6 +142,17 @@ func _getJointKey(mod1 Manager, mod2 Manager) string {
 	return fmt.Sprintf("%s-%s", mod1.KeyString(), mod2.KeyString())
 }
 
+func ensureModuleNotRegistered(mod, newMod BaseManagerInterface) {
+	modSvcType := mod.ServiceType()
+	newModSvcType := newMod.ServiceType()
+	if mod == newMod {
+		log.Fatalf("Module %#v duplicate registered, service type: %q", mod, modSvcType)
+	}
+	if modSvcType != newModSvcType {
+		log.Fatalf("Module %#v already registered, service type is %q.\nSo new module %#v can't be registered, service type is %q", mod, modSvcType, newMod, newModSvcType)
+	}
+}
+
 func _register(version string, mod BaseManagerInterface) {
 	if modules == nil {
 		modules = make(map[string]map[string][]BaseManagerInterface)
@@ -156,9 +167,7 @@ func _register(version string, mod BaseManagerInterface) {
 		mods = make([]BaseManagerInterface, 0)
 	}
 	for _, m := range mods {
-		if m == mod {
-			log.Fatal("Module already registered", mod)
-		}
+		ensureModuleNotRegistered(m, mod)
 	}
 	modtable[mod.KeyString()] = append(mods, mod)
 }
@@ -179,7 +188,7 @@ func _registerJointModule(version string, mod BaseManagerInterface) {
 		}
 		for _, m := range jointMods {
 			if m == jointMod {
-				log.Fatal("Module already registered", jointMod)
+				ensureModuleNotRegistered(m, jointMod)
 			}
 		}
 		modtable[jointKey] = append(jointMods, jointMod)

pkg/mcclient/session.go

@@ -75,7 +75,26 @@ func (this *ClientSession) GetServiceURL(service, endpointType string) (string,
 	if len(this.apiVersion) > 0 && this.apiVersion != DEFAULT_API_VERSION {
 		service = fmt.Sprintf("%s_%s", service, this.apiVersion)
 	}
-	return this.token.GetServiceURL(service, this.region, this.zone, endpointType)
+	url, err := this.token.GetServiceURL(service, this.region, this.zone, endpointType)
+	if err != nil {
+		url, err = this.client.serviceCatalog.GetServiceURL(service, this.region, this.zone, endpointType)
+	}
+	return url, err
+}
+
+func (this *ClientSession) GetServiceURLs(service, endpointType string) ([]string, error) {
+	if len(this.endpointType) > 0 {
+		// session specific endpoint type should override the input endpointType, which is supplied by manager
+		endpointType = this.endpointType
+	}
+	if len(this.apiVersion) > 0 && this.apiVersion != DEFAULT_API_VERSION {
+		service = fmt.Sprintf("%s_%s", service, this.apiVersion)
+	}
+	urls, err := this.token.GetServiceURLs(service, this.region, this.zone, endpointType)
+	if err != nil {
+		urls, err = this.client.serviceCatalog.GetServiceURLs(service, this.region, this.zone, endpointType)
+	}
+	return urls, err
 }
 
 func (this *ClientSession) getBaseUrl(service, endpointType string) (string, error) {

pkg/mcclient/token.go

@@ -11,9 +11,20 @@ type ExternalService struct {
 	Url  string
 }
 
+type Endpoint struct {
+	Id          string
+	RegionId    string
+	ServiceId   string
+	ServiceName string
+	Url         string
+	Interface   string
+}
+
 type TokenCredential interface {
 	gotypes.ISerializable
 
+	IServiceCatalog
+
 	GetTokenString() string
 	GetDomainId() string
 	GetDomainName() string
@@ -30,7 +41,10 @@ type TokenCredential interface {
 	IsAdmin() bool
 	IsSystemAdmin() bool
 	GetRegions() []string
-	GetServiceURL(service, region, zone, endpointType string) (string, error)
+
+	GetServiceCatalog() IServiceCatalog
+
 	GetInternalServices(region string) []string
 	GetExternalServices(region string) []ExternalService
+	GetEndpoints(region string, endpointType string) []Endpoint
 }

pkg/mcclient/token2.go

@@ -133,7 +133,11 @@ func (this *TokenCredentialV2) IsSystemAdmin() bool {
 }
 
 func (this *TokenCredentialV2) GetServiceURL(service, region, zone, endpointType string) (string, error) {
-	return this.ServiceCatalog.getServiceURL(service, region, zone, endpointType)
+	return this.ServiceCatalog.GetServiceURL(service, region, zone, endpointType)
+}
+
+func (this *TokenCredentialV2) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) {
+	return this.ServiceCatalog.GetServiceURLs(service, region, zone, endpointType)
 }
 
 func (this *TokenCredentialV2) GetInternalServices(region string) []string {
@@ -144,6 +148,14 @@ func (this *TokenCredentialV2) GetExternalServices(region string) []ExternalServ
 	return nil
 }
 
+func (this *TokenCredentialV2) GetEndpoints(region string, endpointType string) []Endpoint {
+	return nil
+}
+
+func (this *TokenCredentialV2) GetServiceCatalog() IServiceCatalog {
+	return this.ServiceCatalog
+}
+
 func stringArrayContains(arr []string, needle string) bool {
 	for i := 0; i < len(arr); i++ {
 		if arr[i] == needle {
@@ -207,7 +219,7 @@ func (catalog KeystoneServiceCatalogV2) getServiceEndpoint(service, region, zone
 	return selected, fmt.Errorf("No such service %s", service)
 }
 
-func (catalog KeystoneServiceCatalogV2) getServiceURL(service, region, zone, endpointType string) (string, error) {
+func (catalog KeystoneServiceCatalogV2) GetServiceURL(service, region, zone, endpointType string) (string, error) {
 	ep, err := catalog.getServiceEndpoint(service, region, zone)
 	if err != nil {
 		return "", err
@@ -215,6 +227,14 @@ func (catalog KeystoneServiceCatalogV2) getServiceURL(service, region, zone, end
 	return ep.getURL(endpointType), nil
 }
 
+func (catalog KeystoneServiceCatalogV2) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) {
+	url, err := catalog.GetServiceURL(service, region, zone, endpointType)
+	if err != nil {
+		return nil, err
+	}
+	return []string{url}, nil
+}
+
 func (ep KeystoneEndpointV2) getURL(epType string) string {
 	switch epType {
 	case "publicURL":

pkg/mcclient/token3.go

@@ -141,7 +141,11 @@ func (this *TokenCredentialV3) GetRegions() []string {
 }
 
 func (this *TokenCredentialV3) GetServiceURL(service, region, zone, endpointType string) (string, error) {
-	return this.Token.Catalog.getServiceURL(service, region, zone, endpointType)
+	return this.Token.Catalog.GetServiceURL(service, region, zone, endpointType)
+}
+
+func (this *TokenCredentialV3) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) {
+	return this.Token.Catalog.GetServiceURLs(service, region, zone, endpointType)
 }
 
 func (this *TokenCredentialV3) GetInternalServices(region string) []string {
@@ -152,6 +156,14 @@ func (this *TokenCredentialV3) GetExternalServices(region string) []ExternalServ
 	return this.Token.Catalog.getExternalServices(region)
 }
 
+func (this *TokenCredentialV3) GetEndpoints(region string, endpointType string) []Endpoint {
+	return this.Token.Catalog.getEndpoints(region, endpointType)
+}
+
+func (this *TokenCredentialV3) GetServiceCatalog() IServiceCatalog {
+	return this.Token.Catalog
+}
+
 func (catalog KeystoneServiceCatalogV3) getInternalServices(region string) []string {
 	services := make([]string, 0)
 	for i := 0; i < len(catalog); i++ {
@@ -200,6 +212,27 @@ func (catalog KeystoneServiceCatalogV3) getRegions() []string {
 	return regions
 }
 
+func (catalog KeystoneServiceCatalogV3) getEndpoints(region string, endpointType string) []Endpoint {
+	endpoints := make([]Endpoint, 0)
+	for i := 0; i < len(catalog); i++ {
+		for j := 0; j < len(catalog[i].Endpoints); j++ {
+			endpoint := catalog[i].Endpoints[j]
+			if (endpoint.Region_id == region || strings.HasPrefix(endpoint.Region_id, region+"-")) && endpoint.Interface == endpointType {
+				endpoints = append(endpoints, Endpoint{
+					endpoint.Id,
+					endpoint.Region_id,
+					catalog[i].Id,
+					catalog[i].Name,
+					endpoint.Url,
+					endpoint.Interface,
+				})
+			}
+		}
+	}
+
+	return endpoints
+}
+
 func RegionID(region, zone string) string {
 	if len(zone) > 0 {
 		return fmt.Sprintf("%s%c%s", region, REGION_ZONE_SEP, zone)
@@ -217,7 +250,15 @@ func Id2RegionZone(id string) (string, string) {
 	}
 }
 
-func (catalog KeystoneServiceCatalogV3) getServiceURL(service, region, zone, endpointType string) (string, error) {
+func (catalog KeystoneServiceCatalogV3) GetServiceURL(service, region, zone, endpointType string) (string, error) {
+	urls, err := catalog.GetServiceURLs(service, region, zone, endpointType)
+	if err != nil {
+		return "", err
+	}
+	return urls[rand.Intn(len(urls))], nil
+}
+
+func (catalog KeystoneServiceCatalogV3) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) {
 	if endpointType == "" {
 		endpointType = "internalURL"
 	}
@@ -248,7 +289,7 @@ func (catalog KeystoneServiceCatalogV3) getServiceURL(service, region, zone, end
 						break
 					}
 				} else {
-					return "", fmt.Errorf("No default region")
+					return nil, fmt.Errorf("No default region")
 				}
 			} else {
 				_, ok := regeps[regionzone]
@@ -257,14 +298,14 @@ func (catalog KeystoneServiceCatalogV3) getServiceURL(service, region, zone, end
 				} else {
 					selected, ok = regeps[region]
 					if !ok {
-						return "", fmt.Errorf("No valid %s endpoints for %s in region %s", endpointType, service, RegionID(region, zone))
+						return nil, fmt.Errorf("No valid %s endpoints for %s in region %s", endpointType, service, RegionID(region, zone))
 					}
 				}
 			}
-			return selected[rand.Intn(len(selected))], nil
+			return selected, nil
 		}
 	}
-	return "", fmt.Errorf("No such service %s", service)
+	return nil, fmt.Errorf("No such service %s", service)
 }
 
 func (self *TokenCredentialV3) String() string {

pkg/mcclient/tokensimple.go

@@ -96,6 +96,10 @@ func (self *SSimpleToken) GetServiceURL(service, region, zone, endpointType stri
 	return "", fmt.Errorf("Not available")
 }
 
+func (self *SSimpleToken) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) {
+	return nil, fmt.Errorf("Not available")
+}
+
 func (self *SSimpleToken) GetInternalServices(region string) []string {
 	return nil
 }
@@ -104,6 +108,14 @@ func (self *SSimpleToken) GetExternalServices(region string) []ExternalService {
 	return nil
 }
 
+func (this *SSimpleToken) GetEndpoints(region string, endpointType string) []Endpoint {
+	return nil
+}
+
+func (this *SSimpleToken) GetServiceCatalog() IServiceCatalog {
+	return nil
+}
+
 func SimplifyToken(token TokenCredential) TokenCredential {
 	simToken, ok := token.(*SSimpleToken)
 	if ok {

pkg/scheduler/algorithm/predicates/guest/network_predicate.go

@@ -9,6 +9,7 @@ import (
 	"yunion.io/x/onecloud/pkg/scheduler/api"
 	"yunion.io/x/onecloud/pkg/scheduler/core"
 	networks "yunion.io/x/onecloud/pkg/scheduler/db/models"
+	"yunion.io/x/pkg/util/sets"
 	"yunion.io/x/pkg/utils"
 )
 
@@ -51,9 +52,9 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor
 		return len(d.HostID) > 0
 	}
 
-	// ServerType's value is 'guest' or ''(support all type) will return true.
+	// ServerType's value is 'guest', 'container' or ''(support all type) will return true.
 	isMatchServerType := func(network *networks.NetworkSchedResult) bool {
-		return network.ServerType == "guest" || network.ServerType == ""
+		return sets.NewString("guest", "", "container").Has(network.ServerType)
 	}
 
 	counterOfNetwork := func(u *core.Unit, n *networks.NetworkSchedResult, r int) core.Counter {

pkg/scheduler/api/sched.go

@@ -551,7 +551,15 @@ func newBaremetalDiskConfigFromSimpleJson(sjson *simplejson.Json) (*BaremetalDis
 	rangeArray := make([]int64, 0)
 	ranges := sjson.Get("range").MustArray()
 	for _, size := range ranges {
-		rangeArray = append(rangeArray, size.(int64))
+		sizeNum, ok := size.(json.Number)
+		if !ok {
+			return nil, fmt.Errorf("Invalid range size, not json.Number: %v", size)
+		}
+		rangeSize, err := sizeNum.Int64()
+		if err != nil {
+			return nil, err
+		}
+		rangeArray = append(rangeArray, rangeSize)
 	}
 	baremetalDiskConfig.Range = rangeArray
 

pkg/util/aliyun/aliyun.go

@@ -3,6 +3,7 @@ package aliyun
 import (
 	"github.com/aliyun/alibaba-cloud-sdk-go/sdk"
 	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
+
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/cloudprovider"

pkg/util/aliyun/disk.go

@@ -54,6 +54,16 @@ type SDisk struct {
 	ZoneId                        string
 }
 
+func (self *SDisk) GetMetadata() *jsonutils.JSONDict {
+	data := jsonutils.NewDict()
+
+	// The pricingInfo key structure is 'RegionId::DiskCategory::DiskType
+	priceKey := fmt.Sprintf("%s::%s::%s", self.RegionId, self.Category, self.Type)
+	data.Add(jsonutils.NewString(priceKey), "price_key")
+
+	return data
+}
+
 func (self *SRegion) GetDisks(instanceId string, zoneId string, category string, diskIds []string, offset int, limit int) ([]SDisk, int, error) {
 	if limit > 50 || limit <= 0 {
 		limit = 50
@@ -138,6 +148,12 @@ func (self *SDisk) Refresh() error {
 	return jsonutils.Update(self, new)
 }
 
+func (self *SDisk) ResizeDisk(newSize int64) error {
+	// newSize 单位为 GB. 范围在20 ～2000. 只能往大调。不能调小
+	// https://help.aliyun.com/document_detail/25522.html?spm=a2c4g.11174283.6.897.aHwqkS
+	return self.storage.zone.region.resizeDisk(self.DiskId, newSize)
+}
+
 func (self *SDisk) GetDiskFormat() string {
 	return "vhd"
 }
@@ -185,7 +201,7 @@ func (self *SDisk) GetMountpoint() string {
 	return ""
 }
 
-func (self *SRegion) createDisk(zoneId string, category string, name string, sizeGb int, desc string) (string, error) {
+func (self *SRegion) CreateDisk(zoneId string, category string, name string, sizeGb int, desc string) (string, error) {
 	params := make(map[string]string)
 	params["ZoneId"] = zoneId
 	params["DiskName"] = name
@@ -223,11 +239,24 @@ func (self *SRegion) deleteDisk(diskId string) error {
 	return err
 }
 
+func (self *SRegion) DeleteDisk(diskId string) error {
+	params := make(map[string]string)
+	params["DiskId"] = diskId
+
+	_, err := self.ecsRequest("DeleteDisk", params)
+	return err
+}
+
 func (self *SRegion) resizeDisk(diskId string, size int64) error {
 	params := make(map[string]string)
 	params["DiskId"] = diskId
 	params["NewSize"] = fmt.Sprintf("%d", size)
 
 	_, err := self.ecsRequest("ResizeDisk", params)
-	return err
+	if err != nil {
+		log.Errorf("ResizeDisk %s to %s GiB fail %s", diskId, size, err)
+		return err
+	}
+
+	return nil
 }

pkg/util/aliyun/errors.go

@@ -0,0 +1,13 @@
+package aliyun
+
+import (
+	aliyunerrors "github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors"
+)
+
+func isError(err error, code string) bool {
+	aliyunErr, ok := err.(aliyunerrors.Error)
+	if ! ok {
+		return false
+	}
+	return aliyunErr.ErrorCode() == code
+}

pkg/util/aliyun/host.go

@@ -19,6 +19,10 @@ type SHost struct {
 	zone *SZone
 }
 
+func (self *SHost) GetMetadata() *jsonutils.JSONDict {
+	return nil
+}
+
 func (self *SHost) GetIWires() ([]cloudprovider.ICloudWire, error) {
 	return self.zone.GetIWires()
 }
@@ -157,7 +161,7 @@ func (self *SHost) GetManagerId() string {
 	return self.zone.region.client.providerId
 }
 
-func (self *SHost) getInstanceById(instanceId string) (*SInstance, error) {
+func (self *SHost) GetInstanceById(instanceId string) (*SInstance, error) {
 	inst, err := self.zone.region.GetInstance(instanceId)
 	if err != nil {
 		return nil, err
@@ -171,7 +175,7 @@ func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int,
 	if err != nil {
 		return nil, err
 	}
-	vm, err := self.getInstanceById(vmId)
+	vm, err := self.GetInstanceById(vmId)
 	if err != nil {
 		return nil, err
 	}

pkg/util/aliyun/image.go

@@ -55,6 +55,10 @@ type SImage struct {
 	Usage                string
 }
 
+func (self *SImage) GetMetadata() *jsonutils.JSONDict {
+	return nil
+}
+
 func (self *SImage) GetId() string {
 	return self.ImageId
 }

pkg/util/aliyun/instance.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"time"
 
+	"yunion.io/x/onecloud/pkg/util/seclib2"
+
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/util/osprofile"
@@ -110,6 +112,7 @@ type SInstance struct {
 	InternetMaxBandwidthIn  int
 	InternetMaxBandwidthOut int
 	IoOptimized             bool
+	KeyPairName             string
 	Memory                  int
 	NetworkInterfaces       SNetworkInterfaces
 	OSName                  string
@@ -162,6 +165,20 @@ func (self *SRegion) GetInstances(zoneId string, ids []string, offset int, limit
 	return instances, int(total), nil
 }
 
+func (self *SInstance) GetMetadata() *jsonutils.JSONDict {
+	data := jsonutils.NewDict()
+
+	// The pricingInfo key structure is 'RegionId::InstanceType::NetworkType::OSType::IoOptimized'
+	optimized := "optimized"
+	if !self.IoOptimized {
+		optimized = "none"
+	}
+	priceKey := fmt.Sprintf("%s::%s::%s::%s::%s", self.RegionId, self.InstanceType, self.InstanceNetworkType, self.OSType, optimized)
+	data.Add(jsonutils.NewString(priceKey), "price_key")
+
+	return data
+}
+
 func (self *SInstance) GetCreateTime() time.Time {
 	return self.CreationTime
 }
@@ -322,6 +339,77 @@ func (self *SInstance) GetHypervisor() string {
 	return models.HYPERVISOR_ALIYUN
 }
 
+func (self *SInstance) StartVM() error {
+	err := self.host.zone.region.StartVM(self.InstanceId)
+	if err != nil {
+		return err
+	}
+	return cloudprovider.WaitStatus(self, models.VM_RUNNING, 5*time.Second, 180*time.Second) // 3minutes
+}
+
+func (self *SInstance) StopVM(isForce bool) error {
+	err := self.host.zone.region.StopVM(self.InstanceId, isForce)
+	if err != nil {
+		return err
+	}
+	return cloudprovider.WaitStatus(self, models.VM_READY, 10*time.Second, 300*time.Second) // 5mintues
+}
+
+func (self *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) {
+	url, err := self.host.zone.region.GetInstanceVNCUrl(self.InstanceId)
+	if err != nil {
+		return nil, err
+	}
+	passwd := seclib.RandomPassword(6)
+	err = self.host.zone.region.ModifyInstanceVNCUrlPassword(self.InstanceId, passwd)
+	if err != nil {
+		return nil, err
+	}
+	ret := jsonutils.NewDict()
+	ret.Add(jsonutils.NewString(url), "url")
+	ret.Add(jsonutils.NewString(passwd), "password")
+	ret.Add(jsonutils.NewString("aliyun"), "protocol")
+	ret.Add(jsonutils.NewString(self.InstanceId), "instance_id")
+	return ret, nil
+}
+
+func (self *SInstance) UpdateVM(name string) error {
+	return self.host.zone.region.UpdateVM(self.InstanceId, name)
+}
+
+func (self *SInstance) DeployVM(name string, password string, publicKey string, resetPassword bool, deleteKeypair bool, description string) error {
+	var keypairName string
+	if len(publicKey) > 0 {
+		key, e := self.host.lookUpAliyunKeypair(publicKey)
+		if e != nil {
+			key, e = self.host.importAliyunKeypair(publicKey)
+			if e != nil {
+				return e
+			}
+		}
+
+		keypairName = key
+	}
+
+	return self.host.zone.region.DeployVM(self.InstanceId, name, password, keypairName, resetPassword, deleteKeypair, description)
+}
+
+func (self *SInstance) RebuildRoot(imageId string) error {
+	return self.host.zone.region.ReplaceSystemDisk(self.InstanceId, imageId)
+}
+
+func (self *SInstance) ChangeConfig(instanceId string, ncpu int, vmem int) error {
+	return self.host.zone.region.ChangeVMConfig(self.ZoneId, self.InstanceId, ncpu, vmem, nil)
+}
+
+func (self *SInstance) AttachDisk(diskId string) error {
+	return self.host.zone.region.AttachDisk(self.InstanceId, diskId)
+}
+
+func (self *SInstance) DetachDisk(diskId string) error {
+	return self.host.zone.region.DetachDisk(self.InstanceId, diskId)
+}
+
 func (self *SRegion) GetInstance(instanceId string) (*SInstance, error) {
 	instances, _, err := self.GetInstances("", []string{instanceId}, 0, 1)
 	if err != nil {
@@ -402,7 +490,10 @@ func (self *SRegion) doStopVM(instanceId string, isForce bool) error {
 }
 
 func (self *SRegion) doDeleteVM(instanceId string) error {
-	return self.instanceOperation(instanceId, "DeleteInstance", nil)
+	params := make(map[string]string)
+	params["TerminateSubscription"] = "false"
+	params["Force"] = "true"
+	return self.instanceOperation(instanceId, "DeleteInstance", params)
 }
 
 /*func (self *SRegion) waitInstanceStatus(instanceId string, target string, interval time.Duration, timeout time.Duration) error {
@@ -425,8 +516,13 @@ func (self *SInstance) waitStatus(target string, interval time.Duration, timeout
 }*/
 
 func (self *SRegion) StartVM(instanceId string) error {
-	status, _ := self.GetInstanceStatus(instanceId)
+	status, err := self.GetInstanceStatus(instanceId)
+	if err != nil {
+		log.Errorf("Fail to get instance status on StartVM: %s", err)
+		return err
+	}
 	if status != InstanceStatusStopped {
+		log.Errorf("StartVM: vm status is %s expect %s", status, InstanceStatusStopped)
 		return cloudprovider.ErrInvalidStatus
 	}
 	return self.doStartVM(instanceId)
@@ -437,8 +533,13 @@ func (self *SRegion) StartVM(instanceId string) error {
 }
 
 func (self *SRegion) StopVM(instanceId string, isForce bool) error {
-	status, _ := self.GetInstanceStatus(instanceId)
+	status, err := self.GetInstanceStatus(instanceId)
+	if err != nil {
+		log.Errorf("Fail to get instance status on StopVM: %s", err)
+		return err
+	}
 	if status != InstanceStatusRunning {
+		log.Errorf("StopVM: vm status is %s expect %s", status, InstanceStatusRunning)
 		return cloudprovider.ErrInvalidStatus
 	}
 	return self.doStopVM(instanceId, isForce)
@@ -450,13 +551,13 @@ func (self *SRegion) StopVM(instanceId string, isForce bool) error {
 
 func (self *SRegion) DeleteVM(instanceId string) error {
 	status, err := self.GetInstanceStatus(instanceId)
-	if status == InstanceStatusRunning {
-		err = self.StopVM(instanceId, true)
-		if err != nil {
-			return err
-		}
-	} else if status != InstanceStatusStopped {
-		return cloudprovider.ErrInvalidStatus
+	if err != nil {
+		log.Errorf("Fail to get instance status on DeleteVM: %s", err)
+		return err
+	}
+	log.Debugf("Instance status on delete is %s", status)
+	if status != InstanceStatusStopped {
+		log.Warningf("DeleteVM: vm status is %s expect %s", status, InstanceStatusStopped)
 	}
 	return self.doDeleteVM(instanceId)
 	// if err != nil {
@@ -472,46 +573,128 @@ func (self *SRegion) DeleteVM(instanceId string) error {
 	// }
 }
 
-func (self *SInstance) StartVM() error {
-	err := self.host.zone.region.StartVM(self.InstanceId)
+func (self *SRegion) DeployVM(instanceId string, name string, password string, keypairName string, resetPassword bool, deleteKeypair bool, description string) error {
+	instance, err := self.GetInstance(instanceId)
 	if err != nil {
 		return err
 	}
-	return cloudprovider.WaitStatus(self, models.VM_RUNNING, 5*time.Second, 180*time.Second) // 3minutes
-}
 
-func (self *SInstance) StopVM(isForce bool) error {
-	err := self.host.zone.region.StopVM(self.InstanceId, isForce)
-	if err != nil {
-		return err
+	// 修改密钥时直接返回
+	if deleteKeypair {
+		return self.DetachKeyPair(instanceId, instance.KeyPairName)
+	}
+
+	if len(keypairName) > 0 {
+		return self.AttachKeypair(instanceId, keypairName)
+	}
+
+	params := make(map[string]string)
+	if resetPassword {
+		params["Password"] = seclib2.RandomPassword2(12)
+	}
+	// 指定密码的情况下，使用指定的密码
+	if len(password) > 0 {
+		params["Password"] = password
+	}
+
+	if len(name) > 0 && instance.InstanceName != name {
+		params["InstanceName"] = name
+		params["HostName"] = name
+	}
+
+	if len(description) > 0 && instance.Description != description {
+		params["Description"] = description
+	}
+
+	if len(params) > 0 {
+		return self.modifyInstanceAttribute(instanceId, params)
+	} else {
+		return nil
 	}
-	return cloudprovider.WaitStatus(self, models.VM_READY, 10*time.Second, 300*time.Second) // 5mintues
 }
 
 func (self *SInstance) DeleteVM() error {
-	err := self.host.zone.region.DeleteVM(self.InstanceId)
-	if err != nil {
-		return err
+	for {
+		err := self.host.zone.region.DeleteVM(self.InstanceId)
+		if err != nil {
+			if isError(err, "IncorrectInstanceStatus.Initializing") {
+				log.Infof("The instance is initializing, try later ...")
+				time.Sleep(10 * time.Second)
+			} else {
+				return err
+			}
+		} else {
+			break
+		}
 	}
 	return cloudprovider.WaitDeleted(self, 10*time.Second, 300*time.Second) // 5minutes
 }
 
-func (self *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) {
-	url, err := self.host.zone.region.GetInstanceVNCUrl(self.InstanceId)
-	if err != nil {
-		return nil, err
+func (self *SRegion) UpdateVM(instanceId string, hostname string) error {
+	/*
+			api: ModifyInstanceAttribute
+		    https://help.aliyun.com/document_detail/25503.html?spm=a2c4g.11186623.4.1.DrgpjW
+	*/
+	params := make(map[string]string)
+	params["HostName"] = hostname
+	return self.modifyInstanceAttribute(instanceId, params)
+}
+
+func (self *SRegion) modifyInstanceAttribute(instanceId string, params map[string]string) error {
+	return self.instanceOperation(instanceId, "ModifyInstanceAttribute", params)
+}
+
+func (self *SRegion) ReplaceSystemDisk(instanceId string, image string) error {
+	params := make(map[string]string)
+	params["ImageId"] = image
+	return self.instanceOperation(instanceId, "ReplaceSystemDisk", params)
+}
+
+func (self *SRegion) ChangeVMConfig(zoneId string, instanceId string, ncpu int, vmem int, disks []*SDisk) error {
+	// todo: support change disk config?
+	params := make(map[string]string)
+	instanceTypes, e := self.GetMatchInstanceTypes(ncpu, vmem, 0, zoneId)
+	if e != nil {
+		return e
 	}
-	passwd := seclib.RandomPassword(6)
-	err = self.host.zone.region.ModifyInstanceVNCUrlPassword(self.InstanceId, passwd)
-	if err != nil {
-		return nil, err
+
+	for _, instancetype := range instanceTypes {
+		params["InstanceType"] = instancetype.InstanceTypeId
+		params["ClientToken"] = utils.GenRequestId(20)
+		if err := self.instanceOperation(instanceId, "ModifyInstanceSpec", params); err != nil {
+			log.Errorf("Failed for %s: %s", instancetype.InstanceTypeId, err)
+		} else {
+			return nil
+		}
 	}
-	ret := jsonutils.NewDict()
-	ret.Add(jsonutils.NewString(url), "url")
-	ret.Add(jsonutils.NewString(passwd), "password")
-	ret.Add(jsonutils.NewString("aliyun"), "protocol")
-	ret.Add(jsonutils.NewString(self.InstanceId), "instance_id")
-	return ret, nil
+
+	return fmt.Errorf("Failed to change vm config, specification not supported")
+}
+
+func (self *SRegion) DetachDisk(instanceId string, diskId string) error {
+	params := make(map[string]string)
+	params["InstanceId"] = instanceId
+	params["DiskId"] = diskId
+	_, err := self.ecsRequest("DetachDisk", params)
+	if err != nil {
+		log.Errorf("DetachDisk %s to %s fail %s", diskId, instanceId, err)
+		return err
+	}
+
+	return nil
+}
+
+func (self *SRegion) AttachDisk(instanceId string, diskId string) error {
+	params := make(map[string]string)
+	params["InstanceId"] = instanceId
+	params["DiskId"] = diskId
+	_, err := self.ecsRequest("AttachDisk", params)
+	if err != nil {
+		log.Errorf("AttachDisk %s to %s fail %s", diskId, instanceId, err)
+		return err
+	}
+
+	return nil
 }
 
 func (self *SInstance) SyncSecurityGroup(secgroupId string, name string, rules []secrules.SecurityRule) error {

pkg/util/aliyun/keypair.go

@@ -1,6 +1,7 @@
 package aliyun
 
 import (
+	"encoding/json"
 	"fmt"
 	"yunion.io/x/log"
 )
@@ -62,3 +63,33 @@ func (self *SRegion) ImportKeypair(name string, pubKey string) (*SKeypair, error
 	}
 	return &keypair, nil
 }
+
+func (self *SRegion) AttachKeypair(instanceId string, name string)  error {
+	params := make(map[string]string)
+	params["RegionId"] = self.RegionId
+	params["KeyPairName"] = name
+	instances, _ := json.Marshal(&[...]string{instanceId})
+	params["InstanceIds"] = string(instances)
+	_, err := self.ecsRequest("AttachKeyPair", params)
+	if err != nil {
+		log.Errorf("AttachKeyPair fail %s", err)
+		return err
+	}
+
+	return nil
+}
+
+func (self *SRegion) DetachKeyPair(instanceId string, name string)  error {
+	params := make(map[string]string)
+	params["RegionId"] = self.RegionId
+	params["KeyPairName"] = name
+	instances, _ := json.Marshal(&[...]string{instanceId})
+	params["InstanceIds"] = string(instances)
+	_, err := self.ecsRequest("DetachKeyPair", params)
+	if err != nil {
+		log.Errorf("DetachKeyPair fail %s", err)
+		return err
+	}
+
+	return nil
+}

pkg/util/aliyun/region.go

@@ -9,9 +9,10 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/utils"
+
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
-	"yunion.io/x/pkg/utils"
 )
 
 type SRegion struct {
@@ -35,6 +36,10 @@ func (self *SRegion) GetClient() *SAliyunClient {
 	return self.client
 }
 
+func (self *SRegion) GetMetadata() *jsonutils.JSONDict {
+	return nil
+}
+
 func (self *SRegion) getEcsClient() (*sdk.Client, error) {
 	if self.ecsClient == nil {
 		cli, err := sdk.NewClientWithAccessKey(self.RegionId, self.client.accessKey, self.client.secret)
@@ -564,4 +569,4 @@ func (self *SRegion) updateInstance(instId string, name, desc, passwd, hostname
 
 func (self *SRegion) UpdateInstancePassword(instId string, passwd string) error {
 	return self.updateInstance(instId, "", "", passwd, "")
-}
+}

pkg/util/aliyun/securitygroup.go

@@ -77,6 +77,10 @@ func (v PermissionSet) Less(i, j int) bool {
 	return false
 }
 
+func (self *SSecurityGroup) GetMetadata() *jsonutils.JSONDict {
+	return nil
+}
+
 func (self *SSecurityGroup) GetId() string {
 	return self.SecurityGroupId
 }
@@ -574,3 +578,15 @@ func (self *SRegion) leaveSecurityGroup(secgroupId, instanceId string) error {
 	_, err := self.ecsRequest("LeaveSecurityGroup", params)
 	return err
 }
+
+func (self *SRegion) deleteSecurityGroup(secGrpId string) error {
+	params := make(map[string]string)
+	params["SecurityGroupId"] = secGrpId
+
+	_, err := self.ecsRequest("DeleteSecurityGroup", params)
+	if err != nil {
+		log.Errorf("Delete security group fail %s", err)
+		return err
+	}
+	return nil
+}

pkg/util/aliyun/shell/disk.go

@@ -21,4 +21,15 @@ func init() {
 		printList(disks, total, args.Offset, args.Limit, []string{})
 		return nil
 	})
+
+	type DiskDeleteOptions struct {
+		Instance string `help:"Instance ID"`
+	}
+	shellutils.R(&DiskDeleteOptions{}, "disk-delete", "List disks", func(cli *aliyun.SRegion, args *DiskDeleteOptions) error {
+		e := cli.DeleteDisk(args.Instance)
+		if e != nil {
+			return e
+		}
+		return nil
+	})
 }

pkg/util/aliyun/shell/instance.go

@@ -2,6 +2,7 @@ package shell
 
 import (
 	"fmt"
+
 	"yunion.io/x/onecloud/pkg/util/aliyun"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
 )
@@ -81,8 +82,65 @@ func init() {
 		return nil
 	})
 
+	/*
+		server-change-config 更改系统配置
+		server-reset
+	*/
+	type InstanceDeployOptions struct {
+		ID            string `help:"instance ID"`
+		Name          string `help:"new instance name"`
+		Hostname      string `help:"new hostname"`
+		Keypair       string `help:"Keypair Name"`
+		DeleteKeypair bool   `help:"Remove SSH keypair"`
+		Password      string `help:"new password"`
+		ResetPassword bool   `help:"Force reset password"`
+		Description   string `help:"new instances description"`
+	}
+
+	shellutils.R(&InstanceDeployOptions{}, "instance-deploy", "Deploy keypair/password to a stopped virtual server", func(cli *aliyun.SRegion, args *InstanceDeployOptions) error {
+		err := cli.DeployVM(args.ID, args.Name, args.Password, args.Keypair, args.ResetPassword, args.DeleteKeypair, args.Description)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	type InstanceRebuildRootOptions struct {
+		ID    string `help:"instance ID"`
+		Image string `help:"Image ID"`
+	}
+
+	shellutils.R(&InstanceRebuildRootOptions{}, "instance-rebuild-root", "Reinstall virtual server system image", func(cli *aliyun.SRegion, args *InstanceRebuildRootOptions) error {
+		err := cli.ReplaceSystemDisk(args.ID, args.Image)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	type InstanceChangeConfigOptions struct {
+		ID   string `help:"instance ID"`
+		Ncpu int    `help:"number of CPU"`
+		Vmem int    `help:"MiB of memory"`
+		Disk []int  `help:"Data disk sizes int GB"`
+	}
+
+	shellutils.R(&InstanceChangeConfigOptions{}, "instance-change-config", "Deploy keypair/password to a stopped virtual server", func(cli *aliyun.SRegion, args *InstanceChangeConfigOptions) error {
+		instance, e := cli.GetInstance(args.ID)
+		if e != nil {
+			return e
+		}
+
+		// todo : add create disks
+		err := cli.ChangeVMConfig(instance.ZoneId, args.ID, args.Ncpu, args.Vmem, nil)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
 	type InstanceUpdatePasswordOptions struct {
-		ID string `help:"Instance ID"`
+		ID     string `help:"Instance ID"`
 		PASSWD string `help:"new password"`
 	}
 	shellutils.R(&InstanceUpdatePasswordOptions{}, "instance-update-password", "Update instance password", func(cli *aliyun.SRegion, args *InstanceUpdatePasswordOptions) error {