Merge branch 'release/2.1.0' into feature/rex-logclient-20180822-1018

Makefile

@@ -3,7 +3,7 @@
 REPO_PREFIX := yunion.io/x/onecloud
 VENDOR_PATH := $(REPO_PREFIX)/vendor
 VERSION_PKG := $(VENDOR_PATH)/yunion.io/x/pkg/util/version
-ROOT_DIR := $(shell readlink -f `pwd`)
+ROOT_DIR := $(CURDIR)
 BUILD_DIR := $(ROOT_DIR)/_output
 BIN_DIR := $(BUILD_DIR)/bin
 BUILD_SCRIPT := $(ROOT_DIR)/build/build.sh

cmd/climc/climc.go

@@ -5,28 +5,32 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
 	prompt "github.com/c-bata/go-prompt"
+
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/util/version"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/cmd/climc/promputils"
 	"yunion.io/x/onecloud/cmd/climc/shell"
-	_ "yunion.io/x/onecloud/cmd/climc/shell/k8s"
 	"yunion.io/x/onecloud/pkg/mcclient"
+
+	_ "yunion.io/x/onecloud/cmd/climc/shell/k8s"
 )
 
 type BaseOptions struct {
-	Help       bool   `help:"Show help" short-token:"h"`
-	Debug      bool   `help:"Show debug information"`
-	Version    bool   `help:"Show version"`
-	Timeout    int    `default:"600" help:"Number of seconds to wait for a response"`
-	Insecure   bool   `default:"False" help:"Allow skip server cert verification if URL is https" short-token:"k"`
-	OsUsername string `default:"$OS_USERNAME" help:"Username, defaults to env[OS_USERNAME]"`
-	OsPassword string `default:"$OS_PASSWORD" help:"Password, defaults to env[OS_PASSWORD]"`
+	Help          bool   `help:"Show help" short-token:"h"`
+	Debug         bool   `help:"Show debug information"`
+	Version       bool   `help:"Show version"`
+	Timeout       int    `default:"600" help:"Number of seconds to wait for a response"`
+	Insecure      bool   `default:"false" help:"Allow skip server cert verification if URL is https" short-token:"k"`
+	NoCachedToken bool   `default:"false" help:"Force not use cached token"`
+	OsUsername    string `default:"$OS_USERNAME" help:"Username, defaults to env[OS_USERNAME]"`
+	OsPassword    string `default:"$OS_PASSWORD" help:"Password, defaults to env[OS_PASSWORD]"`
 	// OsProjectId string `default:"$OS_PROJECT_ID" help:"Proejct ID, defaults to env[OS_PROJECT_ID]"`
 	OsProjectName  string `default:"$OS_PROJECT_NAME" help:"Project name, defaults to env[OS_PROJECT_NAME]"`
 	OsDomainName   string `default:"$OS_DOMAIN_NAME" help:"Domain name, defaults to env[OS_DOMAIN_NAME]"`
@@ -122,8 +126,9 @@ func newClientSession(options *BaseOptions) (*mcclient.ClientSession, error) {
 		options.Insecure)
 
 	var cacheToken mcclient.TokenCredential
-	cacheFile, err := os.Open("/tmp/OS_AUTH_CACHE_TOKEN")
-	if err == nil && cacheFile != nil {
+	tokenCachePath := filepath.Join(os.TempDir(), "OS_AUTH_CACHE_TOKEN")
+	cacheFile, err := os.Open(tokenCachePath)
+	if err == nil && cacheFile != nil && !options.NoCachedToken {
 		fileInfo, _ := cacheFile.Stat()
 		dur, err := time.ParseDuration("-24h")
 		if fileInfo != nil && err == nil && fileInfo.ModTime().After(time.Now().Add(dur)) {
@@ -153,12 +158,12 @@ func newClientSession(options *BaseOptions) (*mcclient.ClientSession, error) {
 		if err != nil {
 			fmt.Printf("Marshal token error:%s", err)
 		} else {
-			fo, _ := os.Create("/tmp/OS_AUTH_CACHE_TOKEN")
+			fo, _ := os.Create(tokenCachePath)
 			fo.Write(bytesCacheToken)
 			fo.Close()
 		}
 	} else {
-		fmt.Println("******** Use Token Cache At /tmp/OS_AUTH_CACHE_TOKEN ********")
+		fmt.Printf("******** Use Token Cache At %s ********\n", tokenCachePath)
 	}
 
 	session := client.NewSession(options.OsRegionName,

pkg/mcclient/catalog.go

@@ -0,0 +1,6 @@
+package mcclient
+
+type IServiceCatalog interface {
+	GetServiceURL(service, region, zone, endpointType string) (string, error)
+	GetServiceURLs(service, region, zone, endpointType string) ([]string, error)
+}

pkg/mcclient/mcclient.go

@@ -10,36 +10,16 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/onecloud/pkg/util/httputils"
+	"yunion.io/x/log"
 )
 
-/*const (
-	USER_AGENT = "yunioncloud-go/201708"
-)
-
-var (
-	red    = color.New(color.FgRed, color.Bold).PrintlnFunc()
-	green  = color.New(color.FgGreen, color.Bold).PrintlnFunc()
-	yellow = color.New(color.FgYellow, color.Bold).PrintlnFunc()
-	cyan   = color.New(color.FgHiCyan, color.Bold).PrintlnFunc()
-)*/
-
-/*type JSONClientError struct {
-	Code    int
-	Class   string
-	Details string
-}
-
-func (e *JSONClientError) Error() string {
-	return fmt.Sprintf("JSONClientError: %s %d %s", e.Details, e.Code, e.Class)
-}*/
-
 type Client struct {
 	authUrl string
 	timeout int
 	debug   bool
 
 	httpconn *http.Client
-	// serviceCatalog *KeystoneServiceCatalog
+	serviceCatalog IServiceCatalog
 }
 
 func NewClient(authUrl string, timeout int, debug bool, insecure bool) *Client {
@@ -222,7 +202,7 @@ func (this *Client) _authV3(domainName, uname, passwd, projectId, projectName, t
 	if len(tokenId) == 0 {
 		return nil, fmt.Errorf("No X-Subject-Token in header")
 	}
-	ret, err := UnmarshalV3Token(rbody, tokenId)
+	ret, err := this.unmarshalV3Token(rbody, tokenId)
 	return ret, err
 }
 
@@ -245,7 +225,7 @@ func (this *Client) _authV2(uname, passwd, tenantId, tenantName, token string) (
 	if err != nil {
 		return nil, err
 	}
-	return UnmarshalV2Token(rbody)
+	return this.unmarshalV2Token(rbody)
 }
 
 func (this *Client) Authenticate(uname, passwd, domainName, tenantName string) (TokenCredential, error) {
@@ -255,16 +235,21 @@ func (this *Client) Authenticate(uname, passwd, domainName, tenantName string) (
 	return this._authV2(uname, passwd, "", tenantName, "")
 }
 
-func UnmarshalV3Token(rbody jsonutils.JSONObject, tokenId string) (cred TokenCredential, err error) {
+func (this *Client) unmarshalV3Token(rbody jsonutils.JSONObject, tokenId string) (cred TokenCredential, err error) {
 	cred = &TokenCredentialV3{Id: tokenId}
 	err = rbody.Unmarshal(cred)
 	if err != nil {
 		err = fmt.Errorf("Invalid response when unmarshal V3 Token: %v", err)
 	}
+	cata := cred.GetServiceCatalog()
+	if cata == nil {
+		log.Fatalf("No srvice catalog avaiable")
+	}
+	this.serviceCatalog = cata
 	return
 }
 
-func UnmarshalV2Token(rbody jsonutils.JSONObject) (cred TokenCredential, err error) {
+func (this *Client) unmarshalV2Token(rbody jsonutils.JSONObject) (cred TokenCredential, err error) {
 	access, err := rbody.Get("access")
 	if err == nil {
 		cred = &TokenCredentialV2{}
@@ -272,6 +257,11 @@ func UnmarshalV2Token(rbody jsonutils.JSONObject) (cred TokenCredential, err err
 		if err != nil {
 			err = fmt.Errorf("Invalid response when unmarshal V2 Token: %s", err)
 		}
+		cata := cred.GetServiceCatalog()
+		if cata == nil {
+			log.Fatalf("No srvice catalog avaiable")
+		}
+		this.serviceCatalog = cata
 		return
 	}
 	err = fmt.Errorf("Invalid response: no access object")
@@ -286,7 +276,7 @@ func (this *Client) verifyV3(adminToken, token string) (TokenCredential, error)
 	if err != nil {
 		return nil, err
 	}
-	return UnmarshalV3Token(rbody, token)
+	return this.unmarshalV3Token(rbody, token)
 }
 
 func (this *Client) verifyV2(adminToken, token string) (TokenCredential, error) {
@@ -297,7 +287,7 @@ func (this *Client) verifyV2(adminToken, token string) (TokenCredential, error)
 	if err != nil {
 		return nil, err
 	}
-	return UnmarshalV2Token(rbody)
+	return this.unmarshalV2Token(rbody)
 }
 
 func (this *Client) Verify(adminToken, token string) (cred TokenCredential, err error) {
@@ -320,6 +310,11 @@ func (this *Client) SetProject(tenantId, tenantName string, token TokenCredentia
 }
 
 func (this *Client) NewSession(region, zone, endpointType string, token TokenCredential, apiVersion string) *ClientSession {
+	cata := token.GetServiceCatalog()
+	if cata == nil {
+		log.Fatalf("Missing service catalog in token")
+	}
+	this.serviceCatalog = cata
 	return &ClientSession{client: this, region: region, zone: zone,
 		endpointType: endpointType, token: token,
 		apiVersion: apiVersion,

pkg/mcclient/modules/k8s/cluster.go

@@ -9,7 +9,7 @@ var (
 )
 
 func init() {
-	Clusters = NewManager("cluster", "clusters",
+	Clusters = NewManager("kube_cluster", "kube_clusters",
 		NewResourceCols("mode", "k8s_version", "status", "api_endpoint"),
 		NewColumns())
 	modules.Register(Clusters)

pkg/mcclient/modules/k8s/node.go

@@ -9,6 +9,6 @@ var (
 )
 
 func init() {
-	Nodes = NewManager("node", "nodes", NewResourceCols("cluster", "roles", "address", "status"), NewColumns())
+	Nodes = NewManager("kube_node", "kube_nodes", NewResourceCols("cluster", "roles", "address", "status"), NewColumns())
 	modules.Register(Nodes)
 }

pkg/mcclient/modules/k8s/service.go

@@ -13,7 +13,7 @@ type ServiceManager struct {
 func init() {
 	Services = &ServiceManager{
 		ResourceManager: *NewManager(
-			"service", "services",
+			"k8s_service", "k8s_services",
 			NewNamespaceCols("clusterIP", "selector", "internalEndpoint", "externalEndpoints"),
 			NewClusterCols())}
 	modules.Register(Services)

pkg/mcclient/modules/modules.go

@@ -2,10 +2,10 @@ package modules
 
 import (
 	"fmt"
-	"log"
 	"strings"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
@@ -142,6 +142,17 @@ func _getJointKey(mod1 Manager, mod2 Manager) string {
 	return fmt.Sprintf("%s-%s", mod1.KeyString(), mod2.KeyString())
 }
 
+func ensureModuleNotRegistered(mod, newMod BaseManagerInterface) {
+	modSvcType := mod.ServiceType()
+	newModSvcType := newMod.ServiceType()
+	if mod == newMod {
+		log.Fatalf("Module %#v duplicate registered, service type: %q", mod, modSvcType)
+	}
+	if modSvcType != newModSvcType {
+		log.Fatalf("Module %#v already registered, service type is %q.\nSo new module %#v can't be registered, service type is %q", mod, modSvcType, newMod, newModSvcType)
+	}
+}
+
 func _register(version string, mod BaseManagerInterface) {
 	if modules == nil {
 		modules = make(map[string]map[string][]BaseManagerInterface)
@@ -156,9 +167,7 @@ func _register(version string, mod BaseManagerInterface) {
 		mods = make([]BaseManagerInterface, 0)
 	}
 	for _, m := range mods {
-		if m == mod {
-			log.Fatal("Module already registered", mod)
-		}
+		ensureModuleNotRegistered(m, mod)
 	}
 	modtable[mod.KeyString()] = append(mods, mod)
 }
@@ -179,7 +188,7 @@ func _registerJointModule(version string, mod BaseManagerInterface) {
 		}
 		for _, m := range jointMods {
 			if m == jointMod {
-				log.Fatal("Module already registered", jointMod)
+				ensureModuleNotRegistered(m, jointMod)
 			}
 		}
 		modtable[jointKey] = append(jointMods, jointMod)

pkg/mcclient/session.go

@@ -75,7 +75,26 @@ func (this *ClientSession) GetServiceURL(service, endpointType string) (string,
 	if len(this.apiVersion) > 0 && this.apiVersion != DEFAULT_API_VERSION {
 		service = fmt.Sprintf("%s_%s", service, this.apiVersion)
 	}
-	return this.token.GetServiceURL(service, this.region, this.zone, endpointType)
+	url, err := this.token.GetServiceURL(service, this.region, this.zone, endpointType)
+	if err != nil {
+		url, err = this.client.serviceCatalog.GetServiceURL(service, this.region, this.zone, endpointType)
+	}
+	return url, err
+}
+
+func (this *ClientSession) GetServiceURLs(service, endpointType string) ([]string, error) {
+	if len(this.endpointType) > 0 {
+		// session specific endpoint type should override the input endpointType, which is supplied by manager
+		endpointType = this.endpointType
+	}
+	if len(this.apiVersion) > 0 && this.apiVersion != DEFAULT_API_VERSION {
+		service = fmt.Sprintf("%s_%s", service, this.apiVersion)
+	}
+	urls, err := this.token.GetServiceURLs(service, this.region, this.zone, endpointType)
+	if err != nil {
+		urls, err = this.client.serviceCatalog.GetServiceURLs(service, this.region, this.zone, endpointType)
+	}
+	return urls, err
 }
 
 func (this *ClientSession) getBaseUrl(service, endpointType string) (string, error) {

pkg/mcclient/token.go

@@ -23,6 +23,8 @@ type Endpoint struct {
 type TokenCredential interface {
 	gotypes.ISerializable
 
+	IServiceCatalog
+
 	GetTokenString() string
 	GetDomainId() string
 	GetDomainName() string
@@ -39,7 +41,9 @@ type TokenCredential interface {
 	IsAdmin() bool
 	IsSystemAdmin() bool
 	GetRegions() []string
-	GetServiceURL(service, region, zone, endpointType string) (string, error)
+
+	GetServiceCatalog() IServiceCatalog
+
 	GetInternalServices(region string) []string
 	GetExternalServices(region string) []ExternalService
 	GetEndpoints(region string, endpointType string) []Endpoint

pkg/mcclient/token2.go

@@ -133,7 +133,11 @@ func (this *TokenCredentialV2) IsSystemAdmin() bool {
 }
 
 func (this *TokenCredentialV2) GetServiceURL(service, region, zone, endpointType string) (string, error) {
-	return this.ServiceCatalog.getServiceURL(service, region, zone, endpointType)
+	return this.ServiceCatalog.GetServiceURL(service, region, zone, endpointType)
+}
+
+func (this *TokenCredentialV2) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) {
+	return this.ServiceCatalog.GetServiceURLs(service, region, zone, endpointType)
 }
 
 func (this *TokenCredentialV2) GetInternalServices(region string) []string {
@@ -148,6 +152,10 @@ func (this *TokenCredentialV2) GetEndpoints(region string, endpointType string)
 	return nil
 }
 
+func (this *TokenCredentialV2) GetServiceCatalog() IServiceCatalog {
+	return this.ServiceCatalog
+}
+
 func stringArrayContains(arr []string, needle string) bool {
 	for i := 0; i < len(arr); i++ {
 		if arr[i] == needle {
@@ -211,7 +219,7 @@ func (catalog KeystoneServiceCatalogV2) getServiceEndpoint(service, region, zone
 	return selected, fmt.Errorf("No such service %s", service)
 }
 
-func (catalog KeystoneServiceCatalogV2) getServiceURL(service, region, zone, endpointType string) (string, error) {
+func (catalog KeystoneServiceCatalogV2) GetServiceURL(service, region, zone, endpointType string) (string, error) {
 	ep, err := catalog.getServiceEndpoint(service, region, zone)
 	if err != nil {
 		return "", err
@@ -219,6 +227,14 @@ func (catalog KeystoneServiceCatalogV2) getServiceURL(service, region, zone, end
 	return ep.getURL(endpointType), nil
 }
 
+func (catalog KeystoneServiceCatalogV2) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) {
+	url, err := catalog.GetServiceURL(service, region, zone, endpointType)
+	if err != nil {
+		return nil, err
+	}
+	return []string{url}, nil
+}
+
 func (ep KeystoneEndpointV2) getURL(epType string) string {
 	switch epType {
 	case "publicURL":

pkg/mcclient/token3.go

@@ -141,7 +141,11 @@ func (this *TokenCredentialV3) GetRegions() []string {
 }
 
 func (this *TokenCredentialV3) GetServiceURL(service, region, zone, endpointType string) (string, error) {
-	return this.Token.Catalog.getServiceURL(service, region, zone, endpointType)
+	return this.Token.Catalog.GetServiceURL(service, region, zone, endpointType)
+}
+
+func (this *TokenCredentialV3) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) {
+	return this.Token.Catalog.GetServiceURLs(service, region, zone, endpointType)
 }
 
 func (this *TokenCredentialV3) GetInternalServices(region string) []string {
@@ -156,6 +160,10 @@ func (this *TokenCredentialV3) GetEndpoints(region string, endpointType string)
 	return this.Token.Catalog.getEndpoints(region, endpointType)
 }
 
+func (this *TokenCredentialV3) GetServiceCatalog() IServiceCatalog {
+	return this.Token.Catalog
+}
+
 func (catalog KeystoneServiceCatalogV3) getInternalServices(region string) []string {
 	services := make([]string, 0)
 	for i := 0; i < len(catalog); i++ {
@@ -242,7 +250,15 @@ func Id2RegionZone(id string) (string, string) {
 	}
 }
 
-func (catalog KeystoneServiceCatalogV3) getServiceURL(service, region, zone, endpointType string) (string, error) {
+func (catalog KeystoneServiceCatalogV3) GetServiceURL(service, region, zone, endpointType string) (string, error) {
+	urls, err := catalog.GetServiceURLs(service, region, zone, endpointType)
+	if err != nil {
+		return "", err
+	}
+	return urls[rand.Intn(len(urls))], nil
+}
+
+func (catalog KeystoneServiceCatalogV3) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) {
 	if endpointType == "" {
 		endpointType = "internalURL"
 	}
@@ -273,7 +289,7 @@ func (catalog KeystoneServiceCatalogV3) getServiceURL(service, region, zone, end
 						break
 					}
 				} else {
-					return "", fmt.Errorf("No default region")
+					return nil, fmt.Errorf("No default region")
 				}
 			} else {
 				_, ok := regeps[regionzone]
@@ -282,14 +298,14 @@ func (catalog KeystoneServiceCatalogV3) getServiceURL(service, region, zone, end
 				} else {
 					selected, ok = regeps[region]
 					if !ok {
-						return "", fmt.Errorf("No valid %s endpoints for %s in region %s", endpointType, service, RegionID(region, zone))
+						return nil, fmt.Errorf("No valid %s endpoints for %s in region %s", endpointType, service, RegionID(region, zone))
 					}
 				}
 			}
-			return selected[rand.Intn(len(selected))], nil
+			return selected, nil
 		}
 	}
-	return "", fmt.Errorf("No such service %s", service)
+	return nil, fmt.Errorf("No such service %s", service)
 }
 
 func (self *TokenCredentialV3) String() string {

pkg/mcclient/tokensimple.go

@@ -96,6 +96,10 @@ func (self *SSimpleToken) GetServiceURL(service, region, zone, endpointType stri
 	return "", fmt.Errorf("Not available")
 }
 
+func (self *SSimpleToken) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) {
+	return nil, fmt.Errorf("Not available")
+}
+
 func (self *SSimpleToken) GetInternalServices(region string) []string {
 	return nil
 }
@@ -108,6 +112,10 @@ func (this *SSimpleToken) GetEndpoints(region string, endpointType string) []End
 	return nil
 }
 
+func (this *SSimpleToken) GetServiceCatalog() IServiceCatalog {
+	return nil
+}
+
 func SimplifyToken(token TokenCredential) TokenCredential {
 	simToken, ok := token.(*SSimpleToken)
 	if ok {