Merge branch 'release/2.4.0' of ssh://git.yunion.io/~qiujian/onecloud into feature/qj-region-geographicinfo

cmd/climc/shell/bill_details.go

@@ -0,0 +1,47 @@
+package shell
+
+import (
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func init() {
+
+	type BillDetailListOptions struct {
+		options.BaseListOptions
+		STARTDATE string `help:"start date of the bill_detail"`
+		ENDDATE   string `help:"end date of the bill_detail"`
+		ProjectId string `help:"project id of the bill_detail"`
+	}
+	R(&BillDetailListOptions{}, "billdetail-list", "List all bill details", func(s *mcclient.ClientSession, args *BillDetailListOptions) error {
+		var params *jsonutils.JSONDict
+		{
+			var err error
+			params, err = args.BaseListOptions.Params()
+			if err != nil {
+				return err
+
+			}
+		}
+		if len(args.STARTDATE) > 0 {
+			params.Add(jsonutils.NewString(args.STARTDATE), "start_date")
+		}
+		if len(args.ENDDATE) > 0 {
+			params.Add(jsonutils.NewString(args.ENDDATE), "end_date")
+		}
+
+		if len(args.ProjectId) > 0 {
+			params.Add(jsonutils.NewString(args.ProjectId), "project_id")
+		}
+
+		result, err := modules.BillDetails.List(s, params)
+		if err != nil {
+			return err
+		}
+
+		printList(result, modules.BillDetails.GetColumns(s))
+		return nil
+	})
+}

pkg/appsrv/appparams.go

@@ -0,0 +1,30 @@
+package appsrv
+
+import (
+	"context"
+	"net/http"
+
+	"yunion.io/x/onecloud/pkg/appctx"
+)
+
+const (
+	APP_CONTEXT_KEY_APP_PARAMS = appctx.AppContextKey("app_params")
+)
+
+type SAppParams struct {
+	Name      string
+	SkipLog   bool
+	SkipTrace bool
+	Params    map[string]string
+	Path      []string
+	Request   *http.Request
+}
+
+func AppContextGetParams(ctx context.Context) *SAppParams {
+	val := ctx.Value(APP_CONTEXT_KEY_APP_PARAMS)
+	if val != nil {
+		return val.(*SAppParams)
+	} else {
+		return nil
+	}
+}

pkg/appsrv/appsrv.go

@@ -157,7 +157,7 @@ func (app *Application) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	rid := genRequestId(w, r)
 	lrw := &loggingResponseWriter{w, http.StatusOK}
 	start := time.Now()
-	hi := app.defaultHandle(lrw, r, rid)
+	hi, params := app.defaultHandle(lrw, r, rid)
 	if hi == nil {
 		hi = &app.defHandlerInfo
 	}
@@ -172,7 +172,15 @@ func (app *Application) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	duration := float64(time.Since(start).Nanoseconds()) / 1000000
 	counter.hit += 1
 	counter.duration += duration
-	if !hi.skipLog {
+	skipLog := false
+	if params != nil {
+		if params.SkipLog {
+			skipLog = true
+		}
+	} else if hi.skipLog {
+		skipLog = true
+	}
+	if !skipLog {
 		log.Infof("%d %s %s %s (%s) %.2fms", lrw.status, rid, r.Method, r.URL, r.RemoteAddr, duration)
 	}
 }
@@ -190,7 +198,7 @@ func (app *Application) handleCORS(w http.ResponseWriter, r *http.Request) bool
 	}
 }
 
-func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, rid string) *SHandlerInfo {
+func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, rid string) (*SHandlerInfo, *SAppParams) {
 	segs := SplitPath(r.URL.Path)
 	params := make(map[string]string)
 	w.Header().Set("Server", "Yunion AppServer/Go/2018.4")
@@ -213,6 +221,8 @@ func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, ri
 			if session == nil {
 				session = app.session
 			}
+			appParams := hand.GetAppParams(params, segs)
+			appParams.Request = r
 			session.Run(
 				func() {
 					if ctx.Err() == nil {
@@ -223,9 +233,14 @@ func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, ri
 						if hand.metadata != nil {
 							ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_METADATA, hand.metadata)
 						}
+						ctx = context.WithValue(ctx, APP_CONTEXT_KEY_APP_PARAMS, appParams)
 						func() {
-							span := trace.StartServerTrace(&fw, r, hand.GetName(params), app.GetName(), hand.GetTags())
-							defer span.EndTrace()
+							span := trace.StartServerTrace(&fw, r, appParams.Name, app.GetName(), hand.GetTags())
+							defer func() {
+								if !appParams.SkipTrace {
+									span.EndTrace()
+								}
+							}()
 							ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_TRACE, span)
 							hand.handler(ctx, &fw, r)
 						}()
@@ -249,7 +264,7 @@ func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, ri
 				}
 			}
 			fw.closeChannels()
-			return hand
+			return hand, appParams
 		} else {
 			log.Errorf("Invalid handler for %s", r.URL)
 			httperrors.InternalServerError(w, "Invalid handler %s", r.URL)
@@ -258,7 +273,7 @@ func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, ri
 		log.Errorf("Handler not found")
 		httperrors.NotFoundError(w, "Handler not found")
 	}
-	return nil
+	return nil, nil
 }
 
 func (app *Application) addDefaultHandler(method string, prefix string, handler func(context.Context, http.ResponseWriter, *http.Request), name string) {

pkg/appsrv/cache_test.go

@@ -1,24 +0,0 @@
-package appsrv
-
-import (
-	"testing"
-)
-
-func TestCache(t *testing.T) {
-	c := NewCache(1024)
-	c.Set("123", 123)
-	c.Set("456", 456)
-	v := c.Get("123")
-	if v == nil || v.(int) != 123 {
-		t.Error("Key 123 not found")
-	}
-	v = c.Get("456")
-	if v == nil || v.(int) != 456 {
-		t.Error("Key 456 not found")
-	}
-	c.Set("456", 789)
-	v = c.Get("456")
-	if v == nil || v.(int) != 789 {
-		t.Error("Key 456 not changed")
-	}
-}

pkg/appsrv/context.go

@@ -5,6 +5,7 @@ import (
 	"database/sql"
 
 	"yunion.io/x/onecloud/pkg/appctx"
+	"yunion.io/x/onecloud/pkg/util/hashcache"
 )
 
 func AppContextDB(ctx context.Context) *sql.DB {
@@ -15,12 +16,12 @@ func AppContextDB(ctx context.Context) *sql.DB {
 	return val.(*sql.DB)
 }
 
-func AppContextCache(ctx context.Context) *Cache {
+func AppContextCache(ctx context.Context) *hashcache.Cache {
 	val := ctx.Value(appctx.APP_CONTEXT_KEY_CACHE)
 	if val == nil {
 		return nil
 	}
-	return val.(*Cache)
+	return val.(*hashcache.Cache)
 }
 
 func AppContextApp(ctx context.Context) *Application {

pkg/appsrv/handlerinfo.go

@@ -102,3 +102,12 @@ func (hi *SHandlerInfo) SetSkipLog(skip bool) *SHandlerInfo {
 	hi.skipLog = skip
 	return hi
 }
+
+func (hi *SHandlerInfo) GetAppParams(params map[string]string, path []string) *SAppParams {
+	appParams := SAppParams{}
+	appParams.Name = hi.GetName(params)
+	appParams.SkipLog = hi.skipLog
+	appParams.Params = params
+	appParams.Path = path
+	return &appParams
+}

pkg/cloudcommon/policy/policy.go

@@ -1,6 +1,9 @@
 package policy
 
 import (
+	"fmt"
+	"sort"
+	"strings"
 	"time"
 
 	"yunion.io/x/jsonutils"
@@ -12,6 +15,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/util/conditionparser"
+	"yunion.io/x/onecloud/pkg/util/hashcache"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
@@ -147,6 +151,8 @@ type SPolicyManager struct {
 
 	failedRetryInterval time.Duration
 	refreshInterval     time.Duration
+
+	cache *hashcache.Cache // policy cache
 }
 
 func parseJsonPolicy(obj jsonutils.JSONObject) (string, rbacutils.SRbacPolicy, error) {
@@ -226,6 +232,8 @@ func (manager *SPolicyManager) start(refreshInterval time.Duration, retryInterva
 			Rules: rbacutils.CompactRules(defaultRules),
 		}
 	}
+
+	manager.cache = hashcache.NewCache(2048, manager.refreshInterval/2)
 	manager.sync()
 }
 
@@ -243,7 +251,48 @@ func (manager *SPolicyManager) sync() {
 	time.AfterFunc(manager.refreshInterval, manager.sync)
 }
 
+func queryKey(isAdmin bool, userCred mcclient.TokenCredential, service string, resource string, action string, extra ...string) string {
+	queryKeys := []string{fmt.Sprintf("%v", isAdmin)}
+	queryKeys = append(queryKeys, userCred.GetProjectId(), userCred.GetDomainId(), userCred.GetUserId())
+	roles := userCred.GetRoles()
+	if len(roles) > 0 {
+		sort.Strings(roles)
+	}
+	queryKeys = append(queryKeys, strings.Join(roles, ":"))
+	if rbacutils.WILD_MATCH == service || len(service) == 0 {
+		service = rbacutils.WILD_MATCH
+	}
+	queryKeys = append(queryKeys, service)
+	if rbacutils.WILD_MATCH == resource || len(resource) == 0 {
+		resource = rbacutils.WILD_MATCH
+	}
+	queryKeys = append(queryKeys, resource)
+	if rbacutils.WILD_MATCH == action || len(action) == 0 {
+		action = rbacutils.WILD_MATCH
+	}
+	queryKeys = append(queryKeys, action)
+	if len(extra) > 0 {
+		queryKeys = append(queryKeys, extra...)
+	}
+	return strings.Join(queryKeys, "-")
+}
+
 func (manager *SPolicyManager) Allow(isAdmin bool, userCred mcclient.TokenCredential, service string, resource string, action string, extra ...string) rbacutils.TRbacResult {
+	if manager.cache != nil {
+		key := queryKey(isAdmin, userCred, service, resource, action, extra...)
+		val := manager.cache.Get(key)
+		if val != nil {
+			return val.(rbacutils.TRbacResult)
+		}
+		result := manager.allowWithoutCache(isAdmin, userCred, service, resource, action, extra...)
+		manager.cache.Set(key, result)
+		return result
+	} else {
+		return manager.allowWithoutCache(isAdmin, userCred, service, resource, action, extra...)
+	}
+}
+
+func (manager *SPolicyManager) allowWithoutCache(isAdmin bool, userCred mcclient.TokenCredential, service string, resource string, action string, extra ...string) rbacutils.TRbacResult {
 	var policies map[string]rbacutils.SRbacPolicy
 	if isAdmin {
 		policies = manager.adminPolicies

pkg/compute/guestdrivers/container.go

@@ -26,6 +26,20 @@ func init() {
 	models.RegisterGuestDriver(&driver)
 }
 
+func (self *SContainerDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	vmemSize, vcpuCount, err := models.ValidateMemCpuData(data)
+	if err != nil {
+		return nil, err
+	}
+	if vmemSize > 0 {
+		data.Add(jsonutils.NewInt(int64(vmemSize)), "vmem_size")
+	}
+	if vcpuCount > 0 {
+		data.Add(jsonutils.NewInt(int64(vcpuCount)), "vcpu_count")
+	}
+	return data, nil
+}
+
 func (self *SContainerDriver) newUnsupportOperationError(option string) error {
 	return httperrors.NewUnsupportOperationError("Container not support %s", option)
 }

pkg/compute/models/guests.go

@@ -586,7 +586,7 @@ func (guest *SGuest) ValidateResizeDisk(disk *SDisk, storage *SStorage) error {
 	return guest.GetDriver().ValidateResizeDisk(guest, disk, storage)
 }
 
-func validateMemCpuData(data jsonutils.JSONObject) (int, int, error) {
+func ValidateMemCpuData(data jsonutils.JSONObject) (int, int, error) {
 	vmemSize := 0
 	vcpuCount := 0
 	var err error
@@ -626,7 +626,7 @@ func validateMemCpuData(data jsonutils.JSONObject) (int, int, error) {
 }
 
 func (self *SGuest) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
-	vmemSize, vcpuCount, err := validateMemCpuData(data)
+	vmemSize, vcpuCount, err := ValidateMemCpuData(data)
 	if err != nil {
 		return nil, err
 	}
@@ -740,7 +740,7 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m
 			data.Add(jsonutils.NewInt(int64(sku.MemorySizeMB)), "vmem_size")
 			data.Add(jsonutils.NewInt(int64(sku.CpuCoreCount)), "vcpu_count")
 		} else {
-			vmemSize, vcpuCount, err := validateMemCpuData(data)
+			vmemSize, vcpuCount, err := ValidateMemCpuData(data)
 			if err != nil {
 				return nil, err
 			}

pkg/compute/models/hosts.go

@@ -35,6 +35,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/onecloud/pkg/util/logclient"
+	"yunion.io/x/onecloud/pkg/appsrv"
 )
 
 const (
@@ -2654,6 +2655,14 @@ func (self *SHost) PerformPing(ctx context.Context, userCred mcclient.TokenCrede
 		return nil, fmt.Errorf("Get catalog error")
 	}
 	result.Set("catalog", catalog)
+
+	appParams := appsrv.AppContextGetParams(ctx)
+	if appParams != nil {
+		// skip log&trace, when everything is normal
+		appParams.SkipTrace = true
+		appParams.SkipLog = true
+	}
+
 	return result, nil
 }
 

pkg/compute/models/networks.go

@@ -856,6 +856,10 @@ func (self *SNetwork) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSOND
 			extra.Add(jsonutils.NewString(vpc.GetExternalId()), "vpc_external_id")
 		}
 	}
+	routes := self.GetRoutes()
+	if len(routes) > 0 {
+		extra.Add(jsonutils.Marshal(routes), "routes")
+	}
 
 	return extra
 }

pkg/mcclient/modules/mod_bill_details.go

@@ -0,0 +1,14 @@
+package modules
+
+var (
+	BillDetails ResourceManager
+)
+
+func init() {
+	BillDetails = NewMeterManager("bill_detail", "bill_details",
+		[]string{"bill_id", "account", "platform", "region", "sub_account_project", "res_id",
+			"res_type", "res_name", "start_time", "end_time", "charge_type", "item_rate", "item_fee"},
+		[]string{},
+	)
+	register(&BillDetails)
+}

pkg/util/aliyun/aliyun.go

@@ -10,6 +10,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/pkg/utils"
 )
 
 const (
@@ -70,8 +71,12 @@ func _jsonRequest(client *sdk.Client, domain string, version string, apiName str
 		return nil, err
 	}
 	//{"Code":"InvalidInstanceType.ValueNotSupported","HostId":"ecs.aliyuncs.com","Message":"The specified instanceType beyond the permitted range.","RequestId":"0042EE30-0EDF-48A7-A414-56229D4AD532"}
+	//{"Code":"200","Message":"successful","PageNumber":1,"PageSize":50,"RequestId":"BB4C970C-0E23-48DC-A3B0-EB21FFC70A29","RouterTableList":{"RouterTableListType":[{"CreationTime":"2017-03-19T13:37:40Z","Description":"","ResourceGroupId":"rg-acfmwie3cqoobmi","RouteTableId":"vtb-j6c60lectdi80rk5xz43g","RouteTableName":"","RouteTableType":"System","RouterId":"vrt-j6c00qrol733dg36iq4qj","RouterType":"VRouter","VSwitchIds":{"VSwitchId":["vsw-j6c3gig5ub4fmi2veyrus"]},"VpcId":"vpc-j6c86z3sh8ufhgsxwme0q"}]},"Success":true,"TotalCount":1}
 	if body.Contains("Code") {
-		return nil, fmt.Errorf(body.String())
+		code, _ := body.GetString("Code")
+		if len(code) > 0 && !utils.IsInStringArray(code, []string{"200"}) {
+			return nil, fmt.Errorf(body.String())
+		}
 	}
 	return body, nil
 }

pkg/util/hashcache/cache.go

@@ -1,25 +1,28 @@
-package appsrv
+package hashcache
 
 import (
 	"crypto/md5"
 	"crypto/sha1"
 	"crypto/sha256"
 	"sync"
+	"time"
 )
 
 type cacheNode struct {
-	key   string
-	value interface{}
+	key    string
+	expire time.Time
+	value  interface{}
 }
 
 type Cache struct {
-	table []cacheNode
-	lock  *sync.Mutex
-	size  uint32
+	table      []cacheNode
+	lock       *sync.Mutex
+	size       uint32
+	defaultTtl time.Duration
 }
 
-func NewCache(size uint32) *Cache {
-	ca := &Cache{table: make([]cacheNode, size), lock: &sync.Mutex{}, size: size}
+func NewCache(size uint32, defaultTTL time.Duration) *Cache {
+	ca := &Cache{table: make([]cacheNode, size), lock: &sync.Mutex{}, size: size, defaultTtl: defaultTTL}
 	return ca
 }
 
@@ -51,10 +54,14 @@ func checksum(alg int, key string) uint32 {
 
 func (c *Cache) find(key string) (bool, uint32) {
 	var idx uint32
+	now := time.Now()
 	for _, alg := range []int{HASH_ALG_MD5, HASH_ALG_SHA1, HASH_ALG_SHA256} {
 		idx = checksum(alg, key) % c.size
 		if c.table[idx].key == key {
-			return true, idx
+			if c.table[idx].expire.IsZero() || c.table[idx].expire.After(now) {
+				return true, idx
+			}
+			break
 		}
 	}
 	return false, idx
@@ -74,13 +81,18 @@ func (c *Cache) AtomicGet(key string) interface{} {
 	return c.Get(key)
 }
 
-func (c *Cache) Set(key string, val interface{}) {
+func (c *Cache) Set(key string, val interface{}, expire ...time.Time) {
 	find, idx := c.find(key)
-	if find {
-		c.table[idx].value = val
-	} else {
+	if !find {
 		c.table[idx].key = key
-		c.table[idx].value = val
+	}
+	c.table[idx].value = val
+	if len(expire) > 0 && !expire[0].IsZero() {
+		c.table[idx].expire = expire[0]
+	} else if c.defaultTtl > time.Millisecond {
+		c.table[idx].expire = time.Now().Add(c.defaultTtl)
+	} else {
+		c.table[idx].expire = time.Time{}
 	}
 }
 

pkg/util/hashcache/cache_test.go

@@ -0,0 +1,49 @@
+package hashcache
+
+import (
+	"testing"
+	"time"
+)
+
+func TestCache(t *testing.T) {
+	c := NewCache(1024, time.Second)
+	c.Set("123", 123)
+	c.Set("456", 456)
+	v := c.Get("123")
+	if v == nil || v.(int) != 123 {
+		t.Error("Key 123 not found")
+	}
+	v = c.Get("456")
+	if v == nil || v.(int) != 456 {
+		t.Error("Key 456 not found")
+	}
+	c.Set("456", 789)
+	v = c.Get("456")
+	if v == nil || v.(int) != 789 {
+		t.Error("Key 456 not changed")
+	}
+
+	time.Sleep(time.Second)
+
+	v = c.Get("123")
+	if v != nil {
+		t.Errorf("key 123 shoud expire")
+	}
+	c.Set("123", 1234)
+	c.Set("456", 4567)
+	v = c.Get("123")
+	if v == nil || v.(int) != 1234 {
+		t.Error("Key 123 not found")
+	}
+	v = c.Get("456")
+	if v == nil || v.(int) != 4567 {
+		t.Error("Key 456 not found")
+	}
+
+	time.Sleep(time.Second)
+
+	v = c.Get("123")
+	if v != nil {
+		t.Errorf("key 123 shoud expire")
+	}
+}