Merge branch 'release/2.4.0' into feature/tb-skus-support-private-cloud

pkg/compute/models/capabilities.go

@@ -11,9 +11,9 @@ import (
 )
 
 type SCapabilities struct {
-	Hypervisors        []string
-	StorageTypes       []string
-	GPUModels          []string
+	Hypervisors        []string     `json:",allowempty"`
+	StorageTypes       []string		`json:",allowempty"`
+	GPUModels          []string		`json:",allowempty"`
 	MinNicCount        int
 	MaxNicCount        int
 	MinDataDiskCount   int

pkg/compute/models/hosts.go

@@ -166,7 +166,7 @@ type SHost struct {
 
 	LastPingAt time.Time ``
 
-	ResourceType string `width:"36" charset:"ascii" nullable:"false" list:"admin" update:"admin" create:"admin_required"` // Column(VARCHAR(36, charset='ascii'), nullable=False)
+	ResourceType string `width:"36" charset:"ascii" nullable:"false" list:"admin" update:"admin" create:"admin_optional" default:"shared"` // Column(VARCHAR(36, charset='ascii'), nullable=False)
 
 	RealExternalId string `width:"256" charset:"utf8" get:"admin"`
 }

pkg/compute/models/loadbalancerlistenerrules.go

@@ -39,6 +39,8 @@ type SLoadbalancerListenerRule struct {
 
 	Domain string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"optional"`
 	Path   string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"optional"`
+
+	SLoadbalancerHTTPRateLimiter
 }
 
 func loadbalancerListenerRuleCheckUniqueness(ctx context.Context, lbls *SLoadbalancerListener, domain, path string) error {
@@ -101,6 +103,9 @@ func (man *SLoadbalancerListenerRuleManager) ValidateCreateData(ctx context.Cont
 		"backend_group": backendGroupV,
 		"domain":        domainV.AllowEmpty(true).Default(""),
 		"path":          pathV.Default(""),
+
+		"http_request_rate":         validators.NewNonNegativeValidator("http_request_rate").Default(0),
+		"http_request_rate_per_src": validators.NewNonNegativeValidator("http_request_rate_per_src").Default(0),
 	}
 	for _, v := range keyV {
 		if err := v.Validate(data); err != nil {
@@ -131,10 +136,16 @@ func (lbr *SLoadbalancerListenerRule) AllowPerformStatus(ctx context.Context, us
 
 func (lbr *SLoadbalancerListenerRule) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", lbr.GetOwnerProjectId())
-	backendGroupV.Optional(true)
-	err := backendGroupV.Validate(data)
-	if err != nil {
-		return nil, err
+	keyV := map[string]validators.IValidator{
+		"backend_group":             backendGroupV,
+		"http_request_rate":         validators.NewNonNegativeValidator("http_request_rate"),
+		"http_request_rate_per_src": validators.NewNonNegativeValidator("http_request_rate_per_src"),
+	}
+	for _, v := range keyV {
+		v.Optional(true)
+		if err := v.Validate(data); err != nil {
+			return nil, err
+		}
 	}
 	if backendGroup, ok := backendGroupV.Model.(*SLoadbalancerBackendGroup); ok && backendGroup.Id != lbr.BackendGroupId {
 		listenerM, err := LoadbalancerListenerManager.FetchById(lbr.ListenerId)

pkg/compute/models/loadbalancerlisteners.go

@@ -33,6 +33,11 @@ func init() {
 	}
 }
 
+type SLoadbalancerHTTPRateLimiter struct {
+	HTTPRequestRate       int `nullable:"false" list:"user" create:"optional" update:"user"`
+	HTTPRequestRatePerSrc int `nullable:"false" list:"user" create:"optional" update:"user"`
+}
+
 type SLoadbalancerTCPListener struct{}
 type SLoadbalancerUDPListener struct{}
 
@@ -43,8 +48,6 @@ type SLoadbalancerHTTPListener struct {
 	StickySessionCookie        string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"`
 	StickySessionCookieTimeout int    `nullable:"false" list:"user" create:"optional" update:"user"`
 
-	//XForwardedForSLBIP bool `nullable:"false" list:"user" create:"optional"`
-	//XForwardedForSLBID bool `nullable:"false" list:"user" create:"optional"`
 	XForwardedFor bool `nullable:"false" list:"user" create:"optional" update:"user"`
 	Gzip          bool `nullable:"false" list:"user" create:"optional" update:"user"`
 }
@@ -69,7 +72,6 @@ type SLoadbalancerListener struct {
 	ListenerPort   int    `nullable:"false" list:"user" create:"required"`
 	BackendGroupId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"`
 
-	Bandwidth int    `nullable:"false" list:"user" create:"optional" update:"user"`
 	Scheduler string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user"`
 
 	ClientRequestTimeout  int `nullable:"false" list:"user" create:"optional" update:"user"`
@@ -100,6 +102,8 @@ type SLoadbalancerListener struct {
 	SLoadbalancerUDPListener
 	SLoadbalancerHTTPListener
 	SLoadbalancerHTTPSListener
+
+	SLoadbalancerHTTPRateLimiter
 }
 
 func (man *SLoadbalancerListenerManager) checkListenerUniqueness(ctx context.Context, lb *SLoadbalancer, listenerType string, listenerPort int64) error {
@@ -201,6 +205,9 @@ func (man *SLoadbalancerListenerManager) ValidateCreateData(ctx context.Context,
 
 		"x_forwarded_for": validators.NewBoolValidator("x_forwarded_for").Default(true),
 		"gzip":            validators.NewBoolValidator("gzip").Default(false),
+
+		"http_request_rate":         validators.NewNonNegativeValidator("http_request_rate").Default(0),
+		"http_request_rate_per_src": validators.NewNonNegativeValidator("http_request_rate_per_src").Default(0),
 	}
 	for _, v := range keyV {
 		if err := v.Validate(data); err != nil {
@@ -345,6 +352,9 @@ func (lblis *SLoadbalancerListener) ValidateUpdateData(ctx context.Context, user
 		"x_forwarded_for": validators.NewBoolValidator("x_forwarded_for"),
 		"gzip":            validators.NewBoolValidator("gzip"),
 
+		"http_request_rate":         validators.NewNonNegativeValidator("http_request_rate"),
+		"http_request_rate_per_src": validators.NewNonNegativeValidator("http_request_rate_per_src"),
+
 		"certificate":       certV,
 		"tls_cipher_policy": tlsCipherPolicyV,
 		"enable_http2":      validators.NewBoolValidator("enable_http2").Default(true),

pkg/compute/models/networks.go

@@ -856,6 +856,10 @@ func (self *SNetwork) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSOND
 			extra.Add(jsonutils.NewString(vpc.GetExternalId()), "vpc_external_id")
 		}
 	}
+	routes := self.GetRoutes()
+	if len(routes) > 0 {
+		extra.Add(jsonutils.Marshal(routes), "routes")
+	}
 
 	return extra
 }

pkg/lbagent/models/haproxy.go

@@ -318,6 +318,43 @@ func (b *LoadbalancerCorpus) genHaproxyConfigBackend(data map[string]interface{}
 	return nil
 }
 
+func (b *LoadbalancerCorpus) genHaproxyConfigHttpRate(data map[string]interface{}, requestRate, requestRatePerSrc int) error {
+	periodSecond := 10
+	id := data["id"].(string)
+	dummyBackends := []map[string]string{}
+	rateRules := []string{}
+
+	// order matters here: every src uses up his own quota before touching
+	// the shared one
+	if requestRatePerSrc > 0 {
+		idPerSrc := id + "_persrc"
+		dummyBackends = append(dummyBackends, map[string]string{
+			"id":          idPerSrc,
+			"stick_table": fmt.Sprintf("stick-table type ip size 1m expire 1m store http_req_rate(%ds)", periodSecond),
+		})
+		rateRules = append(rateRules,
+			fmt.Sprintf("http-request deny deny_status 429 if { src_http_req_rate(%s) gt %d }",
+				idPerSrc, requestRatePerSrc*periodSecond),
+			fmt.Sprintf("http-request track-sc0 src table %s",
+				idPerSrc))
+	}
+	if requestRate > 0 {
+		idTotal := id + "_total"
+		dummyBackends = append(dummyBackends, map[string]string{
+			"id":          idTotal,
+			"stick_table": fmt.Sprintf("stick-table type integer size 1 expire 1m store http_req_rate(%ds)", periodSecond),
+		})
+		rateRules = append(rateRules,
+			fmt.Sprintf("http-request deny deny_status 429 if { int(1),table_http_req_rate(%s) gt %d }",
+				idTotal, requestRate*periodSecond),
+			fmt.Sprintf("http-request track-sc1 int(1) table %s",
+				idTotal))
+	}
+	data["rate_rules"] = rateRules
+	data["dummy_backends"] = dummyBackends
+	return nil
+}
+
 func (b *LoadbalancerCorpus) genHaproxyConfigHttp(buf *bytes.Buffer, listener *LoadbalancerListener, opts *AgentParams) error {
 	lb := listener.loadbalancer
 	rules := listener.rules.OrderedEnabledList()
@@ -366,8 +403,10 @@ func (b *LoadbalancerCorpus) genHaproxyConfigHttp(buf *bytes.Buffer, listener *L
 					backendGroup.Name, backendGroup.Id),
 				"id": ruleBackendIdGen(rule.Id),
 			}
-			err := b.genHaproxyConfigBackend(backendData, lb, listener, backendGroup)
-			if err != nil {
+			if err := b.genHaproxyConfigBackend(backendData, lb, listener, backendGroup); err != nil {
+				return err
+			}
+			if err := b.genHaproxyConfigHttpRate(backendData, rule.HTTPRequestRate, rule.HTTPRequestRatePerSrc); err != nil {
 				return err
 			}
 			backends = append(backends, backendData)
@@ -381,8 +420,10 @@ func (b *LoadbalancerCorpus) genHaproxyConfigHttp(buf *bytes.Buffer, listener *L
 					backendGroup.Name, backendGroup.Id),
 				"id": fmt.Sprintf("backends_listener_default-%s", listener.Id),
 			}
-			err := b.genHaproxyConfigBackend(backendData, lb, listener, backendGroup)
-			if err != nil {
+			if err := b.genHaproxyConfigBackend(backendData, lb, listener, backendGroup); err != nil {
+				return err
+			}
+			if err := b.genHaproxyConfigHttpRate(backendData, listener.HTTPRequestRate, listener.HTTPRequestRatePerSrc); err != nil {
 				return err
 			}
 			backends = append(backends, backendData)
@@ -436,12 +477,17 @@ listen {{ .id }}
 
 {{ define "httpListen" -}}
 # {{ .listener_type }} listener: {{ .comment }}
+{{- range .dummy_backends }}
+backend {{ .id }}
+	{{ println .stick_table }}
+{{- end }}
 frontend {{ .id }}
 	bind {{ .bind }}
 	mode http
 	{{- println }}
 	{{- if .log }}	{{ println "option httplog clf" }} {{- end }}
 	{{- if .acl }}	{{ println .acl }} {{- end}}
+	{{- range .rate_rules }}	{{ println . }} {{- end }}
 	{{- if .client_request_timeout }}	timeout http-request {{ println .client_request_timeout }} {{- end}}
 	{{- if .client_idle_timeout }}	timeout http-keep-alive {{ println .client_idle_timeout }} {{- end}}
 	{{- if .xforwardedfor }}	{{ println "option forwardfor" }} {{- end}}
@@ -455,10 +501,15 @@ frontend {{ .id }}
 
 {{ define "backend" -}}
 # {{ .comment }}
+{{- range .dummy_backends }}
+backend {{ .id }}
+	{{ println .stick_table }}
+{{- end }}
 backend {{ .id }}
 	mode {{ .mode }}
 	balance {{ .balanceAlgorithm }}
 	{{- println }}
+	{{- range .rate_rules }}	{{ println . }} {{- end }}
 	{{- if .backend_connect_timeout }}	timeout connect {{ println .backend_connect_timeout }} {{- end}}
 	{{- if .backend_idle_timeout }}	timeout server {{ println .backend_idle_timeout }} {{- end}}
 	{{- if .timeout_check }}	{{ println .timeout_check }} {{- end }}

pkg/mcclient/models/loadbalancers.go

@@ -25,10 +25,8 @@ type LoadbalancerHTTPListener struct {
 	StickySessionCookie        string
 	StickySessionCookieTimeout int
 
-	XForwardedFor      bool
-	XForwardedForSLBIP bool
-	XForwardedForSLBID bool
-	Gzip               bool
+	XForwardedFor bool
+	Gzip          bool
 }
 
 // CACertificate string
@@ -38,11 +36,15 @@ type LoadbalancerHTTPSListener struct {
 	EnableHttp2     bool
 }
 
+type LoadbalancerHTTPRateLimiter struct {
+	HTTPRequestRate       int
+	HTTPRequestRatePerSrc int
+}
+
 type LoadbalancerListener struct {
 	VirtualResource
 
 	LoadbalancerId string
-	Bandwidth      int
 	ListenerType   string
 	ListenerPort   int
 
@@ -79,8 +81,7 @@ type LoadbalancerListener struct {
 	LoadbalancerHTTPListener
 	LoadbalancerHTTPSListener
 
-	XForwardedFor bool
-	Gzip          bool
+	LoadbalancerHTTPRateLimiter
 }
 
 type LoadbalancerListenerRule struct {
@@ -91,6 +92,8 @@ type LoadbalancerListenerRule struct {
 
 	Domain string
 	Path   string
+
+	LoadbalancerHTTPRateLimiter
 }
 
 type LoadbalancerBackendGroup struct {

pkg/mcclient/modules/mod_bill_details.go

@@ -0,0 +1,14 @@
+package modules
+
+var (
+	BillDetails ResourceManager
+)
+
+func init() {
+	BillDetails = NewMeterManager("bill_detail", "bill_details",
+		[]string{"bill_id", "account", "platform", "region", "manager_project", "res_id",
+			"res_type", "res_name", "start_time", "end_time", "charge_type", "item_rate", "item_fee"},
+		[]string{},
+	)
+	register(&BillDetails)
+}

pkg/mcclient/modules/mod_bill_resources.go

@@ -0,0 +1,14 @@
+package modules
+
+var (
+	BillResources ResourceManager
+)
+
+func init() {
+	BillResources = NewMeterManager("bill_resource", "bill_resources",
+		[]string{"account", "platform", "region", "manager_project", "res_id",
+			"res_type", "res_name", "charge_type", "res_fee"},
+		[]string{},
+	)
+	register(&BillResources)
+}

pkg/mcclient/modules/mod_projects.go

@@ -242,6 +242,7 @@ func (this *ProjectManagerV3) DeleteInContexts(session *mcclient.ClientSession,
 	if ctxs == nil {
 		p := jsonutils.NewDict()
 		p.Add(jsonutils.JSONTrue, "admin")
+		p.Add(jsonutils.JSONTrue, "system")
 		p.Add(jsonutils.NewString(id), "tenant")
 		ret, e := Servers.List(session, p)
 		if e != nil {

pkg/mcclient/options/loadbalancerlisteners.go

@@ -9,7 +9,6 @@ type LoadbalancerListenerCreateOptions struct {
 	BackendGroup string
 
 	Scheduler string `required:"true" choices:"rr|wrr|wlc|sch|tch"`
-	Bandwidth *int
 
 	ClientRequestTimeout  *int
 	ClientIdleTimeout     *int
@@ -46,6 +45,9 @@ type LoadbalancerListenerCreateOptions struct {
 	Certificate     string
 	TLSCipherPolicy string
 	EnableHttp2     string `choices:"true|false"`
+
+	HTTPRequestRate       *int
+	HTTPRequestRatePerSrc *int
 }
 
 type LoadbalancerListenerListOptions struct {
@@ -57,7 +59,6 @@ type LoadbalancerListenerListOptions struct {
 	BackendGroup string
 
 	Scheduler string `choices:"rr|wrr|wlc|sch|tch"`
-	Bandwidth *int
 
 	ClientRequestTimeout  *int
 	ClientIdleTimeout     *int
@@ -94,6 +95,9 @@ type LoadbalancerListenerListOptions struct {
 	Certificate     string
 	TLSCipherPolicy string
 	EnableHttp2     string `choices:"true|false"`
+
+	HTTPRequestRate       *int
+	HTTPRequestRatePerSrc *int
 }
 
 type LoadbalancerListenerUpdateOptions struct {
@@ -103,7 +107,6 @@ type LoadbalancerListenerUpdateOptions struct {
 	BackendGroup string
 
 	Scheduler string `choices:"rr|wrr|wlc|sch|tch"`
-	Bandwidth *int
 
 	ClientRequestTimeout  *int
 	ClientIdleTimeout     *int
@@ -140,6 +143,9 @@ type LoadbalancerListenerUpdateOptions struct {
 	Certificate     string
 	TLSCipherPolicy string
 	EnableHttp2     string `choices:"true|false"`
+
+	HTTPRequestRate       *int
+	HTTPRequestRatePerSrc *int
 }
 
 type LoadbalancerListenerGetOptions struct {

pkg/util/aliyun/aliyun.go

@@ -10,6 +10,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/pkg/utils"
 )
 
 const (
@@ -70,8 +71,12 @@ func _jsonRequest(client *sdk.Client, domain string, version string, apiName str
 		return nil, err
 	}
 	//{"Code":"InvalidInstanceType.ValueNotSupported","HostId":"ecs.aliyuncs.com","Message":"The specified instanceType beyond the permitted range.","RequestId":"0042EE30-0EDF-48A7-A414-56229D4AD532"}
+	//{"Code":"200","Message":"successful","PageNumber":1,"PageSize":50,"RequestId":"BB4C970C-0E23-48DC-A3B0-EB21FFC70A29","RouterTableList":{"RouterTableListType":[{"CreationTime":"2017-03-19T13:37:40Z","Description":"","ResourceGroupId":"rg-acfmwie3cqoobmi","RouteTableId":"vtb-j6c60lectdi80rk5xz43g","RouteTableName":"","RouteTableType":"System","RouterId":"vrt-j6c00qrol733dg36iq4qj","RouterType":"VRouter","VSwitchIds":{"VSwitchId":["vsw-j6c3gig5ub4fmi2veyrus"]},"VpcId":"vpc-j6c86z3sh8ufhgsxwme0q"}]},"Success":true,"TotalCount":1}
 	if body.Contains("Code") {
-		return nil, fmt.Errorf(body.String())
+		code, _ := body.GetString("Code")
+		if len(code) > 0 && !utils.IsInStringArray(code, []string{"200"}) {
+			return nil, fmt.Errorf(body.String())
+		}
 	}
 	return body, nil
 }