add: openstack loadbalancer (#7497)

Co-authored-by: lvyangyang <lvyangyang@yunion.cn>

pkg/multicloud/openstack/eip.go

@@ -304,6 +304,16 @@ func (region *SRegion) AssociateEip(instanceId, eipId string) error {
 	return fmt.Errorf("failed to found instnace %s nics for binding eip", instanceId)
 }
 
+func (region *SRegion) AssociateEipWithPortId(portid, eipId string) error {
+	params := map[string]map[string]string{
+		"floatingip": {
+			"port_id": portid,
+		},
+	}
+	_, err := region.vpcUpdate("/v2.0/floatingips/"+eipId, jsonutils.Marshal(params))
+	return err
+}
+
 func (region *SRegion) DisassociateEip(eipId string) error {
 	params, _ := jsonutils.Parse([]byte(`{
 		"floatingip": {

pkg/multicloud/openstack/loadbalabcerpool.go

@@ -0,0 +1,578 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openstack
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+type SLoadbalancerPoolCreateParams struct {
+	LbAlgorithm        string               `json:"lb_algorithm,omitempty"`
+	Protocol           string               `json:"protocol,omitempty"`
+	Description        string               `json:"description,omitempty"`
+	AdminStateUp       bool                 `json:"admin_state_up,omitempty"`
+	SessionPersistence *SSessionPersistence `json:"session_persistence"`
+	LoadbalancerID     string               `json:"loadbalancer_id,omitempty"`
+	ListenerID         string               `json:"listener_id,omitempty"`
+	Name               string               `json:"name,omitempty"`
+	Tags               []string             `json:"tags,omitempty"`
+	TLSContainerRef    string               `json:"tls_container_ref,omitempty"`
+	CaTLSContainerRef  string               `json:"ca_tls_container_ref,omitempty"`
+	CrlContainerRef    string               `json:"crl_container_ref,omitempty"`
+	TLSEnabled         *bool                `json:"tls_enabled,omitempty"`
+	TLSCiphers         string               `json:"tls_ciphers,omitempty"`
+	TLSVersions        []string             `json:"tls_versions,omitempty"`
+}
+
+type SLoadbalancerPoolUpdateParams struct {
+	LbAlgorithm        string               `json:"lb_algorithm,omitempty"`
+	Description        string               `json:"description,omitempty"`
+	AdminStateUp       bool                 `json:"admin_state_up,omitempty"`
+	SessionPersistence *SSessionPersistence `json:"session_persistence"`
+	Name               string               `json:"name,omitempty"`
+	Tags               []string             `json:"tags,omitempty"`
+	TLSContainerRef    string               `json:"tls_container_ref,omitempty"`
+	CaTLSContainerRef  string               `json:"ca_tls_container_ref,omitempty"`
+	CrlContainerRef    string               `json:"crl_container_ref,omitempty"`
+	TLSEnabled         *bool                `json:"tls_enabled,omitempty"`
+	TLSCiphers         string               `json:"tls_ciphers,omitempty"`
+	TLSVersions        []string             `json:"tls_versions,omitempty"`
+}
+
+type SSessionPersistence struct {
+	CookieName string `json:"cookie_name,omitempty"`
+	Type       string `json:"type,omitempty"`
+}
+
+type SLoadbalancerPool struct {
+	region             *SRegion
+	members            []SLoadbalancerMember
+	healthmonitor      *SLoadbalancerHealthmonitor
+	LbAlgorithm        string              `json:"lb_algorithm"`
+	Protocol           string              `json:"protocol"`
+	Description        string              `json:"description"`
+	AdminStateUp       bool                `json:"admin_state_up"`
+	LoadbalancerIds    []SLoadbalancerID   `json:"loadbalancers"`
+	CreatedAt          string              `json:"created_at"`
+	ProvisioningStatus string              `json:"provisioning_status"`
+	UpdatedAt          string              `json:"updated_at"`
+	SessionPersistence SSessionPersistence `json:"session_persistence"`
+	ListenerIds        []SListenerID       `json:"listeners"`
+	MemberIds          []SMemberID         `json:"members"`
+	HealthmonitorID    string              `json:"healthmonitor_id"`
+	ProjectID          string              `json:"project_id"`
+	ID                 string              `json:"id"`
+	OperatingStatus    string              `json:"operating_status"`
+	Name               string              `json:"name"`
+	Tags               []string            `json:"tags"`
+	TLSContainerRef    string              `json:"tls_container_ref"`
+	CaTLSContainerRef  string              `json:"ca_tls_container_ref"`
+	CrlContainerRef    string              `json:"crl_container_ref"`
+	TLSEnabled         bool                `json:"tls_enabled"`
+	TLSCiphers         string              `json:"tls_ciphers"`
+	TLSVersions        []string            `json:"tls_versions"`
+}
+
+func ToOpenstackHealthCheckHttpCode(c string) string {
+	c = strings.TrimSpace(c)
+	segs := strings.Split(c, ",")
+	ret := []string{}
+	for _, seg := range segs {
+		seg = strings.TrimLeft(seg, "http_")
+		seg = strings.TrimSpace(seg)
+		seg = strings.Replace(seg, "xx", "00", -1)
+		ret = append(ret, seg)
+	}
+
+	return strings.Join(ret, ",")
+}
+
+func ToOnecloudHealthCheckHttpCode(c string) string {
+	c = strings.TrimSpace(c)
+	segs := strings.Split(c, ",")
+	ret := []string{}
+	for _, seg := range segs {
+		seg = strings.TrimSpace(seg)
+		seg = strings.Replace(seg, "00", "xx", -1)
+		seg = "http_" + seg
+		ret = append(ret, seg)
+	}
+
+	return strings.Join(ret, ",")
+}
+
+func (pool *SLoadbalancerPool) GetILoadbalancer() cloudprovider.ICloudLoadbalancer {
+	if len(pool.LoadbalancerIds) != 1 {
+		return nil
+	}
+	loadbalancer, err := pool.region.GetLoadbalancerbyId(pool.LoadbalancerIds[0].ID)
+	if err != nil {
+		return nil
+	}
+	return loadbalancer
+}
+
+func (pool *SLoadbalancerPool) GetLoadbalancerId() string {
+	if len(pool.LoadbalancerIds) != 1 {
+		return ""
+	}
+	return pool.LoadbalancerIds[0].ID
+}
+
+func (pool *SLoadbalancerPool) GetProtocolType() string {
+	switch pool.Protocol {
+	case "TCP":
+		return api.LB_LISTENER_TYPE_TCP
+	case "UDP":
+		return api.LB_LISTENER_TYPE_UDP
+	case "HTTP":
+		return api.LB_LISTENER_TYPE_HTTP
+	default:
+		return ""
+	}
+}
+
+func (pool *SLoadbalancerPool) GetScheduler() string {
+	switch pool.LbAlgorithm {
+	case "LEAST_CONNECTIONS":
+		return api.LB_SCHEDULER_WLC
+	case "ROUND_ROBIN":
+		return api.LB_SCHEDULER_WRR
+	case "SOURCE_IP":
+		return api.LB_SCHEDULER_SCH
+	case "SOURCE_IP_PORT":
+		return api.LB_SCHEDULER_TCH
+	default:
+		return ""
+	}
+}
+
+func (pool *SLoadbalancerPool) GetHealthCheck() (*cloudprovider.SLoadbalancerHealthCheck, error) {
+	healthCheck := cloudprovider.SLoadbalancerHealthCheck{}
+	healthCheck.HealthCheckDomain = pool.healthmonitor.DomainName
+	healthCheck.HealthCheckHttpCode = ToOnecloudHealthCheckHttpCode(pool.healthmonitor.ExpectedCodes)
+	healthCheck.HealthCheckInterval = pool.healthmonitor.Delay
+	healthCheck.HealthCheckRise = pool.healthmonitor.MaxRetries
+	healthCheck.HealthCheckFail = pool.healthmonitor.MaxRetriesDown
+	healthCheck.HealthCheckTimeout = pool.healthmonitor.Timeout
+	switch pool.healthmonitor.Type {
+	case "HTTP":
+		healthCheck.HealthCheckType = api.LB_HEALTH_CHECK_HTTP
+	case "HTTPS":
+		healthCheck.HealthCheckType = api.LB_HEALTH_CHECK_HTTPS
+	case "TCP":
+		healthCheck.HealthCheckType = api.LB_HEALTH_CHECK_TCP
+	case "UDP-CONNECT":
+		healthCheck.HealthCheckType = api.LB_HEALTH_CHECK_UDP
+	default:
+		healthCheck.HealthCheckType = ""
+	}
+	healthCheck.HealthCheckURI = pool.healthmonitor.URLPath
+	return &healthCheck, nil
+}
+
+func (pool *SLoadbalancerPool) GetStickySession() (*cloudprovider.SLoadbalancerStickySession, error) {
+	if len(pool.SessionPersistence.Type) == 0 {
+		return nil, nil
+	}
+
+	var stickySessionType string
+	switch pool.SessionPersistence.Type {
+	case "SOURCE_IP":
+		stickySessionType = api.LB_STICKY_SESSION_TYPE_INSERT
+	case "HTTP_COOKIE":
+		stickySessionType = api.LB_STICKY_SESSION_TYPE_INSERT
+	case "APP_COOKIE":
+		stickySessionType = api.LB_STICKY_SESSION_TYPE_SERVER
+	}
+
+	ret := cloudprovider.SLoadbalancerStickySession{
+		StickySession:              api.LB_BOOL_ON,
+		StickySessionCookie:        pool.SessionPersistence.CookieName,
+		StickySessionType:          stickySessionType,
+		StickySessionCookieTimeout: 0,
+	}
+
+	return &ret, nil
+}
+
+func (pool *SLoadbalancerPool) GetName() string {
+	return pool.Name
+}
+
+func (pool *SLoadbalancerPool) GetId() string {
+	return pool.ID
+}
+
+func (pool *SLoadbalancerPool) GetGlobalId() string {
+	return pool.ID
+}
+
+func (pool *SLoadbalancerPool) GetStatus() string {
+	switch pool.ProvisioningStatus {
+	case "ACTIVE":
+		return api.LB_STATUS_ENABLED
+	case "PENDING_CREATE":
+		return api.LB_CREATING
+	case "PENDING_UPDATE":
+		return api.LB_SYNC_CONF
+	case "PENDING_DELETE":
+		return api.LB_STATUS_DELETING
+	case "DELETED":
+		return api.LB_STATUS_DELETED
+	default:
+		return api.LB_STATUS_UNKNOWN
+	}
+}
+
+func (pool *SLoadbalancerPool) IsDefault() bool {
+	return false
+}
+
+func (pool *SLoadbalancerPool) GetType() string {
+	return api.LB_BACKENDGROUP_TYPE_NORMAL
+}
+
+func (pool *SLoadbalancerPool) GetMetadata() *jsonutils.JSONDict {
+	return nil
+}
+
+func (pool *SLoadbalancerPool) IsEmulated() bool {
+	return false
+}
+
+func (region *SRegion) GetLoadbalancerPools() ([]SLoadbalancerPool, error) {
+	pools := []SLoadbalancerPool{}
+	resource := "/v2/lbaas/pools"
+	query := url.Values{}
+	for {
+		resp, err := region.lbList(resource, query)
+		if err != nil {
+			return nil, errors.Wrap(err, "lbList")
+		}
+		part := struct {
+			Pools      []SLoadbalancerPool
+			PoolsLinks SNextLinks
+		}{}
+		err = resp.Unmarshal(&part)
+		if err != nil {
+			return nil, errors.Wrap(err, "resp.Unmarshal")
+		}
+		pools = append(pools, part.Pools...)
+		marker := part.PoolsLinks.GetNextMark()
+		if len(marker) == 0 {
+			break
+		}
+		query.Set("marker", marker)
+	}
+
+	for i := 0; i < len(pools); i++ {
+		pools[i].region = region
+		err := pools[i].fetchLoadbalancerHealthmonitor()
+		if err != nil {
+			return nil, errors.Wrapf(err, "pools[%d].fetchLoadbalancerHealthmonitor()", i)
+		}
+
+	}
+	return pools, nil
+}
+
+func (region *SRegion) GetLoadbalancerPoolById(poolId string) (*SLoadbalancerPool, error) {
+	body, err := region.lbGet(fmt.Sprintf("/v2/lbaas/pools/%s", poolId))
+	if err != nil {
+		return nil, errors.Wrapf(err, "region.lbGet(fmt.Sprintf(/v2/lbaas/pools/%s)", poolId)
+	}
+	pool := SLoadbalancerPool{}
+	err = body.Unmarshal(&pool, "pool")
+	if err != nil {
+		return nil, errors.Wrap(err, "body.Unmarshal")
+	}
+	pool.region = region
+	err = pool.fetchLoadbalancermembers()
+	if err != nil {
+		return nil, errors.Wrap(err, "pool.fetchLoadbalancermembers()")
+	}
+	err = pool.fetchLoadbalancerHealthmonitor()
+	if err != nil {
+		return nil, errors.Wrap(err, "pool.fetchLoadbalancerHealthmonitor()")
+	}
+	return &pool, nil
+}
+
+func (region *SRegion) CreateLoadbalancerPool(group *cloudprovider.SLoadbalancerBackendGroup) (*SLoadbalancerPool, error) {
+	type CreateParams struct {
+		Pool SLoadbalancerPoolCreateParams `json:"pool"`
+	}
+	params := CreateParams{}
+	params.Pool.AdminStateUp = true
+	params.Pool.LbAlgorithm = LB_ALGORITHM_MAP[group.Scheduler]
+	params.Pool.Name = group.Name
+	params.Pool.LoadbalancerID = group.LoadbalancerID
+	// 绑定规则时不能指定listener
+	params.Pool.ListenerID = group.ListenerID
+	params.Pool.Protocol = LB_PROTOCOL_MAP[group.ListenType]
+	params.Pool.SessionPersistence = nil
+	if group.StickySession != nil {
+		session := SSessionPersistence{}
+		session.Type = LB_STICKY_SESSION_MAP[group.StickySession.StickySessionType]
+		if session.Type == "APP_COOKIE" {
+			session.CookieName = group.StickySession.StickySessionCookie
+		}
+		params.Pool.SessionPersistence = &session
+	}
+	body, err := region.lbPost("/v2/lbaas/pools", jsonutils.Marshal(params))
+	if err != nil {
+		return nil, errors.Wrap(err, "region.lbPost(/v2/lbaas/pools)")
+	}
+	spool := SLoadbalancerPool{}
+	spool.region = region
+	err = body.Unmarshal(&spool, "pool")
+	if err != nil {
+		return nil, errors.Wrap(err, "body.Unmarshal(&spool, pool)")
+	}
+	return &spool, nil
+}
+
+func (pool *SLoadbalancerPool) Refresh() error {
+	newPool, err := pool.region.GetLoadbalancerPoolById(pool.ID)
+	if err != nil {
+		return err
+	}
+	return jsonutils.Update(pool, newPool)
+}
+
+func (pool *SLoadbalancerPool) fetchLoadbalancermembers() error {
+	if len(pool.MemberIds) < 1 {
+		return nil
+	}
+	members, err := pool.region.GetLoadbalancerMenbers(pool.ID)
+	if err != nil {
+		return err
+	}
+	pool.members = members
+	return nil
+}
+
+func (pool *SLoadbalancerPool) fetchLoadbalancerHealthmonitor() error {
+	if len(pool.HealthmonitorID) < 1 {
+		return nil
+	}
+	healthmonitor, err := pool.region.GetLoadbalancerHealthmonitorById(pool.HealthmonitorID)
+	if err != nil {
+		return errors.Wrap(err, "pool.region.GetLoadbalancerHealthmonitorById")
+	}
+	pool.healthmonitor = healthmonitor
+	return nil
+}
+
+func (pool *SLoadbalancerPool) GetILoadbalancerBackends() ([]cloudprovider.ICloudLoadbalancerBackend, error) {
+	ibackends := []cloudprovider.ICloudLoadbalancerBackend{}
+	for i := 0; i < len(pool.members); i++ {
+		ibackends = append(ibackends, &pool.members[i])
+	}
+	return ibackends, nil
+}
+
+func (pool *SLoadbalancerPool) GetILoadbalancerBackendById(memberId string) (cloudprovider.ICloudLoadbalancerBackend, error) {
+	for i := 0; i < len(pool.members); i++ {
+		if pool.members[i].ID == memberId {
+			return &pool.members[i], nil
+		}
+	}
+	return nil, errors.Wrapf(cloudprovider.ErrNotFound, "GetILoadbalancerBackendById(%s)", memberId)
+}
+
+func (region *SRegion) UpdateLoadBalancerPool(poolId string, group *cloudprovider.SLoadbalancerBackendGroup) error {
+	type UpdateParams struct {
+		Pool SLoadbalancerPoolUpdateParams `json:"pool"`
+	}
+	params := UpdateParams{}
+	params.Pool.AdminStateUp = true
+	params.Pool.LbAlgorithm = LB_ALGORITHM_MAP[group.Scheduler]
+	params.Pool.Name = group.Name
+	if group.StickySession != nil {
+		session := SSessionPersistence{}
+		session.Type = LB_STICKY_SESSION_MAP[group.StickySession.StickySessionType]
+		if session.Type == "APP_COOKIE" {
+			session.CookieName = group.StickySession.StickySessionCookie
+		}
+		params.Pool.SessionPersistence = &session
+	}
+	_, err := region.lbUpdate(fmt.Sprintf("/v2/lbaas/pools/%s", poolId), jsonutils.Marshal(params))
+	if err != nil {
+		return errors.Wrapf(err, `region.lbUpdate(fmt.Sprintf("/v2/lbaas/pools/%s", poolId), jsonutils.Marshal(params))`)
+	}
+	return nil
+}
+
+func (pool *SLoadbalancerPool) Sync(ctx context.Context, group *cloudprovider.SLoadbalancerBackendGroup) error {
+	lb, err := pool.region.GetLoadbalancerbyId(pool.GetLoadbalancerId())
+	if err != nil {
+		return errors.Wrap(err, "pool.region.GetLoadbalancerbyId(pool.GetLoadbalancerId())")
+	}
+	// ensure loadbalancer status
+	err = waitLbResStatus(lb, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, `waitLbResStatus(lb, 10*time.Second, 8*time.Minute)`)
+	}
+	// ensure pool status
+	err = waitLbResStatus(pool, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, `waitLbResStatus(pool, 10*time.Second, 8*time.Minute)`)
+	}
+	// sync healthmonitor
+	healthmonitor := SLoadbalancerHealthmonitor{}
+	if len(pool.HealthmonitorID) > 0 {
+		oldhealthmonitor, err := pool.region.GetLoadbalancerHealthmonitorById(pool.HealthmonitorID)
+		if err != nil {
+			return errors.Wrap(err, "pool.region.GetLoadbalancerHealthmonitorById(pool.HealthmonitorID)")
+		}
+		// 不能更新健康检查类型，需要删除重建
+		var sHealthCheckType string
+		switch oldhealthmonitor.Type {
+		case "HTTP":
+			sHealthCheckType = api.LB_HEALTH_CHECK_HTTP
+		case "HTTPS":
+			sHealthCheckType = api.LB_HEALTH_CHECK_HTTPS
+		case "TCP":
+			sHealthCheckType = api.LB_HEALTH_CHECK_TCP
+		case "UDP-CONNECT":
+			sHealthCheckType = api.LB_HEALTH_CHECK_UDP
+		default:
+			sHealthCheckType = ""
+		}
+
+		if sHealthCheckType != group.HealthCheck.HealthCheckType {
+			err := pool.region.DeleteLoadbalancerHealthmonitor(pool.HealthmonitorID)
+			if err != nil {
+				return errors.Wrapf(err, "pool.region.DeleteLoadbalancerHealthmonitor(%s)", pool.HealthmonitorID)
+			}
+			// 等待删除结束
+			err = waitLbResStatus(lb, 10*time.Second, 8*time.Minute)
+			if err != nil {
+				return errors.Wrap(err, `waitLbResStatus(lb, 10*time.Second, 8*time.Minute)`)
+			}
+
+			newhealthmonitor, err := pool.region.CreateLoadbalancerHealthmonitor(pool.ID, group.HealthCheck)
+			if err != nil {
+				return errors.Wrapf(err, "pool.region.CreateLoadbalancerHealthmonitor(%s,group.HealthCheck)", pool.ID)
+			}
+			healthmonitor = *newhealthmonitor
+		} else {
+			// ensure healthmonitor status
+			err = waitLbResStatus(oldhealthmonitor, 10*time.Second, 8*time.Minute)
+			if err != nil {
+				return errors.Wrap(err, `waitLbResStatus(oldhealthmonitor, 10*time.Second, 8*time.Minute)`)
+			}
+			oldhealthmonitor, err = pool.region.UpdateLoadbalancerHealthmonitor(pool.HealthmonitorID, group.HealthCheck)
+			if err != nil {
+				return errors.Wrapf(err, `pool.region.UpdateLoadbalancerHealthmonitor(%s, group.HealthCheck)`, pool.HealthmonitorID)
+			}
+			healthmonitor = *oldhealthmonitor
+		}
+	} else {
+		newhealthmonitor, err := pool.region.CreateLoadbalancerHealthmonitor(pool.ID, group.HealthCheck)
+		if err != nil {
+			return errors.Wrapf(err, "pool.region.CreateLoadbalancerHealthmonitor(%s, group.HealthCheck)", pool.ID)
+		}
+		healthmonitor = *newhealthmonitor
+	}
+
+	// ensure pool status
+	err = waitLbResStatus(pool, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, `waitLbResStatus(pool, 10*time.Second, 8*time.Minute)`)
+	}
+	// sync pool
+	err = pool.region.UpdateLoadBalancerPool(pool.ID, group)
+	if err != nil {
+		return errors.Wrapf(err, `pool.region.UpdateLoadBalancerPool(%s, group)`, pool.ID)
+	}
+
+	// wait healthmonitor status
+	err = waitLbResStatus(&healthmonitor, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, `waitLbResStatus(&healthmonitor, 10*time.Second, 8*time.Minute)`)
+	}
+	// wait pool status
+	err = waitLbResStatus(pool, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, `waitLbResStatus(pool,  10*time.Second, 8*time.Minute)`)
+	}
+	return nil
+}
+
+func (region *SRegion) DeleteLoadBalancerPool(poolId string) error {
+	_, err := region.lbDelete(fmt.Sprintf("/v2/lbaas/pools/%s", poolId))
+	if err != nil {
+		return errors.Wrapf(err, "lbDelete(/v2/lbaas/pools/%s)", poolId)
+	}
+	return nil
+}
+
+func (pool *SLoadbalancerPool) Delete(ctx context.Context) error {
+	lb, err := pool.region.GetLoadbalancerbyId(pool.GetLoadbalancerId())
+	if err != nil {
+		return errors.Wrap(err, "pool.region.GetLoadbalancerbyId(pool.GetLoadbalancerId())")
+	}
+	err = waitLbResStatus(lb, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, `waitLbResStatus(lb, 10*time.Second, 8*time.Minute)`)
+	}
+	return pool.region.DeleteLoadBalancerPool(pool.ID)
+}
+
+func (pool *SLoadbalancerPool) AddBackendServer(serverId string, weight, port int) (cloudprovider.ICloudLoadbalancerBackend, error) {
+	// ensure lb status
+	lb, err := pool.region.GetLoadbalancerbyId(pool.GetLoadbalancerId())
+	if err != nil {
+		return nil, errors.Wrap(err, "pool.region.GetLoadbalancerbyId(pool.GetLoadbalancerId())")
+	}
+	err = waitLbResStatus(lb, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return nil, errors.Wrap(err, `waitLbResStatus(lb, 10*time.Second, 8*time.Minute)`)
+	}
+	smemeber, err := pool.region.CreateLoadbalancerMember(pool.ID, serverId, weight, port)
+	if err != nil {
+		return nil, errors.Wrapf(err, `CreateLoadbalancerMember(%s,%s,%s,%s)`, pool.ID, serverId, weight, port)
+	}
+	err = waitLbResStatus(smemeber, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return nil, errors.Wrap(err, `waitLbResStatus(smemeber,  10*time.Second, 8*time.Minute)`)
+	}
+	smemeber.region = pool.region
+	smemeber.poolID = pool.ID
+	pool.members = append(pool.members, *smemeber)
+	return smemeber, nil
+}
+
+// 不是serverId，是memberId
+func (pool *SLoadbalancerPool) RemoveBackendServer(id string, weight, port int) error {
+	return pool.region.DeleteLoadbalancerMember(pool.ID, id)
+}
+
+func (pool *SLoadbalancerPool) GetProjectId() string {
+	return pool.ProjectID
+}

pkg/multicloud/openstack/loadbalanceracl.go

@@ -0,0 +1,188 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openstack
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+type AclEntrys struct {
+	AclEntry []AclEntry
+}
+
+type AclEntry struct {
+	AclEntryComment string
+	AclEntryIP      string
+}
+
+type SLoadbalancerAcl struct {
+	listener *SLoadbalancerListener
+}
+
+func (acl *SLoadbalancerAcl) GetAclListenerID() string {
+	return acl.listener.ID
+}
+
+func (acl *SLoadbalancerAcl) GetName() string {
+	return acl.listener.Name + "AllowedCidrs"
+}
+
+func (acl *SLoadbalancerAcl) GetId() string {
+	return acl.listener.ID
+}
+
+func (acl *SLoadbalancerAcl) GetGlobalId() string {
+	return acl.listener.ID
+}
+
+func (acl *SLoadbalancerAcl) GetStatus() string {
+	return ""
+}
+
+func (acl *SLoadbalancerAcl) GetMetadata() *jsonutils.JSONDict {
+	return nil
+}
+
+func (acl *SLoadbalancerAcl) IsEmulated() bool {
+	return false
+}
+
+func (acl *SLoadbalancerAcl) Refresh() error {
+	return acl.listener.Refresh()
+}
+
+func (acl *SLoadbalancerAcl) GetAclEntries() []cloudprovider.SLoadbalancerAccessControlListEntry {
+	aclEntrys := []cloudprovider.SLoadbalancerAccessControlListEntry{}
+	for i := 0; i < len(acl.listener.AllowedCidrs); i++ {
+		aclEntry := cloudprovider.SLoadbalancerAccessControlListEntry{}
+		aclEntry.CIDR = acl.listener.AllowedCidrs[i]
+		aclEntry.Comment = "AllowedCidr"
+		aclEntrys = append(aclEntrys, aclEntry)
+	}
+	return aclEntrys
+}
+
+func (region *SRegion) UpdateLoadbalancerListenerAllowedCidrs(listenerId string, cidrs []string) error {
+	params := jsonutils.NewDict()
+	listenerParam := jsonutils.NewDict()
+	listenerParam.Add(jsonutils.NewStringArray(cidrs), "allowed_cidrs")
+	params.Add(listenerParam, "listener")
+	_, err := region.lbUpdate(fmt.Sprintf("/v2/lbaas/listeners/%s", listenerId), params)
+	if err != nil {
+		return errors.Wrapf(err, `region.lbUpdate(/v2/lbaas/listeners/%s, params)`, listenerId)
+	}
+	return nil
+}
+
+func (acl *SLoadbalancerAcl) Delete() error {
+	// ensure listener status
+	err := waitLbResStatus(acl.listener, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, `waitLbResStatus(acl.listener, 10*time.Second, 8*time.Minute)`)
+	}
+	err = acl.listener.region.UpdateLoadbalancerListenerAllowedCidrs(acl.listener.ID, []string{})
+	if err != nil {
+		return errors.Wrap(err, `acl.listener.region.UpdateLoadbalancerListenerAllowedCidrs(acl.listener.ID, []string{})`)
+	}
+	err = waitLbResStatus(acl.listener, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, `waitLbResStatus(acl.listener, 10*time.Second, 8*time.Minute)`)
+	}
+	return nil
+}
+
+func (region *SRegion) GetLoadbalancerAclDetail(aclId string) (*SLoadbalancerAcl, error) {
+	listener, err := region.GetLoadbalancerListenerbyId(aclId)
+	if err != nil {
+		return nil, errors.Wrapf(err, "region.GetLoadbalancerListenerbyId(s)", aclId)
+	}
+	acl := SLoadbalancerAcl{}
+	acl.listener = listener
+	return &acl, nil
+}
+
+func (region *SRegion) GetLoadBalancerAcls() ([]SLoadbalancerAcl, error) {
+	listeners, err := region.GetLoadbalancerListeners()
+	if err != nil {
+		return nil, errors.Wrap(err, "region.GetLoadbalancerListeners()")
+	}
+	acls := []SLoadbalancerAcl{}
+	for i := 0; i < len(listeners); i++ {
+		if len(listeners[i].AllowedCidrs) < 1 {
+			continue
+		}
+		acl := new(SLoadbalancerAcl)
+		acl.listener = &listeners[i]
+		acls = append(acls, *acl)
+
+	}
+	return acls, nil
+}
+
+func (region *SRegion) CreateLoadBalancerAcl(acl *cloudprovider.SLoadbalancerAccessControlList) (*SLoadbalancerAcl, error) {
+	if !acl.AccessControlEnable {
+		return nil, errors.Wrap(fmt.Errorf("only support allowed cidrs"), "CreateLoadBalancerAcl")
+	}
+	if len(acl.ListenerId) < 1 {
+		return nil, errors.Wrap(fmt.Errorf("loadbalanceracl must band to a loadbalancerlistener"), "CreateLoadBalancerAcl")
+	}
+	cidrs := []string{}
+	for i := 0; i < len(acl.Entrys); i++ {
+		cidrs = append(cidrs, acl.Entrys[i].CIDR)
+	}
+	sacl, err := region.GetLoadbalancerAclDetail(acl.ListenerId)
+	if err != nil {
+		return nil, errors.Wrapf(err, "region.GetLoadbalancerAclDetail(s)", acl.ListenerId)
+	}
+	// ensure listener status
+	err = waitLbResStatus(sacl.listener, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return nil, errors.Wrap(err, `waitLbResStatus(sacl.listener, 10*time.Second, 8*time.Minute)`)
+	}
+	return sacl, region.UpdateLoadbalancerListenerAllowedCidrs(acl.ListenerId, cidrs)
+}
+
+func (acl *SLoadbalancerAcl) Sync(_acl *cloudprovider.SLoadbalancerAccessControlList) error {
+	// ensure listener status
+	err := waitLbResStatus(acl.listener, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, "waitLbResStatus(acl.listener, 10*time.Second, 8*time.Minute)")
+	}
+
+	cidrs := []string{}
+	for i := 0; i < len(_acl.Entrys); i++ {
+		cidrs = append(cidrs, _acl.Entrys[i].CIDR)
+	}
+	err = acl.listener.region.UpdateLoadbalancerListenerAllowedCidrs(acl.listener.ID, cidrs)
+	if err != nil {
+		return errors.Wrapf(err, "UpdateLoadbalancerListenerAllowedCidrs(%s, cidrs)", acl.listener.ID)
+	}
+	err = waitLbResStatus(acl.listener, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, "waitLbResStatus(acl.listener, 10*time.Second, 8*time.Minute)")
+	}
+	return nil
+}
+
+func (acl *SLoadbalancerAcl) GetProjectId() string {
+	return acl.listener.ProjectID
+}

pkg/multicloud/openstack/loadbalancerl7policy.go

@@ -0,0 +1,169 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openstack
+
+import (
+	"fmt"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+type SLoadbalancerL7PolicyCreateParams struct {
+	Description      string   `json:"description,omitempty"`
+	AdminStateUp     bool     `json:"admin_state_up,omitempty"`
+	ListenerID       string   `json:"listener_id,omitempty"`
+	RedirectPrefix   string   `json:"redirect_prefix,omitempty"`
+	RedirectURL      string   `json:"redirect_url,omitempty"`
+	RedirectPoolID   string   `json:"redirect_pool_id,omitempty"`
+	RedirectHTTPCode *int     `json:"redirect_http_code,omitempty"`
+	Name             string   `json:"name,omitempty"`
+	Action           string   `json:"action,omitempty"`
+	Position         *int     `json:"position,omitempty"`
+	Tags             []string `json:"tags,omitempty"`
+}
+
+type SLoadbalancerL7Policy struct {
+	region             *SRegion
+	l7rules            []SLoadbalancerL7Rule
+	ListenerID         string      `json:"listener_id"`
+	Description        string      `json:"description"`
+	AdminStateUp       bool        `json:"admin_state_up"`
+	RuleIds            []SL7RuleID `json:"rules"`
+	CreatedAt          string      `json:"created_at"`
+	ProvisioningStatus string      `json:"provisioning_status"`
+	UpdatedAt          string      `json:"updated_at"`
+	RedirectHTTPCode   int         `json:"redirect_http_code"`
+	RedirectPoolID     string      `json:"redirect_pool_id"`
+	RedirectPrefix     string      `json:"redirect_prefix"`
+	RedirectURL        string      `json:"redirect_url"`
+	Action             string      `json:"action"`
+	Position           int         `json:"position"`
+	ProjectID          string      `json:"project_id"`
+	ID                 string      `json:"id"`
+	OperatingStatus    string      `json:"operating_status"`
+	Name               string      `json:"name"`
+	Tags               []string    `json:"tags"`
+}
+
+func (region *SRegion) GetLoadbalancerL7PolicybyId(policieId string) (*SLoadbalancerL7Policy, error) {
+	body, err := region.lbGet(fmt.Sprintf("/v2/lbaas/l7policies/%s", policieId))
+	if err != nil {
+		return nil, errors.Wrapf(err, `region.lbGet(/v2/lbaas/l7policies/%s)`, policieId)
+	}
+	l7policy := SLoadbalancerL7Policy{}
+	err = body.Unmarshal(&l7policy, "l7policy")
+	if err != nil {
+		return nil, errors.Wrap(err, "body.Unmarshal(&l7policy, policy)")
+	}
+	l7policy.region = region
+	err = l7policy.fetchLoadbalancerL7Rules()
+	if err != nil {
+		return nil, errors.Wrap(err, "l7policy.fetchLoadbalancerL7Rules()")
+	}
+	return &l7policy, nil
+}
+
+func (region *SRegion) CreateLoadbalancerL7Policy(listenerId string, rule *cloudprovider.SLoadbalancerListenerRule) (*SLoadbalancerL7Policy, error) {
+	type Params struct {
+		L7policy SLoadbalancerL7PolicyCreateParams `json:"l7policy"`
+	}
+	l7policyParams := Params{}
+
+	l7policyParams.L7policy.AdminStateUp = true
+	l7policyParams.L7policy.ListenerID = listenerId
+	l7policyParams.L7policy.Name = rule.Name
+
+	if rule.Redirect == api.LB_REDIRECT_RAW {
+		l7policyParams.L7policy.RedirectHTTPCode = &rule.RedirectCode
+		if len(rule.RedirectPath) < 1 {
+			l7policyParams.L7policy.Action = "REDIRECT_PREFIX"
+			l7policyParams.L7policy.RedirectPrefix = rule.RedirectScheme + "://" + rule.RedirectHost
+		} else {
+			l7policyParams.L7policy.Action = "REDIRECT_TO_URL"
+			l7policyParams.L7policy.RedirectURL = rule.RedirectScheme + "://" + rule.RedirectHost + rule.RedirectPath
+		}
+	} else {
+		l7policyParams.L7policy.Action = "REDIRECT_TO_POOL"
+		l7policyParams.L7policy.RedirectPoolID = rule.BackendGroupID
+	}
+
+	body, err := region.lbPost("/v2/lbaas/l7policies", jsonutils.Marshal(l7policyParams))
+	if err != nil {
+		return nil, errors.Wrap(err, `region.lbPost("/v2/lbaas/l7policies", jsonutils.Marshal(l7policyParams))`)
+	}
+	l7policy := SLoadbalancerL7Policy{}
+	l7policy.region = region
+	err = body.Unmarshal(&l7policy, "l7policy")
+	if err != nil {
+		return nil, errors.Wrap(err, "body.Unmarshal(&l7policy, policy)")
+	}
+	return &l7policy, nil
+}
+
+func (region *SRegion) DeleteLoadbalancerListenerL7policy(policyId string) error {
+	_, err := region.lbDelete(fmt.Sprintf("/v2/lbaas/l7policies/%s", policyId))
+	if err != nil {
+		return errors.Wrapf(err, `region.lbDelete(/v2/lbaas/l7policies/%s)`, policyId)
+	}
+	return nil
+}
+
+func (L7policy *SLoadbalancerL7Policy) GetName() string {
+	return L7policy.Name
+}
+
+func (L7policy *SLoadbalancerL7Policy) GetId() string {
+	return L7policy.ID
+}
+
+func (L7policy *SLoadbalancerL7Policy) GetGlobalId() string {
+	return L7policy.ID
+}
+
+func (L7policy *SLoadbalancerL7Policy) GetStatus() string {
+	switch L7policy.ProvisioningStatus {
+	case "ACTIVE":
+		return api.LB_STATUS_ENABLED
+	case "PENDING_CREATE":
+		return api.LB_CREATING
+	case "PENDING_UPDATE":
+		return api.LB_SYNC_CONF
+	case "PENDING_DELETE":
+		return api.LB_STATUS_DELETING
+	case "DELETED":
+		return api.LB_STATUS_DELETED
+	default:
+		return api.LB_STATUS_UNKNOWN
+	}
+}
+
+func (L7policy *SLoadbalancerL7Policy) Refresh() error {
+	newL7policie, err := L7policy.region.GetLoadbalancerL7PolicybyId(L7policy.ID)
+	if err != nil {
+		return err
+	}
+	return jsonutils.Update(L7policy, newL7policie)
+}
+
+func (L7policy *SLoadbalancerL7Policy) IsEmulated() bool {
+	return false
+}
+func (L7policy *SLoadbalancerL7Policy) GetMetadata() *jsonutils.JSONDict {
+	return nil
+}

pkg/multicloud/openstack/loadbalancerl7rule.go

@@ -0,0 +1,196 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openstack
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+type SLoadbalancerL7RuleCreateParams struct {
+	CompareType  string   `json:"compare_type"`
+	Invert       *bool    `json:"invert"`
+	Type         string   `json:"type"`
+	Value        string   `json:"value"`
+	AdminStateUp bool     `json:"admin_state_up"`
+	Tags         []string `json:"tags"`
+}
+
+type SLoadbalancerL7Rule struct {
+	policy             *SLoadbalancerL7Policy
+	CreatedAt          string   `json:"created_at"`
+	CompareType        string   `json:"compare_type"`
+	ProvisioningStatus string   `json:"provisioning_status"`
+	Invert             bool     `json:"invert"`
+	AdminStateUp       bool     `json:"admin_state_up"`
+	UpdatedAt          string   `json:"updated_at"`
+	Value              string   `json:"value"`
+	Key                string   `json:"key"`
+	ProjectID          string   `json:"project_id"`
+	Type               string   `json:"type"`
+	ID                 string   `json:"id"`
+	OperatingStatus    string   `json:"operating_status"`
+	Tags               []string `json:"tags"`
+}
+
+func (region *SRegion) CreateLoadbalancerL7Rule(l7policyId string, rule *cloudprovider.SLoadbalancerListenerRule) (*SLoadbalancerL7Rule, error) {
+	type Params struct {
+		L7Rule SLoadbalancerL7RuleCreateParams `json:"rule"`
+	}
+	l7ruleParams := Params{}
+	l7ruleParams.L7Rule.AdminStateUp = true
+	l7ruleParams.L7Rule.Type = "PATH"
+	l7ruleParams.L7Rule.Value = rule.Path
+	l7ruleParams.L7Rule.CompareType = "REGEX"
+	body, err := region.lbPost(fmt.Sprintf("/v2/lbaas/l7policies/%s/rules", l7policyId), jsonutils.Marshal(l7ruleParams))
+	if err != nil {
+		return nil, errors.Wrapf(err, `region.lbPost(/v2/lbaas/l7policies/%s/rules), jsonutils.Marshal(l7ruleParams))`, l7policyId)
+	}
+	l7rule := SLoadbalancerL7Rule{}
+	err = body.Unmarshal(&l7rule, "rule")
+	if err != nil {
+		return nil, errors.Wrap(err, `body.Unmarshal(&l7rule, "rule")`)
+	}
+	return &l7rule, nil
+}
+
+func (policy *SLoadbalancerL7Policy) fetchLoadbalancerL7Rules() error {
+	l7rules, err := policy.region.GetLoadbalancerL7Rules(policy.ID)
+	if err != nil {
+		return errors.Wrapf(err, "policy.region.GetLoadbalancerL7Rules(%s)", policy.ID)
+	}
+	for i := 0; i < len(l7rules); i++ {
+		l7rules[i].policy = policy
+	}
+	policy.l7rules = l7rules
+	return nil
+}
+
+func (region *SRegion) GetLoadbalancerL7Rules(policyId string) ([]SLoadbalancerL7Rule, error) {
+	l7rules := []SLoadbalancerL7Rule{}
+	resource := fmt.Sprintf("/v2/lbaas/l7policies/%s/rules", policyId)
+	query := url.Values{}
+	for {
+		resp, err := region.lbList(resource, query)
+		if err != nil {
+			return nil, errors.Wrap(err, "lbList")
+		}
+		part := struct {
+			Rules      []SLoadbalancerL7Rule
+			RulesLinks SNextLinks
+		}{}
+		err = resp.Unmarshal(&part)
+		if err != nil {
+			return nil, errors.Wrap(err, "resp.Unmarshal")
+		}
+		l7rules = append(l7rules, part.Rules...)
+		marker := part.RulesLinks.GetNextMark()
+		if len(marker) == 0 {
+			break
+		}
+		query.Set("marker", marker)
+	}
+
+	return l7rules, nil
+}
+
+func (region *SRegion) GetLoadbalancerL7RulebyId(policyId string, l7ruleId string) (*SLoadbalancerL7Rule, error) {
+	body, err := region.lbGet(fmt.Sprintf("/v2/lbaas/l7policies/%s/rules/%s", policyId, l7ruleId))
+	if err != nil {
+		return nil, errors.Wrapf(err, `region.lbGet(/v2/lbaas/l7policies/%s/rules/%s )`, policyId, l7ruleId)
+	}
+	l7rule := SLoadbalancerL7Rule{}
+	return &l7rule, body.Unmarshal(&l7rule, "rule")
+}
+
+func (l7r *SLoadbalancerL7Rule) GetName() string {
+	return l7r.policy.Name + l7r.ID
+}
+
+func (l7r *SLoadbalancerL7Rule) GetId() string {
+	return l7r.ID
+}
+
+func (l7r *SLoadbalancerL7Rule) GetGlobalId() string {
+	return l7r.ID
+}
+
+func (l7r *SLoadbalancerL7Rule) GetStatus() string {
+	switch l7r.ProvisioningStatus {
+	case "ACTIVE":
+		return api.LB_STATUS_ENABLED
+	case "PENDING_CREATE":
+		return api.LB_CREATING
+	case "PENDING_UPDATE":
+		return api.LB_SYNC_CONF
+	case "PENDING_DELETE":
+		return api.LB_STATUS_DELETING
+	case "DELETED":
+		return api.LB_STATUS_DELETED
+	default:
+		return api.LB_STATUS_UNKNOWN
+	}
+}
+
+func (l7r *SLoadbalancerL7Rule) GetMetadata() *jsonutils.JSONDict {
+	return nil
+}
+
+func (self *SLoadbalancerL7Rule) IsDefault() bool {
+	return false
+}
+
+func (l7r *SLoadbalancerL7Rule) IsEmulated() bool {
+	return false
+}
+
+func (l7r *SLoadbalancerL7Rule) Refresh() error {
+	newL7r, err := l7r.policy.region.GetLoadbalancerL7RulebyId(l7r.policy.ID, l7r.ID)
+	if err != nil {
+		return err
+	}
+	return jsonutils.Update(l7r, newL7r)
+}
+
+func (l7r *SLoadbalancerL7Rule) GetCondition() string {
+	return ""
+}
+
+func (l7r *SLoadbalancerL7Rule) GetDomain() string {
+	return ""
+}
+
+func (l7r *SLoadbalancerL7Rule) GetPath() string {
+	return l7r.Value
+}
+
+func (l7r *SLoadbalancerL7Rule) GetProjectId() string {
+	return ""
+}
+
+func (l7r *SLoadbalancerL7Rule) GetBackendGroupId() string {
+	return l7r.policy.RedirectPoolID
+}
+
+func (l7r *SLoadbalancerL7Rule) Delete(ctx context.Context) error {
+	return l7r.policy.region.DeleteLoadbalancerListenerL7policy(l7r.policy.ID)
+}

pkg/multicloud/openstack/loadbalancerlistener.go

@@ -0,0 +1,766 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openstack
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"strconv"
+	"time"
+
+	"github.com/coredns/coredns/plugin/pkg/log"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+type SLoadbalancerListenerCreateParams struct {
+	Protocol                string         `json:"protocol,omitempty"`
+	Description             string         `json:"description,omitempty"`
+	AdminStateUp            bool           `json:"admin_state_up,omitempty"`
+	ConnectionLimit         *int           `json:"connection_limit,omitempty"`
+	ProtocolPort            string         `json:"protocol_port,omitempty"`
+	LoadbalancerID          string         `json:"loadbalancer_id,omitempty"`
+	DefaultPoolId           string         `json:"default_pool_id,omitempty"`
+	Name                    string         `json:"name,omitempty"`
+	InsertHeaders           SInsertHeaders `json:"insert_headers,omitempty"`
+	DefaultTLSContainerRef  string         `json:"default_tls_container_ref,omitempty"`
+	SniContainerRefs        []string       `json:"sni_container_refs,omitempty"`
+	TimeoutClientData       *int           `json:"timeout_client_data,omitempty"`
+	TimeoutMemberConnect    *int           `json:"timeout_member_connect,omitempty"`
+	TimeoutMemberData       *int           `json:"timeout_member_data,omitempty"`
+	TimeoutTCPInspect       *int           `json:"timeout_tcp_inspect,omitempty"`
+	Tags                    []string       `json:"tags,omitempty"`
+	ClientCaTLSContainerRef string         `json:"client_ca_tls_container_ref,omitempty"`
+	ClientAuthentication    string         `json:"client_authentication,omitempty"`
+	ClientCrlContainerRef   string         `json:"client_crl_container_ref,omitempty"`
+	AllowedCidrs            []string       `json:"allowed_cidrs,omitempty"`
+	TLSCiphers              string         `json:"tls_ciphers,omitempty"`
+	TLSVersions             []string       `json:"tls_versions,omitempty"`
+}
+
+type SLoadbalancerListenerUpdateParams struct {
+	Description             string         `json:"description,omitempty"`
+	AdminStateUp            bool           `json:"admin_state_up,omitempty"`
+	ConnectionLimit         *int           `json:"connection_limit,omitempty"`
+	DefaultPoolId           string         `json:"default_pool_id,omitempty"`
+	Name                    string         `json:"name,omitempty"`
+	InsertHeaders           SInsertHeaders `json:"insert_headers,omitempty"`
+	DefaultTLSContainerRef  string         `json:"default_tls_container_ref,omitempty"`
+	SniContainerRefs        []string       `json:"sni_container_refs,omitempty"`
+	TimeoutClientData       *int           `json:"timeout_client_data,omitempty"`
+	TimeoutMemberConnect    *int           `json:"timeout_member_connect,omitempty"`
+	TimeoutMemberData       *int           `json:"timeout_member_data,omitempty"`
+	TimeoutTCPInspect       *int           `json:"timeout_tcp_inspect,omitempty"`
+	Tags                    []string       `json:"tags,omitempty"`
+	ClientCaTLSContainerRef string         `json:"client_ca_tls_container_ref,omitempty"`
+	ClientAuthentication    string         `json:"client_authentication,omitempty"`
+	ClientCrlContainerRef   string         `json:"client_crl_container_ref,omitempty"`
+	AllowedCidrs            []string       `json:"allowed_cidrs,omitempty"`
+	TLSCiphers              string         `json:"tls_ciphers,omitempty"`
+	TLSVersions             []string       `json:"tls_versions,omitempty"`
+}
+
+type SInsertHeaders struct {
+	XForwardedPort string `json:"X-Forwarded-Port"`
+	XForwardedFor  string `json:"X-Forwarded-For"`
+}
+
+type SLoadbalancerListener struct {
+	region                  *SRegion
+	l7policies              []SLoadbalancerL7Policy
+	pools                   []SLoadbalancerPool
+	Description             string            `json:"description"`
+	AdminStateUp            bool              `json:"admin_state_up"`
+	ProjectID               string            `json:"project_id"`
+	Protocol                string            `json:"protocol"`
+	ProtocolPort            int               `json:"protocol_port"`
+	ProvisioningStatus      string            `json:"provisioning_status"`
+	DefaultTLSContainerRef  string            `json:"default_tls_container_ref"`
+	LoadbalancerIds         []SLoadbalancerID `json:"loadbalancers"`
+	InsertHeaders           SInsertHeaders    `json:"insert_headers"`
+	CreatedAt               string            `json:"created_at"`
+	UpdatedAt               string            `json:"updated_at"`
+	ID                      string            `json:"id"`
+	OperatingStatus         string            `json:"operating_status"`
+	DefaultPoolID           string            `json:"default_pool_id"`
+	SniContainerRefs        []string          `json:"sni_container_refs"`
+	L7PolicieIds            []SL7PolicieID    `json:"l7policies"`
+	Name                    string            `json:"name"`
+	TimeoutClientData       int               `json:"timeout_client_data"`
+	TimeoutMemberConnect    int               `json:"timeout_member_connect"`
+	TimeoutMemberData       int               `json:"timeout_member_data"`
+	TimeoutTCPInspect       int               `json:"timeout_tcp_inspect"`
+	Tags                    []string          `json:"tags"`
+	ClientCaTLSContainerRef string            `json:"client_ca_tls_container_ref"`
+	ClientAuthentication    string            `json:"client_authentication"`
+	ClientCrlContainerRef   string            `json:"client_crl_container_ref"`
+	AllowedCidrs            []string          `json:"allowed_cidrs"`
+	TLSCiphers              string            `json:"tls_ciphers"`
+	TLSVersions             []string          `json:"tls_versions"`
+}
+
+func (listener *SLoadbalancerListener) GetName() string {
+	if len(listener.Name) == 0 {
+		listener.Refresh()
+	}
+	if len(listener.Name) > 0 {
+		return listener.Name
+	}
+	return fmt.Sprintf("HTTP:%d", listener.ProtocolPort)
+}
+
+func (listener *SLoadbalancerListener) GetId() string {
+	return listener.ID
+}
+
+func (listener *SLoadbalancerListener) GetGlobalId() string {
+	return listener.GetId()
+}
+
+func (listener *SLoadbalancerListener) GetStatus() string {
+	switch listener.ProvisioningStatus {
+	case "ACTIVE":
+		return api.LB_STATUS_ENABLED
+	case "PENDING_CREATE":
+		return api.LB_CREATING
+	case "PENDING_UPDATE":
+		return api.LB_SYNC_CONF
+	case "PENDING_DELETE":
+		return api.LB_STATUS_DELETING
+	case "DELETED":
+		return api.LB_STATUS_DELETED
+	default:
+		return api.LB_STATUS_UNKNOWN
+	}
+}
+
+func (listener *SLoadbalancerListener) GetMetadata() *jsonutils.JSONDict {
+	return nil
+}
+
+func (listener *SLoadbalancerListener) IsEmulated() bool {
+	return false
+}
+
+func (listener *SLoadbalancerListener) GetEgressMbps() int {
+
+	return 0
+}
+
+func (region *SRegion) GetLoadbalancerListeners() ([]SLoadbalancerListener, error) {
+	listeners := []SLoadbalancerListener{}
+	resource := "/v2/lbaas/listeners"
+	query := url.Values{}
+	for {
+		resp, err := region.lbList(resource, query)
+		if err != nil {
+			return nil, errors.Wrap(err, "lbList")
+		}
+		part := struct {
+			Listeners      []SLoadbalancerListener
+			ListenersLinks SNextLinks
+		}{}
+		err = resp.Unmarshal(&part)
+		if err != nil {
+			return nil, errors.Wrap(err, "resp.Unmarshal")
+		}
+		listeners = append(listeners, part.Listeners...)
+		marker := part.ListenersLinks.GetNextMark()
+		if len(marker) == 0 {
+			break
+		}
+		query.Set("marker", marker)
+	}
+
+	for i := 0; i < len(listeners); i++ {
+		listeners[i].region = region
+	}
+	for i := 0; i < len(listeners); i++ {
+		err := listeners[i].fetchLoadbalancerListenerL7Policies()
+		if err != nil {
+			return nil, errors.Wrap(err, "listener.fetchLoadbalancerListenerL7Policies()")
+		}
+	}
+
+	for i := 0; i < len(listeners); i++ {
+		err := listeners[i].fetchLoadbalancerPools()
+		if err != nil {
+			return nil, errors.Wrap(err, "listeners[i].fetchLoadbalancerPools()")
+		}
+	}
+
+	return listeners, nil
+}
+
+func (region *SRegion) GetLoadbalancerListenerbyId(listenerId string) (*SLoadbalancerListener, error) {
+	resp, err := region.lbGet(fmt.Sprintf("/v2/lbaas/listeners/%s", listenerId))
+	if err != nil {
+		return nil, errors.Wrapf(err, "region.Get(/v2/lbaas/listeners/%s)", listenerId)
+	}
+	listener := SLoadbalancerListener{}
+	err = resp.Unmarshal(&listener, "listener")
+	if err != nil {
+		return nil, errors.Wrap(err, "resp.Unmarshal(&listener, listener)")
+	}
+	listener.region = region
+	err = listener.fetchLoadbalancerListenerL7Policies()
+	if err != nil {
+		return nil, errors.Wrap(err, "listener.fetchLoadbalancerListenerL7Policies()")
+	}
+
+	err = listener.fetchLoadbalancerPools()
+	if err != nil {
+		return nil, errors.Wrap(err, "listeners[i].fetchLoadbalancerPools()")
+
+	}
+	return &listener, nil
+}
+
+func (region *SRegion) CreateLoadbalancerListener(loadbalancerId string, listenerParams *cloudprovider.SLoadbalancerListener) (*SLoadbalancerListener, error) {
+	type CreateParams struct {
+		Listener SLoadbalancerListenerCreateParams `json:"listener"`
+	}
+	params := CreateParams{}
+	params.Listener.AdminStateUp = true
+	params.Listener.LoadbalancerID = loadbalancerId
+	params.Listener.DefaultPoolId = listenerParams.BackendGroupID
+	params.Listener.Protocol = LB_PROTOCOL_MAP[listenerParams.ListenerType]
+	params.Listener.ProtocolPort = strconv.Itoa(listenerParams.ListenerPort)
+	if listenerParams.ClientIdleTimeout != 0 {
+		// 毫秒单位
+		msClientIdleTimeout := listenerParams.ClientIdleTimeout * 1000
+		params.Listener.TimeoutClientData = &msClientIdleTimeout
+	}
+	if listenerParams.BackendConnectTimeout != 0 {
+		msBackendConnectTimeout := listenerParams.BackendConnectTimeout * 1000
+		params.Listener.TimeoutMemberConnect = &msBackendConnectTimeout
+	}
+	if listenerParams.BackendIdleTimeout != 0 {
+		msBackendIdleTimeout := listenerParams.BackendIdleTimeout * 1000
+		params.Listener.TimeoutMemberData = &msBackendIdleTimeout
+	}
+
+	params.Listener.Name = listenerParams.Name
+	if listenerParams.XForwardedFor {
+		params.Listener.InsertHeaders.XForwardedFor = "true"
+	}
+	body, err := region.lbPost("/v2/lbaas/listeners", jsonutils.Marshal(params))
+	if err != nil {
+		return nil, errors.Wrap(err, "region.Post(/v2/lbaas/listeners)")
+	}
+	slistener := SLoadbalancerListener{}
+	slistener.region = region
+	return &slistener, body.Unmarshal(&slistener, "listener")
+}
+
+func (listener *SLoadbalancerListener) Refresh() error {
+	newlistener, err := listener.region.GetLoadbalancerListenerbyId(listener.ID)
+	if err != nil {
+		return errors.Wrapf(err, "listener.region.GetLoadbalancerListenerbyId(%s)", listener.ID)
+	}
+	return jsonutils.Update(listener, newlistener)
+}
+
+func (listener *SLoadbalancerListener) GetListenerType() string {
+	switch listener.Protocol {
+	case "HTTP":
+		return api.LB_LISTENER_TYPE_HTTP
+	case "HTTPS":
+		return api.LB_LISTENER_TYPE_HTTPS
+	case "TERMINATED_HTTPS":
+		return api.LB_LISTENER_TYPE_TERMINATED_HTTPS
+	case "TCP":
+		return api.LB_LISTENER_TYPE_TCP
+	case "UDP":
+		return api.LB_LISTENER_TYPE_UDP
+	default:
+		return ""
+	}
+}
+
+func (listener *SLoadbalancerListener) GetListenerPort() int {
+	return listener.ProtocolPort
+}
+
+func (listener *SLoadbalancerListener) GetBackendGroupId() string {
+	return listener.DefaultPoolID
+}
+
+func (listener *SLoadbalancerListener) GetBackendServerPort() int {
+	return listener.ProtocolPort
+}
+
+func (listener *SLoadbalancerListener) GetScheduler() string {
+	pool, err := listener.fetchFeaturePool()
+	if err != nil {
+		log.Errorf("GetScheduler():listener.fetchFeaturePool():%s", err)
+		return ""
+	}
+	switch pool.LbAlgorithm {
+	case "ROUND_ROBIN":
+		return api.LB_SCHEDULER_WRR
+	case "LEAST_CONNECTIONS":
+		return api.LB_SCHEDULER_WLC
+	case "SOURCE_IP":
+		return api.LB_SCHEDULER_SCH
+	case "SOURCE_IP_PORT":
+		return api.LB_SCHEDULER_TCH
+	default:
+		return ""
+	}
+}
+
+func (listener *SLoadbalancerListener) GetAclStatus() string {
+	if len(listener.AllowedCidrs) > 0 {
+		return api.LB_BOOL_ON
+	}
+	return api.LB_BOOL_OFF
+}
+
+func (listener *SLoadbalancerListener) GetAclType() string {
+	return api.LB_ACL_TYPE_WHITE
+}
+
+func (listener *SLoadbalancerListener) GetAclId() string {
+	return ""
+}
+
+func (listener *SLoadbalancerListener) GetHealthCheck() string {
+	pool, err := listener.fetchFeaturePool()
+	if err != nil {
+		log.Errorf("GetHealthCheck():listener.fetchFeaturePool():%s", err)
+		return ""
+	}
+	if pool.healthmonitor != nil {
+		return api.LB_BOOL_ON
+	}
+	return api.LB_BOOL_OFF
+
+}
+
+func (listener *SLoadbalancerListener) GetHealthCheckType() string {
+	pool, err := listener.fetchFeaturePool()
+	if err != nil {
+		log.Errorf("GetHealthCheckType():listener.fetchFeaturePool():%s", err)
+		return ""
+	}
+	if pool.healthmonitor == nil {
+		return ""
+	}
+	switch pool.healthmonitor.Type {
+	case "HTTP":
+		return api.LB_HEALTH_CHECK_HTTP
+	case "HTTPS":
+		return api.LB_HEALTH_CHECK_HTTPS
+	case "TCP":
+		return api.LB_HEALTH_CHECK_TCP
+	case "UDP-CONNECT":
+		return api.LB_HEALTH_CHECK_UDP
+	default:
+		return ""
+	}
+}
+
+func (listener *SLoadbalancerListener) GetHealthCheckDomain() string {
+	pool, err := listener.fetchFeaturePool()
+	if err != nil {
+		log.Errorf("GetHealthCheckDomain():listener.fetchFeaturePool():%s", err)
+		return ""
+	}
+	if pool.healthmonitor == nil {
+		return ""
+	}
+	return pool.healthmonitor.DomainName
+}
+
+func (listener *SLoadbalancerListener) GetHealthCheckURI() string {
+	pool, err := listener.fetchFeaturePool()
+	if err != nil {
+		log.Errorf("GetHealthCheckURI():listener.fetchFeaturePool():%s", err)
+		return ""
+	}
+	if pool.healthmonitor == nil {
+		return ""
+	}
+	return pool.healthmonitor.URLPath
+}
+
+func (listener *SLoadbalancerListener) GetHealthCheckCode() string {
+	pool, err := listener.fetchFeaturePool()
+	if err != nil {
+		log.Errorf("GetHealthCheckCode():listener.fetchFeaturePool():%s", err)
+		return ""
+	}
+	if pool.healthmonitor == nil {
+		return ""
+	}
+	return pool.healthmonitor.ExpectedCodes
+}
+
+func (listener *SLoadbalancerListener) GetHealthCheckRise() int {
+	pool, err := listener.fetchFeaturePool()
+	if err != nil {
+		log.Errorf("GetHealthCheckRise():listener.fetchFeaturePool():%s", err)
+		return 0
+	}
+	if pool.healthmonitor == nil {
+		return 0
+	}
+	return pool.healthmonitor.MaxRetries
+}
+
+func (listener *SLoadbalancerListener) GetHealthCheckFail() int {
+	pool, err := listener.fetchFeaturePool()
+	if err != nil {
+		log.Errorf("GetHealthCheckFail():listener.fetchFeaturePool():%s", err)
+		return 0
+	}
+	if pool.healthmonitor == nil {
+		return 0
+	}
+	return pool.healthmonitor.MaxRetriesDown
+}
+
+func (listener *SLoadbalancerListener) GetHealthCheckTimeout() int {
+	pool, err := listener.fetchFeaturePool()
+	if err != nil {
+		log.Errorf("GetHealthCheckTimeout():listener.fetchFeaturePool():%s", err)
+		return 0
+	}
+	if pool.healthmonitor == nil {
+		return 0
+	}
+	return pool.healthmonitor.Timeout
+}
+
+func (listener *SLoadbalancerListener) GetHealthCheckInterval() int {
+	pool, err := listener.fetchFeaturePool()
+	if err != nil {
+		log.Errorf("GetHealthCheckInterval():listener.fetchFeaturePool():%s", err)
+		return 0
+	}
+	if pool.healthmonitor == nil {
+		return 0
+	}
+	return pool.healthmonitor.Delay
+}
+
+func (listener *SLoadbalancerListener) GetHealthCheckReq() string {
+	return ""
+}
+
+func (listener *SLoadbalancerListener) GetHealthCheckExp() string {
+	return ""
+}
+
+func (listener *SLoadbalancerListener) GetStickySession() string {
+	pool, err := listener.fetchFeaturePool()
+	if err != nil {
+		log.Errorf("GetStickySession():listener.fetchFeaturePool():%s", err)
+		return api.LB_BOOL_OFF
+	}
+	stickySession, err := pool.GetStickySession()
+	if err != nil {
+		log.Errorf("GetStickySession():listener.fetchFeaturePool():%s", err)
+		return api.LB_BOOL_OFF
+	}
+	if stickySession == nil {
+		return ""
+	}
+	return stickySession.StickySession
+}
+
+func (listener *SLoadbalancerListener) GetStickySessionType() string {
+	pool, err := listener.fetchFeaturePool()
+	if err != nil {
+		log.Errorf("GetStickySession():listener.fetchFeaturePool():%s", err)
+		return ""
+	}
+	stickySession, err := pool.GetStickySession()
+	if err != nil {
+		log.Errorf("GetStickySession():listener.fetchFeaturePool():%s", err)
+		return ""
+	}
+	if stickySession == nil {
+		return ""
+	}
+	return stickySession.StickySessionType
+}
+
+func (listener *SLoadbalancerListener) GetStickySessionCookie() string {
+	pool, err := listener.fetchFeaturePool()
+	if err != nil {
+		log.Errorf("GetStickySession():listener.fetchFeaturePool():%s", err)
+		return ""
+	}
+	stickySession, err := pool.GetStickySession()
+	if err != nil {
+		log.Errorf("GetStickySession():listener.fetchFeaturePool():%s", err)
+		return ""
+	}
+	if stickySession == nil {
+		return ""
+	}
+	return stickySession.StickySessionCookie
+}
+
+func (listener *SLoadbalancerListener) GetStickySessionCookieTimeout() int {
+	pool, err := listener.fetchFeaturePool()
+	if err != nil {
+		log.Errorf("GetStickySession():listener.fetchFeaturePool():%s", err)
+		return 0
+	}
+	stickySession, err := pool.GetStickySession()
+	if err != nil {
+		log.Errorf("GetStickySession():listener.fetchFeaturePool():%s", err)
+		return 0
+	}
+	if stickySession == nil {
+		return 0
+	}
+	return stickySession.StickySessionCookieTimeout
+}
+
+func (listener *SLoadbalancerListener) XForwardedForEnabled() bool {
+	if listener.InsertHeaders.XForwardedFor == "true" {
+		return true
+	}
+	return false
+}
+
+func (listener *SLoadbalancerListener) GzipEnabled() bool {
+	return false
+}
+
+func (listener *SLoadbalancerListener) GetCertificateId() string {
+	return ""
+}
+
+func (listener *SLoadbalancerListener) GetTLSCipherPolicy() string {
+	return listener.TLSCiphers
+}
+
+func (listener *SLoadbalancerListener) HTTP2Enabled() bool {
+	return false
+}
+
+func (listener *SLoadbalancerListener) fetchLoadbalancerListenerL7Policies() error {
+	l7policies := []SLoadbalancerL7Policy{}
+	for i := 0; i < len(listener.L7PolicieIds); i++ {
+		l7policy, err := listener.region.GetLoadbalancerL7PolicybyId(listener.L7PolicieIds[i].ID)
+		if err != nil {
+			return errors.Wrapf(err, "listener.region.GetLoadbalancerL7PolicybyId(%s)", listener.L7PolicieIds[i].ID)
+		}
+		l7policies = append(l7policies, *l7policy)
+	}
+	listener.l7policies = l7policies
+	return nil
+}
+
+func (listener *SLoadbalancerListener) GetILoadbalancerListenerRules() ([]cloudprovider.ICloudLoadbalancerListenerRule, error) {
+	iRules := []cloudprovider.ICloudLoadbalancerListenerRule{}
+	for i := 0; i < len(listener.l7policies); i++ {
+		for j := 0; j < len(listener.l7policies[i].l7rules); j++ {
+			iRules = append(iRules, &listener.l7policies[i].l7rules[j])
+		}
+	}
+	return iRules, nil
+}
+
+func (region *SRegion) DeleteLoadbalancerListener(listenerId string) error {
+	_, err := region.lbDelete(fmt.Sprintf("/v2/lbaas/listeners/%s", listenerId))
+	if err != nil {
+		return errors.Wrapf(err, `region.lbDelete(fmt.Sprintf("/v2/lbaas/listeners/%s", listenerId))`)
+	}
+	return nil
+}
+
+func (listener *SLoadbalancerListener) Delete(ctx context.Context) error {
+	waitLbResStatus(listener, 10*time.Second, 1*time.Minute)
+	return listener.region.DeleteLoadbalancerListener(listener.ID)
+}
+
+func (listener *SLoadbalancerListener) CreateILoadBalancerListenerRule(rule *cloudprovider.SLoadbalancerListenerRule) (cloudprovider.ICloudLoadbalancerListenerRule, error) {
+	l7policy, err := listener.region.CreateLoadbalancerL7Policy(listener.ID, rule)
+	if err != nil {
+		return nil, errors.Wrapf(err, `listener.region.CreateLoadbalancerL7Policy(%s, rule)`, listener.ID)
+	}
+	// async wait
+	err = waitLbResStatus(l7policy, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return nil, errors.Wrap(err, `waitLbResStatus(l7policy, 10*time.Second, 8*time.Minute)`)
+	}
+
+	l7rule, err := listener.region.CreateLoadbalancerL7Rule(l7policy.ID, rule)
+	if err != nil {
+		return nil, errors.Wrapf(err, `listener.region.CreateLoadbalancerL7Rule(%s, rule)`, l7policy.ID)
+	}
+	l7rule.policy = l7policy
+	// async wait
+	err = waitLbResStatus(l7rule, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return nil, errors.Wrap(err, `waitLbResStatus(l7rule, 10*time.Second, 8*time.Minute)`)
+	}
+	return l7rule, nil
+}
+
+func (listener *SLoadbalancerListener) GetILoadBalancerListenerRuleById(ruleId string) (cloudprovider.ICloudLoadbalancerListenerRule, error) {
+	for i := 0; i < len(listener.l7policies); i++ {
+		for j := 0; j < len(listener.l7policies[i].l7rules); j++ {
+			if listener.l7policies[i].l7rules[j].GetId() == ruleId {
+				return &listener.l7policies[i].l7rules[j], nil
+			}
+		}
+	}
+	return nil, nil
+}
+
+func (listener *SLoadbalancerListener) fetchLoadbalancerPools() error {
+	pools := []SLoadbalancerPool{}
+	if len(listener.DefaultPoolID) > 0 {
+		defaultPool, err := listener.region.GetLoadbalancerPoolById(listener.DefaultPoolID)
+		if err != nil {
+			return errors.Wrapf(err, "listener.region.GetLoadbalancerPoolById(%s)", listener.DefaultPoolID)
+		}
+		pools = append(pools, *defaultPool)
+	}
+	for i := 0; i < len(listener.l7policies); i++ {
+		if len(listener.l7policies[i].RedirectPoolID) > 0 {
+			policyPool, err := listener.region.GetLoadbalancerPoolById(listener.l7policies[i].RedirectPoolID)
+			if err != nil {
+				return errors.Wrapf(err, "listener.region.GetLoadbalancerPoolById(%s)", listener.l7policies[i].RedirectPoolID)
+			}
+			pools = append(pools, *policyPool)
+		}
+	}
+	listener.pools = pools
+	return nil
+}
+
+func (listener *SLoadbalancerListener) fetchFeaturePool() (*SLoadbalancerPool, error) {
+	if len(listener.pools) < 1 {
+		return nil, fmt.Errorf("can't find pool with healthmonitor")
+	}
+	for i := 0; i < len(listener.pools); i++ {
+		if listener.pools[i].healthmonitor != nil {
+			return &listener.pools[i], nil
+		}
+	}
+	return &listener.pools[0], nil
+}
+
+func (region *SRegion) UpdateLoadBalancerListenerAdminStateUp(AdminStateUp bool, loadbalancerListenerId string) error {
+	params := jsonutils.NewDict()
+	poolParam := jsonutils.NewDict()
+	poolParam.Add(jsonutils.NewBool(AdminStateUp), "admin_state_up")
+	params.Add(poolParam, "listener")
+	_, err := region.lbUpdate(fmt.Sprintf("/v2/lbaas/listeners/%s", loadbalancerListenerId), params)
+	if err != nil {
+		return errors.Wrapf(err, `region.lbUpdate(/v2/lbaas/listeners/%s, params)`, loadbalancerListenerId)
+	}
+	return nil
+}
+
+func (region *SRegion) UpdateLoadBalancerListener(loadbalancerListenerId string, lblis *cloudprovider.SLoadbalancerListener) error {
+	type UpdateParams struct {
+		Listener SLoadbalancerListenerUpdateParams `json:"listener"`
+	}
+	params := UpdateParams{}
+	params.Listener.AdminStateUp = true
+	params.Listener.DefaultPoolId = lblis.BackendGroupID
+
+	if lblis.ClientIdleTimeout != 0 {
+		// 毫秒单位
+		msClientIdleTimeout := lblis.ClientIdleTimeout * 1000
+		params.Listener.TimeoutClientData = &msClientIdleTimeout
+	}
+	if lblis.BackendConnectTimeout != 0 {
+		msBackendConnectTimeout := lblis.BackendConnectTimeout * 1000
+		params.Listener.TimeoutMemberConnect = &msBackendConnectTimeout
+	}
+	if lblis.BackendIdleTimeout != 0 {
+		msBackendIdleTimeout := lblis.BackendIdleTimeout * 1000
+		params.Listener.TimeoutMemberData = &msBackendIdleTimeout
+	}
+	params.Listener.Name = lblis.Name
+	if lblis.XForwardedFor {
+		params.Listener.InsertHeaders.XForwardedFor = "true"
+	}
+	_, err := region.lbUpdate(fmt.Sprintf("/v2/lbaas/listeners/%s", loadbalancerListenerId), jsonutils.Marshal(params))
+	if err != nil {
+		return errors.Wrapf(err, `region.lbUpdate(/v2/lbaas/listeners/%s, jsonutils.Marshal(params))`, loadbalancerListenerId)
+	}
+	return nil
+}
+
+func (listener *SLoadbalancerListener) Start() error {
+	// ensure listener status
+	err := waitLbResStatus(listener, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, ` waitLbResStatus(listener, 10*time.Second, 8*time.Minute)`)
+	}
+	err = listener.region.UpdateLoadBalancerListenerAdminStateUp(true, listener.ID)
+	if err != nil {
+		return errors.Wrapf(err, `listener.region.UpdateLoadBalancerListenerAdminStateUp(true, %s)`, listener.ID)
+	}
+	err = waitLbResStatus(listener, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, `waitLbResStatus(listener, 10*time.Second, 8*time.Minute)`)
+	}
+	return nil
+}
+
+func (listener *SLoadbalancerListener) Stop() error {
+	// ensure listener status
+	err := waitLbResStatus(listener, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, ` waitLbResStatus(listener, 10*time.Second, 8*time.Minute)`)
+	}
+	err = listener.region.UpdateLoadBalancerListenerAdminStateUp(false, listener.ID)
+	if err != nil {
+		return errors.Wrapf(err, `listener.region.UpdateLoadBalancerListenerAdminStateUp(false,%s)`, listener.ID)
+	}
+	err = waitLbResStatus(listener, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, `waitLbResStatus(listener, 10*time.Second, 8*time.Minute)`)
+	}
+	return nil
+}
+
+func (listener *SLoadbalancerListener) Sync(ctx context.Context, lblis *cloudprovider.SLoadbalancerListener) error {
+	// ensure listener status
+	err := waitLbResStatus(listener, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, ` waitLbResStatus(listener, 10*time.Second, 8*time.Minute)`)
+	}
+	err = listener.region.UpdateLoadBalancerListener(listener.ID, lblis)
+	if err != nil {
+		return errors.Wrapf(err, `listener.region.UpdateLoadBalancerListener(%s, lblis)`, listener.ID)
+	}
+	err = waitLbResStatus(listener, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, `waitLbResStatus(listener, 10*time.Second, 8*time.Minute)`)
+	}
+	return nil
+}
+
+func (listener *SLoadbalancerListener) GetProjectId() string {
+	return listener.ProjectID
+}

pkg/multicloud/openstack/loadbalancermember.go

@@ -0,0 +1,262 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openstack
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+)
+
+type SLoadbalancerMemberCreateParams struct {
+	Name         string   `json:"name,omitempty"`
+	Weight       *int     `json:"weight,omitempty"`
+	AdminStateUp bool     `json:"admin_state_up,omitempty"`
+	SubnetID     string   `json:"subnet_id,omitempty"`
+	Address      string   `json:"address,omitempty"`
+	ProtocolPort *int     `json:"protocol_port,omitempty"`
+	MonitorPort  *int     `json:"monitor_port,omitempty"`
+	Backup       *bool    `json:"backup,omitempty"`
+	Tags         []string `json:"tags,omitempty"`
+}
+
+type SLoadbalancerMember struct {
+	poolID             string
+	region             *SRegion
+	MonitorPort        int      `json:"monitor_port"`
+	ProjectID          string   `json:"project_id"`
+	Name               string   `json:"name"`
+	Weight             int      `json:"weight"`
+	Backup             bool     `json:"backup"`
+	AdminStateUp       bool     `json:"admin_state_up"`
+	SubnetID           string   `json:"subnet_id"`
+	CreatedAt          string   `json:"created_at"`
+	ProvisioningStatus string   `json:"provisioning_status"`
+	MonitorAddress     string   `json:"monitor_address"`
+	UpdatedAt          string   `json:"updated_at"`
+	Address            string   `json:"address"`
+	ProtocolPort       int      `json:"protocol_port"`
+	ID                 string   `json:"id"`
+	OperatingStatus    string   `json:"operating_status"`
+	Tags               []string `json:"tags"`
+}
+
+func (member *SLoadbalancerMember) GetName() string {
+	return member.Name
+}
+
+func (member *SLoadbalancerMember) GetId() string {
+	return member.ID
+}
+
+func (member *SLoadbalancerMember) GetGlobalId() string {
+	return member.GetId()
+}
+
+func (member *SLoadbalancerMember) GetStatus() string {
+	switch member.ProvisioningStatus {
+	case "ACTIVE":
+		return api.LB_STATUS_ENABLED
+	case "PENDING_CREATE":
+		return api.LB_CREATING
+	case "PENDING_UPDATE":
+		return api.LB_SYNC_CONF
+	case "PENDING_DELETE":
+		return api.LB_STATUS_DELETING
+	case "DELETED":
+		return api.LB_STATUS_DELETED
+	default:
+		return api.LB_STATUS_UNKNOWN
+	}
+}
+
+func (member *SLoadbalancerMember) GetMetadata() *jsonutils.JSONDict {
+	return nil
+}
+
+func (member *SLoadbalancerMember) IsEmulated() bool {
+	return false
+}
+
+func (region *SRegion) GetLoadbalancerMenberById(poolId string, MenberId string) (*SLoadbalancerMember, error) {
+	body, err := region.lbGet(fmt.Sprintf("/v2/lbaas/pools/%s/members/%s", poolId, MenberId))
+	if err != nil {
+		return nil, errors.Wrapf(err, "region.Get(/v2/lbaas/pools/%s/members/%s)", poolId, MenberId)
+	}
+	member := SLoadbalancerMember{}
+	member.region = region
+	member.poolID = poolId
+	return &member, body.Unmarshal(&member, "member")
+}
+
+func (region *SRegion) GetLoadbalancerMenbers(poolId string) ([]SLoadbalancerMember, error) {
+	members := []SLoadbalancerMember{}
+	resource := fmt.Sprintf("/v2/lbaas/pools/%s/members", poolId)
+	query := url.Values{}
+	for {
+		resp, err := region.lbList(resource, query)
+		if err != nil {
+			return nil, errors.Wrap(err, "lbList")
+		}
+		part := struct {
+			Members      []SLoadbalancerMember
+			MembersLinks SNextLinks
+		}{}
+		err = resp.Unmarshal(&part)
+		if err != nil {
+			return nil, errors.Wrap(err, "resp.Unmarshal")
+		}
+		members = append(members, part.Members...)
+		marker := part.MembersLinks.GetNextMark()
+		if len(marker) == 0 {
+			break
+		}
+		query.Set("marker", marker)
+	}
+
+	for i := 0; i < len(members); i++ {
+		members[i].poolID = poolId
+		members[i].region = region
+	}
+	return members, nil
+}
+
+// serverId 转ip,对接不准确
+func (region *SRegion) CreateLoadbalancerMember(poolId, serverId string, weight, port int) (*SLoadbalancerMember, error) {
+	ports, err := region.GetPorts("", serverId)
+	if len(ports) < 1 {
+		return nil, errors.Wrap(err, "server have no port")
+	}
+	fixedip := SFixedIP{}
+	for i := 0; i < len(ports); i++ {
+		if len(ports[i].FixedIps) > 0 {
+			fixedip = ports[i].FixedIps[0]
+			break
+		}
+	}
+	if len(fixedip.IpAddress) < 1 || len(fixedip.SubnetID) < 1 {
+		return nil, errors.Wrap(err, "server have no fixedip")
+	}
+	type CreateParams struct {
+		Member SLoadbalancerMemberCreateParams `json:"member"`
+	}
+	memberParams := CreateParams{}
+	memberParams.Member.AdminStateUp = true
+	memberParams.Member.Address = fixedip.IpAddress
+	memberParams.Member.SubnetID = fixedip.SubnetID
+	memberParams.Member.ProtocolPort = &port
+	memberParams.Member.Weight = &weight
+
+	body, err := region.lbPost(fmt.Sprintf("/v2/lbaas/pools/%s/members", poolId), jsonutils.Marshal(memberParams))
+	if err != nil {
+		return nil, errors.Wrapf(err, `region.lbPost(/v2/lbaas/pools/%s/members, jsonutils.Marshal(memberParams))`, poolId)
+	}
+	member := SLoadbalancerMember{}
+	member.region = region
+	member.poolID = poolId
+	return &member, body.Unmarshal(&member, "member")
+}
+
+func (region *SRegion) DeleteLoadbalancerMember(poolId, memberId string) error {
+	_, err := region.lbDelete(fmt.Sprintf("/v2/lbaas/pools/%s/members/%s", poolId, memberId))
+	if err != nil {
+		return errors.Wrapf(err, "region.lbDelete(fmt.Sprintf(/v2/lbaas/pools/%s/members/%s)", poolId, memberId)
+	}
+	return nil
+}
+
+func (member *SLoadbalancerMember) Refresh() error {
+	newMember, err := member.region.GetLoadbalancerMenberById(member.poolID, member.ID)
+	if err != nil {
+		return err
+	}
+	return jsonutils.Update(member, newMember)
+}
+
+func (member *SLoadbalancerMember) GetWeight() int {
+	return member.Weight
+}
+
+func (member *SLoadbalancerMember) GetPort() int {
+	return member.ProtocolPort
+}
+
+func (member *SLoadbalancerMember) GetBackendType() string {
+	return api.LB_BACKEND_GUEST
+}
+
+func (member *SLoadbalancerMember) GetBackendRole() string {
+	return api.LB_BACKEND_ROLE_DEFAULT
+}
+
+// 网络地址映射设备
+func (member *SLoadbalancerMember) GetBackendId() string {
+	ports, err := member.region.GetPorts("", "")
+	if err != nil {
+		log.Errorln(errors.Wrap(err, "member.region.GetPorts()"))
+	}
+	for i := 0; i < len(ports); i++ {
+		for j := 0; j < len(ports[i].FixedIps); j++ {
+			fixedIP := ports[i].FixedIps[j]
+			if fixedIP.SubnetID == member.SubnetID && fixedIP.IpAddress == member.Address {
+				return ports[i].DeviceID
+			}
+		}
+	}
+	return ""
+}
+
+func (member *SLoadbalancerMember) GetProjectId() string {
+	return member.ProjectID
+}
+
+func (region *SRegion) UpdateLoadBalancerMemberWtight(poolId, memberId string, weight int) error {
+	params := jsonutils.NewDict()
+	poolParam := jsonutils.NewDict()
+	poolParam.Add(jsonutils.NewInt(int64(weight)), "weight")
+	params.Add(poolParam, "member")
+	_, err := region.lbUpdate(fmt.Sprintf("/v2/lbaas/pools/%s/members/%s", poolId, memberId), params)
+	if err != nil {
+		return errors.Wrapf(err, "region.lbUpdate(fmt.Sprintf(/v2/lbaas/pools/%s/members/%s", poolId, memberId)
+	}
+	return nil
+}
+
+func (member *SLoadbalancerMember) SyncConf(ctx context.Context, port, weight int) error {
+	if port > 0 {
+		log.Warningf("Elb backend SyncConf unsupport modify port")
+	}
+	// ensure member status
+	err := waitLbResStatus(member, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, "waitLbResStatus(member, 10*time.Second, 8*time.Minute)")
+	}
+	err = member.region.UpdateLoadBalancerMemberWtight(member.poolID, member.ID, weight)
+	if err != nil {
+		return errors.Wrapf(err, "member.region.UpdateLoadBalancerMemberWtight(%s,%s,%s)", member.poolID, member.ID, weight)
+	}
+	err = waitLbResStatus(member, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, "waitLbResStatus(member, 10*time.Second, 8*time.Minute)")
+	}
+	return nil
+}

pkg/multicloud/openstack/loadbalbacer.go

@@ -0,0 +1,489 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http:// www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openstack
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"time"
+
+	"github.com/coredns/coredns/plugin/pkg/log"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud"
+)
+
+var LB_ALGORITHM_MAP = map[string]string{
+	api.LB_SCHEDULER_RR:  "ROUND_ROBIN",
+	api.LB_SCHEDULER_WRR: "ROUND_ROBIN",
+	api.LB_SCHEDULER_WLC: "LEAST_CONNECTIONS",
+	api.LB_SCHEDULER_SCH: "SOURCE_IP",
+	api.LB_SCHEDULER_TCH: "SOURCE_IP_PORT",
+}
+
+var LB_PROTOCOL_MAP = map[string]string{
+	api.LB_LISTENER_TYPE_HTTP:             "HTTP",
+	api.LB_LISTENER_TYPE_HTTPS:            "HTTPS",
+	api.LB_LISTENER_TYPE_TERMINATED_HTTPS: "TERMINATED_HTTPS",
+	api.LB_LISTENER_TYPE_UDP:              "UDP",
+	api.LB_LISTENER_TYPE_TCP:              "TCP",
+}
+
+var LB_STICKY_SESSION_MAP = map[string]string{
+	api.LB_STICKY_SESSION_TYPE_INSERT: "HTTP_COOKIE",
+	api.LB_STICKY_SESSION_TYPE_SERVER: "APP_COOKIE",
+}
+
+var LB_HEALTHCHECK_TYPE_MAP = map[string]string{
+	api.LB_HEALTH_CHECK_HTTP:  "HTTP",
+	api.LB_HEALTH_CHECK_HTTPS: "HTTPS",
+	api.LB_HEALTH_CHECK_TCP:   "TCP",
+	api.LB_HEALTH_CHECK_UDP:   "UDP_CONNECT",
+}
+
+type SLoadbalancerCreateParams struct {
+	Description      string   `json:"description,omitempty"`
+	AdminStateUp     bool     `json:"admin_state_up,omitempty"`
+	ProjectID        string   `json:"project_id,omitempty"`
+	VipNetworkId     string   `json:"vip_network_id,omitempty"`
+	VipSubnetID      string   `json:"vip_subnet_id,omitempty"`
+	VipAddress       string   `json:"vip_address,omitempty"`
+	Provider         string   `json:"provider,omitempty"`
+	Name             string   `json:"name,omitempty"`
+	VipQosPolicyID   string   `json:"vip_qos_policy_id,omitempty"`
+	AvailabilityZone string   `json:"availability_zone,omitempty"`
+	Tags             []string `json:"tags,omitempty"`
+}
+
+type SLoadbalancerID struct {
+	ID string `json:"id"`
+}
+
+type SPoolID struct {
+	ID string `json:"id"`
+}
+
+type SMemberID struct {
+	ID string `json:"id"`
+}
+
+type SListenerID struct {
+	ID string `json:"id"`
+}
+
+type SL7PolicieID struct {
+	ID string `json:"id"`
+}
+
+type SL7RuleID struct {
+	ID string `json:"id"`
+}
+
+type SLoadbalancer struct {
+	multicloud.SLoadbalancerBase
+	region *SRegion
+
+	Description        string        `json:"description"`
+	AdminStateUp       bool          `json:"admin_state_up"`
+	ProjectID          string        `json:"project_id"`
+	ProvisioningStatus string        `json:"provisioning_status"`
+	FlavorID           string        `json:"flavor_id"`
+	VipSubnetID        string        `json:"vip_subnet_id"`
+	ListenerIds        []SListenerID `json:"listeners"`
+	VipAddress         string        `json:"vip_address"`
+	VipNetworkID       string        `json:"vip_network_id"`
+	VipPortID          string        `json:"vip_port_id"`
+	Provider           string        `json:"provider"`
+	PoolIds            []SPoolID     `json:"pools"`
+	CreatedAt          string        `json:"created_at"`
+	UpdatedAt          string        `json:"updated_at"`
+	ID                 string        `json:"id"`
+	OperatingStatus    string        `json:"operating_status"`
+	Name               string        `json:"name"`
+	VipQosPolicyID     string        `json:"vip_qos_policy_id"`
+	AvailabilityZone   string        `json:"availability_zone"`
+	Tags               []string      `json:"tags"`
+}
+
+func waitLbResStatus(res cloudprovider.ICloudResource, interval time.Duration, timeout time.Duration) error {
+	err := cloudprovider.WaitMultiStatus(res, []string{api.LB_STATUS_ENABLED, api.LB_STATUS_UNKNOWN}, interval, timeout)
+	if err != nil {
+		return errors.Wrap(err, "waitLbResStatus(res, interval, timeout)")
+	}
+	if res.GetStatus() == api.LB_STATUS_UNKNOWN {
+		return errors.Wrap(fmt.Errorf("status error"), "check status")
+	}
+	return nil
+}
+
+func (lb *SLoadbalancer) GetName() string {
+	return lb.Name
+}
+
+func (lb *SLoadbalancer) GetId() string {
+	return lb.ID
+}
+
+func (lb *SLoadbalancer) GetGlobalId() string {
+	return lb.ID
+}
+
+func (lb *SLoadbalancer) GetStatus() string {
+	switch lb.ProvisioningStatus {
+	case "ACTIVE":
+		return api.LB_STATUS_ENABLED
+	case "PENDING_CREATE":
+		return api.LB_CREATING
+	case "PENDING_UPDATE":
+		return api.LB_SYNC_CONF
+	case "PENDING_DELETE":
+		return api.LB_STATUS_DELETING
+	case "DELETED":
+		return api.LB_STATUS_DELETED
+	default:
+		return api.LB_STATUS_UNKNOWN
+	}
+}
+
+func (lb *SLoadbalancer) GetMetadata() *jsonutils.JSONDict {
+	return nil
+}
+
+func (lb *SLoadbalancer) GetAddress() string {
+	return lb.VipAddress
+}
+
+func (lb *SLoadbalancer) GetAddressType() string {
+	eip, err := lb.GetIEIP()
+	if err != nil {
+		return api.LB_ADDR_TYPE_INTRANET
+	}
+	if eip == nil {
+		return api.LB_ADDR_TYPE_INTRANET
+	}
+	return api.LB_ADDR_TYPE_INTERNET
+}
+
+func (lb *SLoadbalancer) GetNetworkType() string {
+	network, err := lb.region.GetVpc(lb.VipNetworkID)
+	if err != nil {
+		log.Error(errors.Wrapf(err, "lb.region.GetNetwork(%s)", lb.VipNetworkID))
+	}
+	if network.NetworkType == "flat" || network.NetworkType == "vlan" {
+		return api.LB_NETWORK_TYPE_CLASSIC
+	}
+	return api.LB_NETWORK_TYPE_VPC
+}
+
+func (lb *SLoadbalancer) GetNetworkIds() []string {
+	return []string{lb.VipSubnetID}
+}
+
+func (lb *SLoadbalancer) GetZoneId() string {
+	return lb.AvailabilityZone
+}
+
+func (lb *SLoadbalancer) IsEmulated() bool {
+	return false
+}
+
+func (lb *SLoadbalancer) GetVpcId() string {
+	return lb.VipNetworkID
+}
+
+func (lb *SLoadbalancer) Refresh() error {
+	loadbalancer, err := lb.region.GetLoadbalancerbyId(lb.ID)
+	if err != nil {
+		return err
+	}
+	return jsonutils.Update(lb, loadbalancer)
+}
+
+func (region *SRegion) GetLoadbalancers() ([]SLoadbalancer, error) {
+	loadbalancers := []SLoadbalancer{}
+	resource := "/v2/lbaas/loadbalancers"
+	query := url.Values{}
+	for {
+		resp, err := region.lbList(resource, query)
+		if err != nil {
+			return nil, errors.Wrap(err, "lbList")
+		}
+		part := struct {
+			Loadbalancers      []SLoadbalancer
+			LoadbalancersLinks SNextLinks
+		}{}
+		err = resp.Unmarshal(&part)
+		if err != nil {
+			return nil, errors.Wrap(err, "resp.Unmarshal")
+		}
+		loadbalancers = append(loadbalancers, part.Loadbalancers...)
+		marker := part.LoadbalancersLinks.GetNextMark()
+		if len(marker) == 0 {
+			break
+		}
+		query.Set("marker", marker)
+	}
+	for i := 0; i < len(loadbalancers); i++ {
+		loadbalancers[i].region = region
+	}
+	return loadbalancers, nil
+}
+
+func (region *SRegion) GetLoadbalancerbyId(loadbalancerId string) (*SLoadbalancer, error) {
+	// region.client.Debug(true)
+	body, err := region.lbGet(fmt.Sprintf("/v2/lbaas/loadbalancers/%s", loadbalancerId))
+	if err != nil {
+		return nil, errors.Wrapf(err, `region.lbGet(/v2/lbaas/loadbalancers/%s)`, loadbalancerId)
+	}
+	loadbalancer := SLoadbalancer{}
+	err = body.Unmarshal(&loadbalancer, "loadbalancer")
+	if err != nil {
+		return nil, errors.Wrap(err, "resp.Unmarshal(loadbalancer)")
+	}
+	loadbalancer.region = region
+	return &loadbalancer, nil
+}
+
+func (region *SRegion) CreateLoadBalancer(loadbalancer *cloudprovider.SLoadbalancer) (*SLoadbalancer, error) {
+	type CreateParams struct {
+		Loadbalancer SLoadbalancerCreateParams `json:"loadbalancer"`
+	}
+	params := CreateParams{}
+	params.Loadbalancer.AdminStateUp = true
+	params.Loadbalancer.AvailabilityZone = loadbalancer.ZoneID
+	params.Loadbalancer.Name = loadbalancer.Name
+	params.Loadbalancer.ProjectID = loadbalancer.ProjectId
+	params.Loadbalancer.VipSubnetID = loadbalancer.NetworkIDs[0]
+	params.Loadbalancer.VipAddress = loadbalancer.Address
+
+	body, err := region.lbPost("/v2/lbaas/loadbalancers", jsonutils.Marshal(params))
+	if err != nil {
+		return nil, errors.Wrap(err, `region.lbPost("/v2/lbaas/loadbalancers", jsonutils.Marshal(params))`)
+	}
+	sloadbalancer := SLoadbalancer{}
+	err = body.Unmarshal(&sloadbalancer, "loadbalancer")
+	if err != nil {
+		return nil, errors.Wrap(err, "body.Unmarshal(sloadbalancer, loadbalancer)")
+	}
+	sloadbalancer.region = region
+	if len(loadbalancer.EipID) > 0 {
+		err = region.AssociateEipWithPortId(sloadbalancer.VipPortID, loadbalancer.EipID)
+		if err != nil {
+			return nil, errors.Wrapf(err, "region.AssociateEipWithPortId(%s, %s)", sloadbalancer.VipPortID, loadbalancer.EipID)
+		}
+	}
+	return &sloadbalancer, nil
+}
+
+func (region *SRegion) DeleteLoadbalancer(loadbalancerId string) error {
+	_, err := region.lbDelete(fmt.Sprintf("/v2/lbaas/loadbalancers/%s?cascade=True", loadbalancerId))
+	if err != nil {
+		return errors.Wrapf(err, `region.lbDelete(/v2/lbaas/loadbalancers/%s?cascade=True)`, loadbalancerId)
+	}
+	return nil
+}
+
+func (lb *SLoadbalancer) Delete(ctx context.Context) error {
+	return lb.region.DeleteLoadbalancer(lb.ID)
+}
+
+func (lb *SLoadbalancer) GetILoadBalancerBackendGroups() ([]cloudprovider.ICloudLoadbalancerBackendGroup, error) {
+	ibackendgroups := []cloudprovider.ICloudLoadbalancerBackendGroup{}
+	for i := 0; i < len(lb.PoolIds); i++ {
+		pool, err := lb.region.GetLoadbalancerPoolById(lb.PoolIds[i].ID)
+		if err != nil {
+			return nil, errors.Wrapf(err, "lb.region.GetLoadbalancerPoolById(%s)", lb.PoolIds[i].ID)
+		}
+		ibackendgroups = append(ibackendgroups, pool)
+	}
+	return ibackendgroups, nil
+}
+
+func (lb *SLoadbalancer) CreateILoadBalancerBackendGroup(group *cloudprovider.SLoadbalancerBackendGroup) (cloudprovider.ICloudLoadbalancerBackendGroup, error) {
+	// ensure lb status
+	err := waitLbResStatus(lb, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return nil, errors.Wrap(err, "waitLbResStatus(lb, api.LB_STATUS_ENABLED, 10*time.Second, 8*time.Minute)")
+	}
+	// create pool
+	spool, err := lb.region.CreateLoadbalancerPool(group)
+	if err != nil {
+		return nil, errors.Wrap(err, "lb.region.CreateLoadbalancerPool")
+	}
+	// wait spool
+	err = waitLbResStatus(spool, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return nil, errors.Wrap(err, "waitLbResStatus(spool,  10*time.Second, 8*time.Minute)")
+	}
+	// create healthmonitor
+	if group.HealthCheck != nil {
+
+		healthmonitor, err := lb.region.CreateLoadbalancerHealthmonitor(spool.ID, group.HealthCheck)
+		if err != nil {
+			return nil, errors.Wrapf(err, "region.CreateLoadbalancerHealthmonitor(%s, group.HealthCheck)", spool.ID)
+		}
+		spool.healthmonitor = healthmonitor
+	}
+	// wait health monitor
+	if spool.healthmonitor != nil {
+		err = waitLbResStatus(spool.healthmonitor, 10*time.Second, 8*time.Minute)
+		if err != nil {
+			return nil, errors.Wrap(err, "waitLbResStatus(spool.healthmonitor,  10*time.Second, 8*time.Minute)")
+		}
+	}
+	return spool, nil
+}
+
+func (lb *SLoadbalancer) CreateILoadBalancerListener(ctx context.Context, listener *cloudprovider.SLoadbalancerListener) (cloudprovider.ICloudLoadbalancerListener, error) {
+	// ensure lb status
+	err := waitLbResStatus(lb, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return nil, errors.Wrap(err, "waitLbResStatus(lb, api.LB_STATUS_ENABLED, 10*time.Second, 8*time.Minute)")
+	}
+	slistener, err := lb.region.CreateLoadbalancerListener(lb.ID, listener)
+	if err != nil {
+		return nil, errors.Wrapf(err, "lb.region.CreateLoadbalancerListener(%s, listener)", lb.ID)
+	}
+	return slistener, nil
+}
+
+func (lb *SLoadbalancer) GetLoadbalancerSpec() string {
+	return lb.Description
+}
+
+func (lb *SLoadbalancer) GetChargeType() string {
+	eip, err := lb.GetIEIP()
+	if err != nil {
+		log.Errorf("lb.GetIEIP():", err)
+	}
+	if err != nil {
+		return eip.GetInternetChargeType()
+	}
+
+	return api.EIP_CHARGE_TYPE_BY_TRAFFIC
+}
+
+func (lb *SLoadbalancer) GetEgressMbps() int {
+	return 0
+}
+
+func (lb *SLoadbalancer) GetILoadBalancerBackendGroupById(poolId string) (cloudprovider.ICloudLoadbalancerBackendGroup, error) {
+	err := lb.Refresh()
+	if err != nil {
+		return nil, errors.Wrap(err, "lb.Refresh()")
+	}
+	index := -1
+	for i := 0; i < len(lb.PoolIds); i++ {
+		if poolId == lb.PoolIds[i].ID {
+			index = i
+		}
+	}
+	if index < 0 {
+		return nil, cloudprovider.ErrNotFound
+	}
+	spool, err := lb.region.GetLoadbalancerPoolById(poolId)
+	if err != nil {
+		return nil, errors.Wrapf(err, "lb.region.GetLoadbalancerPoolById(%s)", poolId)
+	}
+	if spool.GetStatus() == api.LB_STATUS_DELETING {
+		return nil, cloudprovider.ErrNotFound
+	}
+	return spool, nil
+}
+
+func (lb *SLoadbalancer) GetIEIP() (cloudprovider.ICloudEIP, error) {
+	eips, err := lb.region.GetEips("")
+	if err != nil {
+		return nil, errors.Wrapf(err, "lb.region.GetEips()")
+	}
+	for _, eip := range eips {
+		if eip.PortId == lb.VipPortID {
+			return &eip, nil
+		}
+	}
+	return nil, nil
+}
+
+func (region *SRegion) UpdateLoadBalancerAdminStateUp(AdminStateUp bool, loadbalancerId string) error {
+	params := jsonutils.NewDict()
+	poolParam := jsonutils.NewDict()
+	poolParam.Add(jsonutils.NewBool(AdminStateUp), "admin_state_up")
+	params.Add(poolParam, "loadbalancer")
+	_, err := region.lbUpdate(fmt.Sprintf("/v2/lbaas/loadbalancers/%s", loadbalancerId), params)
+	if err != nil {
+		return errors.Wrapf(err, `region.lbUpdate(/v2/lbaas/loadbalancers/%s), params)`, loadbalancerId)
+	}
+	return nil
+}
+
+func (lb *SLoadbalancer) Start() error {
+	// ensure lb status
+	err := waitLbResStatus(lb, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, "waitLbResStatus(lb, api.LB_STATUS_ENABLED, 10*time.Second, 8*time.Minute)")
+	}
+	err = lb.region.UpdateLoadBalancerAdminStateUp(true, lb.ID)
+	if err != nil {
+		return errors.Wrapf(err, "lb.region.UpdateLoadBalancerAdminStateUp(true, %s)", lb.ID)
+	}
+	err = waitLbResStatus(lb, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, "waitLbResStatus(lb,  10*time.Second, 8*time.Minute)")
+	}
+	return nil
+}
+
+func (lb *SLoadbalancer) Stop() error {
+	// ensure lb status
+	err := waitLbResStatus(lb, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, "waitLbResStatus(lb, api.LB_STATUS_ENABLED, 10*time.Second, 8*time.Minute)")
+	}
+	err = lb.region.UpdateLoadBalancerAdminStateUp(false, lb.ID)
+	if err != nil {
+		return errors.Wrapf(err, "lb.region.UpdateLoadBalancerAdminStateUp(false,%s)", lb.ID)
+	}
+	err = waitLbResStatus(lb, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return errors.Wrap(err, "waitLbResStatus(lb,  10*time.Second, 8*time.Minute)")
+	}
+	return nil
+}
+
+func (lb *SLoadbalancer) GetILoadBalancerListenerById(listenerId string) (cloudprovider.ICloudLoadbalancerListener, error) {
+
+	return lb.region.GetLoadbalancerListenerbyId(listenerId)
+}
+
+func (lb *SLoadbalancer) GetILoadBalancerListeners() ([]cloudprovider.ICloudLoadbalancerListener, error) {
+	ilisteners := []cloudprovider.ICloudLoadbalancerListener{}
+	for i := 0; i < len(lb.ListenerIds); i++ {
+		listener, err := lb.region.GetLoadbalancerListenerbyId(lb.ListenerIds[i].ID)
+		if err != nil {
+			return nil, errors.Wrapf(err, "lb.region.GetLoadbalancerListenerbyId(%s)", lb.ListenerIds[i].ID)
+		}
+		ilisteners = append(ilisteners, listener)
+	}
+	return ilisteners, nil
+}
+
+func (lb *SLoadbalancer) GetProjectId() string {
+	return lb.ProjectID
+}

pkg/multicloud/openstack/loadbalbancerhealthmonitor.go

@@ -0,0 +1,226 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openstack
+
+import (
+	"fmt"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+type SLoadbalancerHealthmonitorCreateParams struct {
+	Name           string   `json:"name,omitempty"`
+	AdminStateUp   bool     `json:"admin_state_up"`
+	PoolID         string   `json:"pool_id,omitempty"`
+	Delay          *int     `json:"delay,omitempty"`
+	ExpectedCodes  string   `json:"expected_codes,omitempty"`
+	MaxRetries     *int     `json:"max_retries,omitempty"`
+	HTTPMethod     string   `json:"http_method,omitempty"`
+	Timeout        *int     `json:"timeout,omitempty"`
+	URLPath        string   `json:"url_path,omitempty"`
+	Type           string   `json:"type,omitempty"`
+	MaxRetriesDown *int     `json:"max_retries_down,omitempty"`
+	Tags           []string `json:"tags,omitempty"`
+	HTTPVersion    *float64 `json:"http_version,omitempty"`
+	DomainName     string   `json:"domain_name,omitempty"`
+}
+
+type SLoadbalancerHealthmonitorUpdateParams struct {
+	Name           string   `json:"name,omitempty"`
+	AdminStateUp   bool     `json:"admin_state_up"`
+	Delay          *int     `json:"delay,omitempty"`
+	ExpectedCodes  string   `json:"expected_codes,omitempty"`
+	MaxRetries     *int     `json:"max_retries,omitempty"`
+	HTTPMethod     string   `json:"http_method,omitempty"`
+	Timeout        *int     `json:"timeout,omitempty"`
+	URLPath        string   `json:"url_path,omitempty"`
+	MaxRetriesDown *int     `json:"max_retries_down,omitempty"`
+	Tags           []string `json:"tags,omitempty"`
+	HTTPVersion    *float64 `json:"http_version,omitempty"`
+	DomainName     string   `json:"domain_name,omitempty"`
+}
+
+type SLoadbalancerHealthmonitor struct {
+	region             *SRegion
+	ProjectID          string    `json:"project_id"`
+	Name               string    `json:"name"`
+	AdminStateUp       bool      `json:"admin_state_up"`
+	PoolIds            []SPoolID `json:"pools"`
+	CreatedAt          string    `json:"created_at"`
+	ProvisioningStatus string    `json:"provisioning_status"`
+	UpdatedAt          string    `json:"updated_at"`
+	Delay              int       `json:"delay"`
+	ExpectedCodes      string    `json:"expected_codes"`
+	MaxRetries         int       `json:"max_retries"`
+	HTTPMethod         string    `json:"http_method"`
+	Timeout            int       `json:"timeout"`
+	MaxRetriesDown     int       `json:"max_retries_down"`
+	URLPath            string    `json:"url_path"`
+	Type               string    `json:"type"`
+	ID                 string    `json:"id"`
+	OperatingStatus    string    `json:"operating_status"`
+	Tags               []string  `json:"tags"`
+	HTTPVersion        float64   `json:"http_version"`
+	DomainName         string    `json:"domain_name"`
+}
+
+func (region *SRegion) GetLoadbalancerHealthmonitorById(healthmonitorId string) (*SLoadbalancerHealthmonitor, error) {
+	body, err := region.lbGet(fmt.Sprintf("/v2/lbaas/healthmonitors/%s", healthmonitorId))
+	if err != nil {
+		return nil, errors.Wrapf(err, `region.lbGet(/v2/lbaas/healthmonitors/%s)`, healthmonitorId)
+	}
+	healthmonitor := SLoadbalancerHealthmonitor{}
+	healthmonitor.region = region
+	return &healthmonitor, body.Unmarshal(&healthmonitor, "healthmonitor")
+}
+func (region *SRegion) CreateLoadbalancerHealthmonitor(poolId string, healthcheck *cloudprovider.SLoadbalancerHealthCheck) (*SLoadbalancerHealthmonitor, error) {
+	type CreateParams struct {
+		Healthmonitor SLoadbalancerHealthmonitorCreateParams `json:"healthmonitor"`
+	}
+	params := CreateParams{}
+	params.Healthmonitor.AdminStateUp = true
+	params.Healthmonitor.Delay = &healthcheck.HealthCheckInterval
+	params.Healthmonitor.Timeout = &healthcheck.HealthCheckTimeout
+
+	params.Healthmonitor.MaxRetries = &healthcheck.HealthCheckRise
+	params.Healthmonitor.MaxRetriesDown = &healthcheck.HealthCheckFail
+	params.Healthmonitor.PoolID = poolId
+
+	switch healthcheck.HealthCheckType {
+	case api.LB_HEALTH_CHECK_TCP:
+		params.Healthmonitor.Type = "TCP"
+	case api.LB_HEALTH_CHECK_UDP:
+		params.Healthmonitor.Type = "UDP-CONNECT"
+	case api.LB_HEALTH_CHECK_HTTP:
+		params.Healthmonitor.Type = "HTTP"
+	case api.LB_HEALTH_CHECK_HTTPS:
+		params.Healthmonitor.Type = "HTTPS"
+	case api.LB_HEALTH_CHECK_PING:
+		params.Healthmonitor.Type = "PING"
+	default:
+		params.Healthmonitor.Type = "PING"
+	}
+
+	if params.Healthmonitor.Type == "HTTP" || params.Healthmonitor.Type == "HTTPS" {
+		params.Healthmonitor.HTTPMethod = "GET"
+		httpVersion := 1.1
+		params.Healthmonitor.HTTPVersion = &httpVersion
+		params.Healthmonitor.DomainName = healthcheck.HealthCheckDomain
+		params.Healthmonitor.URLPath = healthcheck.HealthCheckURI
+		params.Healthmonitor.ExpectedCodes = ToOpenstackHealthCheckHttpCode(healthcheck.HealthCheckHttpCode)
+	}
+
+	body, err := region.lbPost("/v2/lbaas/healthmonitors", jsonutils.Marshal(params))
+	if err != nil {
+		return nil, errors.Wrap(err, `region.lbPost("/v2/lbaas/healthmonitors", jsonutils.Marshal(params))`)
+	}
+	shealthmonitor := SLoadbalancerHealthmonitor{}
+	shealthmonitor.region = region
+	err = body.Unmarshal(&shealthmonitor, "healthmonitor")
+	if err != nil {
+		return nil, errors.Wrap(err, "body.Unmarshal(&shealthmonitor, healthmonitor)")
+	}
+	return &shealthmonitor, nil
+}
+
+func (region *SRegion) UpdateLoadbalancerHealthmonitor(healthmonitorId string, healthcheck *cloudprovider.SLoadbalancerHealthCheck) (*SLoadbalancerHealthmonitor, error) {
+	type UpdateParams struct {
+		Healthmonitor SLoadbalancerHealthmonitorUpdateParams `json:"healthmonitor"`
+	}
+	params := UpdateParams{}
+	params.Healthmonitor.AdminStateUp = true
+	params.Healthmonitor.Delay = &healthcheck.HealthCheckInterval
+	params.Healthmonitor.Timeout = &healthcheck.HealthCheckTimeout
+
+	params.Healthmonitor.MaxRetries = &healthcheck.HealthCheckRise
+	params.Healthmonitor.MaxRetriesDown = &healthcheck.HealthCheckFail
+
+	if healthcheck.HealthCheckType == api.LB_HEALTH_CHECK_HTTP || healthcheck.HealthCheckType == api.LB_HEALTH_CHECK_HTTPS {
+		params.Healthmonitor.HTTPMethod = "GET"
+		httpVersion := 1.1
+		params.Healthmonitor.HTTPVersion = &httpVersion
+		params.Healthmonitor.DomainName = healthcheck.HealthCheckDomain
+		params.Healthmonitor.URLPath = healthcheck.HealthCheckURI
+		params.Healthmonitor.ExpectedCodes = ToOpenstackHealthCheckHttpCode(healthcheck.HealthCheckHttpCode)
+	}
+
+	body, err := region.lbUpdate(fmt.Sprintf("/v2/lbaas/healthmonitors/%s", healthmonitorId), jsonutils.Marshal(params))
+	if err != nil {
+		return nil, errors.Wrapf(err, `region.lbUpdate(/v2/lbaas/healthmonitors/%s), jsonutils.Marshal(params))`, healthmonitorId)
+	}
+	shealthmonitor := SLoadbalancerHealthmonitor{}
+	err = body.Unmarshal(&shealthmonitor, "healthmonitor")
+	shealthmonitor.region = region
+	if err != nil {
+		return nil, errors.Wrap(err, "body.Unmarshal(&shealthmonitor, healthmonitor)")
+	}
+	return &shealthmonitor, nil
+}
+
+func (region *SRegion) DeleteLoadbalancerHealthmonitor(healthmonitorId string) error {
+	_, err := region.lbDelete(fmt.Sprintf("/v2/lbaas/healthmonitors/%s", healthmonitorId))
+	if err != nil {
+		return errors.Wrapf(err, `region.lbDelete(/v2/lbaas/healthmonitors/%s )`, healthmonitorId)
+	}
+	return nil
+}
+
+func (healthmonitor *SLoadbalancerHealthmonitor) GetName() string {
+	return healthmonitor.Name
+}
+
+func (healthmonitor *SLoadbalancerHealthmonitor) GetId() string {
+	return healthmonitor.ID
+}
+
+func (healthmonitor *SLoadbalancerHealthmonitor) GetGlobalId() string {
+	return healthmonitor.ID
+}
+
+func (healthmonitor *SLoadbalancerHealthmonitor) GetStatus() string {
+	switch healthmonitor.ProvisioningStatus {
+	case "ACTIVE":
+		return api.LB_STATUS_ENABLED
+	case "PENDING_CREATE":
+		return api.LB_CREATING
+	case "PENDING_UPDATE":
+		return api.LB_SYNC_CONF
+	case "PENDING_DELETE":
+		return api.LB_STATUS_DELETING
+	case "DELETED":
+		return api.LB_STATUS_DELETED
+	default:
+		return api.LB_STATUS_UNKNOWN
+	}
+}
+
+func (healthmonitor *SLoadbalancerHealthmonitor) Refresh() error {
+	newhealthmonitor, err := healthmonitor.region.GetLoadbalancerHealthmonitorById(healthmonitor.ID)
+	if err != nil {
+		return err
+	}
+	return jsonutils.Update(healthmonitor, newhealthmonitor)
+}
+
+func (healthmonitor *SLoadbalancerHealthmonitor) IsEmulated() bool {
+	return false
+}
+func (healthmonitor *SLoadbalancerHealthmonitor) GetMetadata() *jsonutils.JSONDict {
+	return nil
+}

pkg/multicloud/openstack/openstack.go

@@ -38,13 +38,14 @@ const (
 	CLOUD_PROVIDER_OPENSTACK = api.CLOUD_PROVIDER_OPENSTACK
 	OPENSTACK_DEFAULT_REGION = "RegionOne"
 
-	OPENSTACK_SERVICE_COMPUTE  = "compute"
-	OPENSTACK_SERVICE_NETWORK  = "network"
-	OPENSTACK_SERVICE_IDENTITY = "identity"
-	OPENSTACK_SERVICE_VOLUMEV3 = "volumev3"
-	OPENSTACK_SERVICE_VOLUMEV2 = "volumev2"
-	OPENSTACK_SERVICE_VOLUME   = "volume"
-	OPENSTACK_SERVICE_IMAGE    = "image"
+	OPENSTACK_SERVICE_COMPUTE      = "compute"
+	OPENSTACK_SERVICE_NETWORK      = "network"
+	OPENSTACK_SERVICE_IDENTITY     = "identity"
+	OPENSTACK_SERVICE_VOLUMEV3     = "volumev3"
+	OPENSTACK_SERVICE_VOLUMEV2     = "volumev2"
+	OPENSTACK_SERVICE_VOLUME       = "volume"
+	OPENSTACK_SERVICE_IMAGE        = "image"
+	OPENSTACK_SERVICE_LOADBALANCER = "load-balancer"
 
 	ErrNoEndpoint = errors.Error("no valid endpoint")
 )
@@ -373,6 +374,10 @@ func (cli *SOpenStackClient) imageUpload(region, url string, body io.Reader) (*h
 	return session.RawRequest(OPENSTACK_SERVICE_IMAGE, "", httputils.PUT, url, header, body)
 }
 
+func (cli *SOpenStackClient) lbRequest(region string, method httputils.THttpMethod, resource string, query url.Values, body interface{}) (jsonutils.JSONObject, error) {
+	return jsonReuest(cli.tokenCredential, OPENSTACK_SERVICE_LOADBALANCER, region, cli.endpointType, method, resource, query, body, cli.debug)
+}
+
 func (cli *SOpenStackClient) fetchToken() error {
 	if cli.tokenCredential != nil {
 		return nil
@@ -533,7 +538,7 @@ func (self *SOpenStackClient) GetCapabilities() []string {
 		cloudprovider.CLOUD_CAPABILITY_PROJECT,
 		cloudprovider.CLOUD_CAPABILITY_COMPUTE,
 		cloudprovider.CLOUD_CAPABILITY_NETWORK,
-		// cloudprovider.CLOUD_CAPABILITY_LOADBALANCER,
+		cloudprovider.CLOUD_CAPABILITY_LOADBALANCER,
 		// cloudprovider.CLOUD_CAPABILITY_OBJECTSTORE,
 		// cloudprovider.CLOUD_CAPABILITY_RDS,
 		// cloudprovider.CLOUD_CAPABILITY_CACHE,

pkg/multicloud/openstack/region.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"io"
 	"net/url"
+	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
@@ -43,7 +44,15 @@ type SRegion struct {
 }
 
 func (region *SRegion) GetILoadBalancerBackendGroups() ([]cloudprovider.ICloudLoadbalancerBackendGroup, error) {
-	return nil, cloudprovider.ErrNotImplemented
+	backendGroups := []cloudprovider.ICloudLoadbalancerBackendGroup{}
+	pools, err := region.GetLoadbalancerPools()
+	if err != nil {
+		return backendGroups, errors.Wrap(err, "region.GetLoadbalancerPools()")
+	}
+	for i := 0; i < len(pools); i++ {
+		backendGroups = append(backendGroups, &pools[i])
+	}
+	return backendGroups, nil
 }
 
 func (region *SRegion) GetClient() *SOpenStackClient {
@@ -354,6 +363,28 @@ func (region *SRegion) bsCreate(projectId, resource string, params interface{})
 	return region.client.bsCreate(projectId, region.Name, resource, params)
 }
 
+//loadbalancer
+
+func (region *SRegion) lbList(resource string, query url.Values) (jsonutils.JSONObject, error) {
+	return region.client.lbRequest(region.Name, httputils.GET, resource, query, nil)
+}
+
+func (region *SRegion) lbGet(resource string) (jsonutils.JSONObject, error) {
+	return region.client.lbRequest(region.Name, httputils.GET, resource, nil, nil)
+}
+
+func (region *SRegion) lbUpdate(resource string, params interface{}) (jsonutils.JSONObject, error) {
+	return region.client.lbRequest(region.Name, httputils.PUT, resource, nil, params)
+}
+
+func (region *SRegion) lbPost(resource string, params interface{}) (jsonutils.JSONObject, error) {
+	return region.client.lbRequest(region.Name, httputils.POST, resource, nil, params)
+}
+
+func (region *SRegion) lbDelete(resource string) (jsonutils.JSONObject, error) {
+	return region.client.lbRequest(region.Name, httputils.DELETE, resource, nil, nil)
+}
+
 func (region *SRegion) ProjectId() string {
 	return region.client.tokenCredential.GetProjectId()
 }
@@ -413,15 +444,27 @@ func (region *SRegion) GetIEipById(eipId string) (cloudprovider.ICloudEIP, error
 }
 
 func (region *SRegion) GetILoadBalancers() ([]cloudprovider.ICloudLoadbalancer, error) {
-	return nil, cloudprovider.ErrNotImplemented
+	loadbalancers := []cloudprovider.ICloudLoadbalancer{}
+	sloadbalancers, err := region.GetLoadbalancers()
+	if err != nil {
+		return nil, errors.Wrap(err, "region.GetLoadbalancers()")
+	}
+	for i := 0; i < len(sloadbalancers); i++ {
+		loadbalancers = append(loadbalancers, &sloadbalancers[i])
+	}
+	return loadbalancers, nil
 }
 
 func (region *SRegion) GetILoadBalancerById(loadbalancerId string) (cloudprovider.ICloudLoadbalancer, error) {
-	return nil, cloudprovider.ErrNotImplemented
+	sloadbalancer, err := region.GetLoadbalancerbyId(loadbalancerId)
+	if err != nil {
+		return nil, errors.Wrapf(err, "region.GetLoadbalancerbyId(%s)", loadbalancerId)
+	}
+	return sloadbalancer, nil
 }
 
 func (region *SRegion) GetILoadBalancerAclById(aclId string) (cloudprovider.ICloudLoadbalancerAcl, error) {
-	return nil, cloudprovider.ErrNotImplemented
+	return region.GetLoadbalancerAclDetail(aclId)
 }
 
 func (region *SRegion) GetILoadBalancerCertificateById(certId string) (cloudprovider.ICloudLoadbalancerCertificate, error) {
@@ -433,7 +476,15 @@ func (region *SRegion) CreateILoadBalancerCertificate(cert *cloudprovider.SLoadb
 }
 
 func (region *SRegion) GetILoadBalancerAcls() ([]cloudprovider.ICloudLoadbalancerAcl, error) {
-	return nil, cloudprovider.ErrNotImplemented
+	iloadbalancerAcls := []cloudprovider.ICloudLoadbalancerAcl{}
+	acls, err := region.GetLoadBalancerAcls()
+	if err != nil {
+		return nil, errors.Wrap(err, "region.GetLoadBalancerAcls")
+	}
+	for i := 0; i < len(acls); i++ {
+		iloadbalancerAcls = append(iloadbalancerAcls, &acls[i])
+	}
+	return iloadbalancerAcls, nil
 }
 
 func (region *SRegion) GetILoadBalancerCertificates() ([]cloudprovider.ICloudLoadbalancerCertificate, error) {
@@ -441,11 +492,26 @@ func (region *SRegion) GetILoadBalancerCertificates() ([]cloudprovider.ICloudLoa
 }
 
 func (region *SRegion) CreateILoadBalancer(loadbalancer *cloudprovider.SLoadbalancer) (cloudprovider.ICloudLoadbalancer, error) {
-	return nil, cloudprovider.ErrNotImplemented
+	sloadbalancer, err := region.CreateLoadBalancer(loadbalancer)
+	if err != nil {
+		return nil, errors.Wrap(err, "region.CreateLoadBalancer")
+	}
+	return sloadbalancer, nil
 }
 
 func (region *SRegion) CreateILoadBalancerAcl(acl *cloudprovider.SLoadbalancerAccessControlList) (cloudprovider.ICloudLoadbalancerAcl, error) {
-	return nil, cloudprovider.ErrNotImplemented
+	sacl, err := region.CreateLoadBalancerAcl(acl)
+	if err != nil {
+		return nil, errors.Wrap(err, "region.CreateLoadBalancerAcl(acl)")
+	}
+	err = cloudprovider.WaitMultiStatus(sacl.listener, []string{api.LB_STATUS_ENABLED, api.LB_STATUS_UNKNOWN}, 10*time.Second, 8*time.Minute)
+	if err != nil {
+		return nil, errors.Wrap(err, "cloudprovider.WaitMultiStatus")
+	}
+	if sacl.listener.GetStatus() == api.LB_STATUS_UNKNOWN {
+		return nil, errors.Wrap(fmt.Errorf("status error"), "check status")
+	}
+	return sacl, nil
 }
 
 func (region *SRegion) GetISkus() ([]cloudprovider.ICloudSku, error) {

pkg/multicloud/openstack/shell/loadbalancer.go

@@ -0,0 +1,83 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shell
+
+import (
+	"yunion.io/x/onecloud/pkg/multicloud/openstack"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	type LoadbalancerListOptions struct {
+	}
+	type LoadbalancerPoolListOptions struct {
+	}
+	type LoadbalancerListenerListOptions struct {
+	}
+	type LoadbalancerOptions struct {
+		ID string `help:"LoadbalancerId"`
+	}
+	type LoadbalancerListenerOptions struct {
+		ID string `help:"LoadbalancerListenerId"`
+	}
+
+	shellutils.R(&LoadbalancerListOptions{}, "lb-list", "List loadbalancers", func(cli *openstack.SRegion, args *LoadbalancerListOptions) error {
+		loadbalancers, err := cli.GetLoadbalancers()
+		if err != nil {
+			return err
+		}
+		printObject(loadbalancers)
+		return nil
+	})
+	shellutils.R(&LoadbalancerOptions{}, "lb-show", "Show loadbalancer", func(cli *openstack.SRegion, args *LoadbalancerOptions) error {
+		loadbalancer, err := cli.GetLoadbalancerbyId(args.ID)
+		if err != nil {
+			return err
+		}
+		printObject(loadbalancer)
+		return nil
+	})
+	shellutils.R(&LoadbalancerOptions{}, "lb-delete", "delete loadbalancer", func(cli *openstack.SRegion, args *LoadbalancerOptions) error {
+		err := cli.DeleteLoadbalancer(args.ID)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	shellutils.R(&LoadbalancerPoolListOptions{}, "lbpool-list", "List loadbalancers pool", func(cli *openstack.SRegion, args *LoadbalancerPoolListOptions) error {
+		loadbalancers, err := cli.GetLoadbalancerPools()
+		if err != nil {
+			return err
+		}
+		printObject(loadbalancers)
+		return nil
+	})
+
+	shellutils.R(&LoadbalancerListenerListOptions{}, "lblistener-list", "List loadbalancers listener", func(cli *openstack.SRegion, args *LoadbalancerListenerListOptions) error {
+		loadbalancers, err := cli.GetLoadbalancerListeners()
+		if err != nil {
+			return err
+		}
+		printObject(loadbalancers)
+		return nil
+	})
+	shellutils.R(&LoadbalancerOptions{}, "lblistener-delete", "Delete loadbalancer listener", func(cli *openstack.SRegion, args *LoadbalancerOptions) error {
+		err := cli.DeleteLoadbalancerListener(args.ID)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+}

pkg/multicloud/openstack/vpc.go

@@ -62,7 +62,7 @@ type SVpc struct {
 	VlanTransparent       bool
 	Fescription           string
 	IsDefault             bool
-	NetworkType           string `json:"provider:network_type"`
+	NetworkType           string `json:"provider:network_type"` // flat, vlan, vxlan, or gre ...
 	PhysicalNetwork       string `json:"provider:physical_network"`
 	SegmentationId        string `json:"provider:segmentation_id"`
 }