loadbalancer: initial version

build/lbagent/root/etc/yunion/lbagent.conf.sample

@@ -0,0 +1,15 @@
+region = 'Yunion'
+auth_uri = 'http://10.168.222.136:35357/v3'
+admin_user = 'regionadmin'
+admin_password = 'GkZOr6poQgcmMszU'
+admin_tenant_name = 'system'
+
+data_preserve_n = 2
+base_data_dir = "/opt/cloud/workspace/lbagent"
+
+api_lbagent_id = 'lbagent id'
+api_lbagent_hb_interval = 60
+api_lbagent_hb_timeout_relaxation = 120
+
+api_sync_interval = 5
+api_list_batch_size = 2048

build/lbagent/root/opt/yunion/share/lbagent/healthcheck.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+set -e
+
+o_wait=5
+o_req="are you ok?"
+o_exp="ok"
+o_host=
+o_port=
+
+__errmsg() {
+	echo "healthcheck: $*" >&2
+}
+
+while [ "$#" -gt 0 ]; do
+	case "$1" in
+		wait=*|\
+		req=*|\
+		exp=*|\
+		host=*|\
+		port=*)
+			eval "o_$1"
+			;;
+	esac
+	shift
+done
+
+if [ -z "$o_host" -o -z "$o_port" ]; then
+	__errmsg "--host or --port missing"
+	exit 126 # script error are not server error
+fi
+
+got="$(
+	(stdbuf -o0 printf "%s" "$o_req"; sleep "$o_wait") | \
+	nc \
+		--verbose \
+		--udp \
+		--nodns \
+		--wait "$o_wait" \
+	"$o_host" "$o_port"
+)"
+
+if [ "$got" != "$o_exp" ]; then
+	__errmsg "result unmatch, got $got, want $o_exp"
+	exit 1
+fi

build/lbagent/root/opt/yunion/share/lbagent/yunion-lbagent.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=Yunion Loadbalancer Agent
+Documentation=http://doc.yunionyun.com
+After=network.target
+
+[Service]
+Type=simple
+User=root
+Group=root
+ExecStart=/opt/yunion/bin/lbagent --config /etc/yunion/lbagent.conf
+WorkingDirectory=/opt/yunion
+KillMode=process
+Restart=always
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

build/lbagent/vars

@@ -0,0 +1,8 @@
+DESCRIPTION="Yunion Loadbalancer Agent"
+
+REQUIRES=(
+	nmap-ncat
+	"keepalived >= 2.0.0"
+	"haproxy >= 1.8.0"
+	"gobetween >= 0.7.0"
+)

cmd/climc/shell/loadbalanceracls.go

@@ -0,0 +1,115 @@
+package shell
+
+import (
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func init() {
+	lbAclConvert := func(jd *jsonutils.JSONDict) error {
+		jaeso, err := jd.Get("acl_entries")
+		if err != nil {
+			return err
+		}
+		aclEntries := options.AclEntries{}
+		err = jaeso.Unmarshal(&aclEntries)
+		if err != nil {
+			return err
+		}
+		aclTextLines := aclEntries.String()
+		jd.Set("acl_entries", jsonutils.NewString(aclTextLines))
+		return nil
+	}
+
+	printLbAcl := func(jsonObj jsonutils.JSONObject) {
+		jd, ok := jsonObj.(*jsonutils.JSONDict)
+		if !ok {
+			printObject(jsonObj)
+			return
+		}
+		err := lbAclConvert(jd)
+		if err != nil {
+			printObject(jsonObj)
+			return
+		}
+		printObject(jd)
+	}
+	printLbAclList := func(list *modules.ListResult, columns []string) {
+		data := list.Data
+		for _, jsonObj := range data {
+			jd := jsonObj.(*jsonutils.JSONDict)
+			err := lbAclConvert(jd)
+			if err != nil {
+				printList(list, columns)
+			}
+		}
+		printList(list, columns)
+	}
+
+	R(&options.LoadbalancerAclCreateOptions{}, "lbacl-create", "Create lbacl", func(s *mcclient.ClientSession, opts *options.LoadbalancerAclCreateOptions) error {
+		params, err := opts.Params()
+		if err != nil {
+			return err
+		}
+		lbacl, err := modules.LoadbalancerAcls.Create(s, params)
+		if err != nil {
+			return err
+		}
+		printLbAcl(lbacl)
+		return nil
+	})
+	R(&options.LoadbalancerAclGetOptions{}, "lbacl-show", "Show lbacl", func(s *mcclient.ClientSession, opts *options.LoadbalancerAclGetOptions) error {
+		lbacl, err := modules.LoadbalancerAcls.Get(s, opts.ID, nil)
+		if err != nil {
+			return err
+		}
+		printLbAcl(lbacl)
+		return nil
+	})
+	R(&options.LoadbalancerAclListOptions{}, "lbacl-list", "List lbacls", func(s *mcclient.ClientSession, opts *options.LoadbalancerAclListOptions) error {
+		params, err := options.ListStructToParams(opts)
+		if err != nil {
+			return err
+		}
+		result, err := modules.LoadbalancerAcls.List(s, params)
+		if err != nil {
+			return err
+		}
+		printLbAclList(result, modules.LoadbalancerAcls.GetColumns(s))
+		return nil
+	})
+	R(&options.LoadbalancerAclUpdateOptions{}, "lbacl-update", "Update lbacls", func(s *mcclient.ClientSession, opts *options.LoadbalancerAclUpdateOptions) error {
+		params, err := opts.Params()
+		if err != nil {
+			return err
+		}
+		lbacl, err := modules.LoadbalancerAcls.Update(s, opts.ID, params)
+		if err != nil {
+			return err
+		}
+		printLbAcl(lbacl)
+		return nil
+	})
+	R(&options.LoadbalancerAclDeleteOptions{}, "lbacl-delete", "Show lbacl", func(s *mcclient.ClientSession, opts *options.LoadbalancerAclDeleteOptions) error {
+		lbacl, err := modules.LoadbalancerAcls.Delete(s, opts.ID, nil)
+		if err != nil {
+			return err
+		}
+		printLbAcl(lbacl)
+		return nil
+	})
+	R(&options.LoadbalancerAclActionPatchOptions{}, "lbacl-patch", "Patch lbacls", func(s *mcclient.ClientSession, opts *options.LoadbalancerAclActionPatchOptions) error {
+		params, err := opts.Params()
+		if err != nil {
+			return err
+		}
+		lbacl, err := modules.LoadbalancerAcls.PerformAction(s, opts.ID, "patch", params)
+		if err != nil {
+			return err
+		}
+		printLbAcl(lbacl)
+		return nil
+	})
+}

cmd/climc/shell/loadbalanceragents.go

@@ -0,0 +1,107 @@
+package shell
+
+import (
+	"encoding/base64"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func init() {
+	printLbagent := func(data jsonutils.JSONObject) {
+		printObjectRecursiveEx(data, func(data jsonutils.JSONObject) {
+			// "base64 -d" config template
+			keys := []string{
+				"params.keepalived_conf_tmpl",
+				"params.haproxy_conf_tmpl",
+			}
+			d := data.(*jsonutils.JSONDict)
+			for _, key := range keys {
+				if d.Contains(key) {
+					s0, _ := d.GetString(key)
+					b1, err := base64.StdEncoding.DecodeString(s0)
+					if err != nil {
+						log.Errorf("%s: invalid base64 string: %s\n  %s", key, err, s0)
+						return
+					}
+					s1 := string(b1)
+					d.Set(key, jsonutils.NewString(s1))
+				}
+			}
+			printObject(d)
+		})
+	}
+	R(&options.LoadbalancerAgentCreateOptions{}, "lbagent-create", "Create lbagent", func(s *mcclient.ClientSession, opts *options.LoadbalancerAgentCreateOptions) error {
+		params, err := opts.Params()
+		if err != nil {
+			return err
+		}
+		lbagent, err := modules.LoadbalancerAgents.Create(s, params)
+		if err != nil {
+			return err
+		}
+		printLbagent(lbagent)
+		return nil
+	})
+	R(&options.LoadbalancerAgentGetOptions{}, "lbagent-show", "Show lbagent", func(s *mcclient.ClientSession, opts *options.LoadbalancerAgentGetOptions) error {
+		lbagent, err := modules.LoadbalancerAgents.Get(s, opts.ID, nil)
+		if err != nil {
+			return err
+		}
+		printLbagent(lbagent)
+		return nil
+	})
+	R(&options.LoadbalancerAgentListOptions{}, "lbagent-list", "List lbagents", func(s *mcclient.ClientSession, opts *options.LoadbalancerAgentListOptions) error {
+		params, err := options.ListStructToParams(opts)
+		if err != nil {
+			return err
+		}
+		result, err := modules.LoadbalancerAgents.List(s, params)
+		if err != nil {
+			return err
+		}
+		printList(result, modules.LoadbalancerAgents.GetColumns(s))
+		return nil
+	})
+	R(&options.LoadbalancerAgentUpdateOptions{}, "lbagent-update", "Update lbagent", func(s *mcclient.ClientSession, opts *options.LoadbalancerAgentUpdateOptions) error {
+		params, err := options.StructToParams(opts)
+		lbagent, err := modules.LoadbalancerAgents.Update(s, opts.ID, params)
+		if err != nil {
+			return err
+		}
+		printLbagent(lbagent)
+		return nil
+	})
+	R(&options.LoadbalancerAgentDeleteOptions{}, "lbagent-delete", "Show lbagent", func(s *mcclient.ClientSession, opts *options.LoadbalancerAgentDeleteOptions) error {
+		lbagent, err := modules.LoadbalancerAgents.Delete(s, opts.ID, nil)
+		if err != nil {
+			return err
+		}
+		printLbagent(lbagent)
+		return nil
+	})
+	R(&options.LoadbalancerAgentActionHbOptions{}, "lbagent-heartbeat", "Emulate a lbagent heartbeat", func(s *mcclient.ClientSession, opts *options.LoadbalancerAgentActionHbOptions) error {
+		lbagent, err := modules.LoadbalancerAgents.PerformAction(s, opts.ID, "hb", nil)
+		if err != nil {
+			return err
+		}
+		printLbagent(lbagent)
+		return nil
+	})
+	R(&options.LoadbalancerAgentActionPatchParamsOptions{}, "lbagent-params-patch", "Patch lbagent params", func(s *mcclient.ClientSession, opts *options.LoadbalancerAgentActionPatchParamsOptions) error {
+		params, err := opts.Params()
+		if err != nil {
+			return err
+		}
+		lbagent, err := modules.LoadbalancerAgents.PerformAction(s, opts.ID, "params-patch", params)
+		if err != nil {
+			return err
+		}
+		printLbagent(lbagent)
+		return nil
+	})
+}

cmd/climc/shell/loadbalancerbackendgroups.go

@@ -0,0 +1,50 @@
+package shell
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func init() {
+	R(&options.LoadbalancerBackendGroupCreateOptions{}, "lbbackendgroup-create", "Create lbbackendgroup", func(s *mcclient.ClientSession, opts *options.LoadbalancerBackendGroupCreateOptions) error {
+		params, err := options.StructToParams(opts)
+		if err != nil {
+			return err
+		}
+		lbbackendgroup, err := modules.LoadbalancerBackendGroups.Create(s, params)
+		if err != nil {
+			return err
+		}
+		printObject(lbbackendgroup)
+		return nil
+	})
+	R(&options.LoadbalancerBackendGroupGetOptions{}, "lbbackendgroup-show", "Show lbbackendgroup", func(s *mcclient.ClientSession, opts *options.LoadbalancerBackendGroupGetOptions) error {
+		lbbackendgroup, err := modules.LoadbalancerBackendGroups.Get(s, opts.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(lbbackendgroup)
+		return nil
+	})
+	R(&options.LoadbalancerBackendGroupListOptions{}, "lbbackendgroup-list", "List lbbackendgroups", func(s *mcclient.ClientSession, opts *options.LoadbalancerBackendGroupListOptions) error {
+		params, err := options.ListStructToParams(opts)
+		if err != nil {
+			return err
+		}
+		result, err := modules.LoadbalancerBackendGroups.List(s, params)
+		if err != nil {
+			return err
+		}
+		printList(result, modules.LoadbalancerBackendGroups.GetColumns(s))
+		return nil
+	})
+	R(&options.LoadbalancerBackendGroupDeleteOptions{}, "lbbackendgroup-delete", "Show lbbackendgroup", func(s *mcclient.ClientSession, opts *options.LoadbalancerBackendGroupDeleteOptions) error {
+		lbbackendgroup, err := modules.LoadbalancerBackendGroups.Delete(s, opts.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(lbbackendgroup)
+		return nil
+	})
+}

cmd/climc/shell/loadbalancerbackends.go

@@ -0,0 +1,59 @@
+package shell
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func init() {
+	R(&options.LoadbalancerBackendCreateOptions{}, "lbbackend-create", "Create lbbackend", func(s *mcclient.ClientSession, opts *options.LoadbalancerBackendCreateOptions) error {
+		params, err := options.StructToParams(opts)
+		if err != nil {
+			return err
+		}
+		lbbackend, err := modules.LoadbalancerBackends.Create(s, params)
+		if err != nil {
+			return err
+		}
+		printObject(lbbackend)
+		return nil
+	})
+	R(&options.LoadbalancerBackendGetOptions{}, "lbbackend-show", "Show lbbackend", func(s *mcclient.ClientSession, opts *options.LoadbalancerBackendGetOptions) error {
+		lbbackend, err := modules.LoadbalancerBackends.Get(s, opts.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(lbbackend)
+		return nil
+	})
+	R(&options.LoadbalancerBackendListOptions{}, "lbbackend-list", "List lbbackends", func(s *mcclient.ClientSession, opts *options.LoadbalancerBackendListOptions) error {
+		params, err := options.ListStructToParams(opts)
+		if err != nil {
+			return err
+		}
+		result, err := modules.LoadbalancerBackends.List(s, params)
+		if err != nil {
+			return err
+		}
+		printList(result, modules.LoadbalancerBackends.GetColumns(s))
+		return nil
+	})
+	R(&options.LoadbalancerBackendUpdateOptions{}, "lbbackend-update", "Update lbbackend", func(s *mcclient.ClientSession, opts *options.LoadbalancerBackendUpdateOptions) error {
+		params, err := options.StructToParams(opts)
+		lbbackend, err := modules.LoadbalancerBackends.Update(s, opts.ID, params)
+		if err != nil {
+			return err
+		}
+		printObject(lbbackend)
+		return nil
+	})
+	R(&options.LoadbalancerBackendDeleteOptions{}, "lbbackend-delete", "Show lbbackend", func(s *mcclient.ClientSession, opts *options.LoadbalancerBackendDeleteOptions) error {
+		lbbackend, err := modules.LoadbalancerBackends.Delete(s, opts.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(lbbackend)
+		return nil
+	})
+}

cmd/climc/shell/loadbalancercertificates.go

@@ -0,0 +1,62 @@
+package shell
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func init() {
+	R(&options.LoadbalancerCertificateCreateOptions{}, "lbcert-create", "Create lbcert", func(s *mcclient.ClientSession, opts *options.LoadbalancerCertificateCreateOptions) error {
+		params, err := opts.Params()
+		if err != nil {
+			return err
+		}
+		lbcert, err := modules.LoadbalancerCertificates.Create(s, params)
+		if err != nil {
+			return err
+		}
+		printObject(lbcert)
+		return nil
+	})
+	R(&options.LoadbalancerCertificateGetOptions{}, "lbcert-show", "Show lbcert", func(s *mcclient.ClientSession, opts *options.LoadbalancerCertificateGetOptions) error {
+		lbcert, err := modules.LoadbalancerCertificates.Get(s, opts.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(lbcert)
+		return nil
+	})
+	R(&options.LoadbalancerCertificateListOptions{}, "lbcert-list", "List lbcerts", func(s *mcclient.ClientSession, opts *options.LoadbalancerCertificateListOptions) error {
+		params, err := options.ListStructToParams(opts)
+		if err != nil {
+			return err
+		}
+		result, err := modules.LoadbalancerCertificates.List(s, params)
+		if err != nil {
+			return err
+		}
+		printList(result, modules.LoadbalancerCertificates.GetColumns(s))
+		return nil
+	})
+	R(&options.LoadbalancerCertificateUpdateOptions{}, "lbcert-update", "Update lbcert", func(s *mcclient.ClientSession, opts *options.LoadbalancerCertificateUpdateOptions) error {
+		params, err := opts.Params()
+		if err != nil {
+			return err
+		}
+		lbcert, err := modules.LoadbalancerCertificates.Update(s, opts.ID, params)
+		if err != nil {
+			return err
+		}
+		printObject(lbcert)
+		return nil
+	})
+	R(&options.LoadbalancerCertificateDeleteOptions{}, "lbcert-delete", "Show lbcert", func(s *mcclient.ClientSession, opts *options.LoadbalancerCertificateDeleteOptions) error {
+		lbcert, err := modules.LoadbalancerCertificates.Delete(s, opts.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(lbcert)
+		return nil
+	})
+}

cmd/climc/shell/loadbalancerlistenerrules.go

@@ -0,0 +1,69 @@
+package shell
+
+import (
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func init() {
+	R(&options.LoadbalancerListenerRuleCreateOptions{}, "lblistenerrule-create", "Create lblistenerrule", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerRuleCreateOptions) error {
+		params := jsonutils.Marshal(opts)
+		lblistenerrule, err := modules.LoadbalancerListenerRules.Create(s, params)
+		if err != nil {
+			return err
+		}
+		printObject(lblistenerrule)
+		return nil
+	})
+	R(&options.LoadbalancerListenerRuleGetOptions{}, "lblistenerrule-show", "Show lblistenerrule", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerRuleGetOptions) error {
+		lblistenerrule, err := modules.LoadbalancerListenerRules.Get(s, opts.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(lblistenerrule)
+		return nil
+	})
+	R(&options.LoadbalancerListenerRuleListOptions{}, "lblistenerrule-list", "List lblistenerrules", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerRuleListOptions) error {
+		params, err := options.ListStructToParams(opts)
+		if err != nil {
+			return err
+		}
+		result, err := modules.LoadbalancerListenerRules.List(s, params)
+		if err != nil {
+			return err
+		}
+		printList(result, modules.LoadbalancerListenerRules.GetColumns(s))
+		return nil
+	})
+	R(&options.LoadbalancerListenerRuleUpdateOptions{}, "lblistenerrule-update", "Update lblistenerrule", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerRuleUpdateOptions) error {
+		params, err := options.StructToParams(opts)
+		lblistenerrule, err := modules.LoadbalancerListenerRules.Update(s, opts.ID, params)
+		if err != nil {
+			return err
+		}
+		printObject(lblistenerrule)
+		return nil
+	})
+	R(&options.LoadbalancerListenerRuleDeleteOptions{}, "lblistenerrule-delete", "Show lblistenerrule", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerRuleDeleteOptions) error {
+		lblistenerrule, err := modules.LoadbalancerListenerRules.Delete(s, opts.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(lblistenerrule)
+		return nil
+	})
+	R(&options.LoadbalancerListenerRuleActionStatusOptions{}, "lblistenerrule-status", "Change lblistenerrule status", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerRuleActionStatusOptions) error {
+		params, err := options.StructToParams(opts)
+		if err != nil {
+			return err
+		}
+		lblistenerrule, err := modules.LoadbalancerListenerRules.PerformAction(s, opts.ID, "status", params)
+		if err != nil {
+			return err
+		}
+		printObject(lblistenerrule)
+		return nil
+	})
+}

cmd/climc/shell/loadbalancerlisteners.go

@@ -0,0 +1,71 @@
+package shell
+
+import (
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func init() {
+
+	R(&options.LoadbalancerListenerCreateOptions{}, "lblistener-create", "Create lblistener", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerCreateOptions) error {
+		// TODO make a generic one
+		params := jsonutils.Marshal(opts)
+		lblistener, err := modules.LoadbalancerListeners.Create(s, params)
+		if err != nil {
+			return err
+		}
+		printObject(lblistener)
+		return nil
+	})
+	R(&options.LoadbalancerListenerGetOptions{}, "lblistener-show", "Show lblistener", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerGetOptions) error {
+		lblistener, err := modules.LoadbalancerListeners.Get(s, opts.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(lblistener)
+		return nil
+	})
+	R(&options.LoadbalancerListenerListOptions{}, "lblistener-list", "List lblisteners", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerListOptions) error {
+		params, err := options.ListStructToParams(opts)
+		if err != nil {
+			return err
+		}
+		result, err := modules.LoadbalancerListeners.List(s, params)
+		if err != nil {
+			return err
+		}
+		printList(result, modules.LoadbalancerListeners.GetColumns(s))
+		return nil
+	})
+	R(&options.LoadbalancerListenerUpdateOptions{}, "lblistener-update", "Update lblistener", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerUpdateOptions) error {
+		params, err := options.StructToParams(opts)
+		lblistener, err := modules.LoadbalancerListeners.Update(s, opts.ID, params)
+		if err != nil {
+			return err
+		}
+		printObject(lblistener)
+		return nil
+	})
+	R(&options.LoadbalancerListenerDeleteOptions{}, "lblistener-delete", "Show lblistener", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerDeleteOptions) error {
+		lblistener, err := modules.LoadbalancerListeners.Delete(s, opts.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(lblistener)
+		return nil
+	})
+	R(&options.LoadbalancerListenerActionStatusOptions{}, "lblistener-status", "Change lblistener status", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerActionStatusOptions) error {
+		params, err := options.StructToParams(opts)
+		if err != nil {
+			return err
+		}
+		lblistener, err := modules.LoadbalancerListeners.PerformAction(s, opts.ID, "status", params)
+		if err != nil {
+			return err
+		}
+		printObject(lblistener)
+		return nil
+	})
+}

cmd/climc/shell/loadbalancers.go

@@ -0,0 +1,71 @@
+package shell
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func init() {
+	R(&options.LoadbalancerCreateOptions{}, "lb-create", "Create lb", func(s *mcclient.ClientSession, opts *options.LoadbalancerCreateOptions) error {
+		params, err := options.StructToParams(opts)
+		if err != nil {
+			return err
+		}
+		lb, err := modules.Loadbalancers.Create(s, params)
+		if err != nil {
+			return err
+		}
+		printObject(lb)
+		return nil
+	})
+	R(&options.LoadbalancerGetOptions{}, "lb-show", "Show lb", func(s *mcclient.ClientSession, opts *options.LoadbalancerGetOptions) error {
+		lb, err := modules.Loadbalancers.Get(s, opts.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(lb)
+		return nil
+	})
+	R(&options.LoadbalancerListOptions{}, "lb-list", "List lbs", func(s *mcclient.ClientSession, opts *options.LoadbalancerListOptions) error {
+		params, err := options.ListStructToParams(opts)
+		if err != nil {
+			return err
+		}
+		result, err := modules.Loadbalancers.List(s, params)
+		if err != nil {
+			return err
+		}
+		printList(result, modules.Loadbalancers.GetColumns(s))
+		return nil
+	})
+	R(&options.LoadbalancerUpdateOptions{}, "lb-update", "Update lb", func(s *mcclient.ClientSession, opts *options.LoadbalancerUpdateOptions) error {
+		params, err := options.StructToParams(opts)
+		lb, err := modules.Loadbalancers.Update(s, opts.ID, params)
+		if err != nil {
+			return err
+		}
+		printObject(lb)
+		return nil
+	})
+	R(&options.LoadbalancerDeleteOptions{}, "lb-delete", "Show lb", func(s *mcclient.ClientSession, opts *options.LoadbalancerDeleteOptions) error {
+		lb, err := modules.Loadbalancers.Delete(s, opts.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(lb)
+		return nil
+	})
+	R(&options.LoadbalancerActionStatusOptions{}, "lb-status", "Change lb status", func(s *mcclient.ClientSession, opts *options.LoadbalancerActionStatusOptions) error {
+		params, err := options.StructToParams(opts)
+		if err != nil {
+			return err
+		}
+		lb, err := modules.Loadbalancers.PerformAction(s, opts.ID, "status", params)
+		if err != nil {
+			return err
+		}
+		printObject(lb)
+		return nil
+	})
+}

cmd/lbagent/main.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+
+	"yunion.io/x/log"
+	"yunion.io/x/onecloud/pkg/cloudcommon"
+	"yunion.io/x/onecloud/pkg/lbagent"
+)
+
+func main() {
+	opts := &lbagent.Options{}
+	commonOpts := &opts.CommonOpts
+	{
+		cloudcommon.ParseOptions(opts, commonOpts, os.Args, "lbagent.conf")
+		cloudcommon.InitAuth(commonOpts, func() {
+			log.Infof("auth finished ok")
+		})
+	}
+	if err := opts.ValidateThenInit(); err != nil {
+		log.Fatalf("opts validate: %s", err)
+	}
+
+	var haproxyHelper *lbagent.HaproxyHelper
+	var apiHelper *lbagent.ApiHelper
+	var err error
+	{
+		haproxyHelper, err = lbagent.NewHaproxyHelper(opts)
+		if err != nil {
+			log.Fatalf("init haproxy helper failed: %s", err)
+		}
+	}
+	{
+		apiHelper, err = lbagent.NewApiHelper(opts)
+		if err != nil {
+			log.Fatalf("init api helper failed: %s", err)
+		}
+	}
+
+	{
+		wg := &sync.WaitGroup{}
+		cmdChan := make(chan *lbagent.LbagentCmd) // internal
+		ctx, cancelFunc := context.WithCancel(context.Background())
+		ctx = context.WithValue(ctx, "wg", wg)
+		ctx = context.WithValue(ctx, "cmdChan", cmdChan)
+		wg.Add(2)
+		go haproxyHelper.Run(ctx)
+		go apiHelper.Run(ctx)
+
+		go func() {
+			sigChan := make(chan os.Signal)
+			signal.Notify(sigChan, syscall.SIGINT)
+			signal.Notify(sigChan, syscall.SIGTERM)
+			sig := <-sigChan
+			log.Infof("signal received: %s", sig)
+			cancelFunc()
+		}()
+		wg.Wait()
+	}
+}

pkg/compute/handlers.go

@@ -62,6 +62,14 @@ func InitHandlers(app *appsrv.Application) {
 		models.DnsRecordManager,
 		models.ElasticipManager,
 		models.SnapshotManager,
+		models.LoadbalancerManager,
+		models.LoadbalancerListenerManager,
+		models.LoadbalancerListenerRuleManager,
+		models.LoadbalancerBackendGroupManager,
+		models.LoadbalancerBackendManager,
+		models.LoadbalancerCertificateManager,
+		models.LoadbalancerAclManager,
+		models.LoadbalancerAgentManager,
 	} {
 		db.RegisterModelManager(manager)
 		handler := db.NewModelHandler(manager)

pkg/compute/models/loadbalancer_common.go

@@ -0,0 +1,30 @@
+package models
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+type SLoadbalancerStatus struct {
+	RuntimeStatus string `width:"36" charset:"ascii" nullable:"false" default:"init" list:"user"`
+}
+
+type ILoadbalancerSubResourceManager interface {
+	db.IModelManager
+
+	// PreDeleteSubs is to be called by upper manager to PreDelete models managed by this one
+	PreDeleteSubs(ctx context.Context, userCred mcclient.TokenCredential, q *sqlchemy.SQuery)
+}
+
+// TODO
+// notify on post create/update/delete
+type SLoadbalancerNotifier struct{}
+
+func (n *SLoadbalancerNotifier) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) {
+	return
+}

pkg/compute/models/loadbalancer_const.go

@@ -0,0 +1,215 @@
+package models
+
+import (
+	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+)
+
+// Load balancer status transition (for spec status)
+//
+//                      create          start           stop            delete
+//      init            running         -               -               -
+//      running		-		-		stopped		stopped
+//      stopped		-		running		-		-
+//
+// Each entity will have spec and runtime version.  Spec version will increment
+// on entity attribute update.  Runtime version will be filled by the scheduler
+// to the newest spec it has seen and committed
+//
+// When spec and runtime version differ, scheduler will set runtime version to
+// "configuring", "stopping" and will finally transition to a terminal state.
+//
+// In the case of instance has PendingDeleted marked, it is also the
+// scheduler's duty to make the runtime status to stopped and finally the
+// entity in question
+//
+const (
+	LB_STATUS_ENABLED  = "enabled"
+	LB_STATUS_DISABLED = "disabled"
+
+	LB_STATUS_INIT           = "init"
+	LB_STATUS_RUNNING        = "running"
+	LB_STATUS_STOPPED        = "stopped"
+	LB_STATUS_CONFIGURING    = "configuring" // config changes pending
+	LB_STATUS_STOPPING       = "stopping"
+	LB_STATUS_DELETE_PENDING = "delete_pending"
+	LB_STATUS_ERROR          = "error" // bad things happen
+)
+
+var LB_STATUS_SPEC = validators.NewChoices(
+	LB_STATUS_ENABLED,
+	LB_STATUS_DISABLED,
+)
+
+var LB_STATUS_RUNTIME = validators.NewChoices(
+	LB_STATUS_INIT,
+	LB_STATUS_CONFIGURING,
+	LB_STATUS_RUNNING,
+	LB_STATUS_STOPPING,
+	LB_STATUS_STOPPED,
+	LB_STATUS_ERROR,
+)
+
+// Load Balancer network type (vpc or classic) determines viable backend
+// servers (they should be from the same network type as the load balancer).
+//
+// Load Balancer address type (intranet or internet) determins the scope the
+// service provided by load balancer can be accessed.  If it's intranet, then
+// it will only be accessible from the specified network.  If it's internet,
+// then it's public and can be accessed from outside the cloud region
+const (
+	LB_ADDR_TYPE_INTRANET = "intranet"
+	LB_ADDR_TYPE_INTERNET = "internet"
+)
+
+var LB_ADDR_TYPES = validators.NewChoices(
+	LB_ADDR_TYPE_INTERNET,
+	LB_ADDR_TYPE_INTRANET,
+)
+
+const (
+	LB_NETWORK_TYPE_CLASSIC = "classic"
+	LB_NETWORK_TYPE_VPC     = "vpc"
+)
+
+var LB_NETWORK_TYPES = validators.NewChoices(
+	LB_NETWORK_TYPE_CLASSIC,
+	LB_NETWORK_TYPE_VPC,
+)
+
+// TODO https_direct sni
+const (
+	LB_LISTENER_TYPE_TCP   = "tcp"
+	LB_LISTENER_TYPE_UDP   = "udp"
+	LB_LISTENER_TYPE_HTTP  = "http"
+	LB_LISTENER_TYPE_HTTPS = "https"
+)
+
+var LB_LISTENER_TYPES = validators.NewChoices(
+	LB_LISTENER_TYPE_TCP,
+	LB_LISTENER_TYPE_UDP,
+	LB_LISTENER_TYPE_HTTP,
+	LB_LISTENER_TYPE_HTTPS,
+)
+
+const (
+	LB_ACL_TYPE_BLACK = "black"
+	LB_ACL_TYPE_WHITE = "white"
+)
+
+var LB_ACL_TYPES = validators.NewChoices(
+	LB_ACL_TYPE_BLACK,
+	LB_ACL_TYPE_WHITE,
+)
+
+const (
+	LB_TLS_CERT_PUBKEY_ALGO_RSA   = "RSA"
+	LB_TLS_CERT_PUBKEY_ALGO_ECDSA = "ECDSA"
+)
+
+var LB_TLS_CERT_PUBKEY_ALGOS = validators.NewChoices(
+	LB_TLS_CERT_PUBKEY_ALGO_RSA,
+	LB_TLS_CERT_PUBKEY_ALGO_ECDSA,
+)
+
+// TODO may want extra for legacy apps
+const (
+	LB_TLS_CIPHER_POLICY_1_0        = "tls_cipher_policy_1_0"
+	LB_TLS_CIPHER_POLICY_1_1        = "tls_cipher_policy_1_1"
+	LB_TLS_CIPHER_POLICY_1_2        = "tls_cipher_policy_1_2"
+	LB_TLS_CIPHER_POLICY_1_2_strict = "tls_cipher_policy_1_2_strict"
+)
+
+var LB_TLS_CIPHER_POLICIES = validators.NewChoices(
+	LB_TLS_CIPHER_POLICY_1_0,
+	LB_TLS_CIPHER_POLICY_1_1,
+	LB_TLS_CIPHER_POLICY_1_2,
+	LB_TLS_CIPHER_POLICY_1_2_strict,
+)
+
+const (
+	LB_STICKY_SESSION_TYPE_INSERT = "insert"
+	LB_STICKY_SESSION_TYPE_SERVER = "server"
+)
+
+var LB_STICKY_SESSION_TYPES = validators.NewChoices(
+	LB_STICKY_SESSION_TYPE_INSERT,
+	LB_STICKY_SESSION_TYPE_SERVER,
+)
+
+// TODO maybe https check when field need comes ;)
+const (
+	LB_HEALTH_CHECK_TCP  = "tcp"
+	LB_HEALTH_CHECK_UDP  = "udp"
+	LB_HEALTH_CHECK_HTTP = "http"
+)
+
+var LB_HEALTH_CHECK_TYPES = validators.NewChoices(
+	LB_HEALTH_CHECK_TCP,
+	LB_HEALTH_CHECK_UDP,
+	LB_HEALTH_CHECK_HTTP,
+)
+
+var LB_HEALTH_CHECK_TYPES_TCP = validators.NewChoices(
+	LB_HEALTH_CHECK_TCP,
+	LB_HEALTH_CHECK_HTTP,
+)
+
+var LB_HEALTH_CHECK_TYPES_UDP = validators.NewChoices(
+	LB_HEALTH_CHECK_UDP,
+)
+
+const (
+	LB_HEALTH_CHECK_HTTP_CODE_2xx     = "http_2xx"
+	LB_HEALTH_CHECK_HTTP_CODE_3xx     = "http_3xx"
+	LB_HEALTH_CHECK_HTTP_CODE_4xx     = "http_4xx"
+	LB_HEALTH_CHECK_HTTP_CODE_5xx     = "http_5xx"
+	LB_HEALTH_CHECK_HTTP_CODE_DEFAULT = "http_2xx,http_3xx"
+)
+
+var LB_HEALTH_CHECK_HTTP_CODES = validators.NewChoices(
+	LB_HEALTH_CHECK_HTTP_CODE_2xx,
+	LB_HEALTH_CHECK_HTTP_CODE_3xx,
+	LB_HEALTH_CHECK_HTTP_CODE_4xx,
+	LB_HEALTH_CHECK_HTTP_CODE_5xx,
+)
+
+const (
+	LB_BOOL_ON  = "on"
+	LB_BOOL_OFF = "off"
+)
+
+var LB_BOOL_VALUES = validators.NewChoices(
+	LB_BOOL_ON,
+	LB_BOOL_OFF,
+)
+
+//TODO
+//
+// - qch, quic connection id
+// - mh, maglev consistent hash
+const (
+	LB_SCHEDULER_RR  = "rr"  // round robin
+	LB_SCHEDULER_WRR = "wrr" // weighted round robin
+	LB_SCHEDULER_WLC = "wlc" // weighted least connection
+	LB_SCHEDULER_SCH = "sch" // source-ip-based consistent hash
+	LB_SCHEDULER_TCH = "tch" // 4-tuple-based consistent hash
+)
+
+var LB_SCHEDULER_TYPES = validators.NewChoices(
+	LB_SCHEDULER_RR,
+	LB_SCHEDULER_WRR,
+	LB_SCHEDULER_WLC,
+	LB_SCHEDULER_SCH,
+	LB_SCHEDULER_TCH,
+)
+
+// TODO raw type
+const (
+	LB_BACKEND_GUEST = "guest"
+	LB_BACKEND_HOST  = "host"
+)
+
+var LB_BACKEND_TYPES = validators.NewChoices(
+	LB_BACKEND_GUEST,
+	LB_BACKEND_HOST,
+)

pkg/compute/models/loadbalanceracls.go

@@ -0,0 +1,214 @@
+package models
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"reflect"
+	"strings"
+	"unicode"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/gotypes"
+	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+type SLoadbalancerAclEntry struct {
+	Cidr    string
+	Comment string
+}
+
+func (aclEntry *SLoadbalancerAclEntry) Validate(data *jsonutils.JSONDict) error {
+	if strings.Index(aclEntry.Cidr, "/") > 0 {
+		_, ipNet, err := net.ParseCIDR(aclEntry.Cidr)
+		if err != nil {
+			return err
+		}
+		// normalize from 192.168.1.3/24 to 192.168.1.0/24
+		aclEntry.Cidr = ipNet.String()
+	} else {
+		ip := net.ParseIP(aclEntry.Cidr).To4()
+		if ip == nil {
+			return fmt.Errorf("invalid addr %s", aclEntry.Cidr)
+		}
+	}
+	if commentLimit := 128; len(aclEntry.Comment) > commentLimit {
+		return fmt.Errorf("comment too long (%d>=%d)",
+			len(aclEntry.Comment), commentLimit)
+	}
+	for _, r := range aclEntry.Comment {
+		if !unicode.IsPrint(r) {
+			return fmt.Errorf("comment contains non-printable char: %v", r)
+		}
+	}
+	return nil
+}
+
+type SLoadbalancerAclEntries []*SLoadbalancerAclEntry
+
+func (aclEntries *SLoadbalancerAclEntries) String() string {
+	return jsonutils.Marshal(aclEntries).String()
+}
+func (aclEntries *SLoadbalancerAclEntries) IsZero() bool {
+	if len([]*SLoadbalancerAclEntry(*aclEntries)) == 0 {
+		return true
+	}
+	return false
+}
+
+func (aclEntries *SLoadbalancerAclEntries) Validate(data *jsonutils.JSONDict) error {
+	found := map[string]bool{}
+	for _, aclEntry := range *aclEntries {
+		if err := aclEntry.Validate(data); err != nil {
+			return err
+		}
+		if _, ok := found[aclEntry.Cidr]; ok {
+			// error so that the user has a chance to deal with comments
+			return fmt.Errorf("acl cidr duplicate %s", aclEntry.Cidr)
+		}
+		found[aclEntry.Cidr] = true
+	}
+	return nil
+}
+
+type SLoadbalancerAclManager struct {
+	db.SSharableVirtualResourceBaseManager
+}
+
+var LoadbalancerAclManager *SLoadbalancerAclManager
+
+func init() {
+	gotypes.RegisterSerializable(reflect.TypeOf(&SLoadbalancerAclEntries{}), func() gotypes.ISerializable {
+		return &SLoadbalancerAclEntries{}
+	})
+	LoadbalancerAclManager = &SLoadbalancerAclManager{
+		SSharableVirtualResourceBaseManager: db.NewSharableVirtualResourceBaseManager(
+			SLoadbalancerAcl{},
+			"loadbalanceracls_tbl",
+			"loadbalanceracl",
+			"loadbalanceracls",
+		),
+	}
+}
+
+type SLoadbalancerAcl struct {
+	db.SSharableVirtualResourceBase
+
+	AclEntries *SLoadbalancerAclEntries `list:"user" update:"user" create:"required"`
+}
+
+func loadbalancerAclsValidateAclEntries(data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	aclEntries := SLoadbalancerAclEntries{}
+	aclEntriesV := validators.NewStructValidator("acl_entries", &aclEntries)
+	err := aclEntriesV.Validate(data)
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}
+
+func (man *SLoadbalancerAclManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	data, err := loadbalancerAclsValidateAclEntries(data)
+	if err != nil {
+		return nil, err
+	}
+	return man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)
+}
+
+func (lbacl *SLoadbalancerAcl) AllowPerformStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return false
+}
+
+func (lbacl *SLoadbalancerAcl) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	data, err := loadbalancerAclsValidateAclEntries(data)
+	if err != nil {
+		return nil, err
+	}
+	return lbacl.SSharableVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data)
+}
+
+func (lbacl *SLoadbalancerAcl) AllowPerformPatch(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) bool {
+	return lbacl.IsOwner(userCred) || userCred.IsSystemAdmin()
+}
+
+// PerformPatch patches acl entries by adding then deleting the specified acls.
+// This is intended mainly for command line operations.
+func (lbacl *SLoadbalancerAcl) PerformPatch(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	aclEntries := gotypes.DeepCopy(*lbacl.AclEntries).(SLoadbalancerAclEntries)
+	{
+		adds := SLoadbalancerAclEntries{}
+		addsV := validators.NewStructValidator("adds", &adds)
+		addsV.Optional(true)
+		err := addsV.Validate(data)
+		if err != nil {
+			return nil, err
+		}
+		for _, add := range adds {
+			found := false
+			for _, aclEntry := range aclEntries {
+				if aclEntry.Cidr == add.Cidr {
+					found = true
+					aclEntry.Comment = add.Comment
+					break
+				}
+			}
+			if !found {
+				aclEntries = append(aclEntries, add)
+			}
+		}
+	}
+	{
+		dels := SLoadbalancerAclEntries{}
+		delsV := validators.NewStructValidator("dels", &dels)
+		delsV.Optional(true)
+		err := delsV.Validate(data)
+		if err != nil {
+			return nil, err
+		}
+		for _, del := range dels {
+			for i := len(aclEntries) - 1; i >= 0; i-- {
+				aclEntry := aclEntries[i]
+				if aclEntry.Cidr == del.Cidr {
+					aclEntries = append(aclEntries[:i], aclEntries[i+1:]...)
+					break
+				}
+			}
+		}
+	}
+	_, err := lbacl.GetModelManager().TableSpec().Update(lbacl, func() error {
+		lbacl.AclEntries = &aclEntries
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return nil, nil
+}
+
+func (lbacl *SLoadbalancerAcl) ValidateDeleteCondition(ctx context.Context) error {
+	man := LoadbalancerListenerManager
+	t := man.TableSpec().Instance()
+	pdF := t.Field("pending_deleted")
+	lbaclId := lbacl.Id
+	n := t.Query().
+		Filter(sqlchemy.OR(sqlchemy.IsNull(pdF), sqlchemy.IsFalse(pdF))).
+		Equals("acl_id", lbaclId).
+		Count()
+	if n > 0 {
+		return fmt.Errorf("acl %s is still referred to by %d %s",
+			lbaclId, n, man.KeywordPlural())
+	}
+	return nil
+}
+
+func (lbacl *SLoadbalancerAcl) PreDelete(ctx context.Context, userCred mcclient.TokenCredential) {
+	lbacl.DoPendingDelete(ctx, userCred)
+}
+
+func (lbacl *SLoadbalancerAcl) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	return nil
+}

pkg/compute/models/loadbalanceragents.go

@@ -0,0 +1,457 @@
+package models
+
+import (
+	"context"
+	"encoding/base64"
+	"reflect"
+	"text/template"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/gotypes"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+type SLoadbalancerAgentManager struct {
+	db.SStandaloneResourceBaseManager
+	SInfrastructureManager
+}
+
+var LoadbalancerAgentManager *SLoadbalancerAgentManager
+
+func init() {
+	gotypes.RegisterSerializable(reflect.TypeOf(&SLoadbalancerAgentParams{}), func() gotypes.ISerializable {
+		return &SLoadbalancerAgentParams{}
+	})
+	LoadbalancerAgentManager = &SLoadbalancerAgentManager{
+		SStandaloneResourceBaseManager: db.NewStandaloneResourceBaseManager(
+			SLoadbalancerAgent{},
+			"loadbalanceragents_tbl",
+			"loadbalanceragent",
+			"loadbalanceragents",
+		),
+	}
+}
+
+// TODO
+//
+//  - scrub stale backends: Guests with deleted=1
+//  - agent configuration params
+//
+type SLoadbalancerAgent struct {
+	db.SStandaloneResourceBase
+	SInfrastructure
+
+	HbLastSeen time.Time                 `nullable:"true" list:"admin" update:"admin"`
+	HbTimeout  int                       `nullable:"true" list:"admin" update:"admin" create:"optional" default:"3600"`
+	Params     *SLoadbalancerAgentParams `create:"optional" get:"admin"`
+
+	Loadbalancers             time.Time `nullable:"true" list:"admin" update:"admin"`
+	LoadbalancerListeners     time.Time `nullable:"true" list:"admin" update:"admin"`
+	LoadbalancerListenerRules time.Time `nullable:"true" list:"admin" update:"admin"`
+	LoadbalancerBackendGroups time.Time `nullable:"true" list:"admin" update:"admin"`
+	LoadbalancerBackends      time.Time `nullable:"true" list:"admin" update:"admin"`
+	LoadbalancerAcls          time.Time `nullable:"true" list:"admin" update:"admin"`
+	LoadbalancerCertificates  time.Time `nullable:"true" list:"admin" update:"admin"`
+}
+
+type SLoadbalancerAgentParamsVrrp struct {
+	Priority          int
+	VirtualRouterId   int
+	GarpMasterRefresh int
+	Preempt           bool
+	Interface         string
+	AdvertInt         int
+	Pass              string
+}
+
+type SLoadbalancerAgentParamsHaproxy struct {
+	GlobalLog      string
+	GlobalNbthread int
+	LogHttp        bool
+	LogTcp         bool
+	LogNormal      bool
+}
+
+type SLoadbalancerAgentParams struct {
+	KeepalivedConfTmpl string
+	HaproxyConfTmpl    string
+	Vrrp               SLoadbalancerAgentParamsVrrp
+	Haproxy            SLoadbalancerAgentParamsHaproxy
+}
+
+func (p *SLoadbalancerAgentParamsVrrp) Validate(data *jsonutils.JSONDict) error {
+	if len(p.Interface) == 0 || len(p.Interface) > 16 {
+		// TODO printable exclude white space
+		return httperrors.NewInputParameterError("invalid vrrp interface %q", p.Interface)
+	}
+	if len(p.Pass) == 0 || len(p.Pass) > 8 {
+		// TODO printable exclude white space
+		return httperrors.NewInputParameterError("invalid vrrp authentication pass size: %d, want [1,8]", len(p.Pass))
+	}
+	if p.Priority < 1 || p.Priority > 255 {
+		return httperrors.NewInputParameterError("invalid vrrp priority %d: want [1,255]", p.Priority)
+	}
+	if p.VirtualRouterId < 1 || p.VirtualRouterId > 255 {
+		return httperrors.NewInputParameterError("invalid vrrp virtual_router_id %d: want [1,255]", p.VirtualRouterId)
+	}
+	return nil
+}
+
+func (p *SLoadbalancerAgentParamsVrrp) initDefault(data *jsonutils.JSONDict) {
+	if !data.Contains("params", "vrrp", "advert_int") {
+		p.AdvertInt = 1
+	}
+	if !data.Contains("params", "vrrp", "garp_master_refresh") {
+		p.GarpMasterRefresh = 27
+	}
+	if !data.Contains("params", "vrrp", "pass") {
+		p.Pass = "YunionLB"
+	}
+}
+
+func (p *SLoadbalancerAgentParamsHaproxy) Validate(data *jsonutils.JSONDict) error {
+	if p.GlobalNbthread < 1 {
+		p.GlobalNbthread = 1
+	}
+	if p.GlobalNbthread > 64 {
+		// This is a limit imposed by haproxy and arch word size
+		p.GlobalNbthread = 64
+	}
+	return nil
+}
+
+func (p *SLoadbalancerAgentParamsHaproxy) initDefault(data *jsonutils.JSONDict) {
+	if !data.Contains("params", "haproxy", "global_nbthread") {
+		p.GlobalNbthread = 1
+	}
+	if !data.Contains("params", "haproxy", "global_log") {
+		p.GlobalLog = "log /dev/log local0 info"
+	}
+	if !data.Contains("params", "haproxy", "log_http") {
+		p.LogHttp = true
+	}
+	if !data.Contains("params", "haproxy", "log_normal") {
+		p.LogNormal = true
+	}
+}
+
+func (p *SLoadbalancerAgentParams) validateTmpl(k, s string) error {
+	d, err := base64.StdEncoding.DecodeString(s)
+	if err != nil {
+		return httperrors.NewInputParameterError("%s: bad base64 encoded string: %s", k, err)
+	}
+	s = string(d)
+	_, err = template.New("").Parse(s)
+	if err != nil {
+		return httperrors.NewInputParameterError("%s: bad template: %s", k, err)
+	}
+	return nil
+}
+
+func (p *SLoadbalancerAgentParams) initDefault(data *jsonutils.JSONDict) {
+	if p.KeepalivedConfTmpl == "" {
+		p.KeepalivedConfTmpl = loadbalancerKeepalivedConfTmplDefaultEncoded
+	}
+	if p.HaproxyConfTmpl == "" {
+		p.HaproxyConfTmpl = loadbalancerHaproxyConfTmplDefaultEncoded
+	}
+	p.Vrrp.initDefault(data)
+	p.Haproxy.initDefault(data)
+}
+
+func (p *SLoadbalancerAgentParams) Validate(data *jsonutils.JSONDict) error {
+	p.initDefault(data)
+	if err := p.validateTmpl("keepalived_conf_tmpl", p.KeepalivedConfTmpl); err != nil {
+		return err
+	}
+	if err := p.validateTmpl("haproxy_conf_tmpl", p.HaproxyConfTmpl); err != nil {
+		return err
+	}
+	if err := p.Vrrp.Validate(data); err != nil {
+		return err
+	}
+	if err := p.Haproxy.Validate(data); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (p *SLoadbalancerAgentParams) String() string {
+	return jsonutils.Marshal(p).String()
+}
+
+func (p *SLoadbalancerAgentParams) IsZero() bool {
+	if *p == (SLoadbalancerAgentParams{}) {
+		return true
+	}
+	return false
+}
+
+func (man *SLoadbalancerAgentManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	{
+		keyV := map[string]validators.IValidator{
+			"hb_timeout": validators.NewNonNegativeValidator("nb_timeout").Default(3600),
+			"params":     validators.NewStructValidator("params", &SLoadbalancerAgentParams{}),
+		}
+		for _, v := range keyV {
+			if err := v.Validate(data); err != nil {
+				return nil, err
+			}
+		}
+	}
+	return man.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)
+}
+
+func (man *SLoadbalancerAgentManager) CleanPendingDeleteLoadbalancers(ctx context.Context, userCred mcclient.TokenCredential) {
+	agents := []SLoadbalancerAgent{}
+	{
+		// find active agents
+		err := man.Query().All(&agents)
+		if err != nil {
+			log.Errorf("query agents failed")
+			return
+		}
+		i := 0
+		for _, agent := range agents {
+			if !agent.IsActive() {
+				continue
+			}
+			agents[i] = agent
+			i++
+		}
+		agents = agents[:i]
+	}
+	men := map[string]db.IModelManager{
+		"loadbalancers":               LoadbalancerManager,
+		"loadbalancer_listeners":      LoadbalancerListenerManager,
+		"loadbalancer_listener_rules": LoadbalancerListenerRuleManager,
+		"loadbalancer_backend_groups": LoadbalancerBackendGroupManager,
+		"loadbalancer_backends":       LoadbalancerBackendManager,
+		"loadbalancer_acls":           LoadbalancerAclManager,
+		"loadbalancer_certificates":   LoadbalancerCertificateManager,
+	}
+	agentsData := jsonutils.Marshal(&agents).(*jsonutils.JSONArray)
+	for fieldName, man := range men {
+		keyPlural := man.KeywordPlural()
+		now := time.Now()
+		minT := now
+		if len(agents) > 0 {
+			// find min updated_at seen by these active agents
+			for i := 0; i < agentsData.Length(); i++ {
+				agentData, _ := agentsData.GetAt(i)
+				t, err := agentData.GetTime(fieldName)
+				if err != nil {
+					continue
+				}
+				if minT.After(t) {
+					minT = t
+				}
+			}
+			if minT.Equal(now) {
+				log.Warningf("%s: no agents has reported yet", keyPlural)
+				continue
+			}
+		} else {
+			// when no active agents exists, we are free to go
+		}
+		{
+			// find resources pending deleted before minT
+			q := man.Query().IsTrue("pending_deleted").LT("pending_deleted_at", minT)
+			rows, err := q.Rows()
+			if err != nil {
+				log.Errorf("%s: query pending_deleted_at < %s: %s", keyPlural, minT, err)
+				continue
+			}
+			m, err := db.NewModelObject(man)
+			if err != nil {
+				log.Errorf("%s: new model object failed: %s", keyPlural, err)
+				continue
+			}
+			mInitValue := reflect.Indirect(reflect.ValueOf(m))
+			m, _ = db.NewModelObject(man)
+			for rows.Next() {
+				reflect.Indirect(reflect.ValueOf(m)).Set(mInitValue)
+				err := q.Row2Struct(rows, m)
+				if err != nil {
+					log.Errorf("%s: Row2Struct: %s", keyPlural, err)
+					continue
+				}
+				{
+					// find real delete method
+					rv := reflect.Indirect(reflect.ValueOf(m))
+					baseRv := rv.FieldByName("SVirtualResourceBase")
+					if !baseRv.IsValid() {
+						baseRv = rv.FieldByName("SSharableVirtualResourceBase")
+					}
+					if !baseRv.IsValid() {
+						log.Errorf("%s: cannot find base resource field", keyPlural)
+						break // no need to try again
+					}
+					// now update deleted,deleted_at fields
+					realDeleteMethod := baseRv.Addr().MethodByName("Delete")
+					retRv := realDeleteMethod.Call([]reflect.Value{
+						reflect.ValueOf(ctx),
+						reflect.ValueOf(userCred),
+					})
+					err := retRv[0].Interface()
+					if !gotypes.IsNil(err) {
+						log.Errorf("%s: real delete failed: %s", keyPlural, err.(error))
+					}
+				}
+			}
+		}
+	}
+}
+
+func (man *SLoadbalancerAgentManager) AllowCreateItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return userCred.IsSystemAdmin()
+}
+
+func (lbagent *SLoadbalancerAgent) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	keys := map[string]time.Time{
+		"loadbalancers":               lbagent.Loadbalancers,
+		"loadbalancer_listeners":      lbagent.LoadbalancerListeners,
+		"loadbalancer_listener_rules": lbagent.LoadbalancerListenerRules,
+		"loadbalancer_backend_groups": lbagent.LoadbalancerBackendGroups,
+		"loadbalancer_backends":       lbagent.LoadbalancerBackends,
+		"loadbalancer_acls":           lbagent.LoadbalancerAcls,
+		"loadbalancer_certificates":   lbagent.LoadbalancerCertificates,
+	}
+	for k, curValue := range keys {
+		if !data.Contains(k) {
+			continue
+		}
+		newValue, err := data.GetTime(k)
+		if err != nil {
+			return nil, httperrors.NewInputParameterError("%s: time error: %s", k, err)
+		}
+		if newValue.Before(curValue) {
+			// this is possible with objects deleted
+			data.Remove(k)
+			continue
+		}
+		if now := time.Now(); newValue.After(now) {
+			return nil, httperrors.NewInputParameterError("%s: new time is in the future: %s > %s",
+				k, newValue, now)
+		}
+	}
+	data.Set("hb_last_seen", jsonutils.NewTimeString(time.Now()))
+	return data, nil
+}
+
+func (lbagent *SLoadbalancerAgent) AllowPerformHb(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) bool {
+	return userCred.IsSystemAdmin()
+}
+
+func (lbagent *SLoadbalancerAgent) PerformHb(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	_, err := lbagent.GetModelManager().TableSpec().Update(lbagent, func() error {
+		lbagent.HbLastSeen = time.Now()
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return nil, nil
+}
+
+func (lbagent *SLoadbalancerAgent) IsActive() bool {
+	if lbagent.HbLastSeen.IsZero() {
+		return false
+	}
+	duration := time.Since(lbagent.HbLastSeen).Seconds()
+	if int(duration) >= lbagent.HbTimeout {
+		return false
+	}
+	return true
+}
+
+func (lbagent *SLoadbalancerAgent) AllowPerformParamsPatch(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) bool {
+	return userCred.IsSystemAdmin()
+}
+
+func (lbagent *SLoadbalancerAgent) PerformParamsPatch(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	params := gotypes.DeepCopy(*lbagent.Params).(SLoadbalancerAgentParams)
+	d := jsonutils.NewDict()
+	d.Set("params", data)
+	paramsV := validators.NewStructValidator("params", &params)
+	if err := paramsV.Validate(d); err != nil {
+		return nil, err
+	}
+	{
+		_, err := lbagent.GetModelManager().TableSpec().Update(lbagent, func() error {
+			lbagent.Params = &params
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+	return nil, nil
+}
+
+const (
+	loadbalancerKeepalivedConfTmplDefault = `
+global_defs {
+	router_id {{ .agent.id }}
+	#vrrp_strict
+	vrrp_skip_check_adv_addr
+	enable_script_security
+}
+
+vrrp_instance YunionLB {
+	interface {{ .vrrp.interface }}
+	virtual_router_id {{ .vrrp.virtual_router_id }}
+	authentication {
+		auth_type PASS
+		auth_pass {{ .vrrp.pass }}
+	}
+	priority {{ .vrrp.priority }}
+	advert_int {{ .vrrp.advert_int }}
+	garp_master_refresh {{ .vrrp.garp_master_refresh }}
+	{{ if .vrrp.preempt -}} preempt {{- else -}} nopreempt {{- end }}
+	virtual_ipaddress {
+		{{- printf "\n" }}
+		{{- range .vrrp.addresses }}		{{ println . }} {{- end }}
+		{{- printf "\t" -}}
+	}
+}
+`
+	loadbalancerHaproxyConfTmplDefault = `
+global
+	maxconn 20480
+	tune.ssl.default-dh-param 2048
+	{{- println }}
+	{{- if .haproxy.global_stats_socket }}	{{ println .haproxy.global_stats_socket }} {{- end }}
+	{{- if .haproxy.global_nbthread }}	nbthread {{ println .haproxy.global_nbthread }} {{- end }}
+	{{- if .haproxy.global_log }}	{{ println .haproxy.global_log }} {{- end }}
+
+defaults
+	timeout connect 10s
+	timeout client 60s
+	timeout server 60s
+	timeout tunnel 1h
+	{{- println }}
+	{{- if .haproxy.global_log }}	{{ println "log global" }} {{- end }}
+	{{- if not .haproxy.log_normal }}	{{ println "option dontlog-normal" }} {{- end }}
+
+listen stats
+	mode http
+	bind :778
+	stats enable
+	stats hide-version
+	stats realm "Haproxy Statistics"
+	stats auth Yunion:LBStats
+	stats uri /
+`
+)
+
+var (
+	loadbalancerKeepalivedConfTmplDefaultEncoded = base64.StdEncoding.EncodeToString([]byte(loadbalancerKeepalivedConfTmplDefault))
+	loadbalancerHaproxyConfTmplDefaultEncoded    = base64.StdEncoding.EncodeToString([]byte(loadbalancerHaproxyConfTmplDefault))
+)

pkg/compute/models/loadbalancerbackendgroups.go

@@ -0,0 +1,122 @@
+package models
+
+import (
+	"context"
+	"fmt"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
+	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+type SLoadbalancerBackendGroupManager struct {
+	db.SVirtualResourceBaseManager
+}
+
+var LoadbalancerBackendGroupManager *SLoadbalancerBackendGroupManager
+
+func init() {
+	LoadbalancerBackendGroupManager = &SLoadbalancerBackendGroupManager{
+		SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager(
+			SLoadbalancerBackendGroup{},
+			"loadbalancerbackendgroups_tbl",
+			"loadbalancerbackendgroup",
+			"loadbalancerbackendgroups",
+		),
+	}
+}
+
+type SLoadbalancerBackendGroup struct {
+	db.SVirtualResourceBase
+
+	LoadbalancerId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional"`
+}
+
+func (man *SLoadbalancerBackendGroupManager) PreDeleteSubs(ctx context.Context, userCred mcclient.TokenCredential, q *sqlchemy.SQuery) {
+	subs := []SLoadbalancerBackendGroup{}
+	db.FetchModelObjects(man, q, &subs)
+	for _, sub := range subs {
+		sub.PreDelete(ctx, userCred)
+	}
+}
+
+func (man *SLoadbalancerBackendGroupManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
+	q, err := man.SVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query)
+	if err != nil {
+		return nil, err
+	}
+	userProjId := userCred.GetProjectId()
+	data := query.(*jsonutils.JSONDict)
+	{
+		lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", userProjId)
+		lbV.Optional(true)
+		q, err = lbV.QueryFilter(q, data)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return q, nil
+}
+
+func (man *SLoadbalancerBackendGroupManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", ownerProjId)
+	err := lbV.Validate(data)
+	if err != nil {
+		return nil, err
+	}
+	return man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)
+}
+
+func (lbbg *SLoadbalancerBackendGroup) GetLoadbalancer() *SLoadbalancer {
+	lb, _ := LoadbalancerManager.FetchById(lbbg.LoadbalancerId)
+	return lb.(*SLoadbalancer)
+}
+
+func (lbbg *SLoadbalancerBackendGroup) AllowPerformStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return false
+}
+
+func (lbbg *SLoadbalancerBackendGroup) ValidateDeleteCondition(ctx context.Context) error {
+	men := []db.IModelManager{
+		LoadbalancerManager,
+		LoadbalancerListenerManager,
+		LoadbalancerListenerRuleManager,
+	}
+	lbbgId := lbbg.Id
+	for _, man := range men {
+		t := man.TableSpec().Instance()
+		pdF := t.Field("pending_deleted")
+		n := t.Query().
+			Equals("backend_group_id", lbbgId).
+			Filter(sqlchemy.OR(sqlchemy.IsNull(pdF), sqlchemy.IsFalse(pdF))).
+			Count()
+		if n > 0 {
+			return fmt.Errorf("backend group %s is still referred to by %d %s",
+				lbbgId, n, man.KeywordPlural())
+		}
+	}
+	return nil
+}
+
+func (lbbg *SLoadbalancerBackendGroup) PreDelete(ctx context.Context, userCred mcclient.TokenCredential) {
+	lbbg.DoPendingDelete(ctx, userCred)
+	lbbg.PreDeleteSubs(ctx, userCred)
+}
+
+func (lbbg *SLoadbalancerBackendGroup) PreDeleteSubs(ctx context.Context, userCred mcclient.TokenCredential) {
+	subMan := LoadbalancerBackendManager
+	ownerProjId := lbbg.GetOwnerProjectId()
+
+	lockman.LockClass(ctx, subMan, ownerProjId)
+	defer lockman.ReleaseClass(ctx, subMan, ownerProjId)
+	q := subMan.Query().Equals("backend_group_id", lbbg.Id)
+	subMan.PreDeleteSubs(ctx, userCred, q)
+}
+
+func (lbbg *SLoadbalancerBackendGroup) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	return nil
+}

pkg/compute/models/loadbalancerbackends.go

@@ -0,0 +1,194 @@
+package models
+
+import (
+	"context"
+	"fmt"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+type SLoadbalancerBackendManager struct {
+	db.SVirtualResourceBaseManager
+}
+
+var LoadbalancerBackendManager *SLoadbalancerBackendManager
+
+func init() {
+	LoadbalancerBackendManager = &SLoadbalancerBackendManager{
+		SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager(
+			SLoadbalancerBackend{},
+			"loadbalancerbackends_tbl",
+			"loadbalancerbackend",
+			"loadbalancerbackends",
+		),
+	}
+}
+
+type SLoadbalancerBackend struct {
+	db.SVirtualResourceBase
+
+	BackendGroupId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional"`
+	BackendId      string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional"`
+	BackendType    string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional"`
+	Weight         int    `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"`
+	Address        string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional"`
+	Port           int    `nullable:"false" list:"user" create:"required" update:"user"`
+}
+
+func (man *SLoadbalancerBackendManager) PreDeleteSubs(ctx context.Context, userCred mcclient.TokenCredential, q *sqlchemy.SQuery) {
+	subs := []SLoadbalancerBackend{}
+	db.FetchModelObjects(man, q, &subs)
+	for _, sub := range subs {
+		sub.PreDelete(ctx, userCred)
+	}
+}
+
+func (man *SLoadbalancerBackendManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
+	q, err := man.SVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query)
+	if err != nil {
+		return nil, err
+	}
+	userProjId := userCred.GetProjectId()
+	data := query.(*jsonutils.JSONDict)
+	{
+		backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", userProjId)
+		backendGroupV.Optional(true)
+		q, err = backendGroupV.QueryFilter(q, data)
+		if err != nil {
+			return nil, err
+		}
+	}
+	{
+		// NOTE extend this when new backend_type was added
+		backendV := validators.NewModelIdOrNameValidator("backend", "server", userProjId)
+		backendV.Optional(true)
+		q, err = backendV.QueryFilter(q, data)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return q, nil
+}
+
+func (man *SLoadbalancerBackendManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", ownerProjId)
+	backendTypeV := validators.NewStringChoicesValidator("backend_type", LB_BACKEND_TYPES)
+	keyV := map[string]validators.IValidator{
+		"backend_group": backendGroupV,
+		"backend_type":  backendTypeV,
+		"weight":        validators.NewRangeValidator("weight", 1, 256).Default(1),
+		"port":          validators.NewPortValidator("port"),
+	}
+	for _, v := range keyV {
+		if err := v.Validate(data); err != nil {
+			return nil, err
+		}
+	}
+	backendGroup := backendGroupV.Model.(*SLoadbalancerBackendGroup)
+	backendType := backendTypeV.Value
+	var baseName string
+	switch backendType {
+	case LB_BACKEND_GUEST:
+		backendV := validators.NewModelIdOrNameValidator("backend", "server", ownerProjId)
+		err := backendV.Validate(data)
+		if err != nil {
+			return nil, err
+		}
+		guest := backendV.Model.(*SGuest)
+		{
+			// guest zone must match that of loadbalancer's
+			host := guest.GetHost()
+			if host == nil {
+				return nil, fmt.Errorf("error getting host of guest %s", guest.GetId())
+			}
+			lb := backendGroup.GetLoadbalancer()
+			if lb == nil {
+				return nil, fmt.Errorf("error loadbalancer of backend group %s", backendGroup.GetId())
+			}
+			if host.ZoneId != lb.ZoneId {
+				return nil, fmt.Errorf("host zone (%s) != loadbalancer %q zone (%s)",
+					host.Name, host.ZoneId, lb.Name, lb.ZoneId)
+			}
+		}
+		{
+			// get guest intranet address
+			//
+			// NOTE add address hint (cidr) if needed
+			gns := guest.GetNetworks()
+			if len(gns) == 0 {
+				return nil, fmt.Errorf("guest %s has no network attached", guest.GetId())
+			}
+			var address string
+			for _, gn := range gns {
+				if !gn.IsExit() {
+					address = gn.IpAddr
+					break
+				}
+			}
+			if len(address) == 0 {
+				return nil, fmt.Errorf("guest %s has no intranet address attached", guest.GetId())
+			}
+			data.Set("address", jsonutils.NewString(address))
+		}
+		baseName = guest.Name
+	case LB_BACKEND_HOST:
+		if !userCred.IsSystemAdmin() {
+			return nil, fmt.Errorf("only sysadmin can specify host as backend")
+		}
+		backendV := validators.NewModelIdOrNameValidator("backend", "host", userCred.GetProjectId())
+		err := backendV.Validate(data)
+		if err != nil {
+			return nil, err
+		}
+		host := backendV.Model.(*SHost)
+		{
+			if len(host.AccessIp) == 0 {
+				return nil, fmt.Errorf("host %s has no access ip", host.GetId())
+			}
+			data.Set("address", jsonutils.NewString(host.AccessIp))
+		}
+		baseName = host.Name
+	default:
+		return nil, fmt.Errorf("internal error: unexpected backend type %s", backendType)
+	}
+	// name it
+	//
+	// NOTE it's okay for name to be not unique.
+	//
+	//  - Mix in loadbalancer name if needed
+	//  - Use name from input query
+	name := fmt.Sprintf("%s-%s-%s", backendGroup.Name, backendType, baseName)
+	data.Set("name", jsonutils.NewString(name))
+	return man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)
+}
+
+func (lbb *SLoadbalancerBackend) AllowPerformStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return false
+}
+
+func (lbb *SLoadbalancerBackend) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	keyV := map[string]validators.IValidator{
+		"weight": validators.NewRangeValidator("weight", 1, 256),
+		"port":   validators.NewPortValidator("port"),
+	}
+	for _, v := range keyV {
+		v.Optional(true)
+		if err := v.Validate(data); err != nil {
+			return nil, err
+		}
+	}
+	return lbb.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data)
+}
+
+func (lbb *SLoadbalancerBackend) PreDelete(ctx context.Context, userCred mcclient.TokenCredential) {
+	lbb.DoPendingDelete(ctx, userCred)
+}
+
+func (lbb *SLoadbalancerBackend) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	return nil
+}

pkg/compute/models/loadbalancercertificates.go

@@ -0,0 +1,154 @@
+package models
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+type SLoadbalancerCertificateManager struct {
+	db.SVirtualResourceBaseManager
+}
+
+var LoadbalancerCertificateManager *SLoadbalancerCertificateManager
+
+func init() {
+	LoadbalancerCertificateManager = &SLoadbalancerCertificateManager{
+		SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager(
+			SLoadbalancerCertificate{},
+			"loadbalancercertificates_tbl",
+			"loadbalancercertificate",
+			"loadbalancercertificates",
+		),
+	}
+}
+
+// TODO
+//
+//  - notify users of cert expiration
+//  - ca info: self-signed, public ca
+type SLoadbalancerCertificate struct {
+	db.SVirtualResourceBase
+
+	Certificate string `create:"required" list:"admin" update:"user"`
+	PrivateKey  string `create:"required" list:"admin" update:"user"`
+
+	// derived attributes
+	PublicKeyAlgorithm      string    `create:"optional" list:user update:"user"`
+	PublicKeyBitLen         int       `create:"optional" list:user update:"user"`
+	SignatureAlgorithm      string    `create:"optional" list:user update:"user"`
+	FingerprintSha256       string    `create:"optional" list:user update:"user"`
+	NotBefore               time.Time `create:"optional" list:user update:"user"`
+	NotAfter                time.Time `create:"optional" list:user update:"user"`
+	CommonName              string    `create:"optional" list:user update:"user"`
+	SubjectAlternativeNames string    `create:"optional" list:user update:"user"`
+}
+
+func (man *SLoadbalancerCertificateManager) PreDeleteSubs(ctx context.Context, userCred mcclient.TokenCredential, q *sqlchemy.SQuery) {
+	subs := []SLoadbalancerCertificate{}
+	db.FetchModelObjects(man, q, &subs)
+	for _, sub := range subs {
+		sub.PreDelete(ctx, userCred)
+	}
+}
+
+func (man *SLoadbalancerCertificateManager) validateCertKey(ctx context.Context, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	certV := validators.NewCertificateValidator("certificate")
+	pkeyV := validators.NewPrivateKeyValidator("private_key")
+	keyV := map[string]validators.IValidator{
+		"certificate": certV,
+		"private_key": pkeyV,
+	}
+	for _, v := range keyV {
+		if err := v.Validate(data); err != nil {
+			return nil, err
+		}
+	}
+	cert := certV.Certificates[0]
+	certPubKeyAlgo := cert.PublicKeyAlgorithm.String()
+	if !LB_TLS_CERT_PUBKEY_ALGOS.Has(certPubKeyAlgo) {
+		return nil, httperrors.NewInputParameterError("invalid cert pubkey algorithm: %s, want %s",
+			certPubKeyAlgo, LB_TLS_CERT_PUBKEY_ALGOS.String())
+	}
+	err := pkeyV.MatchCertificate(cert)
+	if err != nil {
+		return nil, err
+	}
+	// NOTE subject alternative names also includes email, url, ip addresses,
+	// but we ignore them here.
+	//
+	// NOTE we use white space to separate names
+	data.Set("common_name", jsonutils.NewString(cert.Subject.CommonName))
+	data.Set("subject_alternative_names", jsonutils.NewString(strings.Join(cert.DNSNames, " ")))
+
+	data.Set("not_before", jsonutils.NewTimeString(cert.NotBefore))
+	data.Set("not_after", jsonutils.NewTimeString(cert.NotAfter))
+	data.Set("public_key_algorithm", jsonutils.NewString(certPubKeyAlgo))
+	data.Set("public_key_bit_len", jsonutils.NewInt(int64(certV.PublicKeyBitLen())))
+	data.Set("signature_algorithm", jsonutils.NewString(cert.SignatureAlgorithm.String()))
+	data.Set("fingerprint_sha256", jsonutils.NewString(certV.FingerprintSha256String()))
+	return data, nil
+}
+
+func (man *SLoadbalancerCertificateManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	data, err := man.validateCertKey(ctx, data)
+	if err != nil {
+		return nil, err
+	}
+	return man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)
+}
+
+func (lbcert *SLoadbalancerCertificate) AllowPerformStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return false
+}
+
+func (lbcert *SLoadbalancerCertificate) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	if !data.Contains("certificate") {
+		data.Set("certificate", jsonutils.NewString(lbcert.Certificate))
+	}
+	if !data.Contains("private_key") {
+		data.Set("private_key", jsonutils.NewString(lbcert.PrivateKey))
+	}
+	data, err := LoadbalancerCertificateManager.validateCertKey(ctx, data)
+	if err != nil {
+		return nil, err
+	}
+	return lbcert.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data)
+}
+
+func (lbcert *SLoadbalancerCertificate) ValidateDeleteCondition(ctx context.Context) error {
+	men := []db.IModelManager{
+		LoadbalancerListenerManager,
+	}
+	lbcertId := lbcert.Id
+	for _, man := range men {
+		t := man.TableSpec().Instance()
+		pdF := t.Field("pending_deleted")
+		n := t.Query().
+			Equals("certificate_id", lbcertId).
+			Filter(sqlchemy.OR(sqlchemy.IsNull(pdF), sqlchemy.IsFalse(pdF))).
+			Count()
+		if n > 0 {
+			return fmt.Errorf("certificate %s is still referred to by %d %s",
+				lbcertId, n, man.KeywordPlural())
+		}
+	}
+	return nil
+}
+
+func (lbcert *SLoadbalancerCertificate) PreDelete(ctx context.Context, userCred mcclient.TokenCredential) {
+	lbcert.DoPendingDelete(ctx, userCred)
+}
+
+func (lbcert *SLoadbalancerCertificate) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	return nil
+}

pkg/compute/models/loadbalancerlistenerrules.go

@@ -0,0 +1,145 @@
+package models
+
+import (
+	"context"
+	"fmt"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+type SLoadbalancerListenerRuleManager struct {
+	db.SVirtualResourceBaseManager
+}
+
+var LoadbalancerListenerRuleManager *SLoadbalancerListenerRuleManager
+
+func init() {
+	LoadbalancerListenerRuleManager = &SLoadbalancerListenerRuleManager{
+		SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager(
+			SLoadbalancerListenerRule{},
+			"loadbalancerlistenerrules_tbl",
+			"loadbalancerlistenerrule",
+			"loadbalancerlistenerrules",
+		),
+	}
+}
+
+type SLoadbalancerListenerRule struct {
+	db.SVirtualResourceBase
+
+	ListenerId     string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional"`
+	BackendGroupId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"`
+
+	Domain string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"optional"`
+	Path   string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"optional"`
+}
+
+func loadbalancerListenerRuleCheckUniqueness(ctx context.Context, lbls *SLoadbalancerListener, domain, path string) error {
+	man := db.GetModelManager("loadbalancerlistenerrule")
+	if man == nil {
+		return fmt.Errorf("getting loadbalancer rule manager failed")
+	}
+	q := man.Query().
+		Equals("listener_id", lbls.Id).
+		Equals("domain", domain).
+		Equals("path", path)
+	var lblsr SLoadbalancerListenerRule
+	q.First(&lblsr)
+	if len(lblsr.Id) > 0 {
+		return fmt.Errorf("rule %s/%s already occupied by rule %s(%s)", domain, path, lblsr.Name, lblsr.Id)
+	}
+	return nil
+}
+
+func (man *SLoadbalancerListenerRuleManager) PreDeleteSubs(ctx context.Context, userCred mcclient.TokenCredential, q *sqlchemy.SQuery) {
+	subs := []SLoadbalancerListenerRule{}
+	db.FetchModelObjects(man, q, &subs)
+	for _, sub := range subs {
+		sub.PreDelete(ctx, userCred)
+	}
+}
+
+func (man *SLoadbalancerListenerRuleManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
+	q, err := man.SVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query)
+	if err != nil {
+		return nil, err
+	}
+	userProjId := userCred.GetProjectId()
+	data := query.(*jsonutils.JSONDict)
+	{
+		listenerV := validators.NewModelIdOrNameValidator("listener", "listener", userProjId)
+		listenerV.Optional(true)
+		q, err = listenerV.QueryFilter(q, data)
+		if err != nil {
+			return nil, err
+		}
+	}
+	{
+		backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", userProjId)
+		backendGroupV.Optional(true)
+		q, err = backendGroupV.QueryFilter(q, data)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return q, nil
+}
+
+func (man *SLoadbalancerListenerRuleManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	listenerV := validators.NewModelIdOrNameValidator("listener", "loadbalancerlistener", ownerProjId)
+	backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", ownerProjId)
+	domainV := validators.NewDomainNameValidator("domain")
+	pathV := validators.NewURLPathValidator("path")
+	keyV := map[string]validators.IValidator{
+		"status": validators.NewStringChoicesValidator("status", LB_STATUS_SPEC).Default(LB_STATUS_ENABLED),
+
+		"listener":      listenerV,
+		"backend_group": backendGroupV,
+		"domain":        domainV.AllowEmpty(true).Default(""),
+		"path":          pathV.Default(""),
+	}
+	for _, v := range keyV {
+		if err := v.Validate(data); err != nil {
+			return nil, err
+		}
+	}
+	listener := listenerV.Model.(*SLoadbalancerListener)
+	listenerType := listener.ListenerType
+	if listenerType != LB_LISTENER_TYPE_HTTP && listenerType != LB_LISTENER_TYPE_HTTPS {
+		return nil, fmt.Errorf("listener type must be http/https, got %s", listenerType)
+	}
+	err := loadbalancerListenerRuleCheckUniqueness(ctx, listener, domainV.Value, pathV.Value)
+	if err != nil {
+		return nil, err
+	}
+	return man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)
+}
+
+func (lbr *SLoadbalancerListenerRule) AllowPerformStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return lbr.IsOwner(userCred) || userCred.IsSystemAdmin()
+}
+
+func (lbr *SLoadbalancerListenerRule) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", lbr.GetOwnerProjectId()).Optional(true)
+	err := backendGroupV.Validate(data)
+	if err != nil {
+		return nil, err
+	}
+	return lbr.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data)
+}
+
+func (lbr *SLoadbalancerListenerRule) PreDelete(ctx context.Context, userCred mcclient.TokenCredential) {
+	lbr.SetStatus(userCred, LB_STATUS_DISABLED, "preDelete")
+	lbr.DoPendingDelete(ctx, userCred)
+}
+
+func (lbr *SLoadbalancerListenerRule) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	return nil
+}
+
+// Delete, Update

pkg/compute/models/loadbalancerlisteners.go

@@ -0,0 +1,360 @@
+package models
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
+	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+type SLoadbalancerListenerManager struct {
+	db.SVirtualResourceBaseManager
+}
+
+var LoadbalancerListenerManager *SLoadbalancerListenerManager
+
+func init() {
+	LoadbalancerListenerManager = &SLoadbalancerListenerManager{
+		SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager(
+			SLoadbalancerListener{},
+			"loadbalancerlisteners_tbl",
+			"loadbalancerlistener",
+			"loadbalancerlisteners",
+		),
+	}
+}
+
+type SLoadbalancerTCPListener struct{}
+type SLoadbalancerUDPListener struct{}
+
+// TODO sensible default for knobs
+type SLoadbalancerHTTPListener struct {
+	StickySession              string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"`
+	StickySessionType          string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"`
+	StickySessionCookie        string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"`
+	StickySessionCookieTimeout int    `nullable:"false" list:"user" create:"optional" update:"user"`
+
+	//XForwardedForSLBIP bool `nullable:"false" list:"user" create:"optional"`
+	//XForwardedForSLBID bool `nullable:"false" list:"user" create:"optional"`
+	XForwardedFor bool `nullable:"false" list:"user" create:"optional" update:"user"`
+	Gzip          bool `nullable:"false" list:"user" create:"optional" update:"user"`
+}
+
+// TODO
+//
+//  - CACertificate string
+//  - Certificate2Id // multiple certificates for rsa, ecdsa
+//  - Use certificate for tcp listener
+//  - Customize ciphers?
+type SLoadbalancerHTTPSListener struct {
+	CertificateId   string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional"`
+	TLSCipherPolicy string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional"`
+	EnableHttp2     bool   `create:"optional" list:"user"`
+}
+
+type SLoadbalancerListener struct {
+	db.SVirtualResourceBase
+
+	LoadbalancerId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional"`
+	ListenerType   string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"required"`
+	ListenerPort   int    `nullable:"false" list:"user" create:"required"`
+	BackendGroupId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"`
+
+	Bandwidth int    `nullable:"false" list:"user" create:"optional" update:"user"`
+	Scheduler string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user"`
+
+	ClientRequestTimeout  int `nullable:"false" list:"user" create:"optional" update:"user"`
+	ClientIdleTimeout     int `nullable:"false" list:"user" create:"optional" update:"user"`
+	BackendConnectTimeout int `nullable:"false" list:"user" create:"optional" update:"user"`
+	BackendIdleTimeout    int `nullable:"false" list:"user" create:"optional" update:"user"`
+
+	AclStatus string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"`
+	AclType   string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"`
+	AclId     string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"`
+
+	HealthCheck     string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"`
+	HealthCheckType string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"`
+
+	HealthCheckDomain   string `charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"`
+	HealthCheckURI      string `charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"`
+	HealthCheckHttpCode string `charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"`
+
+	HealthCheckRise     int `nullable:"false" list:"user" create:"optional" update:"user"`
+	HealthCheckFall     int `nullable:"false" list:"user" create:"optional" update:"user"`
+	HealthCheckTimeout  int `nullable:"false" list:"user" create:"optional" update:"user"`
+	HealthCheckInterval int `nullable:"false" list:"user" create:"optional" update:"user"`
+
+	HealthCheckReq string `list:"user" create:"optional" update:"user"`
+	HealthCheckExp string `list:"user" create:"optional" update:"user"`
+
+	SLoadbalancerTCPListener
+	SLoadbalancerUDPListener
+	SLoadbalancerHTTPListener
+	SLoadbalancerHTTPSListener
+}
+
+func (man *SLoadbalancerListenerManager) checkListenerUniqueness(ctx context.Context, lb *SLoadbalancer, listenerType string, listenerPort int64) error {
+	q := man.Query().
+		Equals("loadbalancer_id", lb.Id).
+		Equals("listener_port", listenerPort)
+	switch listenerType {
+	case LB_LISTENER_TYPE_TCP, LB_LISTENER_TYPE_HTTP, LB_LISTENER_TYPE_HTTPS:
+		q = q.NotEquals("listener_type", LB_LISTENER_TYPE_UDP)
+	case LB_LISTENER_TYPE_UDP:
+		q = q.Equals("listener_type", LB_LISTENER_TYPE_UDP)
+	default:
+		return fmt.Errorf("unexpected listener type: %s", listenerType)
+	}
+	var listener SLoadbalancerListener
+	q.First(&listener)
+	if len(listener.Id) > 0 {
+		return fmt.Errorf("%s listener port %d is already taken by listener %s",
+			listenerType, listenerPort, listener.Id)
+	}
+	return nil
+}
+
+func (man *SLoadbalancerListenerManager) PreDeleteSubs(ctx context.Context, userCred mcclient.TokenCredential, q *sqlchemy.SQuery) {
+	subs := []SLoadbalancerListener{}
+	db.FetchModelObjects(man, q, &subs)
+	for _, sub := range subs {
+		sub.PreDelete(ctx, userCred)
+	}
+}
+
+func (man *SLoadbalancerListenerManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
+	q, err := man.SVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query)
+	if err != nil {
+		return nil, err
+	}
+	userProjId := userCred.GetProjectId()
+	data := query.(*jsonutils.JSONDict)
+	{
+		lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", userProjId)
+		lbV.Optional(true)
+		q, err = lbV.QueryFilter(q, data)
+		if err != nil {
+			return nil, err
+		}
+	}
+	{
+		backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", userProjId)
+		backendGroupV.Optional(true)
+		q, err = backendGroupV.QueryFilter(q, data)
+		if err != nil {
+			return nil, err
+		}
+	}
+	{
+		aclV := validators.NewModelIdOrNameValidator("acl", "loadbalanceracl", userProjId)
+		aclV.Optional(true)
+		q, err = aclV.QueryFilter(q, data)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return q, nil
+}
+
+func (man *SLoadbalancerListenerManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", ownerProjId)
+	listenerTypeV := validators.NewStringChoicesValidator("listener_type", LB_LISTENER_TYPES)
+	listenerPortV := validators.NewPortValidator("listener_port")
+	backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", ownerProjId)
+	aclStatusV := validators.NewStringChoicesValidator("acl_status", LB_BOOL_VALUES)
+	aclTypeV := validators.NewStringChoicesValidator("acl_type", LB_ACL_TYPES)
+	aclV := validators.NewModelIdOrNameValidator("acl", "loadbalanceracl", ownerProjId)
+	keyV := map[string]validators.IValidator{
+		"status": validators.NewStringChoicesValidator("status", LB_STATUS_SPEC).Default(LB_STATUS_ENABLED),
+
+		"loadbalancer":  lbV,
+		"listener_type": listenerTypeV,
+		"listener_port": listenerPortV,
+		"backend_group": backendGroupV.Optional(true),
+
+		"acl_status": aclStatusV.Default(LB_BOOL_OFF),
+		"acl_type":   aclTypeV.Optional(true),
+		"acl":        aclV.Optional(true),
+
+		"scheduler": validators.NewStringChoicesValidator("scheduler", LB_SCHEDULER_TYPES),
+		"bandwidth": validators.NewRangeValidator("bandwidth", 0, 10000).Optional(true),
+
+		"client_request_timeout":  validators.NewRangeValidator("client_request_timeout", 0, 600).Default(10),
+		"client_idle_timeout":     validators.NewRangeValidator("client_idle_timeout", 0, 600).Default(90),
+		"backend_connect_timeout": validators.NewRangeValidator("backend_connect_timeout", 0, 180).Default(5),
+		"backend_idle_timeout":    validators.NewRangeValidator("backend_idle_timeout", 0, 600).Default(90),
+
+		"sticky_session":                validators.NewStringChoicesValidator("sticky_session", LB_BOOL_VALUES).Default(LB_BOOL_OFF),
+		"sticky_session_type":           validators.NewStringChoicesValidator("sticky_session_type", LB_STICKY_SESSION_TYPES).Default(LB_STICKY_SESSION_TYPE_INSERT),
+		"sticky_session_cookie":         validators.NewRegexpValidator("sticky_session_cookie", regexp.MustCompile(`\w+`)).Optional(true),
+		"sticky_session_cookie_timeout": validators.NewNonNegativeValidator("sticky_session_cookie_timeout").Optional(true),
+
+		"x_forwarded_for": validators.NewBoolValidator("x_forwarded_for").Default(true),
+		"gzip":            validators.NewBoolValidator("gzip").Default(false),
+	}
+	for _, v := range keyV {
+		if err := v.Validate(data); err != nil {
+			return nil, err
+		}
+	}
+	{
+		lb := lbV.Model.(*SLoadbalancer)
+		listenerPort := listenerPortV.Value
+		listenerType := listenerTypeV.Value
+		err := man.checkListenerUniqueness(ctx, lb, listenerType, listenerPort)
+		if err != nil {
+			// duplicate?
+			return nil, err
+		}
+	}
+	{
+		if listenerTypeV.Value == LB_LISTENER_TYPE_HTTPS {
+			certV := validators.NewModelIdOrNameValidator("certificate", "loadbalancercertificate", ownerProjId)
+			tlsCipherPolicyV := validators.NewStringChoicesValidator("tls_cipher_policy", LB_TLS_CIPHER_POLICIES).Default(LB_TLS_CIPHER_POLICY_1_2)
+			httpsV := map[string]validators.IValidator{
+				"certificate":       certV,
+				"tls_cipher_policy": tlsCipherPolicyV,
+				"enable_http2":      validators.NewBoolValidator("enable_http2").Default(true),
+			}
+			for _, v := range httpsV {
+				if err := v.Validate(data); err != nil {
+					return nil, err
+				}
+			}
+		}
+	}
+	{
+		// health check default depends on input parameters
+		checkTypeV := man.checkTypeV(listenerTypeV.Value)
+		keyVHealth := map[string]validators.IValidator{
+			"health_check":      validators.NewStringChoicesValidator("health_check", LB_BOOL_VALUES).Default(LB_BOOL_ON),
+			"health_check_type": checkTypeV,
+
+			"health_check_domain":    validators.NewDomainNameValidator("domain").AllowEmpty(true).Default(""),
+			"health_check_path":      validators.NewURLPathValidator("path").Default(""),
+			"health_check_http_code": validators.NewStringMultiChoicesValidator("health_check_http_code", LB_HEALTH_CHECK_HTTP_CODES).Sep(",").Default(LB_HEALTH_CHECK_HTTP_CODE_DEFAULT),
+
+			"health_check_rise":     validators.NewRangeValidator("health_check_rise", 1, 1000).Default(3),
+			"health_check_fall":     validators.NewRangeValidator("health_check_fall", 1, 1000).Default(3),
+			"health_check_timeout":  validators.NewRangeValidator("health_check_timeout", 1, 300).Default(5),
+			"health_check_interval": validators.NewRangeValidator("health_check_interval", 1, 1000).Default(5),
+		}
+		for _, v := range keyVHealth {
+			if err := v.Validate(data); err != nil {
+				return nil, err
+			}
+		}
+	}
+	{
+		if aclStatusV.Value == LB_BOOL_ON {
+			acl := aclV.Model.(*SLoadbalancerAcl)
+			if acl == nil {
+				return nil, fmt.Errorf("missing acl")
+			}
+			if len(aclTypeV.Value) == 0 {
+				return nil, fmt.Errorf("missing acl_type")
+			}
+		}
+	}
+	return man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)
+}
+
+func (man *SLoadbalancerListenerManager) checkTypeV(listenerType string) validators.IValidator {
+	switch listenerType {
+	case LB_LISTENER_TYPE_HTTP, LB_LISTENER_TYPE_HTTPS:
+		return validators.NewStringChoicesValidator("health_check_type", LB_HEALTH_CHECK_TYPES_TCP).Default(LB_HEALTH_CHECK_HTTP)
+	case LB_LISTENER_TYPE_TCP:
+		return validators.NewStringChoicesValidator("health_check_type", LB_HEALTH_CHECK_TYPES_TCP).Default(LB_HEALTH_CHECK_TCP)
+	case LB_LISTENER_TYPE_UDP:
+		return validators.NewStringChoicesValidator("health_check_type", LB_HEALTH_CHECK_TYPES_UDP).Default(LB_HEALTH_CHECK_UDP)
+	}
+	// should it happen, panic then
+	return nil
+}
+
+func (lblis *SLoadbalancerListener) AllowPerformStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return lblis.IsOwner(userCred) || userCred.IsSystemAdmin()
+}
+
+func (lblis *SLoadbalancerListener) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	ownerProjId := lblis.GetOwnerProjectId()
+	backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", ownerProjId)
+	aclStatusV := validators.NewStringChoicesValidator("acl_status", LB_BOOL_VALUES)
+	aclTypeV := validators.NewStringChoicesValidator("acl_type", LB_ACL_TYPES)
+	aclV := validators.NewModelIdOrNameValidator("acl", "loadbalanceracl", ownerProjId)
+	certV := validators.NewModelIdOrNameValidator("certificate", "loadbalancercertificate", ownerProjId)
+	tlsCipherPolicyV := validators.NewStringChoicesValidator("tls_cipher_policy", LB_TLS_CIPHER_POLICIES).Default(LB_TLS_CIPHER_POLICY_1_2)
+	keyV := map[string]validators.IValidator{
+		"backend_group": backendGroupV,
+
+		"acl_status": aclStatusV,
+		"acl_type":   aclTypeV,
+		"acl":        aclV,
+
+		"scheduler": validators.NewStringChoicesValidator("scheduler", LB_SCHEDULER_TYPES),
+		"bandwidth": validators.NewRangeValidator("bandwidth", 0, 10000),
+
+		"client_request_timeout":  validators.NewRangeValidator("client_request_timeout", 0, 600),
+		"client_idle_timeout":     validators.NewRangeValidator("client_idle_timeout", 0, 600),
+		"backend_connect_timeout": validators.NewRangeValidator("backend_connect_timeout", 0, 180),
+		"backend_idle_timeout":    validators.NewRangeValidator("backend_idle_timeout", 0, 600),
+
+		"sticky_session":                validators.NewStringChoicesValidator("sticky_session", LB_BOOL_VALUES),
+		"sticky_session_type":           validators.NewStringChoicesValidator("sticky_session_type", LB_STICKY_SESSION_TYPES),
+		"sticky_session_cookie":         validators.NewRegexpValidator("sticky_session_cookie", regexp.MustCompile(`\w+`)),
+		"sticky_session_cookie_timeout": validators.NewNonNegativeValidator("sticky_session_cookie_timeout"),
+
+		"health_check":      validators.NewStringChoicesValidator("health_check", LB_BOOL_VALUES),
+		"health_check_type": LoadbalancerListenerManager.checkTypeV(lblis.ListenerType),
+
+		"health_check_domain":    validators.NewDomainNameValidator("domain").AllowEmpty(true),
+		"health_check_path":      validators.NewURLPathValidator("path"),
+		"health_check_http_code": validators.NewStringMultiChoicesValidator("health_check_http_code", LB_HEALTH_CHECK_HTTP_CODES).Sep(","),
+
+		"health_check_rise":     validators.NewRangeValidator("health_check_rise", 1, 1000),
+		"health_check_fall":     validators.NewRangeValidator("health_check_fall", 1, 1000),
+		"health_check_timeout":  validators.NewRangeValidator("health_check_timeout", 1, 300),
+		"health_check_interval": validators.NewRangeValidator("health_check_interval", 1, 1000),
+
+		"x_forwarded_for": validators.NewBoolValidator("x_forwarded_for"),
+		"gzip":            validators.NewBoolValidator("gzip"),
+
+		"certificate":       certV,
+		"tls_cipher_policy": tlsCipherPolicyV,
+		"enable_http2":      validators.NewBoolValidator("enable_http2").Default(true),
+	}
+	for _, v := range keyV {
+		v.Optional(true)
+		if err := v.Validate(data); err != nil {
+			return nil, err
+		}
+	}
+	return lblis.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data)
+}
+
+func (lblis *SLoadbalancerListener) PreDelete(ctx context.Context, userCred mcclient.TokenCredential) {
+	lblis.SetStatus(userCred, LB_STATUS_DISABLED, "preDelete")
+	lblis.DoPendingDelete(ctx, userCred)
+	lblis.PreDeleteSubs(ctx, userCred)
+}
+
+func (lblis *SLoadbalancerListener) PreDeleteSubs(ctx context.Context, userCred mcclient.TokenCredential) {
+	subMan := LoadbalancerListenerRuleManager
+	ownerProjId := lblis.GetOwnerProjectId()
+
+	lockman.LockClass(ctx, subMan, ownerProjId)
+	defer lockman.ReleaseClass(ctx, subMan, ownerProjId)
+	q := subMan.Query().Equals("listener_id", lblis.Id)
+	subMan.PreDeleteSubs(ctx, userCred, q)
+}
+
+func (lblis *SLoadbalancerListener) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	return nil
+}

pkg/compute/models/loadbalancernetworks.go

@@ -4,10 +4,11 @@ import (
 	"context"
 	"fmt"
 
+	"yunion.io/x/pkg/util/regutils"
+
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/mcclient"
-	"yunion.io/x/pkg/util/regutils"
 )
 
 type SLoadbalancernetworkManager struct {

pkg/compute/models/loadbalancers.go

@@ -0,0 +1,195 @@
+package models
+
+import (
+	"context"
+	"fmt"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
+	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+type SLoadbalancerManager struct {
+	db.SVirtualResourceBaseManager
+}
+
+var LoadbalancerManager *SLoadbalancerManager
+
+func init() {
+	LoadbalancerManager = &SLoadbalancerManager{
+		SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager(
+			SLoadbalancer{},
+			"loadbalancers_tbl",
+			"loadbalancer",
+			"loadbalancers",
+		),
+	}
+}
+
+// TODO build errors on pkg/httperrors/errors.go
+// NewGetManagerError
+// NewMissingArgumentError
+// NewInvalidArgumentError
+//
+// TODO ZoneId or RegionId
+// bandwidth
+// scheduler
+//
+// TODO update backendgroupid
+type SLoadbalancer struct {
+	db.SVirtualResourceBase
+
+	Address     string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"optional"`
+	AddressType string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"optional"`
+	NetworkType string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"optional"`
+	NetworkId   string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required"`
+	ZoneId      string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional"`
+
+	BackendGroupId string `width:"36" charset:"ascii" nullable:"false" list:"user" update:"user" update:"user"`
+}
+
+func (man *SLoadbalancerManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
+	q, err := man.SVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query)
+	if err != nil {
+		return nil, err
+	}
+	userProjId := userCred.GetProjectId()
+	data := query.(*jsonutils.JSONDict)
+	{
+		networkV := validators.NewModelIdOrNameValidator("network", "network", userProjId)
+		networkV.Optional(true)
+		q, err = networkV.QueryFilter(q, data)
+		if err != nil {
+			return nil, err
+		}
+	}
+	{
+		zoneV := validators.NewModelIdOrNameValidator("zone", "zone", userProjId)
+		zoneV.Optional(true)
+		q, err = zoneV.QueryFilter(q, data)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return q, nil
+}
+
+func (man *SLoadbalancerManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	networkV := validators.NewModelIdOrNameValidator("network", "network", ownerProjId)
+	addressV := validators.NewIPv4AddrValidator("address").Optional(true)
+	{
+		keyV := map[string]validators.IValidator{
+			"status": validators.NewStringChoicesValidator("status", LB_STATUS_SPEC).Default(LB_STATUS_ENABLED),
+
+			"address": addressV,
+			"network": networkV,
+		}
+		for _, v := range keyV {
+			if err := v.Validate(data); err != nil {
+				return nil, err
+			}
+		}
+	}
+	{
+		network := networkV.Model.(*SNetwork)
+		if wire := network.GetWire(); wire == nil {
+			return nil, fmt.Errorf("getting wire failed")
+		} else if zone := wire.GetZone(); zone == nil {
+			return nil, fmt.Errorf("getting zone failed")
+		} else {
+			data.Set("zone_id", jsonutils.NewString(zone.GetId()))
+		}
+		// TODO validate network is of classic type
+		data.Set("network_type", jsonutils.NewString(LB_NETWORK_TYPE_CLASSIC))
+		data.Set("address_type", jsonutils.NewString(LB_ADDR_TYPE_INTRANET))
+	}
+	return man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)
+}
+
+func (lb *SLoadbalancer) AllowPerformStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return lb.IsOwner(userCred) || userCred.IsSystemAdmin()
+}
+
+func (lb *SLoadbalancer) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) {
+	lb.SVirtualResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data)
+	// NOTE lb.Id will only be available after BeforeInsert happens
+	// NOTE this means lb.UpdateVersion will be 0, then 1 after creation
+	// NOTE need ways to notify error
+	lb.GetModelManager().TableSpec().Update(lb, func() error {
+		if lb.AddressType == LB_ADDR_TYPE_INTRANET {
+			// TODO support use reserved ip address
+			// TODO prefer ip address from server_type loadbalancer?
+			req := &SLoadbalancerNetworkRequestData{
+				loadbalancer: lb,
+				networkId:    lb.NetworkId,
+				address:      lb.Address,
+			}
+			lnMan := db.GetModelManager("loadbalancernetwork").(*SLoadbalancernetworkManager)
+			ln, err := lnMan.NewLoadbalancerNetwork(ctx, userCred, req)
+			if err != nil {
+				log.Errorf("allocating loadbalancer network failed: %#v", req)
+				return err
+			}
+			lb.Address = ln.IpAddr
+		}
+		return nil
+	})
+}
+
+func (lb *SLoadbalancer) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", lb.GetOwnerProjectId()).Optional(true)
+	err := backendGroupV.Validate(data)
+	if err != nil {
+		return nil, err
+	}
+	return lb.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data)
+}
+
+func (lb *SLoadbalancer) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
+	if len(lb.Address) > 0 {
+		// TODO reserve support
+		req := &SLoadbalancerNetworkDeleteData{
+			loadbalancer: lb,
+		}
+		lnMan := db.GetModelManager("loadbalancernetwork").(*SLoadbalancernetworkManager)
+		err := lnMan.DeleteLoadbalancerNetwork(ctx, userCred, req)
+		if err != nil {
+			return err
+		}
+		lb.Address = ""
+	}
+	// TODO How about mark pending delete and return
+	return nil
+}
+
+func (lb *SLoadbalancer) PreDelete(ctx context.Context, userCred mcclient.TokenCredential) {
+	lb.SetStatus(userCred, LB_STATUS_DISABLED, "preDelete")
+	lb.DoPendingDelete(ctx, userCred)
+	lb.PreDeleteSubs(ctx, userCred)
+}
+
+func (lb *SLoadbalancer) PreDeleteSubs(ctx context.Context, userCred mcclient.TokenCredential) {
+	ownerProjId := lb.GetOwnerProjectId()
+	lbId := lb.Id
+	subMen := []ILoadbalancerSubResourceManager{
+		LoadbalancerListenerManager,
+		LoadbalancerBackendGroupManager,
+	}
+	for _, subMan := range subMen {
+		func(subMan ILoadbalancerSubResourceManager) {
+			lockman.LockClass(ctx, subMan, ownerProjId)
+			defer lockman.ReleaseClass(ctx, subMan, ownerProjId)
+			q := subMan.Query().Equals("loadbalancer_id", lbId)
+			subMan.PreDeleteSubs(ctx, userCred, q)
+		}(subMan)
+	}
+}
+
+func (lb *SLoadbalancer) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	return nil
+}

pkg/compute/options/options.go

@@ -26,6 +26,8 @@ type ComputeOptions struct {
 	PendingDeleteExpireSeconds     int  `default:"259200" help:"How long a pending delete VM/disks cleaned automatically, default 3 days"`
 	PendingDeleteMaxCleanBatchSize int  `default:"50" help:"How many pending delete servers can be clean in a batch"`
 
+	LoadbalancerPendingDeleteCheckInterval int `default:"3600" help:"Interval between checks of pending deleted loadbalancer objects, defaults to 1h"`
+
 	ImageCacheStoragePolicy string `default:"least_used" choices:"best_fit|least_used" help:"Policy to choose storage for image cache, best_fit or least_used"`
 	MetricsRetentionDays    int32  `default:"30" help:"Retention days for monitoring metrics in influxdb"`
 

pkg/compute/service/service.go

@@ -56,6 +56,7 @@ func StartService() {
 			cron := cronman.GetCronJobManager()
 			cron.AddJob1("CleanPendingDeleteServers", time.Duration(options.Options.PendingDeleteCheckSeconds)*time.Second, models.GuestManager.CleanPendingDeleteServers)
 			cron.AddJob1("CleanPendingDeleteDisks", time.Duration(options.Options.PendingDeleteCheckSeconds)*time.Second, models.DiskManager.CleanPendingDeleteDisks)
+			cron.AddJob1("CleanPendingDeleteLoadbalancers", time.Duration(options.Options.LoadbalancerPendingDeleteCheckInterval)*time.Second, models.LoadbalancerAgentManager.CleanPendingDeleteLoadbalancers)
 			cron.AddJob2("AutoDiskSnapshot", options.Options.AutoSnapshotDay, options.Options.AutoSnapshotHour, 0, 0, models.DiskManager.AutoDiskSnapshot)
 
 			cron.Start()

pkg/lbagent/api.go

@@ -0,0 +1,291 @@
+package lbagent
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"yunion.io/x/log"
+
+	agentmodels "yunion.io/x/onecloud/pkg/lbagent/models"
+	agentutils "yunion.io/x/onecloud/pkg/lbagent/utils"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/mcclient/models"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+type ApiHelper struct {
+	opts *Options
+
+	dataDirMan  *agentutils.ConfigDirManager
+	corpus      *agentmodels.LoadbalancerCorpus
+	agentParams *agentmodels.AgentParams
+}
+
+func NewApiHelper(opts *Options) (*ApiHelper, error) {
+	helper := &ApiHelper{
+		opts:       opts,
+		dataDirMan: agentutils.NewConfigDirManager(opts.apiDataStoreDir),
+	}
+	return helper, nil
+}
+
+func (h *ApiHelper) Run(ctx context.Context) {
+	defer func() {
+		log.Infof("api helper bye")
+		wg := ctx.Value("wg").(*sync.WaitGroup)
+		wg.Done()
+	}()
+	h.runInit(ctx)
+	apiSyncTicker := time.NewTicker(time.Duration(h.opts.ApiSyncInterval) * time.Second)
+	hbTicker := time.NewTicker(time.Duration(h.opts.ApiLbagentHbInterval) * time.Second)
+	defer hbTicker.Stop()
+	defer apiSyncTicker.Stop()
+	for {
+		select {
+		case <-hbTicker.C:
+			_, err := h.doHb(ctx)
+			if err != nil {
+				log.Errorf("heartbeat: %s", err)
+			}
+		case <-apiSyncTicker.C:
+			apiDataChanged := h.doSyncApiData(ctx)
+			agentParamsChanged := h.doSyncAgentParams(ctx)
+			if apiDataChanged || agentParamsChanged {
+				h.doUseCorpus(ctx)
+			}
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (h *ApiHelper) adminClientSession(ctx context.Context) *mcclient.ClientSession {
+	region := h.opts.CommonOpts.Region
+	apiVersion := "v2"
+	s := auth.GetAdminSession(region, apiVersion)
+	return s
+}
+
+func (h *ApiHelper) agentPeekOnce(ctx context.Context) (*models.LoadbalancerAgent, error) {
+	s := h.adminClientSession(ctx)
+	data, err := modules.LoadbalancerAgents.Get(s, h.opts.ApiLbagentId, nil)
+	if err != nil {
+		err := fmt.Errorf("agent get error: %s", err)
+		return nil, err
+	}
+	agent := &models.LoadbalancerAgent{}
+	err = data.Unmarshal(agent)
+	if err != nil {
+		err := fmt.Errorf("agent data unmarshal error: %s", err)
+		return nil, err
+	}
+	return agent, nil
+}
+
+type agentPeekResult models.LoadbalancerAgent
+
+func (r *agentPeekResult) staleInFuture(s int) bool {
+	if r.HbLastSeen.IsZero() {
+		return true
+	}
+	duration := time.Since(r.HbLastSeen).Seconds()
+	if int(duration) < s {
+		return true
+	}
+	return false
+}
+
+func (h *ApiHelper) agentPeek(ctx context.Context) *agentPeekResult {
+	doPeekWithLog := func() *models.LoadbalancerAgent {
+		agent, err := h.agentPeekOnce(ctx)
+		if err != nil {
+			log.Errorf("agent peek failed: %s", err)
+		}
+		return agent
+	}
+	agent := doPeekWithLog()
+	if agent == nil {
+		initHbTicker := time.NewTicker(time.Duration(3) * time.Second)
+		defer initHbTicker.Stop()
+	initHbDone:
+		for {
+			select {
+			case <-initHbTicker.C:
+				agent = doPeekWithLog()
+				if agent != nil {
+					break initHbDone
+				}
+			case <-ctx.Done():
+				return nil
+			}
+		}
+	}
+	return (*agentPeekResult)(agent)
+}
+
+func (h *ApiHelper) runInit(ctx context.Context) {
+	r := h.agentPeek(ctx)
+	if r == nil {
+		return
+	}
+	if !r.staleInFuture(h.opts.ApiLbagentHbTimeoutRelaxation) {
+		log.Warningf("agent will stale in %d seconds, re-sync",
+			h.opts.ApiLbagentHbTimeoutRelaxation)
+	} else {
+		h.doHb(ctx)
+		corpus, err := h.loadLocalData(ctx)
+		if err == nil {
+			h.corpus = corpus
+		} else {
+			log.Errorf("load local api data failed: %s", err)
+		}
+	}
+	if changed := h.doSyncApiData(ctx); changed {
+		h.doUseCorpus(ctx)
+	}
+}
+
+func (h *ApiHelper) loadLocalData(ctx context.Context) (*agentmodels.LoadbalancerCorpus, error) {
+	corpus := agentmodels.NewEmptyLoadbalancerCorpus()
+	dataDir := h.dataDirMan.MostRecentSubdir()
+	err := corpus.LoadDir(dataDir)
+	if err != nil {
+		return nil, err
+	}
+	return corpus, nil
+}
+
+func (h *ApiHelper) agentUpdateSeen(ctx context.Context) *models.LoadbalancerAgent {
+	s := h.adminClientSession(ctx)
+	params := h.corpus.MaxSeenUpdatedAtParams()
+	data, err := modules.LoadbalancerAgents.Update(s, h.opts.ApiLbagentId, params)
+	if err != nil {
+		log.Errorf("agent get error: %s", err)
+		return nil
+	}
+	agent := &models.LoadbalancerAgent{}
+	err = data.Unmarshal(agent)
+	if err != nil {
+		log.Errorf("agent data unmarshal error: %s", err)
+		return nil
+	}
+	return agent
+}
+
+func (h *ApiHelper) doHb(ctx context.Context) (*models.LoadbalancerAgent, error) {
+	// TODO check if things changed recently
+	s := h.adminClientSession(ctx)
+	data, err := modules.LoadbalancerAgents.PerformAction(s, h.opts.ApiLbagentId, "hb", nil)
+	if err != nil {
+		err := fmt.Errorf("heartbeat api error: %s", err)
+		return nil, err
+	}
+	agent := &models.LoadbalancerAgent{}
+	err = data.Unmarshal(agent)
+	if err != nil {
+		err := fmt.Errorf("heartbeat data unmarshal error: %s", err)
+		return nil, err
+	}
+	return agent, nil
+}
+
+func (h *ApiHelper) doSyncApiData(ctx context.Context) bool {
+	{
+		stime := time.Now()
+		defer func() {
+			elapsed := time.Since(stime)
+			log.Infof("sync api data done, elapsed: %s", elapsed.String())
+		}()
+	}
+
+	s := h.adminClientSession(ctx)
+	if h.corpus == nil {
+		h.corpus = agentmodels.NewEmptyLoadbalancerCorpus()
+	}
+	r, err := h.corpus.SyncModelSets(s, h.opts.ApiListBatchSize)
+	if err != nil {
+		log.Errorf("sync models: %s", err)
+		return false
+	}
+	if r.Changed {
+		h.agentUpdateSeen(ctx)
+	}
+	if !r.Correct {
+		log.Warningf("sync models: not correct")
+		return false
+	}
+	if r.Changed {
+		err := h.saveCorpus(ctx)
+		if err != nil {
+			log.Errorf("save corpus failed: %s", err)
+			return false
+		}
+		if err := h.dataDirMan.Prune(h.opts.DataPreserveN); err != nil {
+			log.Errorf("prune corpus data dir failed: %s", err)
+			// continue
+		}
+		log.Infof("corpus changed")
+		return true
+	}
+	return false
+}
+
+func (h *ApiHelper) saveCorpus(ctx context.Context) error {
+	_, err := h.dataDirMan.NewDir(func(dir string) error {
+		err := h.corpus.SaveDir(dir)
+		if err != nil {
+			return fmt.Errorf("save to dir %s: %s", dir, err)
+		}
+		return nil
+	})
+	return err
+}
+
+func (h *ApiHelper) doSyncAgentParams(ctx context.Context) bool {
+	agent, err := h.agentPeekOnce(ctx)
+	if err != nil {
+		log.Errorf("agent params get failure: %s", err)
+		return false
+	}
+	agentParams, err := agentmodels.NewAgentParams(agent)
+	if err != nil {
+		log.Errorf("agent params prepare failure: %s", err)
+		return false
+	}
+	if !agentParams.Equal(h.agentParams) {
+		h.agentParams = agentParams
+		return true
+	}
+	return false
+}
+
+func (h *ApiHelper) doUseCorpus(ctx context.Context) {
+	if h.corpus == nil {
+		log.Warningf("agent corpus nil")
+		return
+	}
+	if h.agentParams == nil {
+		log.Warningf("agent params nil")
+		return
+	}
+	log.Infof("make effect new corpus and params")
+	cmdData := &LbagentCmdUseCorpusData{
+		Corpus:      h.corpus,
+		AgentParams: h.agentParams,
+	}
+	cmdData.Wg.Add(1)
+	cmd := &LbagentCmd{
+		Type: LbagentCmdUseCorpus,
+		Data: cmdData,
+	}
+	cmdChan := ctx.Value("cmdChan").(chan *LbagentCmd)
+	select {
+	case cmdChan <- cmd:
+		cmdData.Wg.Wait()
+	case <-ctx.Done():
+		return
+	}
+}

pkg/lbagent/cmd.go

@@ -0,0 +1,24 @@
+package lbagent
+
+import (
+	"sync"
+
+	agentmodels "yunion.io/x/onecloud/pkg/lbagent/models"
+)
+
+type LbagentCmdType uintptr
+
+const (
+	LbagentCmdUseCorpus LbagentCmdType = iota
+)
+
+type LbagentCmdUseCorpusData struct {
+	Corpus      *agentmodels.LoadbalancerCorpus
+	AgentParams *agentmodels.AgentParams
+	Wg          sync.WaitGroup
+}
+
+type LbagentCmd struct {
+	Type LbagentCmdType
+	Data interface{}
+}

pkg/lbagent/gobetween/config.go

@@ -0,0 +1,304 @@
+/**
+ * config.go - config file definitions
+ *
+ * @author Yaroslav Pogrebnyak <yyyaroslav@gmail.com>
+ * @author Gene Ponomarenko <kikomdev@gmail.com>
+ */
+package gobetween
+
+/**
+ * Config file top-level object
+ */
+type Config struct {
+	Logging  LoggingConfig     `toml:"logging" json:"logging"`
+	Api      ApiConfig         `toml:"api" json:"api"`
+	Defaults ConnectionOptions `toml:"defaults" json:"defaults"`
+	Acme     *AcmeConfig       `toml:"acme" json:"acme"`
+	Servers  map[string]Server `toml:"servers" json:"servers"`
+}
+
+/**
+ * Logging config section
+ */
+type LoggingConfig struct {
+	Level  string `toml:"level" json:"level"`
+	Output string `toml:"output" json:"output"`
+}
+
+/**
+ * Api config section
+ */
+type ApiConfig struct {
+	Enabled   bool                `toml:"enabled" json:"enabled"`
+	Bind      string              `toml:"bind" json:"bind"`
+	BasicAuth *ApiBasicAuthConfig `toml:"basic_auth" json:"basic_auth"`
+	Tls       *ApiTlsConfig       `toml:"tls" json:"tls"`
+	Cors      bool                `toml:"cors" json:"cors"`
+}
+
+/**
+ * Api Basic Auth Config
+ */
+type ApiBasicAuthConfig struct {
+	Login    string `toml:"login" json:"login"`
+	Password string `toml:"password" json:"password"`
+}
+
+/**
+ * Api TLS server Config
+ */
+type ApiTlsConfig struct {
+	CertPath string `toml:"cert_path" json:"cert_path"`
+	KeyPath  string `toml:"key_path" json:"key_path"`
+}
+
+/**
+ * Default values can be overridden in server
+ */
+type ConnectionOptions struct {
+	MaxConnections           *int    `toml:"max_connections" json:"max_connections"`
+	ClientIdleTimeout        *string `toml:"client_idle_timeout" json:"client_idle_timeout"`
+	BackendIdleTimeout       *string `toml:"backend_idle_timeout" json:"backend_idle_timeout"`
+	BackendConnectionTimeout *string `toml:"backend_connection_timeout" json:"backend_connection_timeout"`
+}
+
+/**
+ * Acme config
+ */
+type AcmeConfig struct {
+	Challenge string `toml:"challenge" json:"challenge"`
+	HttpBind  string `toml:"http_bind" json:"http_bind"`
+	CacheDir  string `toml:"cache_dir" json:"cache_dir"`
+}
+
+/**
+ * Server section config
+ */
+type Server struct {
+	ConnectionOptions
+
+	// hostname:port
+	Bind string `toml:"bind" json:"bind"`
+
+	// tcp | udp | tls
+	Protocol string `toml:"protocol" json:"protocol"`
+
+	// weight | leastconn | roundrobin
+	Balance string `toml:"balance" json:"balance"`
+
+	// Optional configuration for server name indication
+	Sni *Sni `toml:"sni" json:"sni"`
+
+	// Optional configuration for protocol = tls
+	Tls *Tls `toml:"tls" json:"tls"`
+
+	// Optional configuration for backend_tls_enabled = true
+	BackendsTls *BackendsTls `toml:"backends_tls" json:"backends_tls"`
+
+	// Optional configuration for protocol = udp
+	Udp *Udp `toml:"udp" json:"udp"`
+
+	// Access configuration
+	Access *AccessConfig `toml:"access" json:"access"`
+
+	// ProxyProtocol configuration
+	ProxyProtocol *ProxyProtocol `toml:"proxy_protocol" json:"proxy_protocol"`
+
+	// Discovery configuration
+	Discovery *DiscoveryConfig `toml:"discovery" json:"discovery"`
+
+	// Healthcheck configuration
+	Healthcheck *HealthcheckConfig `toml:"healthcheck" json:"healthcheck"`
+}
+
+/**
+ * ProxyProtocol configurtion
+ */
+type ProxyProtocol struct {
+	Version string `toml:"version" json:"version"`
+}
+
+/**
+ * Server Sni options
+ */
+type Sni struct {
+	HostnameMatchingStrategy   string `toml:"hostname_matching_strategy" json:"hostname_matching_strategy"`
+	UnexpectedHostnameStrategy string `toml:"unexpected_hostname_strategy" json:"unexpected_hostname_strategy"`
+	ReadTimeout                string `toml:"read_timeout" json:"read_timeout"`
+}
+
+/**
+ * Common part of Tls and BackendTls types
+ */
+type tlsCommon struct {
+	Ciphers             []string `toml:"ciphers" json:"ciphers"`
+	PreferServerCiphers bool     `toml:"prefer_server_ciphers" json:"prefer_server_ciphers"`
+	MinVersion          string   `toml:"min_version" json:"min_version"`
+	MaxVersion          string   `toml:"max_version" json:"max_version"`
+	SessionTickets      bool     `toml:"session_tickets" json:"session_tickets"`
+}
+
+/**
+ * Server Tls options
+ * for protocol = "tls"
+ */
+type Tls struct {
+	AcmeHosts []string `toml:"acme_hosts" json:"acme_hosts"`
+	CertPath  string   `toml:"cert_path" json:"cert_path"`
+	KeyPath   string   `toml:"key_path" json:"key_path"`
+	tlsCommon
+}
+
+type BackendsTls struct {
+	IgnoreVerify   bool    `toml:"ignore_verify" json:"ignore_verify"`
+	RootCaCertPath *string `toml:"root_ca_cert_path" json:"root_ca_cert_path"`
+	CertPath       *string `toml:"cert_path" json:"cert_path"`
+	KeyPath        *string `toml:"key_path" json:"key_path"`
+	tlsCommon
+}
+
+/**
+ * Server udp options
+ * for protocol = "udp"
+ */
+type Udp struct {
+	MaxRequests  uint64 `toml:"max_requests" json:"max_requests"`
+	MaxResponses uint64 `toml:"max_responses" json:"max_responses"`
+}
+
+/**
+ * Access configuration
+ */
+type AccessConfig struct {
+	Default string   `toml:"default" json:"default"`
+	Rules   []string `toml:"rules" json:"rules"`
+}
+
+/**
+ * Discovery configuration
+ */
+type DiscoveryConfig struct {
+	Kind       string `toml:"kind" json:"kind"`
+	Failpolicy string `toml:"failpolicy" json:"failpolicy"`
+	Interval   string `toml:"interval" json:"interval"`
+	Timeout    string `toml:"timeout" json:"timeout"`
+
+	/* Depends on Kind */
+
+	*StaticDiscoveryConfig
+	*SrvDiscoveryConfig
+	*DockerDiscoveryConfig
+	*JsonDiscoveryConfig
+	*ExecDiscoveryConfig
+	*PlaintextDiscoveryConfig
+	*ConsulDiscoveryConfig
+	*LXDDiscoveryConfig
+}
+
+type StaticDiscoveryConfig struct {
+	StaticList []string `toml:"static_list" json:"static_list"`
+}
+
+type SrvDiscoveryConfig struct {
+	SrvLookupServer  string `toml:"srv_lookup_server" json:"srv_lookup_server"`
+	SrvLookupPattern string `toml:"srv_lookup_pattern" json:"srv_lookup_pattern"`
+	SrvDnsProtocol   string `toml:"srv_dns_protocol" json:"srv_dns_protocol"`
+}
+
+type ExecDiscoveryConfig struct {
+	ExecCommand []string `toml:"exec_command" json:"exec_command"`
+}
+
+type JsonDiscoveryConfig struct {
+	JsonEndpoint        string `toml:"json_endpoint" json:"json_endpoint"`
+	JsonHostPattern     string `toml:"json_host_pattern" json:"json_host_pattern"`
+	JsonPortPattern     string `toml:"json_port_pattern" json:"json_port_pattern"`
+	JsonWeightPattern   string `toml:"json_weight_pattern" json:"json_weight_pattern"`
+	JsonPriorityPattern string `toml:"json_priority_pattern" json:"json_priority_pattern"`
+	JsonSniPattern      string `toml:"json_sni_pattern" json:"json_sni_pattern"`
+}
+
+type PlaintextDiscoveryConfig struct {
+	PlaintextEndpoint      string `toml:"plaintext_endpoint" json:"plaintext_endpoint"`
+	PlaintextRegexpPattern string `toml:"plaintext_regex_pattern" json:"plaintext_regex_pattern"`
+}
+
+type DockerDiscoveryConfig struct {
+	DockerEndpoint             string `toml:"docker_endpoint" json:"docker_endpoint"`
+	DockerContainerLabel       string `toml:"docker_container_label" json:"docker_container_label"`
+	DockerContainerPrivatePort int64  `toml:"docker_container_private_port" json:"docker_container_private_port"`
+	DockerContainerHostEnvVar  string `toml:"docker_container_host_env_var" json:"docker_container_host_env_var"`
+
+	DockerTlsEnabled    bool   `toml:"docker_tls_enabled" json:"docker_tls_enabled"`
+	DockerTlsCertPath   string `toml:"docker_tls_cert_path" json:"docker_tls_cert_path"`
+	DockerTlsKeyPath    string `toml:"docker_tls_key_path" json:"docker_tls_key_path"`
+	DockerTlsCacertPath string `toml:"docker_tls_cacert_path" json:"docker_tls_cacert_path"`
+}
+
+type ConsulDiscoveryConfig struct {
+	ConsulHost               string `toml:"consul_host" json:"consul_host"`
+	ConsulServiceName        string `toml:"consul_service_name" json:"consul_service_name"`
+	ConsulServiceTag         string `toml:"consul_service_tag" json:"consul_service_tag"`
+	ConsulServicePassingOnly bool   `toml:"consul_service_passing_only" json:"consul_service_passing_only"`
+	ConsulDatacenter         string `toml:"consul_datacenter" json:"consul_datacenter"`
+
+	ConsulAuthUsername string `toml:"consul_auth_username" json:"consul_auth_username"`
+	ConsulAuthPassword string `toml:"consul_auth_password" json:"consul_auth_password"`
+
+	ConsulTlsEnabled    bool   `toml:"consul_tls_enabled" json:"consul_tls_enabled"`
+	ConsulTlsCertPath   string `toml:"consul_tls_cert_path" json:"consul_tls_cert_path"`
+	ConsulTlsKeyPath    string `toml:"consul_tls_key_path" json:"consul_tls_key_path"`
+	ConsulTlsCacertPath string `toml:"consul_tls_cacert_path" json:"consul_tls_cacert_path"`
+}
+
+type LXDDiscoveryConfig struct {
+	LXDServerAddress        string `toml:"lxd_server_address" json:"lxd_server_address"`
+	LXDServerRemoteName     string `toml:"lxd_server_remote_name" json:"lxd_server_remote_name"`
+	LXDServerRemotePassword string `toml:"lxd_server_remote_password" json:"lxd_server_remote_password"`
+
+	LXDConfigDirectory     string `toml:"lxd_config_directory" json:"lxd_config_directory"`
+	LXDGenerateClientCerts bool   `toml:"lxd_generate_client_certs" json:"lxd_generate_client_certs"`
+	LXDAcceptServerCert    bool   `toml:"lxd_accept_server_cert" json:"lxd_accept_server_cert"`
+
+	LXDContainerLabelKey   string `toml:"lxd_container_label_key" json:"lxd_container_label_key"`
+	LXDContainerLabelValue string `toml:"lxd_container_label_value" json:"lxd_container_label_value"`
+
+	LXDContainerPort    int    `toml:"lxd_container_port" json:"lxd_container_port"`
+	LXDContainerPortKey string `toml:"lxd_container_port_key" json:"lxd_container_port_key"`
+
+	LXDContainerInterface    string `toml:"lxd_container_interface" json:"lxd_container_interface"`
+	LXDContainerInterfaceKey string `toml:"lxd_container_interface_key" json:"lxd_container_interface_key"`
+
+	LXDContainerSNIKey      string `toml:"lxd_container_sni_key" json:"lxd_container_sni_key"`
+	LXDContainerAddressType string `toml:"lxd_container_address_type" json:"lxd_container_address_type"`
+}
+
+/**
+ * Healthcheck configuration
+ */
+type HealthcheckConfig struct {
+	Kind     string `toml:"kind" json:"kind"`
+	Interval string `toml:"interval" json:"interval"`
+	Passes   int    `toml:"passes" json:"passes"`
+	Fails    int    `toml:"fails" json:"fails"`
+	Timeout  string `toml:"timeout" json:"timeout"`
+
+	/* Depends on Kind */
+
+	*PingHealthcheckConfig
+	*ExecHealthcheckConfig
+	*UdpHealthcheckConfig
+}
+
+type PingHealthcheckConfig struct{}
+
+type ExecHealthcheckConfig struct {
+	ExecCommand                string `toml:"exec_command" json:"exec_command,omitempty"`
+	ExecExpectedPositiveOutput string `toml:"exec_expected_positive_output" json:"exec_expected_positive_output"`
+	ExecExpectedNegativeOutput string `toml:"exec_expected_negative_output" json:"exec_expected_negative_output"`
+}
+
+type UdpHealthcheckConfig struct {
+	Receive string
+	Send    string
+}

pkg/lbagent/haproxy.go

@@ -0,0 +1,352 @@
+package lbagent
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"syscall"
+
+	"yunion.io/x/log"
+	aggrerrors "yunion.io/x/pkg/util/errors"
+
+	agentmodels "yunion.io/x/onecloud/pkg/lbagent/models"
+	agentutils "yunion.io/x/onecloud/pkg/lbagent/utils"
+)
+
+type HaproxyHelper struct {
+	opts *Options
+
+	configDirMan *agentutils.ConfigDirManager
+}
+
+func NewHaproxyHelper(opts *Options) (*HaproxyHelper, error) {
+	helper := &HaproxyHelper{
+		opts:         opts,
+		configDirMan: agentutils.NewConfigDirManager(opts.haproxyConfigDir),
+	}
+	{
+		// sysctl
+		args := []string{
+			"sysctl", "-w",
+			"net.ipv4.ip_nonlocal_bind=1",
+			"net.ipv4.ip_forward=1",
+		}
+		if err := helper.runCmd(args); err != nil {
+			return nil, fmt.Errorf("sysctl: %s", err)
+		}
+	}
+	if false {
+		// ipvs modules
+		mods := []string{
+			"ip_vs",
+			"ip_vs_rr",
+			"ip_vs_wrr",
+			"ip_vs_lc",
+			"ip_vs_wlc",
+			"ip_vs_sh",
+		}
+		args := []string{"modprobe", ""}
+		for _, mod := range mods {
+			args[1] = mod
+			if err := helper.runCmd(args); err != nil {
+				return nil, fmt.Errorf("modprobe %s: %s", mod, err)
+			}
+		}
+	}
+	return helper, nil
+}
+
+func (h *HaproxyHelper) Run(ctx context.Context) {
+	defer func() {
+		wg := ctx.Value("wg").(*sync.WaitGroup)
+		wg.Done()
+	}()
+	cmdChan := ctx.Value("cmdChan").(chan *LbagentCmd)
+	for {
+		for {
+			select {
+			case <-ctx.Done():
+				log.Infof("haproxy helper bye")
+				return
+			case cmd := <-cmdChan:
+				h.handleCmd(ctx, cmd)
+			}
+		}
+	}
+}
+
+func (h *HaproxyHelper) handleCmd(ctx context.Context, cmd *LbagentCmd) {
+	switch cmd.Type {
+	case LbagentCmdUseCorpus:
+		cmdData := cmd.Data.(*LbagentCmdUseCorpusData)
+		defer cmdData.Wg.Done()
+		h.handleUseCorpusCmd(ctx, cmd)
+	default:
+		log.Warningf("command type ignored: %v", cmd.Type)
+	}
+}
+
+func (h *HaproxyHelper) handleUseCorpusCmd(ctx context.Context, cmd *LbagentCmd) {
+	// haproxy config dir
+	dir, err := h.configDirMan.NewDir(func(dir string) error {
+		cmdData := cmd.Data.(*LbagentCmdUseCorpusData)
+		corpus := cmdData.Corpus
+		agentParams := cmdData.AgentParams
+		{
+			opt := fmt.Sprintf("stats socket %s expose-fd listeners", h.haproxyStatsSocketFile())
+			agentParams.SetHaproxyParams("global_stats_socket", opt)
+		}
+		var genHaproxyConfigsResult *agentmodels.GenHaproxyConfigsResult
+		var err error
+		{
+			// haproxy toplevel global/defaults config
+			err = corpus.GenHaproxyToplevelConfig(dir, agentParams)
+			if err != nil {
+				err = fmt.Errorf("generating haproxy toplevel config failed: %s", err)
+				return err
+			}
+		}
+		{
+			// haproxy configs
+			genHaproxyConfigsResult, err = corpus.GenHaproxyConfigs(dir, agentParams)
+			if err != nil {
+				err = fmt.Errorf("generating haproxy config failed: %s", err)
+				return err
+			}
+		}
+		{
+			// gobetween config
+			opts := &agentmodels.GenGobetweenConfigOptions{
+				LoadbalancersEnabled: genHaproxyConfigsResult.LoadbalancersEnabled,
+				AgentParams:          agentParams,
+			}
+			err := corpus.GenGobetweenConfigs(dir, opts)
+			if err != nil {
+				err = fmt.Errorf("generating gobetween config failed: %s", err)
+				return err
+			}
+		}
+		{
+			// keepalived config
+			opts := &agentmodels.GenKeepalivedConfigOptions{
+				LoadbalancersEnabled: genHaproxyConfigsResult.LoadbalancersEnabled,
+				AgentParams:          agentParams,
+			}
+			err := corpus.GenKeepalivedConfigs(dir, opts)
+			if err != nil {
+				err = fmt.Errorf("generating keepalived config failed: %s", err)
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		log.Errorf("making configs: %s", err)
+		return
+	}
+	if err := h.configDirMan.Prune(h.opts.DataPreserveN); err != nil {
+		log.Errorf("prune configs dir failed: %s", err)
+		// continue
+	}
+	if err := h.useConfigs(ctx, dir); err != nil {
+		log.Errorf("useConfigs: %s", err)
+	}
+}
+
+func (h *HaproxyHelper) useConfigs(ctx context.Context, d string) error {
+	lnF := func(old, new string) error {
+		err := os.RemoveAll(new)
+		if err != nil {
+			return err
+		}
+		err = os.Symlink(old, new)
+		return err
+	}
+	keepalivedConf := filepath.Join(h.opts.haproxyConfigDir, "keepalived.conf")
+	gobetweenJson := filepath.Join(h.opts.haproxyConfigDir, "gobetween.json")
+	haproxyConfD := h.haproxyConfD()
+	dirMap := map[string]string{
+		keepalivedConf: filepath.Join(d, "keepalived.conf"),
+		haproxyConfD:   d,
+		gobetweenJson:  filepath.Join(d, "gobetween.json"),
+	}
+	for new, old := range dirMap {
+		err := lnF(old, new)
+		if err != nil {
+			return err
+		}
+	}
+	{
+		var errs []error
+		var err error
+		{
+			// reload haproxy
+			err = h.reloadHaproxy(ctx)
+			if err != nil {
+				errs = append(errs, err)
+			}
+		}
+		{
+			// reload gobetween
+			err = h.reloadGobetween(ctx)
+			if err != nil {
+				errs = append(errs, err)
+			}
+		}
+		{
+			// reload keepalived
+			err = h.reloadKeepalived(ctx)
+			if err != nil {
+				errs = append(errs, err)
+			}
+		}
+		if len(errs) == 0 {
+			return nil
+		}
+		return aggrerrors.NewAggregate(errs)
+	}
+}
+
+func (h *HaproxyHelper) haproxyConfD() string {
+	return filepath.Join(h.opts.haproxyConfigDir, "haproxy.conf.d")
+}
+
+func (h *HaproxyHelper) haproxyPidFile() string {
+	return filepath.Join(h.opts.haproxyRunDir, "haproxy.pid")
+}
+
+func (h *HaproxyHelper) haproxyStatsSocketFile() string {
+	return filepath.Join(h.opts.haproxyRunDir, "haproxy.sock")
+}
+
+func (h *HaproxyHelper) reloadHaproxy(ctx context.Context) error {
+	// NOTE we may sometimes need to specify a custom the executable path
+	pidFile := h.haproxyPidFile()
+	args := []string{
+		h.opts.HaproxyBin,
+		"-D", // goes daemon
+		"-p", pidFile,
+		"-C", h.haproxyConfD(),
+		"-f", h.haproxyConfD(),
+	}
+	proc := agentutils.ReadPidFile(pidFile)
+	if proc != nil {
+		args = append(args, "-sf", fmt.Sprintf("%d", proc.Pid))
+		{
+			statsSocket := h.haproxyStatsSocketFile()
+			if fi, err := os.Stat(statsSocket); err == nil && fi.Mode()&os.ModeSocket != 0 {
+				args = append(args, "-x", statsSocket)
+			} else {
+				log.Warningf("stats socket %s not found", statsSocket)
+			}
+		}
+		log.Infof("reloading haproxy")
+	} else {
+		log.Infof("starting haproxy")
+	}
+	return h.runCmd(args)
+}
+
+func (h *HaproxyHelper) gobetweenConf() string {
+	return filepath.Join(h.opts.haproxyConfigDir, "gobetween.json")
+}
+
+func (h *HaproxyHelper) gobetweenPidFile() string {
+	return filepath.Join(h.opts.haproxyRunDir, "gobetween.pid")
+}
+
+func (h *HaproxyHelper) reloadGobetween(ctx context.Context) error {
+	pidFile := h.gobetweenPidFile()
+	args := []string{
+		h.opts.GobetweenBin,
+		"--config", h.gobetweenConf(),
+		"--format", "json",
+	}
+	proc := agentutils.ReadPidFile(pidFile)
+	if proc != nil {
+		log.Infof("stopping gobetween(%d)", proc.Pid)
+		proc.Kill()
+		proc.Wait()
+	}
+	log.Infof("starting gobetween")
+	cmd, err := h.startCmd(args)
+	if err != nil {
+		return err
+	}
+	err = agentutils.WritePidFile(cmd.Process.Pid, h.gobetweenPidFile())
+	if err != nil {
+		return fmt.Errorf("writing gobetween pid file: %s", err)
+	}
+	return nil
+}
+
+func (h *HaproxyHelper) keepalivedConf() string {
+	return filepath.Join(h.opts.haproxyConfigDir, "keepalived.conf")
+}
+
+func (h *HaproxyHelper) keepalivedPidFile() string {
+	return filepath.Join(h.opts.haproxyRunDir, "keepalived.pid")
+}
+
+func (h *HaproxyHelper) keepalivedVrrpPidFile() string {
+	return filepath.Join(h.opts.haproxyRunDir, "keepalived_vrrp.pid")
+}
+
+func (h *HaproxyHelper) keepalivedCheckersPidFile() string {
+	return filepath.Join(h.opts.haproxyRunDir, "keepalived_checkers.pid")
+}
+
+func (h *HaproxyHelper) reloadKeepalived(ctx context.Context) error {
+	pidFile := h.keepalivedPidFile()
+	proc := agentutils.ReadPidFile(pidFile)
+	if proc != nil {
+		// send SIGHUP to reload
+		err := proc.Signal(syscall.SIGHUP)
+		if err != nil {
+			return fmt.Errorf("keepalived: send HUP failed: %s", err)
+		}
+		return nil
+	}
+	args := []string{
+		h.opts.KeepalivedBin,
+		"--pid", pidFile,
+		"--vrrp_pid", h.keepalivedVrrpPidFile(),
+		"--checkers_pid", h.keepalivedCheckersPidFile(),
+		"--use-file", h.keepalivedConf(),
+	}
+	return h.runCmd(args)
+}
+
+func (h *HaproxyHelper) runCmd(args []string) error {
+	name := args[0]
+	args = args[1:]
+	cmd := exec.Command(name, args...)
+
+	output, err := cmd.Output()
+	if err != nil {
+		if ee, ok := err.(*exec.ExitError); ok {
+			stdout := string(output)
+			stderr := string(ee.Stderr)
+			return fmt.Errorf("%s: %s\nargs: %s\nstdout: %s\nstderr: %s",
+				name, err, strings.Join(args, " "), stdout, stderr)
+		}
+		return fmt.Errorf("%s: %s", name, err)
+	}
+	return nil
+}
+
+func (h *HaproxyHelper) startCmd(args []string) (*exec.Cmd, error) {
+	name := args[0]
+	args = args[1:]
+	cmd := exec.Command(name, args...)
+
+	err := cmd.Start()
+	if err != nil {
+		return nil, err
+	}
+	return cmd, nil
+}

pkg/lbagent/models/agentparams.go

@@ -0,0 +1,106 @@
+package models
+
+import (
+	"encoding/base64"
+	"fmt"
+	"text/template"
+
+	"yunion.io/x/onecloud/pkg/mcclient/models"
+)
+
+type AgentParams struct {
+	AgentModel           *models.LoadbalancerAgent
+	KeepalivedConfigTmpl *template.Template
+	HaproxyConfigTmpl    *template.Template
+	Data                 map[string]interface{}
+}
+
+func NewAgentParams(agent *models.LoadbalancerAgent) (*AgentParams, error) {
+	b64s := map[string]string{
+		"keepalived_conf_tmpl": agent.Params.KeepalivedConfTmpl,
+		"haproxy_conf_tmpl":    agent.Params.HaproxyConfTmpl,
+	}
+	tmpls := map[string]*template.Template{}
+	for name, b64 := range b64s {
+		d, err := base64.StdEncoding.DecodeString(b64)
+		if err != nil {
+			return nil, fmt.Errorf("%s: invalid base64 string: %s", name, err)
+		}
+		tmpl, err := template.New(name).Parse(string(d))
+		if err != nil {
+			return nil, fmt.Errorf("%s: invalid template: %s", name, err)
+		}
+		tmpls[name] = tmpl
+	}
+	dataAgent := map[string]interface{}{
+		"id":   agent.Id,
+		"name": agent.Name,
+	}
+	dataVrrp := map[string]interface{}{
+		"priority":            agent.Params.Vrrp.Priority,
+		"virtual_router_id":   agent.Params.Vrrp.VirtualRouterId,
+		"garp_master_refresh": agent.Params.Vrrp.GarpMasterRefresh,
+		"preempt":             agent.Params.Vrrp.Preempt,
+		"interface":           agent.Params.Vrrp.Interface,
+		"advert_int":          agent.Params.Vrrp.AdvertInt,
+		"pass":                agent.Params.Vrrp.Pass,
+	}
+	dataHaproxy := map[string]interface{}{
+		"global_log":      agent.Params.Haproxy.GlobalLog,
+		"global_nbthread": agent.Params.Haproxy.GlobalNbthread,
+		"log_http":        agent.Params.Haproxy.LogHttp,
+		"log_tcp":         agent.Params.Haproxy.LogTcp,
+		"log_normal":      agent.Params.Haproxy.LogNormal,
+	}
+	data := map[string]interface{}{
+		"agent":   dataAgent,
+		"vrrp":    dataVrrp,
+		"haproxy": dataHaproxy,
+	}
+	agentParams := &AgentParams{
+		AgentModel:           agent,
+		KeepalivedConfigTmpl: tmpls["keepalived_conf_tmpl"],
+		HaproxyConfigTmpl:    tmpls["haproxy_conf_tmpl"],
+		Data:                 data,
+	}
+	return agentParams, nil
+}
+
+func (p *AgentParams) Equal(p2 *AgentParams) bool {
+	if p == nil && p2 == nil {
+		return true
+	}
+	if p == nil || p2 == nil {
+		return false
+	}
+	agentP := p.AgentModel
+	agentP2 := p2.AgentModel
+	if agentP.Params != agentP2.Params {
+		return false
+	}
+	return true
+}
+
+func (p *AgentParams) setXxParams(xx, k string, v interface{}) map[string]interface{} {
+	var dt map[string]interface{}
+	d, ok := p.Data[xx]
+	if !ok {
+		dt = map[string]interface{}{}
+		p.Data[xx] = dt
+	} else {
+		dt = d.(map[string]interface{})
+	}
+	dt[k] = v
+	return dt
+}
+
+func (p *AgentParams) SetVrrpParams(k string, v interface{}) map[string]interface{} {
+	return p.setXxParams("vrrp", k, v)
+}
+
+func (p *AgentParams) SetHaproxyParams(k string, v interface{}) map[string]interface{} {
+	return p.setXxParams("haproxy", k, v)
+}
+
+func (p *AgentParams) KeepalivedConfig() {
+}

pkg/lbagent/models/corpus.go

@@ -0,0 +1,79 @@
+package models
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+
+	"yunion.io/x/jsonutils"
+
+	agentutils "yunion.io/x/onecloud/pkg/lbagent/utils"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+type LoadbalancerCorpus struct {
+	*ModelSets
+	ModelSetsMaxUpdatedAt *ModelSetsMaxUpdatedAt
+}
+
+func NewEmptyLoadbalancerCorpus() *LoadbalancerCorpus {
+	return &LoadbalancerCorpus{
+		ModelSets:             NewModelSets(),
+		ModelSetsMaxUpdatedAt: NewModelSetsMaxUpdatedAt(),
+	}
+}
+
+func (b *LoadbalancerCorpus) SyncModelSets(s *mcclient.ClientSession, batchSize int) (*ModelSetsUpdateResult, error) {
+	mss := b.ModelSetList()
+	mssNews := NewModelSets()
+	for i, msNew := range mssNews.ModelSetList() {
+		minUpdatedAt := ModelSetMaxUpdatedAt(mss[i])
+		err := GetModels(&GetModelsOptions{
+			ClientSession: s,
+			ModelManager:  msNew.ModelManager(),
+			MinUpdatedAt:  minUpdatedAt,
+			ModelSet:      msNew,
+			BatchListSize: batchSize,
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+	r := b.ModelSets.ApplyUpdates(mssNews)
+	b.ModelSetsMaxUpdatedAt = r.ModelSetsMaxUpdatedAt
+	return r, nil
+}
+
+func (b *LoadbalancerCorpus) MaxSeenUpdatedAtParams() *jsonutils.JSONDict {
+	mssmua := b.ModelSetsMaxUpdatedAt
+	return jsonutils.Marshal(mssmua).(*jsonutils.JSONDict)
+}
+
+func (b *LoadbalancerCorpus) SaveDir(dir string) error {
+	j := jsonutils.Marshal(b)
+	d := j.String()
+	p := filepath.Join(dir, "corpus")
+	err := ioutil.WriteFile(p, []byte(d), agentutils.FileModeFileSensitive)
+	return err
+}
+
+func (b *LoadbalancerCorpus) LoadDir(dir string) error {
+	p := filepath.Join(dir, "corpus")
+	d, err := ioutil.ReadFile(p)
+	if err != nil {
+		return err
+	}
+	jd, err := jsonutils.Parse(d)
+	if err != nil {
+		return fmt.Errorf("%s: json parse failed: %s", p, err)
+	}
+	err = jd.Unmarshal(b)
+	if err != nil {
+		return fmt.Errorf("%s: json unmarshal failed: %s", p, err)
+	}
+	correct := b.join()
+	if !correct {
+		return fmt.Errorf("%s: corpus data has inconsistencies", p)
+	}
+	return nil
+}

pkg/lbagent/models/gobetween.go

@@ -0,0 +1,164 @@
+package models
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/lbagent/gobetween"
+	agentutils "yunion.io/x/onecloud/pkg/lbagent/utils"
+)
+
+type GenGobetweenConfigOptions struct {
+	LoadbalancersEnabled []*Loadbalancer
+	AgentParams          *AgentParams
+
+	Config *gobetween.Config
+}
+
+func (b *LoadbalancerCorpus) GenGobetweenConfigs(dir string, opts *GenGobetweenConfigOptions) error {
+	//agentParams := opts.AgentParams
+	// TODO
+	//  - dynamic password
+	//  - log to remote
+	//  - log to local syslog unix sock
+	//  - respawn
+	opts.Config = &gobetween.Config{
+		Servers: map[string]gobetween.Server{},
+		Api: gobetween.ApiConfig{
+			Enabled: true,
+			Bind:    "localhost:777",
+			BasicAuth: &gobetween.ApiBasicAuthConfig{
+				Login:    "Yunion",
+				Password: "LBStats",
+			},
+		},
+	}
+	for _, lb := range opts.LoadbalancersEnabled {
+		for _, listener := range lb.listeners {
+			if listener.ListenerType != "udp" {
+				continue
+			}
+			if listener.Status != "enabled" {
+				continue
+			}
+			if listener.BackendGroupId == "" {
+				continue
+			}
+			backendGroup := lb.backendGroups[listener.BackendGroupId]
+			if backendGroup == nil || len(backendGroup.backends) == 0 {
+				continue
+			}
+
+			// backends
+			staticList := []string{}
+			for _, backend := range backendGroup.backends {
+				backendS := fmt.Sprintf("%s:%d weight=%d", backend.Address, backend.Port, backend.Weight)
+				staticList = append(staticList, backendS)
+			}
+
+			// scheduler
+			var serverBalance string
+			switch listener.Scheduler {
+			case "rr", "wrr":
+				serverBalance = "roundrobin"
+			case "lc", "wlc":
+				serverBalance = "leastconn"
+			case "sch", "tch":
+				log.Warningf("scheduler %s converted to iphash1", listener.Scheduler)
+				serverBalance = "iphash1"
+			default:
+				log.Warningf("scheduler %s converted to iphash1", listener.Scheduler)
+				serverBalance = "iphash1"
+			}
+
+			// healthcheck
+			var serverHealthcheck *gobetween.HealthcheckConfig
+			if listener.HealthCheck == "on" && listener.HealthCheckType == "udp" {
+				serverHealthcheck = &gobetween.HealthcheckConfig{
+					Kind:     "pingudp",
+					Interval: fmt.Sprintf("%ds", listener.HealthCheckInterval),
+					Timeout:  fmt.Sprintf("%ds", listener.HealthCheckTimeout),
+					Passes:   listener.HealthCheckRise,
+					Fails:    listener.HealthCheckFall,
+					UdpHealthcheckConfig: &gobetween.UdpHealthcheckConfig{
+						Send:    listener.HealthCheckReq,
+						Receive: listener.HealthCheckExp,
+					},
+				}
+			}
+
+			// acl
+			var serverAccess *gobetween.AccessConfig
+			if listener.AclStatus == "on" && listener.AclId != "" {
+				acl := b.LoadbalancerAcls[listener.AclId]
+				if acl == nil {
+					log.Warningf("listener %s(%s): unknown acl %s",
+						listener.Name, listener.Id, listener.AclId)
+					continue
+				}
+				var accessDefault string
+				var accessRulesAction string
+				accessRules := []string{}
+				switch listener.AclType {
+				case "black":
+					accessDefault = "allow"
+					accessRulesAction = "deny"
+				case "white":
+					accessDefault = "deny"
+					accessRulesAction = "allow"
+				default:
+					log.Warningf("listener %s(%s): unknown acl type: %s",
+						listener.Name, listener.Id, listener.AclId)
+					continue
+				}
+				for _, aclEntry := range *acl.AclEntries {
+					rule := fmt.Sprintf("%s %s", accessRulesAction, aclEntry.Cidr)
+					accessRules = append(accessRules, rule)
+				}
+				serverAccess = &gobetween.AccessConfig{
+					Default: accessDefault,
+					Rules:   accessRules,
+				}
+			}
+			pize := func(s string) *string {
+				return &s
+			}
+
+			opts.Config.Servers[listener.Id] = gobetween.Server{
+				Bind:     fmt.Sprintf("%s:%d", lb.Address, listener.ListenerPort),
+				Protocol: "udp",
+				Balance:  serverBalance,
+				Discovery: &gobetween.DiscoveryConfig{
+					Kind: "static",
+					StaticDiscoveryConfig: &gobetween.StaticDiscoveryConfig{
+						StaticList: staticList,
+					},
+				},
+				Healthcheck: serverHealthcheck,
+				Access:      serverAccess,
+				ConnectionOptions: gobetween.ConnectionOptions{
+					ClientIdleTimeout:        pize(fmt.Sprintf("%ds", listener.ClientIdleTimeout)),
+					BackendIdleTimeout:       pize(fmt.Sprintf("%ds", listener.BackendIdleTimeout)),
+					BackendConnectionTimeout: pize(fmt.Sprintf("%ds", listener.BackendConnectTimeout)),
+				},
+			}
+		}
+	}
+	{
+		// write gobetween.json
+		d, err := json.MarshalIndent(opts.Config, "", "  ")
+		if err != nil {
+			return err
+		}
+		p := filepath.Join(dir, "gobetween.json")
+		err = ioutil.WriteFile(p, d, agentutils.FileModeFile)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/lbagent/models/haproxy.go

@@ -0,0 +1,476 @@
+package models
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"yunion.io/x/log"
+
+	agentutils "yunion.io/x/onecloud/pkg/lbagent/utils"
+)
+
+var haproxyConfigErrNop = errors.New("nop haproxy config snippet")
+
+type GenHaproxyConfigsResult struct {
+	LoadbalancersEnabled []*Loadbalancer
+}
+
+func (b *LoadbalancerCorpus) GenHaproxyToplevelConfig(dir string, opts *AgentParams) error {
+	buf := bytes.NewBufferString("# yunion lb auto-generated 00-haproxy.cfg\n")
+	haproxyConfigTmpl := opts.HaproxyConfigTmpl
+	err := haproxyConfigTmpl.Execute(buf, opts.Data)
+	if err != nil {
+		return err
+	}
+	data := buf.Bytes()
+	p := filepath.Join(dir, "00-haproxy.cfg")
+	err = ioutil.WriteFile(p, data, agentutils.FileModeFile)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (b *LoadbalancerCorpus) GenHaproxyConfigs(dir string, opts *AgentParams) (*GenHaproxyConfigsResult, error) {
+	if len(b.LoadbalancerCertificates) > 0 {
+		certsBase := filepath.Join(dir, "certs")
+		certsBaseFinal := filepath.Join(agentutils.DirStagingToFinal(dir), "certs")
+		err := os.MkdirAll(certsBase, agentutils.FileModeDirSensitive)
+		if err != nil {
+			return nil, fmt.Errorf("mkdir %s: %s", certsBase, err)
+		}
+		{
+			p := filepath.Join(dir, "01-haproxy.cfg")
+			lines := []string{
+				"global",
+				fmt.Sprintf("	crt-base %s", certsBaseFinal),
+				"",
+			}
+			s := strings.Join(lines, "\n")
+			err := ioutil.WriteFile(p, []byte(s), agentutils.FileModeFile)
+			if err != nil {
+				return nil, fmt.Errorf("write 01-haproxy.cfg: %s", err)
+			}
+		}
+		for _, lbcert := range b.LoadbalancerCertificates {
+			d := []byte(lbcert.Certificate)
+			if d[len(d)-1] != '\n' {
+				d = append(d, '\n')
+			}
+			d = append(d, []byte(lbcert.PrivateKey)...)
+			fn := fmt.Sprintf("%s.pem", lbcert.Id)
+			p := filepath.Join(certsBase, fn)
+			err := ioutil.WriteFile(p, d, agentutils.FileModeFileSensitive)
+			if err != nil {
+				return nil, fmt.Errorf("write cert %s: %s", lbcert.Id, err)
+			}
+		}
+	}
+	for _, lbacl := range b.LoadbalancerAcls {
+		cidrs := []string{}
+		if lbacl.AclEntries != nil {
+			for _, aclEntry := range *lbacl.AclEntries {
+				cidrs = append(cidrs, aclEntry.Cidr)
+			}
+		}
+		if len(cidrs) > 0 {
+			s := fmt.Sprintf("## loadbalancer acl %s(%s)\n", lbacl.Name, lbacl.Id)
+			s += strings.Join(cidrs, "\n")
+			s += "\n"
+			p := filepath.Join(dir, "acl-"+lbacl.Id)
+			err := ioutil.WriteFile(p, []byte(s), agentutils.FileModeFile)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	r := &GenHaproxyConfigsResult{
+		LoadbalancersEnabled: []*Loadbalancer{},
+	}
+	for _, lb := range b.Loadbalancers {
+		if lb.Status != "enabled" {
+			continue
+		}
+		if lb.Address == "" {
+			continue
+		}
+		if len(lb.listeners) == 0 {
+			continue
+		}
+		buf := bytes.NewBufferString(fmt.Sprintf("## loadbalancer %s(%s)\n\n", lb.Name, lb.Id))
+		hasActiveListener := false
+		for _, listener := range lb.listeners {
+			if listener.Status != "enabled" {
+				continue
+			}
+			var err error
+			switch listener.ListenerType {
+			case "http", "https":
+				err = b.genHaproxyConfigHttp(buf, listener, opts)
+			case "tcp":
+				err = b.genHaproxyConfigTcp(buf, listener, opts)
+			case "udp":
+				// we record it for use in keepalived conf gen
+				r.LoadbalancersEnabled = append(r.LoadbalancersEnabled, lb)
+				continue
+			default:
+				log.Infof("haproxy: ignore listener type %s", listener.ListenerType)
+				continue
+			}
+			if err == haproxyConfigErrNop {
+				continue
+			}
+			if err != nil {
+				return nil, err
+			}
+			hasActiveListener = true
+			buf.WriteString("\n\n") // listeners sep lines
+		}
+		if hasActiveListener {
+			r.LoadbalancersEnabled = append(r.LoadbalancersEnabled, lb)
+
+			d := buf.Bytes()
+			d = d[:len(d)-2]
+			fn := fmt.Sprintf("%s.%s", lb.Id, agentutils.HaproxyCfgExt)
+			p := filepath.Join(dir, fn)
+			err := ioutil.WriteFile(p, d, agentutils.FileModeFile)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+	return r, nil
+}
+
+func (b *LoadbalancerCorpus) genHaproxyConfigCommon(lb *Loadbalancer, listener *LoadbalancerListener, opts *AgentParams) map[string]interface{} {
+	data := map[string]interface{}{
+		"comment":       fmt.Sprintf("%s(%s)", listener.Name, listener.Id),
+		"id":            listener.Id,
+		"listener_type": listener.ListenerType,
+	}
+	{
+		bind := fmt.Sprintf("%s:%d", lb.Address, listener.ListenerPort)
+		if listener.ListenerType == "https" && listener.certificate != nil {
+			bind += fmt.Sprintf(" ssl crt %s.pem", listener.certificate.Id)
+			if listener.TLSCipherPolicy != "" {
+				policy := agentutils.HaproxySslPolicy(listener.TLSCipherPolicy)
+				if policy != nil {
+					bind += fmt.Sprintf(" ssl-min-ver %s", policy.SslMinVer)
+				}
+			}
+			if listener.EnableHttp2 {
+				bind += fmt.Sprintf(" alpn h2,http/1.1")
+			}
+		}
+		data["bind"] = bind
+	}
+	{
+		agentHaproxyParams := opts.AgentModel.Params.Haproxy
+		if agentHaproxyParams.GlobalLog != "" {
+			if listener.ListenerType == "http" && agentHaproxyParams.LogHttp {
+				data["log"] = true
+			} else if listener.ListenerType == "tcp" && agentHaproxyParams.LogTcp {
+				data["log"] = true
+			}
+		}
+	}
+	if listener.AclStatus == "on" {
+		lbacl, ok := b.LoadbalancerAcls[listener.AclId]
+		if ok && lbacl.AclEntries != nil && len(*lbacl.AclEntries) > 0 {
+			var action, cond string
+			switch listener.ListenerType {
+			case "tcp":
+				action = "tcp-request connection reject"
+			case "http", "https":
+				action = "http-request deny"
+			}
+			switch listener.AclType {
+			case "black":
+				cond = "if"
+			case "white":
+				cond = "unless"
+			}
+			if action != "" && cond != "" {
+				acl := fmt.Sprintf("%s %s { src -f acl-%s }", action, cond, listener.AclId)
+				data["acl"] = acl
+			}
+		}
+	}
+	{
+		// NOTE timeout tunnel is not set.  We may need to prepare a
+		// default section for each frontend/listener
+		timeoutsMap := map[string]int{
+			"client_request_timeout": listener.ClientRequestTimeout,
+			"client_idle_timeout":    listener.ClientIdleTimeout,
+		}
+		for k, v := range timeoutsMap {
+			if v > 0 {
+				data[k] = fmt.Sprintf("%ds", v)
+			}
+		}
+	}
+	return data
+}
+
+func (b *LoadbalancerCorpus) genHaproxyConfigBackend(data map[string]interface{}, lb *Loadbalancer, listener *LoadbalancerListener, backendGroup *LoadbalancerBackendGroup) error {
+	var mode string
+	var balanceAlgorithm string
+	var httpCheck, httpCheckExpect string
+	var checkEnable, httpCheckEnable bool
+	var err error
+	{ // mode
+		switch listener.ListenerType {
+		case "tcp":
+			mode = "tcp"
+		case "http", "https":
+			mode = "http"
+		default:
+			return fmt.Errorf("haproxy: unsupported listener type %s", listener.ListenerType)
+		}
+	}
+	{ // balance algorithm
+		balanceAlgorithm, err = agentutils.HaproxyBalanceAlgorithm(listener.Scheduler)
+		if err != nil {
+			return err
+		}
+	}
+	{ // (http) check enabled?
+		if listener.HealthCheck == "on" {
+			checkEnable = true
+			if listener.HealthCheckType == "http" {
+				httpCheckEnable = true
+				httpCheck = agentutils.HaproxyConfigHttpCheck(
+					listener.HealthCheckURI, listener.HealthCheckDomain)
+				httpCheckExpect = agentutils.HaproxyConfigHttpCheckExpect(
+					listener.HealthCheckHttpCode)
+			}
+		}
+	}
+	var stickySessionEnable bool
+	if mode == "http" && listener.StickySession == "on" {
+		// sticky session
+		stickyCookie := ""
+		switch listener.StickySessionType {
+		case "insert":
+			stickyCookie = "cookie SERVERID insert indirect nocache"
+			if maxIdle := listener.StickySessionCookieTimeout; maxIdle > 0 {
+				stickyCookie += fmt.Sprintf(" maxidle %ds", maxIdle)
+			}
+		case "server":
+			cookie := listener.StickySessionCookie
+			if cookie != "" {
+				stickyCookie = fmt.Sprintf("cookie %q rewrite nocache", cookie)
+			}
+		}
+		if stickyCookie != "" {
+			data["stickyCookie"] = stickyCookie
+			stickySessionEnable = true
+		}
+	}
+	{
+		serverLines := []string{}
+		for _, backend := range backendGroup.backends {
+			serverLine := fmt.Sprintf("server %s %s:%d", backend.Id, backend.Address, backend.Port)
+			if listener.Scheduler == "rr" {
+				serverLine += " weight 1"
+			} else {
+				serverLine += fmt.Sprintf(" weight %d", backend.Weight)
+			}
+			if checkEnable {
+				serverLine += fmt.Sprintf(" check rise %d fall %d inter %ds",
+					listener.HealthCheckRise, listener.HealthCheckFall, listener.HealthCheckInterval)
+			}
+			if stickySessionEnable {
+				serverLine += fmt.Sprintf(" cookie %q", backend.Id)
+			}
+			serverLines = append(serverLines, serverLine)
+		}
+		data["servers"] = serverLines
+	}
+	if listener.HealthCheckTimeout > 0 {
+		data["timeout_check"] = fmt.Sprintf("timeout check %ds", listener.HealthCheckTimeout)
+	}
+	{
+		timeoutsMap := map[string]int{
+			"backend_connect_timeout": listener.BackendConnectTimeout,
+			"backend_idle_timeout":    listener.BackendIdleTimeout,
+			"backend_tunnel_timeout":  listener.BackendIdleTimeout,
+		}
+		for k, v := range timeoutsMap {
+			if v > 0 {
+				data[k] = fmt.Sprintf("%ds", v)
+			}
+		}
+	}
+	data["mode"] = mode
+	data["balanceAlgorithm"] = balanceAlgorithm
+	if httpCheckEnable {
+		data["httpCheck"] = httpCheck
+		data["httpCheckExpect"] = httpCheckExpect
+	}
+	return nil
+}
+
+func (b *LoadbalancerCorpus) genHaproxyConfigHttp(buf *bytes.Buffer, listener *LoadbalancerListener, opts *AgentParams) error {
+	lb := listener.loadbalancer
+	rules := LoadbalancerListenerRules{}
+	for id, rule := range listener.rules {
+		if rule.Status != "enabled" {
+			continue
+		}
+		rules[id] = rule
+	}
+	data := b.genHaproxyConfigCommon(lb, listener, opts)
+	ruleBackendIdGen := func(id string) string {
+		return fmt.Sprintf("backends_rule-%s", id)
+	}
+	{
+		// NOTE add X-Real-IP if needed
+		//
+		//	http-request set-header X-Client-IP %[src]
+		//
+		data["xforwardedfor"] = listener.XForwardedFor
+		data["gzip"] = listener.Gzip
+	}
+	{ // use_backend rule.Id if xx
+		ruleLines := []string{}
+		for _, rule := range rules {
+			ruleLine := fmt.Sprintf("use_backend %s", ruleBackendIdGen(rule.Id))
+			if rule.Domain != "" || rule.Path != "" {
+				ruleLine += " if"
+				if rule.Domain != "" {
+					ruleLine += fmt.Sprintf(" { hdr_dom(host) %q }", rule.Domain)
+				}
+				if rule.Path != "" {
+					ruleLine += fmt.Sprintf(" { path_beg %q }", rule.Path)
+				}
+			}
+			ruleLines = append(ruleLines, ruleLine)
+		}
+		data["rules"] = ruleLines
+	}
+	{
+		backends := []interface{}{}
+		// rules backend group
+		for _, rule := range rules {
+			// NOTE dup is ok
+			if rule.BackendGroupId == "" {
+				// just in case
+				continue
+			}
+			backendGroup := lb.backendGroups[rule.BackendGroupId]
+			backendData := map[string]interface{}{
+				"comment": fmt.Sprintf("rule %s(%s) backendGroup %s(%s)",
+					rule.Name, rule.Id,
+					backendGroup.Name, backendGroup.Id),
+				"id": ruleBackendIdGen(rule.Id),
+			}
+			err := b.genHaproxyConfigBackend(backendData, lb, listener, backendGroup)
+			if err != nil {
+				return err
+			}
+			backends = append(backends, backendData)
+		}
+		// default backend group
+		if listener.BackendGroupId != "" {
+			backendGroup := lb.backendGroups[listener.BackendGroupId]
+			backendData := map[string]interface{}{
+				"comment": fmt.Sprintf("listener %s(%s) default backendGroup %s(%s)",
+					listener.Name, listener.Id,
+					backendGroup.Name, backendGroup.Id),
+				"id": fmt.Sprintf("backends_listener_default-%s", listener.Id),
+			}
+			err := b.genHaproxyConfigBackend(backendData, lb, listener, backendGroup)
+			if err != nil {
+				return err
+			}
+			backends = append(backends, backendData)
+			data["default_backend"] = backendData
+		}
+		if len(backends) == 0 {
+			// no backendgroup specified, nothing to serve
+			return haproxyConfigErrNop
+		}
+		data["backends"] = backends
+	}
+	err := haproxyConfigTmpl.ExecuteTemplate(buf, "httpListen", data)
+	return err
+}
+
+func (b *LoadbalancerCorpus) genHaproxyConfigTcp(buf *bytes.Buffer, listener *LoadbalancerListener, opts *AgentParams) error {
+	lb := listener.loadbalancer
+	data := b.genHaproxyConfigCommon(lb, listener, opts)
+	if listener.BackendGroupId != "" {
+		backendGroup := lb.backendGroups[listener.BackendGroupId]
+		backendData := map[string]interface{}{
+			"comment": fmt.Sprintf("listener %s(%s) backendGroup %s(%s)",
+				listener.Name, listener.Id,
+				backendGroup.Name, backendGroup.Id),
+			"id": fmt.Sprintf("backends_listener-%s", listener.Id),
+		}
+		err := b.genHaproxyConfigBackend(backendData, lb, listener, backendGroup)
+		if err != nil {
+			return err
+		}
+		data["backend"] = backendData
+		err = haproxyConfigTmpl.ExecuteTemplate(buf, "tcpListen", data)
+		return err
+	}
+	return haproxyConfigErrNop
+}
+
+var haproxyConfigTmpl = template.Must(template.New("").Parse(`
+{{ define "tcpListen" -}}
+# {{ .listener_type }} listener: {{ .comment }}
+listen {{ .id }}
+	bind {{ .bind }}
+	mode tcp
+	{{- println }}
+	{{- if .log }}	{{ println "option tcplog" }} {{- end }}
+	{{- if .acl }}	{{ println .acl }} {{- end}}
+	{{- if .client_idle_timeout }}	timeout client {{ println .client_idle_timeout }} {{- end}}
+	default_backend {{ .backend.id }}
+{{ template "backend" .backend }}
+{{- end }}
+
+{{ define "httpListen" -}}
+# {{ .listener_type }} listener: {{ .comment }}
+frontend {{ .id }}
+	bind {{ .bind }}
+	mode http
+	{{- println }}
+	{{- if .log }}	{{ println "option httplog clf" }} {{- end }}
+	{{- if .acl }}	{{ println .acl }} {{- end}}
+	{{- if .client_request_timeout }}	timeout http-request {{ println .client_request_timeout }} {{- end}}
+	{{- if .client_idle_timeout }}	timeout http-keep-alive {{ println .client_idle_timeout }} {{- end}}
+	{{- if .xforwardedfor }}	{{ println "option forwardfor" }} {{- end}}
+	{{- if .gzip }}	{{ println "compression algo gzip" }} {{- end}}
+	{{- range .rules }}	{{ println . }} {{- end }}
+	{{- if .default_backend.id }}	default_backend {{ println .default_backend.id }} {{- end }}
+{{- range .backends }}
+{{- template "backend" . }}
+{{- end }}
+{{- end }}
+
+{{ define "backend" -}}
+# {{ .comment }}
+backend {{ .id }}
+	mode {{ .mode }}
+	balance {{ .balanceAlgorithm }}
+	{{- println }}
+	{{- if .backend_connect_timeout }}	timeout connect {{ println .backend_connect_timeout }} {{- end}}
+	{{- if .backend_idle_timeout }}	timeout server {{ println .backend_idle_timeout }} {{- end}}
+	{{- if .timeout_check }}	{{ println .timeout_check }} {{- end }}
+	{{- if .stickyCookie }}	{{ println .stickyCookie }} {{- end }}
+	{{- if .httpCheck }}	{{ println .httpCheck }} {{- end }}
+	{{- if .httpCheckExpect }}	{{ println .httpCheckExpect }} {{- end }}
+	{{- range .servers }}	{{ println . }} {{- end }}
+{{- end }}
+`))

pkg/lbagent/models/keepalived.go

@@ -0,0 +1,157 @@
+package models
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"text/template"
+
+	"yunion.io/x/log"
+
+	agentutils "yunion.io/x/onecloud/pkg/lbagent/utils"
+)
+
+const keepalivedMiscCheckPath = "/opt/yunion/share/lbagent/healthcheck.sh"
+
+type GenKeepalivedConfigOptions struct {
+	LoadbalancersEnabled []*Loadbalancer
+	AgentParams          *AgentParams
+}
+
+func (b *LoadbalancerCorpus) GenKeepalivedConfigs(dir string, opts *GenKeepalivedConfigOptions) error {
+	agentParams := opts.AgentParams
+	{
+		addresses := []string{}
+		for _, lb := range opts.LoadbalancersEnabled {
+			if lb.Status != "enabled" {
+				continue
+			}
+			if lb.Address == "" {
+				continue
+			}
+			addresses = append(addresses, lb.Address)
+		}
+		agentParams.SetVrrpParams("addresses", addresses)
+	}
+	buf := bytes.NewBufferString("# yunion lb auto-generated keepalived.conf\n")
+	{
+		// write global_defs and vrrp_instance
+		keepalivedConfigToplevelTmpl := agentParams.KeepalivedConfigTmpl
+		err := keepalivedConfigToplevelTmpl.Execute(buf, agentParams.Data)
+		if err != nil {
+			return err
+		}
+	}
+	if false {
+		// write virtual_server
+		for _, lb := range opts.LoadbalancersEnabled {
+			for _, listener := range lb.listeners {
+				if listener.Status != "enabled" {
+					continue
+				}
+				if listener.BackendGroupId == "" {
+					continue
+				}
+				if listener.ListenerType == "udp" {
+					dataReals := []map[string]interface{}{}
+					dataVirtual := map[string]interface{}{
+						"virtual_ip":   lb.Address,
+						"virtual_port": listener.ListenerPort,
+					}
+					{
+						// scheduler
+						switch listener.Scheduler {
+						case "rr", "wrr", "lc", "wlc":
+							dataVirtual["scheduler"] = listener.Scheduler
+						case "sch", "tch":
+							log.Warningf("scheduler %s converted to sh", listener.Scheduler)
+							dataVirtual["scheduler"] = "sh"
+						default:
+							log.Warningf("scheduler %s converted to sh", listener.Scheduler)
+							dataVirtual["scheduler"] = "sh"
+						}
+					}
+					udpCheckEnabled := false
+					{
+						if listener.HealthCheck == "on" && listener.HealthCheckType == "udp" && listener.HealthCheckReq != "" {
+							udpCheckEnabled = true
+						}
+					}
+					backendGroup := lb.backendGroups[listener.BackendGroupId]
+					for _, backend := range backendGroup.backends {
+						dataReal := map[string]interface{}{
+							"real_ip":   backend.Address,
+							"real_port": backend.Port,
+							"weight":    backend.Weight,
+						}
+						if udpCheckEnabled {
+							checkMiscArgs := []string{
+								keepalivedMiscCheckPath,
+								fmt.Sprintf("wait=%d", listener.HealthCheckTimeout),
+								fmt.Sprintf("req=%s", listener.HealthCheckReq),
+								fmt.Sprintf("exp=%s", listener.HealthCheckExp),
+								fmt.Sprintf("host=%s", backend.Address),
+								fmt.Sprintf("port=%d", backend.Port),
+							}
+							dataCheckMiscPath := agentutils.KeepalivedConfQuoteScriptArgs(checkMiscArgs)
+							dataCheck := map[string]interface{}{
+								"misc_path":    dataCheckMiscPath,
+								"misc_timeout": listener.HealthCheckTimeout,
+								"interval":     listener.HealthCheckInterval,
+								"fall":         listener.HealthCheckFall,
+								"usergroup":    "nobody",
+							}
+							dataReal["check"] = dataCheck
+						}
+						dataReals = append(dataReals, dataReal)
+					}
+					dataVirtual["real_servers"] = dataReals
+
+					fmt.Fprintf(buf, "\n\n")
+					fmt.Fprintf(buf, "## listener %s(%s) backendGroup %s(%s)\n",
+						listener.Name, listener.Id,
+						backendGroup.Name, backendGroup.Id)
+					keepalivedConfTmpl.ExecuteTemplate(buf, "keepalivedVirtualServerUDP", dataVirtual)
+				}
+			}
+		}
+	}
+	{
+		// write keepalived.conf
+		d := buf.Bytes()
+		p := filepath.Join(dir, "keepalived.conf")
+		err := ioutil.WriteFile(p, d, agentutils.FileModeFile)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// TODO retry for down, up?
+var keepalivedConfTmpl = template.Must(template.New("").Parse(`
+{{- define "keepalivedVirtualServerUDP" -}}
+virtual_server {{ .virtual_ip }} {{ .virtual_port }} {
+	protocol UDP
+	lvs_method NAT
+	lvs_sched {{ .scheduler }}
+	ha_suspend
+	{{- range .real_servers }}
+	real_server {{ .real_ip }} {{ .real_port }} {
+		weight {{ .weight }}
+		inhibit_on_failure
+		{{- if .check }}
+		MISC_CHECK {
+			misc_path {{ .check.misc_path }}
+			misc_timeout {{ .check.misc_timeout }}
+			delay_loop {{ .check.interval }}
+			retry {{ .check.fall }}
+			user {{ .check.usergroup }}
+		}
+		{{- end }}
+	}
+	{{- end }}
+}
+{{ end }}
+`))

pkg/lbagent/models/models.go

@@ -0,0 +1,48 @@
+package models
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient/models"
+)
+
+type IModel interface {
+}
+
+type Loadbalancer struct {
+	*models.Loadbalancer
+
+	listeners     LoadbalancerListeners
+	backendGroups LoadbalancerBackendGroups
+}
+
+type LoadbalancerListener struct {
+	*models.LoadbalancerListener
+
+	loadbalancer *Loadbalancer
+	certificate  *LoadbalancerCertificate
+	rules        LoadbalancerListenerRules
+}
+
+type LoadbalancerListenerRule struct {
+	*models.LoadbalancerListenerRule
+
+	listener *LoadbalancerListener
+}
+
+type LoadbalancerBackendGroup struct {
+	*models.LoadbalancerBackendGroup
+
+	backends     LoadbalancerBackends
+	loadbalancer *Loadbalancer
+}
+
+type LoadbalancerBackend struct {
+	*models.LoadbalancerBackend
+}
+
+type LoadbalancerAcl struct {
+	*models.LoadbalancerAcl
+}
+
+type LoadbalancerCertificate struct {
+	*models.LoadbalancerCertificate
+}

pkg/lbagent/models/modelset.go

@@ -0,0 +1,249 @@
+package models
+
+import (
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/mcclient/models"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+type IModelSet interface {
+	//InitializeFromJSON([]jsonutils.JSONObject) error
+	ModelManager() modules.Manager
+	NewModel() models.IVirtualResource
+	//GetModel(id string) models.IVirtualResource
+	addModelCallback(models.IVirtualResource) error
+}
+
+type Loadbalancers map[string]*Loadbalancer
+type LoadbalancerListeners map[string]*LoadbalancerListener
+type LoadbalancerListenerRules map[string]*LoadbalancerListenerRule
+type LoadbalancerBackendGroups map[string]*LoadbalancerBackendGroup
+type LoadbalancerBackends map[string]*LoadbalancerBackend
+type LoadbalancerAcls map[string]*LoadbalancerAcl
+type LoadbalancerCertificates map[string]*LoadbalancerCertificate
+
+func (set Loadbalancers) ModelManager() modules.Manager {
+	return &modules.Loadbalancers
+}
+
+func (set Loadbalancers) NewModel() models.IVirtualResource {
+	return &models.Loadbalancer{}
+}
+
+func (set Loadbalancers) addModelCallback(i models.IVirtualResource) error {
+	m, _ := i.(*models.Loadbalancer)
+	set[m.Id] = &Loadbalancer{
+		Loadbalancer:  m,
+		listeners:     LoadbalancerListeners{},
+		backendGroups: LoadbalancerBackendGroups{},
+	}
+	return nil
+}
+
+func (ms Loadbalancers) JoinListeners(subEntries LoadbalancerListeners) bool {
+	for _, m := range ms {
+		m.listeners = LoadbalancerListeners{}
+	}
+	correct := true
+	for subId, subEntry := range subEntries {
+		id := subEntry.LoadbalancerId
+		m, ok := ms[id]
+		if !ok {
+			log.Warningf("loadbalancer id %s not found", id)
+			correct = false
+			continue
+		}
+		if _, ok := m.listeners[subId]; ok {
+			log.Warningf("loadbalancer listener id %s already joined", subId)
+			continue
+		}
+		subEntry.loadbalancer = m
+		m.listeners[subId] = subEntry
+	}
+	return correct
+}
+
+func (ms Loadbalancers) JoinBackendGroups(subEntries LoadbalancerBackendGroups) bool {
+	for _, m := range ms {
+		m.backendGroups = LoadbalancerBackendGroups{}
+	}
+	correct := true
+	for subId, subEntry := range subEntries {
+		id := subEntry.LoadbalancerId
+		m, ok := ms[id]
+		if !ok {
+			log.Warningf("loadbalancer id %s not found", id)
+			correct = false
+			continue
+		}
+		if _, ok := m.backendGroups[subId]; ok {
+			log.Warningf("loadbalancer backendgroup id %s already joined", subId)
+			continue
+		}
+		subEntry.loadbalancer = m
+		m.backendGroups[subId] = subEntry
+	}
+	return correct
+}
+
+func (set LoadbalancerListeners) ModelManager() modules.Manager {
+	return &modules.LoadbalancerListeners
+}
+
+func (set LoadbalancerListeners) NewModel() models.IVirtualResource {
+	return &models.LoadbalancerListener{}
+}
+
+func (set LoadbalancerListeners) addModelCallback(i models.IVirtualResource) error {
+	m, _ := i.(*models.LoadbalancerListener)
+	set[m.Id] = &LoadbalancerListener{
+		LoadbalancerListener: m,
+		rules:                LoadbalancerListenerRules{},
+	}
+	return nil
+}
+
+func (ms LoadbalancerListeners) JoinListenerRules(subEntries LoadbalancerListenerRules) bool {
+	for _, m := range ms {
+		m.rules = LoadbalancerListenerRules{}
+	}
+	correct := true
+	for subId, subEntry := range subEntries {
+		id := subEntry.ListenerId
+		m, ok := ms[id]
+		if !ok {
+			log.Warningf("loadbalancer listener id %s not found", id)
+			correct = false
+			continue
+		}
+		if _, ok := m.rules[subId]; ok {
+			log.Warningf("loadbalancer rule id %s already joined", subId)
+			continue
+		}
+		subEntry.listener = m
+		m.rules[subId] = subEntry
+	}
+	return correct
+}
+
+func (ms LoadbalancerListeners) JoinCertificates(subEntries LoadbalancerCertificates) bool {
+	correct := true
+	for _, m := range ms {
+		m.certificate = nil
+		if m.CertificateId != "" {
+			subEntry, ok := subEntries[m.CertificateId]
+			if !ok {
+				log.Warningf("loadbalancer m id %s: cannot find certificate id %s",
+					m.Id, m.CertificateId)
+				correct = false
+				continue
+			}
+			m.certificate = subEntry
+		}
+	}
+	return correct
+}
+
+func (set LoadbalancerListenerRules) ModelManager() modules.Manager {
+	return &modules.LoadbalancerListenerRules
+}
+
+func (set LoadbalancerListenerRules) NewModel() models.IVirtualResource {
+	return &models.LoadbalancerListenerRule{}
+}
+
+func (set LoadbalancerListenerRules) addModelCallback(i models.IVirtualResource) error {
+	m, _ := i.(*models.LoadbalancerListenerRule)
+	set[m.Id] = &LoadbalancerListenerRule{
+		LoadbalancerListenerRule: m,
+	}
+	return nil
+}
+
+func (set LoadbalancerBackendGroups) ModelManager() modules.Manager {
+	return &modules.LoadbalancerBackendGroups
+}
+
+func (set LoadbalancerBackendGroups) NewModel() models.IVirtualResource {
+	return &models.LoadbalancerBackendGroup{}
+}
+
+func (set LoadbalancerBackendGroups) addModelCallback(i models.IVirtualResource) error {
+	m, _ := i.(*models.LoadbalancerBackendGroup)
+	set[m.Id] = &LoadbalancerBackendGroup{
+		LoadbalancerBackendGroup: m,
+		backends:                 LoadbalancerBackends{},
+	}
+	return nil
+}
+
+func (ms LoadbalancerBackendGroups) JoinBackends(subEntries LoadbalancerBackends) bool {
+	for _, m := range ms {
+		m.backends = LoadbalancerBackends{}
+	}
+	correct := true
+	for subId, subEntry := range subEntries {
+		id := subEntry.BackendGroupId
+		m, ok := ms[id]
+		if !ok {
+			log.Warningf("loadbalancer backend group id %s not found", id)
+			correct = false
+			continue
+		}
+		if _, ok := m.backends[subId]; ok {
+			log.Warningf("loadbalancer backend id %s already joined", subId)
+			continue
+		}
+		m.backends[subId] = subEntry
+	}
+	return correct
+}
+
+func (set LoadbalancerBackends) ModelManager() modules.Manager {
+	return &modules.LoadbalancerBackends
+}
+
+func (set LoadbalancerBackends) NewModel() models.IVirtualResource {
+	return &models.LoadbalancerBackend{}
+}
+
+func (set LoadbalancerBackends) addModelCallback(i models.IVirtualResource) error {
+	m, _ := i.(*models.LoadbalancerBackend)
+	set[m.Id] = &LoadbalancerBackend{
+		LoadbalancerBackend: m,
+	}
+	return nil
+}
+
+func (set LoadbalancerAcls) ModelManager() modules.Manager {
+	return &modules.LoadbalancerAcls
+}
+
+func (set LoadbalancerAcls) NewModel() models.IVirtualResource {
+	return &models.LoadbalancerAcl{}
+}
+
+func (set LoadbalancerAcls) addModelCallback(i models.IVirtualResource) error {
+	m, _ := i.(*models.LoadbalancerAcl)
+	set[m.Id] = &LoadbalancerAcl{
+		LoadbalancerAcl: m,
+	}
+	return nil
+}
+
+func (set LoadbalancerCertificates) ModelManager() modules.Manager {
+	return &modules.LoadbalancerCertificates
+}
+
+func (set LoadbalancerCertificates) NewModel() models.IVirtualResource {
+	return &models.LoadbalancerCertificate{}
+}
+
+func (set LoadbalancerCertificates) addModelCallback(i models.IVirtualResource) error {
+	m, _ := i.(*models.LoadbalancerCertificate)
+	set[m.Id] = &LoadbalancerCertificate{
+		LoadbalancerCertificate: m,
+	}
+	return nil
+}

pkg/lbagent/models/modelsets.go

@@ -0,0 +1,138 @@
+package models
+
+import (
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+)
+
+// pluralMap maps from KeyPlurals to underscore-separated field names
+var pluralMap = map[string]string{}
+
+func init() {
+	ss := []string{
+		"loadbalancers",
+		"loadbalancer_listeners",
+		"loadbalancer_listener_rules",
+		"loadbalancer_backend_groups",
+		"loadbalancer_backends",
+		"loadbalancer_acls",
+		"loadbalancer_certificates",
+	}
+	for _, s := range ss {
+		k := strings.Replace(s, "_", "", -1)
+		pluralMap[k] = s
+	}
+}
+
+type ModelSetsMaxUpdatedAt struct {
+	Loadbalancers             time.Time
+	LoadbalancerListeners     time.Time
+	LoadbalancerListenerRules time.Time
+	LoadbalancerBackendGroups time.Time
+	LoadbalancerBackends      time.Time
+	LoadbalancerAcls          time.Time
+	LoadbalancerCertificates  time.Time
+}
+
+func NewModelSetsMaxUpdatedAt() *ModelSetsMaxUpdatedAt {
+	return &ModelSetsMaxUpdatedAt{
+		Loadbalancers:             PseudoZeroTime,
+		LoadbalancerListeners:     PseudoZeroTime,
+		LoadbalancerListenerRules: PseudoZeroTime,
+		LoadbalancerBackendGroups: PseudoZeroTime,
+		LoadbalancerBackends:      PseudoZeroTime,
+		LoadbalancerAcls:          PseudoZeroTime,
+		LoadbalancerCertificates:  PseudoZeroTime,
+	}
+}
+
+type ModelSets struct {
+	Loadbalancers             Loadbalancers
+	LoadbalancerListeners     LoadbalancerListeners
+	LoadbalancerListenerRules LoadbalancerListenerRules
+	LoadbalancerBackendGroups LoadbalancerBackendGroups
+	LoadbalancerBackends      LoadbalancerBackends
+	LoadbalancerAcls          LoadbalancerAcls
+	LoadbalancerCertificates  LoadbalancerCertificates
+}
+
+func NewModelSets() *ModelSets {
+	return &ModelSets{
+		Loadbalancers:             Loadbalancers{},
+		LoadbalancerListeners:     LoadbalancerListeners{},
+		LoadbalancerListenerRules: LoadbalancerListenerRules{},
+		LoadbalancerBackendGroups: LoadbalancerBackendGroups{},
+		LoadbalancerBackends:      LoadbalancerBackends{},
+		LoadbalancerAcls:          LoadbalancerAcls{},
+		LoadbalancerCertificates:  LoadbalancerCertificates{},
+	}
+}
+
+func (mss *ModelSets) ModelSetList() []IModelSet {
+	// it's ordered this way to favour creation, not deletion
+	return []IModelSet{
+		mss.LoadbalancerListenerRules,
+		mss.LoadbalancerListeners,
+		mss.LoadbalancerBackends,
+		mss.LoadbalancerBackendGroups,
+		mss.Loadbalancers,
+		mss.LoadbalancerAcls,
+		mss.LoadbalancerCertificates,
+	}
+}
+
+func (mss *ModelSets) MaxSeenUpdatedAtParams() *jsonutils.JSONDict {
+	d := jsonutils.NewDict()
+	for _, ms := range mss.ModelSetList() {
+		k := ms.ModelManager().KeyString()
+		k = pluralMap[k]
+		t := ModelSetMaxUpdatedAt(ms)
+		if !t.Equal(PseudoZeroTime) {
+			d.Set(k, jsonutils.NewTimeString(t))
+		}
+	}
+	return d
+}
+
+type ModelSetsUpdateResult struct {
+	Correct               bool // all elements referenced are present
+	Changed               bool // any thing changed in the corpus
+	ModelSetsMaxUpdatedAt *ModelSetsMaxUpdatedAt
+}
+
+func (mss *ModelSets) ApplyUpdates(mssNews *ModelSets) *ModelSetsUpdateResult {
+	r := &ModelSetsUpdateResult{
+		Changed: false,
+		Correct: true,
+	}
+	mssmua := NewModelSetsMaxUpdatedAt()
+	mssList := mss.ModelSetList()
+	mssNewsList := mssNews.ModelSetList()
+	for i, mss := range mssList {
+		mssNews := mssNewsList[i]
+		msR := ModelSetApplyUpdates(mss, mssNews)
+		if !r.Changed && msR.Changed {
+			r.Changed = true
+		}
+		{
+			keyPlural := mss.ModelManager().KeyString()
+			ModelSetsMaxUpdatedAtSetField(mssmua, keyPlural, msR.MaxUpdatedAt)
+		}
+	}
+	if r.Changed {
+		r.Correct = mss.join()
+	}
+	r.ModelSetsMaxUpdatedAt = mssmua
+	return r
+}
+
+func (mss *ModelSets) join() bool {
+	correct0 := mss.LoadbalancerBackendGroups.JoinBackends(mss.LoadbalancerBackends)
+	correct1 := mss.LoadbalancerListeners.JoinListenerRules(mss.LoadbalancerListenerRules)
+	correct2 := mss.LoadbalancerListeners.JoinCertificates(mss.LoadbalancerCertificates)
+	correct3 := mss.Loadbalancers.JoinListeners(mss.LoadbalancerListeners)
+	correct4 := mss.Loadbalancers.JoinBackendGroups(mss.LoadbalancerBackendGroups)
+	return correct0 && correct1 && correct2 && correct3 && correct4
+}

pkg/lbagent/models/reflect.go

@@ -0,0 +1,246 @@
+package models
+
+import (
+	"fmt"
+	"reflect"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/util/timeutils"
+	"yunion.io/x/pkg/utils"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/models"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+// A hack to workaround the IsZero() in timeutils.Utcify.  This depends on the
+// fact that database time has a resolution of 1-second
+var PseudoZeroTime = time.Time{}.Add(time.Nanosecond)
+
+type GetModelsOptions struct {
+	ClientSession *mcclient.ClientSession
+	ModelManager  modules.Manager
+	ModelSet      IModelSet
+
+	BatchListSize int
+	MinUpdatedAt  time.Time
+}
+
+func GetModels(opts *GetModelsOptions) error {
+	man := opts.ModelManager
+	manKeyPlural := man.KeyString()
+
+	minUpdatedAt := opts.MinUpdatedAt
+	minUpdatedAtFilter := func(time time.Time) string {
+		// TODO add GE
+		tstr := timeutils.MysqlTime(time)
+		return fmt.Sprintf("updated_at.ge('%s')", tstr)
+	}
+	setNextListParams := func(params *jsonutils.JSONDict, lastUpdatedAt time.Time, lastResult *modules.ListResult) (time.Time, error) {
+		// NOTE: the updated_at field has second-level resolution.
+		// If they all have the same date...
+		var max time.Time
+		nmax := 0
+		n := len(lastResult.Data)
+
+		// find out the max updated_at date in the result set, and how
+		// many in the set has this date
+		for i := n - 1; i >= 0; i-- {
+			j := lastResult.Data[i]
+			updatedAt, err := j.GetTime("updated_at")
+			if err != nil {
+				log.Warningf("%s: updated_at field: %s, %s",
+					manKeyPlural, err, j.String())
+				continue
+			}
+			if max.IsZero() {
+				max = updatedAt
+			}
+			if max.Equal(updatedAt) {
+				nmax += 1
+			}
+		}
+		// error if we do not have valid date
+		if max.IsZero() {
+			return time.Time{}, fmt.Errorf("cannot find next updated_at after '%q'",
+				manKeyPlural, lastUpdatedAt)
+		}
+
+		var newTime time.Time
+		var newOffset int
+		// if not all updated_at date are the same, then we can
+		// continue to the next age.
+		if nmax < n || (!max.Equal(lastUpdatedAt) && !max.Equal(PseudoZeroTime)) {
+			newTime = max
+			newOffset = nmax
+		} else {
+			newTime = lastUpdatedAt
+			newOffset = lastResult.Offset + n
+		}
+		params.Set("filter.0", jsonutils.NewString(minUpdatedAtFilter(newTime)))
+		params.Set("offset", jsonutils.NewInt(int64(newOffset)))
+		return newTime, nil
+	}
+
+	listOptions := options.BaseListOptions{
+		Admin:   options.Bool(true),
+		Filter:  []string{minUpdatedAtFilter(minUpdatedAt)},
+		OrderBy: []string{"updated_at", "id"},
+		Order:   "asc",
+		Limit:   options.Int(opts.BatchListSize),
+		Offset:  options.Int(0),
+	}
+	if !minUpdatedAt.Equal(PseudoZeroTime) {
+		// Only fetching pending deletes when we are doing incremental fetch
+		listOptions.PendingDeleteAll = options.Bool(true)
+	}
+	params, err := listOptions.Params()
+	if err != nil {
+		return fmt.Errorf("%s: making list params: %s", manKeyPlural, err)
+	}
+
+	entriesJson := []jsonutils.JSONObject{}
+	for {
+		var err error
+		listResult, err := opts.ModelManager.List(opts.ClientSession, params)
+		if err != nil {
+			return fmt.Errorf("%s: list failed with updated_at.gt('%s'): %s",
+				manKeyPlural, minUpdatedAt, err)
+		}
+		entriesJson = append(entriesJson, listResult.Data...)
+		if listResult.Offset+len(listResult.Data) >= listResult.Total {
+			break
+		}
+		minUpdatedAt, err = setNextListParams(params, minUpdatedAt, listResult)
+		if err != nil {
+			return fmt.Errorf("%s: %s", manKeyPlural, err)
+		}
+	}
+	{
+		err := InitializeModelSetFromJSON(opts.ModelSet, entriesJson)
+		if err != nil {
+			return fmt.Errorf("%s: initializing model set failed: %s",
+				manKeyPlural, err)
+		}
+	}
+	return nil
+}
+
+func InitializeModelSetFromJSON(set IModelSet, entriesJson []jsonutils.JSONObject) error {
+	setRv := reflect.ValueOf(set)
+	for _, kRv := range setRv.MapKeys() {
+		zRv := reflect.Value{}
+		setRv.SetMapIndex(kRv, zRv)
+	}
+	manKeyPlural := set.ModelManager().KeyString()
+	for _, entryJson := range entriesJson {
+		m := set.NewModel()
+		var err error
+		err = entryJson.Unmarshal(m)
+		if err != nil {
+			return fmt.Errorf("%s: error unmarshaling: %s: %s", manKeyPlural, err, entryJson.String())
+		}
+		{
+			keyRv := reflect.ValueOf(m.GetId())
+			oldMRv := setRv.MapIndex(keyRv)
+			if oldMRv.IsValid() {
+				// check version
+				oldM := oldMRv.Interface().(models.IVirtualResource)
+				oldVersion := oldM.GetUpdateVersion()
+				version := m.GetUpdateVersion()
+				if oldVersion > version {
+					oldUpdatedAt := oldM.GetUpdatedAt()
+					updatedAt := m.GetUpdatedAt()
+					log.Warningf("prefer loadbalancer with update_version %d(%s) to %d(%s)",
+						oldVersion, oldUpdatedAt, version, updatedAt)
+					return nil
+				}
+			}
+		}
+		err = set.addModelCallback(m)
+		if err != nil {
+			return fmt.Errorf("%s: add model: %s", manKeyPlural, err)
+		}
+	}
+	return nil
+}
+
+func ModelSetMaxUpdatedAt(set IModelSet) time.Time {
+	r := PseudoZeroTime
+	setRv := reflect.ValueOf(set)
+	for _, kRv := range setRv.MapKeys() {
+		mRv := setRv.MapIndex(kRv)
+		m := mRv.Interface().(models.IVirtualResource)
+		updatedAt := m.GetUpdatedAt()
+		if r.Before(updatedAt) {
+			r = updatedAt
+		}
+	}
+	return r
+}
+
+type ModelSetUpdateResult struct {
+	Changed      bool
+	MaxUpdatedAt time.Time
+}
+
+// ModelSetApplyUpdates applies bSet to aSet.
+//
+//  - PendingDeleted in bSet are removed from aSet
+//  - Newer models in bSet are updated in aSet
+func ModelSetApplyUpdates(aSet, bSet IModelSet) *ModelSetUpdateResult {
+	r := &ModelSetUpdateResult{
+		Changed: false,
+	}
+	{
+		a := ModelSetMaxUpdatedAt(aSet)
+		b := ModelSetMaxUpdatedAt(bSet)
+		if b.After(a) {
+			r.MaxUpdatedAt = b
+		} else {
+			r.MaxUpdatedAt = a
+		}
+	}
+	aSetRv := reflect.ValueOf(aSet)
+	bSetRv := reflect.ValueOf(bSet)
+	for _, kRv := range bSetRv.MapKeys() {
+		bMRv := bSetRv.MapIndex(kRv)
+		bM := bMRv.Interface().(models.IVirtualResource)
+		aMRv := aSetRv.MapIndex(kRv)
+		if aMRv.IsValid() {
+			aM := aMRv.Interface().(models.IVirtualResource)
+			if bM.GetPendingDeleted() {
+				// oops, deleted
+				aSetRv.SetMapIndex(kRv, reflect.Value{})
+				r.Changed = true
+				continue
+			}
+			if aM.GetUpdateVersion() < bM.GetUpdateVersion() {
+				// oops, updated
+				aSetRv.SetMapIndex(kRv, bMRv)
+				r.Changed = true
+				continue
+			}
+		} else {
+			if bM.GetPendingDeleted() {
+				// hmm, gone before even knowning
+				continue
+			}
+			// oops, new member
+			aSetRv.SetMapIndex(kRv, bMRv)
+			r.Changed = true
+		}
+	}
+	return r
+}
+
+func ModelSetsMaxUpdatedAtSetField(mssmua *ModelSetsMaxUpdatedAt, keyPlural string, t time.Time) {
+	field_name := pluralMap[keyPlural]
+	fieldName := utils.Kebab2Camel(field_name, "_")
+	rv := reflect.Indirect(reflect.ValueOf(mssmua))
+	fieldRv := rv.FieldByName(fieldName)
+	fieldRv.Set(reflect.ValueOf(t))
+}

pkg/lbagent/options.go

@@ -0,0 +1,63 @@
+package lbagent
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon"
+	agentutils "yunion.io/x/onecloud/pkg/lbagent/utils"
+)
+
+type LbagentOptions struct {
+	ApiLbagentId                  string `require:true`
+	ApiLbagentHbInterval          int    `default:10`
+	ApiLbagentHbTimeoutRelaxation int    `default:120 help:"If agent is to stale out in specified seconds in the future, consider it staled to avoid race condition when doing incremental api data fetch"`
+
+	ApiSyncInterval  int
+	ApiListBatchSize int `default:1024`
+
+	DataPreserveN int `default:8 help:"number of recent data to preserve on disk"`
+
+	BaseDataDir      string // `required:true`
+	apiDataStoreDir  string
+	haproxyConfigDir string
+	haproxyRunDir    string
+
+	KeepalivedBin string `default:keepalived`
+	HaproxyBin    string `default:haproxy`
+	GobetweenBin  string `default:gobetween`
+}
+
+type Options struct {
+	CommonOpts cloudcommon.Options
+
+	LbagentOptions
+}
+
+func (opts *Options) ValidateThenInit() error {
+	if opts.ApiListBatchSize <= 0 {
+		return fmt.Errorf("negative api batch list size: %d",
+			opts.ApiListBatchSize)
+	}
+	return opts.initDirs()
+}
+
+func (opts *Options) initDirs() error {
+	opts.apiDataStoreDir = filepath.Join(opts.BaseDataDir, "data")
+	opts.haproxyConfigDir = filepath.Join(opts.BaseDataDir, "configs")
+	opts.haproxyRunDir = filepath.Join(opts.BaseDataDir, "run")
+	dirs := []string{
+		opts.apiDataStoreDir,
+		opts.haproxyConfigDir,
+		opts.haproxyRunDir,
+	}
+	for _, dir := range dirs {
+		err := os.MkdirAll(dir, agentutils.FileModeDir)
+		if err != nil {
+			return fmt.Errorf("mkdir -p %q: %s",
+				dir, err)
+		}
+	}
+	return nil
+}

pkg/lbagent/utils/configdir.go

@@ -0,0 +1,142 @@
+package utils
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+
+	aggrerrors "yunion.io/x/pkg/util/errors"
+)
+
+const (
+	ConfigDirMode       = os.FileMode(0755)
+	configDirFmt        = "20060102.150405.000"
+	configDirFmtStaging = configDirFmt + ".staging"
+)
+
+var (
+	configDirPat = regexp.MustCompile(`^\d{8}\.\d{6}\.\d{3}$`)
+)
+
+type ConfigDirManager struct {
+	baseDir string
+}
+
+// TODO
+//
+//  - set active
+func NewConfigDirManager(baseDir string) *ConfigDirManager {
+	return &ConfigDirManager{
+		baseDir: baseDir,
+	}
+}
+
+type DirFunc func(string) error
+
+// call f() inside 20180830.104050.333.lbcorpus.json.staging
+// move to final name 20180830.104050.333.lbcorpus.json
+func (m *ConfigDirManager) NewDir(f DirFunc) (finalDir string, err error) {
+	stagingDir := m.stagingDir()
+	if stagingDir == "" {
+		err = fmt.Errorf("failed creating staging dir")
+		return
+	}
+	defer func() {
+		if err != nil {
+			os.RemoveAll(stagingDir)
+		}
+	}()
+
+	err = f(stagingDir)
+
+	if err != nil {
+		return
+	}
+	finalDir = DirStagingToFinal(stagingDir)
+	err = os.Rename(stagingDir, finalDir)
+	if err != nil {
+		return
+	}
+	return
+}
+
+func (m *ConfigDirManager) subdirs() []string {
+	dirs := []string{}
+	fis, err := ioutil.ReadDir(m.baseDir)
+	if err != nil {
+		return dirs
+	}
+	for _, fi := range fis {
+		if !fi.IsDir() {
+			continue
+		}
+		dirName := fi.Name()
+		if !configDirPat.Match([]byte(dirName)) {
+			continue
+		}
+		dir := filepath.Join(m.baseDir, dirName)
+		dirs = append(dirs, dir)
+	}
+	return dirs
+}
+
+func (m *ConfigDirManager) MostRecentSubdir() string {
+	dirs := m.subdirs()
+	nDirs := len(dirs)
+	if nDirs == 0 {
+		return ""
+	}
+	return dirs[nDirs-1]
+}
+
+func (m *ConfigDirManager) Prune(nRetain int) error {
+	if nRetain <= 0 {
+		// zero means prune nothing
+		return nil
+	}
+	dirs := m.subdirs()
+	if len(dirs) > nRetain {
+		// retain the most recent n
+		dirs = dirs[:len(dirs)-nRetain]
+		errs := []error{}
+		for _, dir := range dirs {
+			err := os.RemoveAll(dir)
+			if os.IsNotExist(err) {
+				continue
+			}
+			errs = append(errs, err)
+		}
+		if len(errs) > 0 {
+			return aggrerrors.NewAggregate(errs)
+		}
+	}
+	return nil
+}
+
+// stagingDir creates a new staging dir and returns the full path
+func (m *ConfigDirManager) stagingDir() string {
+	for {
+		now := time.Now()
+		dn := now.Format(configDirFmtStaging)
+		path := filepath.Join(m.baseDir, dn)
+		_, err := os.Stat(path)
+		if err != nil && os.IsNotExist(err) {
+			os.MkdirAll(path, ConfigDirMode)
+			return path
+		}
+		time.Sleep(time.Millisecond)
+	}
+	// TODO panic retries
+	return ""
+}
+
+func DirStagingToFinal(s string) string {
+	if strings.HasSuffix(s, ".staging") {
+		return s[:len(s)-8]
+	}
+	return s
+}

pkg/lbagent/utils/filemode.go

@@ -0,0 +1,12 @@
+package utils
+
+import (
+	"os"
+)
+
+const (
+	FileModeDir           = os.FileMode(0755)
+	FileModeFile          = os.FileMode(0644)
+	FileModeDirSensitive  = os.FileMode(0700)
+	FileModeFileSensitive = os.FileMode(0600)
+)

pkg/lbagent/utils/haproxy.go

@@ -0,0 +1,71 @@
+package utils
+
+import (
+	"fmt"
+	"strings"
+)
+
+const HaproxyCfgExt = "cfg"
+
+func HaproxyBalanceAlgorithm(scheduler string) (balance string, err error) {
+	switch scheduler {
+	case "rr", "wrr":
+		balance = "roundrobin"
+	case "wlc":
+		balance = "leastconn"
+	case "sch":
+		balance = "source"
+	case "tch":
+		// NOTE haproxy supports only TCP type proxy
+		balance = "source"
+	default:
+		err = fmt.Errorf("unknown scheduler type %q", scheduler)
+	}
+	return
+}
+
+type HaproxySslPolicyParams struct {
+	SslMinVer string
+	Ciphers   string
+}
+
+// TODO restrict ciphers as noted in https://help.aliyun.com/document_detail/90740.html
+func HaproxySslPolicy(policy string) *HaproxySslPolicyParams {
+	r := &HaproxySslPolicyParams{}
+	switch policy {
+	case "tls_cipher_policy_1_0":
+		r.SslMinVer = "TLSv1.0"
+	case "tls_cipher_policy_1_1":
+		r.SslMinVer = "TLSv1.1"
+	case "tls_cipher_policy_1_2":
+		r.SslMinVer = "TLSv1.2"
+	case "tls_cipher_policy_1_2_strict":
+		r.SslMinVer = "TLSv1.2"
+	default:
+		return nil
+	}
+	return r
+}
+
+func HaproxyConfigHttpCheck(uri, domain string) string {
+	if uri == "" {
+		uri = "/"
+	}
+	s := fmt.Sprintf("option httpchk HEAD %s HTTP/1.0", uri)
+	if domain != "" {
+		s += `\r\nHost:\ ` + domain
+	}
+	return s
+}
+
+func HaproxyConfigHttpCheckExpect(s string) string {
+	ss := []string{}
+	for _, s := range strings.Split(s, ",") {
+		s = s[len("http_"):]
+		s = strings.Replace(s, "x", ".", -1)
+		ss = append(ss, s)
+	}
+	s = strings.Join(ss, "|")
+	s = fmt.Sprintf("http-check expect rstatus %s", s)
+	return s
+}

pkg/lbagent/utils/keepalived.go

@@ -0,0 +1,31 @@
+package utils
+
+import (
+	"fmt"
+	"strings"
+	"unicode"
+)
+
+func KeepalivedConfHexQuote(s string) string {
+	q := ""
+	for _, r := range s {
+		if strings.ContainsRune(`"'\#!`, r) || !unicode.IsPrint(r) {
+			// cannot use \xNN here for keepalived bug
+			q += fmt.Sprintf(`\%03c`, r)
+		} else {
+			q += string(r)
+		}
+	}
+	q = fmt.Sprintf(`"%s"`, q)
+	return q
+}
+
+func KeepalivedConfQuoteScriptArgs(args []string) string {
+	for i, arg := range args {
+		args[i] = KeepalivedConfHexQuote(arg)
+	}
+	s := strings.Join(args, " ")
+	s = strings.Replace(s, `"`, `\"`, -1)
+	s = fmt.Sprintf(`"%s"`, s)
+	return s
+}

pkg/lbagent/utils/pidfile.go

@@ -0,0 +1,36 @@
+package utils
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strconv"
+	"strings"
+	"syscall"
+)
+
+func ReadPidFile(pidFile string) *os.Process {
+	data, err := ioutil.ReadFile(pidFile)
+	if err != nil {
+		return nil
+	}
+	s := strings.TrimSpace(string(data))
+	pid, err := strconv.Atoi(s)
+	if err != nil {
+		return nil
+	}
+	proc, err := os.FindProcess(pid)
+	if err != nil {
+		return nil
+	}
+	if err := proc.Signal(syscall.Signal(0)); err != nil {
+		return nil
+	}
+	return proc
+}
+
+func WritePidFile(pid int, pidFile string) error {
+	data := fmt.Sprintf("%d\n", pid)
+	err := ioutil.WriteFile(pidFile, []byte(data), FileModeFile)
+	return err
+}

pkg/mcclient/models/base.go

@@ -0,0 +1,75 @@
+package models
+
+import (
+	"time"
+)
+
+type IResource interface {
+	GetUpdateVersion() int
+	GetUpdatedAt() time.Time
+}
+
+type IStandaloneResource interface {
+	IResource
+	GetId() string
+}
+
+type IVirtualResource interface {
+	IStandaloneResource
+	GetPendingDeleted() bool
+}
+
+type Resource struct {
+	CreatedAt     time.Time
+	UpdatedAt     time.Time
+	UpdateVersion int
+	DeletedAt     time.Time
+	Deleted       bool
+}
+
+func (r *Resource) GetUpdateVersion() int {
+	return r.UpdateVersion
+}
+
+func (r *Resource) GetUpdatedAt() time.Time {
+	return r.UpdatedAt
+}
+
+type StandaloneResource struct {
+	Resource
+
+	Id          string
+	Name        string
+	ExternalId  string
+	Description string
+	IsEmulated  bool
+}
+
+func (r *StandaloneResource) GetId() string {
+	return r.Id
+}
+
+type StatusStandaloneResource struct {
+	StandaloneResource
+
+	Status string
+}
+
+type VirtualResource struct {
+	StatusStandaloneResource
+
+	ProjectId        string
+	IsSystem         bool
+	PendingDeletedAt time.Time
+	PendingDeleted   bool
+}
+
+func (r *VirtualResource) GetPendingDeleted() bool {
+	return r.PendingDeleted
+}
+
+type SharableVirtualResource struct {
+	VirtualResource
+
+	IsPublic bool
+}

pkg/mcclient/models/loadbalancers.go

@@ -0,0 +1,179 @@
+package models
+
+import (
+	"time"
+)
+
+type Loadbalancer struct {
+	VirtualResource
+
+	Address     string
+	AddressType string
+	NetworkType string
+	NetworkId   string
+	ZoneId      string
+
+	BackendGroupId string
+}
+
+type LoadbalancerTCPListener struct{}
+type LoadbalancerUDPListener struct{}
+
+type LoadbalancerHTTPListener struct {
+	StickySession              string
+	StickySessionType          string
+	StickySessionCookie        string
+	StickySessionCookieTimeout int
+
+	XForwardedFor      bool
+	XForwardedForSLBIP bool
+	XForwardedForSLBID bool
+	Gzip               bool
+}
+
+// CACertificate string
+type LoadbalancerHTTPSListener struct {
+	CertificateId   string
+	TLSCipherPolicy string
+	EnableHttp2     bool
+}
+
+type LoadbalancerListener struct {
+	VirtualResource
+
+	LoadbalancerId string
+	Bandwidth      int
+	ListenerType   string
+	ListenerPort   int
+
+	Scheduler string
+
+	ClientRequestTimeout  int
+	ClientIdleTimeout     int
+	BackendConnectTimeout int
+	BackendIdleTimeout    int
+
+	BackendGroupId string
+
+	AclStatus string
+	AclType   string
+	AclId     string
+
+	HealthCheck     string
+	HealthCheckType string
+
+	HealthCheckDomain   string
+	HealthCheckURI      string
+	HealthCheckHttpCode string
+
+	HealthCheckRise     int
+	HealthCheckFall     int
+	HealthCheckInterval int
+	HealthCheckTimeout  int
+
+	HealthCheckReq string
+	HealthCheckExp string
+
+	LoadbalancerTCPListener
+	LoadbalancerUDPListener
+	LoadbalancerHTTPListener
+	LoadbalancerHTTPSListener
+
+	XForwardedFor bool
+	Gzip          bool
+}
+
+type LoadbalancerListenerRule struct {
+	VirtualResource
+
+	ListenerId     string
+	BackendGroupId string
+
+	Domain string
+	Path   string
+}
+
+type LoadbalancerBackendGroup struct {
+	VirtualResource
+
+	LoadbalancerId string
+}
+
+type LoadbalancerBackend struct {
+	VirtualResource
+
+	BackendGroupId string
+	BackendId      string
+	BackendType    string
+	Weight         int
+	Address        string
+	Port           int
+}
+
+type LoadbalancerAclEntry struct {
+	Cidr    string
+	Comment string
+}
+type LoadbalancerAclEntries []*LoadbalancerAclEntry
+type LoadbalancerAcl struct {
+	SharableVirtualResource
+
+	AclEntries *LoadbalancerAclEntries
+}
+
+type LoadbalancerCertificate struct {
+	VirtualResource
+
+	Certificate string
+	PrivateKey  string
+
+	PublicKeyAlgorithm      string
+	PublicKeyBitLen         int
+	SignatureAlgorithm      string
+	FingerprintSha256       string
+	NotBefore               time.Time
+	NotAfter                time.Time
+	CommonName              string
+	SubjectAlternativeNames string
+}
+
+type LoadbalancerAgent struct {
+	StandaloneResource
+
+	HbLastSeen time.Time
+	HbTimeout  int
+
+	Loadbalancers             time.Time
+	LoadbalancerListeners     time.Time
+	LoadbalancerListenerRules time.Time
+	LoadbalancerBackendGroups time.Time
+	LoadbalancerBackends      time.Time
+	LoadbalancerAcls          time.Time
+	LoadbalancerCertificates  time.Time
+	Params                    LoadbalancerAgentParams
+}
+
+type LoadbalancerAgentParamsVrrp struct {
+	Priority          int
+	VirtualRouterId   int
+	GarpMasterRefresh int
+	Preempt           bool
+	Interface         string
+	AdvertInt         int
+	Pass              string
+}
+
+type LoadbalancerAgentParamsHaproxy struct {
+	GlobalLog      string
+	GlobalNbthread int
+	LogHttp        bool
+	LogTcp         bool
+	LogNormal      bool
+}
+
+type LoadbalancerAgentParams struct {
+	KeepalivedConfTmpl string
+	HaproxyConfTmpl    string
+	Vrrp               LoadbalancerAgentParamsVrrp
+	Haproxy            LoadbalancerAgentParamsHaproxy
+}

pkg/mcclient/modules/mod_loadbalanceracls.go

@@ -0,0 +1,25 @@
+package modules
+
+type LoadbalancerAclManager struct {
+	ResourceManager
+}
+
+var (
+	LoadbalancerAcls LoadbalancerAclManager
+)
+
+func init() {
+	LoadbalancerAcls = LoadbalancerAclManager{
+		NewComputeManager(
+			"loadbalanceracl",
+			"loadbalanceracls",
+			[]string{
+				"id",
+				"name",
+				"acl_entries",
+			},
+			[]string{"tenant"},
+		),
+	}
+	registerCompute(&LoadbalancerAcls)
+}

pkg/mcclient/modules/mod_loadbalanceragents.go

@@ -0,0 +1,35 @@
+package modules
+
+type LoadbalancerAgentManager struct {
+	ResourceManager
+}
+
+var (
+	LoadbalancerAgents LoadbalancerAgentManager
+)
+
+func init() {
+	LoadbalancerAgents = LoadbalancerAgentManager{
+		NewComputeManager(
+			"loadbalanceragent",
+			"loadbalanceragents",
+			[]string{
+				"id",
+				"name",
+
+				"hb_last_seen",
+				"hb_timeout",
+
+				"loadbalancers",
+				"loadbalancer_listeners",
+				"loadbalancer_listener_rules",
+				"loadbalancer_backend_groups",
+				"loadbalancer_backends",
+				"loadbalancer_acls",
+				"loadbalancer_certificates",
+			},
+			[]string{},
+		),
+	}
+	registerCompute(&LoadbalancerAgents)
+}

pkg/mcclient/modules/mod_loadbalancerbackendgroups.go

@@ -0,0 +1,25 @@
+package modules
+
+type LoadbalancerBackendGroupManager struct {
+	ResourceManager
+}
+
+var (
+	LoadbalancerBackendGroups LoadbalancerBackendGroupManager
+)
+
+func init() {
+	LoadbalancerBackendGroups = LoadbalancerBackendGroupManager{
+		NewComputeManager(
+			"loadbalancerbackendgroup",
+			"loadbalancerbackendgroups",
+			[]string{
+				"id",
+				"name",
+				"loadbalancer_id",
+			},
+			[]string{"tenant"},
+		),
+	}
+	registerCompute(&LoadbalancerBackendGroups)
+}

pkg/mcclient/modules/mod_loadbalancerbackends.go

@@ -0,0 +1,30 @@
+package modules
+
+type LoadbalancerBackendManager struct {
+	ResourceManager
+}
+
+var (
+	LoadbalancerBackends LoadbalancerBackendManager
+)
+
+func init() {
+	LoadbalancerBackends = LoadbalancerBackendManager{
+		NewComputeManager(
+			"loadbalancerbackend",
+			"loadbalancerbackends",
+			[]string{
+				"id",
+				"name",
+				"backend_group_id",
+				"backend_id",
+				"backend_type",
+				"address",
+				"port",
+				"weight",
+			},
+			[]string{"tenant"},
+		),
+	}
+	registerCompute(&LoadbalancerBackends)
+}

pkg/mcclient/modules/mod_loadbalancercertificates.go

@@ -0,0 +1,30 @@
+package modules
+
+type LoadbalancerCertificateManager struct {
+	ResourceManager
+}
+
+var (
+	LoadbalancerCertificates LoadbalancerCertificateManager
+)
+
+func init() {
+	LoadbalancerCertificates = LoadbalancerCertificateManager{
+		NewComputeManager(
+			"loadbalancercertificate",
+			"loadbalancercertificates",
+			[]string{
+				"id",
+				"name",
+				"algorithm",
+				"fingerprint",
+				"not_before",
+				"not_after",
+				"common_name",
+				"subject_alternative_Names",
+			},
+			[]string{"tenant"},
+		),
+	}
+	registerCompute(&LoadbalancerCertificates)
+}

pkg/mcclient/modules/mod_loadbalancerlistenerrules.go

@@ -0,0 +1,29 @@
+package modules
+
+type LoadbalancerListenerRuleManager struct {
+	ResourceManager
+}
+
+var (
+	LoadbalancerListenerRules LoadbalancerListenerRuleManager
+)
+
+func init() {
+	LoadbalancerListenerRules = LoadbalancerListenerRuleManager{
+		NewComputeManager(
+			"loadbalancerlistenerrule",
+			"loadbalancerlistenerrules",
+			[]string{
+				"id",
+				"name",
+				"listener_id",
+				"status",
+				"domain",
+				"path",
+				"backend_id",
+			},
+			[]string{"tenant"},
+		),
+	}
+	registerCompute(&LoadbalancerListenerRules)
+}

pkg/mcclient/modules/mod_loadbalancerlisteners.go

@@ -0,0 +1,31 @@
+package modules
+
+type LoadbalancerListenerManager struct {
+	ResourceManager
+}
+
+var (
+	LoadbalancerListeners LoadbalancerListenerManager
+)
+
+func init() {
+	LoadbalancerListeners = LoadbalancerListenerManager{
+		NewComputeManager(
+			"loadbalancerlistener",
+			"loadbalancerlisteners",
+			[]string{
+				"id",
+				"name",
+				"loadbalancer_id",
+				"status",
+				"listener_type",
+				"listener_port",
+				"backend_port",
+				"acl_status",
+				"acl_type",
+			},
+			[]string{"tenant"},
+		),
+	}
+	registerCompute(&LoadbalancerListeners)
+}

pkg/mcclient/modules/mod_loadbalancers.go

@@ -0,0 +1,31 @@
+package modules
+
+type LoadbalancerManager struct {
+	ResourceManager
+}
+
+var (
+	Loadbalancers LoadbalancerManager
+)
+
+func init() {
+	Loadbalancers = LoadbalancerManager{
+		NewComputeManager(
+			"loadbalancer",
+			"loadbalancers",
+			[]string{
+				"id",
+				"name",
+				"status",
+				"address_type",
+				"address",
+				"network_type",
+				"network_id",
+				"zone_id",
+			},
+			[]string{"tenant"},
+		),
+	}
+	registerCompute(&Loadbalancers)
+
+}

pkg/mcclient/options/base.go

@@ -147,6 +147,11 @@ func ListStructToParams(v interface{}) (*jsonutils.JSONDict, error) {
 	return params, nil
 }
 
+const (
+	ListOrderAsc  = "asc"
+	ListOrderDesc = "desc"
+)
+
 type BaseListOptions struct {
 	Limit            *int     `default:"20" help:"Page limit"`
 	Offset           *int     `default:"0" help:"Page offset"`

pkg/mcclient/options/loadbalanceracls.go

@@ -0,0 +1,123 @@
+package options
+
+import (
+	"fmt"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+)
+
+type AclEntry struct {
+	Cidr    string
+	Comment string
+}
+
+type AclEntries []*AclEntry
+
+func NewAclEntry(s string) *AclEntry {
+	tu := strings.SplitN(s, "#", 2)
+	cidr := strings.TrimSpace(tu[0])
+	comment := ""
+	if len(tu) > 1 {
+		comment = tu[1]
+	}
+	aclEntry := &AclEntry{
+		Cidr:    cidr,
+		Comment: comment,
+	}
+	return aclEntry
+}
+
+func NewAclEntries(ss []string) AclEntries {
+	aclEntries := AclEntries{}
+	for _, s := range ss {
+		aclEntry := NewAclEntry(s)
+		aclEntries = append(aclEntries, aclEntry)
+	}
+	return aclEntries
+}
+
+func (entry *AclEntry) String() string {
+	cidr := entry.Cidr
+	comment := entry.Comment
+	if comment != "" {
+		comment = " # " + comment
+	}
+	return fmt.Sprintf("%-15s%s", cidr, comment)
+}
+
+func (entries AclEntries) String() string {
+	ss := []string{}
+	for _, entry := range entries {
+		ss = append(ss, entry.String())
+	}
+	lines := strings.Join(ss, "\n")
+	return lines
+}
+
+type LoadbalancerAclCreateOptions struct {
+	NAME     string
+	AclEntry []string `help:"acl entry with cidr and comment separated by #, e.g. 10.9.0.0/16#no comment" json:"-"`
+}
+
+type LoadbalancerAclGetOptions struct {
+	ID string
+}
+
+type LoadbalancerAclListOptions struct {
+	BaseListOptions
+}
+
+type LoadbalancerAclUpdateOptions struct {
+	ID       string   `json:"-"`
+	AclEntry []string `help:"acl entry with cidr and comment separated by #, e.g. 10.9.0.0/16#no comment" json:"-"`
+}
+
+type LoadbalancerAclDeleteOptions struct {
+	ID string `json:"-"`
+}
+
+type LoadbalancerAclActionPatchOptions struct {
+	ID  string   `json:"-"`
+	Add []string `help:"acl entry with cidr and comment separated by #, e.g. 10.9.0.0/16#no comment" json:"-"`
+	Del []string `help:"acl entry with cidr and comment separated by #, e.g. 10.9.0.0/16#no comment" json:"-"`
+}
+
+func (opts *LoadbalancerAclCreateOptions) Params() (*jsonutils.JSONDict, error) {
+	params, err := optionsStructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	aclEntries := NewAclEntries(opts.AclEntry)
+	aclEntriesJson := jsonutils.Marshal(aclEntries)
+	params.Set("acl_entries", aclEntriesJson)
+	return params, nil
+}
+
+func (opts *LoadbalancerAclUpdateOptions) Params() (*jsonutils.JSONDict, error) {
+	params, err := optionsStructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	aclEntries := NewAclEntries(opts.AclEntry)
+	aclEntriesJson := jsonutils.Marshal(aclEntries)
+	params.Set("acl_entries", aclEntriesJson)
+	return params, nil
+}
+
+func (opts *LoadbalancerAclActionPatchOptions) Params() (*jsonutils.JSONDict, error) {
+	params, err := optionsStructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	m := map[string][]string{
+		"adds": opts.Add,
+		"dels": opts.Del,
+	}
+	for k, ss := range m {
+		aclEntries := NewAclEntries(ss)
+		aclEntriesJson := jsonutils.Marshal(aclEntries)
+		params.Set(k, aclEntriesJson)
+	}
+	return params, nil
+}

pkg/mcclient/options/loadbalanceragents.go

@@ -0,0 +1,113 @@
+package options
+
+import (
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+)
+
+type LoadbalancerAgentParamsOptions struct {
+	KeepalivedConfTmpl string
+	HaproxyConfTmpl    string
+
+	VrrpPriority          *int // required
+	VrrpVirtualRouterId   *int // required
+	VrrpGarpMasterRefresh *int
+	VrrpPreempt           string `choices:true|false`
+	VrrpInterface         string // required
+	VrrpAdvertInt         *int
+	VrrpPass              string
+
+	HaproxyGlobalLog      string
+	HaproxyGlobalNbthread *int   `default:1 help:"enable experimental multi-threading support available since haproxy 1.8"`
+	HaproxyLogHttp        string `choices:true|false`
+	HaproxyLogTcp         string `choices:true|false`
+	HaproxyLogNormal      string `choices:true|false`
+}
+
+func (opts *LoadbalancerAgentParamsOptions) setPrefixedParams(params *jsonutils.JSONDict, pref string) {
+	pref_ := pref + "_"
+	pref_len := len(pref_)
+	pp := jsonutils.NewDict()
+	pMap, _ := params.GetMap()
+	for k, v := range pMap {
+		if strings.HasPrefix(k, pref_) && !strings.HasSuffix(k, "_conf_tmpl") {
+			params.Remove(k)
+			pp.Set(k[pref_len:], v)
+		}
+	}
+	if pp.Length() > 0 {
+		params.Set(pref, pp)
+	}
+}
+
+func (opts *LoadbalancerAgentParamsOptions) Params() (*jsonutils.JSONDict, error) {
+	params, err := optionsStructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	opts.setPrefixedParams(params, "vrrp")
+	opts.setPrefixedParams(params, "haproxy")
+	return params, nil
+}
+
+type LoadbalancerAgentCreateOptions struct {
+	NAME      string
+	HbTimeout *int
+
+	LoadbalancerAgentParamsOptions
+}
+
+func (opts *LoadbalancerAgentCreateOptions) Params() (*jsonutils.JSONDict, error) {
+	params, err := StructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	paramsSub, err := opts.LoadbalancerAgentParamsOptions.Params()
+	if err != nil {
+		return nil, err
+	}
+	params.Set("params", paramsSub)
+	return params, nil
+}
+
+type LoadbalancerAgentListOptions struct {
+	BaseListOptions
+}
+
+type LoadbalancerAgentGetOptions struct {
+	ID string
+}
+
+type LoadbalancerAgentUpdateOptions struct {
+	ID string
+
+	HbTimeout *int
+
+	Loadbalancers             *time.Time
+	LoadbalancerListeners     *time.Time
+	LoadbalancerListenerRules *time.Time
+	LoadbalancerBackendGroups *time.Time
+	LoadbalancerBackends      *time.Time
+	LoadbalancerAcls          *time.Time
+	LoadbalancerCertificates  *time.Time
+}
+
+type LoadbalancerAgentDeleteOptions struct {
+	ID string
+}
+
+type LoadbalancerAgentActionHbOptions struct {
+	ID string
+}
+
+type LoadbalancerAgentActionPatchParamsOptions struct {
+	ID string
+
+	LoadbalancerAgentParamsOptions
+}
+
+func (opts *LoadbalancerAgentActionPatchParamsOptions) Params() (*jsonutils.JSONDict, error) {
+	return opts.LoadbalancerAgentParamsOptions.Params()
+}

pkg/mcclient/options/loadbalancerbackendgroups.go

@@ -0,0 +1,16 @@
+package options
+
+type LoadbalancerBackendGroupCreateOptions struct {
+	NAME         string
+	Loadbalancer string
+}
+type LoadbalancerBackendGroupGetOptions struct {
+	ID string
+}
+type LoadbalancerBackendGroupDeleteOptions struct {
+	ID string
+}
+type LoadbalancerBackendGroupListOptions struct {
+	BaseListOptions
+	Loadbalancer string
+}

pkg/mcclient/options/loadbalancerbackends.go

@@ -0,0 +1,34 @@
+package options
+
+type LoadbalancerBackendCreateOptions struct {
+	BackendGroup string `required:true`
+	Backend      string `required:true`
+	BackendType  string `default:guest`
+	Port         *int   `required:true`
+	Weight       *int   `default:1`
+}
+
+type LoadbalancerBackendListOptions struct {
+	BaseListOptions
+	BackendGroup string
+	Backend      string
+	BackendType  string
+	Weight       *int
+	Address      string
+	Port         *int
+}
+
+type LoadbalancerBackendUpdateOptions struct {
+	ID string
+
+	Weight *int
+	Port   *int
+}
+
+type LoadbalancerBackendGetOptions struct {
+	ID string
+}
+
+type LoadbalancerBackendDeleteOptions struct {
+	ID string
+}

pkg/mcclient/options/loadbalancercertificates.go

@@ -0,0 +1,85 @@
+package options
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	"yunion.io/x/jsonutils"
+)
+
+func loadbalancerCertificateLoadFiles(cert, pkey string, allowEmpty bool) (*jsonutils.JSONDict, error) {
+	params := jsonutils.NewDict()
+	pathM := map[string]string{
+		"certificate": cert,
+		"private_key": pkey,
+	}
+	for fieldName, path := range pathM {
+		if path == "" {
+			if allowEmpty {
+				continue
+			} else {
+				return nil, fmt.Errorf("%s: empty path", fieldName)
+			}
+		}
+		d, err := ioutil.ReadFile(path)
+		if err != nil {
+			return nil, fmt.Errorf("%s: read %s: %s", fieldName, path, err)
+		}
+		if len(d) == 0 {
+			return nil, fmt.Errorf("%s: empty file %s", fieldName, path)
+		}
+		params.Set(fieldName, jsonutils.NewString(string(d)))
+	}
+	return params, nil
+}
+
+type LoadbalancerCertificateCreateOptions struct {
+	NAME string
+
+	Cert string `required:true json:- help:"path to certificate file"`
+	Pkey string `required:true json:- help:"path to private key file"`
+}
+
+func (opts *LoadbalancerCertificateCreateOptions) Params() (*jsonutils.JSONDict, error) {
+	params, err := StructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	paramsCertKey, err := loadbalancerCertificateLoadFiles(opts.Cert, opts.Pkey, false)
+	if err != nil {
+		return nil, err
+	}
+	params.Update(paramsCertKey)
+	return params, nil
+}
+
+type LoadbalancerCertificateGetOptions struct {
+	ID string
+}
+
+type LoadbalancerCertificateDeleteOptions struct {
+	ID string
+}
+
+type LoadbalancerCertificateListOptions struct {
+	BaseListOptions
+
+	PublicKeyAlgorithm string
+	PublicKeyBitLen    *int
+	SignatureAlgorithm string
+}
+
+type LoadbalancerCertificateUpdateOptions struct {
+	ID string
+
+	Cert string `json:- help:"path to certificate file"`
+	Pkey string `json:- help:"path to private key file"`
+}
+
+func (opts *LoadbalancerCertificateUpdateOptions) Params() (*jsonutils.JSONDict, error) {
+	paramsCertKey, err := loadbalancerCertificateLoadFiles(opts.Cert, opts.Pkey, true)
+	if err != nil {
+		return nil, err
+	}
+	return paramsCertKey, nil
+}

pkg/mcclient/options/loadbalancerlistenerrules.go

@@ -0,0 +1,34 @@
+package options
+
+type LoadbalancerListenerRuleCreateOptions struct {
+	NAME         string
+	Listener     string `required:true`
+	BackendGroup string
+	Domain       string
+	Path         string
+}
+
+type LoadbalancerListenerRuleListOptions struct {
+	BaseListOptions
+	Listener string
+	Domain   string
+	Path     string
+}
+
+type LoadbalancerListenerRuleUpdateOptions struct {
+	ID           string
+	BackendGroup string
+}
+
+type LoadbalancerListenerRuleGetOptions struct {
+	ID string
+}
+
+type LoadbalancerListenerRuleDeleteOptions struct {
+	ID string
+}
+
+type LoadbalancerListenerRuleActionStatusOptions struct {
+	ID     string
+	Status string `choices:"enabled|disabled"`
+}

pkg/mcclient/options/loadbalancerlisteners.go

@@ -0,0 +1,155 @@
+package options
+
+type LoadbalancerListenerCreateOptions struct {
+	NAME string
+
+	Loadbalancer string `required:true`
+	ListenerType string `required:true choices:tcp|udp|http|https`
+	ListenerPort *int   `required:true`
+	BackendGroup string
+
+	Scheduler string `required:true choices:"rr|wrr|wlc|sch|tch"`
+	Bandwidth *int
+
+	ClientRequestTimeout  *int
+	ClientIdleTimeout     *int
+	BackendConnectTimeout *int
+	BackendIdleTimeout    *int
+
+	AclStatus string `choices:"on|off"`
+	AclType   string `choices:"black|white"`
+	Acl       string
+
+	HealthCheck     string `choices:"on|off"`
+	HealthCheckType string `choices:"tcp|http"`
+
+	HealthCheckDomain   string
+	HealthCheckURI      string
+	HealthCheckHttpCode string
+
+	HealthCheckRise     *int
+	HealthCheckFall     *int
+	HealthCheckInterval *int
+	HealthCheckTimeout  *int
+
+	HealthCheckReq string
+	HealthCheckExp string
+
+	StickySession              string
+	StickySessionType          string
+	StickySessionCookie        string
+	StickySessionCookieTimeout *int
+
+	XForwardedFor string `choices:true|false`
+	Gzip          string `choices:true|false`
+
+	Certificate     string
+	TLSCipherPolicy string
+	EnableHttp2     string `choices:true|false`
+}
+
+type LoadbalancerListenerListOptions struct {
+	BaseListOptions
+
+	Loadbalancer string
+	ListenerType string `choices:tcp|udp|http|https`
+	ListenerPort *int
+	BackendGroup string
+
+	Scheduler string `choices:"rr|wrr|wlc|sch|tch"`
+	Bandwidth *int
+
+	ClientRequestTimeout  *int
+	ClientIdleTimeout     *int
+	BackendConnectTimeout *int
+	BackendIdleTimeout    *int
+
+	AclStatus string `choices:"on|off"`
+	AclType   string `choices:"black|white"`
+	Acl       string
+
+	HealthCheck     string `choices:"on|off"`
+	HealthCheckType string `choices:"tcp|http"`
+
+	HealthCheckDomain   string
+	HealthCheckURI      string
+	HealthCheckHttpCode string
+
+	HealthCheckRise     *int
+	HealthCheckFall     *int
+	HealthCheckInterval *int
+	HealthCheckTimeout  *int
+
+	HealthCheckReq string
+	HealthCheckExp string
+
+	StickySession              string
+	StickySessionType          string
+	StickySessionCookie        string
+	StickySessionCookieTimeout *int
+
+	XForwardedFor string `choices:true|false`
+	Gzip          string `choices:true|false`
+
+	Certificate     string
+	TLSCipherPolicy string
+	EnableHttp2     string `choices:true|false`
+}
+
+type LoadbalancerListenerUpdateOptions struct {
+	ID string
+
+	BackendGroup string
+
+	Scheduler string `choices:"rr|wrr|wlc|sch|tch"`
+	Bandwidth *int
+
+	ClientRequestTimeout  *int
+	ClientIdleTimeout     *int
+	BackendConnectTimeout *int
+	BackendIdleTimeout    *int
+
+	AclStatus string `choices:"on|off"`
+	AclType   string `choices:"black|white"`
+	Acl       string
+
+	HealthCheck     string `choices:"on|off"`
+	HealthCheckType string `choices:"tcp|http"`
+
+	HealthCheckDomain   string
+	HealthCheckURI      string
+	HealthCheckHttpCode string
+
+	HealthCheckRise     *int
+	HealthCheckFall     *int
+	HealthCheckInterval *int
+	HealthCheckTimeout  *int
+
+	HealthCheckReq string
+	HealthCheckExp string
+
+	StickySession              string
+	StickySessionType          string
+	StickySessionCookie        string
+	StickySessionCookieTimeout *int
+
+	XForwardedFor string `choices:true|false`
+	Gzip          string `choices:true|false`
+
+	Certificate     string
+	TLSCipherPolicy string
+	EnableHttp2     string `choices:true|false`
+}
+
+type LoadbalancerListenerGetOptions struct {
+	ID string
+}
+
+type LoadbalancerListenerDeleteOptions struct {
+	ID string
+}
+
+type LoadbalancerListenerActionStatusOptions struct {
+	ID     string
+	Status string `choices:"enabled|disabled"`
+}

pkg/mcclient/options/loadbalancers.go

@@ -0,0 +1,35 @@
+package options
+
+type LoadbalancerCreateOptions struct {
+	NAME    string
+	Network string
+	Address string
+}
+
+type LoadbalancerGetOptions struct {
+	ID string
+}
+
+type LoadbalancerUpdateOptions struct {
+	ID           string
+	BackendGroup string
+}
+
+type LoadbalancerDeleteOptions struct {
+	ID string
+}
+
+type LoadbalancerListOptions struct {
+	BaseListOptions
+	Zone         string
+	Address      string
+	AddressType  string `choices:"intranet|internet"`
+	NetworkType  string `choices:"classic|vpc"`
+	Network      string
+	BackendGroup string
+}
+
+type LoadbalancerActionStatusOptions struct {
+	ID     string
+	Status string `choices:"enabled|disabled"`
+}