From ca5f4f6dc802385da5a307de150e77f29b8c00e7 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 21 Sep 2018 02:28:40 +0000 Subject: [PATCH 1/4] build.sh: support specifying runtime requires --- build/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build/build.sh b/build/build.sh index 201197583d..0cc7a2c028 100755 --- a/build/build.sh +++ b/build/build.sh @@ -76,6 +76,7 @@ Summary: %{pkgname} Group: Unspecified License: GPL URL: https://www.yunion.io/doc +$(for req in "${REQUIRES[@]}"; do echo Requires: "$req"; done) %if %{use_systemd} Requires: systemd BuildRequires: systemd From a8a19c4de9bd0a25c8303c4c5461224f3a89f3d6 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 19 Sep 2018 15:12:44 +0000 Subject: [PATCH 2/4] validators: validate certs and keys --- pkg/cloudcommon/validators/validators_cert.go | 360 ++++++++++++++++++ .../validators/validators_cert_test.go | 1 + 2 files changed, 361 insertions(+) create mode 100644 pkg/cloudcommon/validators/validators_cert.go create mode 100644 pkg/cloudcommon/validators/validators_cert_test.go diff --git a/pkg/cloudcommon/validators/validators_cert.go b/pkg/cloudcommon/validators/validators_cert.go new file mode 100644 index 0000000000..e61dd051b6 --- /dev/null +++ b/pkg/cloudcommon/validators/validators_cert.go @@ -0,0 +1,360 @@ +package validators + +import ( + "crypto" + "crypto/ecdsa" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/hex" + "encoding/pem" + "fmt" + "reflect" + + "yunion.io/x/jsonutils" +) + +type ValidatorPEM struct { + Validator + Blocks []*pem.Block +} + +func NewPEMValidator(key string) *ValidatorPEM { + v := &ValidatorPEM{ + Validator: Validator{Key: key}, + } + v.parent = v + return v +} + +func (v *ValidatorPEM) getValue() interface{} { + return v.Blocks +} + +func (v *ValidatorPEM) parseFromString(s string) []*pem.Block { + blocks := []*pem.Block{} + for d := []byte(s); ; { + block, rest := pem.Decode(d) + if block == nil { + if len(rest) > 0 { + return nil + } + break + } + blocks = append(blocks, block) + d = rest + } + return blocks +} + +func (v *ValidatorPEM) setDefault(data *jsonutils.JSONDict) bool { + if v.defaultVal == nil { + return false + } + s, ok := v.defaultVal.(string) + if !ok { + return false + } + blocks := v.parseFromString(s) + if blocks != nil { + value := jsonutils.NewString(s) + v.value = value + data.Set(v.Key, value) + v.Blocks = blocks + return true + } + return false +} + +func (v *ValidatorPEM) Validate(data *jsonutils.JSONDict) error { + if err, isSet := v.Validator.validateEx(data); err != nil || !isSet { + return err + } + s, err := v.value.GetString() + if err != nil { + return newInvalidTypeError(v.Key, "pem", err) + } + v.Blocks = v.parseFromString(s) + return nil +} + +type ValidatorCertificate struct { + ValidatorPEM + Certificates []*x509.Certificate +} + +func NewCertificateValidator(key string) *ValidatorCertificate { + v := &ValidatorCertificate{ + ValidatorPEM: *NewPEMValidator(key), + } + v.parent = v + return v +} + +func (v *ValidatorCertificate) getValue() interface{} { + return v.Certificates +} + +func (v *ValidatorCertificate) parseFromBlocks(blocks []*pem.Block) []*x509.Certificate { + certs := []*x509.Certificate{} + for i, block := range blocks { + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil + } + if i > 0 { + certSub := certs[i-1] + if err := certSub.CheckSignatureFrom(cert); err != nil { + return nil + } + } + certs = append(certs, cert) + } + return certs +} + +func (v *ValidatorCertificate) parseFromString(s string) []*x509.Certificate { + blocks := v.ValidatorPEM.parseFromString(s) + if blocks == nil { + return nil + } + v.Blocks = blocks + certs := v.parseFromBlocks(blocks) + return certs +} + +func (v *ValidatorCertificate) setDefault(data *jsonutils.JSONDict) bool { + if v.defaultVal == nil { + return false + } + s, ok := v.defaultVal.(string) + if !ok { + return false + } + certs := v.parseFromString(s) + if certs != nil { + v.setCertificates(certs, data) + return true + } + return false +} + +func (v *ValidatorCertificate) setCertificates(certs []*x509.Certificate, data *jsonutils.JSONDict) { + pems := []byte{} + for _, block := range v.Blocks { + d := pem.EncodeToMemory(block) + pems = append(pems, d...) + } + value := jsonutils.NewString(string(pems)) + v.value = value + data.Set(v.Key, value) + v.Certificates = certs +} + +func (v *ValidatorCertificate) Validate(data *jsonutils.JSONDict) error { + if err, isSet := v.Validator.validateEx(data); err != nil || !isSet { + return err + } + s, err := v.value.GetString() + if err != nil { + return newInvalidTypeError(v.Key, "certificate", err) + } + certs := v.parseFromString(s) + if !v.optional && len(certs) == 0 { + return newInvalidValueError(v.Key, "empty certificate chain") + } + if len(certs) > 0 { + for _, block := range v.Blocks { + if block.Type != "CERTIFICATE" { + err := fmt.Errorf("wrong PEM type: %s", block.Type) + return newInvalidTypeError(v.Key, "certificate", err) + } + } + for i := 0; i < len(certs)-1; i++ { + cert := certs[i] + certP := certs[i+1] + err := cert.CheckSignatureFrom(certP) + if err != nil { + msg := fmt.Sprintf("cannot verify signature of certificate %d in the chain", i+1) + return newInvalidValueError(v.Key, msg) + } + } + v.setCertificates(certs, data) + return nil + } + return nil +} + +func (v *ValidatorCertificate) FingerprintSha256() []byte { + csum := sha256.Sum256(v.Certificates[0].Raw) + return csum[:] +} + +func (v *ValidatorCertificate) FingerprintSha256String() string { + fp := v.FingerprintSha256() + s := hex.EncodeToString(fp) + return s +} + +func (v *ValidatorCertificate) PublicKeyBitLen() int { + cert := v.Certificates[0] + pubkey := cert.PublicKey + switch pub := pubkey.(type) { + case *rsa.PublicKey: + return pub.N.BitLen() + case *ecdsa.PublicKey: + return pub.X.BitLen() + default: + return 0 + } +} + +type ValidatorPrivateKey struct { + ValidatorPEM + PrivateKey crypto.PrivateKey +} + +func NewPrivateKeyValidator(key string) *ValidatorPrivateKey { + v := &ValidatorPrivateKey{ + ValidatorPEM: *NewPEMValidator(key), + } + v.parent = v + return v +} + +func (v *ValidatorPrivateKey) getValue() interface{} { + return v.PrivateKey +} + +func (v *ValidatorPrivateKey) parseFromBlock(block *pem.Block) (crypto.PrivateKey, error) { + const RSA_PRIVATE_KEY = "RSA PRIVATE KEY" + const EC_PRIVATE_KEY = "EC PRIVATE KEY" + const PKCS8_PRIVATE_KEY = "PRIVATE KEY" + + var fuzz bool + switch block.Type { + case PKCS8_PRIVATE_KEY: + goto parsePkcs8 + case RSA_PRIVATE_KEY: + goto parseRsa + case EC_PRIVATE_KEY: + goto parseEc + default: + } + fuzz = true +parsePkcs8: + { + pkey, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err == nil { + switch pkey1 := pkey.(type) { + case *rsa.PrivateKey: + block.Type = RSA_PRIVATE_KEY + block.Bytes = x509.MarshalPKCS1PrivateKey(pkey1) + return pkey1, nil + case *ecdsa.PrivateKey: + block.Type = EC_PRIVATE_KEY + block.Bytes, _ = x509.MarshalECPrivateKey(pkey1) + return pkey1, nil + default: + return nil, newInvalidValueError(v.Key, "unknown private key type") + } + } + if !fuzz { + return nil, newInvalidValueError(v.Key, fmt.Sprintf("invalid pkcs8 private key: %s", err)) + } + } +parseRsa: + { + pkey, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err == nil { + block.Type = RSA_PRIVATE_KEY + return pkey, nil + } + if !fuzz { + return nil, newInvalidValueError(v.Key, fmt.Sprintf("invalid rsa private key: %s", err)) + } + } +parseEc: + { + pkey, err := x509.ParseECPrivateKey(block.Bytes) + if err == nil { + block.Type = EC_PRIVATE_KEY + return pkey, nil + } + if !fuzz { + return nil, newInvalidValueError(v.Key, fmt.Sprintf("invalid ec private key: %s", err)) + } + } + return nil, newInvalidValueError(v.Key, fmt.Sprintf("invalid private key")) +} + +func (v *ValidatorPrivateKey) parseFromString(s string) (crypto.PrivateKey, error) { + blocks := v.ValidatorPEM.parseFromString(s) + if len(blocks) != 1 { + return nil, newInvalidValueError(v.Key, fmt.Sprintf("found %d pem blocks, expecting 1", len(blocks))) + } + v.Blocks = blocks + pkey, err := v.parseFromBlock(blocks[0]) + return pkey, err +} + +func (v *ValidatorPrivateKey) setDefault(data *jsonutils.JSONDict) bool { + if v.defaultVal == nil { + return false + } + s, ok := v.defaultVal.(string) + if !ok { + return false + } + pkey, err := v.parseFromString(s) + if err != nil { + return false + } + v.setPrivateKey(pkey, data) + return true +} + +func (v *ValidatorPrivateKey) setPrivateKey(pkey crypto.PrivateKey, data *jsonutils.JSONDict) { + d := pem.EncodeToMemory(v.Blocks[0]) + value := jsonutils.NewString(string(d)) + v.value = value + data.Set(v.Key, value) + v.PrivateKey = pkey +} + +func (v *ValidatorPrivateKey) Validate(data *jsonutils.JSONDict) error { + if err, isSet := v.Validator.validateEx(data); err != nil || !isSet { + return err + } + s, err := v.value.GetString() + if err != nil { + return newInvalidTypeError(v.Key, "privateKey", err) + } + pkey, err := v.parseFromString(s) + if err != nil { + return err + } + v.setPrivateKey(pkey, data) + return nil +} + +func (v *ValidatorPrivateKey) MatchCertificate(cert *x509.Certificate) error { + switch pkey := v.PrivateKey.(type) { + case *rsa.PrivateKey: + pubkey0 := pkey.Public() + pubkey1, _ := cert.PublicKey.(crypto.PublicKey) // safe conversion + if !reflect.DeepEqual(pubkey0, pubkey1) { + return newInvalidValueError(v.Key, "certificate and rsa key do not match") + } + case *ecdsa.PrivateKey: + pubkey0 := pkey.Public() + pubkey1, _ := cert.PublicKey.(crypto.PublicKey) + if !reflect.DeepEqual(pubkey0, pubkey1) { + return newInvalidValueError(v.Key, "certificate and ec key do not match") + } + default: + // should never happen + return newInvalidValueError(v.Key, "unknown private key type") + } + return nil +} diff --git a/pkg/cloudcommon/validators/validators_cert_test.go b/pkg/cloudcommon/validators/validators_cert_test.go new file mode 100644 index 0000000000..3e4d37f502 --- /dev/null +++ b/pkg/cloudcommon/validators/validators_cert_test.go @@ -0,0 +1 @@ +package validators From aef6996fb09d36d21c00c747be8e9424081b9957 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 8 Aug 2018 15:02:58 +0000 Subject: [PATCH 3/4] loadbalancer: add loadbalancernetworks --- pkg/compute/handlers.go | 1 + pkg/compute/models/loadbalancernetworks.go | 126 +++++++++++++++++++++ pkg/compute/models/networks.go | 13 ++- 3 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 pkg/compute/models/loadbalancernetworks.go diff --git a/pkg/compute/handlers.go b/pkg/compute/handlers.go index 311109892f..a30844dd20 100644 --- a/pkg/compute/handlers.go +++ b/pkg/compute/handlers.go @@ -74,6 +74,7 @@ func InitHandlers(app *appsrv.Application) { models.HoststorageManager, models.HostschedtagManager, models.GuestnetworkManager, + models.LoadbalancernetworkManager, models.GuestdiskManager, models.GroupnetworkManager, models.GroupguestManager, diff --git a/pkg/compute/models/loadbalancernetworks.go b/pkg/compute/models/loadbalancernetworks.go new file mode 100644 index 0000000000..29a5c279d0 --- /dev/null +++ b/pkg/compute/models/loadbalancernetworks.go @@ -0,0 +1,126 @@ +package models + +import ( + "context" + "fmt" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/pkg/util/regutils" +) + +type SLoadbalancernetworkManager struct { + db.SVirtualJointResourceBaseManager +} + +var LoadbalancernetworkManager *SLoadbalancernetworkManager + +func init() { + db.InitManager(func() { + LoadbalancernetworkManager = &SLoadbalancernetworkManager{ + SVirtualJointResourceBaseManager: db.NewVirtualJointResourceBaseManager( + SLoadbalancerNetwork{}, + "loadbalancernetworks_tbl", + "loadbalancernetwork", + "loadbalancernetworks", + LoadbalancerManager, + NetworkManager, + ), + } + }) +} + +type SLoadbalancerNetwork struct { + db.SVirtualJointResourceBase + + LoadbalancerId string `width:"36" charset:"ascii" nullable:"false" key_index:"true" list:"admin"` + NetworkId string `width:"36" charset:"ascii" nullable:"false" key_index:"true" list:"admin"` + IpAddr string `width:"16" charset:"ascii" list:"admin"` +} + +func (ln *SLoadbalancerNetwork) Network() *SNetwork { + network, _ := ln.GetModelManager().FetchById(ln.NetworkId) + if network != nil { + return network.(*SNetwork) + } + return nil +} + +type SLoadbalancerNetworkRequestData struct { + loadbalancer *SLoadbalancer + networkId string + reserved bool // allocate from reserved + address string // the address user intends to use + strategy IPAddlocationDirection // allocate bottom up, top down, randomly +} + +type SLoadbalancerNetworkDeleteData struct { + loadbalancer *SLoadbalancer + reserve bool // reserve after delete +} + +func (m *SLoadbalancernetworkManager) NewLoadbalancerNetwork(ctx context.Context, userCred mcclient.TokenCredential, req *SLoadbalancerNetworkRequestData) (*SLoadbalancerNetwork, error) { + networkMan := db.GetModelManager("network").(*SNetworkManager) + if networkMan == nil { + return nil, fmt.Errorf("failed getting network manager") + } + im, err := networkMan.FetchById(req.networkId) + if err != nil { + return nil, err + } + network := im.(*SNetwork) + ln := &SLoadbalancerNetwork{ + LoadbalancerId: req.loadbalancer.Id, + NetworkId: network.Id, + } + ln.SetModelManager(m) + + lockman.LockObject(ctx, network) + defer lockman.ReleaseObject(ctx, network) + usedMap := network.GetUsedAddresses() + recentReclaimed := map[string]bool{} + ipAddr, err := network.GetFreeIP(ctx, userCred, + usedMap, recentReclaimed, req.address, req.strategy, req.reserved) + if err != nil { + return nil, err + } + ln.IpAddr = ipAddr + err = m.TableSpec().Insert(ln) + if err != nil { + // NOTE no need to free ipAddr as GetFreeIP has no side effect + return nil, err + } + return ln, err +} + +func (m *SLoadbalancernetworkManager) DeleteLoadbalancerNetwork(ctx context.Context, userCred mcclient.TokenCredential, req *SLoadbalancerNetworkDeleteData) error { + q := m.Query().Equals("loadbalancer_id", req.loadbalancer.Id) + lns := []SLoadbalancerNetwork{} + err := db.FetchModelObjects(m, q, &lns) + if err != nil { + return err + } + // TODO pack up errors and continue, then return as a whole + for _, ln := range lns { + err := ln.Delete(ctx, userCred) + if err != nil { + return err + } + if req.reserve && regutils.MatchIP4Addr(ln.IpAddr) { + note := fmt.Sprintf("reserved from loadbalancer delete: %s", + req.loadbalancer.Id) + reservedIpMan := db.GetModelManager("reservedip").(*SReservedipManager) + network := ln.Network() + err := reservedIpMan.ReserveIP(userCred, network, ln.IpAddr, note) + if err != nil { + return err + } + } + } + return nil +} + +func (lbNetwork *SLoadbalancerNetwork) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + return db.DeleteModel(ctx, userCred, lbNetwork) +} diff --git a/pkg/compute/models/networks.go b/pkg/compute/models/networks.go index 52d9b44825..e13d486e45 100644 --- a/pkg/compute/models/networks.go +++ b/pkg/compute/models/networks.go @@ -164,6 +164,7 @@ func (self *SNetwork) GetUsedAddresses() map[string]bool { GroupnetworkManager.Query().SubQuery(), HostnetworkManager.Query().SubQuery(), ReservedipManager.Query().SubQuery(), + LoadbalancernetworkManager.Query().SubQuery(), } { q := tbl.Query(tbl.Field("ip_addr")).Equals("network_id", self.Id) rows, err := q.Rows() @@ -207,6 +208,7 @@ func isIpUsed(ipstr string, addrTable map[string]bool, recentUsedAddrTable map[s func (self *SNetwork) getFreeIP(addrTable map[string]bool, recentUsedAddrTable map[string]bool, candidate string, allocDir IPAddlocationDirection) (string, error) { iprange := self.getIPRange() + // Try candidate first if len(candidate) > 0 { candIP, err := netutils.NewIPV4Addr(candidate) if err != nil { @@ -531,8 +533,15 @@ func (self *SNetwork) isAddressInRange(address netutils.IPV4Addr) bool { } func (self *SNetwork) isAddressUsed(address string) bool { - for _, model := range []db.IModelManager{GuestnetworkManager, GroupnetworkManager, HostnetworkManager, ReservedipManager} { - q := model.Query().Equals("ip_addr", address).Equals("network_id", self.Id) + managers := []db.IModelManager{ + GuestnetworkManager, + GroupnetworkManager, + HostnetworkManager, + ReservedipManager, + LoadbalancernetworkManager, + } + for _, manager := range managers { + q := manager.Query().Equals("ip_addr", address).Equals("network_id", self.Id) if q.Count() > 0 { return true } From af6946ebb5bc0f39a7c2429c9313b6b4053ec6ab Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Sat, 29 Sep 2018 14:08:38 +0000 Subject: [PATCH 4/4] loadbalancer: initial version --- .../root/etc/yunion/lbagent.conf.sample | 15 + .../opt/yunion/share/lbagent/healthcheck.sh | 46 ++ .../share/lbagent/yunion-lbagent.service | 17 + build/lbagent/vars | 8 + cmd/climc/shell/loadbalanceracls.go | 115 +++++ cmd/climc/shell/loadbalanceragents.go | 107 ++++ cmd/climc/shell/loadbalancerbackendgroups.go | 50 ++ cmd/climc/shell/loadbalancerbackends.go | 59 +++ cmd/climc/shell/loadbalancercertificates.go | 62 +++ cmd/climc/shell/loadbalancerlistenerrules.go | 69 +++ cmd/climc/shell/loadbalancerlisteners.go | 71 +++ cmd/climc/shell/loadbalancers.go | 71 +++ cmd/lbagent/main.go | 64 +++ pkg/compute/handlers.go | 8 + pkg/compute/models/loadbalancer_common.go | 30 ++ pkg/compute/models/loadbalancer_const.go | 215 ++++++++ pkg/compute/models/loadbalanceracls.go | 214 ++++++++ pkg/compute/models/loadbalanceragents.go | 457 +++++++++++++++++ .../models/loadbalancerbackendgroups.go | 122 +++++ pkg/compute/models/loadbalancerbackends.go | 194 +++++++ .../models/loadbalancercertificates.go | 154 ++++++ .../models/loadbalancerlistenerrules.go | 145 ++++++ pkg/compute/models/loadbalancerlisteners.go | 360 +++++++++++++ pkg/compute/models/loadbalancernetworks.go | 3 +- pkg/compute/models/loadbalancers.go | 195 +++++++ pkg/compute/options/options.go | 2 + pkg/compute/service/service.go | 1 + pkg/lbagent/api.go | 291 +++++++++++ pkg/lbagent/cmd.go | 24 + pkg/lbagent/gobetween/config.go | 304 +++++++++++ pkg/lbagent/haproxy.go | 352 +++++++++++++ pkg/lbagent/models/agentparams.go | 106 ++++ pkg/lbagent/models/corpus.go | 79 +++ pkg/lbagent/models/gobetween.go | 164 ++++++ pkg/lbagent/models/haproxy.go | 476 ++++++++++++++++++ pkg/lbagent/models/keepalived.go | 157 ++++++ pkg/lbagent/models/models.go | 48 ++ pkg/lbagent/models/modelset.go | 249 +++++++++ pkg/lbagent/models/modelsets.go | 138 +++++ pkg/lbagent/models/reflect.go | 246 +++++++++ pkg/lbagent/options.go | 63 +++ pkg/lbagent/utils/configdir.go | 142 ++++++ pkg/lbagent/utils/filemode.go | 12 + pkg/lbagent/utils/haproxy.go | 71 +++ pkg/lbagent/utils/keepalived.go | 31 ++ pkg/lbagent/utils/pidfile.go | 36 ++ pkg/mcclient/models/base.go | 75 +++ pkg/mcclient/models/loadbalancers.go | 179 +++++++ pkg/mcclient/modules/mod_loadbalanceracls.go | 25 + .../modules/mod_loadbalanceragents.go | 35 ++ .../modules/mod_loadbalancerbackendgroups.go | 25 + .../modules/mod_loadbalancerbackends.go | 30 ++ .../modules/mod_loadbalancercertificates.go | 30 ++ .../modules/mod_loadbalancerlistenerrules.go | 29 ++ .../modules/mod_loadbalancerlisteners.go | 31 ++ pkg/mcclient/modules/mod_loadbalancers.go | 31 ++ pkg/mcclient/options/base.go | 5 + pkg/mcclient/options/loadbalanceracls.go | 123 +++++ pkg/mcclient/options/loadbalanceragents.go | 113 +++++ .../options/loadbalancerbackendgroups.go | 16 + pkg/mcclient/options/loadbalancerbackends.go | 34 ++ .../options/loadbalancercertificates.go | 85 ++++ .../options/loadbalancerlistenerrules.go | 34 ++ pkg/mcclient/options/loadbalancerlisteners.go | 155 ++++++ pkg/mcclient/options/loadbalancers.go | 35 ++ 65 files changed, 6932 insertions(+), 1 deletion(-) create mode 100644 build/lbagent/root/etc/yunion/lbagent.conf.sample create mode 100755 build/lbagent/root/opt/yunion/share/lbagent/healthcheck.sh create mode 100644 build/lbagent/root/opt/yunion/share/lbagent/yunion-lbagent.service create mode 100644 build/lbagent/vars create mode 100644 cmd/climc/shell/loadbalanceracls.go create mode 100644 cmd/climc/shell/loadbalanceragents.go create mode 100644 cmd/climc/shell/loadbalancerbackendgroups.go create mode 100644 cmd/climc/shell/loadbalancerbackends.go create mode 100644 cmd/climc/shell/loadbalancercertificates.go create mode 100644 cmd/climc/shell/loadbalancerlistenerrules.go create mode 100644 cmd/climc/shell/loadbalancerlisteners.go create mode 100644 cmd/climc/shell/loadbalancers.go create mode 100644 cmd/lbagent/main.go create mode 100644 pkg/compute/models/loadbalancer_common.go create mode 100644 pkg/compute/models/loadbalancer_const.go create mode 100644 pkg/compute/models/loadbalanceracls.go create mode 100644 pkg/compute/models/loadbalanceragents.go create mode 100644 pkg/compute/models/loadbalancerbackendgroups.go create mode 100644 pkg/compute/models/loadbalancerbackends.go create mode 100644 pkg/compute/models/loadbalancercertificates.go create mode 100644 pkg/compute/models/loadbalancerlistenerrules.go create mode 100644 pkg/compute/models/loadbalancerlisteners.go create mode 100644 pkg/compute/models/loadbalancers.go create mode 100644 pkg/lbagent/api.go create mode 100644 pkg/lbagent/cmd.go create mode 100644 pkg/lbagent/gobetween/config.go create mode 100644 pkg/lbagent/haproxy.go create mode 100644 pkg/lbagent/models/agentparams.go create mode 100644 pkg/lbagent/models/corpus.go create mode 100644 pkg/lbagent/models/gobetween.go create mode 100644 pkg/lbagent/models/haproxy.go create mode 100644 pkg/lbagent/models/keepalived.go create mode 100644 pkg/lbagent/models/models.go create mode 100644 pkg/lbagent/models/modelset.go create mode 100644 pkg/lbagent/models/modelsets.go create mode 100644 pkg/lbagent/models/reflect.go create mode 100644 pkg/lbagent/options.go create mode 100644 pkg/lbagent/utils/configdir.go create mode 100644 pkg/lbagent/utils/filemode.go create mode 100644 pkg/lbagent/utils/haproxy.go create mode 100644 pkg/lbagent/utils/keepalived.go create mode 100644 pkg/lbagent/utils/pidfile.go create mode 100644 pkg/mcclient/models/base.go create mode 100644 pkg/mcclient/models/loadbalancers.go create mode 100644 pkg/mcclient/modules/mod_loadbalanceracls.go create mode 100644 pkg/mcclient/modules/mod_loadbalanceragents.go create mode 100644 pkg/mcclient/modules/mod_loadbalancerbackendgroups.go create mode 100644 pkg/mcclient/modules/mod_loadbalancerbackends.go create mode 100644 pkg/mcclient/modules/mod_loadbalancercertificates.go create mode 100644 pkg/mcclient/modules/mod_loadbalancerlistenerrules.go create mode 100644 pkg/mcclient/modules/mod_loadbalancerlisteners.go create mode 100644 pkg/mcclient/modules/mod_loadbalancers.go create mode 100644 pkg/mcclient/options/loadbalanceracls.go create mode 100644 pkg/mcclient/options/loadbalanceragents.go create mode 100644 pkg/mcclient/options/loadbalancerbackendgroups.go create mode 100644 pkg/mcclient/options/loadbalancerbackends.go create mode 100644 pkg/mcclient/options/loadbalancercertificates.go create mode 100644 pkg/mcclient/options/loadbalancerlistenerrules.go create mode 100644 pkg/mcclient/options/loadbalancerlisteners.go create mode 100644 pkg/mcclient/options/loadbalancers.go diff --git a/build/lbagent/root/etc/yunion/lbagent.conf.sample b/build/lbagent/root/etc/yunion/lbagent.conf.sample new file mode 100644 index 0000000000..08db09c8d1 --- /dev/null +++ b/build/lbagent/root/etc/yunion/lbagent.conf.sample @@ -0,0 +1,15 @@ +region = 'Yunion' +auth_uri = 'http://10.168.222.136:35357/v3' +admin_user = 'regionadmin' +admin_password = 'GkZOr6poQgcmMszU' +admin_tenant_name = 'system' + +data_preserve_n = 2 +base_data_dir = "/opt/cloud/workspace/lbagent" + +api_lbagent_id = 'lbagent id' +api_lbagent_hb_interval = 60 +api_lbagent_hb_timeout_relaxation = 120 + +api_sync_interval = 5 +api_list_batch_size = 2048 diff --git a/build/lbagent/root/opt/yunion/share/lbagent/healthcheck.sh b/build/lbagent/root/opt/yunion/share/lbagent/healthcheck.sh new file mode 100755 index 0000000000..ba47c44b7b --- /dev/null +++ b/build/lbagent/root/opt/yunion/share/lbagent/healthcheck.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -e + +o_wait=5 +o_req="are you ok?" +o_exp="ok" +o_host= +o_port= + +__errmsg() { + echo "healthcheck: $*" >&2 +} + +while [ "$#" -gt 0 ]; do + case "$1" in + wait=*|\ + req=*|\ + exp=*|\ + host=*|\ + port=*) + eval "o_$1" + ;; + esac + shift +done + +if [ -z "$o_host" -o -z "$o_port" ]; then + __errmsg "--host or --port missing" + exit 126 # script error are not server error +fi + +got="$( + (stdbuf -o0 printf "%s" "$o_req"; sleep "$o_wait") | \ + nc \ + --verbose \ + --udp \ + --nodns \ + --wait "$o_wait" \ + "$o_host" "$o_port" +)" + +if [ "$got" != "$o_exp" ]; then + __errmsg "result unmatch, got $got, want $o_exp" + exit 1 +fi diff --git a/build/lbagent/root/opt/yunion/share/lbagent/yunion-lbagent.service b/build/lbagent/root/opt/yunion/share/lbagent/yunion-lbagent.service new file mode 100644 index 0000000000..c5761d7963 --- /dev/null +++ b/build/lbagent/root/opt/yunion/share/lbagent/yunion-lbagent.service @@ -0,0 +1,17 @@ +[Unit] +Description=Yunion Loadbalancer Agent +Documentation=http://doc.yunionyun.com +After=network.target + +[Service] +Type=simple +User=root +Group=root +ExecStart=/opt/yunion/bin/lbagent --config /etc/yunion/lbagent.conf +WorkingDirectory=/opt/yunion +KillMode=process +Restart=always +RestartSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/build/lbagent/vars b/build/lbagent/vars new file mode 100644 index 0000000000..596b539a66 --- /dev/null +++ b/build/lbagent/vars @@ -0,0 +1,8 @@ +DESCRIPTION="Yunion Loadbalancer Agent" + +REQUIRES=( + nmap-ncat + "keepalived >= 2.0.0" + "haproxy >= 1.8.0" + "gobetween >= 0.7.0" +) diff --git a/cmd/climc/shell/loadbalanceracls.go b/cmd/climc/shell/loadbalanceracls.go new file mode 100644 index 0000000000..085f116d56 --- /dev/null +++ b/cmd/climc/shell/loadbalanceracls.go @@ -0,0 +1,115 @@ +package shell + +import ( + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +func init() { + lbAclConvert := func(jd *jsonutils.JSONDict) error { + jaeso, err := jd.Get("acl_entries") + if err != nil { + return err + } + aclEntries := options.AclEntries{} + err = jaeso.Unmarshal(&aclEntries) + if err != nil { + return err + } + aclTextLines := aclEntries.String() + jd.Set("acl_entries", jsonutils.NewString(aclTextLines)) + return nil + } + + printLbAcl := func(jsonObj jsonutils.JSONObject) { + jd, ok := jsonObj.(*jsonutils.JSONDict) + if !ok { + printObject(jsonObj) + return + } + err := lbAclConvert(jd) + if err != nil { + printObject(jsonObj) + return + } + printObject(jd) + } + printLbAclList := func(list *modules.ListResult, columns []string) { + data := list.Data + for _, jsonObj := range data { + jd := jsonObj.(*jsonutils.JSONDict) + err := lbAclConvert(jd) + if err != nil { + printList(list, columns) + } + } + printList(list, columns) + } + + R(&options.LoadbalancerAclCreateOptions{}, "lbacl-create", "Create lbacl", func(s *mcclient.ClientSession, opts *options.LoadbalancerAclCreateOptions) error { + params, err := opts.Params() + if err != nil { + return err + } + lbacl, err := modules.LoadbalancerAcls.Create(s, params) + if err != nil { + return err + } + printLbAcl(lbacl) + return nil + }) + R(&options.LoadbalancerAclGetOptions{}, "lbacl-show", "Show lbacl", func(s *mcclient.ClientSession, opts *options.LoadbalancerAclGetOptions) error { + lbacl, err := modules.LoadbalancerAcls.Get(s, opts.ID, nil) + if err != nil { + return err + } + printLbAcl(lbacl) + return nil + }) + R(&options.LoadbalancerAclListOptions{}, "lbacl-list", "List lbacls", func(s *mcclient.ClientSession, opts *options.LoadbalancerAclListOptions) error { + params, err := options.ListStructToParams(opts) + if err != nil { + return err + } + result, err := modules.LoadbalancerAcls.List(s, params) + if err != nil { + return err + } + printLbAclList(result, modules.LoadbalancerAcls.GetColumns(s)) + return nil + }) + R(&options.LoadbalancerAclUpdateOptions{}, "lbacl-update", "Update lbacls", func(s *mcclient.ClientSession, opts *options.LoadbalancerAclUpdateOptions) error { + params, err := opts.Params() + if err != nil { + return err + } + lbacl, err := modules.LoadbalancerAcls.Update(s, opts.ID, params) + if err != nil { + return err + } + printLbAcl(lbacl) + return nil + }) + R(&options.LoadbalancerAclDeleteOptions{}, "lbacl-delete", "Show lbacl", func(s *mcclient.ClientSession, opts *options.LoadbalancerAclDeleteOptions) error { + lbacl, err := modules.LoadbalancerAcls.Delete(s, opts.ID, nil) + if err != nil { + return err + } + printLbAcl(lbacl) + return nil + }) + R(&options.LoadbalancerAclActionPatchOptions{}, "lbacl-patch", "Patch lbacls", func(s *mcclient.ClientSession, opts *options.LoadbalancerAclActionPatchOptions) error { + params, err := opts.Params() + if err != nil { + return err + } + lbacl, err := modules.LoadbalancerAcls.PerformAction(s, opts.ID, "patch", params) + if err != nil { + return err + } + printLbAcl(lbacl) + return nil + }) +} diff --git a/cmd/climc/shell/loadbalanceragents.go b/cmd/climc/shell/loadbalanceragents.go new file mode 100644 index 0000000000..cba6a89717 --- /dev/null +++ b/cmd/climc/shell/loadbalanceragents.go @@ -0,0 +1,107 @@ +package shell + +import ( + "encoding/base64" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +func init() { + printLbagent := func(data jsonutils.JSONObject) { + printObjectRecursiveEx(data, func(data jsonutils.JSONObject) { + // "base64 -d" config template + keys := []string{ + "params.keepalived_conf_tmpl", + "params.haproxy_conf_tmpl", + } + d := data.(*jsonutils.JSONDict) + for _, key := range keys { + if d.Contains(key) { + s0, _ := d.GetString(key) + b1, err := base64.StdEncoding.DecodeString(s0) + if err != nil { + log.Errorf("%s: invalid base64 string: %s\n %s", key, err, s0) + return + } + s1 := string(b1) + d.Set(key, jsonutils.NewString(s1)) + } + } + printObject(d) + }) + } + R(&options.LoadbalancerAgentCreateOptions{}, "lbagent-create", "Create lbagent", func(s *mcclient.ClientSession, opts *options.LoadbalancerAgentCreateOptions) error { + params, err := opts.Params() + if err != nil { + return err + } + lbagent, err := modules.LoadbalancerAgents.Create(s, params) + if err != nil { + return err + } + printLbagent(lbagent) + return nil + }) + R(&options.LoadbalancerAgentGetOptions{}, "lbagent-show", "Show lbagent", func(s *mcclient.ClientSession, opts *options.LoadbalancerAgentGetOptions) error { + lbagent, err := modules.LoadbalancerAgents.Get(s, opts.ID, nil) + if err != nil { + return err + } + printLbagent(lbagent) + return nil + }) + R(&options.LoadbalancerAgentListOptions{}, "lbagent-list", "List lbagents", func(s *mcclient.ClientSession, opts *options.LoadbalancerAgentListOptions) error { + params, err := options.ListStructToParams(opts) + if err != nil { + return err + } + result, err := modules.LoadbalancerAgents.List(s, params) + if err != nil { + return err + } + printList(result, modules.LoadbalancerAgents.GetColumns(s)) + return nil + }) + R(&options.LoadbalancerAgentUpdateOptions{}, "lbagent-update", "Update lbagent", func(s *mcclient.ClientSession, opts *options.LoadbalancerAgentUpdateOptions) error { + params, err := options.StructToParams(opts) + lbagent, err := modules.LoadbalancerAgents.Update(s, opts.ID, params) + if err != nil { + return err + } + printLbagent(lbagent) + return nil + }) + R(&options.LoadbalancerAgentDeleteOptions{}, "lbagent-delete", "Show lbagent", func(s *mcclient.ClientSession, opts *options.LoadbalancerAgentDeleteOptions) error { + lbagent, err := modules.LoadbalancerAgents.Delete(s, opts.ID, nil) + if err != nil { + return err + } + printLbagent(lbagent) + return nil + }) + R(&options.LoadbalancerAgentActionHbOptions{}, "lbagent-heartbeat", "Emulate a lbagent heartbeat", func(s *mcclient.ClientSession, opts *options.LoadbalancerAgentActionHbOptions) error { + lbagent, err := modules.LoadbalancerAgents.PerformAction(s, opts.ID, "hb", nil) + if err != nil { + return err + } + printLbagent(lbagent) + return nil + }) + R(&options.LoadbalancerAgentActionPatchParamsOptions{}, "lbagent-params-patch", "Patch lbagent params", func(s *mcclient.ClientSession, opts *options.LoadbalancerAgentActionPatchParamsOptions) error { + params, err := opts.Params() + if err != nil { + return err + } + lbagent, err := modules.LoadbalancerAgents.PerformAction(s, opts.ID, "params-patch", params) + if err != nil { + return err + } + printLbagent(lbagent) + return nil + }) +} diff --git a/cmd/climc/shell/loadbalancerbackendgroups.go b/cmd/climc/shell/loadbalancerbackendgroups.go new file mode 100644 index 0000000000..8cf38963fa --- /dev/null +++ b/cmd/climc/shell/loadbalancerbackendgroups.go @@ -0,0 +1,50 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +func init() { + R(&options.LoadbalancerBackendGroupCreateOptions{}, "lbbackendgroup-create", "Create lbbackendgroup", func(s *mcclient.ClientSession, opts *options.LoadbalancerBackendGroupCreateOptions) error { + params, err := options.StructToParams(opts) + if err != nil { + return err + } + lbbackendgroup, err := modules.LoadbalancerBackendGroups.Create(s, params) + if err != nil { + return err + } + printObject(lbbackendgroup) + return nil + }) + R(&options.LoadbalancerBackendGroupGetOptions{}, "lbbackendgroup-show", "Show lbbackendgroup", func(s *mcclient.ClientSession, opts *options.LoadbalancerBackendGroupGetOptions) error { + lbbackendgroup, err := modules.LoadbalancerBackendGroups.Get(s, opts.ID, nil) + if err != nil { + return err + } + printObject(lbbackendgroup) + return nil + }) + R(&options.LoadbalancerBackendGroupListOptions{}, "lbbackendgroup-list", "List lbbackendgroups", func(s *mcclient.ClientSession, opts *options.LoadbalancerBackendGroupListOptions) error { + params, err := options.ListStructToParams(opts) + if err != nil { + return err + } + result, err := modules.LoadbalancerBackendGroups.List(s, params) + if err != nil { + return err + } + printList(result, modules.LoadbalancerBackendGroups.GetColumns(s)) + return nil + }) + R(&options.LoadbalancerBackendGroupDeleteOptions{}, "lbbackendgroup-delete", "Show lbbackendgroup", func(s *mcclient.ClientSession, opts *options.LoadbalancerBackendGroupDeleteOptions) error { + lbbackendgroup, err := modules.LoadbalancerBackendGroups.Delete(s, opts.ID, nil) + if err != nil { + return err + } + printObject(lbbackendgroup) + return nil + }) +} diff --git a/cmd/climc/shell/loadbalancerbackends.go b/cmd/climc/shell/loadbalancerbackends.go new file mode 100644 index 0000000000..f8e1c4d03e --- /dev/null +++ b/cmd/climc/shell/loadbalancerbackends.go @@ -0,0 +1,59 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +func init() { + R(&options.LoadbalancerBackendCreateOptions{}, "lbbackend-create", "Create lbbackend", func(s *mcclient.ClientSession, opts *options.LoadbalancerBackendCreateOptions) error { + params, err := options.StructToParams(opts) + if err != nil { + return err + } + lbbackend, err := modules.LoadbalancerBackends.Create(s, params) + if err != nil { + return err + } + printObject(lbbackend) + return nil + }) + R(&options.LoadbalancerBackendGetOptions{}, "lbbackend-show", "Show lbbackend", func(s *mcclient.ClientSession, opts *options.LoadbalancerBackendGetOptions) error { + lbbackend, err := modules.LoadbalancerBackends.Get(s, opts.ID, nil) + if err != nil { + return err + } + printObject(lbbackend) + return nil + }) + R(&options.LoadbalancerBackendListOptions{}, "lbbackend-list", "List lbbackends", func(s *mcclient.ClientSession, opts *options.LoadbalancerBackendListOptions) error { + params, err := options.ListStructToParams(opts) + if err != nil { + return err + } + result, err := modules.LoadbalancerBackends.List(s, params) + if err != nil { + return err + } + printList(result, modules.LoadbalancerBackends.GetColumns(s)) + return nil + }) + R(&options.LoadbalancerBackendUpdateOptions{}, "lbbackend-update", "Update lbbackend", func(s *mcclient.ClientSession, opts *options.LoadbalancerBackendUpdateOptions) error { + params, err := options.StructToParams(opts) + lbbackend, err := modules.LoadbalancerBackends.Update(s, opts.ID, params) + if err != nil { + return err + } + printObject(lbbackend) + return nil + }) + R(&options.LoadbalancerBackendDeleteOptions{}, "lbbackend-delete", "Show lbbackend", func(s *mcclient.ClientSession, opts *options.LoadbalancerBackendDeleteOptions) error { + lbbackend, err := modules.LoadbalancerBackends.Delete(s, opts.ID, nil) + if err != nil { + return err + } + printObject(lbbackend) + return nil + }) +} diff --git a/cmd/climc/shell/loadbalancercertificates.go b/cmd/climc/shell/loadbalancercertificates.go new file mode 100644 index 0000000000..21882b5165 --- /dev/null +++ b/cmd/climc/shell/loadbalancercertificates.go @@ -0,0 +1,62 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +func init() { + R(&options.LoadbalancerCertificateCreateOptions{}, "lbcert-create", "Create lbcert", func(s *mcclient.ClientSession, opts *options.LoadbalancerCertificateCreateOptions) error { + params, err := opts.Params() + if err != nil { + return err + } + lbcert, err := modules.LoadbalancerCertificates.Create(s, params) + if err != nil { + return err + } + printObject(lbcert) + return nil + }) + R(&options.LoadbalancerCertificateGetOptions{}, "lbcert-show", "Show lbcert", func(s *mcclient.ClientSession, opts *options.LoadbalancerCertificateGetOptions) error { + lbcert, err := modules.LoadbalancerCertificates.Get(s, opts.ID, nil) + if err != nil { + return err + } + printObject(lbcert) + return nil + }) + R(&options.LoadbalancerCertificateListOptions{}, "lbcert-list", "List lbcerts", func(s *mcclient.ClientSession, opts *options.LoadbalancerCertificateListOptions) error { + params, err := options.ListStructToParams(opts) + if err != nil { + return err + } + result, err := modules.LoadbalancerCertificates.List(s, params) + if err != nil { + return err + } + printList(result, modules.LoadbalancerCertificates.GetColumns(s)) + return nil + }) + R(&options.LoadbalancerCertificateUpdateOptions{}, "lbcert-update", "Update lbcert", func(s *mcclient.ClientSession, opts *options.LoadbalancerCertificateUpdateOptions) error { + params, err := opts.Params() + if err != nil { + return err + } + lbcert, err := modules.LoadbalancerCertificates.Update(s, opts.ID, params) + if err != nil { + return err + } + printObject(lbcert) + return nil + }) + R(&options.LoadbalancerCertificateDeleteOptions{}, "lbcert-delete", "Show lbcert", func(s *mcclient.ClientSession, opts *options.LoadbalancerCertificateDeleteOptions) error { + lbcert, err := modules.LoadbalancerCertificates.Delete(s, opts.ID, nil) + if err != nil { + return err + } + printObject(lbcert) + return nil + }) +} diff --git a/cmd/climc/shell/loadbalancerlistenerrules.go b/cmd/climc/shell/loadbalancerlistenerrules.go new file mode 100644 index 0000000000..403dbd9acc --- /dev/null +++ b/cmd/climc/shell/loadbalancerlistenerrules.go @@ -0,0 +1,69 @@ +package shell + +import ( + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +func init() { + R(&options.LoadbalancerListenerRuleCreateOptions{}, "lblistenerrule-create", "Create lblistenerrule", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerRuleCreateOptions) error { + params := jsonutils.Marshal(opts) + lblistenerrule, err := modules.LoadbalancerListenerRules.Create(s, params) + if err != nil { + return err + } + printObject(lblistenerrule) + return nil + }) + R(&options.LoadbalancerListenerRuleGetOptions{}, "lblistenerrule-show", "Show lblistenerrule", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerRuleGetOptions) error { + lblistenerrule, err := modules.LoadbalancerListenerRules.Get(s, opts.ID, nil) + if err != nil { + return err + } + printObject(lblistenerrule) + return nil + }) + R(&options.LoadbalancerListenerRuleListOptions{}, "lblistenerrule-list", "List lblistenerrules", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerRuleListOptions) error { + params, err := options.ListStructToParams(opts) + if err != nil { + return err + } + result, err := modules.LoadbalancerListenerRules.List(s, params) + if err != nil { + return err + } + printList(result, modules.LoadbalancerListenerRules.GetColumns(s)) + return nil + }) + R(&options.LoadbalancerListenerRuleUpdateOptions{}, "lblistenerrule-update", "Update lblistenerrule", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerRuleUpdateOptions) error { + params, err := options.StructToParams(opts) + lblistenerrule, err := modules.LoadbalancerListenerRules.Update(s, opts.ID, params) + if err != nil { + return err + } + printObject(lblistenerrule) + return nil + }) + R(&options.LoadbalancerListenerRuleDeleteOptions{}, "lblistenerrule-delete", "Show lblistenerrule", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerRuleDeleteOptions) error { + lblistenerrule, err := modules.LoadbalancerListenerRules.Delete(s, opts.ID, nil) + if err != nil { + return err + } + printObject(lblistenerrule) + return nil + }) + R(&options.LoadbalancerListenerRuleActionStatusOptions{}, "lblistenerrule-status", "Change lblistenerrule status", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerRuleActionStatusOptions) error { + params, err := options.StructToParams(opts) + if err != nil { + return err + } + lblistenerrule, err := modules.LoadbalancerListenerRules.PerformAction(s, opts.ID, "status", params) + if err != nil { + return err + } + printObject(lblistenerrule) + return nil + }) +} diff --git a/cmd/climc/shell/loadbalancerlisteners.go b/cmd/climc/shell/loadbalancerlisteners.go new file mode 100644 index 0000000000..2c60b61c05 --- /dev/null +++ b/cmd/climc/shell/loadbalancerlisteners.go @@ -0,0 +1,71 @@ +package shell + +import ( + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +func init() { + + R(&options.LoadbalancerListenerCreateOptions{}, "lblistener-create", "Create lblistener", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerCreateOptions) error { + // TODO make a generic one + params := jsonutils.Marshal(opts) + lblistener, err := modules.LoadbalancerListeners.Create(s, params) + if err != nil { + return err + } + printObject(lblistener) + return nil + }) + R(&options.LoadbalancerListenerGetOptions{}, "lblistener-show", "Show lblistener", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerGetOptions) error { + lblistener, err := modules.LoadbalancerListeners.Get(s, opts.ID, nil) + if err != nil { + return err + } + printObject(lblistener) + return nil + }) + R(&options.LoadbalancerListenerListOptions{}, "lblistener-list", "List lblisteners", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerListOptions) error { + params, err := options.ListStructToParams(opts) + if err != nil { + return err + } + result, err := modules.LoadbalancerListeners.List(s, params) + if err != nil { + return err + } + printList(result, modules.LoadbalancerListeners.GetColumns(s)) + return nil + }) + R(&options.LoadbalancerListenerUpdateOptions{}, "lblistener-update", "Update lblistener", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerUpdateOptions) error { + params, err := options.StructToParams(opts) + lblistener, err := modules.LoadbalancerListeners.Update(s, opts.ID, params) + if err != nil { + return err + } + printObject(lblistener) + return nil + }) + R(&options.LoadbalancerListenerDeleteOptions{}, "lblistener-delete", "Show lblistener", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerDeleteOptions) error { + lblistener, err := modules.LoadbalancerListeners.Delete(s, opts.ID, nil) + if err != nil { + return err + } + printObject(lblistener) + return nil + }) + R(&options.LoadbalancerListenerActionStatusOptions{}, "lblistener-status", "Change lblistener status", func(s *mcclient.ClientSession, opts *options.LoadbalancerListenerActionStatusOptions) error { + params, err := options.StructToParams(opts) + if err != nil { + return err + } + lblistener, err := modules.LoadbalancerListeners.PerformAction(s, opts.ID, "status", params) + if err != nil { + return err + } + printObject(lblistener) + return nil + }) +} diff --git a/cmd/climc/shell/loadbalancers.go b/cmd/climc/shell/loadbalancers.go new file mode 100644 index 0000000000..5ee9d53f1d --- /dev/null +++ b/cmd/climc/shell/loadbalancers.go @@ -0,0 +1,71 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +func init() { + R(&options.LoadbalancerCreateOptions{}, "lb-create", "Create lb", func(s *mcclient.ClientSession, opts *options.LoadbalancerCreateOptions) error { + params, err := options.StructToParams(opts) + if err != nil { + return err + } + lb, err := modules.Loadbalancers.Create(s, params) + if err != nil { + return err + } + printObject(lb) + return nil + }) + R(&options.LoadbalancerGetOptions{}, "lb-show", "Show lb", func(s *mcclient.ClientSession, opts *options.LoadbalancerGetOptions) error { + lb, err := modules.Loadbalancers.Get(s, opts.ID, nil) + if err != nil { + return err + } + printObject(lb) + return nil + }) + R(&options.LoadbalancerListOptions{}, "lb-list", "List lbs", func(s *mcclient.ClientSession, opts *options.LoadbalancerListOptions) error { + params, err := options.ListStructToParams(opts) + if err != nil { + return err + } + result, err := modules.Loadbalancers.List(s, params) + if err != nil { + return err + } + printList(result, modules.Loadbalancers.GetColumns(s)) + return nil + }) + R(&options.LoadbalancerUpdateOptions{}, "lb-update", "Update lb", func(s *mcclient.ClientSession, opts *options.LoadbalancerUpdateOptions) error { + params, err := options.StructToParams(opts) + lb, err := modules.Loadbalancers.Update(s, opts.ID, params) + if err != nil { + return err + } + printObject(lb) + return nil + }) + R(&options.LoadbalancerDeleteOptions{}, "lb-delete", "Show lb", func(s *mcclient.ClientSession, opts *options.LoadbalancerDeleteOptions) error { + lb, err := modules.Loadbalancers.Delete(s, opts.ID, nil) + if err != nil { + return err + } + printObject(lb) + return nil + }) + R(&options.LoadbalancerActionStatusOptions{}, "lb-status", "Change lb status", func(s *mcclient.ClientSession, opts *options.LoadbalancerActionStatusOptions) error { + params, err := options.StructToParams(opts) + if err != nil { + return err + } + lb, err := modules.Loadbalancers.PerformAction(s, opts.ID, "status", params) + if err != nil { + return err + } + printObject(lb) + return nil + }) +} diff --git a/cmd/lbagent/main.go b/cmd/lbagent/main.go new file mode 100644 index 0000000000..1943948512 --- /dev/null +++ b/cmd/lbagent/main.go @@ -0,0 +1,64 @@ +package main + +import ( + "context" + "os" + "os/signal" + "sync" + "syscall" + + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudcommon" + "yunion.io/x/onecloud/pkg/lbagent" +) + +func main() { + opts := &lbagent.Options{} + commonOpts := &opts.CommonOpts + { + cloudcommon.ParseOptions(opts, commonOpts, os.Args, "lbagent.conf") + cloudcommon.InitAuth(commonOpts, func() { + log.Infof("auth finished ok") + }) + } + if err := opts.ValidateThenInit(); err != nil { + log.Fatalf("opts validate: %s", err) + } + + var haproxyHelper *lbagent.HaproxyHelper + var apiHelper *lbagent.ApiHelper + var err error + { + haproxyHelper, err = lbagent.NewHaproxyHelper(opts) + if err != nil { + log.Fatalf("init haproxy helper failed: %s", err) + } + } + { + apiHelper, err = lbagent.NewApiHelper(opts) + if err != nil { + log.Fatalf("init api helper failed: %s", err) + } + } + + { + wg := &sync.WaitGroup{} + cmdChan := make(chan *lbagent.LbagentCmd) // internal + ctx, cancelFunc := context.WithCancel(context.Background()) + ctx = context.WithValue(ctx, "wg", wg) + ctx = context.WithValue(ctx, "cmdChan", cmdChan) + wg.Add(2) + go haproxyHelper.Run(ctx) + go apiHelper.Run(ctx) + + go func() { + sigChan := make(chan os.Signal) + signal.Notify(sigChan, syscall.SIGINT) + signal.Notify(sigChan, syscall.SIGTERM) + sig := <-sigChan + log.Infof("signal received: %s", sig) + cancelFunc() + }() + wg.Wait() + } +} diff --git a/pkg/compute/handlers.go b/pkg/compute/handlers.go index a30844dd20..50453c5816 100644 --- a/pkg/compute/handlers.go +++ b/pkg/compute/handlers.go @@ -62,6 +62,14 @@ func InitHandlers(app *appsrv.Application) { models.DnsRecordManager, models.ElasticipManager, models.SnapshotManager, + models.LoadbalancerManager, + models.LoadbalancerListenerManager, + models.LoadbalancerListenerRuleManager, + models.LoadbalancerBackendGroupManager, + models.LoadbalancerBackendManager, + models.LoadbalancerCertificateManager, + models.LoadbalancerAclManager, + models.LoadbalancerAgentManager, } { db.RegisterModelManager(manager) handler := db.NewModelHandler(manager) diff --git a/pkg/compute/models/loadbalancer_common.go b/pkg/compute/models/loadbalancer_common.go new file mode 100644 index 0000000000..1c9bf411a9 --- /dev/null +++ b/pkg/compute/models/loadbalancer_common.go @@ -0,0 +1,30 @@ +package models + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type SLoadbalancerStatus struct { + RuntimeStatus string `width:"36" charset:"ascii" nullable:"false" default:"init" list:"user"` +} + +type ILoadbalancerSubResourceManager interface { + db.IModelManager + + // PreDeleteSubs is to be called by upper manager to PreDelete models managed by this one + PreDeleteSubs(ctx context.Context, userCred mcclient.TokenCredential, q *sqlchemy.SQuery) +} + +// TODO +// notify on post create/update/delete +type SLoadbalancerNotifier struct{} + +func (n *SLoadbalancerNotifier) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) { + return +} diff --git a/pkg/compute/models/loadbalancer_const.go b/pkg/compute/models/loadbalancer_const.go new file mode 100644 index 0000000000..9367ca626d --- /dev/null +++ b/pkg/compute/models/loadbalancer_const.go @@ -0,0 +1,215 @@ +package models + +import ( + "yunion.io/x/onecloud/pkg/cloudcommon/validators" +) + +// Load balancer status transition (for spec status) +// +// create start stop delete +// init running - - - +// running - - stopped stopped +// stopped - running - - +// +// Each entity will have spec and runtime version. Spec version will increment +// on entity attribute update. Runtime version will be filled by the scheduler +// to the newest spec it has seen and committed +// +// When spec and runtime version differ, scheduler will set runtime version to +// "configuring", "stopping" and will finally transition to a terminal state. +// +// In the case of instance has PendingDeleted marked, it is also the +// scheduler's duty to make the runtime status to stopped and finally the +// entity in question +// +const ( + LB_STATUS_ENABLED = "enabled" + LB_STATUS_DISABLED = "disabled" + + LB_STATUS_INIT = "init" + LB_STATUS_RUNNING = "running" + LB_STATUS_STOPPED = "stopped" + LB_STATUS_CONFIGURING = "configuring" // config changes pending + LB_STATUS_STOPPING = "stopping" + LB_STATUS_DELETE_PENDING = "delete_pending" + LB_STATUS_ERROR = "error" // bad things happen +) + +var LB_STATUS_SPEC = validators.NewChoices( + LB_STATUS_ENABLED, + LB_STATUS_DISABLED, +) + +var LB_STATUS_RUNTIME = validators.NewChoices( + LB_STATUS_INIT, + LB_STATUS_CONFIGURING, + LB_STATUS_RUNNING, + LB_STATUS_STOPPING, + LB_STATUS_STOPPED, + LB_STATUS_ERROR, +) + +// Load Balancer network type (vpc or classic) determines viable backend +// servers (they should be from the same network type as the load balancer). +// +// Load Balancer address type (intranet or internet) determins the scope the +// service provided by load balancer can be accessed. If it's intranet, then +// it will only be accessible from the specified network. If it's internet, +// then it's public and can be accessed from outside the cloud region +const ( + LB_ADDR_TYPE_INTRANET = "intranet" + LB_ADDR_TYPE_INTERNET = "internet" +) + +var LB_ADDR_TYPES = validators.NewChoices( + LB_ADDR_TYPE_INTERNET, + LB_ADDR_TYPE_INTRANET, +) + +const ( + LB_NETWORK_TYPE_CLASSIC = "classic" + LB_NETWORK_TYPE_VPC = "vpc" +) + +var LB_NETWORK_TYPES = validators.NewChoices( + LB_NETWORK_TYPE_CLASSIC, + LB_NETWORK_TYPE_VPC, +) + +// TODO https_direct sni +const ( + LB_LISTENER_TYPE_TCP = "tcp" + LB_LISTENER_TYPE_UDP = "udp" + LB_LISTENER_TYPE_HTTP = "http" + LB_LISTENER_TYPE_HTTPS = "https" +) + +var LB_LISTENER_TYPES = validators.NewChoices( + LB_LISTENER_TYPE_TCP, + LB_LISTENER_TYPE_UDP, + LB_LISTENER_TYPE_HTTP, + LB_LISTENER_TYPE_HTTPS, +) + +const ( + LB_ACL_TYPE_BLACK = "black" + LB_ACL_TYPE_WHITE = "white" +) + +var LB_ACL_TYPES = validators.NewChoices( + LB_ACL_TYPE_BLACK, + LB_ACL_TYPE_WHITE, +) + +const ( + LB_TLS_CERT_PUBKEY_ALGO_RSA = "RSA" + LB_TLS_CERT_PUBKEY_ALGO_ECDSA = "ECDSA" +) + +var LB_TLS_CERT_PUBKEY_ALGOS = validators.NewChoices( + LB_TLS_CERT_PUBKEY_ALGO_RSA, + LB_TLS_CERT_PUBKEY_ALGO_ECDSA, +) + +// TODO may want extra for legacy apps +const ( + LB_TLS_CIPHER_POLICY_1_0 = "tls_cipher_policy_1_0" + LB_TLS_CIPHER_POLICY_1_1 = "tls_cipher_policy_1_1" + LB_TLS_CIPHER_POLICY_1_2 = "tls_cipher_policy_1_2" + LB_TLS_CIPHER_POLICY_1_2_strict = "tls_cipher_policy_1_2_strict" +) + +var LB_TLS_CIPHER_POLICIES = validators.NewChoices( + LB_TLS_CIPHER_POLICY_1_0, + LB_TLS_CIPHER_POLICY_1_1, + LB_TLS_CIPHER_POLICY_1_2, + LB_TLS_CIPHER_POLICY_1_2_strict, +) + +const ( + LB_STICKY_SESSION_TYPE_INSERT = "insert" + LB_STICKY_SESSION_TYPE_SERVER = "server" +) + +var LB_STICKY_SESSION_TYPES = validators.NewChoices( + LB_STICKY_SESSION_TYPE_INSERT, + LB_STICKY_SESSION_TYPE_SERVER, +) + +// TODO maybe https check when field need comes ;) +const ( + LB_HEALTH_CHECK_TCP = "tcp" + LB_HEALTH_CHECK_UDP = "udp" + LB_HEALTH_CHECK_HTTP = "http" +) + +var LB_HEALTH_CHECK_TYPES = validators.NewChoices( + LB_HEALTH_CHECK_TCP, + LB_HEALTH_CHECK_UDP, + LB_HEALTH_CHECK_HTTP, +) + +var LB_HEALTH_CHECK_TYPES_TCP = validators.NewChoices( + LB_HEALTH_CHECK_TCP, + LB_HEALTH_CHECK_HTTP, +) + +var LB_HEALTH_CHECK_TYPES_UDP = validators.NewChoices( + LB_HEALTH_CHECK_UDP, +) + +const ( + LB_HEALTH_CHECK_HTTP_CODE_2xx = "http_2xx" + LB_HEALTH_CHECK_HTTP_CODE_3xx = "http_3xx" + LB_HEALTH_CHECK_HTTP_CODE_4xx = "http_4xx" + LB_HEALTH_CHECK_HTTP_CODE_5xx = "http_5xx" + LB_HEALTH_CHECK_HTTP_CODE_DEFAULT = "http_2xx,http_3xx" +) + +var LB_HEALTH_CHECK_HTTP_CODES = validators.NewChoices( + LB_HEALTH_CHECK_HTTP_CODE_2xx, + LB_HEALTH_CHECK_HTTP_CODE_3xx, + LB_HEALTH_CHECK_HTTP_CODE_4xx, + LB_HEALTH_CHECK_HTTP_CODE_5xx, +) + +const ( + LB_BOOL_ON = "on" + LB_BOOL_OFF = "off" +) + +var LB_BOOL_VALUES = validators.NewChoices( + LB_BOOL_ON, + LB_BOOL_OFF, +) + +//TODO +// +// - qch, quic connection id +// - mh, maglev consistent hash +const ( + LB_SCHEDULER_RR = "rr" // round robin + LB_SCHEDULER_WRR = "wrr" // weighted round robin + LB_SCHEDULER_WLC = "wlc" // weighted least connection + LB_SCHEDULER_SCH = "sch" // source-ip-based consistent hash + LB_SCHEDULER_TCH = "tch" // 4-tuple-based consistent hash +) + +var LB_SCHEDULER_TYPES = validators.NewChoices( + LB_SCHEDULER_RR, + LB_SCHEDULER_WRR, + LB_SCHEDULER_WLC, + LB_SCHEDULER_SCH, + LB_SCHEDULER_TCH, +) + +// TODO raw type +const ( + LB_BACKEND_GUEST = "guest" + LB_BACKEND_HOST = "host" +) + +var LB_BACKEND_TYPES = validators.NewChoices( + LB_BACKEND_GUEST, + LB_BACKEND_HOST, +) diff --git a/pkg/compute/models/loadbalanceracls.go b/pkg/compute/models/loadbalanceracls.go new file mode 100644 index 0000000000..245ea94453 --- /dev/null +++ b/pkg/compute/models/loadbalanceracls.go @@ -0,0 +1,214 @@ +package models + +import ( + "context" + "fmt" + "net" + "reflect" + "strings" + "unicode" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/gotypes" + "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/validators" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type SLoadbalancerAclEntry struct { + Cidr string + Comment string +} + +func (aclEntry *SLoadbalancerAclEntry) Validate(data *jsonutils.JSONDict) error { + if strings.Index(aclEntry.Cidr, "/") > 0 { + _, ipNet, err := net.ParseCIDR(aclEntry.Cidr) + if err != nil { + return err + } + // normalize from 192.168.1.3/24 to 192.168.1.0/24 + aclEntry.Cidr = ipNet.String() + } else { + ip := net.ParseIP(aclEntry.Cidr).To4() + if ip == nil { + return fmt.Errorf("invalid addr %s", aclEntry.Cidr) + } + } + if commentLimit := 128; len(aclEntry.Comment) > commentLimit { + return fmt.Errorf("comment too long (%d>=%d)", + len(aclEntry.Comment), commentLimit) + } + for _, r := range aclEntry.Comment { + if !unicode.IsPrint(r) { + return fmt.Errorf("comment contains non-printable char: %v", r) + } + } + return nil +} + +type SLoadbalancerAclEntries []*SLoadbalancerAclEntry + +func (aclEntries *SLoadbalancerAclEntries) String() string { + return jsonutils.Marshal(aclEntries).String() +} +func (aclEntries *SLoadbalancerAclEntries) IsZero() bool { + if len([]*SLoadbalancerAclEntry(*aclEntries)) == 0 { + return true + } + return false +} + +func (aclEntries *SLoadbalancerAclEntries) Validate(data *jsonutils.JSONDict) error { + found := map[string]bool{} + for _, aclEntry := range *aclEntries { + if err := aclEntry.Validate(data); err != nil { + return err + } + if _, ok := found[aclEntry.Cidr]; ok { + // error so that the user has a chance to deal with comments + return fmt.Errorf("acl cidr duplicate %s", aclEntry.Cidr) + } + found[aclEntry.Cidr] = true + } + return nil +} + +type SLoadbalancerAclManager struct { + db.SSharableVirtualResourceBaseManager +} + +var LoadbalancerAclManager *SLoadbalancerAclManager + +func init() { + gotypes.RegisterSerializable(reflect.TypeOf(&SLoadbalancerAclEntries{}), func() gotypes.ISerializable { + return &SLoadbalancerAclEntries{} + }) + LoadbalancerAclManager = &SLoadbalancerAclManager{ + SSharableVirtualResourceBaseManager: db.NewSharableVirtualResourceBaseManager( + SLoadbalancerAcl{}, + "loadbalanceracls_tbl", + "loadbalanceracl", + "loadbalanceracls", + ), + } +} + +type SLoadbalancerAcl struct { + db.SSharableVirtualResourceBase + + AclEntries *SLoadbalancerAclEntries `list:"user" update:"user" create:"required"` +} + +func loadbalancerAclsValidateAclEntries(data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + aclEntries := SLoadbalancerAclEntries{} + aclEntriesV := validators.NewStructValidator("acl_entries", &aclEntries) + err := aclEntriesV.Validate(data) + if err != nil { + return nil, err + } + return data, nil +} + +func (man *SLoadbalancerAclManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + data, err := loadbalancerAclsValidateAclEntries(data) + if err != nil { + return nil, err + } + return man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data) +} + +func (lbacl *SLoadbalancerAcl) AllowPerformStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return false +} + +func (lbacl *SLoadbalancerAcl) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + data, err := loadbalancerAclsValidateAclEntries(data) + if err != nil { + return nil, err + } + return lbacl.SSharableVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data) +} + +func (lbacl *SLoadbalancerAcl) AllowPerformPatch(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) bool { + return lbacl.IsOwner(userCred) || userCred.IsSystemAdmin() +} + +// PerformPatch patches acl entries by adding then deleting the specified acls. +// This is intended mainly for command line operations. +func (lbacl *SLoadbalancerAcl) PerformPatch(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + aclEntries := gotypes.DeepCopy(*lbacl.AclEntries).(SLoadbalancerAclEntries) + { + adds := SLoadbalancerAclEntries{} + addsV := validators.NewStructValidator("adds", &adds) + addsV.Optional(true) + err := addsV.Validate(data) + if err != nil { + return nil, err + } + for _, add := range adds { + found := false + for _, aclEntry := range aclEntries { + if aclEntry.Cidr == add.Cidr { + found = true + aclEntry.Comment = add.Comment + break + } + } + if !found { + aclEntries = append(aclEntries, add) + } + } + } + { + dels := SLoadbalancerAclEntries{} + delsV := validators.NewStructValidator("dels", &dels) + delsV.Optional(true) + err := delsV.Validate(data) + if err != nil { + return nil, err + } + for _, del := range dels { + for i := len(aclEntries) - 1; i >= 0; i-- { + aclEntry := aclEntries[i] + if aclEntry.Cidr == del.Cidr { + aclEntries = append(aclEntries[:i], aclEntries[i+1:]...) + break + } + } + } + } + _, err := lbacl.GetModelManager().TableSpec().Update(lbacl, func() error { + lbacl.AclEntries = &aclEntries + return nil + }) + if err != nil { + return nil, err + } + return nil, nil +} + +func (lbacl *SLoadbalancerAcl) ValidateDeleteCondition(ctx context.Context) error { + man := LoadbalancerListenerManager + t := man.TableSpec().Instance() + pdF := t.Field("pending_deleted") + lbaclId := lbacl.Id + n := t.Query(). + Filter(sqlchemy.OR(sqlchemy.IsNull(pdF), sqlchemy.IsFalse(pdF))). + Equals("acl_id", lbaclId). + Count() + if n > 0 { + return fmt.Errorf("acl %s is still referred to by %d %s", + lbaclId, n, man.KeywordPlural()) + } + return nil +} + +func (lbacl *SLoadbalancerAcl) PreDelete(ctx context.Context, userCred mcclient.TokenCredential) { + lbacl.DoPendingDelete(ctx, userCred) +} + +func (lbacl *SLoadbalancerAcl) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + return nil +} diff --git a/pkg/compute/models/loadbalanceragents.go b/pkg/compute/models/loadbalanceragents.go new file mode 100644 index 0000000000..068f5bced2 --- /dev/null +++ b/pkg/compute/models/loadbalanceragents.go @@ -0,0 +1,457 @@ +package models + +import ( + "context" + "encoding/base64" + "reflect" + "text/template" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/gotypes" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/validators" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type SLoadbalancerAgentManager struct { + db.SStandaloneResourceBaseManager + SInfrastructureManager +} + +var LoadbalancerAgentManager *SLoadbalancerAgentManager + +func init() { + gotypes.RegisterSerializable(reflect.TypeOf(&SLoadbalancerAgentParams{}), func() gotypes.ISerializable { + return &SLoadbalancerAgentParams{} + }) + LoadbalancerAgentManager = &SLoadbalancerAgentManager{ + SStandaloneResourceBaseManager: db.NewStandaloneResourceBaseManager( + SLoadbalancerAgent{}, + "loadbalanceragents_tbl", + "loadbalanceragent", + "loadbalanceragents", + ), + } +} + +// TODO +// +// - scrub stale backends: Guests with deleted=1 +// - agent configuration params +// +type SLoadbalancerAgent struct { + db.SStandaloneResourceBase + SInfrastructure + + HbLastSeen time.Time `nullable:"true" list:"admin" update:"admin"` + HbTimeout int `nullable:"true" list:"admin" update:"admin" create:"optional" default:"3600"` + Params *SLoadbalancerAgentParams `create:"optional" get:"admin"` + + Loadbalancers time.Time `nullable:"true" list:"admin" update:"admin"` + LoadbalancerListeners time.Time `nullable:"true" list:"admin" update:"admin"` + LoadbalancerListenerRules time.Time `nullable:"true" list:"admin" update:"admin"` + LoadbalancerBackendGroups time.Time `nullable:"true" list:"admin" update:"admin"` + LoadbalancerBackends time.Time `nullable:"true" list:"admin" update:"admin"` + LoadbalancerAcls time.Time `nullable:"true" list:"admin" update:"admin"` + LoadbalancerCertificates time.Time `nullable:"true" list:"admin" update:"admin"` +} + +type SLoadbalancerAgentParamsVrrp struct { + Priority int + VirtualRouterId int + GarpMasterRefresh int + Preempt bool + Interface string + AdvertInt int + Pass string +} + +type SLoadbalancerAgentParamsHaproxy struct { + GlobalLog string + GlobalNbthread int + LogHttp bool + LogTcp bool + LogNormal bool +} + +type SLoadbalancerAgentParams struct { + KeepalivedConfTmpl string + HaproxyConfTmpl string + Vrrp SLoadbalancerAgentParamsVrrp + Haproxy SLoadbalancerAgentParamsHaproxy +} + +func (p *SLoadbalancerAgentParamsVrrp) Validate(data *jsonutils.JSONDict) error { + if len(p.Interface) == 0 || len(p.Interface) > 16 { + // TODO printable exclude white space + return httperrors.NewInputParameterError("invalid vrrp interface %q", p.Interface) + } + if len(p.Pass) == 0 || len(p.Pass) > 8 { + // TODO printable exclude white space + return httperrors.NewInputParameterError("invalid vrrp authentication pass size: %d, want [1,8]", len(p.Pass)) + } + if p.Priority < 1 || p.Priority > 255 { + return httperrors.NewInputParameterError("invalid vrrp priority %d: want [1,255]", p.Priority) + } + if p.VirtualRouterId < 1 || p.VirtualRouterId > 255 { + return httperrors.NewInputParameterError("invalid vrrp virtual_router_id %d: want [1,255]", p.VirtualRouterId) + } + return nil +} + +func (p *SLoadbalancerAgentParamsVrrp) initDefault(data *jsonutils.JSONDict) { + if !data.Contains("params", "vrrp", "advert_int") { + p.AdvertInt = 1 + } + if !data.Contains("params", "vrrp", "garp_master_refresh") { + p.GarpMasterRefresh = 27 + } + if !data.Contains("params", "vrrp", "pass") { + p.Pass = "YunionLB" + } +} + +func (p *SLoadbalancerAgentParamsHaproxy) Validate(data *jsonutils.JSONDict) error { + if p.GlobalNbthread < 1 { + p.GlobalNbthread = 1 + } + if p.GlobalNbthread > 64 { + // This is a limit imposed by haproxy and arch word size + p.GlobalNbthread = 64 + } + return nil +} + +func (p *SLoadbalancerAgentParamsHaproxy) initDefault(data *jsonutils.JSONDict) { + if !data.Contains("params", "haproxy", "global_nbthread") { + p.GlobalNbthread = 1 + } + if !data.Contains("params", "haproxy", "global_log") { + p.GlobalLog = "log /dev/log local0 info" + } + if !data.Contains("params", "haproxy", "log_http") { + p.LogHttp = true + } + if !data.Contains("params", "haproxy", "log_normal") { + p.LogNormal = true + } +} + +func (p *SLoadbalancerAgentParams) validateTmpl(k, s string) error { + d, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return httperrors.NewInputParameterError("%s: bad base64 encoded string: %s", k, err) + } + s = string(d) + _, err = template.New("").Parse(s) + if err != nil { + return httperrors.NewInputParameterError("%s: bad template: %s", k, err) + } + return nil +} + +func (p *SLoadbalancerAgentParams) initDefault(data *jsonutils.JSONDict) { + if p.KeepalivedConfTmpl == "" { + p.KeepalivedConfTmpl = loadbalancerKeepalivedConfTmplDefaultEncoded + } + if p.HaproxyConfTmpl == "" { + p.HaproxyConfTmpl = loadbalancerHaproxyConfTmplDefaultEncoded + } + p.Vrrp.initDefault(data) + p.Haproxy.initDefault(data) +} + +func (p *SLoadbalancerAgentParams) Validate(data *jsonutils.JSONDict) error { + p.initDefault(data) + if err := p.validateTmpl("keepalived_conf_tmpl", p.KeepalivedConfTmpl); err != nil { + return err + } + if err := p.validateTmpl("haproxy_conf_tmpl", p.HaproxyConfTmpl); err != nil { + return err + } + if err := p.Vrrp.Validate(data); err != nil { + return err + } + if err := p.Haproxy.Validate(data); err != nil { + return err + } + return nil +} + +func (p *SLoadbalancerAgentParams) String() string { + return jsonutils.Marshal(p).String() +} + +func (p *SLoadbalancerAgentParams) IsZero() bool { + if *p == (SLoadbalancerAgentParams{}) { + return true + } + return false +} + +func (man *SLoadbalancerAgentManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + { + keyV := map[string]validators.IValidator{ + "hb_timeout": validators.NewNonNegativeValidator("nb_timeout").Default(3600), + "params": validators.NewStructValidator("params", &SLoadbalancerAgentParams{}), + } + for _, v := range keyV { + if err := v.Validate(data); err != nil { + return nil, err + } + } + } + return man.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data) +} + +func (man *SLoadbalancerAgentManager) CleanPendingDeleteLoadbalancers(ctx context.Context, userCred mcclient.TokenCredential) { + agents := []SLoadbalancerAgent{} + { + // find active agents + err := man.Query().All(&agents) + if err != nil { + log.Errorf("query agents failed") + return + } + i := 0 + for _, agent := range agents { + if !agent.IsActive() { + continue + } + agents[i] = agent + i++ + } + agents = agents[:i] + } + men := map[string]db.IModelManager{ + "loadbalancers": LoadbalancerManager, + "loadbalancer_listeners": LoadbalancerListenerManager, + "loadbalancer_listener_rules": LoadbalancerListenerRuleManager, + "loadbalancer_backend_groups": LoadbalancerBackendGroupManager, + "loadbalancer_backends": LoadbalancerBackendManager, + "loadbalancer_acls": LoadbalancerAclManager, + "loadbalancer_certificates": LoadbalancerCertificateManager, + } + agentsData := jsonutils.Marshal(&agents).(*jsonutils.JSONArray) + for fieldName, man := range men { + keyPlural := man.KeywordPlural() + now := time.Now() + minT := now + if len(agents) > 0 { + // find min updated_at seen by these active agents + for i := 0; i < agentsData.Length(); i++ { + agentData, _ := agentsData.GetAt(i) + t, err := agentData.GetTime(fieldName) + if err != nil { + continue + } + if minT.After(t) { + minT = t + } + } + if minT.Equal(now) { + log.Warningf("%s: no agents has reported yet", keyPlural) + continue + } + } else { + // when no active agents exists, we are free to go + } + { + // find resources pending deleted before minT + q := man.Query().IsTrue("pending_deleted").LT("pending_deleted_at", minT) + rows, err := q.Rows() + if err != nil { + log.Errorf("%s: query pending_deleted_at < %s: %s", keyPlural, minT, err) + continue + } + m, err := db.NewModelObject(man) + if err != nil { + log.Errorf("%s: new model object failed: %s", keyPlural, err) + continue + } + mInitValue := reflect.Indirect(reflect.ValueOf(m)) + m, _ = db.NewModelObject(man) + for rows.Next() { + reflect.Indirect(reflect.ValueOf(m)).Set(mInitValue) + err := q.Row2Struct(rows, m) + if err != nil { + log.Errorf("%s: Row2Struct: %s", keyPlural, err) + continue + } + { + // find real delete method + rv := reflect.Indirect(reflect.ValueOf(m)) + baseRv := rv.FieldByName("SVirtualResourceBase") + if !baseRv.IsValid() { + baseRv = rv.FieldByName("SSharableVirtualResourceBase") + } + if !baseRv.IsValid() { + log.Errorf("%s: cannot find base resource field", keyPlural) + break // no need to try again + } + // now update deleted,deleted_at fields + realDeleteMethod := baseRv.Addr().MethodByName("Delete") + retRv := realDeleteMethod.Call([]reflect.Value{ + reflect.ValueOf(ctx), + reflect.ValueOf(userCred), + }) + err := retRv[0].Interface() + if !gotypes.IsNil(err) { + log.Errorf("%s: real delete failed: %s", keyPlural, err.(error)) + } + } + } + } + } +} + +func (man *SLoadbalancerAgentManager) AllowCreateItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return userCred.IsSystemAdmin() +} + +func (lbagent *SLoadbalancerAgent) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + keys := map[string]time.Time{ + "loadbalancers": lbagent.Loadbalancers, + "loadbalancer_listeners": lbagent.LoadbalancerListeners, + "loadbalancer_listener_rules": lbagent.LoadbalancerListenerRules, + "loadbalancer_backend_groups": lbagent.LoadbalancerBackendGroups, + "loadbalancer_backends": lbagent.LoadbalancerBackends, + "loadbalancer_acls": lbagent.LoadbalancerAcls, + "loadbalancer_certificates": lbagent.LoadbalancerCertificates, + } + for k, curValue := range keys { + if !data.Contains(k) { + continue + } + newValue, err := data.GetTime(k) + if err != nil { + return nil, httperrors.NewInputParameterError("%s: time error: %s", k, err) + } + if newValue.Before(curValue) { + // this is possible with objects deleted + data.Remove(k) + continue + } + if now := time.Now(); newValue.After(now) { + return nil, httperrors.NewInputParameterError("%s: new time is in the future: %s > %s", + k, newValue, now) + } + } + data.Set("hb_last_seen", jsonutils.NewTimeString(time.Now())) + return data, nil +} + +func (lbagent *SLoadbalancerAgent) AllowPerformHb(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) bool { + return userCred.IsSystemAdmin() +} + +func (lbagent *SLoadbalancerAgent) PerformHb(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + _, err := lbagent.GetModelManager().TableSpec().Update(lbagent, func() error { + lbagent.HbLastSeen = time.Now() + return nil + }) + if err != nil { + return nil, err + } + return nil, nil +} + +func (lbagent *SLoadbalancerAgent) IsActive() bool { + if lbagent.HbLastSeen.IsZero() { + return false + } + duration := time.Since(lbagent.HbLastSeen).Seconds() + if int(duration) >= lbagent.HbTimeout { + return false + } + return true +} + +func (lbagent *SLoadbalancerAgent) AllowPerformParamsPatch(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) bool { + return userCred.IsSystemAdmin() +} + +func (lbagent *SLoadbalancerAgent) PerformParamsPatch(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + params := gotypes.DeepCopy(*lbagent.Params).(SLoadbalancerAgentParams) + d := jsonutils.NewDict() + d.Set("params", data) + paramsV := validators.NewStructValidator("params", ¶ms) + if err := paramsV.Validate(d); err != nil { + return nil, err + } + { + _, err := lbagent.GetModelManager().TableSpec().Update(lbagent, func() error { + lbagent.Params = ¶ms + return nil + }) + if err != nil { + return nil, err + } + } + return nil, nil +} + +const ( + loadbalancerKeepalivedConfTmplDefault = ` +global_defs { + router_id {{ .agent.id }} + #vrrp_strict + vrrp_skip_check_adv_addr + enable_script_security +} + +vrrp_instance YunionLB { + interface {{ .vrrp.interface }} + virtual_router_id {{ .vrrp.virtual_router_id }} + authentication { + auth_type PASS + auth_pass {{ .vrrp.pass }} + } + priority {{ .vrrp.priority }} + advert_int {{ .vrrp.advert_int }} + garp_master_refresh {{ .vrrp.garp_master_refresh }} + {{ if .vrrp.preempt -}} preempt {{- else -}} nopreempt {{- end }} + virtual_ipaddress { + {{- printf "\n" }} + {{- range .vrrp.addresses }} {{ println . }} {{- end }} + {{- printf "\t" -}} + } +} +` + loadbalancerHaproxyConfTmplDefault = ` +global + maxconn 20480 + tune.ssl.default-dh-param 2048 + {{- println }} + {{- if .haproxy.global_stats_socket }} {{ println .haproxy.global_stats_socket }} {{- end }} + {{- if .haproxy.global_nbthread }} nbthread {{ println .haproxy.global_nbthread }} {{- end }} + {{- if .haproxy.global_log }} {{ println .haproxy.global_log }} {{- end }} + +defaults + timeout connect 10s + timeout client 60s + timeout server 60s + timeout tunnel 1h + {{- println }} + {{- if .haproxy.global_log }} {{ println "log global" }} {{- end }} + {{- if not .haproxy.log_normal }} {{ println "option dontlog-normal" }} {{- end }} + +listen stats + mode http + bind :778 + stats enable + stats hide-version + stats realm "Haproxy Statistics" + stats auth Yunion:LBStats + stats uri / +` +) + +var ( + loadbalancerKeepalivedConfTmplDefaultEncoded = base64.StdEncoding.EncodeToString([]byte(loadbalancerKeepalivedConfTmplDefault)) + loadbalancerHaproxyConfTmplDefaultEncoded = base64.StdEncoding.EncodeToString([]byte(loadbalancerHaproxyConfTmplDefault)) +) diff --git a/pkg/compute/models/loadbalancerbackendgroups.go b/pkg/compute/models/loadbalancerbackendgroups.go new file mode 100644 index 0000000000..676ee2d435 --- /dev/null +++ b/pkg/compute/models/loadbalancerbackendgroups.go @@ -0,0 +1,122 @@ +package models + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/cloudcommon/validators" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type SLoadbalancerBackendGroupManager struct { + db.SVirtualResourceBaseManager +} + +var LoadbalancerBackendGroupManager *SLoadbalancerBackendGroupManager + +func init() { + LoadbalancerBackendGroupManager = &SLoadbalancerBackendGroupManager{ + SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager( + SLoadbalancerBackendGroup{}, + "loadbalancerbackendgroups_tbl", + "loadbalancerbackendgroup", + "loadbalancerbackendgroups", + ), + } +} + +type SLoadbalancerBackendGroup struct { + db.SVirtualResourceBase + + LoadbalancerId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional"` +} + +func (man *SLoadbalancerBackendGroupManager) PreDeleteSubs(ctx context.Context, userCred mcclient.TokenCredential, q *sqlchemy.SQuery) { + subs := []SLoadbalancerBackendGroup{} + db.FetchModelObjects(man, q, &subs) + for _, sub := range subs { + sub.PreDelete(ctx, userCred) + } +} + +func (man *SLoadbalancerBackendGroupManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { + q, err := man.SVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query) + if err != nil { + return nil, err + } + userProjId := userCred.GetProjectId() + data := query.(*jsonutils.JSONDict) + { + lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", userProjId) + lbV.Optional(true) + q, err = lbV.QueryFilter(q, data) + if err != nil { + return nil, err + } + } + return q, nil +} + +func (man *SLoadbalancerBackendGroupManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", ownerProjId) + err := lbV.Validate(data) + if err != nil { + return nil, err + } + return man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data) +} + +func (lbbg *SLoadbalancerBackendGroup) GetLoadbalancer() *SLoadbalancer { + lb, _ := LoadbalancerManager.FetchById(lbbg.LoadbalancerId) + return lb.(*SLoadbalancer) +} + +func (lbbg *SLoadbalancerBackendGroup) AllowPerformStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return false +} + +func (lbbg *SLoadbalancerBackendGroup) ValidateDeleteCondition(ctx context.Context) error { + men := []db.IModelManager{ + LoadbalancerManager, + LoadbalancerListenerManager, + LoadbalancerListenerRuleManager, + } + lbbgId := lbbg.Id + for _, man := range men { + t := man.TableSpec().Instance() + pdF := t.Field("pending_deleted") + n := t.Query(). + Equals("backend_group_id", lbbgId). + Filter(sqlchemy.OR(sqlchemy.IsNull(pdF), sqlchemy.IsFalse(pdF))). + Count() + if n > 0 { + return fmt.Errorf("backend group %s is still referred to by %d %s", + lbbgId, n, man.KeywordPlural()) + } + } + return nil +} + +func (lbbg *SLoadbalancerBackendGroup) PreDelete(ctx context.Context, userCred mcclient.TokenCredential) { + lbbg.DoPendingDelete(ctx, userCred) + lbbg.PreDeleteSubs(ctx, userCred) +} + +func (lbbg *SLoadbalancerBackendGroup) PreDeleteSubs(ctx context.Context, userCred mcclient.TokenCredential) { + subMan := LoadbalancerBackendManager + ownerProjId := lbbg.GetOwnerProjectId() + + lockman.LockClass(ctx, subMan, ownerProjId) + defer lockman.ReleaseClass(ctx, subMan, ownerProjId) + q := subMan.Query().Equals("backend_group_id", lbbg.Id) + subMan.PreDeleteSubs(ctx, userCred, q) +} + +func (lbbg *SLoadbalancerBackendGroup) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + return nil +} diff --git a/pkg/compute/models/loadbalancerbackends.go b/pkg/compute/models/loadbalancerbackends.go new file mode 100644 index 0000000000..490fc50067 --- /dev/null +++ b/pkg/compute/models/loadbalancerbackends.go @@ -0,0 +1,194 @@ +package models + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/validators" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type SLoadbalancerBackendManager struct { + db.SVirtualResourceBaseManager +} + +var LoadbalancerBackendManager *SLoadbalancerBackendManager + +func init() { + LoadbalancerBackendManager = &SLoadbalancerBackendManager{ + SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager( + SLoadbalancerBackend{}, + "loadbalancerbackends_tbl", + "loadbalancerbackend", + "loadbalancerbackends", + ), + } +} + +type SLoadbalancerBackend struct { + db.SVirtualResourceBase + + BackendGroupId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional"` + BackendId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional"` + BackendType string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional"` + Weight int `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"` + Address string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional"` + Port int `nullable:"false" list:"user" create:"required" update:"user"` +} + +func (man *SLoadbalancerBackendManager) PreDeleteSubs(ctx context.Context, userCred mcclient.TokenCredential, q *sqlchemy.SQuery) { + subs := []SLoadbalancerBackend{} + db.FetchModelObjects(man, q, &subs) + for _, sub := range subs { + sub.PreDelete(ctx, userCred) + } +} + +func (man *SLoadbalancerBackendManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { + q, err := man.SVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query) + if err != nil { + return nil, err + } + userProjId := userCred.GetProjectId() + data := query.(*jsonutils.JSONDict) + { + backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", userProjId) + backendGroupV.Optional(true) + q, err = backendGroupV.QueryFilter(q, data) + if err != nil { + return nil, err + } + } + { + // NOTE extend this when new backend_type was added + backendV := validators.NewModelIdOrNameValidator("backend", "server", userProjId) + backendV.Optional(true) + q, err = backendV.QueryFilter(q, data) + if err != nil { + return nil, err + } + } + return q, nil +} + +func (man *SLoadbalancerBackendManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", ownerProjId) + backendTypeV := validators.NewStringChoicesValidator("backend_type", LB_BACKEND_TYPES) + keyV := map[string]validators.IValidator{ + "backend_group": backendGroupV, + "backend_type": backendTypeV, + "weight": validators.NewRangeValidator("weight", 1, 256).Default(1), + "port": validators.NewPortValidator("port"), + } + for _, v := range keyV { + if err := v.Validate(data); err != nil { + return nil, err + } + } + backendGroup := backendGroupV.Model.(*SLoadbalancerBackendGroup) + backendType := backendTypeV.Value + var baseName string + switch backendType { + case LB_BACKEND_GUEST: + backendV := validators.NewModelIdOrNameValidator("backend", "server", ownerProjId) + err := backendV.Validate(data) + if err != nil { + return nil, err + } + guest := backendV.Model.(*SGuest) + { + // guest zone must match that of loadbalancer's + host := guest.GetHost() + if host == nil { + return nil, fmt.Errorf("error getting host of guest %s", guest.GetId()) + } + lb := backendGroup.GetLoadbalancer() + if lb == nil { + return nil, fmt.Errorf("error loadbalancer of backend group %s", backendGroup.GetId()) + } + if host.ZoneId != lb.ZoneId { + return nil, fmt.Errorf("host zone (%s) != loadbalancer %q zone (%s)", + host.Name, host.ZoneId, lb.Name, lb.ZoneId) + } + } + { + // get guest intranet address + // + // NOTE add address hint (cidr) if needed + gns := guest.GetNetworks() + if len(gns) == 0 { + return nil, fmt.Errorf("guest %s has no network attached", guest.GetId()) + } + var address string + for _, gn := range gns { + if !gn.IsExit() { + address = gn.IpAddr + break + } + } + if len(address) == 0 { + return nil, fmt.Errorf("guest %s has no intranet address attached", guest.GetId()) + } + data.Set("address", jsonutils.NewString(address)) + } + baseName = guest.Name + case LB_BACKEND_HOST: + if !userCred.IsSystemAdmin() { + return nil, fmt.Errorf("only sysadmin can specify host as backend") + } + backendV := validators.NewModelIdOrNameValidator("backend", "host", userCred.GetProjectId()) + err := backendV.Validate(data) + if err != nil { + return nil, err + } + host := backendV.Model.(*SHost) + { + if len(host.AccessIp) == 0 { + return nil, fmt.Errorf("host %s has no access ip", host.GetId()) + } + data.Set("address", jsonutils.NewString(host.AccessIp)) + } + baseName = host.Name + default: + return nil, fmt.Errorf("internal error: unexpected backend type %s", backendType) + } + // name it + // + // NOTE it's okay for name to be not unique. + // + // - Mix in loadbalancer name if needed + // - Use name from input query + name := fmt.Sprintf("%s-%s-%s", backendGroup.Name, backendType, baseName) + data.Set("name", jsonutils.NewString(name)) + return man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data) +} + +func (lbb *SLoadbalancerBackend) AllowPerformStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return false +} + +func (lbb *SLoadbalancerBackend) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + keyV := map[string]validators.IValidator{ + "weight": validators.NewRangeValidator("weight", 1, 256), + "port": validators.NewPortValidator("port"), + } + for _, v := range keyV { + v.Optional(true) + if err := v.Validate(data); err != nil { + return nil, err + } + } + return lbb.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data) +} + +func (lbb *SLoadbalancerBackend) PreDelete(ctx context.Context, userCred mcclient.TokenCredential) { + lbb.DoPendingDelete(ctx, userCred) +} + +func (lbb *SLoadbalancerBackend) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + return nil +} diff --git a/pkg/compute/models/loadbalancercertificates.go b/pkg/compute/models/loadbalancercertificates.go new file mode 100644 index 0000000000..d1603881a6 --- /dev/null +++ b/pkg/compute/models/loadbalancercertificates.go @@ -0,0 +1,154 @@ +package models + +import ( + "context" + "fmt" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/validators" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type SLoadbalancerCertificateManager struct { + db.SVirtualResourceBaseManager +} + +var LoadbalancerCertificateManager *SLoadbalancerCertificateManager + +func init() { + LoadbalancerCertificateManager = &SLoadbalancerCertificateManager{ + SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager( + SLoadbalancerCertificate{}, + "loadbalancercertificates_tbl", + "loadbalancercertificate", + "loadbalancercertificates", + ), + } +} + +// TODO +// +// - notify users of cert expiration +// - ca info: self-signed, public ca +type SLoadbalancerCertificate struct { + db.SVirtualResourceBase + + Certificate string `create:"required" list:"admin" update:"user"` + PrivateKey string `create:"required" list:"admin" update:"user"` + + // derived attributes + PublicKeyAlgorithm string `create:"optional" list:user update:"user"` + PublicKeyBitLen int `create:"optional" list:user update:"user"` + SignatureAlgorithm string `create:"optional" list:user update:"user"` + FingerprintSha256 string `create:"optional" list:user update:"user"` + NotBefore time.Time `create:"optional" list:user update:"user"` + NotAfter time.Time `create:"optional" list:user update:"user"` + CommonName string `create:"optional" list:user update:"user"` + SubjectAlternativeNames string `create:"optional" list:user update:"user"` +} + +func (man *SLoadbalancerCertificateManager) PreDeleteSubs(ctx context.Context, userCred mcclient.TokenCredential, q *sqlchemy.SQuery) { + subs := []SLoadbalancerCertificate{} + db.FetchModelObjects(man, q, &subs) + for _, sub := range subs { + sub.PreDelete(ctx, userCred) + } +} + +func (man *SLoadbalancerCertificateManager) validateCertKey(ctx context.Context, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + certV := validators.NewCertificateValidator("certificate") + pkeyV := validators.NewPrivateKeyValidator("private_key") + keyV := map[string]validators.IValidator{ + "certificate": certV, + "private_key": pkeyV, + } + for _, v := range keyV { + if err := v.Validate(data); err != nil { + return nil, err + } + } + cert := certV.Certificates[0] + certPubKeyAlgo := cert.PublicKeyAlgorithm.String() + if !LB_TLS_CERT_PUBKEY_ALGOS.Has(certPubKeyAlgo) { + return nil, httperrors.NewInputParameterError("invalid cert pubkey algorithm: %s, want %s", + certPubKeyAlgo, LB_TLS_CERT_PUBKEY_ALGOS.String()) + } + err := pkeyV.MatchCertificate(cert) + if err != nil { + return nil, err + } + // NOTE subject alternative names also includes email, url, ip addresses, + // but we ignore them here. + // + // NOTE we use white space to separate names + data.Set("common_name", jsonutils.NewString(cert.Subject.CommonName)) + data.Set("subject_alternative_names", jsonutils.NewString(strings.Join(cert.DNSNames, " "))) + + data.Set("not_before", jsonutils.NewTimeString(cert.NotBefore)) + data.Set("not_after", jsonutils.NewTimeString(cert.NotAfter)) + data.Set("public_key_algorithm", jsonutils.NewString(certPubKeyAlgo)) + data.Set("public_key_bit_len", jsonutils.NewInt(int64(certV.PublicKeyBitLen()))) + data.Set("signature_algorithm", jsonutils.NewString(cert.SignatureAlgorithm.String())) + data.Set("fingerprint_sha256", jsonutils.NewString(certV.FingerprintSha256String())) + return data, nil +} + +func (man *SLoadbalancerCertificateManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + data, err := man.validateCertKey(ctx, data) + if err != nil { + return nil, err + } + return man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data) +} + +func (lbcert *SLoadbalancerCertificate) AllowPerformStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return false +} + +func (lbcert *SLoadbalancerCertificate) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + if !data.Contains("certificate") { + data.Set("certificate", jsonutils.NewString(lbcert.Certificate)) + } + if !data.Contains("private_key") { + data.Set("private_key", jsonutils.NewString(lbcert.PrivateKey)) + } + data, err := LoadbalancerCertificateManager.validateCertKey(ctx, data) + if err != nil { + return nil, err + } + return lbcert.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data) +} + +func (lbcert *SLoadbalancerCertificate) ValidateDeleteCondition(ctx context.Context) error { + men := []db.IModelManager{ + LoadbalancerListenerManager, + } + lbcertId := lbcert.Id + for _, man := range men { + t := man.TableSpec().Instance() + pdF := t.Field("pending_deleted") + n := t.Query(). + Equals("certificate_id", lbcertId). + Filter(sqlchemy.OR(sqlchemy.IsNull(pdF), sqlchemy.IsFalse(pdF))). + Count() + if n > 0 { + return fmt.Errorf("certificate %s is still referred to by %d %s", + lbcertId, n, man.KeywordPlural()) + } + } + return nil +} + +func (lbcert *SLoadbalancerCertificate) PreDelete(ctx context.Context, userCred mcclient.TokenCredential) { + lbcert.DoPendingDelete(ctx, userCred) +} + +func (lbcert *SLoadbalancerCertificate) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + return nil +} diff --git a/pkg/compute/models/loadbalancerlistenerrules.go b/pkg/compute/models/loadbalancerlistenerrules.go new file mode 100644 index 0000000000..ed6d75353d --- /dev/null +++ b/pkg/compute/models/loadbalancerlistenerrules.go @@ -0,0 +1,145 @@ +package models + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/validators" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type SLoadbalancerListenerRuleManager struct { + db.SVirtualResourceBaseManager +} + +var LoadbalancerListenerRuleManager *SLoadbalancerListenerRuleManager + +func init() { + LoadbalancerListenerRuleManager = &SLoadbalancerListenerRuleManager{ + SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager( + SLoadbalancerListenerRule{}, + "loadbalancerlistenerrules_tbl", + "loadbalancerlistenerrule", + "loadbalancerlistenerrules", + ), + } +} + +type SLoadbalancerListenerRule struct { + db.SVirtualResourceBase + + ListenerId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional"` + BackendGroupId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"` + + Domain string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"optional"` + Path string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"optional"` +} + +func loadbalancerListenerRuleCheckUniqueness(ctx context.Context, lbls *SLoadbalancerListener, domain, path string) error { + man := db.GetModelManager("loadbalancerlistenerrule") + if man == nil { + return fmt.Errorf("getting loadbalancer rule manager failed") + } + q := man.Query(). + Equals("listener_id", lbls.Id). + Equals("domain", domain). + Equals("path", path) + var lblsr SLoadbalancerListenerRule + q.First(&lblsr) + if len(lblsr.Id) > 0 { + return fmt.Errorf("rule %s/%s already occupied by rule %s(%s)", domain, path, lblsr.Name, lblsr.Id) + } + return nil +} + +func (man *SLoadbalancerListenerRuleManager) PreDeleteSubs(ctx context.Context, userCred mcclient.TokenCredential, q *sqlchemy.SQuery) { + subs := []SLoadbalancerListenerRule{} + db.FetchModelObjects(man, q, &subs) + for _, sub := range subs { + sub.PreDelete(ctx, userCred) + } +} + +func (man *SLoadbalancerListenerRuleManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { + q, err := man.SVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query) + if err != nil { + return nil, err + } + userProjId := userCred.GetProjectId() + data := query.(*jsonutils.JSONDict) + { + listenerV := validators.NewModelIdOrNameValidator("listener", "listener", userProjId) + listenerV.Optional(true) + q, err = listenerV.QueryFilter(q, data) + if err != nil { + return nil, err + } + } + { + backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", userProjId) + backendGroupV.Optional(true) + q, err = backendGroupV.QueryFilter(q, data) + if err != nil { + return nil, err + } + } + return q, nil +} + +func (man *SLoadbalancerListenerRuleManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + listenerV := validators.NewModelIdOrNameValidator("listener", "loadbalancerlistener", ownerProjId) + backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", ownerProjId) + domainV := validators.NewDomainNameValidator("domain") + pathV := validators.NewURLPathValidator("path") + keyV := map[string]validators.IValidator{ + "status": validators.NewStringChoicesValidator("status", LB_STATUS_SPEC).Default(LB_STATUS_ENABLED), + + "listener": listenerV, + "backend_group": backendGroupV, + "domain": domainV.AllowEmpty(true).Default(""), + "path": pathV.Default(""), + } + for _, v := range keyV { + if err := v.Validate(data); err != nil { + return nil, err + } + } + listener := listenerV.Model.(*SLoadbalancerListener) + listenerType := listener.ListenerType + if listenerType != LB_LISTENER_TYPE_HTTP && listenerType != LB_LISTENER_TYPE_HTTPS { + return nil, fmt.Errorf("listener type must be http/https, got %s", listenerType) + } + err := loadbalancerListenerRuleCheckUniqueness(ctx, listener, domainV.Value, pathV.Value) + if err != nil { + return nil, err + } + return man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data) +} + +func (lbr *SLoadbalancerListenerRule) AllowPerformStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return lbr.IsOwner(userCred) || userCred.IsSystemAdmin() +} + +func (lbr *SLoadbalancerListenerRule) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", lbr.GetOwnerProjectId()).Optional(true) + err := backendGroupV.Validate(data) + if err != nil { + return nil, err + } + return lbr.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data) +} + +func (lbr *SLoadbalancerListenerRule) PreDelete(ctx context.Context, userCred mcclient.TokenCredential) { + lbr.SetStatus(userCred, LB_STATUS_DISABLED, "preDelete") + lbr.DoPendingDelete(ctx, userCred) +} + +func (lbr *SLoadbalancerListenerRule) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + return nil +} + +// Delete, Update diff --git a/pkg/compute/models/loadbalancerlisteners.go b/pkg/compute/models/loadbalancerlisteners.go new file mode 100644 index 0000000000..a72f3ea187 --- /dev/null +++ b/pkg/compute/models/loadbalancerlisteners.go @@ -0,0 +1,360 @@ +package models + +import ( + "context" + "fmt" + "regexp" + + "yunion.io/x/jsonutils" + "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/cloudcommon/validators" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type SLoadbalancerListenerManager struct { + db.SVirtualResourceBaseManager +} + +var LoadbalancerListenerManager *SLoadbalancerListenerManager + +func init() { + LoadbalancerListenerManager = &SLoadbalancerListenerManager{ + SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager( + SLoadbalancerListener{}, + "loadbalancerlisteners_tbl", + "loadbalancerlistener", + "loadbalancerlisteners", + ), + } +} + +type SLoadbalancerTCPListener struct{} +type SLoadbalancerUDPListener struct{} + +// TODO sensible default for knobs +type SLoadbalancerHTTPListener struct { + StickySession string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"` + StickySessionType string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"` + StickySessionCookie string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"` + StickySessionCookieTimeout int `nullable:"false" list:"user" create:"optional" update:"user"` + + //XForwardedForSLBIP bool `nullable:"false" list:"user" create:"optional"` + //XForwardedForSLBID bool `nullable:"false" list:"user" create:"optional"` + XForwardedFor bool `nullable:"false" list:"user" create:"optional" update:"user"` + Gzip bool `nullable:"false" list:"user" create:"optional" update:"user"` +} + +// TODO +// +// - CACertificate string +// - Certificate2Id // multiple certificates for rsa, ecdsa +// - Use certificate for tcp listener +// - Customize ciphers? +type SLoadbalancerHTTPSListener struct { + CertificateId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional"` + TLSCipherPolicy string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional"` + EnableHttp2 bool `create:"optional" list:"user"` +} + +type SLoadbalancerListener struct { + db.SVirtualResourceBase + + LoadbalancerId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional"` + ListenerType string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"required"` + ListenerPort int `nullable:"false" list:"user" create:"required"` + BackendGroupId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"` + + Bandwidth int `nullable:"false" list:"user" create:"optional" update:"user"` + Scheduler string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user"` + + ClientRequestTimeout int `nullable:"false" list:"user" create:"optional" update:"user"` + ClientIdleTimeout int `nullable:"false" list:"user" create:"optional" update:"user"` + BackendConnectTimeout int `nullable:"false" list:"user" create:"optional" update:"user"` + BackendIdleTimeout int `nullable:"false" list:"user" create:"optional" update:"user"` + + AclStatus string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"` + AclType string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"` + AclId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"` + + HealthCheck string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"` + HealthCheckType string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"` + + HealthCheckDomain string `charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"` + HealthCheckURI string `charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"` + HealthCheckHttpCode string `charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"` + + HealthCheckRise int `nullable:"false" list:"user" create:"optional" update:"user"` + HealthCheckFall int `nullable:"false" list:"user" create:"optional" update:"user"` + HealthCheckTimeout int `nullable:"false" list:"user" create:"optional" update:"user"` + HealthCheckInterval int `nullable:"false" list:"user" create:"optional" update:"user"` + + HealthCheckReq string `list:"user" create:"optional" update:"user"` + HealthCheckExp string `list:"user" create:"optional" update:"user"` + + SLoadbalancerTCPListener + SLoadbalancerUDPListener + SLoadbalancerHTTPListener + SLoadbalancerHTTPSListener +} + +func (man *SLoadbalancerListenerManager) checkListenerUniqueness(ctx context.Context, lb *SLoadbalancer, listenerType string, listenerPort int64) error { + q := man.Query(). + Equals("loadbalancer_id", lb.Id). + Equals("listener_port", listenerPort) + switch listenerType { + case LB_LISTENER_TYPE_TCP, LB_LISTENER_TYPE_HTTP, LB_LISTENER_TYPE_HTTPS: + q = q.NotEquals("listener_type", LB_LISTENER_TYPE_UDP) + case LB_LISTENER_TYPE_UDP: + q = q.Equals("listener_type", LB_LISTENER_TYPE_UDP) + default: + return fmt.Errorf("unexpected listener type: %s", listenerType) + } + var listener SLoadbalancerListener + q.First(&listener) + if len(listener.Id) > 0 { + return fmt.Errorf("%s listener port %d is already taken by listener %s", + listenerType, listenerPort, listener.Id) + } + return nil +} + +func (man *SLoadbalancerListenerManager) PreDeleteSubs(ctx context.Context, userCred mcclient.TokenCredential, q *sqlchemy.SQuery) { + subs := []SLoadbalancerListener{} + db.FetchModelObjects(man, q, &subs) + for _, sub := range subs { + sub.PreDelete(ctx, userCred) + } +} + +func (man *SLoadbalancerListenerManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { + q, err := man.SVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query) + if err != nil { + return nil, err + } + userProjId := userCred.GetProjectId() + data := query.(*jsonutils.JSONDict) + { + lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", userProjId) + lbV.Optional(true) + q, err = lbV.QueryFilter(q, data) + if err != nil { + return nil, err + } + } + { + backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", userProjId) + backendGroupV.Optional(true) + q, err = backendGroupV.QueryFilter(q, data) + if err != nil { + return nil, err + } + } + { + aclV := validators.NewModelIdOrNameValidator("acl", "loadbalanceracl", userProjId) + aclV.Optional(true) + q, err = aclV.QueryFilter(q, data) + if err != nil { + return nil, err + } + } + return q, nil +} + +func (man *SLoadbalancerListenerManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", ownerProjId) + listenerTypeV := validators.NewStringChoicesValidator("listener_type", LB_LISTENER_TYPES) + listenerPortV := validators.NewPortValidator("listener_port") + backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", ownerProjId) + aclStatusV := validators.NewStringChoicesValidator("acl_status", LB_BOOL_VALUES) + aclTypeV := validators.NewStringChoicesValidator("acl_type", LB_ACL_TYPES) + aclV := validators.NewModelIdOrNameValidator("acl", "loadbalanceracl", ownerProjId) + keyV := map[string]validators.IValidator{ + "status": validators.NewStringChoicesValidator("status", LB_STATUS_SPEC).Default(LB_STATUS_ENABLED), + + "loadbalancer": lbV, + "listener_type": listenerTypeV, + "listener_port": listenerPortV, + "backend_group": backendGroupV.Optional(true), + + "acl_status": aclStatusV.Default(LB_BOOL_OFF), + "acl_type": aclTypeV.Optional(true), + "acl": aclV.Optional(true), + + "scheduler": validators.NewStringChoicesValidator("scheduler", LB_SCHEDULER_TYPES), + "bandwidth": validators.NewRangeValidator("bandwidth", 0, 10000).Optional(true), + + "client_request_timeout": validators.NewRangeValidator("client_request_timeout", 0, 600).Default(10), + "client_idle_timeout": validators.NewRangeValidator("client_idle_timeout", 0, 600).Default(90), + "backend_connect_timeout": validators.NewRangeValidator("backend_connect_timeout", 0, 180).Default(5), + "backend_idle_timeout": validators.NewRangeValidator("backend_idle_timeout", 0, 600).Default(90), + + "sticky_session": validators.NewStringChoicesValidator("sticky_session", LB_BOOL_VALUES).Default(LB_BOOL_OFF), + "sticky_session_type": validators.NewStringChoicesValidator("sticky_session_type", LB_STICKY_SESSION_TYPES).Default(LB_STICKY_SESSION_TYPE_INSERT), + "sticky_session_cookie": validators.NewRegexpValidator("sticky_session_cookie", regexp.MustCompile(`\w+`)).Optional(true), + "sticky_session_cookie_timeout": validators.NewNonNegativeValidator("sticky_session_cookie_timeout").Optional(true), + + "x_forwarded_for": validators.NewBoolValidator("x_forwarded_for").Default(true), + "gzip": validators.NewBoolValidator("gzip").Default(false), + } + for _, v := range keyV { + if err := v.Validate(data); err != nil { + return nil, err + } + } + { + lb := lbV.Model.(*SLoadbalancer) + listenerPort := listenerPortV.Value + listenerType := listenerTypeV.Value + err := man.checkListenerUniqueness(ctx, lb, listenerType, listenerPort) + if err != nil { + // duplicate? + return nil, err + } + } + { + if listenerTypeV.Value == LB_LISTENER_TYPE_HTTPS { + certV := validators.NewModelIdOrNameValidator("certificate", "loadbalancercertificate", ownerProjId) + tlsCipherPolicyV := validators.NewStringChoicesValidator("tls_cipher_policy", LB_TLS_CIPHER_POLICIES).Default(LB_TLS_CIPHER_POLICY_1_2) + httpsV := map[string]validators.IValidator{ + "certificate": certV, + "tls_cipher_policy": tlsCipherPolicyV, + "enable_http2": validators.NewBoolValidator("enable_http2").Default(true), + } + for _, v := range httpsV { + if err := v.Validate(data); err != nil { + return nil, err + } + } + } + } + { + // health check default depends on input parameters + checkTypeV := man.checkTypeV(listenerTypeV.Value) + keyVHealth := map[string]validators.IValidator{ + "health_check": validators.NewStringChoicesValidator("health_check", LB_BOOL_VALUES).Default(LB_BOOL_ON), + "health_check_type": checkTypeV, + + "health_check_domain": validators.NewDomainNameValidator("domain").AllowEmpty(true).Default(""), + "health_check_path": validators.NewURLPathValidator("path").Default(""), + "health_check_http_code": validators.NewStringMultiChoicesValidator("health_check_http_code", LB_HEALTH_CHECK_HTTP_CODES).Sep(",").Default(LB_HEALTH_CHECK_HTTP_CODE_DEFAULT), + + "health_check_rise": validators.NewRangeValidator("health_check_rise", 1, 1000).Default(3), + "health_check_fall": validators.NewRangeValidator("health_check_fall", 1, 1000).Default(3), + "health_check_timeout": validators.NewRangeValidator("health_check_timeout", 1, 300).Default(5), + "health_check_interval": validators.NewRangeValidator("health_check_interval", 1, 1000).Default(5), + } + for _, v := range keyVHealth { + if err := v.Validate(data); err != nil { + return nil, err + } + } + } + { + if aclStatusV.Value == LB_BOOL_ON { + acl := aclV.Model.(*SLoadbalancerAcl) + if acl == nil { + return nil, fmt.Errorf("missing acl") + } + if len(aclTypeV.Value) == 0 { + return nil, fmt.Errorf("missing acl_type") + } + } + } + return man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data) +} + +func (man *SLoadbalancerListenerManager) checkTypeV(listenerType string) validators.IValidator { + switch listenerType { + case LB_LISTENER_TYPE_HTTP, LB_LISTENER_TYPE_HTTPS: + return validators.NewStringChoicesValidator("health_check_type", LB_HEALTH_CHECK_TYPES_TCP).Default(LB_HEALTH_CHECK_HTTP) + case LB_LISTENER_TYPE_TCP: + return validators.NewStringChoicesValidator("health_check_type", LB_HEALTH_CHECK_TYPES_TCP).Default(LB_HEALTH_CHECK_TCP) + case LB_LISTENER_TYPE_UDP: + return validators.NewStringChoicesValidator("health_check_type", LB_HEALTH_CHECK_TYPES_UDP).Default(LB_HEALTH_CHECK_UDP) + } + // should it happen, panic then + return nil +} + +func (lblis *SLoadbalancerListener) AllowPerformStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return lblis.IsOwner(userCred) || userCred.IsSystemAdmin() +} + +func (lblis *SLoadbalancerListener) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + ownerProjId := lblis.GetOwnerProjectId() + backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", ownerProjId) + aclStatusV := validators.NewStringChoicesValidator("acl_status", LB_BOOL_VALUES) + aclTypeV := validators.NewStringChoicesValidator("acl_type", LB_ACL_TYPES) + aclV := validators.NewModelIdOrNameValidator("acl", "loadbalanceracl", ownerProjId) + certV := validators.NewModelIdOrNameValidator("certificate", "loadbalancercertificate", ownerProjId) + tlsCipherPolicyV := validators.NewStringChoicesValidator("tls_cipher_policy", LB_TLS_CIPHER_POLICIES).Default(LB_TLS_CIPHER_POLICY_1_2) + keyV := map[string]validators.IValidator{ + "backend_group": backendGroupV, + + "acl_status": aclStatusV, + "acl_type": aclTypeV, + "acl": aclV, + + "scheduler": validators.NewStringChoicesValidator("scheduler", LB_SCHEDULER_TYPES), + "bandwidth": validators.NewRangeValidator("bandwidth", 0, 10000), + + "client_request_timeout": validators.NewRangeValidator("client_request_timeout", 0, 600), + "client_idle_timeout": validators.NewRangeValidator("client_idle_timeout", 0, 600), + "backend_connect_timeout": validators.NewRangeValidator("backend_connect_timeout", 0, 180), + "backend_idle_timeout": validators.NewRangeValidator("backend_idle_timeout", 0, 600), + + "sticky_session": validators.NewStringChoicesValidator("sticky_session", LB_BOOL_VALUES), + "sticky_session_type": validators.NewStringChoicesValidator("sticky_session_type", LB_STICKY_SESSION_TYPES), + "sticky_session_cookie": validators.NewRegexpValidator("sticky_session_cookie", regexp.MustCompile(`\w+`)), + "sticky_session_cookie_timeout": validators.NewNonNegativeValidator("sticky_session_cookie_timeout"), + + "health_check": validators.NewStringChoicesValidator("health_check", LB_BOOL_VALUES), + "health_check_type": LoadbalancerListenerManager.checkTypeV(lblis.ListenerType), + + "health_check_domain": validators.NewDomainNameValidator("domain").AllowEmpty(true), + "health_check_path": validators.NewURLPathValidator("path"), + "health_check_http_code": validators.NewStringMultiChoicesValidator("health_check_http_code", LB_HEALTH_CHECK_HTTP_CODES).Sep(","), + + "health_check_rise": validators.NewRangeValidator("health_check_rise", 1, 1000), + "health_check_fall": validators.NewRangeValidator("health_check_fall", 1, 1000), + "health_check_timeout": validators.NewRangeValidator("health_check_timeout", 1, 300), + "health_check_interval": validators.NewRangeValidator("health_check_interval", 1, 1000), + + "x_forwarded_for": validators.NewBoolValidator("x_forwarded_for"), + "gzip": validators.NewBoolValidator("gzip"), + + "certificate": certV, + "tls_cipher_policy": tlsCipherPolicyV, + "enable_http2": validators.NewBoolValidator("enable_http2").Default(true), + } + for _, v := range keyV { + v.Optional(true) + if err := v.Validate(data); err != nil { + return nil, err + } + } + return lblis.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data) +} + +func (lblis *SLoadbalancerListener) PreDelete(ctx context.Context, userCred mcclient.TokenCredential) { + lblis.SetStatus(userCred, LB_STATUS_DISABLED, "preDelete") + lblis.DoPendingDelete(ctx, userCred) + lblis.PreDeleteSubs(ctx, userCred) +} + +func (lblis *SLoadbalancerListener) PreDeleteSubs(ctx context.Context, userCred mcclient.TokenCredential) { + subMan := LoadbalancerListenerRuleManager + ownerProjId := lblis.GetOwnerProjectId() + + lockman.LockClass(ctx, subMan, ownerProjId) + defer lockman.ReleaseClass(ctx, subMan, ownerProjId) + q := subMan.Query().Equals("listener_id", lblis.Id) + subMan.PreDeleteSubs(ctx, userCred, q) +} + +func (lblis *SLoadbalancerListener) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + return nil +} diff --git a/pkg/compute/models/loadbalancernetworks.go b/pkg/compute/models/loadbalancernetworks.go index 29a5c279d0..30e533c467 100644 --- a/pkg/compute/models/loadbalancernetworks.go +++ b/pkg/compute/models/loadbalancernetworks.go @@ -4,10 +4,11 @@ import ( "context" "fmt" + "yunion.io/x/pkg/util/regutils" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/pkg/util/regutils" ) type SLoadbalancernetworkManager struct { diff --git a/pkg/compute/models/loadbalancers.go b/pkg/compute/models/loadbalancers.go new file mode 100644 index 0000000000..c4a766dfc1 --- /dev/null +++ b/pkg/compute/models/loadbalancers.go @@ -0,0 +1,195 @@ +package models + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/cloudcommon/validators" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type SLoadbalancerManager struct { + db.SVirtualResourceBaseManager +} + +var LoadbalancerManager *SLoadbalancerManager + +func init() { + LoadbalancerManager = &SLoadbalancerManager{ + SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager( + SLoadbalancer{}, + "loadbalancers_tbl", + "loadbalancer", + "loadbalancers", + ), + } +} + +// TODO build errors on pkg/httperrors/errors.go +// NewGetManagerError +// NewMissingArgumentError +// NewInvalidArgumentError +// +// TODO ZoneId or RegionId +// bandwidth +// scheduler +// +// TODO update backendgroupid +type SLoadbalancer struct { + db.SVirtualResourceBase + + Address string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"optional"` + AddressType string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"optional"` + NetworkType string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"optional"` + NetworkId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required"` + ZoneId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional"` + + BackendGroupId string `width:"36" charset:"ascii" nullable:"false" list:"user" update:"user" update:"user"` +} + +func (man *SLoadbalancerManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { + q, err := man.SVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query) + if err != nil { + return nil, err + } + userProjId := userCred.GetProjectId() + data := query.(*jsonutils.JSONDict) + { + networkV := validators.NewModelIdOrNameValidator("network", "network", userProjId) + networkV.Optional(true) + q, err = networkV.QueryFilter(q, data) + if err != nil { + return nil, err + } + } + { + zoneV := validators.NewModelIdOrNameValidator("zone", "zone", userProjId) + zoneV.Optional(true) + q, err = zoneV.QueryFilter(q, data) + if err != nil { + return nil, err + } + } + return q, nil +} + +func (man *SLoadbalancerManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + networkV := validators.NewModelIdOrNameValidator("network", "network", ownerProjId) + addressV := validators.NewIPv4AddrValidator("address").Optional(true) + { + keyV := map[string]validators.IValidator{ + "status": validators.NewStringChoicesValidator("status", LB_STATUS_SPEC).Default(LB_STATUS_ENABLED), + + "address": addressV, + "network": networkV, + } + for _, v := range keyV { + if err := v.Validate(data); err != nil { + return nil, err + } + } + } + { + network := networkV.Model.(*SNetwork) + if wire := network.GetWire(); wire == nil { + return nil, fmt.Errorf("getting wire failed") + } else if zone := wire.GetZone(); zone == nil { + return nil, fmt.Errorf("getting zone failed") + } else { + data.Set("zone_id", jsonutils.NewString(zone.GetId())) + } + // TODO validate network is of classic type + data.Set("network_type", jsonutils.NewString(LB_NETWORK_TYPE_CLASSIC)) + data.Set("address_type", jsonutils.NewString(LB_ADDR_TYPE_INTRANET)) + } + return man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data) +} + +func (lb *SLoadbalancer) AllowPerformStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return lb.IsOwner(userCred) || userCred.IsSystemAdmin() +} + +func (lb *SLoadbalancer) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) { + lb.SVirtualResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data) + // NOTE lb.Id will only be available after BeforeInsert happens + // NOTE this means lb.UpdateVersion will be 0, then 1 after creation + // NOTE need ways to notify error + lb.GetModelManager().TableSpec().Update(lb, func() error { + if lb.AddressType == LB_ADDR_TYPE_INTRANET { + // TODO support use reserved ip address + // TODO prefer ip address from server_type loadbalancer? + req := &SLoadbalancerNetworkRequestData{ + loadbalancer: lb, + networkId: lb.NetworkId, + address: lb.Address, + } + lnMan := db.GetModelManager("loadbalancernetwork").(*SLoadbalancernetworkManager) + ln, err := lnMan.NewLoadbalancerNetwork(ctx, userCred, req) + if err != nil { + log.Errorf("allocating loadbalancer network failed: %#v", req) + return err + } + lb.Address = ln.IpAddr + } + return nil + }) +} + +func (lb *SLoadbalancer) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", lb.GetOwnerProjectId()).Optional(true) + err := backendGroupV.Validate(data) + if err != nil { + return nil, err + } + return lb.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data) +} + +func (lb *SLoadbalancer) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { + if len(lb.Address) > 0 { + // TODO reserve support + req := &SLoadbalancerNetworkDeleteData{ + loadbalancer: lb, + } + lnMan := db.GetModelManager("loadbalancernetwork").(*SLoadbalancernetworkManager) + err := lnMan.DeleteLoadbalancerNetwork(ctx, userCred, req) + if err != nil { + return err + } + lb.Address = "" + } + // TODO How about mark pending delete and return + return nil +} + +func (lb *SLoadbalancer) PreDelete(ctx context.Context, userCred mcclient.TokenCredential) { + lb.SetStatus(userCred, LB_STATUS_DISABLED, "preDelete") + lb.DoPendingDelete(ctx, userCred) + lb.PreDeleteSubs(ctx, userCred) +} + +func (lb *SLoadbalancer) PreDeleteSubs(ctx context.Context, userCred mcclient.TokenCredential) { + ownerProjId := lb.GetOwnerProjectId() + lbId := lb.Id + subMen := []ILoadbalancerSubResourceManager{ + LoadbalancerListenerManager, + LoadbalancerBackendGroupManager, + } + for _, subMan := range subMen { + func(subMan ILoadbalancerSubResourceManager) { + lockman.LockClass(ctx, subMan, ownerProjId) + defer lockman.ReleaseClass(ctx, subMan, ownerProjId) + q := subMan.Query().Equals("loadbalancer_id", lbId) + subMan.PreDeleteSubs(ctx, userCred, q) + }(subMan) + } +} + +func (lb *SLoadbalancer) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + return nil +} diff --git a/pkg/compute/options/options.go b/pkg/compute/options/options.go index 7fe0d7e98e..4557d435ac 100644 --- a/pkg/compute/options/options.go +++ b/pkg/compute/options/options.go @@ -26,6 +26,8 @@ type ComputeOptions struct { PendingDeleteExpireSeconds int `default:"259200" help:"How long a pending delete VM/disks cleaned automatically, default 3 days"` PendingDeleteMaxCleanBatchSize int `default:"50" help:"How many pending delete servers can be clean in a batch"` + LoadbalancerPendingDeleteCheckInterval int `default:"3600" help:"Interval between checks of pending deleted loadbalancer objects, defaults to 1h"` + ImageCacheStoragePolicy string `default:"least_used" choices:"best_fit|least_used" help:"Policy to choose storage for image cache, best_fit or least_used"` MetricsRetentionDays int32 `default:"30" help:"Retention days for monitoring metrics in influxdb"` diff --git a/pkg/compute/service/service.go b/pkg/compute/service/service.go index 8f4f46834a..c33e89f0ce 100644 --- a/pkg/compute/service/service.go +++ b/pkg/compute/service/service.go @@ -56,6 +56,7 @@ func StartService() { cron := cronman.GetCronJobManager() cron.AddJob1("CleanPendingDeleteServers", time.Duration(options.Options.PendingDeleteCheckSeconds)*time.Second, models.GuestManager.CleanPendingDeleteServers) cron.AddJob1("CleanPendingDeleteDisks", time.Duration(options.Options.PendingDeleteCheckSeconds)*time.Second, models.DiskManager.CleanPendingDeleteDisks) + cron.AddJob1("CleanPendingDeleteLoadbalancers", time.Duration(options.Options.LoadbalancerPendingDeleteCheckInterval)*time.Second, models.LoadbalancerAgentManager.CleanPendingDeleteLoadbalancers) cron.AddJob2("AutoDiskSnapshot", options.Options.AutoSnapshotDay, options.Options.AutoSnapshotHour, 0, 0, models.DiskManager.AutoDiskSnapshot) cron.Start() diff --git a/pkg/lbagent/api.go b/pkg/lbagent/api.go new file mode 100644 index 0000000000..0a93eae015 --- /dev/null +++ b/pkg/lbagent/api.go @@ -0,0 +1,291 @@ +package lbagent + +import ( + "context" + "fmt" + "sync" + "time" + + "yunion.io/x/log" + + agentmodels "yunion.io/x/onecloud/pkg/lbagent/models" + agentutils "yunion.io/x/onecloud/pkg/lbagent/utils" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/models" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type ApiHelper struct { + opts *Options + + dataDirMan *agentutils.ConfigDirManager + corpus *agentmodels.LoadbalancerCorpus + agentParams *agentmodels.AgentParams +} + +func NewApiHelper(opts *Options) (*ApiHelper, error) { + helper := &ApiHelper{ + opts: opts, + dataDirMan: agentutils.NewConfigDirManager(opts.apiDataStoreDir), + } + return helper, nil +} + +func (h *ApiHelper) Run(ctx context.Context) { + defer func() { + log.Infof("api helper bye") + wg := ctx.Value("wg").(*sync.WaitGroup) + wg.Done() + }() + h.runInit(ctx) + apiSyncTicker := time.NewTicker(time.Duration(h.opts.ApiSyncInterval) * time.Second) + hbTicker := time.NewTicker(time.Duration(h.opts.ApiLbagentHbInterval) * time.Second) + defer hbTicker.Stop() + defer apiSyncTicker.Stop() + for { + select { + case <-hbTicker.C: + _, err := h.doHb(ctx) + if err != nil { + log.Errorf("heartbeat: %s", err) + } + case <-apiSyncTicker.C: + apiDataChanged := h.doSyncApiData(ctx) + agentParamsChanged := h.doSyncAgentParams(ctx) + if apiDataChanged || agentParamsChanged { + h.doUseCorpus(ctx) + } + case <-ctx.Done(): + return + } + } +} + +func (h *ApiHelper) adminClientSession(ctx context.Context) *mcclient.ClientSession { + region := h.opts.CommonOpts.Region + apiVersion := "v2" + s := auth.GetAdminSession(region, apiVersion) + return s +} + +func (h *ApiHelper) agentPeekOnce(ctx context.Context) (*models.LoadbalancerAgent, error) { + s := h.adminClientSession(ctx) + data, err := modules.LoadbalancerAgents.Get(s, h.opts.ApiLbagentId, nil) + if err != nil { + err := fmt.Errorf("agent get error: %s", err) + return nil, err + } + agent := &models.LoadbalancerAgent{} + err = data.Unmarshal(agent) + if err != nil { + err := fmt.Errorf("agent data unmarshal error: %s", err) + return nil, err + } + return agent, nil +} + +type agentPeekResult models.LoadbalancerAgent + +func (r *agentPeekResult) staleInFuture(s int) bool { + if r.HbLastSeen.IsZero() { + return true + } + duration := time.Since(r.HbLastSeen).Seconds() + if int(duration) < s { + return true + } + return false +} + +func (h *ApiHelper) agentPeek(ctx context.Context) *agentPeekResult { + doPeekWithLog := func() *models.LoadbalancerAgent { + agent, err := h.agentPeekOnce(ctx) + if err != nil { + log.Errorf("agent peek failed: %s", err) + } + return agent + } + agent := doPeekWithLog() + if agent == nil { + initHbTicker := time.NewTicker(time.Duration(3) * time.Second) + defer initHbTicker.Stop() + initHbDone: + for { + select { + case <-initHbTicker.C: + agent = doPeekWithLog() + if agent != nil { + break initHbDone + } + case <-ctx.Done(): + return nil + } + } + } + return (*agentPeekResult)(agent) +} + +func (h *ApiHelper) runInit(ctx context.Context) { + r := h.agentPeek(ctx) + if r == nil { + return + } + if !r.staleInFuture(h.opts.ApiLbagentHbTimeoutRelaxation) { + log.Warningf("agent will stale in %d seconds, re-sync", + h.opts.ApiLbagentHbTimeoutRelaxation) + } else { + h.doHb(ctx) + corpus, err := h.loadLocalData(ctx) + if err == nil { + h.corpus = corpus + } else { + log.Errorf("load local api data failed: %s", err) + } + } + if changed := h.doSyncApiData(ctx); changed { + h.doUseCorpus(ctx) + } +} + +func (h *ApiHelper) loadLocalData(ctx context.Context) (*agentmodels.LoadbalancerCorpus, error) { + corpus := agentmodels.NewEmptyLoadbalancerCorpus() + dataDir := h.dataDirMan.MostRecentSubdir() + err := corpus.LoadDir(dataDir) + if err != nil { + return nil, err + } + return corpus, nil +} + +func (h *ApiHelper) agentUpdateSeen(ctx context.Context) *models.LoadbalancerAgent { + s := h.adminClientSession(ctx) + params := h.corpus.MaxSeenUpdatedAtParams() + data, err := modules.LoadbalancerAgents.Update(s, h.opts.ApiLbagentId, params) + if err != nil { + log.Errorf("agent get error: %s", err) + return nil + } + agent := &models.LoadbalancerAgent{} + err = data.Unmarshal(agent) + if err != nil { + log.Errorf("agent data unmarshal error: %s", err) + return nil + } + return agent +} + +func (h *ApiHelper) doHb(ctx context.Context) (*models.LoadbalancerAgent, error) { + // TODO check if things changed recently + s := h.adminClientSession(ctx) + data, err := modules.LoadbalancerAgents.PerformAction(s, h.opts.ApiLbagentId, "hb", nil) + if err != nil { + err := fmt.Errorf("heartbeat api error: %s", err) + return nil, err + } + agent := &models.LoadbalancerAgent{} + err = data.Unmarshal(agent) + if err != nil { + err := fmt.Errorf("heartbeat data unmarshal error: %s", err) + return nil, err + } + return agent, nil +} + +func (h *ApiHelper) doSyncApiData(ctx context.Context) bool { + { + stime := time.Now() + defer func() { + elapsed := time.Since(stime) + log.Infof("sync api data done, elapsed: %s", elapsed.String()) + }() + } + + s := h.adminClientSession(ctx) + if h.corpus == nil { + h.corpus = agentmodels.NewEmptyLoadbalancerCorpus() + } + r, err := h.corpus.SyncModelSets(s, h.opts.ApiListBatchSize) + if err != nil { + log.Errorf("sync models: %s", err) + return false + } + if r.Changed { + h.agentUpdateSeen(ctx) + } + if !r.Correct { + log.Warningf("sync models: not correct") + return false + } + if r.Changed { + err := h.saveCorpus(ctx) + if err != nil { + log.Errorf("save corpus failed: %s", err) + return false + } + if err := h.dataDirMan.Prune(h.opts.DataPreserveN); err != nil { + log.Errorf("prune corpus data dir failed: %s", err) + // continue + } + log.Infof("corpus changed") + return true + } + return false +} + +func (h *ApiHelper) saveCorpus(ctx context.Context) error { + _, err := h.dataDirMan.NewDir(func(dir string) error { + err := h.corpus.SaveDir(dir) + if err != nil { + return fmt.Errorf("save to dir %s: %s", dir, err) + } + return nil + }) + return err +} + +func (h *ApiHelper) doSyncAgentParams(ctx context.Context) bool { + agent, err := h.agentPeekOnce(ctx) + if err != nil { + log.Errorf("agent params get failure: %s", err) + return false + } + agentParams, err := agentmodels.NewAgentParams(agent) + if err != nil { + log.Errorf("agent params prepare failure: %s", err) + return false + } + if !agentParams.Equal(h.agentParams) { + h.agentParams = agentParams + return true + } + return false +} + +func (h *ApiHelper) doUseCorpus(ctx context.Context) { + if h.corpus == nil { + log.Warningf("agent corpus nil") + return + } + if h.agentParams == nil { + log.Warningf("agent params nil") + return + } + log.Infof("make effect new corpus and params") + cmdData := &LbagentCmdUseCorpusData{ + Corpus: h.corpus, + AgentParams: h.agentParams, + } + cmdData.Wg.Add(1) + cmd := &LbagentCmd{ + Type: LbagentCmdUseCorpus, + Data: cmdData, + } + cmdChan := ctx.Value("cmdChan").(chan *LbagentCmd) + select { + case cmdChan <- cmd: + cmdData.Wg.Wait() + case <-ctx.Done(): + return + } +} diff --git a/pkg/lbagent/cmd.go b/pkg/lbagent/cmd.go new file mode 100644 index 0000000000..c04af764eb --- /dev/null +++ b/pkg/lbagent/cmd.go @@ -0,0 +1,24 @@ +package lbagent + +import ( + "sync" + + agentmodels "yunion.io/x/onecloud/pkg/lbagent/models" +) + +type LbagentCmdType uintptr + +const ( + LbagentCmdUseCorpus LbagentCmdType = iota +) + +type LbagentCmdUseCorpusData struct { + Corpus *agentmodels.LoadbalancerCorpus + AgentParams *agentmodels.AgentParams + Wg sync.WaitGroup +} + +type LbagentCmd struct { + Type LbagentCmdType + Data interface{} +} diff --git a/pkg/lbagent/gobetween/config.go b/pkg/lbagent/gobetween/config.go new file mode 100644 index 0000000000..0056cf8840 --- /dev/null +++ b/pkg/lbagent/gobetween/config.go @@ -0,0 +1,304 @@ +/** + * config.go - config file definitions + * + * @author Yaroslav Pogrebnyak + * @author Gene Ponomarenko + */ +package gobetween + +/** + * Config file top-level object + */ +type Config struct { + Logging LoggingConfig `toml:"logging" json:"logging"` + Api ApiConfig `toml:"api" json:"api"` + Defaults ConnectionOptions `toml:"defaults" json:"defaults"` + Acme *AcmeConfig `toml:"acme" json:"acme"` + Servers map[string]Server `toml:"servers" json:"servers"` +} + +/** + * Logging config section + */ +type LoggingConfig struct { + Level string `toml:"level" json:"level"` + Output string `toml:"output" json:"output"` +} + +/** + * Api config section + */ +type ApiConfig struct { + Enabled bool `toml:"enabled" json:"enabled"` + Bind string `toml:"bind" json:"bind"` + BasicAuth *ApiBasicAuthConfig `toml:"basic_auth" json:"basic_auth"` + Tls *ApiTlsConfig `toml:"tls" json:"tls"` + Cors bool `toml:"cors" json:"cors"` +} + +/** + * Api Basic Auth Config + */ +type ApiBasicAuthConfig struct { + Login string `toml:"login" json:"login"` + Password string `toml:"password" json:"password"` +} + +/** + * Api TLS server Config + */ +type ApiTlsConfig struct { + CertPath string `toml:"cert_path" json:"cert_path"` + KeyPath string `toml:"key_path" json:"key_path"` +} + +/** + * Default values can be overridden in server + */ +type ConnectionOptions struct { + MaxConnections *int `toml:"max_connections" json:"max_connections"` + ClientIdleTimeout *string `toml:"client_idle_timeout" json:"client_idle_timeout"` + BackendIdleTimeout *string `toml:"backend_idle_timeout" json:"backend_idle_timeout"` + BackendConnectionTimeout *string `toml:"backend_connection_timeout" json:"backend_connection_timeout"` +} + +/** + * Acme config + */ +type AcmeConfig struct { + Challenge string `toml:"challenge" json:"challenge"` + HttpBind string `toml:"http_bind" json:"http_bind"` + CacheDir string `toml:"cache_dir" json:"cache_dir"` +} + +/** + * Server section config + */ +type Server struct { + ConnectionOptions + + // hostname:port + Bind string `toml:"bind" json:"bind"` + + // tcp | udp | tls + Protocol string `toml:"protocol" json:"protocol"` + + // weight | leastconn | roundrobin + Balance string `toml:"balance" json:"balance"` + + // Optional configuration for server name indication + Sni *Sni `toml:"sni" json:"sni"` + + // Optional configuration for protocol = tls + Tls *Tls `toml:"tls" json:"tls"` + + // Optional configuration for backend_tls_enabled = true + BackendsTls *BackendsTls `toml:"backends_tls" json:"backends_tls"` + + // Optional configuration for protocol = udp + Udp *Udp `toml:"udp" json:"udp"` + + // Access configuration + Access *AccessConfig `toml:"access" json:"access"` + + // ProxyProtocol configuration + ProxyProtocol *ProxyProtocol `toml:"proxy_protocol" json:"proxy_protocol"` + + // Discovery configuration + Discovery *DiscoveryConfig `toml:"discovery" json:"discovery"` + + // Healthcheck configuration + Healthcheck *HealthcheckConfig `toml:"healthcheck" json:"healthcheck"` +} + +/** + * ProxyProtocol configurtion + */ +type ProxyProtocol struct { + Version string `toml:"version" json:"version"` +} + +/** + * Server Sni options + */ +type Sni struct { + HostnameMatchingStrategy string `toml:"hostname_matching_strategy" json:"hostname_matching_strategy"` + UnexpectedHostnameStrategy string `toml:"unexpected_hostname_strategy" json:"unexpected_hostname_strategy"` + ReadTimeout string `toml:"read_timeout" json:"read_timeout"` +} + +/** + * Common part of Tls and BackendTls types + */ +type tlsCommon struct { + Ciphers []string `toml:"ciphers" json:"ciphers"` + PreferServerCiphers bool `toml:"prefer_server_ciphers" json:"prefer_server_ciphers"` + MinVersion string `toml:"min_version" json:"min_version"` + MaxVersion string `toml:"max_version" json:"max_version"` + SessionTickets bool `toml:"session_tickets" json:"session_tickets"` +} + +/** + * Server Tls options + * for protocol = "tls" + */ +type Tls struct { + AcmeHosts []string `toml:"acme_hosts" json:"acme_hosts"` + CertPath string `toml:"cert_path" json:"cert_path"` + KeyPath string `toml:"key_path" json:"key_path"` + tlsCommon +} + +type BackendsTls struct { + IgnoreVerify bool `toml:"ignore_verify" json:"ignore_verify"` + RootCaCertPath *string `toml:"root_ca_cert_path" json:"root_ca_cert_path"` + CertPath *string `toml:"cert_path" json:"cert_path"` + KeyPath *string `toml:"key_path" json:"key_path"` + tlsCommon +} + +/** + * Server udp options + * for protocol = "udp" + */ +type Udp struct { + MaxRequests uint64 `toml:"max_requests" json:"max_requests"` + MaxResponses uint64 `toml:"max_responses" json:"max_responses"` +} + +/** + * Access configuration + */ +type AccessConfig struct { + Default string `toml:"default" json:"default"` + Rules []string `toml:"rules" json:"rules"` +} + +/** + * Discovery configuration + */ +type DiscoveryConfig struct { + Kind string `toml:"kind" json:"kind"` + Failpolicy string `toml:"failpolicy" json:"failpolicy"` + Interval string `toml:"interval" json:"interval"` + Timeout string `toml:"timeout" json:"timeout"` + + /* Depends on Kind */ + + *StaticDiscoveryConfig + *SrvDiscoveryConfig + *DockerDiscoveryConfig + *JsonDiscoveryConfig + *ExecDiscoveryConfig + *PlaintextDiscoveryConfig + *ConsulDiscoveryConfig + *LXDDiscoveryConfig +} + +type StaticDiscoveryConfig struct { + StaticList []string `toml:"static_list" json:"static_list"` +} + +type SrvDiscoveryConfig struct { + SrvLookupServer string `toml:"srv_lookup_server" json:"srv_lookup_server"` + SrvLookupPattern string `toml:"srv_lookup_pattern" json:"srv_lookup_pattern"` + SrvDnsProtocol string `toml:"srv_dns_protocol" json:"srv_dns_protocol"` +} + +type ExecDiscoveryConfig struct { + ExecCommand []string `toml:"exec_command" json:"exec_command"` +} + +type JsonDiscoveryConfig struct { + JsonEndpoint string `toml:"json_endpoint" json:"json_endpoint"` + JsonHostPattern string `toml:"json_host_pattern" json:"json_host_pattern"` + JsonPortPattern string `toml:"json_port_pattern" json:"json_port_pattern"` + JsonWeightPattern string `toml:"json_weight_pattern" json:"json_weight_pattern"` + JsonPriorityPattern string `toml:"json_priority_pattern" json:"json_priority_pattern"` + JsonSniPattern string `toml:"json_sni_pattern" json:"json_sni_pattern"` +} + +type PlaintextDiscoveryConfig struct { + PlaintextEndpoint string `toml:"plaintext_endpoint" json:"plaintext_endpoint"` + PlaintextRegexpPattern string `toml:"plaintext_regex_pattern" json:"plaintext_regex_pattern"` +} + +type DockerDiscoveryConfig struct { + DockerEndpoint string `toml:"docker_endpoint" json:"docker_endpoint"` + DockerContainerLabel string `toml:"docker_container_label" json:"docker_container_label"` + DockerContainerPrivatePort int64 `toml:"docker_container_private_port" json:"docker_container_private_port"` + DockerContainerHostEnvVar string `toml:"docker_container_host_env_var" json:"docker_container_host_env_var"` + + DockerTlsEnabled bool `toml:"docker_tls_enabled" json:"docker_tls_enabled"` + DockerTlsCertPath string `toml:"docker_tls_cert_path" json:"docker_tls_cert_path"` + DockerTlsKeyPath string `toml:"docker_tls_key_path" json:"docker_tls_key_path"` + DockerTlsCacertPath string `toml:"docker_tls_cacert_path" json:"docker_tls_cacert_path"` +} + +type ConsulDiscoveryConfig struct { + ConsulHost string `toml:"consul_host" json:"consul_host"` + ConsulServiceName string `toml:"consul_service_name" json:"consul_service_name"` + ConsulServiceTag string `toml:"consul_service_tag" json:"consul_service_tag"` + ConsulServicePassingOnly bool `toml:"consul_service_passing_only" json:"consul_service_passing_only"` + ConsulDatacenter string `toml:"consul_datacenter" json:"consul_datacenter"` + + ConsulAuthUsername string `toml:"consul_auth_username" json:"consul_auth_username"` + ConsulAuthPassword string `toml:"consul_auth_password" json:"consul_auth_password"` + + ConsulTlsEnabled bool `toml:"consul_tls_enabled" json:"consul_tls_enabled"` + ConsulTlsCertPath string `toml:"consul_tls_cert_path" json:"consul_tls_cert_path"` + ConsulTlsKeyPath string `toml:"consul_tls_key_path" json:"consul_tls_key_path"` + ConsulTlsCacertPath string `toml:"consul_tls_cacert_path" json:"consul_tls_cacert_path"` +} + +type LXDDiscoveryConfig struct { + LXDServerAddress string `toml:"lxd_server_address" json:"lxd_server_address"` + LXDServerRemoteName string `toml:"lxd_server_remote_name" json:"lxd_server_remote_name"` + LXDServerRemotePassword string `toml:"lxd_server_remote_password" json:"lxd_server_remote_password"` + + LXDConfigDirectory string `toml:"lxd_config_directory" json:"lxd_config_directory"` + LXDGenerateClientCerts bool `toml:"lxd_generate_client_certs" json:"lxd_generate_client_certs"` + LXDAcceptServerCert bool `toml:"lxd_accept_server_cert" json:"lxd_accept_server_cert"` + + LXDContainerLabelKey string `toml:"lxd_container_label_key" json:"lxd_container_label_key"` + LXDContainerLabelValue string `toml:"lxd_container_label_value" json:"lxd_container_label_value"` + + LXDContainerPort int `toml:"lxd_container_port" json:"lxd_container_port"` + LXDContainerPortKey string `toml:"lxd_container_port_key" json:"lxd_container_port_key"` + + LXDContainerInterface string `toml:"lxd_container_interface" json:"lxd_container_interface"` + LXDContainerInterfaceKey string `toml:"lxd_container_interface_key" json:"lxd_container_interface_key"` + + LXDContainerSNIKey string `toml:"lxd_container_sni_key" json:"lxd_container_sni_key"` + LXDContainerAddressType string `toml:"lxd_container_address_type" json:"lxd_container_address_type"` +} + +/** + * Healthcheck configuration + */ +type HealthcheckConfig struct { + Kind string `toml:"kind" json:"kind"` + Interval string `toml:"interval" json:"interval"` + Passes int `toml:"passes" json:"passes"` + Fails int `toml:"fails" json:"fails"` + Timeout string `toml:"timeout" json:"timeout"` + + /* Depends on Kind */ + + *PingHealthcheckConfig + *ExecHealthcheckConfig + *UdpHealthcheckConfig +} + +type PingHealthcheckConfig struct{} + +type ExecHealthcheckConfig struct { + ExecCommand string `toml:"exec_command" json:"exec_command,omitempty"` + ExecExpectedPositiveOutput string `toml:"exec_expected_positive_output" json:"exec_expected_positive_output"` + ExecExpectedNegativeOutput string `toml:"exec_expected_negative_output" json:"exec_expected_negative_output"` +} + +type UdpHealthcheckConfig struct { + Receive string + Send string +} diff --git a/pkg/lbagent/haproxy.go b/pkg/lbagent/haproxy.go new file mode 100644 index 0000000000..a5c3254f38 --- /dev/null +++ b/pkg/lbagent/haproxy.go @@ -0,0 +1,352 @@ +package lbagent + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "syscall" + + "yunion.io/x/log" + aggrerrors "yunion.io/x/pkg/util/errors" + + agentmodels "yunion.io/x/onecloud/pkg/lbagent/models" + agentutils "yunion.io/x/onecloud/pkg/lbagent/utils" +) + +type HaproxyHelper struct { + opts *Options + + configDirMan *agentutils.ConfigDirManager +} + +func NewHaproxyHelper(opts *Options) (*HaproxyHelper, error) { + helper := &HaproxyHelper{ + opts: opts, + configDirMan: agentutils.NewConfigDirManager(opts.haproxyConfigDir), + } + { + // sysctl + args := []string{ + "sysctl", "-w", + "net.ipv4.ip_nonlocal_bind=1", + "net.ipv4.ip_forward=1", + } + if err := helper.runCmd(args); err != nil { + return nil, fmt.Errorf("sysctl: %s", err) + } + } + if false { + // ipvs modules + mods := []string{ + "ip_vs", + "ip_vs_rr", + "ip_vs_wrr", + "ip_vs_lc", + "ip_vs_wlc", + "ip_vs_sh", + } + args := []string{"modprobe", ""} + for _, mod := range mods { + args[1] = mod + if err := helper.runCmd(args); err != nil { + return nil, fmt.Errorf("modprobe %s: %s", mod, err) + } + } + } + return helper, nil +} + +func (h *HaproxyHelper) Run(ctx context.Context) { + defer func() { + wg := ctx.Value("wg").(*sync.WaitGroup) + wg.Done() + }() + cmdChan := ctx.Value("cmdChan").(chan *LbagentCmd) + for { + for { + select { + case <-ctx.Done(): + log.Infof("haproxy helper bye") + return + case cmd := <-cmdChan: + h.handleCmd(ctx, cmd) + } + } + } +} + +func (h *HaproxyHelper) handleCmd(ctx context.Context, cmd *LbagentCmd) { + switch cmd.Type { + case LbagentCmdUseCorpus: + cmdData := cmd.Data.(*LbagentCmdUseCorpusData) + defer cmdData.Wg.Done() + h.handleUseCorpusCmd(ctx, cmd) + default: + log.Warningf("command type ignored: %v", cmd.Type) + } +} + +func (h *HaproxyHelper) handleUseCorpusCmd(ctx context.Context, cmd *LbagentCmd) { + // haproxy config dir + dir, err := h.configDirMan.NewDir(func(dir string) error { + cmdData := cmd.Data.(*LbagentCmdUseCorpusData) + corpus := cmdData.Corpus + agentParams := cmdData.AgentParams + { + opt := fmt.Sprintf("stats socket %s expose-fd listeners", h.haproxyStatsSocketFile()) + agentParams.SetHaproxyParams("global_stats_socket", opt) + } + var genHaproxyConfigsResult *agentmodels.GenHaproxyConfigsResult + var err error + { + // haproxy toplevel global/defaults config + err = corpus.GenHaproxyToplevelConfig(dir, agentParams) + if err != nil { + err = fmt.Errorf("generating haproxy toplevel config failed: %s", err) + return err + } + } + { + // haproxy configs + genHaproxyConfigsResult, err = corpus.GenHaproxyConfigs(dir, agentParams) + if err != nil { + err = fmt.Errorf("generating haproxy config failed: %s", err) + return err + } + } + { + // gobetween config + opts := &agentmodels.GenGobetweenConfigOptions{ + LoadbalancersEnabled: genHaproxyConfigsResult.LoadbalancersEnabled, + AgentParams: agentParams, + } + err := corpus.GenGobetweenConfigs(dir, opts) + if err != nil { + err = fmt.Errorf("generating gobetween config failed: %s", err) + return err + } + } + { + // keepalived config + opts := &agentmodels.GenKeepalivedConfigOptions{ + LoadbalancersEnabled: genHaproxyConfigsResult.LoadbalancersEnabled, + AgentParams: agentParams, + } + err := corpus.GenKeepalivedConfigs(dir, opts) + if err != nil { + err = fmt.Errorf("generating keepalived config failed: %s", err) + return err + } + } + return nil + }) + if err != nil { + log.Errorf("making configs: %s", err) + return + } + if err := h.configDirMan.Prune(h.opts.DataPreserveN); err != nil { + log.Errorf("prune configs dir failed: %s", err) + // continue + } + if err := h.useConfigs(ctx, dir); err != nil { + log.Errorf("useConfigs: %s", err) + } +} + +func (h *HaproxyHelper) useConfigs(ctx context.Context, d string) error { + lnF := func(old, new string) error { + err := os.RemoveAll(new) + if err != nil { + return err + } + err = os.Symlink(old, new) + return err + } + keepalivedConf := filepath.Join(h.opts.haproxyConfigDir, "keepalived.conf") + gobetweenJson := filepath.Join(h.opts.haproxyConfigDir, "gobetween.json") + haproxyConfD := h.haproxyConfD() + dirMap := map[string]string{ + keepalivedConf: filepath.Join(d, "keepalived.conf"), + haproxyConfD: d, + gobetweenJson: filepath.Join(d, "gobetween.json"), + } + for new, old := range dirMap { + err := lnF(old, new) + if err != nil { + return err + } + } + { + var errs []error + var err error + { + // reload haproxy + err = h.reloadHaproxy(ctx) + if err != nil { + errs = append(errs, err) + } + } + { + // reload gobetween + err = h.reloadGobetween(ctx) + if err != nil { + errs = append(errs, err) + } + } + { + // reload keepalived + err = h.reloadKeepalived(ctx) + if err != nil { + errs = append(errs, err) + } + } + if len(errs) == 0 { + return nil + } + return aggrerrors.NewAggregate(errs) + } +} + +func (h *HaproxyHelper) haproxyConfD() string { + return filepath.Join(h.opts.haproxyConfigDir, "haproxy.conf.d") +} + +func (h *HaproxyHelper) haproxyPidFile() string { + return filepath.Join(h.opts.haproxyRunDir, "haproxy.pid") +} + +func (h *HaproxyHelper) haproxyStatsSocketFile() string { + return filepath.Join(h.opts.haproxyRunDir, "haproxy.sock") +} + +func (h *HaproxyHelper) reloadHaproxy(ctx context.Context) error { + // NOTE we may sometimes need to specify a custom the executable path + pidFile := h.haproxyPidFile() + args := []string{ + h.opts.HaproxyBin, + "-D", // goes daemon + "-p", pidFile, + "-C", h.haproxyConfD(), + "-f", h.haproxyConfD(), + } + proc := agentutils.ReadPidFile(pidFile) + if proc != nil { + args = append(args, "-sf", fmt.Sprintf("%d", proc.Pid)) + { + statsSocket := h.haproxyStatsSocketFile() + if fi, err := os.Stat(statsSocket); err == nil && fi.Mode()&os.ModeSocket != 0 { + args = append(args, "-x", statsSocket) + } else { + log.Warningf("stats socket %s not found", statsSocket) + } + } + log.Infof("reloading haproxy") + } else { + log.Infof("starting haproxy") + } + return h.runCmd(args) +} + +func (h *HaproxyHelper) gobetweenConf() string { + return filepath.Join(h.opts.haproxyConfigDir, "gobetween.json") +} + +func (h *HaproxyHelper) gobetweenPidFile() string { + return filepath.Join(h.opts.haproxyRunDir, "gobetween.pid") +} + +func (h *HaproxyHelper) reloadGobetween(ctx context.Context) error { + pidFile := h.gobetweenPidFile() + args := []string{ + h.opts.GobetweenBin, + "--config", h.gobetweenConf(), + "--format", "json", + } + proc := agentutils.ReadPidFile(pidFile) + if proc != nil { + log.Infof("stopping gobetween(%d)", proc.Pid) + proc.Kill() + proc.Wait() + } + log.Infof("starting gobetween") + cmd, err := h.startCmd(args) + if err != nil { + return err + } + err = agentutils.WritePidFile(cmd.Process.Pid, h.gobetweenPidFile()) + if err != nil { + return fmt.Errorf("writing gobetween pid file: %s", err) + } + return nil +} + +func (h *HaproxyHelper) keepalivedConf() string { + return filepath.Join(h.opts.haproxyConfigDir, "keepalived.conf") +} + +func (h *HaproxyHelper) keepalivedPidFile() string { + return filepath.Join(h.opts.haproxyRunDir, "keepalived.pid") +} + +func (h *HaproxyHelper) keepalivedVrrpPidFile() string { + return filepath.Join(h.opts.haproxyRunDir, "keepalived_vrrp.pid") +} + +func (h *HaproxyHelper) keepalivedCheckersPidFile() string { + return filepath.Join(h.opts.haproxyRunDir, "keepalived_checkers.pid") +} + +func (h *HaproxyHelper) reloadKeepalived(ctx context.Context) error { + pidFile := h.keepalivedPidFile() + proc := agentutils.ReadPidFile(pidFile) + if proc != nil { + // send SIGHUP to reload + err := proc.Signal(syscall.SIGHUP) + if err != nil { + return fmt.Errorf("keepalived: send HUP failed: %s", err) + } + return nil + } + args := []string{ + h.opts.KeepalivedBin, + "--pid", pidFile, + "--vrrp_pid", h.keepalivedVrrpPidFile(), + "--checkers_pid", h.keepalivedCheckersPidFile(), + "--use-file", h.keepalivedConf(), + } + return h.runCmd(args) +} + +func (h *HaproxyHelper) runCmd(args []string) error { + name := args[0] + args = args[1:] + cmd := exec.Command(name, args...) + + output, err := cmd.Output() + if err != nil { + if ee, ok := err.(*exec.ExitError); ok { + stdout := string(output) + stderr := string(ee.Stderr) + return fmt.Errorf("%s: %s\nargs: %s\nstdout: %s\nstderr: %s", + name, err, strings.Join(args, " "), stdout, stderr) + } + return fmt.Errorf("%s: %s", name, err) + } + return nil +} + +func (h *HaproxyHelper) startCmd(args []string) (*exec.Cmd, error) { + name := args[0] + args = args[1:] + cmd := exec.Command(name, args...) + + err := cmd.Start() + if err != nil { + return nil, err + } + return cmd, nil +} diff --git a/pkg/lbagent/models/agentparams.go b/pkg/lbagent/models/agentparams.go new file mode 100644 index 0000000000..be4c5a6ba3 --- /dev/null +++ b/pkg/lbagent/models/agentparams.go @@ -0,0 +1,106 @@ +package models + +import ( + "encoding/base64" + "fmt" + "text/template" + + "yunion.io/x/onecloud/pkg/mcclient/models" +) + +type AgentParams struct { + AgentModel *models.LoadbalancerAgent + KeepalivedConfigTmpl *template.Template + HaproxyConfigTmpl *template.Template + Data map[string]interface{} +} + +func NewAgentParams(agent *models.LoadbalancerAgent) (*AgentParams, error) { + b64s := map[string]string{ + "keepalived_conf_tmpl": agent.Params.KeepalivedConfTmpl, + "haproxy_conf_tmpl": agent.Params.HaproxyConfTmpl, + } + tmpls := map[string]*template.Template{} + for name, b64 := range b64s { + d, err := base64.StdEncoding.DecodeString(b64) + if err != nil { + return nil, fmt.Errorf("%s: invalid base64 string: %s", name, err) + } + tmpl, err := template.New(name).Parse(string(d)) + if err != nil { + return nil, fmt.Errorf("%s: invalid template: %s", name, err) + } + tmpls[name] = tmpl + } + dataAgent := map[string]interface{}{ + "id": agent.Id, + "name": agent.Name, + } + dataVrrp := map[string]interface{}{ + "priority": agent.Params.Vrrp.Priority, + "virtual_router_id": agent.Params.Vrrp.VirtualRouterId, + "garp_master_refresh": agent.Params.Vrrp.GarpMasterRefresh, + "preempt": agent.Params.Vrrp.Preempt, + "interface": agent.Params.Vrrp.Interface, + "advert_int": agent.Params.Vrrp.AdvertInt, + "pass": agent.Params.Vrrp.Pass, + } + dataHaproxy := map[string]interface{}{ + "global_log": agent.Params.Haproxy.GlobalLog, + "global_nbthread": agent.Params.Haproxy.GlobalNbthread, + "log_http": agent.Params.Haproxy.LogHttp, + "log_tcp": agent.Params.Haproxy.LogTcp, + "log_normal": agent.Params.Haproxy.LogNormal, + } + data := map[string]interface{}{ + "agent": dataAgent, + "vrrp": dataVrrp, + "haproxy": dataHaproxy, + } + agentParams := &AgentParams{ + AgentModel: agent, + KeepalivedConfigTmpl: tmpls["keepalived_conf_tmpl"], + HaproxyConfigTmpl: tmpls["haproxy_conf_tmpl"], + Data: data, + } + return agentParams, nil +} + +func (p *AgentParams) Equal(p2 *AgentParams) bool { + if p == nil && p2 == nil { + return true + } + if p == nil || p2 == nil { + return false + } + agentP := p.AgentModel + agentP2 := p2.AgentModel + if agentP.Params != agentP2.Params { + return false + } + return true +} + +func (p *AgentParams) setXxParams(xx, k string, v interface{}) map[string]interface{} { + var dt map[string]interface{} + d, ok := p.Data[xx] + if !ok { + dt = map[string]interface{}{} + p.Data[xx] = dt + } else { + dt = d.(map[string]interface{}) + } + dt[k] = v + return dt +} + +func (p *AgentParams) SetVrrpParams(k string, v interface{}) map[string]interface{} { + return p.setXxParams("vrrp", k, v) +} + +func (p *AgentParams) SetHaproxyParams(k string, v interface{}) map[string]interface{} { + return p.setXxParams("haproxy", k, v) +} + +func (p *AgentParams) KeepalivedConfig() { +} diff --git a/pkg/lbagent/models/corpus.go b/pkg/lbagent/models/corpus.go new file mode 100644 index 0000000000..b0c51f30e7 --- /dev/null +++ b/pkg/lbagent/models/corpus.go @@ -0,0 +1,79 @@ +package models + +import ( + "fmt" + "io/ioutil" + "path/filepath" + + "yunion.io/x/jsonutils" + + agentutils "yunion.io/x/onecloud/pkg/lbagent/utils" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type LoadbalancerCorpus struct { + *ModelSets + ModelSetsMaxUpdatedAt *ModelSetsMaxUpdatedAt +} + +func NewEmptyLoadbalancerCorpus() *LoadbalancerCorpus { + return &LoadbalancerCorpus{ + ModelSets: NewModelSets(), + ModelSetsMaxUpdatedAt: NewModelSetsMaxUpdatedAt(), + } +} + +func (b *LoadbalancerCorpus) SyncModelSets(s *mcclient.ClientSession, batchSize int) (*ModelSetsUpdateResult, error) { + mss := b.ModelSetList() + mssNews := NewModelSets() + for i, msNew := range mssNews.ModelSetList() { + minUpdatedAt := ModelSetMaxUpdatedAt(mss[i]) + err := GetModels(&GetModelsOptions{ + ClientSession: s, + ModelManager: msNew.ModelManager(), + MinUpdatedAt: minUpdatedAt, + ModelSet: msNew, + BatchListSize: batchSize, + }) + if err != nil { + return nil, err + } + } + r := b.ModelSets.ApplyUpdates(mssNews) + b.ModelSetsMaxUpdatedAt = r.ModelSetsMaxUpdatedAt + return r, nil +} + +func (b *LoadbalancerCorpus) MaxSeenUpdatedAtParams() *jsonutils.JSONDict { + mssmua := b.ModelSetsMaxUpdatedAt + return jsonutils.Marshal(mssmua).(*jsonutils.JSONDict) +} + +func (b *LoadbalancerCorpus) SaveDir(dir string) error { + j := jsonutils.Marshal(b) + d := j.String() + p := filepath.Join(dir, "corpus") + err := ioutil.WriteFile(p, []byte(d), agentutils.FileModeFileSensitive) + return err +} + +func (b *LoadbalancerCorpus) LoadDir(dir string) error { + p := filepath.Join(dir, "corpus") + d, err := ioutil.ReadFile(p) + if err != nil { + return err + } + jd, err := jsonutils.Parse(d) + if err != nil { + return fmt.Errorf("%s: json parse failed: %s", p, err) + } + err = jd.Unmarshal(b) + if err != nil { + return fmt.Errorf("%s: json unmarshal failed: %s", p, err) + } + correct := b.join() + if !correct { + return fmt.Errorf("%s: corpus data has inconsistencies", p) + } + return nil +} diff --git a/pkg/lbagent/models/gobetween.go b/pkg/lbagent/models/gobetween.go new file mode 100644 index 0000000000..792cff22ce --- /dev/null +++ b/pkg/lbagent/models/gobetween.go @@ -0,0 +1,164 @@ +package models + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "path/filepath" + + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/lbagent/gobetween" + agentutils "yunion.io/x/onecloud/pkg/lbagent/utils" +) + +type GenGobetweenConfigOptions struct { + LoadbalancersEnabled []*Loadbalancer + AgentParams *AgentParams + + Config *gobetween.Config +} + +func (b *LoadbalancerCorpus) GenGobetweenConfigs(dir string, opts *GenGobetweenConfigOptions) error { + //agentParams := opts.AgentParams + // TODO + // - dynamic password + // - log to remote + // - log to local syslog unix sock + // - respawn + opts.Config = &gobetween.Config{ + Servers: map[string]gobetween.Server{}, + Api: gobetween.ApiConfig{ + Enabled: true, + Bind: "localhost:777", + BasicAuth: &gobetween.ApiBasicAuthConfig{ + Login: "Yunion", + Password: "LBStats", + }, + }, + } + for _, lb := range opts.LoadbalancersEnabled { + for _, listener := range lb.listeners { + if listener.ListenerType != "udp" { + continue + } + if listener.Status != "enabled" { + continue + } + if listener.BackendGroupId == "" { + continue + } + backendGroup := lb.backendGroups[listener.BackendGroupId] + if backendGroup == nil || len(backendGroup.backends) == 0 { + continue + } + + // backends + staticList := []string{} + for _, backend := range backendGroup.backends { + backendS := fmt.Sprintf("%s:%d weight=%d", backend.Address, backend.Port, backend.Weight) + staticList = append(staticList, backendS) + } + + // scheduler + var serverBalance string + switch listener.Scheduler { + case "rr", "wrr": + serverBalance = "roundrobin" + case "lc", "wlc": + serverBalance = "leastconn" + case "sch", "tch": + log.Warningf("scheduler %s converted to iphash1", listener.Scheduler) + serverBalance = "iphash1" + default: + log.Warningf("scheduler %s converted to iphash1", listener.Scheduler) + serverBalance = "iphash1" + } + + // healthcheck + var serverHealthcheck *gobetween.HealthcheckConfig + if listener.HealthCheck == "on" && listener.HealthCheckType == "udp" { + serverHealthcheck = &gobetween.HealthcheckConfig{ + Kind: "pingudp", + Interval: fmt.Sprintf("%ds", listener.HealthCheckInterval), + Timeout: fmt.Sprintf("%ds", listener.HealthCheckTimeout), + Passes: listener.HealthCheckRise, + Fails: listener.HealthCheckFall, + UdpHealthcheckConfig: &gobetween.UdpHealthcheckConfig{ + Send: listener.HealthCheckReq, + Receive: listener.HealthCheckExp, + }, + } + } + + // acl + var serverAccess *gobetween.AccessConfig + if listener.AclStatus == "on" && listener.AclId != "" { + acl := b.LoadbalancerAcls[listener.AclId] + if acl == nil { + log.Warningf("listener %s(%s): unknown acl %s", + listener.Name, listener.Id, listener.AclId) + continue + } + var accessDefault string + var accessRulesAction string + accessRules := []string{} + switch listener.AclType { + case "black": + accessDefault = "allow" + accessRulesAction = "deny" + case "white": + accessDefault = "deny" + accessRulesAction = "allow" + default: + log.Warningf("listener %s(%s): unknown acl type: %s", + listener.Name, listener.Id, listener.AclId) + continue + } + for _, aclEntry := range *acl.AclEntries { + rule := fmt.Sprintf("%s %s", accessRulesAction, aclEntry.Cidr) + accessRules = append(accessRules, rule) + } + serverAccess = &gobetween.AccessConfig{ + Default: accessDefault, + Rules: accessRules, + } + } + pize := func(s string) *string { + return &s + } + + opts.Config.Servers[listener.Id] = gobetween.Server{ + Bind: fmt.Sprintf("%s:%d", lb.Address, listener.ListenerPort), + Protocol: "udp", + Balance: serverBalance, + Discovery: &gobetween.DiscoveryConfig{ + Kind: "static", + StaticDiscoveryConfig: &gobetween.StaticDiscoveryConfig{ + StaticList: staticList, + }, + }, + Healthcheck: serverHealthcheck, + Access: serverAccess, + ConnectionOptions: gobetween.ConnectionOptions{ + ClientIdleTimeout: pize(fmt.Sprintf("%ds", listener.ClientIdleTimeout)), + BackendIdleTimeout: pize(fmt.Sprintf("%ds", listener.BackendIdleTimeout)), + BackendConnectionTimeout: pize(fmt.Sprintf("%ds", listener.BackendConnectTimeout)), + }, + } + } + } + { + // write gobetween.json + d, err := json.MarshalIndent(opts.Config, "", " ") + if err != nil { + return err + } + p := filepath.Join(dir, "gobetween.json") + err = ioutil.WriteFile(p, d, agentutils.FileModeFile) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/lbagent/models/haproxy.go b/pkg/lbagent/models/haproxy.go new file mode 100644 index 0000000000..a33d24dc6f --- /dev/null +++ b/pkg/lbagent/models/haproxy.go @@ -0,0 +1,476 @@ +package models + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "text/template" + + "yunion.io/x/log" + + agentutils "yunion.io/x/onecloud/pkg/lbagent/utils" +) + +var haproxyConfigErrNop = errors.New("nop haproxy config snippet") + +type GenHaproxyConfigsResult struct { + LoadbalancersEnabled []*Loadbalancer +} + +func (b *LoadbalancerCorpus) GenHaproxyToplevelConfig(dir string, opts *AgentParams) error { + buf := bytes.NewBufferString("# yunion lb auto-generated 00-haproxy.cfg\n") + haproxyConfigTmpl := opts.HaproxyConfigTmpl + err := haproxyConfigTmpl.Execute(buf, opts.Data) + if err != nil { + return err + } + data := buf.Bytes() + p := filepath.Join(dir, "00-haproxy.cfg") + err = ioutil.WriteFile(p, data, agentutils.FileModeFile) + if err != nil { + return err + } + return nil +} + +func (b *LoadbalancerCorpus) GenHaproxyConfigs(dir string, opts *AgentParams) (*GenHaproxyConfigsResult, error) { + if len(b.LoadbalancerCertificates) > 0 { + certsBase := filepath.Join(dir, "certs") + certsBaseFinal := filepath.Join(agentutils.DirStagingToFinal(dir), "certs") + err := os.MkdirAll(certsBase, agentutils.FileModeDirSensitive) + if err != nil { + return nil, fmt.Errorf("mkdir %s: %s", certsBase, err) + } + { + p := filepath.Join(dir, "01-haproxy.cfg") + lines := []string{ + "global", + fmt.Sprintf(" crt-base %s", certsBaseFinal), + "", + } + s := strings.Join(lines, "\n") + err := ioutil.WriteFile(p, []byte(s), agentutils.FileModeFile) + if err != nil { + return nil, fmt.Errorf("write 01-haproxy.cfg: %s", err) + } + } + for _, lbcert := range b.LoadbalancerCertificates { + d := []byte(lbcert.Certificate) + if d[len(d)-1] != '\n' { + d = append(d, '\n') + } + d = append(d, []byte(lbcert.PrivateKey)...) + fn := fmt.Sprintf("%s.pem", lbcert.Id) + p := filepath.Join(certsBase, fn) + err := ioutil.WriteFile(p, d, agentutils.FileModeFileSensitive) + if err != nil { + return nil, fmt.Errorf("write cert %s: %s", lbcert.Id, err) + } + } + } + for _, lbacl := range b.LoadbalancerAcls { + cidrs := []string{} + if lbacl.AclEntries != nil { + for _, aclEntry := range *lbacl.AclEntries { + cidrs = append(cidrs, aclEntry.Cidr) + } + } + if len(cidrs) > 0 { + s := fmt.Sprintf("## loadbalancer acl %s(%s)\n", lbacl.Name, lbacl.Id) + s += strings.Join(cidrs, "\n") + s += "\n" + p := filepath.Join(dir, "acl-"+lbacl.Id) + err := ioutil.WriteFile(p, []byte(s), agentutils.FileModeFile) + if err != nil { + return nil, err + } + } + } + + r := &GenHaproxyConfigsResult{ + LoadbalancersEnabled: []*Loadbalancer{}, + } + for _, lb := range b.Loadbalancers { + if lb.Status != "enabled" { + continue + } + if lb.Address == "" { + continue + } + if len(lb.listeners) == 0 { + continue + } + buf := bytes.NewBufferString(fmt.Sprintf("## loadbalancer %s(%s)\n\n", lb.Name, lb.Id)) + hasActiveListener := false + for _, listener := range lb.listeners { + if listener.Status != "enabled" { + continue + } + var err error + switch listener.ListenerType { + case "http", "https": + err = b.genHaproxyConfigHttp(buf, listener, opts) + case "tcp": + err = b.genHaproxyConfigTcp(buf, listener, opts) + case "udp": + // we record it for use in keepalived conf gen + r.LoadbalancersEnabled = append(r.LoadbalancersEnabled, lb) + continue + default: + log.Infof("haproxy: ignore listener type %s", listener.ListenerType) + continue + } + if err == haproxyConfigErrNop { + continue + } + if err != nil { + return nil, err + } + hasActiveListener = true + buf.WriteString("\n\n") // listeners sep lines + } + if hasActiveListener { + r.LoadbalancersEnabled = append(r.LoadbalancersEnabled, lb) + + d := buf.Bytes() + d = d[:len(d)-2] + fn := fmt.Sprintf("%s.%s", lb.Id, agentutils.HaproxyCfgExt) + p := filepath.Join(dir, fn) + err := ioutil.WriteFile(p, d, agentutils.FileModeFile) + if err != nil { + return nil, err + } + } + } + return r, nil +} + +func (b *LoadbalancerCorpus) genHaproxyConfigCommon(lb *Loadbalancer, listener *LoadbalancerListener, opts *AgentParams) map[string]interface{} { + data := map[string]interface{}{ + "comment": fmt.Sprintf("%s(%s)", listener.Name, listener.Id), + "id": listener.Id, + "listener_type": listener.ListenerType, + } + { + bind := fmt.Sprintf("%s:%d", lb.Address, listener.ListenerPort) + if listener.ListenerType == "https" && listener.certificate != nil { + bind += fmt.Sprintf(" ssl crt %s.pem", listener.certificate.Id) + if listener.TLSCipherPolicy != "" { + policy := agentutils.HaproxySslPolicy(listener.TLSCipherPolicy) + if policy != nil { + bind += fmt.Sprintf(" ssl-min-ver %s", policy.SslMinVer) + } + } + if listener.EnableHttp2 { + bind += fmt.Sprintf(" alpn h2,http/1.1") + } + } + data["bind"] = bind + } + { + agentHaproxyParams := opts.AgentModel.Params.Haproxy + if agentHaproxyParams.GlobalLog != "" { + if listener.ListenerType == "http" && agentHaproxyParams.LogHttp { + data["log"] = true + } else if listener.ListenerType == "tcp" && agentHaproxyParams.LogTcp { + data["log"] = true + } + } + } + if listener.AclStatus == "on" { + lbacl, ok := b.LoadbalancerAcls[listener.AclId] + if ok && lbacl.AclEntries != nil && len(*lbacl.AclEntries) > 0 { + var action, cond string + switch listener.ListenerType { + case "tcp": + action = "tcp-request connection reject" + case "http", "https": + action = "http-request deny" + } + switch listener.AclType { + case "black": + cond = "if" + case "white": + cond = "unless" + } + if action != "" && cond != "" { + acl := fmt.Sprintf("%s %s { src -f acl-%s }", action, cond, listener.AclId) + data["acl"] = acl + } + } + } + { + // NOTE timeout tunnel is not set. We may need to prepare a + // default section for each frontend/listener + timeoutsMap := map[string]int{ + "client_request_timeout": listener.ClientRequestTimeout, + "client_idle_timeout": listener.ClientIdleTimeout, + } + for k, v := range timeoutsMap { + if v > 0 { + data[k] = fmt.Sprintf("%ds", v) + } + } + } + return data +} + +func (b *LoadbalancerCorpus) genHaproxyConfigBackend(data map[string]interface{}, lb *Loadbalancer, listener *LoadbalancerListener, backendGroup *LoadbalancerBackendGroup) error { + var mode string + var balanceAlgorithm string + var httpCheck, httpCheckExpect string + var checkEnable, httpCheckEnable bool + var err error + { // mode + switch listener.ListenerType { + case "tcp": + mode = "tcp" + case "http", "https": + mode = "http" + default: + return fmt.Errorf("haproxy: unsupported listener type %s", listener.ListenerType) + } + } + { // balance algorithm + balanceAlgorithm, err = agentutils.HaproxyBalanceAlgorithm(listener.Scheduler) + if err != nil { + return err + } + } + { // (http) check enabled? + if listener.HealthCheck == "on" { + checkEnable = true + if listener.HealthCheckType == "http" { + httpCheckEnable = true + httpCheck = agentutils.HaproxyConfigHttpCheck( + listener.HealthCheckURI, listener.HealthCheckDomain) + httpCheckExpect = agentutils.HaproxyConfigHttpCheckExpect( + listener.HealthCheckHttpCode) + } + } + } + var stickySessionEnable bool + if mode == "http" && listener.StickySession == "on" { + // sticky session + stickyCookie := "" + switch listener.StickySessionType { + case "insert": + stickyCookie = "cookie SERVERID insert indirect nocache" + if maxIdle := listener.StickySessionCookieTimeout; maxIdle > 0 { + stickyCookie += fmt.Sprintf(" maxidle %ds", maxIdle) + } + case "server": + cookie := listener.StickySessionCookie + if cookie != "" { + stickyCookie = fmt.Sprintf("cookie %q rewrite nocache", cookie) + } + } + if stickyCookie != "" { + data["stickyCookie"] = stickyCookie + stickySessionEnable = true + } + } + { + serverLines := []string{} + for _, backend := range backendGroup.backends { + serverLine := fmt.Sprintf("server %s %s:%d", backend.Id, backend.Address, backend.Port) + if listener.Scheduler == "rr" { + serverLine += " weight 1" + } else { + serverLine += fmt.Sprintf(" weight %d", backend.Weight) + } + if checkEnable { + serverLine += fmt.Sprintf(" check rise %d fall %d inter %ds", + listener.HealthCheckRise, listener.HealthCheckFall, listener.HealthCheckInterval) + } + if stickySessionEnable { + serverLine += fmt.Sprintf(" cookie %q", backend.Id) + } + serverLines = append(serverLines, serverLine) + } + data["servers"] = serverLines + } + if listener.HealthCheckTimeout > 0 { + data["timeout_check"] = fmt.Sprintf("timeout check %ds", listener.HealthCheckTimeout) + } + { + timeoutsMap := map[string]int{ + "backend_connect_timeout": listener.BackendConnectTimeout, + "backend_idle_timeout": listener.BackendIdleTimeout, + "backend_tunnel_timeout": listener.BackendIdleTimeout, + } + for k, v := range timeoutsMap { + if v > 0 { + data[k] = fmt.Sprintf("%ds", v) + } + } + } + data["mode"] = mode + data["balanceAlgorithm"] = balanceAlgorithm + if httpCheckEnable { + data["httpCheck"] = httpCheck + data["httpCheckExpect"] = httpCheckExpect + } + return nil +} + +func (b *LoadbalancerCorpus) genHaproxyConfigHttp(buf *bytes.Buffer, listener *LoadbalancerListener, opts *AgentParams) error { + lb := listener.loadbalancer + rules := LoadbalancerListenerRules{} + for id, rule := range listener.rules { + if rule.Status != "enabled" { + continue + } + rules[id] = rule + } + data := b.genHaproxyConfigCommon(lb, listener, opts) + ruleBackendIdGen := func(id string) string { + return fmt.Sprintf("backends_rule-%s", id) + } + { + // NOTE add X-Real-IP if needed + // + // http-request set-header X-Client-IP %[src] + // + data["xforwardedfor"] = listener.XForwardedFor + data["gzip"] = listener.Gzip + } + { // use_backend rule.Id if xx + ruleLines := []string{} + for _, rule := range rules { + ruleLine := fmt.Sprintf("use_backend %s", ruleBackendIdGen(rule.Id)) + if rule.Domain != "" || rule.Path != "" { + ruleLine += " if" + if rule.Domain != "" { + ruleLine += fmt.Sprintf(" { hdr_dom(host) %q }", rule.Domain) + } + if rule.Path != "" { + ruleLine += fmt.Sprintf(" { path_beg %q }", rule.Path) + } + } + ruleLines = append(ruleLines, ruleLine) + } + data["rules"] = ruleLines + } + { + backends := []interface{}{} + // rules backend group + for _, rule := range rules { + // NOTE dup is ok + if rule.BackendGroupId == "" { + // just in case + continue + } + backendGroup := lb.backendGroups[rule.BackendGroupId] + backendData := map[string]interface{}{ + "comment": fmt.Sprintf("rule %s(%s) backendGroup %s(%s)", + rule.Name, rule.Id, + backendGroup.Name, backendGroup.Id), + "id": ruleBackendIdGen(rule.Id), + } + err := b.genHaproxyConfigBackend(backendData, lb, listener, backendGroup) + if err != nil { + return err + } + backends = append(backends, backendData) + } + // default backend group + if listener.BackendGroupId != "" { + backendGroup := lb.backendGroups[listener.BackendGroupId] + backendData := map[string]interface{}{ + "comment": fmt.Sprintf("listener %s(%s) default backendGroup %s(%s)", + listener.Name, listener.Id, + backendGroup.Name, backendGroup.Id), + "id": fmt.Sprintf("backends_listener_default-%s", listener.Id), + } + err := b.genHaproxyConfigBackend(backendData, lb, listener, backendGroup) + if err != nil { + return err + } + backends = append(backends, backendData) + data["default_backend"] = backendData + } + if len(backends) == 0 { + // no backendgroup specified, nothing to serve + return haproxyConfigErrNop + } + data["backends"] = backends + } + err := haproxyConfigTmpl.ExecuteTemplate(buf, "httpListen", data) + return err +} + +func (b *LoadbalancerCorpus) genHaproxyConfigTcp(buf *bytes.Buffer, listener *LoadbalancerListener, opts *AgentParams) error { + lb := listener.loadbalancer + data := b.genHaproxyConfigCommon(lb, listener, opts) + if listener.BackendGroupId != "" { + backendGroup := lb.backendGroups[listener.BackendGroupId] + backendData := map[string]interface{}{ + "comment": fmt.Sprintf("listener %s(%s) backendGroup %s(%s)", + listener.Name, listener.Id, + backendGroup.Name, backendGroup.Id), + "id": fmt.Sprintf("backends_listener-%s", listener.Id), + } + err := b.genHaproxyConfigBackend(backendData, lb, listener, backendGroup) + if err != nil { + return err + } + data["backend"] = backendData + err = haproxyConfigTmpl.ExecuteTemplate(buf, "tcpListen", data) + return err + } + return haproxyConfigErrNop +} + +var haproxyConfigTmpl = template.Must(template.New("").Parse(` +{{ define "tcpListen" -}} +# {{ .listener_type }} listener: {{ .comment }} +listen {{ .id }} + bind {{ .bind }} + mode tcp + {{- println }} + {{- if .log }} {{ println "option tcplog" }} {{- end }} + {{- if .acl }} {{ println .acl }} {{- end}} + {{- if .client_idle_timeout }} timeout client {{ println .client_idle_timeout }} {{- end}} + default_backend {{ .backend.id }} +{{ template "backend" .backend }} +{{- end }} + +{{ define "httpListen" -}} +# {{ .listener_type }} listener: {{ .comment }} +frontend {{ .id }} + bind {{ .bind }} + mode http + {{- println }} + {{- if .log }} {{ println "option httplog clf" }} {{- end }} + {{- if .acl }} {{ println .acl }} {{- end}} + {{- if .client_request_timeout }} timeout http-request {{ println .client_request_timeout }} {{- end}} + {{- if .client_idle_timeout }} timeout http-keep-alive {{ println .client_idle_timeout }} {{- end}} + {{- if .xforwardedfor }} {{ println "option forwardfor" }} {{- end}} + {{- if .gzip }} {{ println "compression algo gzip" }} {{- end}} + {{- range .rules }} {{ println . }} {{- end }} + {{- if .default_backend.id }} default_backend {{ println .default_backend.id }} {{- end }} +{{- range .backends }} +{{- template "backend" . }} +{{- end }} +{{- end }} + +{{ define "backend" -}} +# {{ .comment }} +backend {{ .id }} + mode {{ .mode }} + balance {{ .balanceAlgorithm }} + {{- println }} + {{- if .backend_connect_timeout }} timeout connect {{ println .backend_connect_timeout }} {{- end}} + {{- if .backend_idle_timeout }} timeout server {{ println .backend_idle_timeout }} {{- end}} + {{- if .timeout_check }} {{ println .timeout_check }} {{- end }} + {{- if .stickyCookie }} {{ println .stickyCookie }} {{- end }} + {{- if .httpCheck }} {{ println .httpCheck }} {{- end }} + {{- if .httpCheckExpect }} {{ println .httpCheckExpect }} {{- end }} + {{- range .servers }} {{ println . }} {{- end }} +{{- end }} +`)) diff --git a/pkg/lbagent/models/keepalived.go b/pkg/lbagent/models/keepalived.go new file mode 100644 index 0000000000..fd9f7531ed --- /dev/null +++ b/pkg/lbagent/models/keepalived.go @@ -0,0 +1,157 @@ +package models + +import ( + "bytes" + "fmt" + "io/ioutil" + "path/filepath" + "text/template" + + "yunion.io/x/log" + + agentutils "yunion.io/x/onecloud/pkg/lbagent/utils" +) + +const keepalivedMiscCheckPath = "/opt/yunion/share/lbagent/healthcheck.sh" + +type GenKeepalivedConfigOptions struct { + LoadbalancersEnabled []*Loadbalancer + AgentParams *AgentParams +} + +func (b *LoadbalancerCorpus) GenKeepalivedConfigs(dir string, opts *GenKeepalivedConfigOptions) error { + agentParams := opts.AgentParams + { + addresses := []string{} + for _, lb := range opts.LoadbalancersEnabled { + if lb.Status != "enabled" { + continue + } + if lb.Address == "" { + continue + } + addresses = append(addresses, lb.Address) + } + agentParams.SetVrrpParams("addresses", addresses) + } + buf := bytes.NewBufferString("# yunion lb auto-generated keepalived.conf\n") + { + // write global_defs and vrrp_instance + keepalivedConfigToplevelTmpl := agentParams.KeepalivedConfigTmpl + err := keepalivedConfigToplevelTmpl.Execute(buf, agentParams.Data) + if err != nil { + return err + } + } + if false { + // write virtual_server + for _, lb := range opts.LoadbalancersEnabled { + for _, listener := range lb.listeners { + if listener.Status != "enabled" { + continue + } + if listener.BackendGroupId == "" { + continue + } + if listener.ListenerType == "udp" { + dataReals := []map[string]interface{}{} + dataVirtual := map[string]interface{}{ + "virtual_ip": lb.Address, + "virtual_port": listener.ListenerPort, + } + { + // scheduler + switch listener.Scheduler { + case "rr", "wrr", "lc", "wlc": + dataVirtual["scheduler"] = listener.Scheduler + case "sch", "tch": + log.Warningf("scheduler %s converted to sh", listener.Scheduler) + dataVirtual["scheduler"] = "sh" + default: + log.Warningf("scheduler %s converted to sh", listener.Scheduler) + dataVirtual["scheduler"] = "sh" + } + } + udpCheckEnabled := false + { + if listener.HealthCheck == "on" && listener.HealthCheckType == "udp" && listener.HealthCheckReq != "" { + udpCheckEnabled = true + } + } + backendGroup := lb.backendGroups[listener.BackendGroupId] + for _, backend := range backendGroup.backends { + dataReal := map[string]interface{}{ + "real_ip": backend.Address, + "real_port": backend.Port, + "weight": backend.Weight, + } + if udpCheckEnabled { + checkMiscArgs := []string{ + keepalivedMiscCheckPath, + fmt.Sprintf("wait=%d", listener.HealthCheckTimeout), + fmt.Sprintf("req=%s", listener.HealthCheckReq), + fmt.Sprintf("exp=%s", listener.HealthCheckExp), + fmt.Sprintf("host=%s", backend.Address), + fmt.Sprintf("port=%d", backend.Port), + } + dataCheckMiscPath := agentutils.KeepalivedConfQuoteScriptArgs(checkMiscArgs) + dataCheck := map[string]interface{}{ + "misc_path": dataCheckMiscPath, + "misc_timeout": listener.HealthCheckTimeout, + "interval": listener.HealthCheckInterval, + "fall": listener.HealthCheckFall, + "usergroup": "nobody", + } + dataReal["check"] = dataCheck + } + dataReals = append(dataReals, dataReal) + } + dataVirtual["real_servers"] = dataReals + + fmt.Fprintf(buf, "\n\n") + fmt.Fprintf(buf, "## listener %s(%s) backendGroup %s(%s)\n", + listener.Name, listener.Id, + backendGroup.Name, backendGroup.Id) + keepalivedConfTmpl.ExecuteTemplate(buf, "keepalivedVirtualServerUDP", dataVirtual) + } + } + } + } + { + // write keepalived.conf + d := buf.Bytes() + p := filepath.Join(dir, "keepalived.conf") + err := ioutil.WriteFile(p, d, agentutils.FileModeFile) + if err != nil { + return err + } + } + return nil +} + +// TODO retry for down, up? +var keepalivedConfTmpl = template.Must(template.New("").Parse(` +{{- define "keepalivedVirtualServerUDP" -}} +virtual_server {{ .virtual_ip }} {{ .virtual_port }} { + protocol UDP + lvs_method NAT + lvs_sched {{ .scheduler }} + ha_suspend + {{- range .real_servers }} + real_server {{ .real_ip }} {{ .real_port }} { + weight {{ .weight }} + inhibit_on_failure + {{- if .check }} + MISC_CHECK { + misc_path {{ .check.misc_path }} + misc_timeout {{ .check.misc_timeout }} + delay_loop {{ .check.interval }} + retry {{ .check.fall }} + user {{ .check.usergroup }} + } + {{- end }} + } + {{- end }} +} +{{ end }} +`)) diff --git a/pkg/lbagent/models/models.go b/pkg/lbagent/models/models.go new file mode 100644 index 0000000000..4191fefc3c --- /dev/null +++ b/pkg/lbagent/models/models.go @@ -0,0 +1,48 @@ +package models + +import ( + "yunion.io/x/onecloud/pkg/mcclient/models" +) + +type IModel interface { +} + +type Loadbalancer struct { + *models.Loadbalancer + + listeners LoadbalancerListeners + backendGroups LoadbalancerBackendGroups +} + +type LoadbalancerListener struct { + *models.LoadbalancerListener + + loadbalancer *Loadbalancer + certificate *LoadbalancerCertificate + rules LoadbalancerListenerRules +} + +type LoadbalancerListenerRule struct { + *models.LoadbalancerListenerRule + + listener *LoadbalancerListener +} + +type LoadbalancerBackendGroup struct { + *models.LoadbalancerBackendGroup + + backends LoadbalancerBackends + loadbalancer *Loadbalancer +} + +type LoadbalancerBackend struct { + *models.LoadbalancerBackend +} + +type LoadbalancerAcl struct { + *models.LoadbalancerAcl +} + +type LoadbalancerCertificate struct { + *models.LoadbalancerCertificate +} diff --git a/pkg/lbagent/models/modelset.go b/pkg/lbagent/models/modelset.go new file mode 100644 index 0000000000..7bc0ec9dc3 --- /dev/null +++ b/pkg/lbagent/models/modelset.go @@ -0,0 +1,249 @@ +package models + +import ( + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/mcclient/models" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type IModelSet interface { + //InitializeFromJSON([]jsonutils.JSONObject) error + ModelManager() modules.Manager + NewModel() models.IVirtualResource + //GetModel(id string) models.IVirtualResource + addModelCallback(models.IVirtualResource) error +} + +type Loadbalancers map[string]*Loadbalancer +type LoadbalancerListeners map[string]*LoadbalancerListener +type LoadbalancerListenerRules map[string]*LoadbalancerListenerRule +type LoadbalancerBackendGroups map[string]*LoadbalancerBackendGroup +type LoadbalancerBackends map[string]*LoadbalancerBackend +type LoadbalancerAcls map[string]*LoadbalancerAcl +type LoadbalancerCertificates map[string]*LoadbalancerCertificate + +func (set Loadbalancers) ModelManager() modules.Manager { + return &modules.Loadbalancers +} + +func (set Loadbalancers) NewModel() models.IVirtualResource { + return &models.Loadbalancer{} +} + +func (set Loadbalancers) addModelCallback(i models.IVirtualResource) error { + m, _ := i.(*models.Loadbalancer) + set[m.Id] = &Loadbalancer{ + Loadbalancer: m, + listeners: LoadbalancerListeners{}, + backendGroups: LoadbalancerBackendGroups{}, + } + return nil +} + +func (ms Loadbalancers) JoinListeners(subEntries LoadbalancerListeners) bool { + for _, m := range ms { + m.listeners = LoadbalancerListeners{} + } + correct := true + for subId, subEntry := range subEntries { + id := subEntry.LoadbalancerId + m, ok := ms[id] + if !ok { + log.Warningf("loadbalancer id %s not found", id) + correct = false + continue + } + if _, ok := m.listeners[subId]; ok { + log.Warningf("loadbalancer listener id %s already joined", subId) + continue + } + subEntry.loadbalancer = m + m.listeners[subId] = subEntry + } + return correct +} + +func (ms Loadbalancers) JoinBackendGroups(subEntries LoadbalancerBackendGroups) bool { + for _, m := range ms { + m.backendGroups = LoadbalancerBackendGroups{} + } + correct := true + for subId, subEntry := range subEntries { + id := subEntry.LoadbalancerId + m, ok := ms[id] + if !ok { + log.Warningf("loadbalancer id %s not found", id) + correct = false + continue + } + if _, ok := m.backendGroups[subId]; ok { + log.Warningf("loadbalancer backendgroup id %s already joined", subId) + continue + } + subEntry.loadbalancer = m + m.backendGroups[subId] = subEntry + } + return correct +} + +func (set LoadbalancerListeners) ModelManager() modules.Manager { + return &modules.LoadbalancerListeners +} + +func (set LoadbalancerListeners) NewModel() models.IVirtualResource { + return &models.LoadbalancerListener{} +} + +func (set LoadbalancerListeners) addModelCallback(i models.IVirtualResource) error { + m, _ := i.(*models.LoadbalancerListener) + set[m.Id] = &LoadbalancerListener{ + LoadbalancerListener: m, + rules: LoadbalancerListenerRules{}, + } + return nil +} + +func (ms LoadbalancerListeners) JoinListenerRules(subEntries LoadbalancerListenerRules) bool { + for _, m := range ms { + m.rules = LoadbalancerListenerRules{} + } + correct := true + for subId, subEntry := range subEntries { + id := subEntry.ListenerId + m, ok := ms[id] + if !ok { + log.Warningf("loadbalancer listener id %s not found", id) + correct = false + continue + } + if _, ok := m.rules[subId]; ok { + log.Warningf("loadbalancer rule id %s already joined", subId) + continue + } + subEntry.listener = m + m.rules[subId] = subEntry + } + return correct +} + +func (ms LoadbalancerListeners) JoinCertificates(subEntries LoadbalancerCertificates) bool { + correct := true + for _, m := range ms { + m.certificate = nil + if m.CertificateId != "" { + subEntry, ok := subEntries[m.CertificateId] + if !ok { + log.Warningf("loadbalancer m id %s: cannot find certificate id %s", + m.Id, m.CertificateId) + correct = false + continue + } + m.certificate = subEntry + } + } + return correct +} + +func (set LoadbalancerListenerRules) ModelManager() modules.Manager { + return &modules.LoadbalancerListenerRules +} + +func (set LoadbalancerListenerRules) NewModel() models.IVirtualResource { + return &models.LoadbalancerListenerRule{} +} + +func (set LoadbalancerListenerRules) addModelCallback(i models.IVirtualResource) error { + m, _ := i.(*models.LoadbalancerListenerRule) + set[m.Id] = &LoadbalancerListenerRule{ + LoadbalancerListenerRule: m, + } + return nil +} + +func (set LoadbalancerBackendGroups) ModelManager() modules.Manager { + return &modules.LoadbalancerBackendGroups +} + +func (set LoadbalancerBackendGroups) NewModel() models.IVirtualResource { + return &models.LoadbalancerBackendGroup{} +} + +func (set LoadbalancerBackendGroups) addModelCallback(i models.IVirtualResource) error { + m, _ := i.(*models.LoadbalancerBackendGroup) + set[m.Id] = &LoadbalancerBackendGroup{ + LoadbalancerBackendGroup: m, + backends: LoadbalancerBackends{}, + } + return nil +} + +func (ms LoadbalancerBackendGroups) JoinBackends(subEntries LoadbalancerBackends) bool { + for _, m := range ms { + m.backends = LoadbalancerBackends{} + } + correct := true + for subId, subEntry := range subEntries { + id := subEntry.BackendGroupId + m, ok := ms[id] + if !ok { + log.Warningf("loadbalancer backend group id %s not found", id) + correct = false + continue + } + if _, ok := m.backends[subId]; ok { + log.Warningf("loadbalancer backend id %s already joined", subId) + continue + } + m.backends[subId] = subEntry + } + return correct +} + +func (set LoadbalancerBackends) ModelManager() modules.Manager { + return &modules.LoadbalancerBackends +} + +func (set LoadbalancerBackends) NewModel() models.IVirtualResource { + return &models.LoadbalancerBackend{} +} + +func (set LoadbalancerBackends) addModelCallback(i models.IVirtualResource) error { + m, _ := i.(*models.LoadbalancerBackend) + set[m.Id] = &LoadbalancerBackend{ + LoadbalancerBackend: m, + } + return nil +} + +func (set LoadbalancerAcls) ModelManager() modules.Manager { + return &modules.LoadbalancerAcls +} + +func (set LoadbalancerAcls) NewModel() models.IVirtualResource { + return &models.LoadbalancerAcl{} +} + +func (set LoadbalancerAcls) addModelCallback(i models.IVirtualResource) error { + m, _ := i.(*models.LoadbalancerAcl) + set[m.Id] = &LoadbalancerAcl{ + LoadbalancerAcl: m, + } + return nil +} + +func (set LoadbalancerCertificates) ModelManager() modules.Manager { + return &modules.LoadbalancerCertificates +} + +func (set LoadbalancerCertificates) NewModel() models.IVirtualResource { + return &models.LoadbalancerCertificate{} +} + +func (set LoadbalancerCertificates) addModelCallback(i models.IVirtualResource) error { + m, _ := i.(*models.LoadbalancerCertificate) + set[m.Id] = &LoadbalancerCertificate{ + LoadbalancerCertificate: m, + } + return nil +} diff --git a/pkg/lbagent/models/modelsets.go b/pkg/lbagent/models/modelsets.go new file mode 100644 index 0000000000..58e3c0f58a --- /dev/null +++ b/pkg/lbagent/models/modelsets.go @@ -0,0 +1,138 @@ +package models + +import ( + "strings" + "time" + + "yunion.io/x/jsonutils" +) + +// pluralMap maps from KeyPlurals to underscore-separated field names +var pluralMap = map[string]string{} + +func init() { + ss := []string{ + "loadbalancers", + "loadbalancer_listeners", + "loadbalancer_listener_rules", + "loadbalancer_backend_groups", + "loadbalancer_backends", + "loadbalancer_acls", + "loadbalancer_certificates", + } + for _, s := range ss { + k := strings.Replace(s, "_", "", -1) + pluralMap[k] = s + } +} + +type ModelSetsMaxUpdatedAt struct { + Loadbalancers time.Time + LoadbalancerListeners time.Time + LoadbalancerListenerRules time.Time + LoadbalancerBackendGroups time.Time + LoadbalancerBackends time.Time + LoadbalancerAcls time.Time + LoadbalancerCertificates time.Time +} + +func NewModelSetsMaxUpdatedAt() *ModelSetsMaxUpdatedAt { + return &ModelSetsMaxUpdatedAt{ + Loadbalancers: PseudoZeroTime, + LoadbalancerListeners: PseudoZeroTime, + LoadbalancerListenerRules: PseudoZeroTime, + LoadbalancerBackendGroups: PseudoZeroTime, + LoadbalancerBackends: PseudoZeroTime, + LoadbalancerAcls: PseudoZeroTime, + LoadbalancerCertificates: PseudoZeroTime, + } +} + +type ModelSets struct { + Loadbalancers Loadbalancers + LoadbalancerListeners LoadbalancerListeners + LoadbalancerListenerRules LoadbalancerListenerRules + LoadbalancerBackendGroups LoadbalancerBackendGroups + LoadbalancerBackends LoadbalancerBackends + LoadbalancerAcls LoadbalancerAcls + LoadbalancerCertificates LoadbalancerCertificates +} + +func NewModelSets() *ModelSets { + return &ModelSets{ + Loadbalancers: Loadbalancers{}, + LoadbalancerListeners: LoadbalancerListeners{}, + LoadbalancerListenerRules: LoadbalancerListenerRules{}, + LoadbalancerBackendGroups: LoadbalancerBackendGroups{}, + LoadbalancerBackends: LoadbalancerBackends{}, + LoadbalancerAcls: LoadbalancerAcls{}, + LoadbalancerCertificates: LoadbalancerCertificates{}, + } +} + +func (mss *ModelSets) ModelSetList() []IModelSet { + // it's ordered this way to favour creation, not deletion + return []IModelSet{ + mss.LoadbalancerListenerRules, + mss.LoadbalancerListeners, + mss.LoadbalancerBackends, + mss.LoadbalancerBackendGroups, + mss.Loadbalancers, + mss.LoadbalancerAcls, + mss.LoadbalancerCertificates, + } +} + +func (mss *ModelSets) MaxSeenUpdatedAtParams() *jsonutils.JSONDict { + d := jsonutils.NewDict() + for _, ms := range mss.ModelSetList() { + k := ms.ModelManager().KeyString() + k = pluralMap[k] + t := ModelSetMaxUpdatedAt(ms) + if !t.Equal(PseudoZeroTime) { + d.Set(k, jsonutils.NewTimeString(t)) + } + } + return d +} + +type ModelSetsUpdateResult struct { + Correct bool // all elements referenced are present + Changed bool // any thing changed in the corpus + ModelSetsMaxUpdatedAt *ModelSetsMaxUpdatedAt +} + +func (mss *ModelSets) ApplyUpdates(mssNews *ModelSets) *ModelSetsUpdateResult { + r := &ModelSetsUpdateResult{ + Changed: false, + Correct: true, + } + mssmua := NewModelSetsMaxUpdatedAt() + mssList := mss.ModelSetList() + mssNewsList := mssNews.ModelSetList() + for i, mss := range mssList { + mssNews := mssNewsList[i] + msR := ModelSetApplyUpdates(mss, mssNews) + if !r.Changed && msR.Changed { + r.Changed = true + } + { + keyPlural := mss.ModelManager().KeyString() + ModelSetsMaxUpdatedAtSetField(mssmua, keyPlural, msR.MaxUpdatedAt) + } + } + if r.Changed { + r.Correct = mss.join() + } + r.ModelSetsMaxUpdatedAt = mssmua + return r +} + +func (mss *ModelSets) join() bool { + correct0 := mss.LoadbalancerBackendGroups.JoinBackends(mss.LoadbalancerBackends) + correct1 := mss.LoadbalancerListeners.JoinListenerRules(mss.LoadbalancerListenerRules) + correct2 := mss.LoadbalancerListeners.JoinCertificates(mss.LoadbalancerCertificates) + correct3 := mss.Loadbalancers.JoinListeners(mss.LoadbalancerListeners) + correct4 := mss.Loadbalancers.JoinBackendGroups(mss.LoadbalancerBackendGroups) + return correct0 && correct1 && correct2 && correct3 && correct4 +} diff --git a/pkg/lbagent/models/reflect.go b/pkg/lbagent/models/reflect.go new file mode 100644 index 0000000000..e579f7a6da --- /dev/null +++ b/pkg/lbagent/models/reflect.go @@ -0,0 +1,246 @@ +package models + +import ( + "fmt" + "reflect" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/util/timeutils" + "yunion.io/x/pkg/utils" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/models" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +// A hack to workaround the IsZero() in timeutils.Utcify. This depends on the +// fact that database time has a resolution of 1-second +var PseudoZeroTime = time.Time{}.Add(time.Nanosecond) + +type GetModelsOptions struct { + ClientSession *mcclient.ClientSession + ModelManager modules.Manager + ModelSet IModelSet + + BatchListSize int + MinUpdatedAt time.Time +} + +func GetModels(opts *GetModelsOptions) error { + man := opts.ModelManager + manKeyPlural := man.KeyString() + + minUpdatedAt := opts.MinUpdatedAt + minUpdatedAtFilter := func(time time.Time) string { + // TODO add GE + tstr := timeutils.MysqlTime(time) + return fmt.Sprintf("updated_at.ge('%s')", tstr) + } + setNextListParams := func(params *jsonutils.JSONDict, lastUpdatedAt time.Time, lastResult *modules.ListResult) (time.Time, error) { + // NOTE: the updated_at field has second-level resolution. + // If they all have the same date... + var max time.Time + nmax := 0 + n := len(lastResult.Data) + + // find out the max updated_at date in the result set, and how + // many in the set has this date + for i := n - 1; i >= 0; i-- { + j := lastResult.Data[i] + updatedAt, err := j.GetTime("updated_at") + if err != nil { + log.Warningf("%s: updated_at field: %s, %s", + manKeyPlural, err, j.String()) + continue + } + if max.IsZero() { + max = updatedAt + } + if max.Equal(updatedAt) { + nmax += 1 + } + } + // error if we do not have valid date + if max.IsZero() { + return time.Time{}, fmt.Errorf("cannot find next updated_at after '%q'", + manKeyPlural, lastUpdatedAt) + } + + var newTime time.Time + var newOffset int + // if not all updated_at date are the same, then we can + // continue to the next age. + if nmax < n || (!max.Equal(lastUpdatedAt) && !max.Equal(PseudoZeroTime)) { + newTime = max + newOffset = nmax + } else { + newTime = lastUpdatedAt + newOffset = lastResult.Offset + n + } + params.Set("filter.0", jsonutils.NewString(minUpdatedAtFilter(newTime))) + params.Set("offset", jsonutils.NewInt(int64(newOffset))) + return newTime, nil + } + + listOptions := options.BaseListOptions{ + Admin: options.Bool(true), + Filter: []string{minUpdatedAtFilter(minUpdatedAt)}, + OrderBy: []string{"updated_at", "id"}, + Order: "asc", + Limit: options.Int(opts.BatchListSize), + Offset: options.Int(0), + } + if !minUpdatedAt.Equal(PseudoZeroTime) { + // Only fetching pending deletes when we are doing incremental fetch + listOptions.PendingDeleteAll = options.Bool(true) + } + params, err := listOptions.Params() + if err != nil { + return fmt.Errorf("%s: making list params: %s", manKeyPlural, err) + } + + entriesJson := []jsonutils.JSONObject{} + for { + var err error + listResult, err := opts.ModelManager.List(opts.ClientSession, params) + if err != nil { + return fmt.Errorf("%s: list failed with updated_at.gt('%s'): %s", + manKeyPlural, minUpdatedAt, err) + } + entriesJson = append(entriesJson, listResult.Data...) + if listResult.Offset+len(listResult.Data) >= listResult.Total { + break + } + minUpdatedAt, err = setNextListParams(params, minUpdatedAt, listResult) + if err != nil { + return fmt.Errorf("%s: %s", manKeyPlural, err) + } + } + { + err := InitializeModelSetFromJSON(opts.ModelSet, entriesJson) + if err != nil { + return fmt.Errorf("%s: initializing model set failed: %s", + manKeyPlural, err) + } + } + return nil +} + +func InitializeModelSetFromJSON(set IModelSet, entriesJson []jsonutils.JSONObject) error { + setRv := reflect.ValueOf(set) + for _, kRv := range setRv.MapKeys() { + zRv := reflect.Value{} + setRv.SetMapIndex(kRv, zRv) + } + manKeyPlural := set.ModelManager().KeyString() + for _, entryJson := range entriesJson { + m := set.NewModel() + var err error + err = entryJson.Unmarshal(m) + if err != nil { + return fmt.Errorf("%s: error unmarshaling: %s: %s", manKeyPlural, err, entryJson.String()) + } + { + keyRv := reflect.ValueOf(m.GetId()) + oldMRv := setRv.MapIndex(keyRv) + if oldMRv.IsValid() { + // check version + oldM := oldMRv.Interface().(models.IVirtualResource) + oldVersion := oldM.GetUpdateVersion() + version := m.GetUpdateVersion() + if oldVersion > version { + oldUpdatedAt := oldM.GetUpdatedAt() + updatedAt := m.GetUpdatedAt() + log.Warningf("prefer loadbalancer with update_version %d(%s) to %d(%s)", + oldVersion, oldUpdatedAt, version, updatedAt) + return nil + } + } + } + err = set.addModelCallback(m) + if err != nil { + return fmt.Errorf("%s: add model: %s", manKeyPlural, err) + } + } + return nil +} + +func ModelSetMaxUpdatedAt(set IModelSet) time.Time { + r := PseudoZeroTime + setRv := reflect.ValueOf(set) + for _, kRv := range setRv.MapKeys() { + mRv := setRv.MapIndex(kRv) + m := mRv.Interface().(models.IVirtualResource) + updatedAt := m.GetUpdatedAt() + if r.Before(updatedAt) { + r = updatedAt + } + } + return r +} + +type ModelSetUpdateResult struct { + Changed bool + MaxUpdatedAt time.Time +} + +// ModelSetApplyUpdates applies bSet to aSet. +// +// - PendingDeleted in bSet are removed from aSet +// - Newer models in bSet are updated in aSet +func ModelSetApplyUpdates(aSet, bSet IModelSet) *ModelSetUpdateResult { + r := &ModelSetUpdateResult{ + Changed: false, + } + { + a := ModelSetMaxUpdatedAt(aSet) + b := ModelSetMaxUpdatedAt(bSet) + if b.After(a) { + r.MaxUpdatedAt = b + } else { + r.MaxUpdatedAt = a + } + } + aSetRv := reflect.ValueOf(aSet) + bSetRv := reflect.ValueOf(bSet) + for _, kRv := range bSetRv.MapKeys() { + bMRv := bSetRv.MapIndex(kRv) + bM := bMRv.Interface().(models.IVirtualResource) + aMRv := aSetRv.MapIndex(kRv) + if aMRv.IsValid() { + aM := aMRv.Interface().(models.IVirtualResource) + if bM.GetPendingDeleted() { + // oops, deleted + aSetRv.SetMapIndex(kRv, reflect.Value{}) + r.Changed = true + continue + } + if aM.GetUpdateVersion() < bM.GetUpdateVersion() { + // oops, updated + aSetRv.SetMapIndex(kRv, bMRv) + r.Changed = true + continue + } + } else { + if bM.GetPendingDeleted() { + // hmm, gone before even knowning + continue + } + // oops, new member + aSetRv.SetMapIndex(kRv, bMRv) + r.Changed = true + } + } + return r +} + +func ModelSetsMaxUpdatedAtSetField(mssmua *ModelSetsMaxUpdatedAt, keyPlural string, t time.Time) { + field_name := pluralMap[keyPlural] + fieldName := utils.Kebab2Camel(field_name, "_") + rv := reflect.Indirect(reflect.ValueOf(mssmua)) + fieldRv := rv.FieldByName(fieldName) + fieldRv.Set(reflect.ValueOf(t)) +} diff --git a/pkg/lbagent/options.go b/pkg/lbagent/options.go new file mode 100644 index 0000000000..5f9ad42d6a --- /dev/null +++ b/pkg/lbagent/options.go @@ -0,0 +1,63 @@ +package lbagent + +import ( + "fmt" + "os" + "path/filepath" + + "yunion.io/x/onecloud/pkg/cloudcommon" + agentutils "yunion.io/x/onecloud/pkg/lbagent/utils" +) + +type LbagentOptions struct { + ApiLbagentId string `require:true` + ApiLbagentHbInterval int `default:10` + ApiLbagentHbTimeoutRelaxation int `default:120 help:"If agent is to stale out in specified seconds in the future, consider it staled to avoid race condition when doing incremental api data fetch"` + + ApiSyncInterval int + ApiListBatchSize int `default:1024` + + DataPreserveN int `default:8 help:"number of recent data to preserve on disk"` + + BaseDataDir string // `required:true` + apiDataStoreDir string + haproxyConfigDir string + haproxyRunDir string + + KeepalivedBin string `default:keepalived` + HaproxyBin string `default:haproxy` + GobetweenBin string `default:gobetween` +} + +type Options struct { + CommonOpts cloudcommon.Options + + LbagentOptions +} + +func (opts *Options) ValidateThenInit() error { + if opts.ApiListBatchSize <= 0 { + return fmt.Errorf("negative api batch list size: %d", + opts.ApiListBatchSize) + } + return opts.initDirs() +} + +func (opts *Options) initDirs() error { + opts.apiDataStoreDir = filepath.Join(opts.BaseDataDir, "data") + opts.haproxyConfigDir = filepath.Join(opts.BaseDataDir, "configs") + opts.haproxyRunDir = filepath.Join(opts.BaseDataDir, "run") + dirs := []string{ + opts.apiDataStoreDir, + opts.haproxyConfigDir, + opts.haproxyRunDir, + } + for _, dir := range dirs { + err := os.MkdirAll(dir, agentutils.FileModeDir) + if err != nil { + return fmt.Errorf("mkdir -p %q: %s", + dir, err) + } + } + return nil +} diff --git a/pkg/lbagent/utils/configdir.go b/pkg/lbagent/utils/configdir.go new file mode 100644 index 0000000000..5e4eb97ca5 --- /dev/null +++ b/pkg/lbagent/utils/configdir.go @@ -0,0 +1,142 @@ +package utils + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + aggrerrors "yunion.io/x/pkg/util/errors" +) + +const ( + ConfigDirMode = os.FileMode(0755) + configDirFmt = "20060102.150405.000" + configDirFmtStaging = configDirFmt + ".staging" +) + +var ( + configDirPat = regexp.MustCompile(`^\d{8}\.\d{6}\.\d{3}$`) +) + +type ConfigDirManager struct { + baseDir string +} + +// TODO +// +// - set active +func NewConfigDirManager(baseDir string) *ConfigDirManager { + return &ConfigDirManager{ + baseDir: baseDir, + } +} + +type DirFunc func(string) error + +// call f() inside 20180830.104050.333.lbcorpus.json.staging +// move to final name 20180830.104050.333.lbcorpus.json +func (m *ConfigDirManager) NewDir(f DirFunc) (finalDir string, err error) { + stagingDir := m.stagingDir() + if stagingDir == "" { + err = fmt.Errorf("failed creating staging dir") + return + } + defer func() { + if err != nil { + os.RemoveAll(stagingDir) + } + }() + + err = f(stagingDir) + + if err != nil { + return + } + finalDir = DirStagingToFinal(stagingDir) + err = os.Rename(stagingDir, finalDir) + if err != nil { + return + } + return +} + +func (m *ConfigDirManager) subdirs() []string { + dirs := []string{} + fis, err := ioutil.ReadDir(m.baseDir) + if err != nil { + return dirs + } + for _, fi := range fis { + if !fi.IsDir() { + continue + } + dirName := fi.Name() + if !configDirPat.Match([]byte(dirName)) { + continue + } + dir := filepath.Join(m.baseDir, dirName) + dirs = append(dirs, dir) + } + return dirs +} + +func (m *ConfigDirManager) MostRecentSubdir() string { + dirs := m.subdirs() + nDirs := len(dirs) + if nDirs == 0 { + return "" + } + return dirs[nDirs-1] +} + +func (m *ConfigDirManager) Prune(nRetain int) error { + if nRetain <= 0 { + // zero means prune nothing + return nil + } + dirs := m.subdirs() + if len(dirs) > nRetain { + // retain the most recent n + dirs = dirs[:len(dirs)-nRetain] + errs := []error{} + for _, dir := range dirs { + err := os.RemoveAll(dir) + if os.IsNotExist(err) { + continue + } + errs = append(errs, err) + } + if len(errs) > 0 { + return aggrerrors.NewAggregate(errs) + } + } + return nil +} + +// stagingDir creates a new staging dir and returns the full path +func (m *ConfigDirManager) stagingDir() string { + for { + now := time.Now() + dn := now.Format(configDirFmtStaging) + path := filepath.Join(m.baseDir, dn) + _, err := os.Stat(path) + if err != nil && os.IsNotExist(err) { + os.MkdirAll(path, ConfigDirMode) + return path + } + time.Sleep(time.Millisecond) + } + // TODO panic retries + return "" +} + +func DirStagingToFinal(s string) string { + if strings.HasSuffix(s, ".staging") { + return s[:len(s)-8] + } + return s +} diff --git a/pkg/lbagent/utils/filemode.go b/pkg/lbagent/utils/filemode.go new file mode 100644 index 0000000000..0bb572ebb9 --- /dev/null +++ b/pkg/lbagent/utils/filemode.go @@ -0,0 +1,12 @@ +package utils + +import ( + "os" +) + +const ( + FileModeDir = os.FileMode(0755) + FileModeFile = os.FileMode(0644) + FileModeDirSensitive = os.FileMode(0700) + FileModeFileSensitive = os.FileMode(0600) +) diff --git a/pkg/lbagent/utils/haproxy.go b/pkg/lbagent/utils/haproxy.go new file mode 100644 index 0000000000..1817fd69b3 --- /dev/null +++ b/pkg/lbagent/utils/haproxy.go @@ -0,0 +1,71 @@ +package utils + +import ( + "fmt" + "strings" +) + +const HaproxyCfgExt = "cfg" + +func HaproxyBalanceAlgorithm(scheduler string) (balance string, err error) { + switch scheduler { + case "rr", "wrr": + balance = "roundrobin" + case "wlc": + balance = "leastconn" + case "sch": + balance = "source" + case "tch": + // NOTE haproxy supports only TCP type proxy + balance = "source" + default: + err = fmt.Errorf("unknown scheduler type %q", scheduler) + } + return +} + +type HaproxySslPolicyParams struct { + SslMinVer string + Ciphers string +} + +// TODO restrict ciphers as noted in https://help.aliyun.com/document_detail/90740.html +func HaproxySslPolicy(policy string) *HaproxySslPolicyParams { + r := &HaproxySslPolicyParams{} + switch policy { + case "tls_cipher_policy_1_0": + r.SslMinVer = "TLSv1.0" + case "tls_cipher_policy_1_1": + r.SslMinVer = "TLSv1.1" + case "tls_cipher_policy_1_2": + r.SslMinVer = "TLSv1.2" + case "tls_cipher_policy_1_2_strict": + r.SslMinVer = "TLSv1.2" + default: + return nil + } + return r +} + +func HaproxyConfigHttpCheck(uri, domain string) string { + if uri == "" { + uri = "/" + } + s := fmt.Sprintf("option httpchk HEAD %s HTTP/1.0", uri) + if domain != "" { + s += `\r\nHost:\ ` + domain + } + return s +} + +func HaproxyConfigHttpCheckExpect(s string) string { + ss := []string{} + for _, s := range strings.Split(s, ",") { + s = s[len("http_"):] + s = strings.Replace(s, "x", ".", -1) + ss = append(ss, s) + } + s = strings.Join(ss, "|") + s = fmt.Sprintf("http-check expect rstatus %s", s) + return s +} diff --git a/pkg/lbagent/utils/keepalived.go b/pkg/lbagent/utils/keepalived.go new file mode 100644 index 0000000000..374222c4db --- /dev/null +++ b/pkg/lbagent/utils/keepalived.go @@ -0,0 +1,31 @@ +package utils + +import ( + "fmt" + "strings" + "unicode" +) + +func KeepalivedConfHexQuote(s string) string { + q := "" + for _, r := range s { + if strings.ContainsRune(`"'\#!`, r) || !unicode.IsPrint(r) { + // cannot use \xNN here for keepalived bug + q += fmt.Sprintf(`\%03c`, r) + } else { + q += string(r) + } + } + q = fmt.Sprintf(`"%s"`, q) + return q +} + +func KeepalivedConfQuoteScriptArgs(args []string) string { + for i, arg := range args { + args[i] = KeepalivedConfHexQuote(arg) + } + s := strings.Join(args, " ") + s = strings.Replace(s, `"`, `\"`, -1) + s = fmt.Sprintf(`"%s"`, s) + return s +} diff --git a/pkg/lbagent/utils/pidfile.go b/pkg/lbagent/utils/pidfile.go new file mode 100644 index 0000000000..ad74db6d92 --- /dev/null +++ b/pkg/lbagent/utils/pidfile.go @@ -0,0 +1,36 @@ +package utils + +import ( + "fmt" + "io/ioutil" + "os" + "strconv" + "strings" + "syscall" +) + +func ReadPidFile(pidFile string) *os.Process { + data, err := ioutil.ReadFile(pidFile) + if err != nil { + return nil + } + s := strings.TrimSpace(string(data)) + pid, err := strconv.Atoi(s) + if err != nil { + return nil + } + proc, err := os.FindProcess(pid) + if err != nil { + return nil + } + if err := proc.Signal(syscall.Signal(0)); err != nil { + return nil + } + return proc +} + +func WritePidFile(pid int, pidFile string) error { + data := fmt.Sprintf("%d\n", pid) + err := ioutil.WriteFile(pidFile, []byte(data), FileModeFile) + return err +} diff --git a/pkg/mcclient/models/base.go b/pkg/mcclient/models/base.go new file mode 100644 index 0000000000..911f3e340a --- /dev/null +++ b/pkg/mcclient/models/base.go @@ -0,0 +1,75 @@ +package models + +import ( + "time" +) + +type IResource interface { + GetUpdateVersion() int + GetUpdatedAt() time.Time +} + +type IStandaloneResource interface { + IResource + GetId() string +} + +type IVirtualResource interface { + IStandaloneResource + GetPendingDeleted() bool +} + +type Resource struct { + CreatedAt time.Time + UpdatedAt time.Time + UpdateVersion int + DeletedAt time.Time + Deleted bool +} + +func (r *Resource) GetUpdateVersion() int { + return r.UpdateVersion +} + +func (r *Resource) GetUpdatedAt() time.Time { + return r.UpdatedAt +} + +type StandaloneResource struct { + Resource + + Id string + Name string + ExternalId string + Description string + IsEmulated bool +} + +func (r *StandaloneResource) GetId() string { + return r.Id +} + +type StatusStandaloneResource struct { + StandaloneResource + + Status string +} + +type VirtualResource struct { + StatusStandaloneResource + + ProjectId string + IsSystem bool + PendingDeletedAt time.Time + PendingDeleted bool +} + +func (r *VirtualResource) GetPendingDeleted() bool { + return r.PendingDeleted +} + +type SharableVirtualResource struct { + VirtualResource + + IsPublic bool +} diff --git a/pkg/mcclient/models/loadbalancers.go b/pkg/mcclient/models/loadbalancers.go new file mode 100644 index 0000000000..4ae4aaae00 --- /dev/null +++ b/pkg/mcclient/models/loadbalancers.go @@ -0,0 +1,179 @@ +package models + +import ( + "time" +) + +type Loadbalancer struct { + VirtualResource + + Address string + AddressType string + NetworkType string + NetworkId string + ZoneId string + + BackendGroupId string +} + +type LoadbalancerTCPListener struct{} +type LoadbalancerUDPListener struct{} + +type LoadbalancerHTTPListener struct { + StickySession string + StickySessionType string + StickySessionCookie string + StickySessionCookieTimeout int + + XForwardedFor bool + XForwardedForSLBIP bool + XForwardedForSLBID bool + Gzip bool +} + +// CACertificate string +type LoadbalancerHTTPSListener struct { + CertificateId string + TLSCipherPolicy string + EnableHttp2 bool +} + +type LoadbalancerListener struct { + VirtualResource + + LoadbalancerId string + Bandwidth int + ListenerType string + ListenerPort int + + Scheduler string + + ClientRequestTimeout int + ClientIdleTimeout int + BackendConnectTimeout int + BackendIdleTimeout int + + BackendGroupId string + + AclStatus string + AclType string + AclId string + + HealthCheck string + HealthCheckType string + + HealthCheckDomain string + HealthCheckURI string + HealthCheckHttpCode string + + HealthCheckRise int + HealthCheckFall int + HealthCheckInterval int + HealthCheckTimeout int + + HealthCheckReq string + HealthCheckExp string + + LoadbalancerTCPListener + LoadbalancerUDPListener + LoadbalancerHTTPListener + LoadbalancerHTTPSListener + + XForwardedFor bool + Gzip bool +} + +type LoadbalancerListenerRule struct { + VirtualResource + + ListenerId string + BackendGroupId string + + Domain string + Path string +} + +type LoadbalancerBackendGroup struct { + VirtualResource + + LoadbalancerId string +} + +type LoadbalancerBackend struct { + VirtualResource + + BackendGroupId string + BackendId string + BackendType string + Weight int + Address string + Port int +} + +type LoadbalancerAclEntry struct { + Cidr string + Comment string +} +type LoadbalancerAclEntries []*LoadbalancerAclEntry +type LoadbalancerAcl struct { + SharableVirtualResource + + AclEntries *LoadbalancerAclEntries +} + +type LoadbalancerCertificate struct { + VirtualResource + + Certificate string + PrivateKey string + + PublicKeyAlgorithm string + PublicKeyBitLen int + SignatureAlgorithm string + FingerprintSha256 string + NotBefore time.Time + NotAfter time.Time + CommonName string + SubjectAlternativeNames string +} + +type LoadbalancerAgent struct { + StandaloneResource + + HbLastSeen time.Time + HbTimeout int + + Loadbalancers time.Time + LoadbalancerListeners time.Time + LoadbalancerListenerRules time.Time + LoadbalancerBackendGroups time.Time + LoadbalancerBackends time.Time + LoadbalancerAcls time.Time + LoadbalancerCertificates time.Time + Params LoadbalancerAgentParams +} + +type LoadbalancerAgentParamsVrrp struct { + Priority int + VirtualRouterId int + GarpMasterRefresh int + Preempt bool + Interface string + AdvertInt int + Pass string +} + +type LoadbalancerAgentParamsHaproxy struct { + GlobalLog string + GlobalNbthread int + LogHttp bool + LogTcp bool + LogNormal bool +} + +type LoadbalancerAgentParams struct { + KeepalivedConfTmpl string + HaproxyConfTmpl string + Vrrp LoadbalancerAgentParamsVrrp + Haproxy LoadbalancerAgentParamsHaproxy +} diff --git a/pkg/mcclient/modules/mod_loadbalanceracls.go b/pkg/mcclient/modules/mod_loadbalanceracls.go new file mode 100644 index 0000000000..84e0314de6 --- /dev/null +++ b/pkg/mcclient/modules/mod_loadbalanceracls.go @@ -0,0 +1,25 @@ +package modules + +type LoadbalancerAclManager struct { + ResourceManager +} + +var ( + LoadbalancerAcls LoadbalancerAclManager +) + +func init() { + LoadbalancerAcls = LoadbalancerAclManager{ + NewComputeManager( + "loadbalanceracl", + "loadbalanceracls", + []string{ + "id", + "name", + "acl_entries", + }, + []string{"tenant"}, + ), + } + registerCompute(&LoadbalancerAcls) +} diff --git a/pkg/mcclient/modules/mod_loadbalanceragents.go b/pkg/mcclient/modules/mod_loadbalanceragents.go new file mode 100644 index 0000000000..69b7b38036 --- /dev/null +++ b/pkg/mcclient/modules/mod_loadbalanceragents.go @@ -0,0 +1,35 @@ +package modules + +type LoadbalancerAgentManager struct { + ResourceManager +} + +var ( + LoadbalancerAgents LoadbalancerAgentManager +) + +func init() { + LoadbalancerAgents = LoadbalancerAgentManager{ + NewComputeManager( + "loadbalanceragent", + "loadbalanceragents", + []string{ + "id", + "name", + + "hb_last_seen", + "hb_timeout", + + "loadbalancers", + "loadbalancer_listeners", + "loadbalancer_listener_rules", + "loadbalancer_backend_groups", + "loadbalancer_backends", + "loadbalancer_acls", + "loadbalancer_certificates", + }, + []string{}, + ), + } + registerCompute(&LoadbalancerAgents) +} diff --git a/pkg/mcclient/modules/mod_loadbalancerbackendgroups.go b/pkg/mcclient/modules/mod_loadbalancerbackendgroups.go new file mode 100644 index 0000000000..9aa1b690c7 --- /dev/null +++ b/pkg/mcclient/modules/mod_loadbalancerbackendgroups.go @@ -0,0 +1,25 @@ +package modules + +type LoadbalancerBackendGroupManager struct { + ResourceManager +} + +var ( + LoadbalancerBackendGroups LoadbalancerBackendGroupManager +) + +func init() { + LoadbalancerBackendGroups = LoadbalancerBackendGroupManager{ + NewComputeManager( + "loadbalancerbackendgroup", + "loadbalancerbackendgroups", + []string{ + "id", + "name", + "loadbalancer_id", + }, + []string{"tenant"}, + ), + } + registerCompute(&LoadbalancerBackendGroups) +} diff --git a/pkg/mcclient/modules/mod_loadbalancerbackends.go b/pkg/mcclient/modules/mod_loadbalancerbackends.go new file mode 100644 index 0000000000..d5e6829800 --- /dev/null +++ b/pkg/mcclient/modules/mod_loadbalancerbackends.go @@ -0,0 +1,30 @@ +package modules + +type LoadbalancerBackendManager struct { + ResourceManager +} + +var ( + LoadbalancerBackends LoadbalancerBackendManager +) + +func init() { + LoadbalancerBackends = LoadbalancerBackendManager{ + NewComputeManager( + "loadbalancerbackend", + "loadbalancerbackends", + []string{ + "id", + "name", + "backend_group_id", + "backend_id", + "backend_type", + "address", + "port", + "weight", + }, + []string{"tenant"}, + ), + } + registerCompute(&LoadbalancerBackends) +} diff --git a/pkg/mcclient/modules/mod_loadbalancercertificates.go b/pkg/mcclient/modules/mod_loadbalancercertificates.go new file mode 100644 index 0000000000..6ac5f42b39 --- /dev/null +++ b/pkg/mcclient/modules/mod_loadbalancercertificates.go @@ -0,0 +1,30 @@ +package modules + +type LoadbalancerCertificateManager struct { + ResourceManager +} + +var ( + LoadbalancerCertificates LoadbalancerCertificateManager +) + +func init() { + LoadbalancerCertificates = LoadbalancerCertificateManager{ + NewComputeManager( + "loadbalancercertificate", + "loadbalancercertificates", + []string{ + "id", + "name", + "algorithm", + "fingerprint", + "not_before", + "not_after", + "common_name", + "subject_alternative_Names", + }, + []string{"tenant"}, + ), + } + registerCompute(&LoadbalancerCertificates) +} diff --git a/pkg/mcclient/modules/mod_loadbalancerlistenerrules.go b/pkg/mcclient/modules/mod_loadbalancerlistenerrules.go new file mode 100644 index 0000000000..c62c3229fc --- /dev/null +++ b/pkg/mcclient/modules/mod_loadbalancerlistenerrules.go @@ -0,0 +1,29 @@ +package modules + +type LoadbalancerListenerRuleManager struct { + ResourceManager +} + +var ( + LoadbalancerListenerRules LoadbalancerListenerRuleManager +) + +func init() { + LoadbalancerListenerRules = LoadbalancerListenerRuleManager{ + NewComputeManager( + "loadbalancerlistenerrule", + "loadbalancerlistenerrules", + []string{ + "id", + "name", + "listener_id", + "status", + "domain", + "path", + "backend_id", + }, + []string{"tenant"}, + ), + } + registerCompute(&LoadbalancerListenerRules) +} diff --git a/pkg/mcclient/modules/mod_loadbalancerlisteners.go b/pkg/mcclient/modules/mod_loadbalancerlisteners.go new file mode 100644 index 0000000000..f08a5370ac --- /dev/null +++ b/pkg/mcclient/modules/mod_loadbalancerlisteners.go @@ -0,0 +1,31 @@ +package modules + +type LoadbalancerListenerManager struct { + ResourceManager +} + +var ( + LoadbalancerListeners LoadbalancerListenerManager +) + +func init() { + LoadbalancerListeners = LoadbalancerListenerManager{ + NewComputeManager( + "loadbalancerlistener", + "loadbalancerlisteners", + []string{ + "id", + "name", + "loadbalancer_id", + "status", + "listener_type", + "listener_port", + "backend_port", + "acl_status", + "acl_type", + }, + []string{"tenant"}, + ), + } + registerCompute(&LoadbalancerListeners) +} diff --git a/pkg/mcclient/modules/mod_loadbalancers.go b/pkg/mcclient/modules/mod_loadbalancers.go new file mode 100644 index 0000000000..11b00d1f7f --- /dev/null +++ b/pkg/mcclient/modules/mod_loadbalancers.go @@ -0,0 +1,31 @@ +package modules + +type LoadbalancerManager struct { + ResourceManager +} + +var ( + Loadbalancers LoadbalancerManager +) + +func init() { + Loadbalancers = LoadbalancerManager{ + NewComputeManager( + "loadbalancer", + "loadbalancers", + []string{ + "id", + "name", + "status", + "address_type", + "address", + "network_type", + "network_id", + "zone_id", + }, + []string{"tenant"}, + ), + } + registerCompute(&Loadbalancers) + +} diff --git a/pkg/mcclient/options/base.go b/pkg/mcclient/options/base.go index ef0663e5e7..eac81c7b70 100644 --- a/pkg/mcclient/options/base.go +++ b/pkg/mcclient/options/base.go @@ -147,6 +147,11 @@ func ListStructToParams(v interface{}) (*jsonutils.JSONDict, error) { return params, nil } +const ( + ListOrderAsc = "asc" + ListOrderDesc = "desc" +) + type BaseListOptions struct { Limit *int `default:"20" help:"Page limit"` Offset *int `default:"0" help:"Page offset"` diff --git a/pkg/mcclient/options/loadbalanceracls.go b/pkg/mcclient/options/loadbalanceracls.go new file mode 100644 index 0000000000..dce9a15b94 --- /dev/null +++ b/pkg/mcclient/options/loadbalanceracls.go @@ -0,0 +1,123 @@ +package options + +import ( + "fmt" + "strings" + + "yunion.io/x/jsonutils" +) + +type AclEntry struct { + Cidr string + Comment string +} + +type AclEntries []*AclEntry + +func NewAclEntry(s string) *AclEntry { + tu := strings.SplitN(s, "#", 2) + cidr := strings.TrimSpace(tu[0]) + comment := "" + if len(tu) > 1 { + comment = tu[1] + } + aclEntry := &AclEntry{ + Cidr: cidr, + Comment: comment, + } + return aclEntry +} + +func NewAclEntries(ss []string) AclEntries { + aclEntries := AclEntries{} + for _, s := range ss { + aclEntry := NewAclEntry(s) + aclEntries = append(aclEntries, aclEntry) + } + return aclEntries +} + +func (entry *AclEntry) String() string { + cidr := entry.Cidr + comment := entry.Comment + if comment != "" { + comment = " # " + comment + } + return fmt.Sprintf("%-15s%s", cidr, comment) +} + +func (entries AclEntries) String() string { + ss := []string{} + for _, entry := range entries { + ss = append(ss, entry.String()) + } + lines := strings.Join(ss, "\n") + return lines +} + +type LoadbalancerAclCreateOptions struct { + NAME string + AclEntry []string `help:"acl entry with cidr and comment separated by #, e.g. 10.9.0.0/16#no comment" json:"-"` +} + +type LoadbalancerAclGetOptions struct { + ID string +} + +type LoadbalancerAclListOptions struct { + BaseListOptions +} + +type LoadbalancerAclUpdateOptions struct { + ID string `json:"-"` + AclEntry []string `help:"acl entry with cidr and comment separated by #, e.g. 10.9.0.0/16#no comment" json:"-"` +} + +type LoadbalancerAclDeleteOptions struct { + ID string `json:"-"` +} + +type LoadbalancerAclActionPatchOptions struct { + ID string `json:"-"` + Add []string `help:"acl entry with cidr and comment separated by #, e.g. 10.9.0.0/16#no comment" json:"-"` + Del []string `help:"acl entry with cidr and comment separated by #, e.g. 10.9.0.0/16#no comment" json:"-"` +} + +func (opts *LoadbalancerAclCreateOptions) Params() (*jsonutils.JSONDict, error) { + params, err := optionsStructToParams(opts) + if err != nil { + return nil, err + } + aclEntries := NewAclEntries(opts.AclEntry) + aclEntriesJson := jsonutils.Marshal(aclEntries) + params.Set("acl_entries", aclEntriesJson) + return params, nil +} + +func (opts *LoadbalancerAclUpdateOptions) Params() (*jsonutils.JSONDict, error) { + params, err := optionsStructToParams(opts) + if err != nil { + return nil, err + } + aclEntries := NewAclEntries(opts.AclEntry) + aclEntriesJson := jsonutils.Marshal(aclEntries) + params.Set("acl_entries", aclEntriesJson) + return params, nil +} + +func (opts *LoadbalancerAclActionPatchOptions) Params() (*jsonutils.JSONDict, error) { + params, err := optionsStructToParams(opts) + if err != nil { + return nil, err + } + m := map[string][]string{ + "adds": opts.Add, + "dels": opts.Del, + } + for k, ss := range m { + aclEntries := NewAclEntries(ss) + aclEntriesJson := jsonutils.Marshal(aclEntries) + params.Set(k, aclEntriesJson) + } + return params, nil +} diff --git a/pkg/mcclient/options/loadbalanceragents.go b/pkg/mcclient/options/loadbalanceragents.go new file mode 100644 index 0000000000..46d0263cd6 --- /dev/null +++ b/pkg/mcclient/options/loadbalanceragents.go @@ -0,0 +1,113 @@ +package options + +import ( + "strings" + "time" + + "yunion.io/x/jsonutils" +) + +type LoadbalancerAgentParamsOptions struct { + KeepalivedConfTmpl string + HaproxyConfTmpl string + + VrrpPriority *int // required + VrrpVirtualRouterId *int // required + VrrpGarpMasterRefresh *int + VrrpPreempt string `choices:true|false` + VrrpInterface string // required + VrrpAdvertInt *int + VrrpPass string + + HaproxyGlobalLog string + HaproxyGlobalNbthread *int `default:1 help:"enable experimental multi-threading support available since haproxy 1.8"` + HaproxyLogHttp string `choices:true|false` + HaproxyLogTcp string `choices:true|false` + HaproxyLogNormal string `choices:true|false` +} + +func (opts *LoadbalancerAgentParamsOptions) setPrefixedParams(params *jsonutils.JSONDict, pref string) { + pref_ := pref + "_" + pref_len := len(pref_) + pp := jsonutils.NewDict() + pMap, _ := params.GetMap() + for k, v := range pMap { + if strings.HasPrefix(k, pref_) && !strings.HasSuffix(k, "_conf_tmpl") { + params.Remove(k) + pp.Set(k[pref_len:], v) + } + } + if pp.Length() > 0 { + params.Set(pref, pp) + } +} + +func (opts *LoadbalancerAgentParamsOptions) Params() (*jsonutils.JSONDict, error) { + params, err := optionsStructToParams(opts) + if err != nil { + return nil, err + } + opts.setPrefixedParams(params, "vrrp") + opts.setPrefixedParams(params, "haproxy") + return params, nil +} + +type LoadbalancerAgentCreateOptions struct { + NAME string + HbTimeout *int + + LoadbalancerAgentParamsOptions +} + +func (opts *LoadbalancerAgentCreateOptions) Params() (*jsonutils.JSONDict, error) { + params, err := StructToParams(opts) + if err != nil { + return nil, err + } + paramsSub, err := opts.LoadbalancerAgentParamsOptions.Params() + if err != nil { + return nil, err + } + params.Set("params", paramsSub) + return params, nil +} + +type LoadbalancerAgentListOptions struct { + BaseListOptions +} + +type LoadbalancerAgentGetOptions struct { + ID string +} + +type LoadbalancerAgentUpdateOptions struct { + ID string + + HbTimeout *int + + Loadbalancers *time.Time + LoadbalancerListeners *time.Time + LoadbalancerListenerRules *time.Time + LoadbalancerBackendGroups *time.Time + LoadbalancerBackends *time.Time + LoadbalancerAcls *time.Time + LoadbalancerCertificates *time.Time +} + +type LoadbalancerAgentDeleteOptions struct { + ID string +} + +type LoadbalancerAgentActionHbOptions struct { + ID string +} + +type LoadbalancerAgentActionPatchParamsOptions struct { + ID string + + LoadbalancerAgentParamsOptions +} + +func (opts *LoadbalancerAgentActionPatchParamsOptions) Params() (*jsonutils.JSONDict, error) { + return opts.LoadbalancerAgentParamsOptions.Params() +} diff --git a/pkg/mcclient/options/loadbalancerbackendgroups.go b/pkg/mcclient/options/loadbalancerbackendgroups.go new file mode 100644 index 0000000000..58051415aa --- /dev/null +++ b/pkg/mcclient/options/loadbalancerbackendgroups.go @@ -0,0 +1,16 @@ +package options + +type LoadbalancerBackendGroupCreateOptions struct { + NAME string + Loadbalancer string +} +type LoadbalancerBackendGroupGetOptions struct { + ID string +} +type LoadbalancerBackendGroupDeleteOptions struct { + ID string +} +type LoadbalancerBackendGroupListOptions struct { + BaseListOptions + Loadbalancer string +} diff --git a/pkg/mcclient/options/loadbalancerbackends.go b/pkg/mcclient/options/loadbalancerbackends.go new file mode 100644 index 0000000000..76ceb3b1c4 --- /dev/null +++ b/pkg/mcclient/options/loadbalancerbackends.go @@ -0,0 +1,34 @@ +package options + +type LoadbalancerBackendCreateOptions struct { + BackendGroup string `required:true` + Backend string `required:true` + BackendType string `default:guest` + Port *int `required:true` + Weight *int `default:1` +} + +type LoadbalancerBackendListOptions struct { + BaseListOptions + BackendGroup string + Backend string + BackendType string + Weight *int + Address string + Port *int +} + +type LoadbalancerBackendUpdateOptions struct { + ID string + + Weight *int + Port *int +} + +type LoadbalancerBackendGetOptions struct { + ID string +} + +type LoadbalancerBackendDeleteOptions struct { + ID string +} diff --git a/pkg/mcclient/options/loadbalancercertificates.go b/pkg/mcclient/options/loadbalancercertificates.go new file mode 100644 index 0000000000..38b19c7d76 --- /dev/null +++ b/pkg/mcclient/options/loadbalancercertificates.go @@ -0,0 +1,85 @@ +package options + +import ( + "fmt" + "io/ioutil" + + "yunion.io/x/jsonutils" +) + +func loadbalancerCertificateLoadFiles(cert, pkey string, allowEmpty bool) (*jsonutils.JSONDict, error) { + params := jsonutils.NewDict() + pathM := map[string]string{ + "certificate": cert, + "private_key": pkey, + } + for fieldName, path := range pathM { + if path == "" { + if allowEmpty { + continue + } else { + return nil, fmt.Errorf("%s: empty path", fieldName) + } + } + d, err := ioutil.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("%s: read %s: %s", fieldName, path, err) + } + if len(d) == 0 { + return nil, fmt.Errorf("%s: empty file %s", fieldName, path) + } + params.Set(fieldName, jsonutils.NewString(string(d))) + } + return params, nil +} + +type LoadbalancerCertificateCreateOptions struct { + NAME string + + Cert string `required:true json:- help:"path to certificate file"` + Pkey string `required:true json:- help:"path to private key file"` +} + +func (opts *LoadbalancerCertificateCreateOptions) Params() (*jsonutils.JSONDict, error) { + params, err := StructToParams(opts) + if err != nil { + return nil, err + } + paramsCertKey, err := loadbalancerCertificateLoadFiles(opts.Cert, opts.Pkey, false) + if err != nil { + return nil, err + } + params.Update(paramsCertKey) + return params, nil +} + +type LoadbalancerCertificateGetOptions struct { + ID string +} + +type LoadbalancerCertificateDeleteOptions struct { + ID string +} + +type LoadbalancerCertificateListOptions struct { + BaseListOptions + + PublicKeyAlgorithm string + PublicKeyBitLen *int + SignatureAlgorithm string +} + +type LoadbalancerCertificateUpdateOptions struct { + ID string + + Cert string `json:- help:"path to certificate file"` + Pkey string `json:- help:"path to private key file"` +} + +func (opts *LoadbalancerCertificateUpdateOptions) Params() (*jsonutils.JSONDict, error) { + paramsCertKey, err := loadbalancerCertificateLoadFiles(opts.Cert, opts.Pkey, true) + if err != nil { + return nil, err + } + return paramsCertKey, nil +} diff --git a/pkg/mcclient/options/loadbalancerlistenerrules.go b/pkg/mcclient/options/loadbalancerlistenerrules.go new file mode 100644 index 0000000000..7d2e95cb69 --- /dev/null +++ b/pkg/mcclient/options/loadbalancerlistenerrules.go @@ -0,0 +1,34 @@ +package options + +type LoadbalancerListenerRuleCreateOptions struct { + NAME string + Listener string `required:true` + BackendGroup string + Domain string + Path string +} + +type LoadbalancerListenerRuleListOptions struct { + BaseListOptions + Listener string + Domain string + Path string +} + +type LoadbalancerListenerRuleUpdateOptions struct { + ID string + BackendGroup string +} + +type LoadbalancerListenerRuleGetOptions struct { + ID string +} + +type LoadbalancerListenerRuleDeleteOptions struct { + ID string +} + +type LoadbalancerListenerRuleActionStatusOptions struct { + ID string + Status string `choices:"enabled|disabled"` +} diff --git a/pkg/mcclient/options/loadbalancerlisteners.go b/pkg/mcclient/options/loadbalancerlisteners.go new file mode 100644 index 0000000000..e2bfe8f45e --- /dev/null +++ b/pkg/mcclient/options/loadbalancerlisteners.go @@ -0,0 +1,155 @@ +package options + +type LoadbalancerListenerCreateOptions struct { + NAME string + + Loadbalancer string `required:true` + ListenerType string `required:true choices:tcp|udp|http|https` + ListenerPort *int `required:true` + BackendGroup string + + Scheduler string `required:true choices:"rr|wrr|wlc|sch|tch"` + Bandwidth *int + + ClientRequestTimeout *int + ClientIdleTimeout *int + BackendConnectTimeout *int + BackendIdleTimeout *int + + AclStatus string `choices:"on|off"` + AclType string `choices:"black|white"` + Acl string + + HealthCheck string `choices:"on|off"` + HealthCheckType string `choices:"tcp|http"` + + HealthCheckDomain string + HealthCheckURI string + HealthCheckHttpCode string + + HealthCheckRise *int + HealthCheckFall *int + HealthCheckInterval *int + HealthCheckTimeout *int + + HealthCheckReq string + HealthCheckExp string + + StickySession string + StickySessionType string + StickySessionCookie string + StickySessionCookieTimeout *int + + XForwardedFor string `choices:true|false` + Gzip string `choices:true|false` + + Certificate string + TLSCipherPolicy string + EnableHttp2 string `choices:true|false` +} + +type LoadbalancerListenerListOptions struct { + BaseListOptions + + Loadbalancer string + ListenerType string `choices:tcp|udp|http|https` + ListenerPort *int + BackendGroup string + + Scheduler string `choices:"rr|wrr|wlc|sch|tch"` + Bandwidth *int + + ClientRequestTimeout *int + ClientIdleTimeout *int + BackendConnectTimeout *int + BackendIdleTimeout *int + + AclStatus string `choices:"on|off"` + AclType string `choices:"black|white"` + Acl string + + HealthCheck string `choices:"on|off"` + HealthCheckType string `choices:"tcp|http"` + + HealthCheckDomain string + HealthCheckURI string + HealthCheckHttpCode string + + HealthCheckRise *int + HealthCheckFall *int + HealthCheckInterval *int + HealthCheckTimeout *int + + HealthCheckReq string + HealthCheckExp string + + StickySession string + StickySessionType string + StickySessionCookie string + StickySessionCookieTimeout *int + + XForwardedFor string `choices:true|false` + Gzip string `choices:true|false` + + Certificate string + TLSCipherPolicy string + EnableHttp2 string `choices:true|false` +} + +type LoadbalancerListenerUpdateOptions struct { + ID string + + BackendGroup string + + Scheduler string `choices:"rr|wrr|wlc|sch|tch"` + Bandwidth *int + + ClientRequestTimeout *int + ClientIdleTimeout *int + BackendConnectTimeout *int + BackendIdleTimeout *int + + AclStatus string `choices:"on|off"` + AclType string `choices:"black|white"` + Acl string + + HealthCheck string `choices:"on|off"` + HealthCheckType string `choices:"tcp|http"` + + HealthCheckDomain string + HealthCheckURI string + HealthCheckHttpCode string + + HealthCheckRise *int + HealthCheckFall *int + HealthCheckInterval *int + HealthCheckTimeout *int + + HealthCheckReq string + HealthCheckExp string + + StickySession string + StickySessionType string + StickySessionCookie string + StickySessionCookieTimeout *int + + XForwardedFor string `choices:true|false` + Gzip string `choices:true|false` + + Certificate string + TLSCipherPolicy string + EnableHttp2 string `choices:true|false` +} + +type LoadbalancerListenerGetOptions struct { + ID string +} + +type LoadbalancerListenerDeleteOptions struct { + ID string +} + +type LoadbalancerListenerActionStatusOptions struct { + ID string + Status string `choices:"enabled|disabled"` +} diff --git a/pkg/mcclient/options/loadbalancers.go b/pkg/mcclient/options/loadbalancers.go new file mode 100644 index 0000000000..185c321dbd --- /dev/null +++ b/pkg/mcclient/options/loadbalancers.go @@ -0,0 +1,35 @@ +package options + +type LoadbalancerCreateOptions struct { + NAME string + Network string + Address string +} + +type LoadbalancerGetOptions struct { + ID string +} + +type LoadbalancerUpdateOptions struct { + ID string + BackendGroup string +} + +type LoadbalancerDeleteOptions struct { + ID string +} + +type LoadbalancerListOptions struct { + BaseListOptions + Zone string + Address string + AddressType string `choices:"intranet|internet"` + NetworkType string `choices:"classic|vpc"` + Network string + BackendGroup string +} + +type LoadbalancerActionStatusOptions struct { + ID string + Status string `choices:"enabled|disabled"` +}