From 99e2a1a948dc7db90158e37044b717d3dd2f131e Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Thu, 12 Dec 2019 16:07:17 +0000 Subject: [PATCH] hostman: add OvnHelper --- pkg/hostman/guestman/qemu-kvm.go | 12 ++ pkg/hostman/hostinfo/hostbridge/ovs.go | 25 ++- pkg/hostman/hostinfo/hostinfo.go | 25 +++ pkg/hostman/hostinfo/hostovn.go | 159 +++++++++++++++++++ pkg/hostman/hostinfo/hostovn_test.go | 67 ++++++++ pkg/hostman/options/options.go | 4 + pkg/hostman/system_service/ovn_conroller.go | 33 ++++ pkg/hostman/system_service/system_service.go | 19 +-- 8 files changed, 330 insertions(+), 14 deletions(-) create mode 100644 pkg/hostman/hostinfo/hostovn.go create mode 100644 pkg/hostman/hostinfo/hostovn_test.go create mode 100644 pkg/hostman/system_service/ovn_conroller.go diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index a14249813a..84c9cfd330 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -785,6 +785,18 @@ func (s *SKVMGuestInstance) SaveDesc(desc jsonutils.JSONObject) error { if !ok { return fmt.Errorf("Unknown desc format, not JSONDict") } + { + // fill in ovn vpc nic bridge field + nics, _ := s.Desc.GetArray("nics") + ovnBridge := options.HostOptions.OvnIntegrationBridge + for _, nic := range nics { + vpcProvider, _ := nic.GetString("vpc", "provider") + if vpcProvider == compute.VPC_PROVIDER_OVN { + nicjd := nic.(*jsonutils.JSONDict) + nicjd.Set("bridge", jsonutils.NewString(ovnBridge)) + } + } + } if err := fileutils2.FilePutContents(s.GetDescFilePath(), desc.String(), false); err != nil { log.Errorln(err) } diff --git a/pkg/hostman/hostinfo/hostbridge/ovs.go b/pkg/hostman/hostinfo/hostbridge/ovs.go index a5c45cafe7..50baa9ddd1 100644 --- a/pkg/hostman/hostinfo/hostbridge/ovs.go +++ b/pkg/hostman/hostinfo/hostbridge/ovs.go @@ -22,6 +22,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/hostman/system_service" "yunion.io/x/onecloud/pkg/util/bwutils" @@ -116,19 +117,26 @@ func (o *SOVSBridgeDriver) GenerateIfupScripts(scriptPath string, nic jsonutils. func (o *SOVSBridgeDriver) getUpScripts(nic jsonutils.JSONObject) (string, error) { var ( - bridge, _ = nic.GetString("bridge") - ifname, _ = nic.GetString("ifname") - ip, _ = nic.GetString("ip") - mac, _ = nic.GetString("mac") - vlan, _ = nic.Int("vlan") + bridge, _ = nic.GetString("bridge") + ifname, _ = nic.GetString("ifname") + ip, _ = nic.GetString("ip") + mac, _ = nic.GetString("mac") + netId, _ = nic.GetString("net_id") + vlan, _ = nic.Int("vlan") + vpcProvider, _ = nic.GetString("vpc", "provider") ) + if vpcProvider == compute.VPC_PROVIDER_OVN { + bridge = options.HostOptions.OvnIntegrationBridge + } + s := "#!/bin/bash\n\n" s += fmt.Sprintf("SWITCH='%s'\n", bridge) s += fmt.Sprintf("IF='%s'\n", ifname) s += fmt.Sprintf("IP='%s'\n", ip) s += fmt.Sprintf("MAC='%s'\n", mac) s += fmt.Sprintf("VLAN_ID=%d\n", vlan) + s += fmt.Sprintf("NET_ID=%s\n", netId) limit, burst, err := bwutils.GetOvsBwValues(nic) if err != nil { return "", err @@ -154,6 +162,9 @@ func (o *SOVSBridgeDriver) getUpScripts(nic jsonutils.JSONObject) (string, error s += " TAG=\"tag=$VLAN_ID\"\n" s += "fi\n" s += "ovs-vsctl add-port $SWITCH $IF $TAG\n" + if vpcProvider == compute.VPC_PROVIDER_OVN { + s += "ovs-vsctl set Interface $IF external_ids:iface-id=iface-$NET_ID-$IF\n" + } s += "PORT=$(ovs-ofctl show $SWITCH | grep -w $IF)\n" s += "PORT=$(echo $PORT | awk 'BEGIN{FS=\"(\"}{print $1}')\n" s += "OFCTL=$(ovs-vsctl get-controller $SWITCH)\n" @@ -264,3 +275,7 @@ func NewOVSBridgeDriver(bridge, inter, ip string) (*SOVSBridgeDriver, error) { ovsDrv.drv = ovsDrv return ovsDrv, nil } + +func NewOVSBridgeDriverByName(bridge string) (*SOVSBridgeDriver, error) { + return NewOVSBridgeDriver(bridge, "", "") +} diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index 50ed8e6700..4b5d5b4e36 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -93,6 +93,14 @@ func (h *SHostInfo) GetBridgeDev(bridge string) hostbridge.IBridgeDriver { return n.BridgeDev } } + if bridge == options.HostOptions.OvnIntegrationBridge { + drv, err := hostbridge.NewOVSBridgeDriverByName(bridge) + if err != nil { + log.Errorf("create ovn bridge driver: %v", err) + return nil + } + return drv + } return nil } @@ -133,6 +141,9 @@ func (h *SHostInfo) Init() error { if err := h.parseConfig(); err != nil { return err } + if err := h.setupOvnChassis(); err != nil { + return err + } log.Infof("Start detectHostInfo") if err := h.detectHostInfo(); err != nil { return err @@ -140,6 +151,19 @@ func (h *SHostInfo) Init() error { return nil } +func (h *SHostInfo) setupOvnChassis() error { + opts := &options.HostOptions + if opts.BridgeDriver != hostbridge.DRV_OPEN_VSWITCH { + return nil + } + log.Infof("Start setting up ovn chassis") + oh := NewOvnHelper(h) + if err := oh.Init(); err != nil { + return err + } + return nil +} + func (h *SHostInfo) generateLocalNetworkConfig() (string, error) { netIp, dev, err := netutils2.DefaultSrcIpDev() if err != nil { @@ -846,6 +870,7 @@ func (h *SHostInfo) updateHostRecord(hostId string) { } content.Set("__meta__", jsonutils.Marshal(h.getSysInfo())) content.Set("version", jsonutils.NewString(version.GetShortString())) + content.Set("ovn_version", jsonutils.NewString(MustGetOvnVersion())) var ( res jsonutils.JSONObject diff --git a/pkg/hostman/hostinfo/hostovn.go b/pkg/hostman/hostinfo/hostovn.go new file mode 100644 index 0000000000..a3e2d5c843 --- /dev/null +++ b/pkg/hostman/hostinfo/hostovn.go @@ -0,0 +1,159 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostinfo + +import ( + "fmt" + + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/hostman/options" + "yunion.io/x/onecloud/pkg/hostman/system_service" + "yunion.io/x/onecloud/pkg/util/netutils2" + "yunion.io/x/onecloud/pkg/util/procutils" +) + +const ( + ErrOvnService = errors.Error("ovn controller") + ErrOvnConfig = errors.Error("ovn controller configuration") +) + +type OvnHelper struct { + hi *SHostInfo +} + +func NewOvnHelper(hi *SHostInfo) *OvnHelper { + oh := &OvnHelper{ + hi: hi, + } + return oh +} + +func (oh *OvnHelper) Init() (err error) { + defer func() { + if panicVal := recover(); panicVal != nil { + err = panicVal.(error) + } + }() + oh.mustPrepOvsdbConfig() + oh.mustPrepService() + return nil +} + +func (oh *OvnHelper) mustPrepOvsdbConfig() { + var ( + args = []string{"set", "Open_vSwitch", "."} + opts = &options.HostOptions + ) + { + if opts.OvnIntegrationBridge == "" { + panic(errors.Wrap(ErrOvnConfig, "bad config: ovn_integration_bridge")) + } + args = append(args, fmt.Sprintf("external_ids:ovn-bridge=%s", + opts.OvnIntegrationBridge)) + } + { + encapIp := opts.OvnEncapIp + if encapIp == "" { + var err error + encapIp, err = netutils2.MyIP() + if err != nil { + panic(errors.Wrap(ErrOvnConfig, "determine default encap ip")) + } + } + args = append(args, "external_ids:ovn-encap-type=geneve") + args = append(args, fmt.Sprintf("external_ids:ovn-encap-ip=%s", encapIp)) + } + { + if opts.OvnSouthDatabase == "" { + panic(errors.Wrap(ErrOvnConfig, "bad config: ovn_south_database")) + } + args = append(args, fmt.Sprintf("external_ids:ovn-remote=%s", + opts.OvnSouthDatabase)) + } + output, err := procutils.NewCommand("ovs-vsctl", args...).Output() + if err != nil { + panic(errors.Wrapf(err, "configuring ovn-controller: %s", string(output))) + } +} + +func (oh *OvnHelper) mustPrepService() { + ovn := system_service.GetService("ovn-controller") + if !ovn.IsInstalled() { + panic(errors.Wrap(ErrOvnService, "not installed")) + } + if ovn.IsEnabled() { + // - ovn-controller Requires "openvswitch.service" + // - openvswitch service should be disabled on startup + if err := ovn.Disable(); err != nil { + panic(errors.Wrap(err, "disable ovn-controller on startup")) + } + } + if err := ovn.Start(false); err != nil { + panic(errors.Wrap(err, "start ovn-controller")) + } +} + +func MustGetOvnVersion() string { + output, err := procutils.NewCommand("ovn-controller", "--version").Output() + if err != nil { + return "" + } + return ovnExtractVersion(string(output)) +} + +func ovnExtractVersion(in string) string { + r := make([]rune, 0, 8) + var ( + dot = false + ndot = 0 + digit = 0 + ) + reset := func() { + dot = false + ndot = 0 + digit = 0 + } + for _, c := range in { + switch { + case c == '.': + if dot || digit == 0 { + reset() + continue + } + r = append(r, c) + dot = true + ndot += 1 + digit = 0 + case c >= '0' && c <= '9': + dot = false + if digit < 3 { + r = append(r, c) + digit += 1 + continue + } + reset() + default: + if ndot > 0 && ndot < 3 { + return string(r) + } + reset() + } + } + if ndot > 0 && ndot < 3 { + return string(r) + } + return "" +} diff --git a/pkg/hostman/hostinfo/hostovn_test.go b/pkg/hostman/hostinfo/hostovn_test.go new file mode 100644 index 0000000000..7929824742 --- /dev/null +++ b/pkg/hostman/hostinfo/hostovn_test.go @@ -0,0 +1,67 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostinfo + +import ( + "testing" +) + +func TestMustGetOvnVersion(t *testing.T) { + cases := []struct { + in string + out string + }{ + { + in: ` +ovn-controller (Open vSwitch) 2.9 +OpenFlow versions 0x4:0x4 +`, + out: "2.9", + }, + { + in: ` +ovn-controller (Open vSwitch) 2.9.6 +OpenFlow versions 0x4:0x4 +`, + out: "2.9.6", + }, + { + in: ` +ovn-controller (Open vSwitch) 2.9.100 +OpenFlow versions 0x4:0x4 +`, + out: "2.9.100", + }, + { + in: ` +ovn-controller (Open vSwitch) 2.9.1000 +OpenFlow versions 0x4:0x4 +`, + out: "", + }, + { + in: ` +ovn-controller (Open vSwitch) 2.9.6.1 +`, + out: "", + }, + } + for _, c := range cases { + got := ovnExtractVersion(c.in) + if got != c.out { + t.Fatalf("got: %s, want: %s, input:\n%s", got, c.out, c.in) + } + } +} diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index 802d2df4fd..e5f6e7394e 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -110,6 +110,10 @@ type SHostOptions struct { EnableRemoteExecutor bool `help:"Enable remote executor" default:"false"` ExecutorSocketPath string `help:"Executor socket path" default:"/var/run/exec.sock"` CommonConfigFile string `help:"common config file for container"` + + OvnSouthDatabase string `help:"address for accessing ovn south database" default:"unix:/var/run/openvswitch/ovnsb_db.sock"` + OvnIntegrationBridge string `help:"name of integration bridge for logical ports" default:"brvpc"` + OvnEncapIp string `help:"encap ip for ovn datapath. Default to output src address of default route"` } var HostOptions SHostOptions diff --git a/pkg/hostman/system_service/ovn_conroller.go b/pkg/hostman/system_service/ovn_conroller.go new file mode 100644 index 0000000000..a52e962967 --- /dev/null +++ b/pkg/hostman/system_service/ovn_conroller.go @@ -0,0 +1,33 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package system_service + +type SOvnController struct { + *SBaseSystemService +} + +func NewOvnControllerService() *SOvnController { + return &SOvnController{ + NewBaseSystemService("ovn-controller", nil), + } +} + +func (s *SOvnController) Reload(kwargs map[string]interface{}) error { + return s.reload(s.GetConfig(kwargs), s.GetConfigFile()) +} + +func (s *SOvnController) BgReload(kwargs map[string]interface{}) { + go s.reload(s.GetConfig(kwargs), s.GetConfigFile()) +} diff --git a/pkg/hostman/system_service/system_service.go b/pkg/hostman/system_service/system_service.go index df422f8983..4978209402 100644 --- a/pkg/hostman/system_service/system_service.go +++ b/pkg/hostman/system_service/system_service.go @@ -44,15 +44,16 @@ var serviceMap map[string]ISystemService func Init() { serviceMap = map[string]ISystemService{ - "ntpd": NewNtpdService(), - "telegraf": NewTelegrafService(), - "host_sdnagent": NewHostSdnagentService(), - "openvswitch": NewOpenvswitchService(), - "fluentbit": NewFluentbitService(), - "kube_agent": NewKubeAgentService(), - "lxcfs": NewLxcfsService(), - "docker": NewDockerService(), - "host-deployer": NewHostDeployerService(), + "ntpd": NewNtpdService(), + "telegraf": NewTelegrafService(), + "host_sdnagent": NewHostSdnagentService(), + "openvswitch": NewOpenvswitchService(), + "ovn-controller": NewOvnControllerService(), + "fluentbit": NewFluentbitService(), + "kube_agent": NewKubeAgentService(), + "lxcfs": NewLxcfsService(), + "docker": NewDockerService(), + "host-deployer": NewHostDeployerService(), } }