From cc56d09cebef1ef96b3a8a9c791b6f3c4ca99348 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Mon, 29 Apr 2019 20:30:14 +0800 Subject: [PATCH] golang host support linux bridge --- pkg/hostman/guestman/qemu-kvm.go | 2 +- pkg/hostman/hostinfo/hostbridge/hostbridge.go | 385 +++--------------- .../hostinfo/hostbridge/linux_bridge.go | 161 ++++++++ pkg/hostman/hostinfo/hostbridge/ovs.go | 319 +++++++++++++++ pkg/hostman/hostinfo/hostinfohelper.go | 12 +- 5 files changed, 553 insertions(+), 326 deletions(-) create mode 100644 pkg/hostman/hostinfo/hostbridge/linux_bridge.go create mode 100644 pkg/hostman/hostinfo/hostbridge/ovs.go diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index 37c3b863ea..aa20e818d5 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -225,7 +225,7 @@ func (s *SKVMGuestInstance) asyncScriptStart(ctx context.Context, params interfa return nil, hostutils.ParamsError } - hostbridge.CleanDeletedPorts() + hostbridge.CleanDeletedPorts(options.HostOptions.BridgeDriver) time.Sleep(100 * time.Millisecond) var isStarted, tried = false, 0 diff --git a/pkg/hostman/hostinfo/hostbridge/hostbridge.go b/pkg/hostman/hostinfo/hostbridge/hostbridge.go index ce8a43a2ce..f63f76220d 100644 --- a/pkg/hostman/hostinfo/hostbridge/hostbridge.go +++ b/pkg/hostman/hostinfo/hostbridge/hostbridge.go @@ -18,7 +18,6 @@ import ( "fmt" "net" "os" - "strings" "syscall" "yunion.io/x/jsonutils" @@ -26,27 +25,34 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/hostman/options" - "yunion.io/x/onecloud/pkg/hostman/system_service" - "yunion.io/x/onecloud/pkg/util/bwutils" "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/netutils2" - "yunion.io/x/onecloud/pkg/util/ovsutils" "yunion.io/x/onecloud/pkg/util/procutils" ) type IBridgeDriver interface { ConfirmToConfig(bool, []string) (bool, error) - Setup() error - Exists() bool - Interfaces() []string - - FetchConfig() GetMac() string + FetchConfig() + Setup(IBridgeDriver) error + SetupAddresses(net.IPMask) error + SetupSlaveAddresses([][]string) error + SetupRoutes(routes [][]string) error + BringupInterface() error + + Exists() (bool, error) + Interfaces() ([]string, error) + WarmupConfig() error + CleanupConfig() + SetupBridgeDev() error + SetupInterface() error + GenerateIfupScripts(scriptPath string, nic jsonutils.JSONObject) error GenerateIfdownScripts(scriptPath string, nic jsonutils.JSONObject) error RegisterHostlocalServer(mac, ip string) error - WarmupConfig() error - CleanupConfig() + + getUpScripts(nic jsonutils.JSONObject) (string, error) + getDownScripts(nic jsonutils.JSONObject) (string, error) } type SBaseBridgeDriver struct { @@ -207,66 +213,35 @@ func (d *SBaseBridgeDriver) SetupRoutes(routes [][]string) error { return nil } -type SOVSBridgeDriver struct { - SBaseBridgeDriver -} - -func (o *SOVSBridgeDriver) CleanupConfig() { - ovsutils.CleanAllHiddenPorts() - // if enableopenflowcontroller ... -} - -func (o *SOVSBridgeDriver) Exists() bool { - data, err := procutils.NewCommand("ovs-vsctl", "list-br").Run() - if err != nil { - log.Errorln(err) - return false - } - for _, d := range strings.Split(string(data), "\n") { - if strings.TrimSpace(d) == o.bridge.String() { - return true - } - } - return false -} - -func (o *SOVSBridgeDriver) Interfaces() []string { - data, err := procutils.NewCommand("ovs-vsctl", "list-ifaces", o.bridge.String()).Run() - if err != nil { - log.Errorln(err) - return nil - } - - var infs = make([]string, 0) - for _, d := range strings.Split(string(data), "\n") { - if len(strings.TrimSpace(d)) > 0 { - infs = append(infs, strings.TrimSpace(d)) - } - } - return infs -} - -func (o *SOVSBridgeDriver) Setup() error { +func (d *SBaseBridgeDriver) Setup(o IBridgeDriver) error { var routes [][]string var slaveAddrs [][]string - if o.inter != nil && len(o.inter.Addr) > 0 { - routes = o.inter.GetRoutes(true) - slaveAddrs = o.inter.GetSlaveAddresses() + if d.inter != nil && len(d.inter.Addr) > 0 { + routes = d.inter.GetRoutes(true) + slaveAddrs = d.inter.GetSlaveAddresses() } - if !o.Exists() { + exist, err := o.Exists() + if err != nil { + return err + } + if !exist { if err := o.SetupBridgeDev(); err != nil { return err } } - if o.inter != nil && !utils.IsInStringArray(o.inter.String(), o.Interfaces()) { + infs, err := o.Interfaces() + if err != nil { + return err + } + if d.inter != nil && !utils.IsInStringArray(d.inter.String(), infs) { if err := o.SetupInterface(); err != nil { return err } } - if len(o.bridge.Addr) == 0 { - if len(o.ip) > 0 { - if err := o.SetupAddresses(o.inter.Mask); err != nil { + if len(d.bridge.Addr) == 0 { + if len(d.ip) > 0 { + if err := o.SetupAddresses(d.inter.Mask); err != nil { return err } if len(slaveAddrs) > 0 { @@ -289,301 +264,65 @@ func (o *SOVSBridgeDriver) Setup() error { return o.BringupInterface() } -func (o *SOVSBridgeDriver) SetupInterface() error { - if o.inter != nil && !utils.IsInStringArray(o.inter.String(), o.Interfaces()) { - output, err := procutils.NewCommand("ovs-vsctl", "--", "--may-exist", - "add-port", o.bridge.String(), o.inter.String()).Run() - if err != nil { - return fmt.Errorf("Failed to add interface %s", output) - } - } - return nil +func (d *SBaseBridgeDriver) CleanupConfig() { + // pass } -func (o *SOVSBridgeDriver) SetupBridgeDev() error { - if !o.Exists() { - _, err := procutils.NewCommand("ovs-vsctl", "--", "--may-exist", "add-br", o.bridge.String()).Run() - return err - } - return nil -} - -func (o *SOVSBridgeDriver) GenerateIfdownScripts(scriptPath string, nic jsonutils.JSONObject) error { - script, err := o.getDownScripts(nic) - if err != nil { - log.Errorln(err) - return err - } - return o.saveFileExecutable(scriptPath, script) -} - -func (o *SOVSBridgeDriver) GenerateIfupScripts(scriptPath string, nic jsonutils.JSONObject) error { - script, err := o.getUpScripts(nic) - if err != nil { - log.Errorln(err) - return err - } - return o.saveFileExecutable(scriptPath, script) -} - -func (o *SOVSBridgeDriver) saveFileExecutable(scriptPath, script string) error { +func (d *SBaseBridgeDriver) saveFileExecutable(scriptPath, script string) error { if err := fileutils2.FilePutContents(scriptPath, script, false); err != nil { return err } return os.Chmod(scriptPath, syscall.S_IRUSR|syscall.S_IWUSR|syscall.S_IXUSR) } -func (o *SOVSBridgeDriver) getUpScripts(nic jsonutils.JSONObject) (string, error) { - var ( - bridge, _ = nic.GetString("bridge") - ifname, _ = nic.GetString("ifname") - ip, _ = nic.GetString("ip") - mac, _ = nic.GetString("mac") - vlan, _ = nic.Int("vlan") - ) - - s := "#!/bin/bash\n\n" - s += fmt.Sprintf("SWITCH='%s'\n", bridge) - s += fmt.Sprintf("IF='%s'\n", ifname) - s += fmt.Sprintf("IP='%s'\n", ip) - s += fmt.Sprintf("MAC='%s'\n", mac) - s += fmt.Sprintf("VLAN_ID=%d\n", vlan) - limit, burst, err := bwutils.GetOvsBwValues(nic) +func (d *SBaseBridgeDriver) generateIfdownScripts(driver IBridgeDriver, scriptPath string, nic jsonutils.JSONObject) error { + script, err := driver.getDownScripts(nic) if err != nil { - return "", err + log.Errorln(err) + return err } - s += fmt.Sprintf("LIMIT=%d\n", limit) - s += fmt.Sprintf("BURST=%d\n", burst) - bwDownload, err := bwutils.GetDownloadBwValue(nic, options.HostOptions.BwDownloadBandwidth) + return d.saveFileExecutable(scriptPath, script) +} + +func (d *SBaseBridgeDriver) generateIfupScripts(driver IBridgeDriver, scriptPath string, nic jsonutils.JSONObject) error { + script, err := driver.getUpScripts(nic) if err != nil { - return "", err + log.Errorln(err) + return err } - s += fmt.Sprintf("LIMIT_DOWNLOAD='%dmbit'\n", bwDownload) - if options.HostOptions.TunnelPaddingBytes > 0 { - s += fmt.Sprintf("/sbin/ifconfig $IF mtu %d\n", - 1500+options.HostOptions.TunnelPaddingBytes) - } - s += "/sbin/ifconfig $IF 0.0.0.0 up\n" - s += "ovs-vsctl list-ifaces $SWITCH | grep -w $IF > /dev/null 2>&1\n" - s += "if [ $? -eq '0' ]; then\n" - s += " ovs-vsctl del-port $SWITCH $IF\n" - s += "fi\n" - s += "if [ \"$VLAN_ID\" -ne \"1\" ]; then\n" - s += " TAG=\"tag=$VLAN_ID\"\n" - s += "fi\n" - s += "ovs-vsctl add-port $SWITCH $IF $TAG\n" - s += "PORT=$(ovs-ofctl show $SWITCH | grep -w $IF)\n" - s += "PORT=$(echo $PORT | awk 'BEGIN{FS=\"(\"}{print $1}')\n" - s += "OFCTL=$(ovs-vsctl get-controller $SWITCH)\n" - s += "if [ -z \"$OFCTL\" ]; then\n" - s += " ovs-vsctl set Interface $IF ingress_policing_rate=$LIMIT\n" - s += " ovs-vsctl set Interface $IF ingress_policing_burst=$BURST\n" - for _, r := range o.GetOfRules(nic) { - s += " " + o.AddFlow(r.cond, r.priority, r.actions) - } - s += "fi\n" - s += "if [ $LIMIT_DOWNLOAD != \"0mbit\" ]; then\n" - s += " tc qdisc del dev $IF root 2>/dev/null\n" - s += " tc qdisc add dev $IF root handle 1: htb default 10\n" - s += " tc class add dev $IF parent 1: classid 1:1 htb " + - "rate $LIMIT_DOWNLOAD ceil $LIMIT_DOWNLOAD\n" - s += " tc class add dev $IF parent 1:1 classid 1:10 htb " + - "rate $LIMIT_DOWNLOAD ceil $LIMIT_DOWNLOAD\n" - s += "fi\n" - return s, nil + return d.saveFileExecutable(scriptPath, script) } -func (o *SOVSBridgeDriver) getDownScripts(nic jsonutils.JSONObject) (string, error) { - var ( - bridge, _ = nic.GetString("bridge") - ifname, _ = nic.GetString("ifname") - ip, _ = nic.GetString("ip") - mac, _ = nic.GetString("mac") - vlan, _ = nic.Int("vlan") - ) - - s := "#!/bin/bash\n\n" - s += fmt.Sprintf("SWITCH='%s'\n", bridge) - s += fmt.Sprintf("IF='%s'\n", ifname) - s += fmt.Sprintf("IP='%s'\n", ip) - s += fmt.Sprintf("MAC='%s'\n", mac) - s += fmt.Sprintf("VLAN_ID=%d\n", vlan) - s += "PORT=$(ovs-ofctl show $SWITCH | grep -w $IF)\n" - s += "if [ $? -ne '0' ]; then\n" - s += " exit 0\n" - s += "fi\n" - s += "OFCTL=$(ovs-vsctl get-controller $SWITCH)\n" - s += "PORT=$(echo $PORT | awk 'BEGIN{FS=\"(\"}{print $1}')\n" - s += "if [ -z \"$OFCTL\" ]; then\n" - for _, r := range o.GetOfRules(nic) { - s += " " + o.DelFlow(r.cond) - } - s += "fi\n" - s += "/sbin/ifconfig $IF 0.0.0.0 down\n" - s += "ovs-vsctl -- --if-exists del-port $SWITCH $IF\n" - return s, nil -} - -type SRule struct { - priority int - cond string - actions string -} - -func (o *SOVSBridgeDriver) AddFlow(cond string, priority int, actions string) string { - s := "" - s += fmt.Sprintf("ovs-ofctl add-flow $SWITCH \"%s", cond) - s += fmt.Sprintf(" priority=%d", priority) - s += fmt.Sprintf(" actions=%s\"\n", actions) - return s -} - -func (o *SOVSBridgeDriver) DoAddFlow(cond string, pri int, actions, swt string) error { - _, err := procutils.NewCommand("ovs-ofctl", "add-flow", swt, - fmt.Sprintf("%s priority=%d actions=%s", cond, pri, actions)).Run() - return err -} - -func (o *SOVSBridgeDriver) DelFlow(cond string) string { - return fmt.Sprintf("ovs-ofctl del-flows $SWITCH \"%s\"\n", cond) -} - -func (o *SOVSBridgeDriver) GetOfRules(nic jsonutils.JSONObject) []SRule { - rules := []SRule{} - metadataPort := o.GetMetadataServerPort() - rules = append(rules, - SRule{9000, fmt.Sprintf("table=0 in_port=local tcp nw_dst=$IP tp_src=%d", metadataPort), - "mod_nw_src=169.254.169.254,mod_tp_src:80,output:$PORT"}, - SRule{9500, "table=0 in_port=$PORT udp tp_src=68 tp_dst=67", "local"}, - SRule{8000, "table=0 in_port=$PORT", "resubmit(,1)"}, - ) - if vlan, _ := nic.Int("vlan"); vlan != 1 { - rules = append(rules, - SRule{4901, "table=1 dl_dst=$MAC,dl_vlan=$VLAN_ID", "strip_vlan,output:$PORT"}) - } - rules = append(rules, - SRule{4900, "table=1 dl_dst=$MAC", "output:$PORT"}) - return rules -} - -func (o *SOVSBridgeDriver) GetMetadataServerPort() int { +func (d *SBaseBridgeDriver) GetMetadataServerPort() int { return options.HostOptions.Port + 1000 } -func (o *SOVSBridgeDriver) RegisterHostlocalServer(mac, ip string) error { - if !options.HostOptions.EnableOpenflowController { - metadataPort := o.GetMetadataServerPort() - if err := o.DoAddFlow("table=0 ipv6", 20000, "drop", o.bridge.String()); err != nil { - log.Errorln(err) - return err - } - if err := o.DoAddFlow("table=0 tcp nw_dst=169.254.169.254 tp_dst=80", 10000, - fmt.Sprintf("mod_dl_dst:%s,mod_nw_dst:%s,mod_tp_dst:%d,local", - mac, ip, metadataPort), - o.bridge.String()); err != nil { - log.Errorln(err) - return err - } - log.Infof("OVS: metadata server %s:%d", ip, metadataPort) - - k8sCidr := options.HostOptions.K8sClusterCidr - if len(k8sCidr) > 0 { - addr, mask, err := netutils2.PrefixSplit(k8sCidr) - if err != nil { - log.Errorln(err) - return err - } - k8sCidr = fmt.Sprintf("%s/%d", addr, mask) - log.Infof("OVS: Kubernetes cluster IP range: %s", k8sCidr) - err = o.DoAddFlow(fmt.Sprintf("table=0 ip,nw_dst=%s", k8sCidr), - 10050, fmt.Sprintf("mod_dl_dst:%s,local", mac), o.bridge.String()) - if err != nil { - log.Errorln(err) - return err - } - } - - err := o.DoAddFlow("table=0", 0, "resubmit(,1)", o.bridge.String()) - if err != nil { - log.Errorln(err) - return err - } - err = o.DoAddFlow("table=1", 0, "normal", o.bridge.String()) - if err != nil { - log.Errorln(err) - return err - } - } +func (d *SBaseBridgeDriver) WarmupConfig() error { return nil } -func (o *SOVSBridgeDriver) ovsSetParams(params map[string]map[string]string) { - for tbl, tblval := range params { - for k, v := range tblval { - procutils.NewCommand("ovs-vsctl", "set", tbl, o.bridge.String(), - fmt.Sprintf("%s=%s", k, v)).Run() - } - } -} - -func (o *SOVSBridgeDriver) WarmupConfig() error { - // if options.OvsSflowBridges ... - if options.HostOptions.EnableOpenflowController { - // ... - } else { - params := map[string]map[string]string{ - "bridge": { - "stp_enable": "false", - "fail_mode": "standalone", - "other-config:flow-eviction-threshold": "2500", - }, - } - o.ovsSetParams(params) - } - return nil -} - -func OVSPrepare() error { - ovs := system_service.GetService("openvswitch") - if !ovs.IsInstalled() { - return fmt.Errorf("Service openvswitch not installed!") - } - if !ovs.IsActive() { - return ovs.Start(false) - } - return nil -} - -func CleanOvsBridge() { - ovsutils.CleanAllHiddenPorts() -} - -func NewOVSBridgeDriver(bridge, inter, ip string) (*SOVSBridgeDriver, error) { - base, err := NewBaseBridgeDriver(bridge, inter, ip) - if err != nil { - return nil, err - } - return &SOVSBridgeDriver{*base}, nil -} - func NewDriver(bridgeDriver, bridge, inter, ip string) (IBridgeDriver, error) { if bridgeDriver == "openvswitch" { return NewOVSBridgeDriver(bridge, inter, ip) - } else { - return nil, fmt.Errorf("Not Implentment") + } else if bridgeDriver == "linux_bridge" { + return NewLinuxBridgeDeriver(bridge, inter, ip) } + return nil, fmt.Errorf("Dirver %s not found", bridgeDriver) } func Prepare(bridgeDriver string) error { if bridgeDriver == "openvswitch" { return OVSPrepare() - } else { - return fmt.Errorf("Not Implentment") + } else if bridgeDriver == "linux_bridge" { + return LinuxBridgePrepare() } + return fmt.Errorf("Dirver %s not found", bridgeDriver) } -func CleanDeletedPorts() { - if options.HostOptions.BridgeDriver == "openvswitch" { - CleanOvsBridge() +func CleanDeletedPorts(bridgeDriver string) { + if bridgeDriver == "openvswitch" { + cleanOvsBridge() + } else if bridgeDriver == "linux_bridge" { + cleanLinuxBridge() } } diff --git a/pkg/hostman/hostinfo/hostbridge/linux_bridge.go b/pkg/hostman/hostinfo/hostbridge/linux_bridge.go new file mode 100644 index 0000000000..cf9d91c1dd --- /dev/null +++ b/pkg/hostman/hostinfo/hostbridge/linux_bridge.go @@ -0,0 +1,161 @@ +package hostbridge + +import ( + "bytes" + "fmt" + "regexp" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/hostman/options" + "yunion.io/x/onecloud/pkg/util/procutils" + "yunion.io/x/pkg/utils" +) + +func NewLinuxBridgeDeriver(bridge, inter, ip string) (*SLinuxBridgeDriver, error) { + base, err := NewBaseBridgeDriver(bridge, inter, ip) + if err != nil { + return nil, err + } + return &SLinuxBridgeDriver{*base}, nil +} + +func LinuxBridgePrepare() error { + return nil +} + +func cleanLinuxBridge() { + // pass +} + +type SLinuxBridgeDriver struct { + SBaseBridgeDriver +} + +func (l *SLinuxBridgeDriver) Exists() (bool, error) { + data, err := procutils.NewCommand("brctl", "show").Run() + if err != nil { + return false, err + } + + re := regexp.MustCompile(`\s+`) + for _, line := range bytes.Split(data, []byte{'\n'}) { + info := re.Split(string(line), -1) + if info[0] == l.bridge.String() { + return true, nil + } + } + return false, nil +} + +func (l *SLinuxBridgeDriver) Interfaces() ([]string, error) { + data, err := procutils.NewCommand("brctl", "show", l.bridge.String()).Run() + if err != nil { + return nil, err + } + + infs := make([]string, 0) + re := regexp.MustCompile(`\s+`) + for _, line := range bytes.Split(data, []byte{'\n'}) { + info := re.Split(string(line), -1) + infs = append(infs, info[len(info)-1]) + } + return infs, nil +} + +func (l *SLinuxBridgeDriver) GenerateIfdownScripts(scriptPath string, nic jsonutils.JSONObject) error { + return l.generateIfdownScripts(l, scriptPath, nic) +} + +func (l *SLinuxBridgeDriver) GenerateIfupScripts(scriptPath string, nic jsonutils.JSONObject) error { + return l.generateIfupScripts(l, scriptPath, nic) +} + +func (l *SLinuxBridgeDriver) getUpScripts(nic jsonutils.JSONObject) (string, error) { + s := "#!/bin/bash\n\n" + s += fmt.Sprintf("switch='%s'\n", l.bridge) + if options.HostOptions.TunnelPaddingBytes > 0 { + s += fmt.Sprintf("/sbin/ifconfig $1 mtu %d\n", 1500+options.HostOptions.TunnelPaddingBytes) + } + s += "/sbin/ifconfig $1 0.0.0.0 up\n" + s += "brctl addif ${switch} $1\n" + return s, nil +} + +func (l *SLinuxBridgeDriver) getDownScripts(nic jsonutils.JSONObject) (string, error) { + s := "#!/bin/sh\n\n" + s += fmt.Sprintf("switch='%s'\n", l.bridge) + s += "brctl show ${switch} | grep $1\n" + s += "if [ $? -ne '0' ]; then\n" + s += " exit 0\n" + s += "fi\n" + s += "/sbin/ifconfig $1 0.0.0.0 down\n" + s += "brctl delif ${switch} $1\n" + return s, nil +} + +func (l *SLinuxBridgeDriver) SetupBridgeDev() error { + exist, err := l.Exists() + if err != nil { + return err + } + if !exist { + _, err := procutils.NewCommand("brctl", "addbr", l.bridge.String()).Run() + if err != nil { + return fmt.Errorf("Failed to create bridge %s", l.bridge) + } + } + return nil +} + +func (l *SLinuxBridgeDriver) RegisterHostlocalServer(mac, ip string) error { + metadataPort := l.GetMetadataServerPort() + metadataServerLoc := fmt.Sprintf("%s:%d", ip, metadataPort) + hostDnsServerLoc := fmt.Sprintf("%s:%d", ip, 53) + + cmd := "iptables -t nat -F" + cmd1 := strings.Split(cmd, " ") + output, err := procutils.NewCommand(cmd1[0], cmd1[1:]...).Run() + if err != nil { + log.Errorf("Clean iptables failed: %s", output) + return err + } + + cmd = "iptables -t nat -A PREROUTING -s 0.0.0.0/0" + cmd += " -d 169.254.169.254/32 -p tcp -m tcp --dport 80" + cmd += fmt.Sprintf(" -j DNAT --to-destination %s", metadataServerLoc) + cmd1 = strings.Split(cmd, " ") + output, err = procutils.NewCommand(cmd1[0], cmd1[1:]...).Run() + if err != nil { + log.Errorf("Inject DNAT rule failed: %s", output) + return err + } + + cmd = "sysctl -w net.ipv4.ip_forward=1" + cmd1 = strings.Split(cmd, " ") + output, err = procutils.NewCommand(cmd1[0], cmd1[1:]...).Run() + if err != nil { + log.Errorf("Enable ip forwarding failed: %s", output) + return err + } + + log.Infof("Bridge: metadata server=%s", metadataServerLoc) + log.Infof("Bridge: host dns server=%s", hostDnsServerLoc) + return nil +} + +func (l *SLinuxBridgeDriver) SetupInterface() error { + infs, err := l.Interfaces() + if err != nil { + return err + } + if l.inter != nil && !utils.IsInStringArray(l.inter.String(), infs) { + _, err := procutils.NewCommand( + "brctl", "addif", l.bridge.String(), l.inter.String()).Run() + if err != nil { + return fmt.Errorf("Failed to add interface %s", l.inter) + } + } + return nil +} diff --git a/pkg/hostman/hostinfo/hostbridge/ovs.go b/pkg/hostman/hostinfo/hostbridge/ovs.go new file mode 100644 index 0000000000..48d3086a6e --- /dev/null +++ b/pkg/hostman/hostinfo/hostbridge/ovs.go @@ -0,0 +1,319 @@ +package hostbridge + +import ( + "fmt" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/utils" + + "yunion.io/x/onecloud/pkg/hostman/options" + "yunion.io/x/onecloud/pkg/hostman/system_service" + "yunion.io/x/onecloud/pkg/util/bwutils" + "yunion.io/x/onecloud/pkg/util/netutils2" + "yunion.io/x/onecloud/pkg/util/ovsutils" + "yunion.io/x/onecloud/pkg/util/procutils" +) + +type SOVSBridgeDriver struct { + SBaseBridgeDriver +} + +func (o *SOVSBridgeDriver) CleanupConfig() { + ovsutils.CleanAllHiddenPorts() + // if enableopenflowcontroller ... +} + +func (o *SOVSBridgeDriver) Exists() (bool, error) { + data, err := procutils.NewCommand("ovs-vsctl", "list-br").Run() + if err != nil { + return false, err + } + for _, d := range strings.Split(string(data), "\n") { + if strings.TrimSpace(d) == o.bridge.String() { + return true, nil + } + } + return false, nil +} + +func (o *SOVSBridgeDriver) Interfaces() ([]string, error) { + data, err := procutils.NewCommand("ovs-vsctl", "list-ifaces", o.bridge.String()).Run() + if err != nil { + return nil, err + } + + var infs = make([]string, 0) + for _, d := range strings.Split(string(data), "\n") { + if len(strings.TrimSpace(d)) > 0 { + infs = append(infs, strings.TrimSpace(d)) + } + } + return infs, nil +} + +func (o *SOVSBridgeDriver) SetupInterface() error { + infs, err := o.Interfaces() + if err != nil { + return err + } + + if o.inter != nil && !utils.IsInStringArray(o.inter.String(), infs) { + output, err := procutils.NewCommand("ovs-vsctl", "--", "--may-exist", + "add-port", o.bridge.String(), o.inter.String()).Run() + if err != nil { + return fmt.Errorf("Failed to add interface %s", output) + } + } + return nil +} + +func (o *SOVSBridgeDriver) SetupBridgeDev() error { + exist, err := o.Exists() + if err != nil { + return err + } + if !exist { + _, err := procutils.NewCommand("ovs-vsctl", "--", "--may-exist", "add-br", o.bridge.String()).Run() + return err + } + return nil +} + +func (o *SOVSBridgeDriver) GenerateIfdownScripts(scriptPath string, nic jsonutils.JSONObject) error { + return o.generateIfdownScripts(o, scriptPath, nic) +} + +func (o *SOVSBridgeDriver) GenerateIfupScripts(scriptPath string, nic jsonutils.JSONObject) error { + return o.generateIfupScripts(o, scriptPath, nic) +} + +func (o *SOVSBridgeDriver) getUpScripts(nic jsonutils.JSONObject) (string, error) { + var ( + bridge, _ = nic.GetString("bridge") + ifname, _ = nic.GetString("ifname") + ip, _ = nic.GetString("ip") + mac, _ = nic.GetString("mac") + vlan, _ = nic.Int("vlan") + ) + + s := "#!/bin/bash\n\n" + s += fmt.Sprintf("SWITCH='%s'\n", bridge) + s += fmt.Sprintf("IF='%s'\n", ifname) + s += fmt.Sprintf("IP='%s'\n", ip) + s += fmt.Sprintf("MAC='%s'\n", mac) + s += fmt.Sprintf("VLAN_ID=%d\n", vlan) + limit, burst, err := bwutils.GetOvsBwValues(nic) + if err != nil { + return "", err + } + s += fmt.Sprintf("LIMIT=%d\n", limit) + s += fmt.Sprintf("BURST=%d\n", burst) + bwDownload, err := bwutils.GetDownloadBwValue(nic, options.HostOptions.BwDownloadBandwidth) + if err != nil { + return "", err + } + s += fmt.Sprintf("LIMIT_DOWNLOAD='%dmbit'\n", bwDownload) + if options.HostOptions.TunnelPaddingBytes > 0 { + s += fmt.Sprintf("/sbin/ifconfig $IF mtu %d\n", + 1500+options.HostOptions.TunnelPaddingBytes) + } + s += "/sbin/ifconfig $IF 0.0.0.0 up\n" + s += "ovs-vsctl list-ifaces $SWITCH | grep -w $IF > /dev/null 2>&1\n" + s += "if [ $? -eq '0' ]; then\n" + s += " ovs-vsctl del-port $SWITCH $IF\n" + s += "fi\n" + s += "if [ \"$VLAN_ID\" -ne \"1\" ]; then\n" + s += " TAG=\"tag=$VLAN_ID\"\n" + s += "fi\n" + s += "ovs-vsctl add-port $SWITCH $IF $TAG\n" + s += "PORT=$(ovs-ofctl show $SWITCH | grep -w $IF)\n" + s += "PORT=$(echo $PORT | awk 'BEGIN{FS=\"(\"}{print $1}')\n" + s += "OFCTL=$(ovs-vsctl get-controller $SWITCH)\n" + s += "if [ -z \"$OFCTL\" ]; then\n" + s += " ovs-vsctl set Interface $IF ingress_policing_rate=$LIMIT\n" + s += " ovs-vsctl set Interface $IF ingress_policing_burst=$BURST\n" + for _, r := range o.GetOfRules(nic) { + s += " " + o.AddFlow(r.cond, r.priority, r.actions) + } + s += "fi\n" + s += "if [ $LIMIT_DOWNLOAD != \"0mbit\" ]; then\n" + s += " tc qdisc del dev $IF root 2>/dev/null\n" + s += " tc qdisc add dev $IF root handle 1: htb default 10\n" + s += " tc class add dev $IF parent 1: classid 1:1 htb " + + "rate $LIMIT_DOWNLOAD ceil $LIMIT_DOWNLOAD\n" + s += " tc class add dev $IF parent 1:1 classid 1:10 htb " + + "rate $LIMIT_DOWNLOAD ceil $LIMIT_DOWNLOAD\n" + s += "fi\n" + return s, nil +} + +func (o *SOVSBridgeDriver) getDownScripts(nic jsonutils.JSONObject) (string, error) { + var ( + bridge, _ = nic.GetString("bridge") + ifname, _ = nic.GetString("ifname") + ip, _ = nic.GetString("ip") + mac, _ = nic.GetString("mac") + vlan, _ = nic.Int("vlan") + ) + + s := "#!/bin/bash\n\n" + s += fmt.Sprintf("SWITCH='%s'\n", bridge) + s += fmt.Sprintf("IF='%s'\n", ifname) + s += fmt.Sprintf("IP='%s'\n", ip) + s += fmt.Sprintf("MAC='%s'\n", mac) + s += fmt.Sprintf("VLAN_ID=%d\n", vlan) + s += "PORT=$(ovs-ofctl show $SWITCH | grep -w $IF)\n" + s += "if [ $? -ne '0' ]; then\n" + s += " exit 0\n" + s += "fi\n" + s += "OFCTL=$(ovs-vsctl get-controller $SWITCH)\n" + s += "PORT=$(echo $PORT | awk 'BEGIN{FS=\"(\"}{print $1}')\n" + s += "if [ -z \"$OFCTL\" ]; then\n" + for _, r := range o.GetOfRules(nic) { + s += " " + o.DelFlow(r.cond) + } + s += "fi\n" + s += "/sbin/ifconfig $IF 0.0.0.0 down\n" + s += "ovs-vsctl -- --if-exists del-port $SWITCH $IF\n" + return s, nil +} + +type SRule struct { + priority int + cond string + actions string +} + +func (o *SOVSBridgeDriver) AddFlow(cond string, priority int, actions string) string { + s := "" + s += fmt.Sprintf("ovs-ofctl add-flow $SWITCH \"%s", cond) + s += fmt.Sprintf(" priority=%d", priority) + s += fmt.Sprintf(" actions=%s\"\n", actions) + return s +} + +func (o *SOVSBridgeDriver) DoAddFlow(cond string, pri int, actions, swt string) error { + _, err := procutils.NewCommand("ovs-ofctl", "add-flow", swt, + fmt.Sprintf("%s priority=%d actions=%s", cond, pri, actions)).Run() + return err +} + +func (o *SOVSBridgeDriver) DelFlow(cond string) string { + return fmt.Sprintf("ovs-ofctl del-flows $SWITCH \"%s\"\n", cond) +} + +func (o *SOVSBridgeDriver) GetOfRules(nic jsonutils.JSONObject) []SRule { + rules := []SRule{} + metadataPort := o.GetMetadataServerPort() + rules = append(rules, + SRule{9000, fmt.Sprintf("table=0 in_port=local tcp nw_dst=$IP tp_src=%d", metadataPort), + "mod_nw_src=169.254.169.254,mod_tp_src:80,output:$PORT"}, + SRule{9500, "table=0 in_port=$PORT udp tp_src=68 tp_dst=67", "local"}, + SRule{8000, "table=0 in_port=$PORT", "resubmit(,1)"}, + ) + if vlan, _ := nic.Int("vlan"); vlan != 1 { + rules = append(rules, + SRule{4901, "table=1 dl_dst=$MAC,dl_vlan=$VLAN_ID", "strip_vlan,output:$PORT"}) + } + rules = append(rules, + SRule{4900, "table=1 dl_dst=$MAC", "output:$PORT"}) + return rules +} + +func (o *SOVSBridgeDriver) RegisterHostlocalServer(mac, ip string) error { + if !options.HostOptions.EnableOpenflowController { + metadataPort := o.GetMetadataServerPort() + if err := o.DoAddFlow("table=0 ipv6", 20000, "drop", o.bridge.String()); err != nil { + log.Errorln(err) + return err + } + if err := o.DoAddFlow("table=0 tcp nw_dst=169.254.169.254 tp_dst=80", 10000, + fmt.Sprintf("mod_dl_dst:%s,mod_nw_dst:%s,mod_tp_dst:%d,local", + mac, ip, metadataPort), + o.bridge.String()); err != nil { + log.Errorln(err) + return err + } + log.Infof("OVS: metadata server %s:%d", ip, metadataPort) + + k8sCidr := options.HostOptions.K8sClusterCidr + if len(k8sCidr) > 0 { + addr, mask, err := netutils2.PrefixSplit(k8sCidr) + if err != nil { + log.Errorln(err) + return err + } + k8sCidr = fmt.Sprintf("%s/%d", addr, mask) + log.Infof("OVS: Kubernetes cluster IP range: %s", k8sCidr) + err = o.DoAddFlow(fmt.Sprintf("table=0 ip,nw_dst=%s", k8sCidr), + 10050, fmt.Sprintf("mod_dl_dst:%s,local", mac), o.bridge.String()) + if err != nil { + log.Errorln(err) + return err + } + } + + err := o.DoAddFlow("table=0", 0, "resubmit(,1)", o.bridge.String()) + if err != nil { + log.Errorln(err) + return err + } + err = o.DoAddFlow("table=1", 0, "normal", o.bridge.String()) + if err != nil { + log.Errorln(err) + return err + } + } + return nil +} + +func (o *SOVSBridgeDriver) ovsSetParams(params map[string]map[string]string) { + for tbl, tblval := range params { + for k, v := range tblval { + procutils.NewCommand("ovs-vsctl", "set", tbl, o.bridge.String(), + fmt.Sprintf("%s=%s", k, v)).Run() + } + } +} + +func (o *SOVSBridgeDriver) WarmupConfig() error { + // if options.OvsSflowBridges ... + if options.HostOptions.EnableOpenflowController { + // ... + } else { + params := map[string]map[string]string{ + "bridge": { + "stp_enable": "false", + "fail_mode": "standalone", + "other-config:flow-eviction-threshold": "2500", + }, + } + o.ovsSetParams(params) + } + return nil +} + +func OVSPrepare() error { + ovs := system_service.GetService("openvswitch") + if !ovs.IsInstalled() { + return fmt.Errorf("Service openvswitch not installed!") + } + if !ovs.IsActive() { + return ovs.Start(false) + } + return nil +} + +func cleanOvsBridge() { + ovsutils.CleanAllHiddenPorts() +} + +func NewOVSBridgeDriver(bridge, inter, ip string) (*SOVSBridgeDriver, error) { + base, err := NewBaseBridgeDriver(bridge, inter, ip) + if err != nil { + return nil, err + } + return &SOVSBridgeDriver{*base}, nil +} diff --git a/pkg/hostman/hostinfo/hostinfohelper.go b/pkg/hostman/hostinfo/hostinfohelper.go index 9edca5e1b1..ab843bfb41 100644 --- a/pkg/hostman/hostinfo/hostinfohelper.go +++ b/pkg/hostman/hostinfo/hostinfohelper.go @@ -266,14 +266,22 @@ func NewNIC(desc string) (*SNIC, error) { return nil, err } - confirm, err := nic.BridgeDev.ConfirmToConfig(nic.BridgeDev.Exists(), nic.BridgeDev.Interfaces()) + exist, err := nic.BridgeDev.Exists() + if err != nil { + return nil, err + } + infs, err := nic.BridgeDev.Interfaces() + if err != nil { + return nil, err + } + confirm, err := nic.BridgeDev.ConfirmToConfig(exist, infs) if err != nil { log.Errorln(err) return nil, err } if !confirm { log.Infof("Not confirm to configuration") - if err = nic.BridgeDev.Setup(); err != nil { + if err = nic.BridgeDev.Setup(nic.BridgeDev); err != nil { log.Errorln(err) return nil, err }