opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-24 16:03:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #700 from wanyaoqi/feature/wyq/support-linux-bridge

Browse Source 
feature: golang host support linux bridge
This commit is contained in:
yunion-ci-robot
2019-05-02 10:35:13 +08:00
committed by GitHub
parent 222b9805fd cc56d09ceb
commit a4724da796
5 changed files with 553 additions and 326 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/hostman/guestman/qemu-kvm.go
+1 -1
View File
@@ -225,7 +225,7 @@ func (s *SKVMGuestInstance) asyncScriptStart(ctx context.Context, params interfa
return nil, hostutils.ParamsError
}
hostbridge.CleanDeletedPorts()
hostbridge.CleanDeletedPorts(options.HostOptions.BridgeDriver)
time.Sleep(100 * time.Millisecond)
var isStarted, tried = false, 0
pkg/hostman/hostinfo/hostbridge/hostbridge.go
+62 -323
View File
@@ -18,7 +18,6 @@ import (
"fmt"
"net"
"os"
"strings"
"syscall"
"yunion.io/x/jsonutils"
@@ -26,27 +25,34 @@ import (
"yunion.io/x/pkg/utils"
"yunion.io/x/onecloud/pkg/hostman/options"
"yunion.io/x/onecloud/pkg/hostman/system_service"
"yunion.io/x/onecloud/pkg/util/bwutils"
"yunion.io/x/onecloud/pkg/util/fileutils2"
"yunion.io/x/onecloud/pkg/util/netutils2"
"yunion.io/x/onecloud/pkg/util/ovsutils"
"yunion.io/x/onecloud/pkg/util/procutils"
)
type IBridgeDriver interface {
ConfirmToConfig(bool, []string) (bool, error)
Setup() error
Exists() bool
Interfaces() []string
FetchConfig()
GetMac() string
FetchConfig()
Setup(IBridgeDriver) error
SetupAddresses(net.IPMask) error
SetupSlaveAddresses([][]string) error
SetupRoutes(routes [][]string) error
BringupInterface() error
Exists() (bool, error)
Interfaces() ([]string, error)
WarmupConfig() error
CleanupConfig()
SetupBridgeDev() error
SetupInterface() error
GenerateIfupScripts(scriptPath string, nic jsonutils.JSONObject) error
GenerateIfdownScripts(scriptPath string, nic jsonutils.JSONObject) error
RegisterHostlocalServer(mac, ip string) error
WarmupConfig() error
CleanupConfig()
getUpScripts(nic jsonutils.JSONObject) (string, error)
getDownScripts(nic jsonutils.JSONObject) (string, error)
}
type SBaseBridgeDriver struct {
@@ -207,66 +213,35 @@ func (d *SBaseBridgeDriver) SetupRoutes(routes [][]string) error {
return nil
}
type SOVSBridgeDriver struct {
SBaseBridgeDriver
}
func (o *SOVSBridgeDriver) CleanupConfig() {
ovsutils.CleanAllHiddenPorts()
// if enableopenflowcontroller ...
}
func (o *SOVSBridgeDriver) Exists() bool {
data, err := procutils.NewCommand("ovs-vsctl", "list-br").Run()
if err != nil {
log.Errorln(err)
return false
}
for _, d := range strings.Split(string(data), "\n") {
if strings.TrimSpace(d) == o.bridge.String() {
return true
}
}
return false
}
func (o *SOVSBridgeDriver) Interfaces() []string {
data, err := procutils.NewCommand("ovs-vsctl", "list-ifaces", o.bridge.String()).Run()
if err != nil {
log.Errorln(err)
return nil
}
var infs = make([]string, 0)
for _, d := range strings.Split(string(data), "\n") {
if len(strings.TrimSpace(d)) > 0 {
infs = append(infs, strings.TrimSpace(d))
}
}
return infs
}
func (o *SOVSBridgeDriver) Setup() error {
func (d *SBaseBridgeDriver) Setup(o IBridgeDriver) error {
var routes [][]string
var slaveAddrs [][]string
if o.inter != nil && len(o.inter.Addr) > 0 {
routes = o.inter.GetRoutes(true)
slaveAddrs = o.inter.GetSlaveAddresses()
if d.inter != nil && len(d.inter.Addr) > 0 {
routes = d.inter.GetRoutes(true)
slaveAddrs = d.inter.GetSlaveAddresses()
}
if !o.Exists() {
exist, err := o.Exists()
if err != nil {
return err
}
if !exist {
if err := o.SetupBridgeDev(); err != nil {
return err
}
}
if o.inter != nil && !utils.IsInStringArray(o.inter.String(), o.Interfaces()) {
infs, err := o.Interfaces()
if err != nil {
return err
}
if d.inter != nil && !utils.IsInStringArray(d.inter.String(), infs) {
if err := o.SetupInterface(); err != nil {
return err
}
}
if len(o.bridge.Addr) == 0 {
if len(o.ip) > 0 {
if err := o.SetupAddresses(o.inter.Mask); err != nil {
if len(d.bridge.Addr) == 0 {
if len(d.ip) > 0 {
if err := o.SetupAddresses(d.inter.Mask); err != nil {
return err
}
if len(slaveAddrs) > 0 {
@@ -289,301 +264,65 @@ func (o *SOVSBridgeDriver) Setup() error {
return o.BringupInterface()
}
func (o *SOVSBridgeDriver) SetupInterface() error {
if o.inter != nil && !utils.IsInStringArray(o.inter.String(), o.Interfaces()) {
output, err := procutils.NewCommand("ovs-vsctl", "--", "--may-exist",
"add-port", o.bridge.String(), o.inter.String()).Run()
if err != nil {
return fmt.Errorf("Failed to add interface %s", output)
}
}
return nil
func (d *SBaseBridgeDriver) CleanupConfig() {
// pass
}
func (o *SOVSBridgeDriver) SetupBridgeDev() error {
if !o.Exists() {
_, err := procutils.NewCommand("ovs-vsctl", "--", "--may-exist", "add-br", o.bridge.String()).Run()
return err
}
return nil
}
func (o *SOVSBridgeDriver) GenerateIfdownScripts(scriptPath string, nic jsonutils.JSONObject) error {
script, err := o.getDownScripts(nic)
if err != nil {
log.Errorln(err)
return err
}
return o.saveFileExecutable(scriptPath, script)
}
func (o *SOVSBridgeDriver) GenerateIfupScripts(scriptPath string, nic jsonutils.JSONObject) error {
script, err := o.getUpScripts(nic)
if err != nil {
log.Errorln(err)
return err
}
return o.saveFileExecutable(scriptPath, script)
}
func (o *SOVSBridgeDriver) saveFileExecutable(scriptPath, script string) error {
func (d *SBaseBridgeDriver) saveFileExecutable(scriptPath, script string) error {
if err := fileutils2.FilePutContents(scriptPath, script, false); err != nil {
return err
}
return os.Chmod(scriptPath, syscall.S_IRUSR|syscall.S_IWUSR|syscall.S_IXUSR)
}
func (o *SOVSBridgeDriver) getUpScripts(nic jsonutils.JSONObject) (string, error) {
var (
bridge, _ = nic.GetString("bridge")
ifname, _ = nic.GetString("ifname")
ip, _ = nic.GetString("ip")
mac, _ = nic.GetString("mac")
vlan, _ = nic.Int("vlan")
)
s := "#!/bin/bash\n\n"
s += fmt.Sprintf("SWITCH='%s'\n", bridge)
s += fmt.Sprintf("IF='%s'\n", ifname)
s += fmt.Sprintf("IP='%s'\n", ip)
s += fmt.Sprintf("MAC='%s'\n", mac)
s += fmt.Sprintf("VLAN_ID=%d\n", vlan)
limit, burst, err := bwutils.GetOvsBwValues(nic)
func (d *SBaseBridgeDriver) generateIfdownScripts(driver IBridgeDriver, scriptPath string, nic jsonutils.JSONObject) error {
script, err := driver.getDownScripts(nic)
if err != nil {
return "", err
log.Errorln(err)
return err
}
s += fmt.Sprintf("LIMIT=%d\n", limit)
s += fmt.Sprintf("BURST=%d\n", burst)
bwDownload, err := bwutils.GetDownloadBwValue(nic, options.HostOptions.BwDownloadBandwidth)
return d.saveFileExecutable(scriptPath, script)
}
func (d *SBaseBridgeDriver) generateIfupScripts(driver IBridgeDriver, scriptPath string, nic jsonutils.JSONObject) error {
script, err := driver.getUpScripts(nic)
if err != nil {
return "", err
log.Errorln(err)
return err
}
s += fmt.Sprintf("LIMIT_DOWNLOAD='%dmbit'\n", bwDownload)
if options.HostOptions.TunnelPaddingBytes > 0 {
s += fmt.Sprintf("/sbin/ifconfig $IF mtu %d\n",
1500+options.HostOptions.TunnelPaddingBytes)
}
s += "/sbin/ifconfig $IF 0.0.0.0 up\n"
s += "ovs-vsctl list-ifaces $SWITCH | grep -w $IF > /dev/null 2>&1\n"
s += "if [ $? -eq '0' ]; then\n"
s += " ovs-vsctl del-port $SWITCH $IF\n"
s += "fi\n"
s += "if [ \"$VLAN_ID\" -ne \"1\" ]; then\n"
s += " TAG=\"tag=$VLAN_ID\"\n"
s += "fi\n"
s += "ovs-vsctl add-port $SWITCH $IF $TAG\n"
s += "PORT=$(ovs-ofctl show $SWITCH | grep -w $IF)\n"
s += "PORT=$(echo $PORT | awk 'BEGIN{FS=\"(\"}{print $1}')\n"
s += "OFCTL=$(ovs-vsctl get-controller $SWITCH)\n"
s += "if [ -z \"$OFCTL\" ]; then\n"
s += " ovs-vsctl set Interface $IF ingress_policing_rate=$LIMIT\n"
s += " ovs-vsctl set Interface $IF ingress_policing_burst=$BURST\n"
for _, r := range o.GetOfRules(nic) {
s += " " + o.AddFlow(r.cond, r.priority, r.actions)
}
s += "fi\n"
s += "if [ $LIMIT_DOWNLOAD != \"0mbit\" ]; then\n"
s += " tc qdisc del dev $IF root 2>/dev/null\n"
s += " tc qdisc add dev $IF root handle 1: htb default 10\n"
s += " tc class add dev $IF parent 1: classid 1:1 htb " +
"rate $LIMIT_DOWNLOAD ceil $LIMIT_DOWNLOAD\n"
s += " tc class add dev $IF parent 1:1 classid 1:10 htb " +
"rate $LIMIT_DOWNLOAD ceil $LIMIT_DOWNLOAD\n"
s += "fi\n"
return s, nil
return d.saveFileExecutable(scriptPath, script)
}
func (o *SOVSBridgeDriver) getDownScripts(nic jsonutils.JSONObject) (string, error) {
var (
bridge, _ = nic.GetString("bridge")
ifname, _ = nic.GetString("ifname")
ip, _ = nic.GetString("ip")
mac, _ = nic.GetString("mac")
vlan, _ = nic.Int("vlan")
)
s := "#!/bin/bash\n\n"
s += fmt.Sprintf("SWITCH='%s'\n", bridge)
s += fmt.Sprintf("IF='%s'\n", ifname)
s += fmt.Sprintf("IP='%s'\n", ip)
s += fmt.Sprintf("MAC='%s'\n", mac)
s += fmt.Sprintf("VLAN_ID=%d\n", vlan)
s += "PORT=$(ovs-ofctl show $SWITCH | grep -w $IF)\n"
s += "if [ $? -ne '0' ]; then\n"
s += " exit 0\n"
s += "fi\n"
s += "OFCTL=$(ovs-vsctl get-controller $SWITCH)\n"
s += "PORT=$(echo $PORT | awk 'BEGIN{FS=\"(\"}{print $1}')\n"
s += "if [ -z \"$OFCTL\" ]; then\n"
for _, r := range o.GetOfRules(nic) {
s += " " + o.DelFlow(r.cond)
}
s += "fi\n"
s += "/sbin/ifconfig $IF 0.0.0.0 down\n"
s += "ovs-vsctl -- --if-exists del-port $SWITCH $IF\n"
return s, nil
}
type SRule struct {
priority int
cond string
actions string
}
func (o *SOVSBridgeDriver) AddFlow(cond string, priority int, actions string) string {
s := ""
s += fmt.Sprintf("ovs-ofctl add-flow $SWITCH \"%s", cond)
s += fmt.Sprintf(" priority=%d", priority)
s += fmt.Sprintf(" actions=%s\"\n", actions)
return s
}
func (o *SOVSBridgeDriver) DoAddFlow(cond string, pri int, actions, swt string) error {
_, err := procutils.NewCommand("ovs-ofctl", "add-flow", swt,
fmt.Sprintf("%s priority=%d actions=%s", cond, pri, actions)).Run()
return err
}
func (o *SOVSBridgeDriver) DelFlow(cond string) string {
return fmt.Sprintf("ovs-ofctl del-flows $SWITCH \"%s\"\n", cond)
}
func (o *SOVSBridgeDriver) GetOfRules(nic jsonutils.JSONObject) []SRule {
rules := []SRule{}
metadataPort := o.GetMetadataServerPort()
rules = append(rules,
SRule{9000, fmt.Sprintf("table=0 in_port=local tcp nw_dst=$IP tp_src=%d", metadataPort),
"mod_nw_src=169.254.169.254,mod_tp_src:80,output:$PORT"},
SRule{9500, "table=0 in_port=$PORT udp tp_src=68 tp_dst=67", "local"},
SRule{8000, "table=0 in_port=$PORT", "resubmit(,1)"},
)
if vlan, _ := nic.Int("vlan"); vlan != 1 {
rules = append(rules,
SRule{4901, "table=1 dl_dst=$MAC,dl_vlan=$VLAN_ID", "strip_vlan,output:$PORT"})
}
rules = append(rules,
SRule{4900, "table=1 dl_dst=$MAC", "output:$PORT"})
return rules
}
func (o *SOVSBridgeDriver) GetMetadataServerPort() int {
func (d *SBaseBridgeDriver) GetMetadataServerPort() int {
return options.HostOptions.Port + 1000
}
func (o *SOVSBridgeDriver) RegisterHostlocalServer(mac, ip string) error {
if !options.HostOptions.EnableOpenflowController {
metadataPort := o.GetMetadataServerPort()
if err := o.DoAddFlow("table=0 ipv6", 20000, "drop", o.bridge.String()); err != nil {
log.Errorln(err)
return err
}
if err := o.DoAddFlow("table=0 tcp nw_dst=169.254.169.254 tp_dst=80", 10000,
fmt.Sprintf("mod_dl_dst:%s,mod_nw_dst:%s,mod_tp_dst:%d,local",
mac, ip, metadataPort),
o.bridge.String()); err != nil {
log.Errorln(err)
return err
}
log.Infof("OVS: metadata server %s:%d", ip, metadataPort)
k8sCidr := options.HostOptions.K8sClusterCidr
if len(k8sCidr) > 0 {
addr, mask, err := netutils2.PrefixSplit(k8sCidr)
if err != nil {
log.Errorln(err)
return err
}
k8sCidr = fmt.Sprintf("%s/%d", addr, mask)
log.Infof("OVS: Kubernetes cluster IP range: %s", k8sCidr)
err = o.DoAddFlow(fmt.Sprintf("table=0 ip,nw_dst=%s", k8sCidr),
10050, fmt.Sprintf("mod_dl_dst:%s,local", mac), o.bridge.String())
if err != nil {
log.Errorln(err)
return err
}
}
err := o.DoAddFlow("table=0", 0, "resubmit(,1)", o.bridge.String())
if err != nil {
log.Errorln(err)
return err
}
err = o.DoAddFlow("table=1", 0, "normal", o.bridge.String())
if err != nil {
log.Errorln(err)
return err
}
}
func (d *SBaseBridgeDriver) WarmupConfig() error {
return nil
}
func (o *SOVSBridgeDriver) ovsSetParams(params map[string]map[string]string) {
for tbl, tblval := range params {
for k, v := range tblval {
procutils.NewCommand("ovs-vsctl", "set", tbl, o.bridge.String(),
fmt.Sprintf("%s=%s", k, v)).Run()
}
}
}
func (o *SOVSBridgeDriver) WarmupConfig() error {
// if options.OvsSflowBridges ...
if options.HostOptions.EnableOpenflowController {
// ...
} else {
params := map[string]map[string]string{
"bridge": {
"stp_enable": "false",
"fail_mode": "standalone",
"other-config:flow-eviction-threshold": "2500",
},
}
o.ovsSetParams(params)
}
return nil
}
func OVSPrepare() error {
ovs := system_service.GetService("openvswitch")
if !ovs.IsInstalled() {
return fmt.Errorf("Service openvswitch not installed!")
}
if !ovs.IsActive() {
return ovs.Start(false)
}
return nil
}
func CleanOvsBridge() {
ovsutils.CleanAllHiddenPorts()
}
func NewOVSBridgeDriver(bridge, inter, ip string) (*SOVSBridgeDriver, error) {
base, err := NewBaseBridgeDriver(bridge, inter, ip)
if err != nil {
return nil, err
}
return &SOVSBridgeDriver{*base}, nil
}
func NewDriver(bridgeDriver, bridge, inter, ip string) (IBridgeDriver, error) {
if bridgeDriver == "openvswitch" {
return NewOVSBridgeDriver(bridge, inter, ip)
} else {
return nil, fmt.Errorf("Not Implentment")
} else if bridgeDriver == "linux_bridge" {
return NewLinuxBridgeDeriver(bridge, inter, ip)
}
return nil, fmt.Errorf("Dirver %s not found", bridgeDriver)
}
func Prepare(bridgeDriver string) error {
if bridgeDriver == "openvswitch" {
return OVSPrepare()
} else {
return fmt.Errorf("Not Implentment")
} else if bridgeDriver == "linux_bridge" {
return LinuxBridgePrepare()
}
return fmt.Errorf("Dirver %s not found", bridgeDriver)
}
func CleanDeletedPorts() {
if options.HostOptions.BridgeDriver == "openvswitch" {
CleanOvsBridge()
func CleanDeletedPorts(bridgeDriver string) {
if bridgeDriver == "openvswitch" {
cleanOvsBridge()
} else if bridgeDriver == "linux_bridge" {
cleanLinuxBridge()
}
}
pkg/hostman/hostinfo/hostbridge/linux_bridge.go
+161
View File
@@ -0,0 +1,161 @@
package hostbridge
import (
"bytes"
"fmt"
"regexp"
"strings"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/onecloud/pkg/hostman/options"
"yunion.io/x/onecloud/pkg/util/procutils"
"yunion.io/x/pkg/utils"
)
func NewLinuxBridgeDeriver(bridge, inter, ip string) (*SLinuxBridgeDriver, error) {
base, err := NewBaseBridgeDriver(bridge, inter, ip)
if err != nil {
return nil, err
}
return &SLinuxBridgeDriver{*base}, nil
}
func LinuxBridgePrepare() error {
return nil
}
func cleanLinuxBridge() {
// pass
}
type SLinuxBridgeDriver struct {
SBaseBridgeDriver
}
func (l *SLinuxBridgeDriver) Exists() (bool, error) {
data, err := procutils.NewCommand("brctl", "show").Run()
if err != nil {
return false, err
}
re := regexp.MustCompile(`\s+`)
for _, line := range bytes.Split(data, []byte{'\n'}) {
info := re.Split(string(line), -1)
if info[0] == l.bridge.String() {
return true, nil
}
}
return false, nil
}
func (l *SLinuxBridgeDriver) Interfaces() ([]string, error) {
data, err := procutils.NewCommand("brctl", "show", l.bridge.String()).Run()
if err != nil {
return nil, err
}
infs := make([]string, 0)
re := regexp.MustCompile(`\s+`)
for _, line := range bytes.Split(data, []byte{'\n'}) {
info := re.Split(string(line), -1)
infs = append(infs, info[len(info)-1])
}
return infs, nil
}
func (l *SLinuxBridgeDriver) GenerateIfdownScripts(scriptPath string, nic jsonutils.JSONObject) error {
return l.generateIfdownScripts(l, scriptPath, nic)
}
func (l *SLinuxBridgeDriver) GenerateIfupScripts(scriptPath string, nic jsonutils.JSONObject) error {
return l.generateIfupScripts(l, scriptPath, nic)
}
func (l *SLinuxBridgeDriver) getUpScripts(nic jsonutils.JSONObject) (string, error) {
s := "#!/bin/bash\n\n"
s += fmt.Sprintf("switch='%s'\n", l.bridge)
if options.HostOptions.TunnelPaddingBytes > 0 {
s += fmt.Sprintf("/sbin/ifconfig $1 mtu %d\n", 1500+options.HostOptions.TunnelPaddingBytes)
}
s += "/sbin/ifconfig $1 0.0.0.0 up\n"
s += "brctl addif ${switch} $1\n"
return s, nil
}
func (l *SLinuxBridgeDriver) getDownScripts(nic jsonutils.JSONObject) (string, error) {
s := "#!/bin/sh\n\n"
s += fmt.Sprintf("switch='%s'\n", l.bridge)
s += "brctl show ${switch} | grep $1\n"
s += "if [ $? -ne '0' ]; then\n"
s += " exit 0\n"
s += "fi\n"
s += "/sbin/ifconfig $1 0.0.0.0 down\n"
s += "brctl delif ${switch} $1\n"
return s, nil
}
func (l *SLinuxBridgeDriver) SetupBridgeDev() error {
exist, err := l.Exists()
if err != nil {
return err
}
if !exist {
_, err := procutils.NewCommand("brctl", "addbr", l.bridge.String()).Run()
if err != nil {
return fmt.Errorf("Failed to create bridge %s", l.bridge)
}
}
return nil
}
func (l *SLinuxBridgeDriver) RegisterHostlocalServer(mac, ip string) error {
metadataPort := l.GetMetadataServerPort()
metadataServerLoc := fmt.Sprintf("%s:%d", ip, metadataPort)
hostDnsServerLoc := fmt.Sprintf("%s:%d", ip, 53)
cmd := "iptables -t nat -F"
cmd1 := strings.Split(cmd, " ")
output, err := procutils.NewCommand(cmd1[0], cmd1[1:]...).Run()
if err != nil {
log.Errorf("Clean iptables failed: %s", output)
return err
}
cmd = "iptables -t nat -A PREROUTING -s 0.0.0.0/0"
cmd += " -d 169.254.169.254/32 -p tcp -m tcp --dport 80"
cmd += fmt.Sprintf(" -j DNAT --to-destination %s", metadataServerLoc)
cmd1 = strings.Split(cmd, " ")
output, err = procutils.NewCommand(cmd1[0], cmd1[1:]...).Run()
if err != nil {
log.Errorf("Inject DNAT rule failed: %s", output)
return err
}
cmd = "sysctl -w net.ipv4.ip_forward=1"
cmd1 = strings.Split(cmd, " ")
output, err = procutils.NewCommand(cmd1[0], cmd1[1:]...).Run()
if err != nil {
log.Errorf("Enable ip forwarding failed: %s", output)
return err
}
log.Infof("Bridge: metadata server=%s", metadataServerLoc)
log.Infof("Bridge: host dns server=%s", hostDnsServerLoc)
return nil
}
func (l *SLinuxBridgeDriver) SetupInterface() error {
infs, err := l.Interfaces()
if err != nil {
return err
}
if l.inter != nil && !utils.IsInStringArray(l.inter.String(), infs) {
_, err := procutils.NewCommand(
"brctl", "addif", l.bridge.String(), l.inter.String()).Run()
if err != nil {
return fmt.Errorf("Failed to add interface %s", l.inter)
}
}
return nil
}
pkg/hostman/hostinfo/hostbridge/ovs.go
+319
View File
@@ -0,0 +1,319 @@
package hostbridge
import (
"fmt"
"strings"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/utils"
"yunion.io/x/onecloud/pkg/hostman/options"
"yunion.io/x/onecloud/pkg/hostman/system_service"
"yunion.io/x/onecloud/pkg/util/bwutils"
"yunion.io/x/onecloud/pkg/util/netutils2"
"yunion.io/x/onecloud/pkg/util/ovsutils"
"yunion.io/x/onecloud/pkg/util/procutils"
)
type SOVSBridgeDriver struct {
SBaseBridgeDriver
}
func (o *SOVSBridgeDriver) CleanupConfig() {
ovsutils.CleanAllHiddenPorts()
// if enableopenflowcontroller ...
}
func (o *SOVSBridgeDriver) Exists() (bool, error) {
data, err := procutils.NewCommand("ovs-vsctl", "list-br").Run()
if err != nil {
return false, err
}
for _, d := range strings.Split(string(data), "\n") {
if strings.TrimSpace(d) == o.bridge.String() {
return true, nil
}
}
return false, nil
}
func (o *SOVSBridgeDriver) Interfaces() ([]string, error) {
data, err := procutils.NewCommand("ovs-vsctl", "list-ifaces", o.bridge.String()).Run()
if err != nil {
return nil, err
}
var infs = make([]string, 0)
for _, d := range strings.Split(string(data), "\n") {
if len(strings.TrimSpace(d)) > 0 {
infs = append(infs, strings.TrimSpace(d))
}
}
return infs, nil
}
func (o *SOVSBridgeDriver) SetupInterface() error {
infs, err := o.Interfaces()
if err != nil {
return err
}
if o.inter != nil && !utils.IsInStringArray(o.inter.String(), infs) {
output, err := procutils.NewCommand("ovs-vsctl", "--", "--may-exist",
"add-port", o.bridge.String(), o.inter.String()).Run()
if err != nil {
return fmt.Errorf("Failed to add interface %s", output)
}
}
return nil
}
func (o *SOVSBridgeDriver) SetupBridgeDev() error {
exist, err := o.Exists()
if err != nil {
return err
}
if !exist {
_, err := procutils.NewCommand("ovs-vsctl", "--", "--may-exist", "add-br", o.bridge.String()).Run()
return err
}
return nil
}
func (o *SOVSBridgeDriver) GenerateIfdownScripts(scriptPath string, nic jsonutils.JSONObject) error {
return o.generateIfdownScripts(o, scriptPath, nic)
}
func (o *SOVSBridgeDriver) GenerateIfupScripts(scriptPath string, nic jsonutils.JSONObject) error {
return o.generateIfupScripts(o, scriptPath, nic)
}
func (o *SOVSBridgeDriver) getUpScripts(nic jsonutils.JSONObject) (string, error) {
var (
bridge, _ = nic.GetString("bridge")
ifname, _ = nic.GetString("ifname")
ip, _ = nic.GetString("ip")
mac, _ = nic.GetString("mac")
vlan, _ = nic.Int("vlan")
)
s := "#!/bin/bash\n\n"
s += fmt.Sprintf("SWITCH='%s'\n", bridge)
s += fmt.Sprintf("IF='%s'\n", ifname)
s += fmt.Sprintf("IP='%s'\n", ip)
s += fmt.Sprintf("MAC='%s'\n", mac)
s += fmt.Sprintf("VLAN_ID=%d\n", vlan)
limit, burst, err := bwutils.GetOvsBwValues(nic)
if err != nil {
return "", err
}
s += fmt.Sprintf("LIMIT=%d\n", limit)
s += fmt.Sprintf("BURST=%d\n", burst)
bwDownload, err := bwutils.GetDownloadBwValue(nic, options.HostOptions.BwDownloadBandwidth)
if err != nil {
return "", err
}
s += fmt.Sprintf("LIMIT_DOWNLOAD='%dmbit'\n", bwDownload)
if options.HostOptions.TunnelPaddingBytes > 0 {
s += fmt.Sprintf("/sbin/ifconfig $IF mtu %d\n",
1500+options.HostOptions.TunnelPaddingBytes)
}
s += "/sbin/ifconfig $IF 0.0.0.0 up\n"
s += "ovs-vsctl list-ifaces $SWITCH | grep -w $IF > /dev/null 2>&1\n"
s += "if [ $? -eq '0' ]; then\n"
s += " ovs-vsctl del-port $SWITCH $IF\n"
s += "fi\n"
s += "if [ \"$VLAN_ID\" -ne \"1\" ]; then\n"
s += " TAG=\"tag=$VLAN_ID\"\n"
s += "fi\n"
s += "ovs-vsctl add-port $SWITCH $IF $TAG\n"
s += "PORT=$(ovs-ofctl show $SWITCH | grep -w $IF)\n"
s += "PORT=$(echo $PORT | awk 'BEGIN{FS=\"(\"}{print $1}')\n"
s += "OFCTL=$(ovs-vsctl get-controller $SWITCH)\n"
s += "if [ -z \"$OFCTL\" ]; then\n"
s += " ovs-vsctl set Interface $IF ingress_policing_rate=$LIMIT\n"
s += " ovs-vsctl set Interface $IF ingress_policing_burst=$BURST\n"
for _, r := range o.GetOfRules(nic) {
s += " " + o.AddFlow(r.cond, r.priority, r.actions)
}
s += "fi\n"
s += "if [ $LIMIT_DOWNLOAD != \"0mbit\" ]; then\n"
s += " tc qdisc del dev $IF root 2>/dev/null\n"
s += " tc qdisc add dev $IF root handle 1: htb default 10\n"
s += " tc class add dev $IF parent 1: classid 1:1 htb " +
"rate $LIMIT_DOWNLOAD ceil $LIMIT_DOWNLOAD\n"
s += " tc class add dev $IF parent 1:1 classid 1:10 htb " +
"rate $LIMIT_DOWNLOAD ceil $LIMIT_DOWNLOAD\n"
s += "fi\n"
return s, nil
}
func (o *SOVSBridgeDriver) getDownScripts(nic jsonutils.JSONObject) (string, error) {
var (
bridge, _ = nic.GetString("bridge")
ifname, _ = nic.GetString("ifname")
ip, _ = nic.GetString("ip")
mac, _ = nic.GetString("mac")
vlan, _ = nic.Int("vlan")
)
s := "#!/bin/bash\n\n"
s += fmt.Sprintf("SWITCH='%s'\n", bridge)
s += fmt.Sprintf("IF='%s'\n", ifname)
s += fmt.Sprintf("IP='%s'\n", ip)
s += fmt.Sprintf("MAC='%s'\n", mac)
s += fmt.Sprintf("VLAN_ID=%d\n", vlan)
s += "PORT=$(ovs-ofctl show $SWITCH | grep -w $IF)\n"
s += "if [ $? -ne '0' ]; then\n"
s += " exit 0\n"
s += "fi\n"
s += "OFCTL=$(ovs-vsctl get-controller $SWITCH)\n"
s += "PORT=$(echo $PORT | awk 'BEGIN{FS=\"(\"}{print $1}')\n"
s += "if [ -z \"$OFCTL\" ]; then\n"
for _, r := range o.GetOfRules(nic) {
s += " " + o.DelFlow(r.cond)
}
s += "fi\n"
s += "/sbin/ifconfig $IF 0.0.0.0 down\n"
s += "ovs-vsctl -- --if-exists del-port $SWITCH $IF\n"
return s, nil
}
type SRule struct {
priority int
cond string
actions string
}
func (o *SOVSBridgeDriver) AddFlow(cond string, priority int, actions string) string {
s := ""
s += fmt.Sprintf("ovs-ofctl add-flow $SWITCH \"%s", cond)
s += fmt.Sprintf(" priority=%d", priority)
s += fmt.Sprintf(" actions=%s\"\n", actions)
return s
}
func (o *SOVSBridgeDriver) DoAddFlow(cond string, pri int, actions, swt string) error {
_, err := procutils.NewCommand("ovs-ofctl", "add-flow", swt,
fmt.Sprintf("%s priority=%d actions=%s", cond, pri, actions)).Run()
return err
}
func (o *SOVSBridgeDriver) DelFlow(cond string) string {
return fmt.Sprintf("ovs-ofctl del-flows $SWITCH \"%s\"\n", cond)
}
func (o *SOVSBridgeDriver) GetOfRules(nic jsonutils.JSONObject) []SRule {
rules := []SRule{}
metadataPort := o.GetMetadataServerPort()
rules = append(rules,
SRule{9000, fmt.Sprintf("table=0 in_port=local tcp nw_dst=$IP tp_src=%d", metadataPort),
"mod_nw_src=169.254.169.254,mod_tp_src:80,output:$PORT"},
SRule{9500, "table=0 in_port=$PORT udp tp_src=68 tp_dst=67", "local"},
SRule{8000, "table=0 in_port=$PORT", "resubmit(,1)"},
)
if vlan, _ := nic.Int("vlan"); vlan != 1 {
rules = append(rules,
SRule{4901, "table=1 dl_dst=$MAC,dl_vlan=$VLAN_ID", "strip_vlan,output:$PORT"})
}
rules = append(rules,
SRule{4900, "table=1 dl_dst=$MAC", "output:$PORT"})
return rules
}
func (o *SOVSBridgeDriver) RegisterHostlocalServer(mac, ip string) error {
if !options.HostOptions.EnableOpenflowController {
metadataPort := o.GetMetadataServerPort()
if err := o.DoAddFlow("table=0 ipv6", 20000, "drop", o.bridge.String()); err != nil {
log.Errorln(err)
return err
}
if err := o.DoAddFlow("table=0 tcp nw_dst=169.254.169.254 tp_dst=80", 10000,
fmt.Sprintf("mod_dl_dst:%s,mod_nw_dst:%s,mod_tp_dst:%d,local",
mac, ip, metadataPort),
o.bridge.String()); err != nil {
log.Errorln(err)
return err
}
log.Infof("OVS: metadata server %s:%d", ip, metadataPort)
k8sCidr := options.HostOptions.K8sClusterCidr
if len(k8sCidr) > 0 {
addr, mask, err := netutils2.PrefixSplit(k8sCidr)
if err != nil {
log.Errorln(err)
return err
}
k8sCidr = fmt.Sprintf("%s/%d", addr, mask)
log.Infof("OVS: Kubernetes cluster IP range: %s", k8sCidr)
err = o.DoAddFlow(fmt.Sprintf("table=0 ip,nw_dst=%s", k8sCidr),
10050, fmt.Sprintf("mod_dl_dst:%s,local", mac), o.bridge.String())
if err != nil {
log.Errorln(err)
return err
}
}
err := o.DoAddFlow("table=0", 0, "resubmit(,1)", o.bridge.String())
if err != nil {
log.Errorln(err)
return err
}
err = o.DoAddFlow("table=1", 0, "normal", o.bridge.String())
if err != nil {
log.Errorln(err)
return err
}
}
return nil
}
func (o *SOVSBridgeDriver) ovsSetParams(params map[string]map[string]string) {
for tbl, tblval := range params {
for k, v := range tblval {
procutils.NewCommand("ovs-vsctl", "set", tbl, o.bridge.String(),
fmt.Sprintf("%s=%s", k, v)).Run()
}
}
}
func (o *SOVSBridgeDriver) WarmupConfig() error {
// if options.OvsSflowBridges ...
if options.HostOptions.EnableOpenflowController {
// ...
} else {
params := map[string]map[string]string{
"bridge": {
"stp_enable": "false",
"fail_mode": "standalone",
"other-config:flow-eviction-threshold": "2500",
},
}
o.ovsSetParams(params)
}
return nil
}
func OVSPrepare() error {
ovs := system_service.GetService("openvswitch")
if !ovs.IsInstalled() {
return fmt.Errorf("Service openvswitch not installed!")
}
if !ovs.IsActive() {
return ovs.Start(false)
}
return nil
}
func cleanOvsBridge() {
ovsutils.CleanAllHiddenPorts()
}
func NewOVSBridgeDriver(bridge, inter, ip string) (*SOVSBridgeDriver, error) {
base, err := NewBaseBridgeDriver(bridge, inter, ip)
if err != nil {
return nil, err
}
return &SOVSBridgeDriver{*base}, nil
}
pkg/hostman/hostinfo/hostinfohelper.go
+10 -2
View File
@@ -266,14 +266,22 @@ func NewNIC(desc string) (*SNIC, error) {
return nil, err
}
confirm, err := nic.BridgeDev.ConfirmToConfig(nic.BridgeDev.Exists(), nic.BridgeDev.Interfaces())
exist, err := nic.BridgeDev.Exists()
if err != nil {
return nil, err
}
infs, err := nic.BridgeDev.Interfaces()
if err != nil {
return nil, err
}
confirm, err := nic.BridgeDev.ConfirmToConfig(exist, infs)
if err != nil {
log.Errorln(err)
return nil, err
}
if !confirm {
log.Infof("Not confirm to configuration")
if err = nic.BridgeDev.Setup(); err != nil {
if err = nic.BridgeDev.Setup(nic.BridgeDev); err != nil {
log.Errorln(err)
return nil, err
}