diff --git a/pkg/vpcagent/ovn/external_access.go b/pkg/vpcagent/ovn/external_access.go new file mode 100644 index 0000000000..2adc98b641 --- /dev/null +++ b/pkg/vpcagent/ovn/external_access.go @@ -0,0 +1,30 @@ +package ovn + +import ( + apis "yunion.io/x/onecloud/pkg/apis/compute" + agentmodels "yunion.io/x/onecloud/pkg/vpcagent/models" +) + +func vpcHasDistgw(vpc *agentmodels.Vpc) bool { + mode := vpc.ExternalAccessMode + switch mode { + case + apis.VPC_EXTERNAL_ACCESS_MODE_DISTGW, + apis.VPC_EXTERNAL_ACCESS_MODE_EIP_DISTGW: + return true + default: + return false + } +} + +func vpcHasEipgw(vpc *agentmodels.Vpc) bool { + mode := vpc.ExternalAccessMode + switch mode { + case + apis.VPC_EXTERNAL_ACCESS_MODE_EIP, + apis.VPC_EXTERNAL_ACCESS_MODE_EIP_DISTGW: + return true + default: + return false + } +} diff --git a/pkg/vpcagent/ovn/keeper.go b/pkg/vpcagent/ovn/keeper.go index fb402b6260..852f5e49d7 100644 --- a/pkg/vpcagent/ovn/keeper.go +++ b/pkg/vpcagent/ovn/keeper.go @@ -85,37 +85,87 @@ func (keeper *OVNNorthboundKeeper) ClaimVpc(ctx context.Context, vpc *agentmodel vpcLr := &ovn_nb.LogicalRouter{ Name: vpcLrName(vpc.Id), } - vpcHostLs := &ovn_nb.LogicalSwitch{ - Name: vpcHostLsName(vpc.Id), - } - vpcRhp := &ovn_nb.LogicalRouterPort{ - Name: vpcRhpName(vpc.Id), - Mac: apis.VpcMappedGatewayMac, - Networks: []string{fmt.Sprintf("%s/%d", apis.VpcMappedGatewayIP(), apis.VpcMappedIPMask)}, - } - vpcHrp := &ovn_nb.LogicalSwitchPort{ - Name: vpcHrpName(vpc.Id), - Type: "router", - Addresses: []string{"router"}, - Options: map[string]string{ - "router-port": vpcRhpName(vpc.Id), - }, - } - allFound, args := cmp(&keeper.DB, ocVersion, - vpcLr, - vpcHostLs, - vpcRhp, - vpcHrp, + irows := []types.IRow{vpcLr} + + // distgw + var ( + vpcHostLs *ovn_nb.LogicalSwitch + vpcRhp *ovn_nb.LogicalRouterPort + vpcHrp *ovn_nb.LogicalSwitchPort ) + if vpcHasDistgw(vpc) { + vpcHostLs = &ovn_nb.LogicalSwitch{ + Name: vpcHostLsName(vpc.Id), + } + vpcRhp = &ovn_nb.LogicalRouterPort{ + Name: vpcRhpName(vpc.Id), + Mac: apis.VpcMappedGatewayMac, + Networks: []string{fmt.Sprintf("%s/%d", apis.VpcMappedGatewayIP(), apis.VpcMappedIPMask)}, + } + vpcHrp = &ovn_nb.LogicalSwitchPort{ + Name: vpcHrpName(vpc.Id), + Type: "router", + Addresses: []string{"router"}, + Options: map[string]string{ + "router-port": vpcRhpName(vpc.Id), + }, + } + irows = append(irows, + vpcHostLs, + vpcRhp, + vpcHrp, + ) + } + + // eipgw + var ( + vpcEipLs *ovn_nb.LogicalSwitch + vpcRep *ovn_nb.LogicalRouterPort + vpcErp *ovn_nb.LogicalSwitchPort + ) + if vpcHasEipgw(vpc) { + vpcEipLs = &ovn_nb.LogicalSwitch{ + Name: vpcEipLsName(vpc.Id), + } + vpcRep = &ovn_nb.LogicalRouterPort{ + Name: vpcRepName(vpc.Id), + Mac: apis.VpcEipGatewayMac, + Networks: []string{fmt.Sprintf("%s/%d", apis.VpcEipGatewayIP(), apis.VpcEipGatewayIPMask)}, + } + vpcErp = &ovn_nb.LogicalSwitchPort{ + Name: vpcErpName(vpc.Id), + Type: "router", + Addresses: []string{"router"}, + Options: map[string]string{ + "router-port": vpcRepName(vpc.Id), + }, + } + irows = append(irows, + vpcEipLs, + vpcRep, + vpcErp, + ) + } + + allFound, args := cmp(&keeper.DB, ocVersion, irows...) if allFound { return nil } args = append(args, ovnCreateArgs(vpcLr, vpcLr.Name)...) - args = append(args, ovnCreateArgs(vpcHostLs, vpcHostLs.Name)...) - args = append(args, ovnCreateArgs(vpcRhp, vpcRhp.Name)...) - args = append(args, ovnCreateArgs(vpcHrp, vpcHrp.Name)...) - args = append(args, "--", "add", "Logical_Switch", vpcHostLs.Name, "ports", "@"+vpcHrp.Name) - args = append(args, "--", "add", "Logical_Router", vpcLr.Name, "ports", "@"+vpcRhp.Name) + if vpcHasDistgw(vpc) { + args = append(args, ovnCreateArgs(vpcHostLs, vpcHostLs.Name)...) + args = append(args, ovnCreateArgs(vpcRhp, vpcRhp.Name)...) + args = append(args, ovnCreateArgs(vpcHrp, vpcHrp.Name)...) + args = append(args, "--", "add", "Logical_Switch", vpcHostLs.Name, "ports", "@"+vpcHrp.Name) + args = append(args, "--", "add", "Logical_Router", vpcLr.Name, "ports", "@"+vpcRhp.Name) + } + if vpcHasEipgw(vpc) { + args = append(args, ovnCreateArgs(vpcEipLs, vpcEipLs.Name)...) + args = append(args, ovnCreateArgs(vpcRep, vpcRep.Name)...) + args = append(args, ovnCreateArgs(vpcErp, vpcErp.Name)...) + args = append(args, "--", "add", "Logical_Switch", vpcEipLs.Name, "ports", "@"+vpcErp.Name) + args = append(args, "--", "add", "Logical_Router", vpcLr.Name, "ports", "@"+vpcRep.Name) + } return keeper.cli.Must(ctx, "ClaimVpc", args) } @@ -222,18 +272,48 @@ func (keeper *OVNNorthboundKeeper) ClaimVpcHost(ctx context.Context, vpc *agentm return keeper.cli.Must(ctx, "ClaimVpcHost", args) } +func (keeper *OVNNorthboundKeeper) ClaimVpcEipgw(ctx context.Context, vpc *agentmodels.Vpc) error { + var ( + ocVersion = fmt.Sprintf("%s.%d", vpc.UpdatedAt, vpc.UpdateVersion) + eipgwVip = apis.VpcEipGatewayIP3().String() + ) + vpcEipLsp := &ovn_nb.LogicalSwitchPort{ + Name: vpcEipLspName(vpc.Id, eipgwVip), + Addresses: []string{fmt.Sprintf("%s %s", apis.VpcEipGatewayMac3, eipgwVip)}, + } + if m := keeper.DB.LogicalSwitchPort.FindOneMatchNonZeros(vpcEipLsp); m != nil { + m.SetExternalId(externalKeyOcVersion, ocVersion) + return nil + } else { + args := []string{ + "--bare", "--columns=_uuid", "find", vpcEipLsp.OvsdbTableName(), + fmt.Sprintf("name=%q", vpcEipLsp.Name), + } + res := keeper.cli.Must(ctx, "find vpcEipLsp", args) + vpcEipLspUuid := strings.TrimSpace(res.Output) + if vpcEipLspUuid != "" { + return nil + } + } + var args []string + args = append(args, ovnCreateArgs(vpcEipLsp, vpcEipLsp.Name)...) + args = append(args, "--", "add", "Logical_Switch", vpcEipLsName(vpc.Id), "ports", "@"+vpcEipLsp.Name) + return keeper.cli.Must(ctx, "ClaimVpcEipgw", args) +} + func (keeper *OVNNorthboundKeeper) ClaimGuestnetwork(ctx context.Context, guestnetwork *agentmodels.Guestnetwork) error { var ( guest = guestnetwork.Guest network = guestnetwork.Network vpc = network.Vpc host = guest.Host + eip = guestnetwork.Elasticip - lportName = gnpName(guestnetwork.NetworkId, guestnetwork.Ifname) - ocVersion = fmt.Sprintf("%s.%d", guestnetwork.UpdatedAt, guestnetwork.UpdateVersion) - ocGnrRef = fmt.Sprintf("gnr/%s/%s/%s", vpc.Id, guestnetwork.GuestId, guestnetwork.Ifname) - ocAclRef = fmt.Sprintf("acl/%s/%s/%s", network.Id, guestnetwork.GuestId, guestnetwork.Ifname) - dhcpOpt string + lportName = gnpName(guestnetwork.NetworkId, guestnetwork.Ifname) + ocVersion = fmt.Sprintf("%s.%d", guestnetwork.UpdatedAt, guestnetwork.UpdateVersion) + ocGnrDefaultRef = fmt.Sprintf("gnrDefault/%s/%s/%s", vpc.Id, guestnetwork.GuestId, guestnetwork.Ifname) + ocAclRef = fmt.Sprintf("acl/%s/%s/%s", network.Id, guestnetwork.GuestId, guestnetwork.Ifname) + dhcpOpt string ) { @@ -264,14 +344,33 @@ func (keeper *OVNNorthboundKeeper) ClaimGuestnetwork(ctx context.Context, guestn Dhcpv4Options: &dhcpOpt, } - gnrPolicy := "src-ip" - gnr := &ovn_nb.LogicalRouterStaticRoute{ - Policy: &gnrPolicy, - IpPrefix: guestnetwork.IpAddr + "/32", - Nexthop: host.OvnMappedIpAddr, - ExternalIds: map[string]string{ - externalKeyOcRef: ocGnrRef, - }, + var gnrDefault *ovn_nb.LogicalRouterStaticRoute + { + gnrDefaultPolicy := "src-ip" + ptr := func(s string) *string { + return &s + } + if eip != nil && vpcHasEipgw(vpc) { + gnrDefault = &ovn_nb.LogicalRouterStaticRoute{ + Policy: &gnrDefaultPolicy, + IpPrefix: guestnetwork.IpAddr + "/32", + Nexthop: apis.VpcEipGatewayIP3().String(), + OutputPort: ptr(vpcRepName(vpc.Id)), + ExternalIds: map[string]string{ + externalKeyOcRef: ocGnrDefaultRef, + }, + } + } else if vpcHasDistgw(vpc) { + gnrDefault = &ovn_nb.LogicalRouterStaticRoute{ + Policy: &gnrDefaultPolicy, + IpPrefix: guestnetwork.IpAddr + "/32", + Nexthop: host.OvnMappedIpAddr, + OutputPort: ptr(vpcRhpName(vpc.Id)), + ExternalIds: map[string]string{ + externalKeyOcRef: ocGnrDefaultRef, + }, + } + } } var acls []*ovn_nb.ACL @@ -291,7 +390,10 @@ func (keeper *OVNNorthboundKeeper) ClaimGuestnetwork(ctx context.Context, guestn } irows := []types.IRow{ - gnp, gnr, + gnp, + } + if gnrDefault != nil { + irows = append(irows, gnrDefault) } for _, acl := range acls { irows = append(irows, acl) @@ -300,10 +402,13 @@ func (keeper *OVNNorthboundKeeper) ClaimGuestnetwork(ctx context.Context, guestn if allFound { return nil } + args = append(args, ovnCreateArgs(gnp, gnp.Name)...) - args = append(args, ovnCreateArgs(gnr, "gnr")...) args = append(args, "--", "add", "Logical_Switch", netLsName(guestnetwork.NetworkId), "ports", "@"+gnp.Name) - args = append(args, "--", "add", "Logical_Router", vpcLrName(vpc.Id), "static_routes", "@gnr") + if gnrDefault != nil { + args = append(args, ovnCreateArgs(gnrDefault, "gnrDefault")...) + args = append(args, "--", "add", "Logical_Router", vpcLrName(vpc.Id), "static_routes", "@gnrDefault") + } for i, acl := range acls { ref := fmt.Sprintf("acl%d", i) args = append(args, ovnCreateArgs(acl, ref)...) diff --git a/pkg/vpcagent/ovn/names.go b/pkg/vpcagent/ovn/names.go index 0897140738..4f9cb62d9c 100644 --- a/pkg/vpcagent/ovn/names.go +++ b/pkg/vpcagent/ovn/names.go @@ -8,6 +8,7 @@ func vpcLrName(vpcId string) string { return fmt.Sprintf("vpc-r/%s", vpcId) } +// distgw func vpcHostLsName(vpcId string) string { return fmt.Sprintf("vpc-h/%s", vpcId) } @@ -24,6 +25,23 @@ func vpcHostLspName(vpcId string, hostId string) string { return fmt.Sprintf("vpc-h/%s/%s", vpcId, hostId) } +// eipgw +func vpcEipLsName(vpcId string) string { + return fmt.Sprintf("vpc-e/%s", vpcId) +} + +func vpcRepName(vpcId string) string { + return fmt.Sprintf("vpc-re/%s", vpcId) +} + +func vpcErpName(vpcId string) string { + return fmt.Sprintf("vpc-er/%s", vpcId) +} + +func vpcEipLspName(vpcId string, eipgwId string) string { + return fmt.Sprintf("vpc-ep/%s/%s", vpcId, eipgwId) +} + func netLsName(netId string) string { return fmt.Sprintf("subnet/%s", netId) } diff --git a/pkg/vpcagent/ovn/worker.go b/pkg/vpcagent/ovn/worker.go index 85d10b3313..3022e59864 100644 --- a/pkg/vpcagent/ovn/worker.go +++ b/pkg/vpcagent/ovn/worker.go @@ -120,41 +120,47 @@ func (w *Worker) run(ctx context.Context, mss *agentmodels.ModelSets) (err error continue } ovndb.ClaimVpc(ctx, vpc) + if vpcHasEipgw(vpc) { + ovndb.ClaimVpcEipgw(ctx, vpc) + } for _, network := range vpc.Networks { ovndb.ClaimNetwork(ctx, network) for _, guestnetwork := range network.Guestnetworks { if guestnetwork.Guest == nil { continue } - var ( - guest = guestnetwork.Guest - network = guestnetwork.Network - vpc = network.Vpc - host = guest.Host - ) - if host.OvnVersion == "" { - // Just in case. This should never happen - log.Errorf("host %s(%s) of vpc guestnetwork (%s,%s) has no ovn support", - host.Id, host.Name, guestnetwork.NetworkId, guestnetwork.IpAddr) - continue - } - if host.OvnMappedIpAddr == "" { - // trigger ovn mapped ip addr allocation - apiVersion := "v2" - s := auth.GetAdminSession(ctx, w.opts.Region, apiVersion) - j, err := mcclient_modules.Hosts.Update(s, host.Id, nil) - if err != nil { - log.Errorf("host %s(%s) dummy update err: %v", host.Id, host.Name, err) - continue - } - j.Unmarshal(host) // update local copy in place - if host.OvnMappedIpAddr == "" { - log.Errorf("host %s(%s) has no mapped addr", host.Id, host.Name) - continue - } - } - ovndb.ClaimVpcHost(ctx, vpc, host) + if vpcHasDistgw(vpc) { + var ( + guest = guestnetwork.Guest + network = guestnetwork.Network + vpc = network.Vpc + host = guest.Host + ) + if host.OvnVersion == "" { + // Just in case. This should never happen + log.Errorf("host %s(%s) of vpc guestnetwork (%s,%s) has no ovn support", + host.Id, host.Name, guestnetwork.NetworkId, guestnetwork.IpAddr) + continue + } + if host.OvnMappedIpAddr == "" { + // trigger ovn mapped ip addr allocation + apiVersion := "v2" + s := auth.GetAdminSession(ctx, w.opts.Region, apiVersion) + j, err := mcclient_modules.Hosts.Update(s, host.Id, nil) + if err != nil { + log.Errorf("host %s(%s) dummy update err: %v", host.Id, host.Name, err) + continue + } + j.Unmarshal(host) // update local copy in place + if host.OvnMappedIpAddr == "" { + log.Errorf("host %s(%s) has no mapped addr", host.Id, host.Name) + continue + } + } + + ovndb.ClaimVpcHost(ctx, vpc, host) + } ovndb.ClaimGuestnetwork(ctx, guestnetwork) } }