aliyun: 同步routetables

pkg/cloudprovider/resources.go

@@ -221,6 +221,24 @@ type ICloudSecurityGroup interface {
 	GetRules() ([]secrules.SecurityRule, error)
 }
 
+type ICloudRouteTable interface {
+	ICloudResource
+	GetManagerId() string
+
+	GetDescription() string
+	GetRegionId() string
+	GetVpcId() string
+	GetType() string
+	GetIRoutes() ([]ICloudRoute, error)
+}
+
+type ICloudRoute interface {
+	GetType() string
+	GetCidr() string
+	GetNextHopType() string
+	GetNextHop() string
+}
+
 type ICloudDisk interface {
 	ICloudResource
 	IBillingResource
@@ -266,6 +284,7 @@ type ICloudVpc interface {
 	// GetStatus() string
 	GetIWires() ([]ICloudWire, error)
 	GetISecurityGroups() ([]ICloudSecurityGroup, error)
+	GetIRouteTables() ([]ICloudRouteTable, error)
 
 	GetManagerId() string
 

pkg/compute/models/routetables.go

@@ -9,10 +9,12 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/gotypes"
+	"yunion.io/x/pkg/util/compare"
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
@@ -91,6 +93,7 @@ func init() {
 
 type SRouteTable struct {
 	db.SVirtualResourceBase
+	SManagedResourceBase
 
 	VpcId         string   `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required"`
 	CloudregionId string   `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional"`
@@ -250,5 +253,124 @@ func (rt *SRouteTable) GetCustomizeColumns(ctx context.Context, userCred mcclien
 
 func (rt *SRouteTable) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
 	extra := rt.GetCustomizeColumns(ctx, userCred, query)
+	extra = rt.SManagedResourceBase.getExtraDetails(ctx, extra)
 	return extra
 }
+
+func (man *SRouteTableManager) SyncRouteTables(ctx context.Context, userCred mcclient.TokenCredential, vpc *SVpc, cloudRouteTables []cloudprovider.ICloudRouteTable) ([]SRouteTable, []cloudprovider.ICloudRouteTable, compare.SyncResult) {
+	localRouteTables := make([]SRouteTable, 0)
+	remoteRouteTables := make([]cloudprovider.ICloudRouteTable, 0)
+	syncResult := compare.SyncResult{}
+
+	dbRouteTables := []SRouteTable{}
+	if err := db.FetchModelObjects(man, man.Query(), &dbRouteTables); err != nil {
+		syncResult.Error(err)
+		return nil, nil, syncResult
+	}
+	removed := make([]SRouteTable, 0)
+	commondb := make([]SRouteTable, 0)
+	commonext := make([]cloudprovider.ICloudRouteTable, 0)
+	added := make([]cloudprovider.ICloudRouteTable, 0)
+	if false {
+		for _, rt := range cloudRouteTables {
+			log.Errorf("%s, %s", rt.GetName(), rt.GetGlobalId())
+			routes, err := rt.GetIRoutes()
+			if err != nil {
+				log.Errorf("get routes err: %s", err)
+				continue
+			}
+			for i, route := range routes {
+				log.Errorf("route %d: %#v ", i, route)
+			}
+		}
+	}
+	if err := compare.CompareSets(dbRouteTables, cloudRouteTables, &removed, &commondb, &commonext, &added); err != nil {
+		syncResult.Error(err)
+		return nil, nil, syncResult
+	}
+
+	for i := 0; i < len(commondb); i += 1 {
+		err := commondb[i].SyncWithCloudRouteTable(userCred, vpc, commonext[i])
+		if err != nil {
+			syncResult.UpdateError(err)
+			continue
+		}
+		localRouteTables = append(localRouteTables, commondb[i])
+		remoteRouteTables = append(remoteRouteTables, commonext[i])
+		syncResult.Update()
+	}
+
+	for i := 0; i < len(added); i += 1 {
+		routeTableNew, err := man.insertFromCloud(userCred, vpc, added[i])
+		if err != nil {
+			syncResult.AddError(err)
+			continue
+		}
+		localRouteTables = append(localRouteTables, *routeTableNew)
+		remoteRouteTables = append(remoteRouteTables, added[i])
+		syncResult.Add()
+	}
+	return localRouteTables, remoteRouteTables, syncResult
+}
+
+func (man *SRouteTableManager) newRouteTableFromCloud(userCred mcclient.TokenCredential, vpc *SVpc, cloudRouteTable cloudprovider.ICloudRouteTable) (*SRouteTable, error) {
+	routes := []*SRoute{}
+	{
+		cloudRoutes, err := cloudRouteTable.GetIRoutes()
+		if err != nil {
+			return nil, err
+		}
+		for _, cloudRoute := range cloudRoutes {
+			route := &SRoute{
+				Type:        cloudRoute.GetType(),
+				Cidr:        cloudRoute.GetCidr(),
+				NextHopType: cloudRoute.GetNextHopType(),
+				NextHopId:   cloudRoute.GetNextHop(),
+			}
+			routes = append(routes, route)
+		}
+	}
+	routeTable := &SRouteTable{
+		CloudregionId: vpc.CloudregionId,
+		VpcId:         vpc.Id,
+		Type:          cloudRouteTable.GetType(),
+		Routes:        (*SRoutes)(&routes),
+	}
+	routeTable.Name = cloudRouteTable.GetName()
+	routeTable.ManagerId = vpc.ManagerId
+	routeTable.ExternalId = cloudRouteTable.GetGlobalId()
+	routeTable.Description = cloudRouteTable.GetDescription()
+	routeTable.ProjectId = userCred.GetProjectId()
+	routeTable.SetModelManager(man)
+	return routeTable, nil
+}
+
+func (man *SRouteTableManager) insertFromCloud(userCred mcclient.TokenCredential, vpc *SVpc, cloudRouteTable cloudprovider.ICloudRouteTable) (*SRouteTable, error) {
+	routeTable, err := man.newRouteTableFromCloud(userCred, vpc, cloudRouteTable)
+	if err != nil {
+		return nil, err
+	}
+	if err := man.TableSpec().Insert(routeTable); err != nil {
+		return nil, err
+	}
+	return routeTable, nil
+}
+
+func (self *SRouteTable) SyncWithCloudRouteTable(userCred mcclient.TokenCredential, vpc *SVpc, cloudRouteTable cloudprovider.ICloudRouteTable) error {
+	man := self.GetModelManager().(*SRouteTableManager)
+	routeTable, err := man.newRouteTableFromCloud(userCred, vpc, cloudRouteTable)
+	if err != nil {
+		return err
+	}
+	_, err = man.TableSpec().Update(self, func() error {
+		self.CloudregionId = routeTable.CloudregionId
+		self.VpcId = vpc.Id
+		self.Type = routeTable.Type
+		self.Routes = routeTable.Routes
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}

pkg/compute/tasks/cloud_provider_sync_info_task.go

@@ -209,6 +209,7 @@ func syncRegionVPCs(ctx context.Context, provider *models.SCloudprovider, task *
 	for j := 0; j < len(localVpcs); j += 1 {
 		syncVpcWires(ctx, provider, task, &localVpcs[j], remoteVpcs[j], syncRange)
 		syncVpcSecGroup(ctx, provider, task, &localVpcs[j], remoteVpcs[j], syncRange)
+		syncVpcRouteTables(ctx, provider, task, &localVpcs[j], remoteVpcs[j], syncRange)
 	}
 }
 
@@ -230,6 +231,24 @@ func syncVpcSecGroup(ctx context.Context, provider *models.SCloudprovider, task
 	}
 }
 
+func syncVpcRouteTables(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, localVpc *models.SVpc, remoteVpc cloudprovider.ICloudVpc, syncRange *models.SSyncRange) {
+	routeTables, err := remoteVpc.GetIRouteTables()
+	if err != nil {
+		msg := fmt.Sprintf("GetIRouteTables for vpc %s failed %s", remoteVpc.GetId(), err)
+		log.Errorf(msg)
+		logSyncFailed(provider, task, msg)
+		return
+	}
+	_, _, result := models.RouteTableManager.SyncRouteTables(ctx, task.GetUserCred(), localVpc, routeTables)
+	msg := result.Result()
+	notes := fmt.Sprintf("SyncRouteTables for VPC %s result: %s", localVpc.Name, msg)
+	log.Infof(notes)
+	if result.IsError() {
+		logSyncFailed(provider, task, msg)
+		return
+	}
+}
+
 func syncVpcWires(ctx context.Context, provider *models.SCloudprovider, task taskman.ITask, localVpc *models.SVpc, remoteVpc cloudprovider.ICloudVpc, syncRange *models.SSyncRange) {
 	wires, err := remoteVpc.GetIWires()
 	if err != nil {

pkg/util/aliyun/routetable.go

@@ -1,7 +1,13 @@
 package aliyun
 
 import (
+	"fmt"
 	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 )
 
 // {"CreationTime":"2017-03-19T13:37:40Z","RouteEntrys":{"RouteEntry":[{"DestinationCidrBlock":"172.31.32.0/20","InstanceId":"","NextHopType":"local","NextHops":{"NextHop":[]},"RouteTableId":"vtb-j6c60lectdi80rk5xz43g","Status":"Available","Type":"System"},{"DestinationCidrBlock":"100.64.0.0/10","InstanceId":"","NextHopType":"service","NextHops":{"NextHop":[]},"RouteTableId":"vtb-j6c60lectdi80rk5xz43g","Status":"Available","Type":"System"}]},"RouteTableId":"vtb-j6c60lectdi80rk5xz43g","RouteTableType":"System","VRouterId":"vrt-j6c00qrol733dg36iq4qj"}
@@ -11,23 +17,199 @@ type SNextHops struct {
 }
 
 type SRouteEntry struct {
-	DestinationCidrBlock string
-	InstanceId           string
-	NextHopType          string
-	NextHops             SNextHops
+	routeTable *SRouteTable
+
 	RouteTableId         string
-	Status               string
 	Type                 string
+	DestinationCidrBlock string
+	NextHopType          string
+	InstanceId           string
+	NextHops             SNextHops
+}
+
+func (route *SRouteEntry) GetType() string {
+	return route.Type
+}
+
+func (route *SRouteEntry) GetCidr() string {
+	return route.DestinationCidrBlock
+}
+
+func (route *SRouteEntry) GetNextHopType() string {
+	return route.NextHopType
+}
+
+func (route *SRouteEntry) GetNextHop() string {
+	return route.InstanceId
 }
 
 type SRouteEntrys struct {
-	RouteEntry []SRouteEntry
+	RouteEntry []*SRouteEntry
 }
 
 type SRouteTable struct {
-	CreationTime   time.Time
-	RouteEntrys    SRouteEntrys
+	region *SRegion
+	vpc    *SVpc
+	routes []cloudprovider.ICloudRoute
+
+	VpcId        string
+	CreationTime time.Time
+	RouteEntrys  SRouteEntrys
+	VRouterId    string
+	Description  string
+
 	RouteTableId   string
+	RouteTableName string
 	RouteTableType string
-	VRouterId      string
+	RouterId       string
+	RouterType     string
+	VSwitchIds     SRouteTableVSwitchIds
+}
+
+type SRouteTableVSwitchIds struct {
+	VSwitchId []string
+}
+
+type sDescribeRouteTablesResponseRouteTables struct {
+	RouteTable []SRouteTable
+}
+
+type sDescribeRouteTablesResponse struct {
+	RouteTables sDescribeRouteTablesResponseRouteTables
+	TotalCount  int
+}
+
+func (self *SRouteTable) GetDescription() string {
+	return self.Description
+}
+
+func (self *SRouteTable) GetId() string {
+	return self.GetGlobalId()
+}
+
+func (self *SRouteTable) GetGlobalId() string {
+	return self.RouteTableId
+}
+
+func (self *SRouteTable) GetManagerId() string {
+	return self.region.client.providerId
+}
+
+func (self *SRouteTable) GetName() string {
+	return self.RouteTableName
+}
+
+func (self *SRouteTable) GetMetadata() *jsonutils.JSONDict {
+	return nil
+}
+
+func (self *SRouteTable) GetRegionId() string {
+	return self.region.RegionId
+}
+
+func (self *SRouteTable) GetType() string {
+	return self.RouteTableType
+}
+
+func (self *SRouteTable) GetVpcId() string {
+	return self.VpcId
+}
+
+func (self *SRouteTable) GetIRoutes() ([]cloudprovider.ICloudRoute, error) {
+	if self.routes == nil {
+		err := self.fetchRoutes()
+		if err != nil {
+			return nil, err
+		}
+	}
+	return self.routes, nil
+}
+
+func (self *SRouteTable) GetStatus() string {
+	return ""
+}
+
+func (self *SRouteTable) IsEmulated() bool {
+	return false
+}
+
+func (self *SRouteTable) Refresh() error {
+	return nil
+}
+
+func (self *SRouteTable) fetchRoutes() error {
+	routes := make([]*SRouteEntry, 0)
+	for {
+		parts, total, err := self.RemoteGetRoutes(len(routes), 50)
+		if err != nil {
+			return err
+		}
+		routes = append(routes, parts...)
+		if len(routes) >= total {
+			break
+		}
+	}
+	self.routes = make([]cloudprovider.ICloudRoute, len(routes))
+	for i := 0; i < len(routes); i++ {
+		routes[i].routeTable = self
+		self.routes[i] = routes[i]
+	}
+	return nil
+}
+
+func (self *SRouteTable) RemoteGetRoutes(offset int, limit int) ([]*SRouteEntry, int, error) {
+	if limit > 50 || limit <= 0 {
+		limit = 50
+	}
+	params := make(map[string]string)
+	params["RouteTableId"] = self.RouteTableId
+	params["PageSize"] = fmt.Sprintf("%d", limit)
+	params["PageNumber"] = fmt.Sprintf("%d", (offset/limit)+1)
+
+	body, err := self.region.ecsRequest("DescribeRouteTables", params)
+	if err != nil {
+		log.Errorf("RemoteGetRoutes fail %s", err)
+		return nil, 0, err
+	}
+
+	resp := sDescribeRouteTablesResponse{}
+	err = body.Unmarshal(&resp)
+	if err != nil {
+		log.Errorf("Unmarshal routeEntrys fail %s", err)
+		return nil, 0, err
+	}
+	routeTables := resp.RouteTables.RouteTable
+	if len(routeTables) != 1 {
+		return nil, 0, fmt.Errorf("expecting 1 route table, got %d", len(routeTables))
+	}
+	routeTable := routeTables[0]
+	return routeTable.RouteEntrys.RouteEntry, resp.TotalCount, nil
+}
+
+func (self *SVpc) RemoteGetRouteTableList(offset int, limit int) ([]*SRouteTable, int, error) {
+	if limit > 50 || limit <= 0 {
+		limit = 50
+	}
+	params := make(map[string]string)
+	params["VpcId"] = self.VpcId
+	params["PageSize"] = fmt.Sprintf("%d", limit)
+	params["PageNumber"] = fmt.Sprintf("%d", (offset/limit)+1)
+
+	body, err := self.apiRequest("DescribeRouteTableList", params)
+	if err != nil {
+		log.Errorf("RemoteGetRouteTableList fail %s", err)
+		return nil, 0, err
+	}
+
+	routeTables := make([]*SRouteTable, 0)
+	err = body.Unmarshal(&routeTables, "RouterTableList", "RouterTableListType")
+	if err != nil {
+		log.Errorf("Unmarshal routeTables fail %s", err)
+		return nil, 0, err
+	}
+	for _, routeTable := range routeTables {
+		routeTable.region = self.region
+	}
+	total, _ := body.Int("TotalCount")
+	return routeTables, int(total), nil
 }

pkg/util/aliyun/vpc.go

@@ -30,7 +30,8 @@ type SVpc struct {
 
 	iwires []cloudprovider.ICloudWire
 
-	secgroups []cloudprovider.ICloudSecurityGroup
+	secgroups   []cloudprovider.ICloudSecurityGroup
+	routeTables []cloudprovider.ICloudRouteTable
 
 	CidrBlock    string
 	CreationTime time.Time
@@ -191,6 +192,36 @@ func (self *SVpc) GetISecurityGroups() ([]cloudprovider.ICloudSecurityGroup, err
 	return self.secgroups, nil
 }
 
+func (self *SVpc) fetchRouteTables() error {
+	routeTables := make([]*SRouteTable, 0)
+	for {
+		parts, total, err := self.RemoteGetRouteTableList(len(routeTables), 50)
+		if err != nil {
+			return err
+		}
+		routeTables = append(routeTables, parts...)
+		if len(routeTables) >= total {
+			break
+		}
+	}
+	self.routeTables = make([]cloudprovider.ICloudRouteTable, len(routeTables))
+	for i := 0; i < len(routeTables); i++ {
+		routeTables[i].vpc = self
+		self.routeTables[i] = routeTables[i]
+	}
+	return nil
+}
+
+func (self *SVpc) GetIRouteTables() ([]cloudprovider.ICloudRouteTable, error) {
+	if self.routeTables == nil {
+		err := self.fetchRouteTables()
+		if err != nil {
+			return nil, err
+		}
+	}
+	return self.routeTables, nil
+}
+
 func (self *SVpc) GetManagerId() string {
 	return self.region.client.providerId
 }

pkg/util/aws/vpc.go

@@ -103,6 +103,11 @@ func (self *SVpc) GetISecurityGroups() ([]cloudprovider.ICloudSecurityGroup, err
 	return self.secgroups, nil
 }
 
+func (self *SVpc) GetIRouteTables() ([]cloudprovider.ICloudRouteTable, error) {
+	rts := []cloudprovider.ICloudRouteTable{}
+	return rts, nil
+}
+
 func (self *SVpc) GetManagerId() string {
 	return self.region.client.providerId
 }

pkg/util/azure/classic_vpc.go

@@ -140,6 +140,11 @@ func (self *SClassicVpc) GetISecurityGroups() ([]cloudprovider.ICloudSecurityGro
 	return self.secgroups, nil
 }
 
+func (self *SClassicVpc) GetIRouteTables() ([]cloudprovider.ICloudRouteTable, error) {
+	rts := []cloudprovider.ICloudRouteTable{}
+	return rts, nil
+}
+
 func (self *SClassicVpc) fetchWires() error {
 	networks := make([]cloudprovider.ICloudNetwork, len(self.Properties.Subnets))
 	wire := SClassicWire{zone: self.region.izones[0].(*SZone), vpc: self}

pkg/util/azure/vpc.go

@@ -148,6 +148,11 @@ func (self *SVpc) GetISecurityGroups() ([]cloudprovider.ICloudSecurityGroup, err
 	return self.secgroups, nil
 }
 
+func (self *SVpc) GetIRouteTables() ([]cloudprovider.ICloudRouteTable, error) {
+	rts := []cloudprovider.ICloudRouteTable{}
+	return rts, nil
+}
+
 func (self *SVpc) fetchWires() error {
 	networks := make([]cloudprovider.ICloudNetwork, len(*self.Properties.Subnets))
 	if len(self.region.izones) == 0 {

pkg/util/qcloud/vpc.go

@@ -86,6 +86,11 @@ func (self *SVpc) GetISecurityGroups() ([]cloudprovider.ICloudSecurityGroup, err
 	return isecgroups, nil
 }
 
+func (self *SVpc) GetIRouteTables() ([]cloudprovider.ICloudRouteTable, error) {
+	rts := []cloudprovider.ICloudRouteTable{}
+	return rts, nil
+}
+
 func (self *SVpc) getWireByZoneId(zoneId string) *SWire {
 	for i := 0; i <= len(self.iwires); i++ {
 		wire := self.iwires[i].(*SWire)