feat(apimap): fetch vpcagent net topo info from DB (#17023)

build/docker/Dockerfile.apimap

@@ -0,0 +1,3 @@
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5-1
+
+ADD ./_output/alpine-build/bin/apimap /opt/yunion/bin/apimap

cmd/apimap/main.go

@@ -0,0 +1,21 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import "yunion.io/x/onecloud/pkg/apimap/service"
+
+func main() {
+	service.StartService()
+}

cmd/climc/main.go

@@ -18,6 +18,7 @@ import (
 	"yunion.io/x/onecloud/cmd/climc/entry"
 	_ "yunion.io/x/onecloud/cmd/climc/shell"
 	_ "yunion.io/x/onecloud/cmd/climc/shell/ansible"
+	_ "yunion.io/x/onecloud/cmd/climc/shell/apimap"
 	_ "yunion.io/x/onecloud/cmd/climc/shell/cloudevent"
 	_ "yunion.io/x/onecloud/cmd/climc/shell/cloudid"
 	_ "yunion.io/x/onecloud/cmd/climc/shell/cloudnet"

cmd/climc/shell/apimap/apimap.go

@@ -0,0 +1,36 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package apimap
+
+import (
+	"fmt"
+
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/apimap"
+)
+
+func init() {
+	type GetOptions struct{}
+	shell.R(new(GetOptions), "apimap-vpcagent", "Show net map for vpcagent", func(s *mcclient.ClientSession, _ *GetOptions) error {
+		ret, err := apimap.APIMap.GetVPCAgentTopo(s)
+		if err != nil {
+			return err
+		}
+		fmt.Print(ret.YAMLString())
+		return nil
+	})
+
+}

pkg/apihelper/db.go

@@ -0,0 +1,127 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package apihelper
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+type GetDBModelsOptions struct {
+	modelOptions   *GetModelsOptions
+	modelDBManager db.IModelManager
+}
+
+func (o GetDBModelsOptions) IncludeOtherCloudEnv() bool {
+	return o.modelOptions.InCludeOtherCloudEnv
+}
+
+func (o GetDBModelsOptions) GetModelSet() IModelSet {
+	return o.modelOptions.ModelSet
+}
+
+func (o GetDBModelsOptions) GetMinUpdatedAt() time.Time {
+	return o.modelOptions.MinUpdatedAt
+}
+
+func (o GetDBModelsOptions) IncludeDetails() bool {
+	return o.modelOptions.IncludeDetails
+}
+
+func (o GetDBModelsOptions) IncludeEmulated() bool {
+	return o.modelOptions.IncludeEmulated
+}
+
+func (o GetDBModelsOptions) BatchListSize() int {
+	return o.modelOptions.BatchListSize
+}
+
+func (o GetDBModelsOptions) InCludeOtherCloudEnv() bool {
+	return o.modelOptions.InCludeOtherCloudEnv
+}
+
+func GetDBModels(opts *GetDBModelsOptions) error {
+	man := opts.modelDBManager
+	manKeyPlural := man.KeywordPlural()
+
+	limit := opts.BatchListSize()
+	// limit := 5
+	listOptions := options.BaseListOptions{
+		System:       options.Bool(true),
+		Admin:        options.Bool(true),
+		Scope:        "system",
+		Details:      options.Bool(opts.IncludeDetails()),
+		ShowEmulated: options.Bool(opts.IncludeEmulated()),
+		OrderBy:      []string{"updated_at"},
+		Order:        "asc",
+		Limit:        &limit,
+	}
+	if !opts.InCludeOtherCloudEnv() {
+		listOptions.Filter = append(listOptions.Filter,
+			"manager_id.isnullorempty()",  // len(manager_id) > 0 is for pubcloud objects
+			"external_id.isnullorempty()", // len(external_id) > 0 is for pubcloud objects
+		)
+		listOptions.CloudEnv = "onpremise"
+	}
+	if inter, ok := opts.GetModelSet().(IModelSetFilter); ok {
+		filter := inter.ModelFilter()
+		listOptions.Filter = append(listOptions.Filter, filter...)
+	}
+	params, err := listOptions.Params()
+	if err != nil {
+		return fmt.Errorf("%s: making list params: %s", manKeyPlural, err)
+	}
+	if inter, ok := opts.GetModelSet().(IModelListParam); ok {
+		filter := inter.ModelParamFilter()
+		params.Update(filter)
+	}
+	//XXX
+	//params.Set(api.LBAGENT_QUERY_ORIG_KEY, jsonutils.NewString(api.LBAGENT_QUERY_ORIG_VAL))
+
+	entriesJson := []jsonutils.JSONObject{}
+	for {
+		log.Debugf("list %s with params: %s", manKeyPlural, params.String())
+		var err error
+		listResult, err := db.ListItems(man, context.Background(), auth.AdminCredential(), params, nil)
+		if err != nil {
+			return fmt.Errorf("%s: list failed: %s",
+				manKeyPlural, err)
+		}
+		entriesJson = append(entriesJson, listResult.Data...)
+		if listResult.Offset+len(listResult.Data) >= listResult.Total {
+			break
+		} else {
+			offset := listResult.Offset + len(listResult.Data)
+			params.Set("offset", jsonutils.NewInt(int64(offset)))
+		}
+	}
+	{
+		err := InitializeModelSetFromJSON(opts.GetModelSet(), entriesJson)
+		if err != nil {
+			return fmt.Errorf("%s: initializing model set failed: %s",
+				manKeyPlural, err)
+		}
+	}
+
+	return nil
+}

pkg/apihelper/interface.go

@@ -16,6 +16,7 @@ package apihelper
 
 import (
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	mcclient "yunion.io/x/onecloud/pkg/mcclient"
@@ -35,6 +36,11 @@ type IModelSets interface {
 	CopyJoined() IModelSets
 }
 
+type IDBModelSets interface {
+	IModelSets
+	FetchFromAPIMap(s *mcclient.ClientSession) (IModelSets, error)
+}
+
 type IModelSet interface {
 	ModelManager() mcclient_modulebase.IBaseManager
 	NewModel() db.IModel
@@ -42,6 +48,11 @@ type IModelSet interface {
 	Copy() IModelSet
 }
 
+type IDBModelSet interface {
+	IModelSet
+	DBModelManager() db.IModelManager
+}
+
 type IModelSetEmulatedIncluder interface {
 	IncludeEmulated() bool
 }
@@ -58,7 +69,27 @@ type IModelListSetParams interface {
 	SetModelListParams(params *jsonutils.JSONDict) *jsonutils.JSONDict
 }
 
-func SyncModelSets(mssOld IModelSets, s *mcclient.ClientSession, opt *Options) (r ModelSetsUpdateResult, err error) {
+func SyncModelSets(mssOld IModelSets, s *mcclient.ClientSession, opt *Options) (ModelSetsUpdateResult, error) {
+	var (
+		mssNews IModelSets
+		err     error
+	)
+	if mssDB, ok := mssOld.(IDBModelSets); ok && !opt.FetchFromComputeService {
+		mssNews, err = mssDB.FetchFromAPIMap(s)
+		if err != nil {
+			return ModelSetsUpdateResult{}, errors.Wrap(err, "FetchFromAPIMap")
+		}
+	} else {
+		mssNews, err = syncModelSets(mssOld, s, opt)
+		if err != nil {
+			return ModelSetsUpdateResult{}, errors.Wrap(err, "syncModelSets")
+		}
+	}
+	r := mssOld.ApplyUpdates(mssNews)
+	return r, nil
+}
+
+func syncModelSets(mssOld IModelSets, s *mcclient.ClientSession, opt *Options) (IModelSets, error) {
 	mss := mssOld.ModelSetList()
 	mssNews := mssOld.NewEmpty()
 	for i, msNew := range mssNews.ModelSetList() {
@@ -69,7 +100,7 @@ func SyncModelSets(mssOld IModelSets, s *mcclient.ClientSession, opt *Options) (
 		if optProvider, ok := msNew.(IModelSetEmulatedIncluder); ok {
 			includeEmulated = optProvider.IncludeEmulated()
 		}
-		err = GetModels(&GetModelsOptions{
+		err := GetModels(&GetModelsOptions{
 			ClientSession: s,
 			ModelManager:  msNew.ModelManager(),
 			MinUpdatedAt:  minUpdatedAt,
@@ -81,7 +112,49 @@ func SyncModelSets(mssOld IModelSets, s *mcclient.ClientSession, opt *Options) (
 			InCludeOtherCloudEnv: opt.IncludeOtherCloudEnv,
 		})
 		if err != nil {
-			return
+			return nil, errors.Wrap(err, "GetModels")
+		}
+	}
+	return mssNews, nil
+}
+
+func SyncDBModelSets(mssOld IModelSets, s *mcclient.ClientSession, opt *Options) (r ModelSetsUpdateResult, err error) {
+	mss := mssOld.ModelSetList()
+	mssNews := mssOld.NewEmpty()
+	for i, msNew := range mssNews.ModelSetList() {
+		var (
+			minUpdatedAt    = ModelSetMaxUpdatedAt(mss[i])
+			includeEmulated = false
+		)
+		if optProvider, ok := msNew.(IModelSetEmulatedIncluder); ok {
+			includeEmulated = optProvider.IncludeEmulated()
+		}
+		msi, ok := msNew.(IDBModelSet)
+		opts := &GetModelsOptions{
+			ClientSession: s,
+			ModelManager:  msNew.ModelManager(),
+			MinUpdatedAt:  minUpdatedAt,
+			ModelSet:      msNew,
+			BatchListSize: opt.ListBatchSize,
+
+			IncludeDetails:       opt.IncludeDetails,
+			IncludeEmulated:      includeEmulated,
+			InCludeOtherCloudEnv: opt.IncludeOtherCloudEnv,
+		}
+		if ok {
+			dbOpts := &GetDBModelsOptions{
+				modelOptions:   opts,
+				modelDBManager: msi.DBModelManager(),
+			}
+			err = GetDBModels(dbOpts)
+			if err != nil {
+				return
+			}
+		} else {
+			err = GetModels(opts)
+			if err != nil {
+				return
+			}
 		}
 	}
 	r = mssOld.ApplyUpdates(mssNews)

pkg/apihelper/options.go

@@ -21,9 +21,10 @@ import (
 type Options struct {
 	common_options.CommonOptions
 
-	SyncIntervalSeconds  int
-	RunDelayMilliseconds int
-	ListBatchSize        int
-	IncludeDetails       bool
-	IncludeOtherCloudEnv bool
+	SyncIntervalSeconds     int
+	RunDelayMilliseconds    int
+	ListBatchSize           int
+	IncludeDetails          bool
+	IncludeOtherCloudEnv    bool
+	FetchFromComputeService bool
 }

pkg/apimap/models/vpcagent/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vpcagent // import "yunion.io/x/onecloud/pkg/apimap/models/vpcagent"

pkg/apimap/models/vpcagent/vpcagent.go

@@ -0,0 +1,48 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vpcagent
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/apihelper"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/vpcagent/models"
+)
+
+type Result struct {
+	Models  *models.ModelSets
+	Correct bool
+	Changed bool
+}
+
+func GetTopoResult(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (interface{}, error) {
+	mss := models.NewModelSets()
+	s := auth.GetAdminSession(ctx, "")
+	r, err := apihelper.SyncDBModelSets(mss, s, &apihelper.Options{
+		ListBatchSize:        1024,
+		IncludeDetails:       false,
+		IncludeOtherCloudEnv: false,
+	})
+	ret := &Result{
+		Models:  mss,
+		Correct: r.Correct,
+		Changed: r.Changed,
+	}
+	return ret, err
+}

pkg/apimap/options/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package options // import "yunion.io/x/onecloud/pkg/apimap/options"

pkg/apimap/options/options.go

@@ -0,0 +1,55 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package options
+
+import (
+	"os"
+
+	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
+	"yunion.io/x/onecloud/pkg/compute/options"
+)
+
+type SOptions struct {
+	options.ComputeOptions
+}
+
+var (
+	opts SOptions
+)
+
+func GetOptions() *SOptions {
+	return &opts
+}
+
+func Init() {
+	common_options.ParseOptions(&opts, os.Args, "apimap.conf", "apimap")
+	options.Options = opts.ComputeOptions
+}
+func OnOptionsChange(oldO, newO interface{}) bool {
+	oldOpts := oldO.(*SOptions)
+	newOpts := newO.(*SOptions)
+
+	changed := false
+	if common_options.OnCommonOptionsChange(&oldOpts.CommonOptions, &newOpts.CommonOptions) {
+		changed = true
+	}
+	if common_options.OnDBOptionsChange(&oldOpts.DBOptions, &newOpts.DBOptions) {
+		changed = true
+	}
+
+	options.Options = newOpts.ComputeOptions
+
+	return changed
+}

pkg/apimap/service/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package service // import "yunion.io/x/onecloud/pkg/apimap/service"

pkg/apimap/service/service.go

@@ -0,0 +1,69 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package service
+
+import (
+	"context"
+	"net/http"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/apimap/models/vpcagent"
+	"yunion.io/x/onecloud/pkg/apimap/options"
+	compute_api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/appsrv"
+	common_app "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
+	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
+	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/scheduler/service"
+)
+
+func StartService() {
+	options.Init()
+	opt := options.GetOptions()
+
+	// hack: set GuestManager recordChecksum to false
+	models.GuestManager.SetEnableRecordChecksum(false)
+
+	service.StartServiceWrapper(&opt.DBOptions, &opt.CommonOptions, func(app *appsrv.Application) error {
+		common_options.StartOptionManager(&opt, opt.ConfigSyncPeriodSeconds, compute_api.SERVICE_TYPE, compute_api.SERVICE_VERSION, options.OnOptionsChange)
+		InitHandlers(app)
+		common_app.ServeForever(app, &opt.BaseOptions)
+		return nil
+	})
+}
+
+func InitHandlers(app *appsrv.Application) {
+	app.AddHandler2("GET", "/vpcagent", auth.Authenticate(vpcAgentHandler), nil, "get_vpcagent_topo", nil)
+}
+
+func vpcAgentHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	userCred := auth.FetchUserCredential(ctx, policy.FilterPolicyCredential)
+	query, err := jsonutils.ParseQueryString(r.URL.RawQuery)
+	if err != nil {
+		httperrors.GeneralServerError(ctx, w, err)
+		return
+	}
+
+	result, err := vpcagent.GetTopoResult(ctx, userCred, query)
+	if err != nil {
+		httperrors.GeneralServerError(ctx, w, err)
+		return
+	}
+	appsrv.SendJSON(w, jsonutils.Marshal(result))
+}

pkg/apis/const.go

@@ -41,6 +41,8 @@ const (
 
 	SERVICE_TYPE_SCHEDULEDTASK = "scheduledtask"
 
+	SERVICE_TYPE_APIMAP = "apimap"
+
 	STATUS_UPDATE_TAGS        = "update_tags"
 	STATUS_UPDATE_TAGS_FAILED = "update_tags_fail"
 

pkg/cloudcommon/db/models.go

@@ -112,7 +112,7 @@ func tableSpecId(tableSpec ITableSpec) string {
 }
 
 func CheckSync(autoSync bool, enableChecksumTables bool, skipInitChecksum bool) bool {
-	log.Infof("Start check database schema ...")
+	log.Infof("Start check database schema: autoSync(%v), enableChecksumTables(%v), skipInitChecksum(%v)", autoSync, enableChecksumTables, skipInitChecksum)
 	inSync := true
 
 	var err error

pkg/mcclient/modules/apimap/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package apimap // import "yunion.io/x/onecloud/pkg/mcclient/modules/apimap"

pkg/mcclient/modules/apimap/vpcagent.go

@@ -0,0 +1,42 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package apimap
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+var (
+	APIMap APIMapManager
+)
+
+func init() {
+	APIMap = APIMapManager{
+		ResourceManager: modules.NewAPIMapManager("", "", nil, nil),
+	}
+}
+
+type APIMapManager struct {
+	modulebase.ResourceManager
+}
+
+func (m APIMapManager) GetVPCAgentTopo(s *mcclient.ClientSession) (jsonutils.JSONObject, error) {
+	_, ret, err := modulebase.JsonRequest(m.ResourceManager, s, "GET", "/vpcagent", nil, nil)
+	return ret, err
+}

pkg/mcclient/modules/managers.go

@@ -174,3 +174,9 @@ func NewScheduledtaskManager(keyword, keywordPlural string, columns, adminColumn
 		BaseManager: *modulebase.NewBaseManager(apis.SERVICE_TYPE_SCHEDULEDTASK, "", "", columns, adminColumns),
 		Keyword:     keyword, KeywordPlural: keywordPlural}
 }
+
+func NewAPIMapManager(keyword, keywordPlural string, columns, adminColumns []string) modulebase.ResourceManager {
+	return modulebase.ResourceManager{
+		BaseManager: *modulebase.NewBaseManager(apis.SERVICE_TYPE_APIMAP, "", "", columns, adminColumns),
+		Keyword:     keyword, KeywordPlural: keywordPlural}
+}

pkg/scheduler/service/service.go

@@ -30,6 +30,7 @@ import (
 	_ "yunion.io/x/sqlchemy/backends"
 
 	compute_api "yunion.io/x/onecloud/pkg/apis/scheduler"
+	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/cloudcommon"
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
@@ -54,17 +55,7 @@ import (
 	"yunion.io/x/onecloud/pkg/util/gin/middleware"
 )
 
-func StartService() error {
-	o.Init()
-	dbOpts := o.Options.DBOptions
-
-	o.Options.Port = o.Options.SchedulerPort
-	// init region compute models
-	cloudcommon.InitDB(&dbOpts)
-	defer cloudcommon.CloseDB()
-
-	db.InitAllManagers()
-
+func EnsureDBSync(opt *common_options.DBOptions) {
 	checkDBSyncRetries := 5
 	count := 1
 	for {
@@ -79,54 +70,80 @@ func StartService() error {
 		}
 		count++
 	}
+}
+
+func StartServiceWrapper(
+	dbOpts *common_options.DBOptions,
+	commonOpts *common_options.CommonOptions,
+	sf func(app *appsrv.Application) error) error {
+	// init region compute models
+	cloudcommon.InitDB(dbOpts)
+	defer cloudcommon.CloseDB()
+
+	db.InitAllManagers()
+
+	EnsureDBSync(dbOpts)
 
-	commonOpts := &o.Options.CommonOptions
 	app_common.InitAuth(commonOpts, func() {
 		log.Infof("Auth complete!!")
 	})
 
-	common_options.StartOptionManager(&o.Options, o.Options.ConfigSyncPeriodSeconds, compute_api.SERVICE_TYPE, compute_api.SERVICE_VERSION, o.OnOptionsChange)
-
-	// gin http framework mode configuration
-	ginMode := "release"
-	if o.Options.LogLevel == "debug" {
-		ginMode = "debug"
-	}
-	gin.SetMode(ginMode)
-
 	if err := computemodels.InitDB(); err != nil {
 		log.Fatalf("InitDB fail: %s", err)
 	}
 
-	app := app_common.InitApp(&o.Options.BaseOptions, true)
+	app := app_common.InitApp(&commonOpts.BaseOptions, true)
 	db.AppDBInit(app)
 
-	startSched := func() {
-		stopEverything := make(chan struct{})
-		ctx := context.Background()
-		go skuman.Start(utils.ToDuration(o.Options.SkuRefreshInterval))
-		go schedtag.Start(ctx, utils.ToDuration("30s"))
+	return sf(app)
+}
 
-		for _, f := range []func(ctx context.Context){
-			cloudregion.Manager.Start,
-			zone.Manager.Start,
-			cloudprovider.Manager.Start,
-			cloudaccount.Manager.Start,
-			wire.Manager.Start,
-			network.Manager.Start,
-			hostwire.GetManager().Start,
-			netinterface.GetManager().Start,
-		} {
-			f(ctx)
+func StartService() error {
+	o.Init()
+	dbOpts := o.Options.DBOptions
+	commonOpts := &o.Options.CommonOptions
+	o.Options.Port = o.Options.SchedulerPort
+	o.Options.AutoSyncTable = false
+	o.Options.EnableDBChecksumTables = false
+	o.Options.DBChecksumSkipInit = true
+
+	return StartServiceWrapper(&dbOpts, commonOpts, func(_ *appsrv.Application) error {
+		common_options.StartOptionManager(&o.Options, o.Options.ConfigSyncPeriodSeconds, compute_api.SERVICE_TYPE, compute_api.SERVICE_VERSION, o.OnOptionsChange)
+
+		// gin http framework mode configuration
+		ginMode := "release"
+		if o.Options.LogLevel == "debug" {
+			ginMode = "debug"
 		}
+		gin.SetMode(ginMode)
 
-		time.Sleep(5 * time.Second)
+		startSched := func() {
+			stopEverything := make(chan struct{})
+			ctx := context.Background()
+			go skuman.Start(utils.ToDuration(o.Options.SkuRefreshInterval))
+			go schedtag.Start(ctx, utils.ToDuration("30s"))
 
-		schedman.InitAndStart(stopEverything)
-	}
-	startSched()
-	//InitHandlers(app)
-	return startHTTP(&o.Options)
+			for _, f := range []func(ctx context.Context){
+				cloudregion.Manager.Start,
+				zone.Manager.Start,
+				cloudprovider.Manager.Start,
+				cloudaccount.Manager.Start,
+				wire.Manager.Start,
+				network.Manager.Start,
+				hostwire.GetManager().Start,
+				netinterface.GetManager().Start,
+			} {
+				f(ctx)
+			}
+
+			time.Sleep(5 * time.Second)
+
+			schedman.InitAndStart(stopEverything)
+		}
+		startSched()
+		//InitHandlers(app)
+		return startHTTP(&o.Options)
+	})
 }
 
 func startHTTP(opt *o.SchedulerOptions) error {

pkg/vpcagent/models/modelset.go

@@ -22,6 +22,7 @@ import (
 	"yunion.io/x/onecloud/pkg/apihelper"
 	computeapis "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/compute/models"
 	mcclient_modulebase "yunion.io/x/onecloud/pkg/mcclient/modulebase"
 	mcclient_modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
 )
@@ -57,6 +58,10 @@ func (set Vpcs) ModelManager() mcclient_modulebase.IBaseManager {
 	return &mcclient_modules.Vpcs
 }
 
+func (set Vpcs) DBModelManager() db.IModelManager {
+	return models.VpcManager
+}
+
 func (set Vpcs) NewModel() db.IModel {
 	return &Vpc{}
 }
@@ -153,6 +158,10 @@ func (set Wires) ModelManager() mcclient_modulebase.IBaseManager {
 	return &mcclient_modules.Wires
 }
 
+func (set Wires) DBModelManager() db.IModelManager {
+	return models.WireManager
+}
+
 func (set Wires) NewModel() db.IModel {
 	return &Wire{}
 }
@@ -192,6 +201,10 @@ func (set Guests) ModelManager() mcclient_modulebase.IBaseManager {
 	return &mcclient_modules.Servers
 }
 
+func (set Guests) DBModelManager() db.IModelManager {
+	return models.GuestManager
+}
+
 func (set Guests) NewModel() db.IModel {
 	return &Guest{}
 }
@@ -297,6 +310,10 @@ func (set Hosts) ModelManager() mcclient_modulebase.IBaseManager {
 	return &mcclient_modules.Hosts
 }
 
+func (set Hosts) DBModelManager() db.IModelManager {
+	return models.HostManager
+}
+
 func (set Hosts) NewModel() db.IModel {
 	return &Host{}
 }
@@ -318,6 +335,10 @@ func (set Networks) ModelManager() mcclient_modulebase.IBaseManager {
 	return &mcclient_modules.Networks
 }
 
+func (set Networks) DBModelManager() db.IModelManager {
+	return models.NetworkManager
+}
+
 func (set Networks) NewModel() db.IModel {
 	return &Network{}
 }
@@ -430,6 +451,10 @@ func (set Guestnetworks) ModelManager() mcclient_modulebase.IBaseManager {
 	return &mcclient_modules.Servernetworks
 }
 
+func (set Guestnetworks) DBModelManager() db.IModelManager {
+	return models.GuestnetworkManager
+}
+
 func (set Guestnetworks) NewModel() db.IModel {
 	return &Guestnetwork{}
 }
@@ -530,6 +555,10 @@ func (set NetworkAddresses) ModelManager() mcclient_modulebase.IBaseManager {
 	return &mcclient_modules.NetworkAddresses
 }
 
+func (set NetworkAddresses) DBModelManager() db.IModelManager {
+	return models.NetworkAddressManager
+}
+
 func (set NetworkAddresses) NewModel() db.IModel {
 	return &NetworkAddress{}
 }
@@ -551,6 +580,10 @@ func (set SecurityGroups) ModelManager() mcclient_modulebase.IBaseManager {
 	return &mcclient_modules.SecGroups
 }
 
+func (set SecurityGroups) DBModelManager() db.IModelManager {
+	return models.SecurityGroupManager
+}
+
 func (set SecurityGroups) NewModel() db.IModel {
 	return &SecurityGroup{}
 }
@@ -599,6 +632,10 @@ func (set SecurityGroupRules) ModelManager() mcclient_modulebase.IBaseManager {
 	return &mcclient_modules.SecGroupRules
 }
 
+func (set SecurityGroupRules) DBModelManager() db.IModelManager {
+	return models.SecurityGroupRuleManager
+}
+
 func (set SecurityGroupRules) NewModel() db.IModel {
 	return &SecurityGroupRule{}
 }
@@ -620,6 +657,10 @@ func (set Guestsecgroups) ModelManager() mcclient_modulebase.IBaseManager {
 	return &mcclient_modules.Serversecgroups
 }
 
+func (set Guestsecgroups) DBModelManager() db.IModelManager {
+	return models.GuestsecgroupManager
+}
+
 func (set Guestsecgroups) NewModel() db.IModel {
 	return &Guestsecgroup{}
 }
@@ -682,6 +723,10 @@ func (set Elasticips) ModelManager() mcclient_modulebase.IBaseManager {
 	return &mcclient_modules.Elasticips
 }
 
+func (set Elasticips) DBModelManager() db.IModelManager {
+	return models.ElasticipManager
+}
+
 func (set Elasticips) NewModel() db.IModel {
 	return &Elasticip{}
 }
@@ -707,6 +752,10 @@ func (set DnsRecords) ModelManager() mcclient_modulebase.IBaseManager {
 	return &mcclient_modules.DNSRecords
 }
 
+func (set DnsRecords) DBModelManager() db.IModelManager {
+	return models.DnsRecordManager
+}
+
 func (set DnsRecords) NewModel() db.IModel {
 	return &DnsRecord{}
 }
@@ -728,6 +777,10 @@ func (set RouteTables) ModelManager() mcclient_modulebase.IBaseManager {
 	return &mcclient_modules.RouteTables
 }
 
+func (set RouteTables) DBModelManager() db.IModelManager {
+	return models.RouteTableManager
+}
+
 func (set RouteTables) NewModel() db.IModel {
 	return &RouteTable{}
 }
@@ -749,6 +802,10 @@ func (set Groupguests) ModelManager() mcclient_modulebase.IBaseManager {
 	return &mcclient_modules.InstanceGroupGuests
 }
 
+func (set Groupguests) DBModelManager() db.IModelManager {
+	return models.GroupguestManager
+}
+
 func (set Groupguests) NewModel() db.IModel {
 	return &Groupguest{}
 }
@@ -771,6 +828,10 @@ func (set LoadbalancerNetworks) ModelManager() mcclient_modulebase.IBaseManager
 	return &mcclient_modules.Loadbalancernetworks
 }
 
+func (set LoadbalancerNetworks) DBModelManager() db.IModelManager {
+	return models.LoadbalancernetworkManager
+}
+
 func (set LoadbalancerNetworks) NewModel() db.IModel {
 	return &LoadbalancerNetwork{}
 }
@@ -793,6 +854,10 @@ func (set Groupnetworks) ModelManager() mcclient_modulebase.IBaseManager {
 	return &mcclient_modules.InstancegroupNetworks
 }
 
+func (set Groupnetworks) DBModelManager() db.IModelManager {
+	return models.GroupnetworkManager
+}
+
 func (set Groupnetworks) NewModel() db.IModel {
 	return &Groupnetwork{}
 }
@@ -842,6 +907,10 @@ func (set Groups) ModelManager() mcclient_modulebase.IBaseManager {
 	return &mcclient_modules.InstanceGroups
 }
 
+func (set Groups) DBModelManager() db.IModelManager {
+	return models.GroupManager
+}
+
 func (set Groups) NewModel() db.IModel {
 	return &Group{}
 }
@@ -938,6 +1007,10 @@ func (set LoadbalancerListeners) ModelManager() mcclient_modulebase.IBaseManager
 	return &mcclient_modules.LoadbalancerListeners
 }
 
+func (set LoadbalancerListeners) DBModelManager() db.IModelManager {
+	return models.LoadbalancerListenerManager
+}
+
 func (set LoadbalancerListeners) NewModel() db.IModel {
 	return &LoadbalancerListener{}
 }
@@ -974,6 +1047,10 @@ func (set LoadbalancerAcls) ModelManager() mcclient_modulebase.IBaseManager {
 	return &mcclient_modules.LoadbalancerAcls
 }
 
+func (set LoadbalancerAcls) DBModelManager() db.IModelManager {
+	return models.LoadbalancerAclManager
+}
+
 func (set LoadbalancerAcls) NewModel() db.IModel {
 	return &LoadbalancerAcl{}
 }

pkg/vpcagent/models/modelsets.go

@@ -19,8 +19,11 @@ import (
 	"time"
 
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/apihelper"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/apimap"
 )
 
 type ModelSetsMaxUpdatedAt struct {
@@ -150,6 +153,7 @@ func (mss *ModelSets) ModelSetList() []apihelper.IModelSet {
 
 		mss.Groupguests,
 		mss.Groupnetworks,
+		mss.Groups,
 
 		mss.LoadbalancerNetworks,
 		mss.LoadbalancerListeners,
@@ -181,6 +185,7 @@ func (mss *ModelSets) copy_() *ModelSets {
 
 		Groupguests:   mss.Groupguests.Copy().(Groupguests),
 		Groupnetworks: mss.Groupnetworks.Copy().(Groupnetworks),
+		Groups:        mss.Groups.Copy().(Groups),
 
 		LoadbalancerNetworks:  mss.LoadbalancerNetworks.Copy().(LoadbalancerNetworks),
 		LoadbalancerListeners: mss.LoadbalancerListeners.Copy().(LoadbalancerListeners),
@@ -219,6 +224,18 @@ func (mss *ModelSets) ApplyUpdates(mssNews apihelper.IModelSets) apihelper.Model
 	return r
 }
 
+func (mss *ModelSets) FetchFromAPIMap(s *mcclient.ClientSession) (apihelper.IModelSets, error) {
+	mssNews := mss.NewEmpty()
+	ret, err := apimap.APIMap.GetVPCAgentTopo(s)
+	if err != nil {
+		return nil, errors.Wrap(err, "GetVPCAgentTopo")
+	}
+	if err := ret.Unmarshal(mssNews, "models"); err != nil {
+		return nil, errors.Wrap(err, "Unmarshal topo")
+	}
+	return mssNews, nil
+}
+
 func (mss *ModelSets) join() bool {
 	mss.Guests.initJoin()
 	mss.Groups = Groups{}

pkg/vpcagent/options/options.go

@@ -34,9 +34,10 @@ const (
 type VpcAgentOptions struct {
 	VpcProvider string `default:"ovn"`
 
-	APISyncIntervalSeconds  int `default:"10"`
-	APIRunDelayMilliseconds int `default:"100"`
-	APIListBatchSize        int `default:"1024"`
+	APISyncIntervalSeconds      int  `default:"10"`
+	APIRunDelayMilliseconds     int  `default:"100"`
+	APIListBatchSize            int  `default:"1024"`
+	FetchDataFromComputeService bool `default:"false"`
 
 	OvnWorkerCheckInterval int    `default:"180"`
 	OvnNorthDatabase       string `help:"address for accessing ovn north database.  Default to local unix socket"`

pkg/vpcagent/ovn/worker.go

@@ -45,11 +45,12 @@ type Worker struct {
 func NewWorker(opts *options.Options) worker.IWorker {
 	modelSets := agentmodels.NewModelSets()
 	apiOpts := &apihelper.Options{
-		CommonOptions:        opts.CommonOptions,
-		SyncIntervalSeconds:  opts.APISyncIntervalSeconds,
-		RunDelayMilliseconds: opts.APIRunDelayMilliseconds,
-		ListBatchSize:        opts.APIListBatchSize,
-		IncludeDetails:       false,
+		CommonOptions:           opts.CommonOptions,
+		SyncIntervalSeconds:     opts.APISyncIntervalSeconds,
+		RunDelayMilliseconds:    opts.APIRunDelayMilliseconds,
+		ListBatchSize:           opts.APIListBatchSize,
+		FetchFromComputeService: opts.FetchDataFromComputeService,
+		IncludeDetails:          false,
 
 		IncludeOtherCloudEnv: false,
 	}