Merge pull request #822 in YUNIONIO/onecloud from ~ZHOUYOUSONG/onecloud:bugfix/yousong-options to release/2.4.0

* commit '603e1be98c293ce1e378c16947d720758d768803':
  treewide: 重新组织cloudcommon options
  cloudcommon: 在InitDB()中初始化DEBUG_SQLCHEMY
  appsrv: FetchStruct(): 使用jsonutils

pkg/appsrv/fetch.go

@@ -1,7 +1,6 @@
 package appsrv
 
 import (
-	"encoding/json"
 	"io/ioutil"
 	"net/http"
 
@@ -19,7 +18,11 @@ func FetchStruct(req *http.Request, v interface{}) error {
 		return e
 	}
 	if len(b) > 0 {
-		return json.Unmarshal(b, v)
+		obj, err := jsonutils.Parse(b)
+		if err != nil {
+			return err
+		}
+		return obj.Unmarshal(v)
 	} else {
 		return nil
 	}

pkg/cloudcommon/app.go

@@ -8,7 +8,7 @@ import (
 	"yunion.io/x/onecloud/pkg/appsrv"
 )
 
-func InitApp(options *Options, dbAccess bool) *appsrv.Application {
+func InitApp(options *CommonOptions, dbAccess bool) *appsrv.Application {
 	// cache := appsrv.NewCache(options.AuthTokenCacheSize)
 	app := appsrv.NewApplication(options.ApplicationID, options.RequestWorkerCount, dbAccess)
 	app.CORSAllowHosts(options.CorsHosts)
@@ -20,9 +20,8 @@ func InitApp(options *Options, dbAccess bool) *appsrv.Application {
 	return app
 }
 
-func ServeForever(app *appsrv.Application, options *Options) {
+func ServeForever(app *appsrv.Application, options *CommonOptions) {
 	AppDBInit(app)
-
 	addr := net.JoinHostPort(options.Address, strconv.Itoa(options.Port))
 	proto := "http"
 	if options.EnableSsl {

pkg/cloudcommon/auth.go

@@ -9,7 +9,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 )
 
-func InitAuth(options *Options, authComplete auth.AuthCompletedCallback) {
+func InitAuth(options *CommonOptions, authComplete auth.AuthCompletedCallback) {
 
 	if len(options.AuthURL) == 0 {
 		fmt.Println("Missing AuthURL")

pkg/cloudcommon/database.go

@@ -24,6 +24,10 @@ func InitDB(options *DBOptions) {
 		consts.EnableGlobalVirtualResourceNamespace()
 	}
 
+	if options.DebugSqlchemy {
+		sqlchemy.DEBUG_SQLCHEMY = true
+	}
+
 	dialect, sqlStr, err := options.GetDBConnection()
 	if err != nil {
 		log.Fatalf("Invalid SqlConnection string: %s", options.SqlConnection)

pkg/cloudcommon/options.go

@@ -12,7 +12,7 @@ import (
 	"yunion.io/x/structarg"
 )
 
-type Options struct {
+type CommonOptions struct {
 	Port    int    `help:"The port that the service runs on"`
 	Address string `help:"The IP address to serve on (set to 0.0.0.0 for all interfaces)" default:"0.0.0.0"`
 
@@ -54,15 +54,13 @@ type DBOptions struct {
 
 	GlobalVirtualResourceNamespace bool `help:"Per project namespace or global namespace for virtual resources"`
 	DebugSqlchemy                  bool `default:"False" help:"Print SQL executed by sqlchemy"`
-
-	Options
 }
 
 func (this *DBOptions) GetDBConnection() (dialect, connstr string, err error) {
 	return utils.TransSQLAchemyURL(this.SqlConnection)
 }
 
-func ParseOptions(optStruct interface{}, optionsRef *Options, args []string, configFileName string) {
+func ParseOptions(optStruct interface{}, optionsRef *CommonOptions, args []string, configFileName string) {
 	if len(consts.GetServiceType()) == 0 {
 		log.Fatalf("ServiceType not initialized!")
 	}

pkg/compute/options/options.go

@@ -79,6 +79,7 @@ type ComputeOptions struct {
 	HostOfflineMaxSeconds        int `help:"Maximal seconds interval that a host considered offline during which it did not ping region, default is 3 minues" default:"180"`
 	HostOfflineDetectionInterval int `help:"Interval to check offline hosts, defualt is half a minute" default:"30"`
 
+	cloudcommon.CommonOptions
 	cloudcommon.DBOptions
 }
 

pkg/compute/service/service.go

@@ -7,7 +7,6 @@ import (
 	"yunion.io/x/onecloud/pkg/compute/skus"
 
 	"yunion.io/x/log"
-	"yunion.io/x/sqlchemy"
 
 	_ "github.com/go-sql-driver/mysql"
 	_ "yunion.io/x/onecloud/pkg/compute/guestdrivers"
@@ -31,45 +30,44 @@ import (
 func StartService() {
 	consts.SetServiceType("compute")
 
-	cloudcommon.ParseOptions(&options.Options, &options.Options.Options, os.Args, "region.conf")
+	opts := &options.Options
+	commonOpts := &options.Options.CommonOptions
+	dbOpts := &options.Options.DBOptions
+	cloudcommon.ParseOptions(&options.Options, commonOpts, os.Args, "region.conf")
 
-	if options.Options.DebugSqlchemy {
-		sqlchemy.DEBUG_SQLCHEMY = true
+	if opts.PortV2 > 0 {
+		log.Infof("Port V2 %d is specified, use v2 port", opts.PortV2)
+		commonOpts.Port = opts.PortV2
 	}
 
-	if options.Options.PortV2 > 0 {
-		log.Infof("Port V2 %d is specified, use v2 port", options.Options.PortV2)
-		options.Options.Port = options.Options.PortV2
-	}
-
-	cloudcommon.InitAuth(&options.Options.Options, func() {
+	cloudcommon.InitAuth(commonOpts, func() {
 		log.Infof("Auth complete!!")
 	})
 
-	cloudcommon.InitDB(&options.Options.DBOptions)
+	cloudcommon.InitDB(dbOpts)
 	defer cloudcommon.CloseDB()
 
-	app := cloudcommon.InitApp(&options.Options.Options, true)
+	app := cloudcommon.InitApp(commonOpts, true)
 
 	compute.InitHandlers(app)
 
-	if db.CheckSync(options.Options.AutoSyncTable) {
+	if db.CheckSync(opts.AutoSyncTable) {
 		err := models.InitDB()
 		if err == nil {
 			cron := cronman.GetCronJobManager()
-			cron.AddJob1("CleanPendingDeleteServers", time.Duration(options.Options.PendingDeleteCheckSeconds)*time.Second, models.GuestManager.CleanPendingDeleteServers)
-			cron.AddJob1("CleanPendingDeleteDisks", time.Duration(options.Options.PendingDeleteCheckSeconds)*time.Second, models.DiskManager.CleanPendingDeleteDisks)
-			cron.AddJob1("CleanPendingDeleteLoadbalancers", time.Duration(options.Options.LoadbalancerPendingDeleteCheckInterval)*time.Second, models.LoadbalancerAgentManager.CleanPendingDeleteLoadbalancers)
-			cron.AddJob1("CleanPendingDeleteServers", time.Duration(options.Options.PrepaidExpireCheckSeconds)*time.Second, models.GuestManager.DeleteExpiredPrepaidServers)
-			cron.AddJob1("StartHostPingDetectionTask", time.Duration(options.Options.HostOfflineDetectionInterval)*time.Second, models.HostManager.PingDetectionTask)
+			cron.AddJob1("CleanPendingDeleteServers", time.Duration(opts.PendingDeleteCheckSeconds)*time.Second, models.GuestManager.CleanPendingDeleteServers)
+			cron.AddJob1("CleanPendingDeleteDisks", time.Duration(opts.PendingDeleteCheckSeconds)*time.Second, models.DiskManager.CleanPendingDeleteDisks)
+			cron.AddJob1("CleanPendingDeleteLoadbalancers", time.Duration(opts.LoadbalancerPendingDeleteCheckInterval)*time.Second, models.LoadbalancerAgentManager.CleanPendingDeleteLoadbalancers)
+			cron.AddJob1("CleanPendingDeleteServers", time.Duration(opts.PrepaidExpireCheckSeconds)*time.Second, models.GuestManager.DeleteExpiredPrepaidServers)
+			cron.AddJob1("StartHostPingDetectionTask", time.Duration(opts.HostOfflineDetectionInterval)*time.Second, models.HostManager.PingDetectionTask)
 
-			cron.AddJob2("AutoDiskSnapshot", options.Options.AutoSnapshotDay, options.Options.AutoSnapshotHour, 0, 0, models.DiskManager.AutoDiskSnapshot, false)
-			cron.AddJob2("SyncSkus", options.Options.SyncSkusDay, options.Options.SyncSkusHour, 0, 0, skus.SyncSkus, true)
+			cron.AddJob2("AutoDiskSnapshot", opts.AutoSnapshotDay, opts.AutoSnapshotHour, 0, 0, models.DiskManager.AutoDiskSnapshot, false)
+			cron.AddJob2("SyncSkus", opts.SyncSkusDay, opts.SyncSkusHour, 0, 0, skus.SyncSkus, true)
 
 			cron.Start()
 			defer cron.Stop()
 
-			cloudcommon.ServeForever(app, &options.Options.Options)
+			cloudcommon.ServeForever(app, commonOpts)
 		} else {
 			log.Errorf("InitDB fail: %s", err)
 		}

pkg/lbagent/options.go

@@ -31,7 +31,7 @@ type LbagentOptions struct {
 }
 
 type Options struct {
-	CommonOpts cloudcommon.Options
+	CommonOpts cloudcommon.CommonOptions
 
 	LbagentOptions
 }

pkg/webconsole/options/options.go

@@ -9,7 +9,7 @@ var (
 )
 
 type WebConsoleOptions struct {
-	cloudcommon.Options
+	cloudcommon.CommonOptions
 
 	ApiServer       string `help:"API server url to handle websocket connection, usually with public access" default:"http://webconsole.yunion.io"`
 	KubectlPath     string `help:"kubectl binary path used to connect k8s cluster" default:"/usr/bin/kubectl"`

pkg/webconsole/service/service.go

@@ -27,28 +27,31 @@ func ensureBinExists(binPath string) {
 func StartService() {
 	consts.SetServiceType("webconsole")
 
-	cloudcommon.ParseOptions(&o.Options, &o.Options.Options, os.Args, "webconsole.conf")
+	opts := &o.Options
+	commonOpts := &o.Options.CommonOptions
+	cloudcommon.ParseOptions(opts, commonOpts, os.Args, "webconsole.conf")
 
-	if o.Options.ApiServer == "" {
+	if opts.ApiServer == "" {
 		log.Fatalf("--api-server must specified")
 	}
-	_, err := url.Parse(o.Options.ApiServer)
+	_, err := url.Parse(opts.ApiServer)
 	if err != nil {
-		log.Fatalf("invalid --api-server %s", o.Options.ApiServer)
+		log.Fatalf("invalid --api-server %s", opts.ApiServer)
 	}
 
-	for _, binPath := range []string{o.Options.KubectlPath, o.Options.IpmitoolPath, o.Options.SshToolPath, o.Options.SshpassToolPath} {
+	for _, binPath := range []string{opts.KubectlPath, opts.IpmitoolPath, opts.SshToolPath, opts.SshpassToolPath} {
 		ensureBinExists(binPath)
 	}
 
-	cloudcommon.InitAuth(&o.Options.Options, func() {
+	cloudcommon.InitAuth(commonOpts, func() {
 		log.Infof("Auth complete")
 	})
 	start()
 }
 
 func start() {
-	app := cloudcommon.InitApp(&o.Options.Options, false)
+	commonOpts := &o.Options.CommonOptions
+	app := cloudcommon.InitApp(commonOpts, false)
 	webconsole.InitHandlers(app)
 
 	root := mux.NewRouter()

pkg/yunionconf/options/options.go

@@ -3,6 +3,7 @@ package options
 import "yunion.io/x/onecloud/pkg/cloudcommon"
 
 type YunionConfOptions struct {
+	cloudcommon.CommonOptions
 	cloudcommon.DBOptions
 }
 

pkg/yunionconf/service/service.go

@@ -18,25 +18,28 @@ import (
 func StartService() {
 	consts.SetServiceType("yunionconf")
 
-	cloudcommon.ParseOptions(&options.Options, &options.Options.Options, os.Args, "yunionconf.conf")
-	cloudcommon.InitAuth(&options.Options.Options, func() {
+	opts := &options.Options
+	commonOpts := &options.Options.CommonOptions
+	dbOpts := &options.Options.DBOptions
+	cloudcommon.ParseOptions(opts, commonOpts, os.Args, "yunionconf.conf")
+	cloudcommon.InitAuth(commonOpts, func() {
 		log.Infof("Auth complete!!")
 	})
 
-	if options.Options.GlobalVirtualResourceNamespace {
+	if opts.GlobalVirtualResourceNamespace {
 		consts.EnableGlobalVirtualResourceNamespace()
 	}
 
-	cloudcommon.InitDB(&options.Options.DBOptions)
+	cloudcommon.InitDB(dbOpts)
 	defer cloudcommon.CloseDB()
 
-	app := cloudcommon.InitApp(&options.Options.Options, true)
+	app := cloudcommon.InitApp(commonOpts, true)
 	yunionconf.InitHandlers(app)
 
-	if db.CheckSync(options.Options.AutoSyncTable) {
+	if db.CheckSync(opts.AutoSyncTable) {
 		err := models.InitDB()
 		if err == nil {
-			cloudcommon.ServeForever(app, &options.Options.Options)
+			cloudcommon.ServeForever(app, commonOpts)
 		} else {
 			log.Errorf("InitDB fail: %s", err)
 		}