Merge pull request #177 in YUNIONIO/onecloud from ~TANGBIN/onecloud:feature/tb-paramter-climc to release/2.2.0

* commit '96713696b82ebe866f7d91c867d051325d04deea':
  add auth complete callback
  bugfix import path error
  add yunionconf service
  fix help desc string
  bugfix
  add parameter climc
  delete&update parameter in context
  temp!!withbugs
  add climc

build/yunionconf/root/etc/yunion/yunionconf.conf.sample

@@ -0,0 +1,8 @@
+auth_url = http://10.168.222.251:35357/v3
+admin_user = sysadmin
+admin_passwd = MxqhTC2VKe067jtD
+admin_project = system
+port = 9889
+cors_hosts = *
+sql_connection = 'mysql+pymysql://yunionconf:PASSWORD@10.168.222.1:3306/yunionconf?charset=utf8'
+auto_sync_table = True

build/yunionconf/root/opt/yunion/share/yunionconf/yunion-yunionconf.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Yunion Conf Service
+Documentation=http://doc.yunionyun.com
+After=network.target
+
+[Service]
+Type=simple
+User=yunion
+Group=yunion
+ExecStart=/opt/yunion/bin/yunionconf --config /etc/yunion/yunionconf.conf
+WorkingDirectory=/opt/yunion
+KillMode=process
+
+[Install]
+WantedBy=multi-user.target

build/yunionconf/vars

@@ -0,0 +1 @@
+DESCRIPTION="Yunion Conf Service"

cmd/climc/shell/parameters.go

@@ -0,0 +1,158 @@
+package shell
+
+import (
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func init() {
+	type ParametersListOptions struct {
+		NamespaceId string `help:"List parameter of specificated namespace id, ADMIN only"`
+		User        string `help:"List parameter of specificated user id, ADMIN only"`
+		Service     string `help:"List parameter of specificated service id, ADMIN only"`
+		options.BaseListOptions
+	}
+
+	R(&ParametersListOptions{}, "parameter-list", "list parameters", func(s *mcclient.ClientSession, args *ParametersListOptions) error {
+		params, err := options.ListStructToParams(args)
+		if err != nil {
+			return err
+		}
+
+		var result *modules.ListResult
+		if len(args.NamespaceId) > 0 {
+			params.Add(jsonutils.NewString(args.NamespaceId), "namespace_id")
+			result, err = modules.Parameters.List(s, params)
+		} else if len(args.User) > 0 {
+			result, err = modules.Parameters.ListInContext(s, params, &modules.UsersV3, args.User)
+		} else if len(args.Service) > 0 {
+			result, err = modules.Parameters.ListInContext(s, params, &modules.ServicesV3, args.Service)
+		} else {
+			result, err = modules.Parameters.List(s, params)
+		}
+
+		if err != nil {
+			return err
+		}
+		printList(result, modules.Parameters.GetColumns(s))
+		return nil
+	})
+
+	type ParametersShowOptions struct {
+		NamespaceId string `help:"Show parameter of specificated namespace id, ADMIN only"`
+		User        string `help:"Show parameter of specificated user id, ADMIN only"`
+		Service     string `help:"Show parameter of specificated service id, ADMIN only"`
+		NAME        string `help:"The name of parameter"`
+	}
+
+	R(&ParametersShowOptions{}, "parameter-show", "show a parameter", func(s *mcclient.ClientSession, args *ParametersShowOptions) error {
+		params := jsonutils.NewDict()
+		if len(args.NamespaceId) > 0 {
+			params.Add(jsonutils.JSONTrue, "admin")
+			params.Add(jsonutils.NewString(args.NamespaceId), "namespace_id")
+		}
+
+		var parameter jsonutils.JSONObject
+		var err error
+		if len(args.NamespaceId) > 0 {
+			params.Add(jsonutils.NewString(args.NamespaceId), "namespace_id")
+			parameter, err = modules.Parameters.Get(s, args.NAME, params)
+		} else if len(args.User) > 0 {
+			parameter, err = modules.Parameters.GetInContext(s, args.NAME, params, &modules.UsersV3, args.User)
+		} else if len(args.Service) > 0 {
+			parameter, err = modules.Parameters.GetInContext(s, args.NAME, params, &modules.ServicesV3, args.Service)
+		} else {
+			parameter, err = modules.Parameters.Get(s, args.NAME, params)
+		}
+
+		if err != nil {
+			return err
+		}
+		printObject(parameter)
+		return nil
+	})
+
+	type ParametersCreateOptions struct {
+		User    string `help:"Create parameter for specificated user id, ADMIN only"`
+		Service string `help:"Create parameter for specificated service id, ADMIN only"`
+		NAME    string `help:"The name of parameter"`
+		VALUE   string `help:"The content of parameter"`
+	}
+
+	R(&ParametersCreateOptions{}, "parameter-create", "create a parameter", func(s *mcclient.ClientSession, args *ParametersCreateOptions) error {
+		params := jsonutils.NewDict()
+		params.Add(jsonutils.NewString(args.NAME), "name")
+		params.Add(jsonutils.NewString(args.VALUE), "value")
+
+		if len(args.User) > 0 {
+			params.Add(jsonutils.NewString(args.User), "user_id")
+		} else if len(args.Service) > 0 {
+			params.Add(jsonutils.NewString(args.Service), "service_id")
+		}
+
+		parameter, err := modules.Parameters.Create(s, params)
+		if err != nil {
+			return err
+		}
+		printObject(parameter)
+		return nil
+	})
+
+	type ParametersUpdateOptions struct {
+		User    string `help:"Update parameter of specificated user id, ADMIN only"`
+		Service string `help:"Update parameter of specificated service id, ADMIN only"`
+		NAME    string `help:"The name of parameter"`
+		VALUE   string `help:"The content of parameter"`
+	}
+
+	R(&ParametersUpdateOptions{}, "parameter-update", "update parameter", func(s *mcclient.ClientSession, args *ParametersUpdateOptions) error {
+		params := jsonutils.NewDict()
+		if len(args.VALUE) > 0 {
+			params.Add(jsonutils.NewString(args.VALUE), "value")
+		}
+
+		var parameter jsonutils.JSONObject
+		var err error
+		if len(args.User) > 0 {
+			parameter, err = modules.Parameters.PutInContext(s, args.NAME, params, &modules.UsersV3, args.User)
+		} else if len(args.Service) > 0 {
+			parameter, err = modules.Parameters.PutInContext(s, args.NAME, params, &modules.ServicesV3, args.Service)
+		} else {
+			parameter, err = modules.Parameters.Put(s, args.NAME, params)
+		}
+
+		if err != nil {
+			return err
+		}
+		printObject(parameter)
+		return nil
+	})
+
+	type ParametersDeleteOptions struct {
+		User    string `help:"Delete parameter of specificated user id, ADMIN only"`
+		Service string `help:"Delete parameter of specificated service id, ADMIN only"`
+		NAME    string `help:"The name of parameter"`
+	}
+
+	R(&ParametersDeleteOptions{}, "parameter-delete", "delete notice", func(s *mcclient.ClientSession, args *ParametersDeleteOptions) error {
+		params := jsonutils.NewDict()
+
+		var parameter jsonutils.JSONObject
+		var err error
+		if len(args.User) > 0 {
+			parameter, err = modules.Parameters.DeleteInContext(s, args.NAME, params, &modules.UsersV3, args.User)
+		} else if len(args.Service) > 0 {
+			parameter, err = modules.Parameters.DeleteInContext(s, args.NAME, params, &modules.ServicesV3, args.Service)
+		} else {
+			parameter, err = modules.Parameters.Delete(s, args.NAME, nil)
+		}
+
+		if err != nil {
+			return err
+		}
+		printObject(parameter)
+		return nil
+	})
+}

cmd/yunionconf/main.go

@@ -0,0 +1,14 @@
+package main
+
+/*
+Yunion Conf Service
+“参数服务”在服务器端为指定用户持久化存储和管理个性化参数，例如 控制台的配置，列表的colume配置等，从而实现产品的个性化配置。
+*/
+
+import (
+	"yunion.io/x/onecloud/pkg/yunionconf/service"
+)
+
+func main() {
+	service.StartService()
+}

pkg/mcclient/modules/managers.go

@@ -115,3 +115,11 @@ func NewYunionAgentManager(keyword, keywordPlural string, columns, adminColumns
 			serviceType:  "yunionagent"},
 		Keyword: keyword, KeywordPlural: keywordPlural}
 }
+
+func NewYunionConfManager(keyword, keywordPlural string, columns, adminColumns []string) ResourceManager {
+	return ResourceManager{
+		BaseManager: BaseManager{columns: columns,
+			adminColumns: adminColumns,
+			serviceType:  "yunionconf"},
+		Keyword: keyword, KeywordPlural: keywordPlural}
+}

pkg/mcclient/modules/mod_parameters.go

@@ -0,0 +1,17 @@
+package modules
+
+type ParametersManager struct {
+	ResourceManager
+}
+
+var (
+	Parameters ParametersManager
+)
+
+func init() {
+	Parameters = ParametersManager{NewYunionConfManager("parameter", "parameters",
+		[]string{"id", "created_at", "update_at", "name", "value"},
+		[]string{"namespace", "namespace_id", "created_by", "updated_by"},
+	)}
+	register(&Parameters)
+}

pkg/mcclient/session.go

@@ -155,6 +155,10 @@ func (this *ClientSession) GetRegion() string {
 	return this.region
 }
 
+func (this *ClientSession) GetUserId() string {
+	return this.token.GetUserId()
+}
+
 func (this *ClientSession) GetTenantId() string {
 	return this.token.GetTenantId()
 }

pkg/yunionconf/handlers.go

@@ -0,0 +1,20 @@
+package yunionconf
+
+import (
+	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/appsrv/dispatcher"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/yunionconf/models"
+)
+
+func InitHandlers(app *appsrv.Application) {
+	for _, manager := range []db.IModelManager{
+		models.ParameterManager,
+	} {
+		db.RegisterModelManager(manager)
+		handler := db.NewModelHandler(manager)
+		dispatcher.AddModelDispatcher("", app, handler)
+		dispatcher.AddModelDispatcher("/users/<user_id>", app, handler)
+		dispatcher.AddModelDispatcher("/services/<service_id>", app, handler)
+	}
+}

pkg/yunionconf/models/initdb.go

@@ -0,0 +1,19 @@
+package models
+
+import (
+	"yunion.io/x/log"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+)
+
+func InitDB() error {
+	for _, manager := range []db.IModelManager{
+		ParameterManager,
+	} {
+		err := manager.InitializeData()
+		if err != nil {
+			log.Errorf("Manager %s initializeData fail %s", manager.Keyword(), err)
+			return err
+		}
+	}
+	return nil
+}

pkg/yunionconf/models/parameters.go

@@ -0,0 +1,214 @@
+package models
+
+import (
+	"context"
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/pkg/util/timeutils"
+	"yunion.io/x/pkg/utils"
+	"yunion.io/x/sqlchemy"
+)
+
+const (
+	NAMESPACE_USER    = "user"
+	NAMESPACE_SERVICE = "service"
+)
+
+type SParameterManager struct {
+	db.SResourceBaseManager
+}
+
+type SParameter struct {
+	db.SResourceBase
+
+	Id          int64  `primary:"true" auto_increment:"true" list:"user"`                                          // = Column(BigInteger, primary_key=True)
+	CreatedBy   string `width:"128" charset:"ascii" nullable:"false" create:"required" list:"user"`                // Column(VARCHAR(length=128, charset='ascii'), nullable=False)
+	UpdatedBy   string `width:"128" charset:"ascii" nullable:"false" create:"required" update:"user" list:"user"`  // Column(VARCHAR(length=128, charset='ascii'), nullable=False)  "user"/ serviceName/ "admin"
+	Namespace   string `width:"64" charset:"ascii" default:"user" nullable:"false" create:"required" list:"admin"` // Column(VARCHAR(length=128, charset='ascii'), nullable=False)  user_id / serviceid
+	NamespaceId string `width:"128" charset:"ascii" nullable:"false" index:"true" create:"required" list:"admin"`  // Column(VARCHAR(length=128, charset='ascii'), nullable=False)
+	Name        string `width:"128" charset:"ascii" nullable:"false" index:"true" create:"required" list:"user"`   // Column(VARCHAR(length=128, charset='ascii'), nullable=false)
+	Value       string `charset:"utf8" create:"required" update:"user" update:"user" list:"user"`                  // Column(VARCHAR(charset='utf-8'))
+}
+
+var ParameterManager *SParameterManager
+
+func init() {
+	ParameterManager = &SParameterManager{SResourceBaseManager: db.NewResourceBaseManager(SParameter{}, "paramters_tbl", "parameter", "parameters")}
+}
+
+func isAdminQuery(query jsonutils.JSONObject) bool {
+	admin_fields := [3]string{"namespace_id", "user_id", "service_id"}
+
+	for _, field := range admin_fields {
+		if s, _ := query.GetString(field); len(s) > 0 {
+			return true
+		}
+	}
+
+	return false
+}
+
+func getNamespaceInContext(userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (namespace string, namespaceId string, err error) {
+	// 优先匹配上线文中的参数, /users/<user_id>/parameters  /services/<service_id>/parameters
+	if uid, _ := query.GetString("user_id"); len(uid) > 0 {
+		return NAMESPACE_USER, uid, nil
+	} else if sid, _ := query.GetString("service_id"); len(sid) > 0 {
+		return NAMESPACE_SERVICE, sid, nil
+	}
+
+	// 匹配/parameters中的参数
+	if uid, _ := data.GetString("user_id"); len(uid) > 0 {
+		return NAMESPACE_USER, uid, nil
+	} else if sid, _ := data.GetString("service_id"); len(sid) > 0 {
+		return NAMESPACE_SERVICE, sid, nil
+	} else {
+		return NAMESPACE_USER, userCred.GetUserId(), nil
+	}
+}
+
+func getNamespace(userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (string, string, error) {
+	var namespace, namespace_id string
+	if userCred.IsSystemAdmin() {
+		if name, nameId, e := getNamespaceInContext(userCred, query, data); e != nil {
+			return "", "", e
+		} else {
+			namespace = name
+			namespace_id = nameId
+		}
+	} else {
+		namespace = NAMESPACE_USER
+		namespace_id = userCred.GetUserId()
+	}
+
+	return namespace, namespace_id, nil
+}
+
+func (manager *SParameterManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+	if !isAdminQuery(query) {
+		return true
+	}
+
+	return userCred.IsSystemAdmin()
+}
+
+func (manager *SParameterManager) AllowCreateItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	if !isAdminQuery(query) {
+		return true
+	}
+
+	return userCred.IsSystemAdmin()
+}
+
+func (manager *SParameterManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	// check duplication
+	name, _ := data.GetString("name")
+	uid := userCred.GetUserId()
+	if len(uid) < 0 {
+		return nil, httperrors.NewUserNotFoundError("user not found")
+	}
+
+	namespace, namespace_id, e := getNamespace(userCred, query, data)
+	if e != nil {
+		return nil, e
+	}
+
+	// check duplication, 同一个namespace下,name不能 重复
+	q := manager.Query().Equals("name", name).Equals("namespace_id", namespace_id)
+	if q.Count() > 0 {
+		return nil, httperrors.NewDuplicateNameError("paramter %s has been created", name)
+	}
+
+	data.Add(jsonutils.NewString(uid), "created_by")
+	data.Add(jsonutils.NewString(uid), "updated_by")
+	data.Add(jsonutils.NewString(namespace), "namespace")
+	data.Add(jsonutils.NewString(namespace_id), "namespace_id")
+	return data, nil
+}
+
+func (manager *SParameterManager) GetOwnerId(userCred mcclient.TokenCredential) string {
+	return userCred.GetUserId()
+}
+
+func (manager *SParameterManager) FilterByOwner(q *sqlchemy.SQuery, owner string) *sqlchemy.SQuery {
+	return q.Equals("created_by", owner)
+}
+
+func (manager *SParameterManager) FilterById(q *sqlchemy.SQuery, idStr string) *sqlchemy.SQuery {
+	return q.Equals("name", idStr)
+}
+
+func (manager *SParameterManager) FilterByName(q *sqlchemy.SQuery, name string) *sqlchemy.SQuery {
+	return q.Equals("name", name)
+}
+
+func (manager *SParameterManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
+	if userCred.IsSystemAdmin() {
+		if id, _ := query.GetString("namespace_id"); len(id) > 0 {
+			q = q.Equals("namespace_id", id)
+		} else if id, _ := query.GetString("service_id"); len(id) > 0 {
+			q = q.Equals("namespace_id", id).Equals("namespace", NAMESPACE_SERVICE)
+		} else if id, _ := query.GetString("user_id"); len(id) > 0 {
+			q = q.Equals("namespace_id", id).Equals("namespace", NAMESPACE_USER)
+		} else {
+			//  not admin
+			admin, _ := query.GetString("admin")
+			if !utils.ToBool(admin) {
+				q = q.Equals("namespace_id", userCred.GetUserId()).Equals("namespace", NAMESPACE_USER)
+			}
+		}
+
+		return q, nil
+	}
+	return q.Equals("namespace_id", userCred.GetUserId()).Equals("namespace", NAMESPACE_USER), nil
+}
+
+func (model *SParameter) IsOwner(userCred mcclient.TokenCredential) bool {
+	return model.CreatedBy == userCred.GetUserId() || (model.NamespaceId == userCred.GetUserId() && model.Namespace == NAMESPACE_USER)
+}
+
+func (model *SParameter) AllowUpdateItem(ctx context.Context, userCred mcclient.TokenCredential) bool {
+	return model.IsOwner(userCred) || userCred.IsSystemAdmin()
+}
+
+func (model *SParameter) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	uid := userCred.GetUserId()
+	if len(uid) < 0 {
+		return nil, httperrors.NewUserNotFoundError("user not found")
+	}
+
+	namespace, namespace_id, e := getNamespace(userCred, query, data)
+	if e != nil {
+		return nil, e
+	}
+	data.Add(jsonutils.NewString(uid), "updated_by")
+	data.Add(jsonutils.NewString(namespace_id), "namespace_id")
+	data.Add(jsonutils.NewString(namespace), "namespace")
+	return data, nil
+}
+
+func (model *SParameter) AllowDeleteItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return model.IsOwner(userCred) || userCred.IsSystemAdmin()
+}
+
+func (model *SParameter) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
+	return model.Delete(ctx, userCred)
+}
+
+func (model *SParameter) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	_, err := model.GetModelManager().TableSpec().Update(model, func() error {
+		model.Deleted = true
+		model.DeletedAt = timeutils.UtcNow()
+		return nil
+	})
+	if err != nil {
+		log.Errorf("PendingDelete fail %s", err)
+	}
+	return err
+}
+
+func (model *SParameter) AllowGetDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+	return model.IsOwner(userCred) || userCred.IsSystemAdmin()
+}

pkg/yunionconf/options/options.go

@@ -0,0 +1,11 @@
+package options
+
+import "yunion.io/x/onecloud/pkg/cloudcommon"
+
+type YunionConfOptions struct {
+	cloudcommon.DBOptions
+}
+
+var (
+	Options YunionConfOptions
+)

pkg/yunionconf/service/service.go

@@ -0,0 +1,40 @@
+package service
+
+import (
+	"os"
+
+	_ "github.com/go-sql-driver/mysql"
+
+	"yunion.io/x/log"
+	"yunion.io/x/onecloud/pkg/cloudcommon"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/yunionconf"
+	"yunion.io/x/onecloud/pkg/yunionconf/models"
+	"yunion.io/x/onecloud/pkg/yunionconf/options"
+)
+
+func StartService() {
+	cloudcommon.ParseOptions(&options.Options, &options.Options.Options, os.Args, "yunionconf.conf")
+	cloudcommon.InitAuth(&options.Options.Options, func() {
+		log.Infof("Auth complete!!")
+	})
+
+	if options.Options.GlobalVirtualResourceNamespace {
+		db.EnableGlobalVirtualResourceNamespace()
+	}
+
+	cloudcommon.InitDB(&options.Options.DBOptions)
+	defer cloudcommon.CloseDB()
+
+	app := cloudcommon.InitApp(&options.Options.Options)
+	yunionconf.InitHandlers(app)
+
+	if db.CheckSync(options.Options.AutoSyncTable) {
+		err := models.InitDB()
+		if err == nil {
+			cloudcommon.ServeForever(app, &options.Options.Options)
+		} else {
+			log.Errorf("InitDB fail: %s", err)
+		}
+	}
+}