feature: keystone add service config API and allow config services by

API

cmd/climc/shell/services.go

@@ -15,10 +15,14 @@
 package shell
 
 import (
+	"fmt"
+	"strings"
+
 	"yunion.io/x/jsonutils"
 
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
 )
 
 func init() {
@@ -144,4 +148,69 @@ func init() {
 		printObject(srv)
 		return nil
 	})
+
+	type ServiceConfigShowOptions struct {
+		SERVICE string `help:"service name or id"`
+	}
+	R(&ServiceConfigShowOptions{}, "service-config-show", "Show configs of a service", func(s *mcclient.ClientSession, args *ServiceConfigShowOptions) error {
+		conf, err := modules.ServicesV3.GetSpecific(s, args.SERVICE, "config", nil)
+		if err != nil {
+			return err
+		}
+		fmt.Println(conf.PrettyString())
+		return nil
+	})
+
+	type ServiceConfigOptions struct {
+		SERVICE string   `help:"service name or id"`
+		Config  []string `help:"config key=value pair"`
+		Remove  bool     `help:"remove config"`
+	}
+	R(&ServiceConfigOptions{}, "service-config", "Add config to service", func(s *mcclient.ClientSession, args *ServiceConfigOptions) error {
+		config := jsonutils.NewDict()
+		if args.Remove {
+			config.Add(jsonutils.NewString("remove"), "action")
+		} else {
+			config.Add(jsonutils.NewString("update"), "action")
+		}
+		for _, c := range args.Config {
+			pos := strings.IndexByte(c, '=')
+			if pos < 0 {
+				return fmt.Errorf("%s is not a key=value pair", c)
+			}
+			key := strings.TrimSpace(c[:pos])
+			value := strings.TrimSpace(c[pos+1:])
+			config.Add(jsonutils.NewString(value), "config", "default", key)
+		}
+		nconf, err := modules.ServicesV3.PerformAction(s, args.SERVICE, "config", config)
+		if err != nil {
+			return err
+		}
+		fmt.Println(nconf.PrettyString())
+		return nil
+	})
+
+	type ServiceConfigYamlOptions struct {
+		SERVICE string `help:"service name or id"`
+		YAML    string `help:"config yaml file"`
+	}
+	R(&ServiceConfigYamlOptions{}, "service-config-yaml", "Config service with a yaml file", func(s *mcclient.ClientSession, args *ServiceConfigYamlOptions) error {
+		content, err := fileutils2.FileGetContents(args.YAML)
+		if err != nil {
+			return err
+		}
+		yamlJson, err := jsonutils.ParseYAML(content)
+		if err != nil {
+			return err
+		}
+		config := jsonutils.NewDict()
+		config.Add(yamlJson, "config", "default")
+		nconf, err := modules.ServicesV3.PerformAction(s, args.SERVICE, "config", config)
+		if err != nil {
+			return err
+		}
+		fmt.Println(nconf.PrettyString())
+		return nil
+	})
+
 }

pkg/apis/compute/consts.go

@@ -0,0 +1,20 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compute
+
+const (
+	SERVICE_TYPE    = "compute"
+	SERVICE_VERSION = "v2"
+)

pkg/apis/compute/reservedip.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package compute
 
 const (

pkg/apis/identity/config.go

@@ -21,7 +21,7 @@ const (
 	QUeryScopeSub = "sub"
 )
 
-type TIdentityProviderConfigs map[string]map[string]jsonutils.JSONObject
+type TConfigs map[string]map[string]jsonutils.JSONObject
 
 type SLDAPIdpConfigBaseOptions struct {
 	Url      string `json:"url,omitempty" help:"LDAP server URL" required:"true"`

pkg/apis/identity/consts.go

@@ -83,7 +83,36 @@ var (
 		IdentityDriverLDAP,
 	}
 
-	SensitiveDomainConfigMap = map[string]string{
-		"ldap": "password",
+	SensitiveDomainConfigMap = map[string][]string{
+		"ldap": []string{
+			"password",
+		},
+	}
+
+	BlacklistOptionMap = map[string][]string{
+		"default": []string{
+			"region",
+			"sql_connection",
+			"config",
+			"application_id",
+			"log_level",
+			"temp_path",
+			"auto_sync_table",
+			"address",
+			"port",
+			"admin_port",
+			"notify_admin_users",
+			"session_endpoint_type",
+			"admin_password",
+			"admin_project",
+			"admin_user",
+			"auth_url",
+			"default_aws_instance_type_file",
+			"port_v2",
+			"enable_ssl",
+			"ssl_certfile",
+			"ssl_keyfile",
+			"ssl_ca_certs",
+		},
 	}
 )

pkg/apis/identity/resource.go

@@ -0,0 +1,21 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package identity
+
+const (
+	IDENTITY_PROVIDER_TABLE          = "identity_provider"
+	IDENTITY_PROVIDER_RESOURCE_TYPE  = "identity_provider"
+	IDENTITY_PROVIDER_RESOURCE_TYPES = "identity_providers"
+)

pkg/apis/image/consts.go

@@ -17,7 +17,8 @@ package image
 type TImageType string
 
 const (
-	SERVICE_TYPE = "image"
+	SERVICE_TYPE    = "image"
+	SERVICE_VERSION = ""
 
 	// https://docs.openstack.org/glance/pike/user/statuses.html
 	//

pkg/cloudcommon/app/mergeconf.go

@@ -0,0 +1,97 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package app
+
+import (
+	"context"
+	"database/sql"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func getServiceIdByType(s *mcclient.ClientSession, typeStr string, verStr string) (string, error) {
+	params := jsonutils.NewDict()
+	if len(verStr) > 0 {
+		typeStr += "_" + verStr
+	}
+	params.Add(jsonutils.NewString(typeStr), "type")
+	result, err := modules.ServicesV3.List(s, params)
+	if err != nil {
+		return "", errors.Wrap(err, "modules.ServicesV3.List")
+	}
+	if len(result.Data) == 0 {
+		return "", errors.Wrap(sql.ErrNoRows, "modules.ServicesV3.List")
+	} else if len(result.Data) > 1 {
+		return "", errors.Wrap(sqlchemy.ErrDuplicateEntry, "modules.ServicesV3.List")
+	}
+	return result.Data[0].GetString("id")
+}
+
+func getServiceConfig(s *mcclient.ClientSession, serviceId string) (jsonutils.JSONObject, error) {
+	conf, err := modules.ServicesV3.GetSpecific(s, serviceId, "config", nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "modules.ServicesV3.GetSpecific config")
+	}
+	defConf, _ := conf.Get("config", "default")
+	return defConf, nil
+}
+
+func MergeServiceConfig(opts interface{}, serviceType string, serviceVersion string) error {
+	merged := false
+	conf := jsonutils.Marshal(opts).(*jsonutils.JSONDict)
+	region, _ := conf.GetString("region")
+	epType, _ := conf.GetString("session_endpoint_type")
+	s := auth.AdminSession(context.Background(), region, "", epType, "")
+	serviceId, _ := getServiceIdByType(s, serviceType, serviceVersion)
+	if len(serviceId) > 0 {
+		serviceConf, err := getServiceConfig(s, serviceId)
+		if err != nil {
+			return errors.Wrap(err, "getServiceConfig")
+		}
+		conf.Update(serviceConf)
+		merged = true
+	}
+	commonServiceId, _ := getServiceIdByType(s, consts.COMMON_SERVICE, "")
+	if len(commonServiceId) > 0 {
+		commonConf, err := getServiceConfig(s, commonServiceId)
+		if err != nil {
+			return errors.Wrap(err, "getServiceConfig common service")
+		}
+		conf.Update(commonConf)
+		merged = true
+	}
+	if merged {
+		err := conf.Unmarshal(opts)
+		if err != nil {
+			return errors.Wrap(err, "conf.Unmarshal")
+		}
+		if len(serviceId) > 0 {
+			nconf := jsonutils.NewDict()
+			nconf.Add(conf, "config", "default")
+			_, err := modules.ServicesV3.PerformAction(s, serviceId, "config", nconf)
+			if err != nil {
+				return errors.Wrap(err, "modules.ServicesV3.PerformAction")
+			}
+		}
+	}
+	return nil
+}

pkg/cloudcommon/consts/consts.go

@@ -19,6 +19,8 @@ import (
 )
 
 var (
+	COMMON_SERVICE = "common"
+
 	globalRegion = ""
 
 	globalServiceType = ""

pkg/compute/models/skus_tools.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package models
 
 import (

pkg/compute/service/service.go

@@ -22,6 +22,7 @@ import (
 
 	"yunion.io/x/log"
 
+	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon"
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
 	"yunion.io/x/onecloud/pkg/cloudcommon/cronman"
@@ -43,7 +44,7 @@ func StartService() {
 	commonOpts := &options.Options.CommonOptions
 	baseOpts := &options.Options.BaseOptions
 	dbOpts := &options.Options.DBOptions
-	common_options.ParseOptions(opts, os.Args, "region.conf", "compute")
+	common_options.ParseOptions(opts, os.Args, "region.conf", api.SERVICE_TYPE)
 
 	if opts.PortV2 > 0 {
 		log.Infof("Port V2 %d is specified, use v2 port", opts.PortV2)
@@ -62,7 +63,12 @@ func StartService() {
 	db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB)
 	defer cloudcommon.CloseDB()
 
-	err := setInfluxdbRetentionPolicy()
+	err := app_common.MergeServiceConfig(opts, api.SERVICE_TYPE, api.SERVICE_VERSION)
+	if err != nil {
+		log.Fatalf("Fail to merge service config %s", err)
+	}
+
+	err = setInfluxdbRetentionPolicy()
 	if err != nil {
 		log.Errorf("setInfluxdbRetentionPolicy fail: %s", err)
 	}

pkg/image/service/service.go

@@ -93,6 +93,11 @@ func StartService() {
 
 	db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB)
 
+	err := app_common.MergeServiceConfig(opts, api.SERVICE_TYPE, api.SERVICE_VERSION)
+	if err != nil {
+		log.Fatalf("Fail to merge service config %s", err)
+	}
+
 	go models.CheckImages()
 
 	if len(options.Options.DeployServerSocketPath) > 0 {

pkg/keystone/driver/base.go

@@ -34,7 +34,7 @@ func GetDriverClass(drv string) IIdentityBackendClass {
 	return nil
 }
 
-func GetDriver(driver string, idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TIdentityProviderConfigs) (IIdentityBackend, error) {
+func GetDriver(driver string, idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TConfigs) (IIdentityBackend, error) {
 	drvCls := GetDriverClass(driver)
 	if drvCls == nil {
 		return nil, ErrNoSuchDriver
@@ -45,7 +45,7 @@ func GetDriver(driver string, idpId, idpName, template, targetDomainId string, a
 type SBaseIdentityDriver struct {
 	object.SObject
 
-	Config   api.TIdentityProviderConfigs
+	Config   api.TConfigs
 	IdpId    string
 	IdpName  string
 	Template string
@@ -58,7 +58,7 @@ func (base *SBaseIdentityDriver) IIdentityBackend() IIdentityBackend {
 	return base.GetVirtualObject().(IIdentityBackend)
 }
 
-func NewBaseIdentityDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TIdentityProviderConfigs) (SBaseIdentityDriver, error) {
+func NewBaseIdentityDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TConfigs) (SBaseIdentityDriver, error) {
 	drv := SBaseIdentityDriver{}
 	drv.IdpId = idpId
 	drv.IdpName = idpName

pkg/keystone/driver/cas/cas.go

@@ -40,7 +40,7 @@ type SCASDriver struct {
 	isDebug bool
 }
 
-func NewCASDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TIdentityProviderConfigs) (driver.IIdentityBackend, error) {
+func NewCASDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TConfigs) (driver.IIdentityBackend, error) {
 	base, err := driver.NewBaseIdentityDriver(idpId, idpName, template, targetDomainId, autoCreateProject, conf)
 	if err != nil {
 		return nil, errors.Wrap(err, "NewBaseIdentityDriver")

pkg/keystone/driver/cas/class.go

@@ -29,7 +29,7 @@ func (self *SCASDriverClass) SyncMethod() string {
 	return api.IdentityProviderSyncOnAuth
 }
 
-func (self *SCASDriverClass) NewDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TIdentityProviderConfigs) (driver.IIdentityBackend, error) {
+func (self *SCASDriverClass) NewDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TConfigs) (driver.IIdentityBackend, error) {
 	return NewCASDriver(idpId, idpName, template, targetDomainId, autoCreateProject, conf)
 }
 

pkg/keystone/driver/driver.go

@@ -25,7 +25,7 @@ type IIdentityBackendClass interface {
 	SingletonInstance() bool
 	SyncMethod() string
 	Name() string
-	NewDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TIdentityProviderConfigs) (IIdentityBackend, error)
+	NewDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TConfigs) (IIdentityBackend, error)
 }
 
 type IIdentityBackend interface {

pkg/keystone/driver/ldap/class.go

@@ -29,7 +29,7 @@ func (self *SLDAPDriverClass) SyncMethod() string {
 	return api.IdentityProviderSyncFull
 }
 
-func (self *SLDAPDriverClass) NewDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TIdentityProviderConfigs) (driver.IIdentityBackend, error) {
+func (self *SLDAPDriverClass) NewDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TConfigs) (driver.IIdentityBackend, error) {
 	return NewLDAPDriver(idpId, idpName, template, targetDomainId, autoCreateProject, conf)
 }
 

pkg/keystone/driver/ldap/ldap.go

@@ -39,7 +39,7 @@ type SLDAPDriver struct {
 	ldapConfig *api.SLDAPIdpConfigOptions
 }
 
-func NewLDAPDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TIdentityProviderConfigs) (driver.IIdentityBackend, error) {
+func NewLDAPDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TConfigs) (driver.IIdentityBackend, error) {
 	base, err := driver.NewBaseIdentityDriver(idpId, idpName, template, targetDomainId, autoCreateProject, conf)
 	if err != nil {
 		return nil, errors.Wrap(err, "NewBaseIdentityDriver")

pkg/keystone/driver/sql/class.go

@@ -29,7 +29,7 @@ func (self *SSQLDriverClass) SyncMethod() string {
 	return api.IdentityProviderSyncLocal
 }
 
-func (self *SSQLDriverClass) NewDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TIdentityProviderConfigs) (driver.IIdentityBackend, error) {
+func (self *SSQLDriverClass) NewDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TConfigs) (driver.IIdentityBackend, error) {
 	return NewSQLDriver(idpId, idpName, template, targetDomainId, autoCreateProject, conf)
 }
 

pkg/keystone/driver/sql/sql.go

@@ -30,7 +30,7 @@ type SSQLDriver struct {
 	driver.SBaseIdentityDriver
 }
 
-func NewSQLDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TIdentityProviderConfigs) (driver.IIdentityBackend, error) {
+func NewSQLDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TConfigs) (driver.IIdentityBackend, error) {
 	base, err := driver.NewBaseIdentityDriver(idpId, idpName, template, targetDomainId, autoCreateProject, conf)
 	if err != nil {
 		return nil, err

pkg/keystone/models/configs.go

@@ -15,21 +15,22 @@
 package models
 
 import (
-	"context"
 	"database/sql"
 	"sort"
 
-	"github.com/pkg/errors"
-
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/utils"
 
 	api "yunion.io/x/onecloud/pkg/apis/identity"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
-	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/keystone/options"
 )
 
 type SConfigOptionManager struct {
 	db.SResourceBaseManager
+	IsSensitive bool
 }
 
 var (
@@ -45,6 +46,7 @@ func init() {
 			"sensitive_config",
 			"sensitive_configs",
 		),
+		IsSensitive: true,
 	}
 	SensitiveConfigManager.SetVirtualObject(SensitiveConfigManager)
 	WhitelistedConfigManager = &SConfigOptionManager{
@@ -54,6 +56,7 @@ func init() {
 			"whitelisted_config",
 			"whitelisted_configs",
 		),
+		IsSensitive: false,
 	}
 	WhitelistedConfigManager.SetVirtualObject(WhitelistedConfigManager)
 }
@@ -72,20 +75,29 @@ func init() {
 type SConfigOption struct {
 	db.SResourceBase
 
-	IdpId  string `name:"domain_id" width:"64" charset:"ascii" primary:"true"`
-	Group  string `width:"255" charset:"utf8" primary:"true"`
-	Option string `width:"255" charset:"utf8" primary:"true"`
+	ResType string `width:"32" charset:"ascii" nullable:"false" default:"identity_provider" primary:"true"`
+	ResId   string `name:"domain_id" width:"64" charset:"ascii" primary:"true"`
+	Group   string `width:"255" charset:"utf8" primary:"true"`
+	Option  string `width:"255" charset:"utf8" primary:"true"`
 
 	Value jsonutils.JSONObject `nullable:"false"`
 }
 
-func (manager *SConfigOptionManager) fetchConfigs(idpId string, groups []string, options []string) (TConfigOptions, error) {
-	q := manager.Query().Equals("domain_id", idpId)
+func (manager *SConfigOptionManager) fetchConfigs(model db.IModel, groups []string, options []string) (TConfigOptions, error) {
+	q := manager.Query().Equals("res_type", model.Keyword()).Equals("domain_id", model.GetId())
 	if len(groups) > 0 {
-		q = q.In("group", groups)
+		if len(groups) == 1 {
+			q = q.Equals("group", groups[0])
+		} else {
+			q = q.In("group", groups)
+		}
 	}
 	if len(options) > 0 {
-		q = q.In("option", options)
+		if len(options) == 1 {
+			q = q.Equals("option", options[0])
+		} else {
+			q = q.In("option", options)
+		}
 	}
 	opts := make(TConfigOptions, 0)
 	err := db.FetchModelObjects(manager, q, &opts)
@@ -96,8 +108,8 @@ func (manager *SConfigOptionManager) fetchConfigs(idpId string, groups []string,
 	return opts, nil
 }
 
-func config2map(opts []SConfigOption) api.TIdentityProviderConfigs {
-	conf := make(api.TIdentityProviderConfigs)
+func config2map(opts []SConfigOption) api.TConfigs {
+	conf := make(api.TConfigs)
 	for i := range opts {
 		opt := opts[i]
 		if _, ok := conf[opt.Group]; !ok {
@@ -108,12 +120,39 @@ func config2map(opts []SConfigOption) api.TIdentityProviderConfigs {
 	return conf
 }
 
-func (manager *SConfigOptionManager) deleteConfig(ctx context.Context, userCred mcclient.TokenCredential, idpId string) error {
-	return manager.syncConfig(ctx, userCred, idpId, nil)
+func (manager *SConfigOptionManager) deleteConfigs(model db.IModel) error {
+	return manager.syncConfigs(model, nil)
 }
 
-func (manager *SConfigOptionManager) syncConfig(ctx context.Context, userCred mcclient.TokenCredential, idpId string, newOpts TConfigOptions) error {
-	oldOpts, err := manager.fetchConfigs(idpId, nil, nil)
+func (manager *SConfigOptionManager) updateConfigs(newOpts TConfigOptions) error {
+	for i := range newOpts {
+		err := manager.TableSpec().InsertOrUpdate(&newOpts[i])
+		if err != nil {
+			return errors.Wrap(err, "Insert")
+		}
+	}
+	return nil
+}
+
+func (manager *SConfigOptionManager) removeConfigs(model db.IModel, newOpts TConfigOptions) error {
+	oldOpts, err := manager.fetchConfigs(model, nil, nil)
+	if err != nil {
+		return errors.Wrap(err, "fetchOldConfigs")
+	}
+	_, updated1, _, _ := compareConfigOptions(oldOpts, newOpts)
+	for i := range updated1 {
+		_, err := db.Update(&updated1[i], func() error {
+			return updated1[i].MarkDelete()
+		})
+		if err != nil {
+			return errors.Wrap(err, "Delete")
+		}
+	}
+	return nil
+}
+
+func (manager *SConfigOptionManager) syncConfigs(model db.IModel, newOpts TConfigOptions) error {
+	oldOpts, err := manager.fetchConfigs(model, nil, nil)
 	if err != nil {
 		return errors.Wrap(err, "fetchOldConfigs")
 	}
@@ -144,17 +183,20 @@ func (manager *SConfigOptionManager) syncConfig(ctx context.Context, userCred mc
 	return nil
 }
 
-func getConfigOptions(conf api.TIdentityProviderConfigs, idpId string, sensitiveList map[string]string) (TConfigOptions, TConfigOptions) {
+func getConfigOptions(conf api.TConfigs, model db.IModel, blackList map[string][]string, sensitiveList map[string][]string) (TConfigOptions, TConfigOptions) {
 	options := make(TConfigOptions, 0)
 	sensitive := make(TConfigOptions, 0)
 	for group, groupConf := range conf {
 		for optKey, optVal := range groupConf {
 			opt := SConfigOption{}
-			opt.IdpId = idpId
+			opt.ResType = model.Keyword()
+			opt.ResId = model.GetId()
 			opt.Group = group
 			opt.Option = optKey
 			opt.Value = optVal
-			if v, ok := sensitiveList[group]; ok && v == optKey {
+			if v, ok := blackList[group]; ok && utils.IsInStringArray(optKey, v) {
+				// skip
+			} else if v, ok := sensitiveList[group]; ok && utils.IsInStringArray(optKey, v) {
 				sensitive = append(sensitive, opt)
 			} else {
 				options = append(options, opt)
@@ -227,8 +269,10 @@ func compareConfigOptions(opts1, opts2 TConfigOptions) (deleted, updated1, updat
 	return
 }
 
-func (manager *SConfigOptionManager) getDriver(idStr string) (string, error) {
-	opts, err := manager.fetchConfigs(idStr, []string{"identity"}, []string{"driver"})
+func (manager *SConfigOptionManager) getDriver(idpId string) (string, error) {
+	idp, _ := db.NewModelObject(IdentityProviderManager)
+	idp.(*SIdentityProvider).Id = idpId
+	opts, err := manager.fetchConfigs(idp, []string{"identity"}, []string{"driver"})
 	if err != nil {
 		return "", errors.Wrap(err, "WhitelistedConfigManager.fetchConfigs")
 	}
@@ -237,3 +281,96 @@ func (manager *SConfigOptionManager) getDriver(idStr string) (string, error) {
 	}
 	return api.IdentityDriverSQL, nil
 }
+
+func GetConfigs(model db.IModel, all bool) (api.TConfigs, error) {
+	opts, err := WhitelistedConfigManager.fetchConfigs(model, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+	if all {
+		opts2, err := SensitiveConfigManager.fetchConfigs(model, nil, nil)
+		if err != nil {
+			return nil, err
+		}
+		opts = append(opts, opts2...)
+	}
+	return config2map(opts), nil
+}
+
+func saveConfigs(action string, model db.IModel, opts api.TConfigs, blackList map[string][]string, sensitiveConfs map[string][]string) error {
+	whiteListedOpts, sensitiveOpts := getConfigOptions(opts, model, blackList, sensitiveConfs)
+	if action == "update" {
+		err := WhitelistedConfigManager.updateConfigs(whiteListedOpts)
+		if err != nil {
+			return errors.Wrap(err, "WhitelistedConfigManager.updateConfig")
+		}
+		err = SensitiveConfigManager.updateConfigs(sensitiveOpts)
+		if err != nil {
+			return errors.Wrap(err, "SensitiveConfigManager.updateConfig")
+		}
+	} else if action == "remove" {
+		err := WhitelistedConfigManager.removeConfigs(model, whiteListedOpts)
+		if err != nil {
+			return errors.Wrap(err, "WhitelistedConfigManager.updateConfig")
+		}
+		err = SensitiveConfigManager.removeConfigs(model, sensitiveOpts)
+		if err != nil {
+			return errors.Wrap(err, "SensitiveConfigManager.updateConfig")
+		}
+	} else {
+		err := WhitelistedConfigManager.syncConfigs(model, whiteListedOpts)
+		if err != nil {
+			return errors.Wrap(err, "WhitelistedConfigManager.syncConfig")
+		}
+		err = SensitiveConfigManager.syncConfigs(model, sensitiveOpts)
+		if err != nil {
+			return errors.Wrap(err, "SensitiveConfigManager.syncConfig")
+		}
+	}
+	return nil
+}
+
+func MergeServiceConfig(opts *options.SKeystoneOptions) error {
+	merged := false
+	conf := jsonutils.Marshal(opts).(*jsonutils.JSONDict)
+	service, _ := ServiceManager.fetchServiceByType(api.SERVICE_TYPE)
+	if service != nil {
+		serviceConf, err := GetConfigs(service, false)
+		if err != nil {
+			return errors.Wrap(err, "GetConfigs service")
+		}
+		serviceConfJson := jsonutils.Marshal(serviceConf["default"])
+		conf.Update(serviceConfJson)
+		merged = true
+	}
+	commonService, _ := ServiceManager.fetchServiceByType(consts.COMMON_SERVICE)
+	if commonService != nil {
+		commonConf, err := GetConfigs(commonService, false)
+		if err != nil {
+			return errors.Wrap(err, "GetConfigs commonService")
+		}
+		commonConfJson := jsonutils.Marshal(commonConf["default"])
+		conf.Update(commonConfJson)
+		merged = true
+	}
+	if merged {
+		err := conf.Unmarshal(opts)
+		if err != nil {
+			return errors.Wrap(err, "conf.Unmarshal")
+		}
+		if service != nil {
+			nconf := jsonutils.NewDict()
+			nconf.Add(conf, "default")
+			tconf := api.TConfigs{}
+			err = nconf.Unmarshal(tconf)
+			if err != nil {
+				return errors.Wrap(err, "conf.Unmarshal(tconf)")
+			}
+			err = saveConfigs("", service, tconf, api.BlacklistOptionMap, nil)
+			if err != nil {
+				return errors.Wrap(err, "saveConfigs")
+			}
+		}
+	}
+	return nil
+}

pkg/keystone/models/identity_provider.go

@@ -50,9 +50,9 @@ func init() {
 	IdentityProviderManager = &SIdentityProviderManager{
 		SEnabledStatusStandaloneResourceBaseManager: db.NewEnabledStatusStandaloneResourceBaseManager(
 			SIdentityProvider{},
-			"identity_provider",
-			"identity_provider",
-			"identity_providers",
+			api.IDENTITY_PROVIDER_TABLE,
+			api.IDENTITY_PROVIDER_RESOURCE_TYPE,
+			api.IDENTITY_PROVIDER_RESOURCE_TYPES,
 		),
 	}
 	IdentityProviderManager.SetVirtualObject(IdentityProviderManager)
@@ -201,7 +201,7 @@ func (self *SIdentityProvider) AllowGetDetailsConfig(ctx context.Context, userCr
 }
 
 func (self *SIdentityProvider) GetDetailsConfig(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	conf, err := self.GetConfig(false)
+	conf, err := GetConfigs(self, false)
 	if err != nil {
 		return nil, err
 	}
@@ -210,21 +210,6 @@ func (self *SIdentityProvider) GetDetailsConfig(ctx context.Context, userCred mc
 	return result, nil
 }
 
-func (self *SIdentityProvider) GetConfig(all bool) (api.TIdentityProviderConfigs, error) {
-	opts, err := WhitelistedConfigManager.fetchConfigs(self.Id, nil, nil)
-	if err != nil {
-		return nil, err
-	}
-	if all {
-		opts2, err := SensitiveConfigManager.fetchConfigs(self.Id, nil, nil)
-		if err != nil {
-			return nil, err
-		}
-		opts = append(opts, opts2...)
-	}
-	return config2map(opts), nil
-}
-
 func (ident *SIdentityProvider) AllowPerformConfig(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) bool {
 	return db.IsAdminAllowUpdateSpec(userCred, ident, "config")
 }
@@ -236,13 +221,13 @@ func (ident *SIdentityProvider) PerformConfig(ctx context.Context, userCred mccl
 	if ident.SyncStatus != api.IdentitySyncStatusIdle {
 		return nil, httperrors.NewInvalidStatusError("cannot update config when not idle")
 	}
-	opts := api.TIdentityProviderConfigs{}
+	opts := api.TConfigs{}
 	err := data.Unmarshal(&opts, "config")
 	if err != nil {
 		return nil, httperrors.NewInputParameterError("invalid input data")
 	}
-
-	err = ident.saveConfig(ctx, userCred, opts)
+	action, _ := data.GetString("action")
+	err = saveConfigs(action, ident, opts, nil, api.SensitiveDomainConfigMap)
 	if err != nil {
 		return nil, httperrors.NewInternalServerError("saveConfig fail %s", err)
 	}
@@ -251,19 +236,6 @@ func (ident *SIdentityProvider) PerformConfig(ctx context.Context, userCred mccl
 	return ident.GetDetailsConfig(ctx, userCred, query)
 }
 
-func (ident *SIdentityProvider) saveConfig(ctx context.Context, userCred mcclient.TokenCredential, opts api.TIdentityProviderConfigs) error {
-	whiteListedOpts, sensitiveOpts := getConfigOptions(opts, ident.Id, api.SensitiveDomainConfigMap)
-	err := WhitelistedConfigManager.syncConfig(ctx, userCred, ident.Id, whiteListedOpts)
-	if err != nil {
-		return errors.Wrap(err, "WhitelistedConfigManager.syncConfig")
-	}
-	err = SensitiveConfigManager.syncConfig(ctx, userCred, ident.Id, sensitiveOpts)
-	if err != nil {
-		return errors.Wrap(err, "SensitiveConfigManager.syncConfig")
-	}
-	return nil
-}
-
 func (manager *SIdentityProviderManager) getDriveInstanceCount(drvName string) (int, error) {
 	return manager.Query().Equals("driver", drvName).CountWithError()
 }
@@ -320,7 +292,7 @@ func (manager *SIdentityProviderManager) ValidateCreateData(ctx context.Context,
 		data.Set("target_domain_id", jsonutils.NewString(domain.Id))
 	}
 
-	opts := api.TIdentityProviderConfigs{}
+	opts := api.TConfigs{}
 	err := data.Unmarshal(&opts, "config")
 	if err != nil {
 		return nil, httperrors.NewInputParameterError("parse config error: %s", err)
@@ -338,13 +310,13 @@ func (ident *SIdentityProvider) PostCreate(ctx context.Context, userCred mcclien
 
 	logclient.AddActionLogWithContext(ctx, ident, logclient.ACT_CREATE, data, userCred, true)
 
-	opts := api.TIdentityProviderConfigs{}
+	opts := api.TConfigs{}
 	err := data.Unmarshal(&opts, "config")
 	if err != nil {
 		log.Errorf("parse config error %s", err)
 		return
 	}
-	err = ident.saveConfig(ctx, userCred, opts)
+	err = saveConfigs("", ident, opts, nil, api.SensitiveDomainConfigMap)
 	if err != nil {
 		log.Errorf("saveConfig fail %s", err)
 		return
@@ -611,12 +583,12 @@ func (self *SIdentityProvider) getDomains() ([]SDomain, error) {
 	return domains, nil
 }
 
-func (ident *SIdentityProvider) deleteConfig(ctx context.Context, userCred mcclient.TokenCredential) error {
-	err := WhitelistedConfigManager.deleteConfig(ctx, userCred, ident.Id)
+func (ident *SIdentityProvider) deleteConfigs(ctx context.Context, userCred mcclient.TokenCredential) error {
+	err := WhitelistedConfigManager.deleteConfigs(ident)
 	if err != nil {
 		return errors.Wrap(err, "WhitelistedConfigManager.deleteConfig")
 	}
-	err = SensitiveConfigManager.deleteConfig(ctx, userCred, ident.Id)
+	err = SensitiveConfigManager.deleteConfigs(ident)
 	if err != nil {
 		return errors.Wrap(err, "SensitiveConfigManager.deleteConfig")
 	}
@@ -642,7 +614,7 @@ func (self *SIdentityProvider) Purge(ctx context.Context, userCred mcclient.Toke
 			return errors.Wrap(err, "delete domain")
 		}
 	}
-	err = self.deleteConfig(ctx, userCred)
+	err = self.deleteConfigs(ctx, userCred)
 	if err != nil {
 		return errors.Wrap(err, "self.deleteConfig")
 	}

pkg/keystone/models/passwords.go

@@ -140,8 +140,8 @@ func (manager *SPasswordManager) savePassword(localUserId int, password string,
 	rec.Password = shaPassword(password)
 	now := time.Now()
 	rec.CreatedAtInt = now.UnixNano() / 1000
-	if o.Options.PasswordExpirationDays > 0 && !isSystemAccount {
-		rec.ExpiresAt = now.Add(24 * time.Hour * time.Duration(o.Options.PasswordExpirationDays))
+	if o.Options.PasswordExpirationSeconds > 0 && !isSystemAccount {
+		rec.ExpiresAt = now.Add(time.Second * time.Duration(o.Options.PasswordExpirationSeconds))
 		rec.ExpiresAtInt = rec.ExpiresAt.UnixNano() / 1000
 	}
 	err = manager.TableSpec().Insert(&rec)

pkg/keystone/models/services.go

@@ -16,10 +16,14 @@ package models
 
 import (
 	"context"
+	"database/sql"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/tristate"
+	"yunion.io/x/sqlchemy"
 
+	api "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -136,3 +140,57 @@ func (service *SService) PostDelete(ctx context.Context, userCred mcclient.Token
 	service.SStandaloneResourceBase.PostDelete(ctx, userCred)
 	logclient.AddActionLogWithContext(ctx, service, logclient.ACT_DELETE, nil, userCred, true)
 }
+
+func (service *SService) AllowGetDetailsConfig(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+	return db.IsAdminAllowGetSpec(userCred, service, "config")
+}
+
+func (service *SService) GetDetailsConfig(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	conf, err := GetConfigs(service, false)
+	if err != nil {
+		return nil, err
+	}
+	result := jsonutils.NewDict()
+	result.Add(jsonutils.Marshal(conf), "config")
+	return result, nil
+}
+
+func (service *SService) AllowPerformConfig(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) bool {
+	return db.IsAdminAllowUpdateSpec(userCred, service, "config")
+}
+
+func (service *SService) PerformConfig(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (jsonutils.JSONObject, error) {
+	action, _ := data.GetString("action")
+	opts := api.TConfigs{}
+	err := data.Unmarshal(&opts, "config")
+	if err != nil {
+		return nil, httperrors.NewInputParameterError("invalid input data")
+	}
+	err = saveConfigs(action, service, opts, api.BlacklistOptionMap, nil)
+	if err != nil {
+		return nil, httperrors.NewInternalServerError("saveConfig fail %s", err)
+	}
+	return service.GetDetailsConfig(ctx, userCred, query)
+}
+
+func (manager *SServiceManager) fetchServiceByType(typeStr string) (*SService, error) {
+	q := manager.Query().Equals("type", typeStr)
+	cnt, err := q.CountWithError()
+	if err != nil && errors.Cause(err) != sql.ErrNoRows {
+		return nil, errors.Wrap(err, "CountWithError")
+	}
+	if cnt == 0 {
+		return nil, sql.ErrNoRows
+	} else if cnt > 1 {
+		return nil, sqlchemy.ErrDuplicateEntry
+	}
+	srvObj, err := db.NewModelObject(manager)
+	if err != nil {
+		return nil, errors.Wrap(err, "db.NewModelObject")
+	}
+	err = q.First(srvObj)
+	if err != nil {
+		return nil, errors.Wrap(err, "q.First")
+	}
+	return srvObj.(*SService), nil
+}

pkg/keystone/models/syncworker.go

@@ -44,7 +44,7 @@ func submitIdpSyncTask(ctx context.Context, userCred mcclient.TokenCredential, i
 		idp.SetSyncStatus(ctx, userCred, api.IdentitySyncStatusSyncing)
 		defer idp.SetSyncStatus(ctx, userCred, api.IdentitySyncStatusIdle)
 
-		conf, err := idp.GetConfig(true)
+		conf, err := GetConfigs(idp, true)
 		if err != nil {
 			log.Errorf("GetConfig for idp %s fail %s", idp.Name, err)
 			idp.MarkDisconnected(ctx, userCred)

pkg/keystone/options/options.go

@@ -38,8 +38,8 @@ type SKeystoneOptions struct {
 
 	FetchProjectResourceCountIntervalSeconds int `help:"frequency tp fetch project resource counts" default:"900"`
 
-	PasswordExpirationDays     int `help:"password expires after the duration"`
-	PasswordMinimalLength      int `help:"password minimal length"`
+	PasswordExpirationSeconds  int `help:"password expires after the duration in seconds"`
+	PasswordMinimalLength      int `help:"password minimal length" default:"6"`
 	PasswordUniqueHistoryCheck int `help:"password must be unique in last N passwords"`
 
 	PasswordErrorLockCount int `help:"lock user account if given number of failed auth"`

pkg/keystone/service/service.go

@@ -21,6 +21,8 @@ import (
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/golang-plus/uuid"
 
+	"yunion.io/x/log"
+
 	api "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/cloudcommon"
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
@@ -74,6 +76,11 @@ func StartService() {
 
 	app_common.InitBaseAuth(&opts.BaseOptions)
 
+	err := models.MergeServiceConfig(opts)
+	if err != nil {
+		log.Fatalf("Fail to merge service config: %s", err)
+	}
+
 	if !opts.IsSlaveNode {
 		cron := cronman.InitCronJobManager(true, opts.CronJobWorkerCount)
 

pkg/keystone/tokens/auth.go

@@ -150,7 +150,7 @@ func authUserByIdentity(ctx context.Context, ident mcclient.SAuthenticationIdent
 		return nil, errors.Error(fmt.Sprintf("invalid idp status %s", idp.Status))
 	}
 
-	conf, err := idp.GetConfig(true)
+	conf, err := models.GetConfigs(idp, true)
 	if err != nil {
 		return nil, errors.Wrap(err, "GetConfig")
 	}
@@ -184,7 +184,7 @@ func authUserByCASV3(ctx context.Context, input mcclient.SAuthenticationInputV3)
 		return nil, errors.Error("more than 1 cas identity providers?")
 	}
 	idp := &idps[0]
-	conf, err := idp.GetConfig(true)
+	conf, err := models.GetConfigs(idp, true)
 	if err != nil {
 		return nil, errors.Wrap(err, "idp.GetConfig")
 	}

pkg/mcclient/modules/k8s/rbac.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package k8s
 
 import (

pkg/mcclient/options/k8s/job.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package k8s
 
 import (

pkg/mcclient/options/k8s/pod_template.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package k8s
 
 import (

pkg/mcclient/options/k8s/statefulset.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package k8s
 
 import (