Merge branch 'master' into hotfix/rex-host-arm64-alpine-base-image

cmd/climc/shell/monitor/commonalert.go

@@ -1,47 +1,16 @@
 package monitor
 
 import (
-	"yunion.io/x/jsonutils"
-
-	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
-	"yunion.io/x/onecloud/pkg/mcclient/options"
+	options "yunion.io/x/onecloud/pkg/mcclient/options/monitor"
 )
 
 func init() {
-	type CommonAlertListOptions struct {
-		options.BaseListOptions
-		// 报警类型
-		AlertType string `help:"common alert type" choices:"normal|system"`
-		Level     string `help:"common alert notify level" choices:"normal|important|fatal"`
-	}
-	R(&CommonAlertListOptions{}, "commonalert-list", "List commonalert", func(s *mcclient.ClientSession,
-		args *CommonAlertListOptions) error {
-		params, err := options.ListStructToParams(args)
-		if err != nil {
-			return err
-		}
-		result, err := modules.CommonAlertManager.List(s, params)
-		if err != nil {
-			return nil
-		}
-		printList(result, modules.CommonAlertManager.GetColumns(s))
-		return nil
-	})
-
-	type CommonAlertDeleteOptions struct {
-		ID    string `help:"ID of alart"`
-		Force bool   `help:"force to delete alert"`
-	}
-	R(&CommonAlertDeleteOptions{}, "commonalert-delete", "List commonalert", func(s *mcclient.ClientSession,
-		args *CommonAlertDeleteOptions) error {
-		params := jsonutils.NewDict()
-		params.Add(jsonutils.NewBool(args.Force), "force")
-		object, err := modules.CommonAlertManager.Delete(s, args.ID, params)
-		if err != nil {
-			return err
-		}
-		printObject(object)
-		return nil
-	})
+	cmd := shell.NewResourceCmd(modules.CommonAlertManager)
+	cmd.List(new(options.CommonAlertListOptions))
+	cmd.Show(new(options.CommonAlertShowOptions))
+	cmd.Perform("enable", &options.CommonAlertShowOptions{})
+	cmd.Perform("disable", &options.CommonAlertShowOptions{})
+	cmd.Delete(new(options.CommonAlertDeleteOptions))
 }

cmd/climc/shell/monitor/suggestsysrule.go

@@ -15,70 +15,19 @@
 package monitor
 
 import (
-	"yunion.io/x/onecloud/pkg/mcclient"
-	"yunion.io/x/onecloud/pkg/mcclient/modules/monitor"
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	options "yunion.io/x/onecloud/pkg/mcclient/options/monitor"
 )
 
 func init() {
-	aN := cmdN("suggestsysrule")
-	R(&options.SuggestRuleListOptions{}, aN("list"), "List all suggestsysrules",
-		func(s *mcclient.ClientSession, args *options.SuggestRuleListOptions) error {
-			params, err := args.Params()
-			if err != nil {
-				return err
-			}
-			ret, err := monitor.SuggestSysRuleManager.List(s, params)
-			if err != nil {
-				return err
-			}
-			printList(ret, monitor.SuggestSysRuleManager.GetColumns(s))
-			return nil
-		})
 
-	R(&options.SuggestRuleCreateOptions{}, aN("create"), "Create suggestsys rule",
-		func(s *mcclient.ClientSession, args *options.SuggestRuleCreateOptions) error {
-			params, err := args.Params()
-			if err != nil {
-				return err
-			}
-			ret, err := monitor.SuggestSysRuleManager.Create(s, params)
-			if err != nil {
-				return err
-			}
-			printObject(ret)
-			return nil
-		})
-
-	R(&options.SuggestRuleShowOptions{}, aN("show"), "Show details of a alert rule",
-		func(s *mcclient.ClientSession, args *options.SuggestRuleShowOptions) error {
-			ret, err := monitor.SuggestSysRuleManager.Get(s, args.ID, nil)
-			if err != nil {
-				return err
-			}
-			printObject(ret)
-			return nil
-		})
-
-	R(&options.SuggestRuleUpdateOptions{}, aN("update"), "Update a alert rule",
-		func(s *mcclient.ClientSession, args *options.SuggestRuleUpdateOptions) error {
-			params, err := args.Params()
-			if err != nil {
-				return err
-			}
-			ret, err := monitor.SuggestSysRuleManager.Update(s, args.ID, params)
-			if err != nil {
-				return err
-			}
-			printObject(ret)
-			return nil
-		})
-
-	R(&options.SuggestRuleDeleteOptions{}, aN("delete"), "Delete alerts",
-		func(s *mcclient.ClientSession, args *options.SuggestRuleDeleteOptions) error {
-			ret := monitor.SuggestSysRuleManager.BatchDelete(s, args.ID, nil)
-			printBatchResults(ret, monitor.SuggestSysRuleManager.GetColumns(s))
-			return nil
-		})
+	cmd := shell.NewResourceCmd(modules.SuggestSysRuleManager)
+	cmd.List(new(options.SuggestRuleListOptions))
+	cmd.Show(new(options.SuggestRuleShowOptions))
+	cmd.Perform("enable", new(options.SuggestRuleShowOptions))
+	cmd.Perform("disable", new(options.SuggestRuleShowOptions))
+	cmd.Delete(new(options.CommonAlertDeleteOptions))
+	cmd.Perform("config", new(options.SuggestRuleConfigOptions))
 
 }

pkg/apigateway/clientman/authtoken_test.go

@@ -15,20 +15,17 @@
 package clientman
 
 import (
-	"crypto/rand"
-	"crypto/rsa"
 	"reflect"
 	"testing"
 )
 
 func TestEncoeDecode(t *testing.T) {
-	key, err := rsa.GenerateKey(rand.Reader, 2048)
+	SetupTest()
 	token := SAuthToken{
 		token:      `gAAAAABe-gUMAawOPrP-mA4jY6-b1UPalPJw9WlZJVqHZMtc3IBKUOvHTbKm60YyZQtnVBa3O3QDfS2ss5_Xwi_n0L-jfuUstguLHfDyztAvT_IAKupw8YNK0FvJg25LKC4IR3bmDzCNzTwMO-rEeb4ha2e1vkGOwko9GT1Bn-xN7UM2qeEsm5PiLBg0ZTMuv4Jm5RWIXk2K`,
 		verifyTotp: true,
 		enableTotp: false,
 	}
-	setPrivateKey(key)
 	et := token.encodeBytes()
 	plainEt := compressString(et)
 	encEt := EncryptString(et)

pkg/apigateway/clientman/clientman.go

@@ -15,6 +15,8 @@
 package clientman
 
 import (
+	"crypto/rand"
+	"crypto/rsa"
 	"io/ioutil"
 
 	"github.com/pkg/errors"
@@ -38,3 +40,8 @@ func InitClient() error {
 
 	return nil
 }
+
+func SetupTest() {
+	key, _ := rsa.GenerateKey(rand.Reader, 2048)
+	setPrivateKey(key)
+}

pkg/apigateway/handler/auth.go

@@ -71,6 +71,7 @@ func (h *AuthHandlers) AddMethods() {
 		NewHP(handleOIDCAuth, "oidc", "auth"),
 		NewHP(handleOIDCConfiguration, "oidc", ".well-known", "openid-configuration"),
 		NewHP(handleOIDCJWKeys, "oidc", "keys"),
+		NewHP(handleOIDCUserInfo, "oidc", "user"),
 	)
 	h.AddByMethod(POST, nil,
 		NewHP(h.initTotpSecrets, "initcredential"),
@@ -91,8 +92,6 @@ func (h *AuthHandlers) AddMethods() {
 		NewHP(h.getResources, "scoped_resources"),
 		NewHP(fetchIdpBasicConfig, "idp", "<idp_id>", "info"),
 		NewHP(fetchIdpSAMLMetadata, "idp", "<idp_id>", "saml-metadata"),
-		// oidc
-		NewHP(handleOIDCUserInfo, "oidc", "user"),
 	)
 	h.AddByMethod(POST, FetchAuthToken,
 		NewHP(h.resetUserPassword, "password"),
@@ -251,22 +250,17 @@ func doTenantLogin(ctx context.Context, req *http.Request, body jsonutils.JSONOb
 
 func fetchUserInfoFromToken(ctx context.Context, req *http.Request, token mcclient.TokenCredential) (jsonutils.JSONObject, error) {
 	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
-	info, err := modules.UsersV3.Get(s, token.GetUserId(), nil)
+	return fetchUserInfoById(s, token.GetUserId())
+}
+
+func fetchUserInfoById(s *mcclient.ClientSession, userId string) (jsonutils.JSONObject, error) {
+	info, err := modules.UsersV3.Get(s, userId, nil)
 	if err != nil {
 		return nil, errors.Wrap(err, "UsersV3.Get")
 	}
 	return info, nil
 }
 
-func FetchProjectMetadata(ctx context.Context, req *http.Request, pid string) (jsonutils.JSONObject, error) {
-	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
-	meta, err := modules.Projects.GetSpecific(s, pid, "metadata", nil)
-	if err != nil {
-		return nil, errors.Wrap(err, "GetProjectMetadata")
-	}
-	return meta, nil
-}
-
 func isUserEnableTotp(userInfo jsonutils.JSONObject) bool {
 	return jsonutils.QueryBoolean(userInfo, "enable_mfa", false)
 }
@@ -759,9 +753,14 @@ func getUserInfo(ctx context.Context, req *http.Request) (*jsonutils.JSONDict, e
 		log.Errorf("modules.UsersV3.Get fail %s", err)
 		return nil, fmt.Errorf("not found user %s", token.GetUserId())
 	}*/
-	usr, err := fetchUserInfoFromToken(ctx, req, token)
+	// usr, err := fetchUserInfoFromToken(ctx, req, token)
+	return getUserInfo2(s, token.GetUserId(), token.GetProjectId(), token.GetLoginIp())
+}
+
+func getUserInfo2(s *mcclient.ClientSession, uid string, pid string, loginIp string) (*jsonutils.JSONDict, error) {
+	usr, err := fetchUserInfoById(s, uid)
 	if err != nil {
-		return nil, errors.Wrapf(err, "fetchUserInfoFromToken %s", token.GetUserId())
+		return nil, errors.Wrapf(err, "fetchUserInfoFromToken %s", uid)
 	}
 	data := jsonutils.NewDict()
 	for _, k := range []string{
@@ -779,21 +778,37 @@ func getUserInfo(ctx context.Context, req *http.Request) (*jsonutils.JSONDict, e
 			data.Add(v, k)
 		}
 	}
-	data.Add(jsonutils.NewString(token.GetDomainId()), "domain", "id")
-	data.Add(jsonutils.NewString(token.GetDomainName()), "domain", "name")
-	data.Add(jsonutils.NewStringArray(auth.AdminCredential().GetRegions()), "regions")
-	data.Add(jsonutils.NewStringArray(token.GetRoles()), "roles")
-	data.Add(jsonutils.NewString(token.GetProjectName()), "projectName")
-	data.Add(jsonutils.NewString(token.GetProjectId()), "projectId")
-	data.Add(jsonutils.NewString(token.GetProjectDomain()), "projectDomain")
-	data.Add(jsonutils.NewString(token.GetProjectDomainId()), "projectDomainId")
+	usrId, _ := usr.GetString("id")
+	usrName, _ := usr.GetString("name")
+	usrDomainId, _ := usr.GetString("domain_id")
+	usrDomainName, _ := usr.GetString("project_domain")
+
+	data.Add(jsonutils.NewString(usrDomainId), "domain", "id")
+	data.Add(jsonutils.NewString(usrDomainName), "domain", "name")
+	data.Add(jsonutils.NewStringArray(auth.AdminCredential().GetRegions()), "regions")
+
+	var projName string
+	var projDomainId string
+	if len(pid) > 0 {
+		projInfo, err := modules.Projects.GetById(s, pid, nil)
+		if err != nil {
+			return nil, errors.Wrapf(err, "fetchProjectById %s", pid)
+		}
+
+		projName, _ = projInfo.GetString("name")
+		projId, _ := projInfo.GetString("id")
+		projDomainId, _ = projInfo.GetString("domain_id")
+		projDomainName, _ := projInfo.GetString("project_domain")
+		data.Add(jsonutils.NewString(projName), "projectName")
+		data.Add(jsonutils.NewString(projId), "projectId")
+		data.Add(jsonutils.NewString(projDomainName), "projectDomain")
+		data.Add(jsonutils.NewString(projDomainId), "projectDomainId")
+
+		pmeta, err := projInfo.Get("metadata")
+		if pmeta != nil {
+			data.Add(pmeta, "project_meta")
+		}
 
-	pmeta, err := FetchProjectMetadata(ctx, req, token.GetProjectId())
-	if err != nil {
-		return nil, errors.Wrap(err, "FetchProjectMetadata")
-	}
-	if pmeta != nil {
-		data.Add(pmeta, "project_meta")
 	}
 
 	log.Infof("getUserInfo modules.RoleAssignments.List")
@@ -802,11 +817,12 @@ func getUserInfo(ctx context.Context, req *http.Request) (*jsonutils.JSONDict, e
 	query.Add(jsonutils.JSONNull, "include_names")
 	query.Add(jsonutils.JSONNull, "include_system")
 	query.Add(jsonutils.NewInt(0), "limit")
-	query.Add(jsonutils.NewString(token.GetUserId()), "user", "id")
+	query.Add(jsonutils.NewString(uid), "user", "id")
 	roleAssigns, err := modules.RoleAssignments.List(s, query)
 	if err != nil {
 		return nil, errors.Wrapf(err, "get RoleAssignments list")
 	}
+	currentRoles := make([]string, 0)
 	projects := make(map[string]*projectRoles)
 	for _, roleAssign := range roleAssigns.Data {
 		roleId, _ := roleAssign.GetString("role", "id")
@@ -815,6 +831,9 @@ func getUserInfo(ctx context.Context, req *http.Request) (*jsonutils.JSONDict, e
 		projectName, _ := roleAssign.GetString("scope", "project", "name")
 		domainId, _ := roleAssign.GetString("scope", "project", "domain", "id")
 		domain, _ := roleAssign.GetString("scope", "project", "domain", "name")
+		if projectId == pid {
+			currentRoles = append(currentRoles, roleName)
+		}
 		_, ok := projects[projectId]
 		if ok {
 			projects[projectId].add(roleId, roleName)
@@ -822,31 +841,38 @@ func getUserInfo(ctx context.Context, req *http.Request) (*jsonutils.JSONDict, e
 			projects[projectId] = newProjectRoles(projectId, projectName, roleId, roleName, domainId, domain)
 		}
 	}
+
+	data.Add(jsonutils.NewStringArray(currentRoles), "roles")
+
 	projJson := jsonutils.NewArray()
 	for _, proj := range projects {
 		projJson.Add(proj.json(
-			token.GetUserName(),
-			token.GetUserId(),
-			token.GetDomainName(),
-			token.GetDomainId(),
-			token.GetLoginIp(),
+			usrName,
+			usrId,
+			usrDomainName,
+			usrDomainId,
+			loginIp,
 		))
 	}
 	data.Add(projJson, "projects")
 
-	for _, scope := range []rbacutils.TRbacScope{
-		rbacutils.ScopeSystem,
-		rbacutils.ScopeDomain,
-		rbacutils.ScopeProject,
-	} {
-		p := policy.PolicyManager.MatchedPolicyNames(scope, token)
-		data.Add(jsonutils.NewStringArray(p), fmt.Sprintf("%s_policies", scope))
-		if scope == rbacutils.ScopeSystem {
-			data.Add(jsonutils.NewStringArray(p), "admin_policies")
-		} else if scope == rbacutils.ScopeProject {
-			data.Add(jsonutils.NewStringArray(p), "policies")
+	if len(pid) > 0 {
+		ident := rbacutils.NewRbacIdentity2(projDomainId, projName, currentRoles, loginIp)
+		for _, scope := range []rbacutils.TRbacScope{
+			rbacutils.ScopeSystem,
+			rbacutils.ScopeDomain,
+			rbacutils.ScopeProject,
+		} {
+			p := policy.PolicyManager.MatchedPolicyNames(scope, ident)
+			data.Add(jsonutils.NewStringArray(p), fmt.Sprintf("%s_policies", scope))
+			if scope == rbacutils.ScopeSystem {
+				data.Add(jsonutils.NewStringArray(p), "admin_policies")
+			} else if scope == rbacutils.ScopeProject {
+				data.Add(jsonutils.NewStringArray(p), "policies")
+			}
 		}
 	}
+
 	allPolicies := policy.PolicyManager.AllPolicies()
 	data.Add(jsonutils.Marshal(allPolicies), "all_policies")
 
@@ -854,7 +880,7 @@ func getUserInfo(ctx context.Context, req *http.Request) (*jsonutils.JSONDict, e
 	menus := jsonutils.NewArray()
 	k8s := jsonutils.NewArray()
 
-	curReg := FetchRegion(req)
+	curReg := s.GetRegion()
 	srvCat := auth.Client().GetServiceCatalog()
 	var allsrv []string
 	var alleps []mcclient.ExternalService
@@ -908,13 +934,13 @@ func getUserInfo(ctx context.Context, req *http.Request) (*jsonutils.JSONDict, e
 	}
 
 	log.Infof("getUserInfo modules.Hosts.Get")
-	s2 := auth.GetSession(ctx, token, FetchRegion(req), "v2")
+	// s2 := auth.GetSession(ctx, token, FetchRegion(req), "v2")
 	params := jsonutils.NewDict()
 	params.Add(jsonutils.NewString("host_type"), "field")
 	params.Add(jsonutils.NewString("system"), "scope")
 	params.Add(jsonutils.JSONTrue, "usable")
 	params.Add(jsonutils.JSONTrue, "show_emulated")
-	cap, err := modules.Hosts.Get(s2, "distinct-field", params)
+	cap, err := modules.Hosts.Get(s, "distinct-field", params)
 	if err != nil {
 		log.Errorf("modules.Servers.Get distinct-field fail %s", err)
 	} else {

pkg/apigateway/handler/oidc.go

@@ -45,7 +45,10 @@ import (
 )
 
 const (
+	// OIDC code expires in 5 minutes
 	OIDC_CODE_EXPIRE_SECONDS = 300
+	// OIDC token expires in 2 hours
+	OIDC_TOKEN_EXPIRE_SECONDS = 7200
 )
 
 func getLoginCallbackParam() string {
@@ -72,7 +75,7 @@ func addQuery(urlstr string, qs jsonutils.JSONObject) string {
 func handleOIDCAuth(ctx context.Context, w http.ResponseWriter, req *http.Request) {
 	ctx, err := fetchAndSetAuthContext(ctx, w, req)
 	if err != nil {
-		// redirect to login page
+		// not login redirect to login page
 		qs := jsonutils.NewDict()
 		oUrl := req.URL.String()
 		if !strings.HasPrefix(oUrl, "http") {
@@ -135,8 +138,10 @@ func doOIDCAuth(ctx context.Context, req *http.Request, query jsonutils.JSONObje
 		return oidcAuth, "", errors.Wrap(httperrors.ErrInvalidCredential, "redirect uri not match")
 	}
 
+	token := AppContextToken(ctx)
+
 	cliIp := netutils2.GetHttpRequestIp(req)
-	codeInfo := newOIDCClientInfo(cliIp)
+	codeInfo := newOIDCClientInfo(token, cliIp, FetchRegion(req))
 	code := clientman.EncryptString(codeInfo.toBytes())
 
 	return oidcAuth, code, nil
@@ -155,12 +160,20 @@ func handleOIDCToken(ctx context.Context, w http.ResponseWriter, req *http.Reque
 type SOIDCClientInfo struct {
 	Timestamp int64
 	Ip        netutils.IPV4Addr
+	UserId    string
+	ProjectId string
+	Region    string
 }
 
 func (i SOIDCClientInfo) toBytes() []byte {
-	enc := make([]byte, 12)
+	enc := make([]byte, 12+1+len(i.UserId)+1+len(i.ProjectId)+len(i.Region))
 	binary.LittleEndian.PutUint64(enc, uint64(i.Timestamp))
 	binary.LittleEndian.PutUint32(enc[8:], uint32(i.Ip))
+	enc[12] = byte(len(i.UserId))
+	enc[13] = byte(len(i.ProjectId))
+	copy(enc[14:], i.UserId)
+	copy(enc[14+len(i.UserId):], i.ProjectId)
+	copy(enc[14+len(i.UserId)+len(i.ProjectId):], i.Region)
 	return enc
 }
 
@@ -171,23 +184,67 @@ func (i SOIDCClientInfo) isExpired() bool {
 	return false
 }
 
+func (i SOIDCClientInfo) expiresAt(secs int) time.Time {
+	expires := i.Timestamp + int64(secs)*int64(time.Second)
+	esecs := expires / int64(time.Second)
+	nsecs := expires - esecs*int64(time.Second)
+	return time.Unix(esecs, nsecs)
+}
+
 func decodeOIDCClientInfo(enc []byte) (SOIDCClientInfo, error) {
 	info := SOIDCClientInfo{}
-	if len(enc) != 8+4 {
+	if len(enc) < 8+4+1 {
 		return info, errors.Wrap(httperrors.ErrInvalidCredential, "code byte length must be 12")
 	}
 	info.Timestamp = int64(binary.LittleEndian.Uint64(enc))
 	info.Ip = netutils.IPV4Addr(binary.LittleEndian.Uint32(enc[8:]))
+	info.UserId = string(enc[14 : 14+int(enc[12])])
+	info.ProjectId = string(enc[14+int(enc[12]) : 14+int(enc[12])+int(enc[13])])
+	info.Region = string(enc[14+int(enc[12])+int(enc[13]):])
 	return info, nil
 }
 
-func newOIDCClientInfo(ipstr string) SOIDCClientInfo {
+func newOIDCClientInfo(token mcclient.TokenCredential, ipstr string, region string) SOIDCClientInfo {
 	info := SOIDCClientInfo{}
 	info.Timestamp = time.Now().UnixNano()
 	info.Ip, _ = netutils.NewIPV4Addr(ipstr)
+	info.UserId = token.GetUserId()
+	info.ProjectId = token.GetProjectId()
+	info.Region = region
 	return info
 }
 
+type SOIDCClientToken struct {
+	Info SOIDCClientInfo
+}
+
+func (t SOIDCClientToken) encode() string {
+	json := jsonutils.NewDict()
+	json.Add(jsonutils.NewString(string(t.Info.toBytes())), "info")
+	return clientman.EncryptString([]byte(json.String()))
+}
+
+func decodeOIDCClientToken(token string) (SOIDCClientToken, error) {
+	ret := SOIDCClientToken{}
+	tBytes, err := clientman.DecryptString(token)
+	if err != nil {
+		return ret, errors.Wrap(err, "DecryptString")
+	}
+	json, err := jsonutils.Parse(tBytes)
+	if err != nil {
+		return ret, errors.Wrap(err, "json.Parse")
+	}
+	info, err := json.GetString("info")
+	if err != nil {
+		return ret, errors.Wrap(err, "getString(info)")
+	}
+	ret.Info, err = decodeOIDCClientInfo([]byte(info))
+	if err != nil {
+		return ret, errors.Wrap(err, "decodeOIDCClientInfo")
+	}
+	return ret, nil
+}
+
 func validateOIDCToken(ctx context.Context, req *http.Request) (oidcutils.SOIDCAccessTokenResponse, error) {
 	var tokenResp oidcutils.SOIDCAccessTokenResponse
 	bodyBytes, err := appsrv.Fetch(req)
@@ -253,35 +310,29 @@ func validateOIDCToken(ctx context.Context, req *http.Request) (oidcutils.SOIDCA
 		return tokenResp, errors.Wrap(httperrors.ErrInvalidCredential, "client secret not match")
 	}
 
-	token, err := auth.Client().AuthenticateByAccessKey(clientId, clientSecret, codeInfo.Ip.String())
-	if err != nil {
-		return tokenResp, errors.Wrap(err, "invalid client_id/client_secret")
+	token := SOIDCClientToken{
+		Info: codeInfo,
 	}
 
 	tokenResp = token2AccessTokenResponse(token, clientId)
 	return tokenResp, nil
 }
 
-func token2AccessTokenResponse(token mcclient.TokenCredential, clientId string) oidcutils.SOIDCAccessTokenResponse {
+func token2AccessTokenResponse(token SOIDCClientToken, clientId string) oidcutils.SOIDCAccessTokenResponse {
 	resp := oidcutils.SOIDCAccessTokenResponse{}
-	resp.AccessToken = token2AccessToken(token)
+	resp.AccessToken = token.encode()
 	resp.TokenType = oidcutils.OIDC_BEARER_TOKEN_TYPE
 	resp.IdToken, _ = token2IdToken(token, clientId)
-	resp.ExpiresIn = int(token.GetExpires().Unix() - time.Now().Unix())
+	resp.ExpiresIn = int(token.Info.expiresAt(OIDC_TOKEN_EXPIRE_SECONDS).Unix() - time.Now().Unix())
 	return resp
 }
 
-func token2AccessToken(token mcclient.TokenCredential) string {
-	authToken := clientman.NewAuthToken(token.GetTokenString(), false, false)
-	return authToken.Encode()
-}
-
-func token2IdToken(token mcclient.TokenCredential, clientId string) (string, error) {
+func token2IdToken(token SOIDCClientToken, clientId string) (string, error) {
 	jwtToken := jwt.New()
 	jwtToken.Set(jwt.IssuerKey, options.Options.ApiServer)
-	jwtToken.Set(jwt.SubjectKey, token.GetUserId())
+	jwtToken.Set(jwt.SubjectKey, token.Info.UserId)
 	jwtToken.Set(jwt.AudienceKey, clientId)
-	jwtToken.Set(jwt.ExpirationKey, token.GetExpires().Unix())
+	jwtToken.Set(jwt.ExpirationKey, token.Info.expiresAt(OIDC_TOKEN_EXPIRE_SECONDS).Unix())
 	jwtToken.Set(jwt.IssuedAtKey, time.Now().Unix())
 	return clientman.SignJWT(jwtToken)
 }
@@ -334,7 +385,24 @@ func handleOIDCJWKeys(ctx context.Context, w http.ResponseWriter, req *http.Requ
 }
 
 func handleOIDCUserInfo(ctx context.Context, w http.ResponseWriter, req *http.Request) {
-	data, err := getUserInfo(ctx, req)
+	tokenHdr := getAuthToken(req)
+	if len(tokenHdr) == 0 {
+		httperrors.InvalidCredentialError(ctx, w, "No token in header")
+		return
+	}
+	token, err := decodeOIDCClientToken(tokenHdr)
+	if err != nil {
+		log.Errorf("decodeOIDCClientToken %s fail %s", tokenHdr, err)
+		httperrors.InvalidCredentialError(ctx, w, "Token in header invalid")
+		return
+	}
+	if token.Info.expiresAt(OIDC_TOKEN_EXPIRE_SECONDS).Before(time.Now()) {
+		httperrors.InvalidCredentialError(ctx, w, "Token expired")
+		return
+	}
+
+	s := auth.GetAdminSession(ctx, token.Info.Region, "")
+	data, err := getUserInfo2(s, token.Info.UserId, token.Info.ProjectId, token.Info.Ip.String())
 	if err != nil {
 		httperrors.NotFoundError(ctx, w, "%v", err)
 		return

pkg/apigateway/handler/oidc_test.go

@@ -0,0 +1,93 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handler
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"yunion.io/x/pkg/util/netutils"
+
+	"yunion.io/x/onecloud/pkg/apigateway/clientman"
+)
+
+func TestClientInfo(t *testing.T) {
+	clientman.SetupTest()
+
+	cases := []struct {
+		ip      string
+		user    string
+		project string
+	}{
+		{
+			ip:      "0.0.0.0",
+			user:    "sysadmin",
+			project: "system",
+		},
+		{
+			ip:      "10.168.26.253",
+			user:    "ab9502de-c6b6-4150-880b-d0e3e6ba8ec8",
+			project: "a2049cfadf4c40888b9da136faba5cc8",
+		},
+	}
+	for _, c := range cases {
+		info := SOIDCClientInfo{}
+		info.Timestamp = time.Now().UnixNano()
+		info.Ip, _ = netutils.NewIPV4Addr(c.ip)
+		info.UserId = c.user
+		info.ProjectId = c.project
+
+		msg := info.toBytes()
+		if len(msg) != 14+len(info.UserId)+len(info.ProjectId)+len(info.Region) {
+			t.Fatalf("incorrect msg size")
+		}
+
+		info2, err := decodeOIDCClientInfo(msg)
+		if err != nil {
+			t.Fatalf("decode error %s", err)
+		}
+
+		if info2.Timestamp != info.Timestamp {
+			t.Fatalf("incorrect timestamp")
+		}
+		if info2.Ip.String() != info.Ip.String() {
+			t.Fatalf("incorrect ip")
+		}
+		if info2.UserId != info.UserId {
+			t.Fatalf("incorrect user id")
+		}
+		if info2.ProjectId != info.ProjectId {
+			t.Fatalf("incorrect project id")
+		}
+		if info2.Region != info.Region {
+			t.Fatalf("incorrect region id")
+		}
+
+		token := SOIDCClientToken{
+			Info: info,
+		}
+
+		tokenStr := token.encode()
+		token2, err := decodeOIDCClientToken(tokenStr)
+		if err != nil {
+			t.Fatalf("decodeOIDCClientToken fail %s", err)
+		}
+
+		if !reflect.DeepEqual(token2.Info, info2) {
+			t.Fatalf("token2 info not equal to info2")
+		}
+	}
+}

pkg/apis/compute/dnszonecache.go

@@ -22,6 +22,7 @@ const (
 	DNS_ZONE_CACHE_STATUS_CREATING      = "creating"
 	DNS_ZONE_CACHE_STATUS_CREATE_FAILED = "create_failed"
 	DNS_ZONE_CACHE_STATUS_AVAILABLE     = "available"
+	DNS_ZONE_CACHE_STATUS_UNKNOWN       = "unknown"
 )
 
 type DnsZoneCacheCreateInput struct {

pkg/apis/compute/loadbalancer.go

@@ -201,7 +201,10 @@ type LoadbalancerResourceInfo struct {
 	// 可用区ID
 	ZoneId string `json:"zone_id"`
 
-	ZoneResourceInfoBase
+	ZoneResourceInfo
+
+	// cloud provider info
+	ManagedResourceInfo
 }
 
 type LoadbalancerResourceInput struct {

pkg/apis/monitor/metric.go

@@ -24,6 +24,7 @@ var MetricUnit = []string{"%", "bps", "Mbps", "Bps", "count/s", "count", "ms", "
 
 type MetricMeasurementCreateInput struct {
 	apis.StandaloneResourceCreateInput
+	apis.EnabledBaseResourceCreateInput
 
 	ResType     string `json:"res_type"`
 	DisplayName string `json:"display_name"`

pkg/apis/monitor/suggestsysrule.go

@@ -94,7 +94,6 @@ type SSuggestSysAlertSetting struct {
 }
 
 type EIPUnused struct {
-	//Status string `json:"status"`
 }
 
 type DiskUnused struct {

pkg/cloudcommon/db/scoperesource.go

@@ -116,7 +116,7 @@ func (m *SScopedResourceBaseManager) FilterByOwner(q *sqlchemy.SQuery, userCred
 	}
 	switch scope {
 	case rbacutils.ScopeDomain:
-		q = q.Filter(sqlchemy.OR(
+		/* q = q.Filter(sqlchemy.OR(
 			// share to system
 			sqlchemy.AND(
 				sqlchemy.IsNullOrEmpty(q.Field("domain_id")),
@@ -124,9 +124,10 @@ func (m *SScopedResourceBaseManager) FilterByOwner(q *sqlchemy.SQuery, userCred
 			),
 			// share to this domain or its sub-projects
 			sqlchemy.Equals(q.Field("domain_id"), userCred.GetProjectDomainId()),
-		))
+		)) */
+		q = q.Equals("domain_id", userCred.GetProjectDomainId())
 	case rbacutils.ScopeProject:
-		q = q.Filter(sqlchemy.OR(
+		/* q = q.Filter(sqlchemy.OR(
 			// share to system
 			sqlchemy.AND(
 				sqlchemy.IsNullOrEmpty(q.Field("domain_id")),
@@ -142,7 +143,8 @@ func (m *SScopedResourceBaseManager) FilterByOwner(q *sqlchemy.SQuery, userCred
 				sqlchemy.Equals(q.Field("domain_id"), userCred.GetProjectDomainId()),
 				sqlchemy.Equals(q.Field("tenant_id"), userCred.GetProjectId()),
 			),
-		))
+		)) */
+		q = q.Equals("tenant_id", userCred.GetProjectId())
 	}
 	return q
 }

pkg/compute/models/buckets.go

@@ -716,11 +716,7 @@ func (bucket *SBucket) GetDetailsObjects(
 	}
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return output, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return output, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return output, errors.Wrap(err, "GetIBucket")
 	}
 	prefix := input.Prefix
 	isRecursive := false
@@ -794,11 +790,7 @@ func (bucket *SBucket) PerformTempUrl(
 
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return output, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return output, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return output, errors.Wrap(err, "GetIBucket")
 	}
 	tmpUrl, err := iBucket.GetTempUrl(method, key, time.Duration(expire)*time.Second)
 	if err != nil {
@@ -842,11 +834,7 @@ func (bucket *SBucket) PerformMakedir(
 
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return nil, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return nil, errors.Wrap(err, "GetIBucket")
 	}
 
 	_, err = cloudprovider.GetIObject(iBucket, key+"/")
@@ -913,11 +901,7 @@ func (bucket *SBucket) PerformDelete(
 
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return nil, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return nil, errors.Wrap(err, "GetIBucket")
 	}
 	ok := jsonutils.NewDict()
 	results := modulebase.BatchDo(keyStrs, func(key string) (jsonutils.JSONObject, error) {
@@ -977,11 +961,7 @@ func (bucket *SBucket) PerformUpload(
 
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return nil, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return nil, errors.Wrap(err, "GetIBucket")
 	}
 
 	meta := cloudprovider.FetchMetaFromHttpHeader(cloudprovider.META_HEADER_PREFIX, appParams.Request.Header)
@@ -1185,11 +1165,7 @@ func (bucket *SBucket) PerformSync(
 
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return nil, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return nil, errors.Wrap(err, "GetIBucket")
 	}
 
 	err = bucket.syncWithCloudBucket(ctx, userCred, iBucket, nil, statsOnly)
@@ -1233,11 +1209,7 @@ func (bucket *SBucket) GetDetailsAcl(
 	}
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return output, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return output, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return output, errors.Wrap(err, "GetIBucket")
 	}
 	objKey := input.Key
 	var acl cloudprovider.TBucketACLType
@@ -1321,11 +1293,7 @@ func (bucket *SBucket) PerformLimit(
 
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return nil, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return nil, errors.Wrap(err, "GetIBucket")
 	}
 
 	err = iBucket.SetLimit(limit)
@@ -1438,11 +1406,7 @@ func (bucket *SBucket) processObjectsActionInput(input api.BucketObjectsActionIn
 	}
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return nil, nil, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return nil, nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return nil, nil, errors.Wrap(err, "GetIBucket")
 	}
 	objects := make([]cloudprovider.ICloudObject, 0)
 	for _, key := range input.Key {

pkg/compute/models/cloudaccounts.go

@@ -361,6 +361,9 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre
 	if err != nil {
 		return output, errors.Wrap(err, "FetchOwnerId in PerformPrepareNets")
 	}
+	if ownerId == nil {
+		ownerId = userCred
+	}
 	input.CloudaccountCreateInput, err = scm.ValidateCreateData(ctx, userCred, ownerId, query, input.CloudaccountCreateInput)
 	if err != nil {
 		return output, err

pkg/compute/models/dns_recordsets.go

@@ -152,40 +152,78 @@ func (manager *SDnsRecordSetManager) ValidateCreateData(ctx context.Context, use
 	}
 
 	// 处理重复的记录
-	dupedRecordsets := make([]SDnsRecordSet, 0)
-	err = DnsRecordSetManager.Query().Equals("dns_zone_id", input.DnsZoneId).Equals("name", input.Name).Equals("dns_type", input.DnsType).All(&dupedRecordsets)
-	if err != nil && errors.Cause(err) != sql.ErrNoRows {
-		return input, httperrors.NewGeneralError(err)
-	}
+
+	// CNAME  dnsName不能和其他类型record相同
+
+	// 同dnsName 同dnsType重复检查
 	// 检查dnsrecord 是否通过policy重复
 	// simple类型不能重复，不能和其他policy重复
 	// 不同类型policy不能重复
 	// 同类型policy的dnsrecord重复时，需要通过policyvalue区别
-	for i := range dupedRecordsets {
-		sq := DnsRecordSetTrafficPolicyManager.Query("dns_traffic_policy_id").Equals("dns_recordset_id", dupedRecordsets[i].Id)
-		q := DnsTrafficPolicyManager.Query().In("id", sq.SubQuery())
-		policies := []SDnsTrafficPolicy{}
-		err := db.FetchModelObjects(DnsTrafficPolicyManager, q, &policies)
-		if err != nil {
-			return input, httperrors.NewGeneralError(errors.Wrap(err, "db.FetchModelObjects"))
-		}
-		if len(policies) < 1 || len(input.TrafficPolicies) < 1 {
-			return input, httperrors.NewNotSupportedError("duplicated dnsrecord with simple policy not support")
-		}
-		for j := range policies {
-			for k := range input.TrafficPolicies {
-				if policies[j].Provider != input.TrafficPolicies[k].Provider {
-					continue
-				}
-				if strings.Contains(policies[j].Name, "Simple") ||
-					strings.Contains(input.TrafficPolicies[k].PolicyType, "Simple") ||
-					policies[j].PolicyType != input.TrafficPolicies[k].PolicyType {
-					return input, httperrors.NewNotSupportedError("duplicated dnsrecord with different policyType not support")
-				}
-				if policies[j].PolicyValue == input.TrafficPolicies[k].PolicyValue {
-					return input, httperrors.NewNotSupportedError("duplicated dnsrecord with same policyValue not support")
-				}
+
+	// validate name type
+	q := DnsRecordSetManager.Query().Equals("dns_zone_id", input.DnsZoneId).Equals("name", input.Name)
+	recordTypeQuery := q
+	switch input.DnsType {
+	case "CNAME":
+		recordTypeQuery = recordTypeQuery.NotEquals("dns_type", "CNAME")
+	default:
+		recordTypeQuery = recordTypeQuery.Equals("dns_type", "CNAME")
+	}
+
+	cnt, err := recordTypeQuery.CountWithError()
+	if err != nil {
+		return input, httperrors.NewGeneralError(err)
+	}
+	if cnt > 0 {
+		return input, httperrors.NewNotSupportedError("duplicated with CNAME dnsrecord name not support")
+	}
+
+	//validate policy
+
+	policyQuery := DnsRecordSetManager.Query().Equals("dns_zone_id", input.DnsZoneId).Equals("name", input.Name).Equals("dns_type", input.DnsType)
+	if len(input.TrafficPolicies) > 0 {
+		for i := range input.TrafficPolicies {
+			dupeDnsrecord := DnsRecordSetManager.Query().Equals("dns_zone_id", input.DnsZoneId).Equals("name", input.Name).Equals("dns_type", input.DnsType).SubQuery()
+			provider := input.TrafficPolicies[i].Provider
+			policyType := input.TrafficPolicies[i].PolicyType
+			policyValue := input.TrafficPolicies[i].PolicyValue
+
+			policyQuery := dupeDnsrecord.Query()
+			dnsrecordTrafficPolicies := DnsRecordSetTrafficPolicyManager.Query().SubQuery()
+			dnstrafficPolicy := DnsTrafficPolicyManager.Query().SubQuery()
+
+			policyQuery = policyQuery.LeftJoin(dnsrecordTrafficPolicies, sqlchemy.Equals(dupeDnsrecord.Field("id"), dnsrecordTrafficPolicies.Field("dns_recordset_id")))
+			policyQuery = policyQuery.LeftJoin(dnstrafficPolicy, sqlchemy.Equals(dnsrecordTrafficPolicies.Field("dns_traffic_policy_id"), dnstrafficPolicy.Field("id")))
+			policyQuery = policyQuery.Filter(sqlchemy.OR(
+				sqlchemy.IsNullOrEmpty(dnsrecordTrafficPolicies.Field("dns_traffic_policy_id")), // 无trafficPolicy的重复dnsrecord
+				sqlchemy.Contains(dnstrafficPolicy.Field("name"), "Simple"),                     // 简单 trafficPolicy的重复dnsrecord
+				sqlchemy.AND(
+					sqlchemy.Equals(dnstrafficPolicy.Field("provider"), provider), // 只看相同provider
+					sqlchemy.OR(
+						sqlchemy.NotEquals(dnstrafficPolicy.Field("policy_type"), policyType), // policy类型不同的重复dnsrecord
+						sqlchemy.AND(
+							sqlchemy.Equals(dnstrafficPolicy.Field("policy_type"), policyType), // policy类型相同但未通过policyvalue区分的的重复dnsrecord
+							sqlchemy.Equals(dnstrafficPolicy.Field("policy_value"), policyValue)),
+					),
+				),
+			))
+			policyQuery.DebugQuery()
+			cnt, err = policyQuery.CountWithError()
+			if err != nil {
+				return input, httperrors.NewGeneralError(err)
 			}
+			if cnt > 0 {
+				return input, httperrors.NewNotSupportedError("duplicated dnsrecord with existed dnsrecord can not distinguish by %s policy", provider)
+			}
+		}
+	} else {
+		cnt, err = policyQuery.CountWithError()
+		if err != nil {
+			return input, httperrors.NewGeneralError(err)
+		}
+		if cnt > 0 {
+			return input, httperrors.NewNotSupportedError("duplicated dnsrecord with existed dnsrecord not support")
 		}
 	}
 
@@ -352,9 +390,6 @@ func (manager *SDnsRecordSetManager) FilterByUniqValues(q *sqlchemy.SQuery, valu
 	if len(uniq.DnsName) > 0 {
 		q = q.Equals("name", uniq.DnsName)
 	}
-	if uniq.DnsType == "CNAME" {
-		return q
-	}
 	if len(uniq.DnsType) > 0 {
 		q = q.Equals("dns_type", uniq.DnsType)
 	}
@@ -387,6 +422,14 @@ func (manager *SDnsRecordSetManager) ResourceScope() rbacutils.TRbacScope {
 	return rbacutils.ScopeDomain
 }
 
+func (self *SDnsRecordSet) IsSharable(reqUsrId mcclient.IIdentityProvider) bool {
+	dnsZone, err := self.GetDnsZone()
+	if err != nil {
+		return false
+	}
+	return dnsZone.IsSharable(reqUsrId)
+}
+
 func (self *SDnsRecordSet) GetOwnerId() mcclient.IIdentityProvider {
 	dnsZone, err := self.GetDnsZone()
 	if err != nil {
@@ -436,25 +479,23 @@ func (self *SDnsRecordSet) ValidateUpdateData(ctx context.Context, userCred mccl
 		return input, httperrors.NewGeneralError(errors.Wrapf(err, "GetDnsZone"))
 	}
 
+	if len(input.DnsType) == 0 {
+		input.DnsType = self.DnsType
+	}
+	if len(input.DnsValue) == 0 {
+		input.DnsValue = self.DnsValue
+	}
+	if input.TTL == nil {
+		input.TTL = &self.TTL
+	}
+	if input.MxPriority == nil {
+		input.MxPriority = &self.MxPriority
+	}
 	recordset := api.SDnsRecordSet{}
 	recordset.DnsType = input.DnsType
 	recordset.DnsValue = input.DnsValue
-	if len(recordset.DnsType) == 0 {
-		recordset.DnsType = self.DnsType
-	}
-	if len(recordset.DnsValue) == 0 {
-		recordset.DnsValue = self.DnsValue
-	}
-	if input.TTL != nil {
-		recordset.TTL = *input.TTL
-	} else {
-		recordset.TTL = self.TTL
-	}
-	if input.MxPriority != nil {
-		recordset.MxPriority = *input.MxPriority
-	} else {
-		recordset.MxPriority = self.MxPriority
-	}
+	recordset.TTL = *input.TTL
+	recordset.MxPriority = *input.MxPriority
 
 	err = recordset.ValidateDnsrecordValue()
 	if err != nil {
@@ -466,61 +507,95 @@ func (self *SDnsRecordSet) ValidateUpdateData(ctx context.Context, userCred mccl
 	}
 
 	// 处理重复的记录
-	dupedRecordsets := make([]SDnsRecordSet, 0)
-	q := DnsRecordSetManager.Query().Equals("dns_zone_id", dnsZone.Id).Equals("name", input.Name).Equals("dns_type", input.DnsType).NotEquals("id", self.Id)
-	err = q.All(&dupedRecordsets)
-	if err != nil && errors.Cause(err) != sql.ErrNoRows {
-		return input, httperrors.NewGeneralError(err)
-	}
+
+	// CNAME  dnsName不能和其他类型record相同
+
+	// 同dnsName 同dnsType重复检查
 	// 检查dnsrecord 是否通过policy重复
 	// simple类型不能重复，不能和其他policy重复
 	// 不同类型policy不能重复
 	// 同类型policy的dnsrecord重复时，需要通过policyvalue区别
 
-	oldPolicies, err := self.GetDnsTrafficPolicies()
+	// validate name type
+	q := DnsRecordSetManager.Query().Equals("dns_zone_id", dnsZone.Id).NotEquals("id", self.Id).Equals("name", input.Name)
+	recordTypeQuery := q
+	switch input.DnsType {
+	case "CNAME":
+		recordTypeQuery = recordTypeQuery.NotEquals("dns_type", "CNAME")
+	default:
+		recordTypeQuery = recordTypeQuery.Equals("dns_type", "CNAME")
+	}
+
+	cnt, err := recordTypeQuery.CountWithError()
 	if err != nil {
 		return input, httperrors.NewGeneralError(err)
 	}
-	for i := range dupedRecordsets {
-		sq := DnsRecordSetTrafficPolicyManager.Query("dns_traffic_policy_id").Equals("dns_recordset_id", dupedRecordsets[i].Id)
-		q := DnsTrafficPolicyManager.Query().In("id", sq.SubQuery())
-		policies := []SDnsTrafficPolicy{}
-		err := db.FetchModelObjects(DnsTrafficPolicyManager, q, &policies)
+	if cnt > 0 {
+		return input, httperrors.NewNotSupportedError("duplicated with CNAME dnsrecord name not support")
+	}
+
+	//validate policy
+	policies, err := self.GetDnsTrafficPolicies()
+	if err != nil {
+		return input, httperrors.NewGeneralError(err)
+	}
+	policyMap := map[string]SDnsTrafficPolicy{}
+	for i := range policies {
+		policyMap[policies[i].Provider] = policies[i]
+	}
+	for i := range input.TrafficPolicies {
+		inputPolicy := input.TrafficPolicies[i]
+		policyMap[inputPolicy.Provider] = SDnsTrafficPolicy{
+			Provider:    inputPolicy.Provider,
+			PolicyType:  inputPolicy.PolicyType,
+			PolicyValue: inputPolicy.PolicyValue,
+			Options:     inputPolicy.PolicyOptions,
+		}
+	}
+
+	policyQuery := DnsRecordSetManager.Query().Equals("dns_zone_id", dnsZone.Id).NotEquals("id", self.Id).Equals("name", input.Name).Equals("dns_type", input.DnsType)
+	if len(policyMap) > 0 {
+		for i := range policyMap {
+			dupeDnsrecord := DnsRecordSetManager.Query().Equals("dns_zone_id", dnsZone.Id).NotEquals("id", self.Id).Equals("name", input.Name).Equals("dns_type", input.DnsType).SubQuery()
+			provider := policyMap[i].Provider
+			policyType := policyMap[i].PolicyType
+			policyValue := policyMap[i].PolicyValue
+
+			policyQuery := dupeDnsrecord.Query()
+			dnsrecordTrafficPolicies := DnsRecordSetTrafficPolicyManager.Query().SubQuery()
+			dnstrafficPolicy := DnsTrafficPolicyManager.Query().SubQuery()
+
+			policyQuery = policyQuery.LeftJoin(dnsrecordTrafficPolicies, sqlchemy.Equals(dupeDnsrecord.Field("id"), dnsrecordTrafficPolicies.Field("dns_recordset_id")))
+			policyQuery = policyQuery.LeftJoin(dnstrafficPolicy, sqlchemy.Equals(dnsrecordTrafficPolicies.Field("dns_traffic_policy_id"), dnstrafficPolicy.Field("id")))
+			policyQuery = policyQuery.Filter(sqlchemy.OR(
+				sqlchemy.IsNullOrEmpty(dnsrecordTrafficPolicies.Field("dns_traffic_policy_id")), // 无trafficPolicy的重复dnsrecord
+				sqlchemy.Contains(dnstrafficPolicy.Field("name"), "Simple"),                     // 简单 trafficPolicy的重复dnsrecord
+				sqlchemy.AND(
+					sqlchemy.Equals(dnstrafficPolicy.Field("provider"), provider), // 只看相同provider
+					sqlchemy.OR(
+						sqlchemy.NotEquals(dnstrafficPolicy.Field("policy_type"), policyType), // policy类型不同的重复dnsrecord
+						sqlchemy.AND(
+							sqlchemy.Equals(dnstrafficPolicy.Field("policy_type"), policyType), // policy类型相同但未通过policyvalue区分的的重复dnsrecord
+							sqlchemy.Equals(dnstrafficPolicy.Field("policy_value"), policyValue)),
+					),
+				),
+			))
+			policyQuery.DebugQuery()
+			cnt, err = policyQuery.CountWithError()
+			if err != nil {
+				return input, httperrors.NewGeneralError(err)
+			}
+			if cnt > 0 {
+				return input, httperrors.NewNotSupportedError("duplicated dnsrecord with existed dnsrecord can not distinguish by %s policy", provider)
+			}
+		}
+	} else {
+		cnt, err = policyQuery.CountWithError()
 		if err != nil {
-			return input, httperrors.NewGeneralError(errors.Wrap(err, "db.FetchModelObjects"))
+			return input, httperrors.NewGeneralError(err)
 		}
-		if len(policies) < 1 || (len(input.TrafficPolicies) < 1 && len(oldPolicies) < 1) {
-			return input, httperrors.NewNotSupportedError("duplicated dnsrecord with simple policy not support")
-		}
-		for j := range policies {
-			for k := range input.TrafficPolicies {
-				if policies[j].Provider != input.TrafficPolicies[k].Provider {
-					continue
-				}
-				if strings.Contains(policies[j].Name, "Simple") ||
-					strings.Contains(input.TrafficPolicies[k].PolicyType, "Simple") ||
-					policies[j].PolicyType != input.TrafficPolicies[k].PolicyType {
-					return input, httperrors.NewNotSupportedError("duplicated dnsrecord with different policyType not support")
-				}
-				if policies[j].PolicyValue == input.TrafficPolicies[k].PolicyValue {
-					return input, httperrors.NewNotSupportedError("duplicated dnsrecord with same policyValue not support")
-				}
-			}
-			if len(input.TrafficPolicies) < 1 {
-				for k := range oldPolicies {
-					if policies[j].Provider != input.TrafficPolicies[k].Provider {
-						continue
-					}
-					if strings.Contains(policies[j].Name, "Simple") ||
-						strings.Contains(oldPolicies[k].PolicyType, "Simple") ||
-						policies[j].PolicyType != oldPolicies[k].PolicyType {
-						return input, httperrors.NewNotSupportedError("duplicated dnsrecord with different policyType not support")
-					}
-					if policies[j].PolicyValue == oldPolicies[k].PolicyValue {
-						return input, httperrors.NewNotSupportedError("duplicated dnsrecord with same policyValue not support")
-					}
-				}
-			}
+		if cnt > 0 {
+			return input, httperrors.NewNotSupportedError("duplicated dnsrecord with existed dnsrecord not support")
 		}
 	}
 	return input, nil

pkg/compute/models/dns_zones.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"strings"
 	"time"
 
 	"yunion.io/x/jsonutils"
@@ -127,6 +128,9 @@ func (manager *SDnsZoneManager) ValidateCreateData(ctx context.Context, userCred
 			}
 			input.CloudaccountId = account.GetId()
 		}
+		if !strings.ContainsRune(input.Name, '.') {
+			return input, httperrors.NewNotSupportedError("top level public domain name %s not support", input.Name)
+		}
 	default:
 		return input, httperrors.NewInputParameterError("unknown zone type %s", input.ZoneType)
 	}
@@ -469,6 +473,7 @@ func (self *SDnsZone) newCache(ctx context.Context, userCred mcclient.TokenCrede
 	cache.CloudaccountId = accountId
 	cache.DnsZoneId = self.Id
 	if ext != nil {
+		cache.Status = ext.GetStatus()
 		cache.ExternalId = ext.GetGlobalId()
 		cache.ProductType = string(ext.GetDnsProductType())
 	}

pkg/compute/models/loadbalancerresource.go

@@ -167,23 +167,28 @@ func (manager *SLoadbalancerResourceBaseManager) FetchCustomizeColumns(
 
 	vpcList := make([]interface{}, len(rows))
 	zoneList := make([]interface{}, len(rows))
+	manList := make([]interface{}, len(rows))
 	for i := range rows {
 		rows[i] = api.LoadbalancerResourceInfo{}
 		if lb, ok := lbs[lbIds[i]]; ok {
 			rows[i].Loadbalancer = lb.Name
 			rows[i].VpcId = lb.VpcId
 			rows[i].ZoneId = lb.ZoneId
+			rows[i].ManagerId = lb.ManagerId
 		}
 		vpcList[i] = &SVpcResourceBase{rows[i].VpcId}
 		zoneList[i] = &SZoneResourceBase{rows[i].ZoneId}
+		manList[i] = &SManagedResourceBase{rows[i].ManagerId}
 	}
 
 	vpcRows := manager.SVpcResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, vpcList, fields, isList)
 	zoneRows := manager.SZoneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, zoneList, fields, isList)
+	manRows := manager.SManagedResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, manList, fields, isList)
 
 	for i := range rows {
 		rows[i].VpcResourceInfo = vpcRows[i]
-		rows[i].ZoneResourceInfoBase = zoneRows[i].ZoneResourceInfoBase
+		rows[i].ZoneResourceInfo = zoneRows[i]
+		rows[i].ManagedResourceInfo = manRows[i]
 	}
 	return rows
 }

pkg/compute/models/networks.go

@@ -1942,10 +1942,10 @@ func (manager *SNetworkManager) ListItemFilter(
 		q = q.In("ifname_hint", input.IfnameHint)
 	}
 	if len(input.GuestIpStart) > 0 {
-		q = q.In("guest_ip_start", input.GuestIpStart)
+		q = q.Filter(sqlchemy.ContainsAny(q.Field("guest_ip_start"), input.GuestIpStart))
 	}
 	if len(input.GuestIpEnd) > 0 {
-		q = q.In("guest_ip_end", input.GuestIpEnd)
+		q = q.Filter(sqlchemy.ContainsAny(q.Field("guest_ip_end"), input.GuestIpEnd))
 	}
 	if len(input.GuestIpMask) > 0 {
 		q = q.In("guest_ip_mask", input.GuestIpMask)
@@ -2206,9 +2206,17 @@ func (self *SNetwork) PerformMerge(ctx context.Context, userCred mcclient.TokenC
 	ipSS, _ := netutils.NewIPV4Addr(self.GuestIpStart)
 	ipSE, _ := netutils.NewIPV4Addr(self.GuestIpEnd)
 
-	if ipNE.StepUp() == ipSS {
+	wireNets := make([]SNetwork, 0)
+	q := NetworkManager.Query().Equals("wire_id", self.WireId).NotEquals("id", self.Id).NotEquals("id", net.Id)
+	err = db.FetchModelObjects(NetworkManager, q, &wireNets)
+	if err != nil && errors.Cause(err) != sql.ErrNoRows {
+		logclient.AddActionLogWithContext(ctx, self, logclient.ACT_MERGE, err.Error(), userCred, false)
+		return nil, errors.Wrap(err, "Query nets of same wire")
+	}
+
+	if ipNE.StepUp() == ipSS || (ipNE.StepUp() < ipSS && !isOverlapNetworks(wireNets, ipNE.StepUp(), ipSS.StepDown())) {
 		startIp, endIp = net.GuestIpStart, self.GuestIpEnd
-	} else if ipSE.StepUp() == ipNS {
+	} else if ipSE.StepUp() == ipNS || (ipSE.StepUp() < ipNS && !isOverlapNetworks(wireNets, ipSE.StepUp(), ipNS.StepDown())) {
 		startIp, endIp = self.GuestIpStart, net.GuestIpEnd
 	} else {
 		note := "Incontinuity Network for %s and %s"

pkg/compute/models/wires.go

@@ -28,6 +28,7 @@ import (
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
+	"yunion.io/x/onecloud/pkg/apis"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
@@ -362,8 +363,9 @@ func (self *SWire) syncWithCloudWire(ctx context.Context, userCred mcclient.Toke
 
 		if self.IsEmulated {
 			self.DomainId = vpc.DomainId
-			self.IsPublic = vpc.IsPublic
-			self.PublicScope = vpc.PublicScope
+			// self.IsPublic = vpc.IsPublic
+			// self.PublicScope = vpc.PublicScope
+			// self.PublicSrc = vpc.PublicSrc
 		}
 
 		return nil
@@ -375,6 +377,8 @@ func (self *SWire) syncWithCloudWire(ctx context.Context, userCred mcclient.Toke
 	if provider != nil && !self.IsEmulated {
 		SyncCloudDomain(userCred, self, provider.GetOwnerId())
 		self.SyncShareState(ctx, userCred, provider.getAccountShareInfo())
+	} else if self.IsEmulated {
+		self.SaveSharedInfo(apis.TOwnerSource(vpc.PublicSrc), ctx, userCred, vpc.GetSharedInfo())
 	}
 
 	db.OpsLog.LogSyncUpdate(self, diff, userCred)
@@ -418,6 +422,7 @@ func (manager *SWireManager) newFromCloudWire(ctx context.Context, userCred mccl
 	wire.DomainId = vpc.DomainId
 	wire.IsPublic = vpc.IsPublic
 	wire.PublicScope = vpc.PublicScope
+	wire.PublicSrc = vpc.PublicSrc
 
 	err = manager.TableSpec().Insert(ctx, &wire)
 	if err != nil {

pkg/compute/policy/defaults.go

@@ -131,6 +131,30 @@ var (
 					Action:   PolicyActionList,
 					Result:   rbacutils.Allow,
 				},
+				{
+					Service:  api.SERVICE_TYPE,
+					Resource: "dns_recordsets",
+					Action:   PolicyActionList,
+					Result:   rbacutils.Allow,
+				},
+				{
+					Service:  api.SERVICE_TYPE,
+					Resource: "dns_recodsets",
+					Action:   PolicyActionGet,
+					Result:   rbacutils.Allow,
+				},
+				{
+					Service:  api.SERVICE_TYPE,
+					Resource: "dns_zonecaches",
+					Action:   PolicyActionList,
+					Result:   rbacutils.Allow,
+				},
+				{
+					Service:  api.SERVICE_TYPE,
+					Resource: "dns_zonecaches",
+					Action:   PolicyActionGet,
+					Result:   rbacutils.Allow,
+				},
 			},
 		},
 		{

pkg/compute/tasks/dnszone_syncstatus_task.go

@@ -51,9 +51,15 @@ func (self *DnsZoneSyncstatusTask) OnInit(ctx context.Context, obj db.IStandalon
 
 	for i := range caches {
 		if len(caches[i].ExternalId) > 0 {
-			_, err := caches[i].GetICloudDnsZone()
+			status := api.DNS_ZONE_CACHE_STATUS_UNKNOWN
+			iZone, err := caches[i].GetICloudDnsZone()
 			if err != nil {
 				logclient.AddActionLogWithContext(ctx, &caches[i], logclient.ACT_SYNC_STATUS, errors.Wrapf(err, "GetICloudDnsZone"), self.UserCred, false)
+			} else {
+				status = iZone.GetStatus()
+			}
+			if caches[i].Status != status {
+				caches[i].SetStatus(self.GetUserCred(), status, "")
 			}
 		}
 	}

pkg/dns/dns.go

@@ -87,7 +87,8 @@ type SRegionDNS struct {
 	Region        string
 	K8sSkip       bool
 
-	K8sManager *k8s.SKubeClusterManager
+	K8sManager            *k8s.SKubeClusterManager
+	primaryZoneLabelCount int
 }
 
 func New() *SRegionDNS {
@@ -402,9 +403,13 @@ func (r *SRegionDNS) queryLocalDnsRecords(req *recordRequest) (recs []msg.Servic
 }
 
 func (r *SRegionDNS) isMyDomain(req *recordRequest) bool {
-	zones := []string{fmt.Sprintf("%s.", r.PrimaryZone)}
-	zone := plugin.Zones(zones).Matches(req.state.Name())
-	if zone != "" {
+	qname := req.state.Name()
+	qnameLabelCount := dns.CountLabel(qname)
+	if qnameLabelCount <= r.primaryZoneLabelCount {
+		return false
+	}
+	matched := dns.CompareDomainName(r.PrimaryZone, qname)
+	if matched == r.primaryZoneLabelCount {
 		return true
 	}
 	return false

pkg/dns/setup.go

@@ -21,6 +21,7 @@ import (
 	"github.com/coredns/coredns/plugin"
 	"github.com/coredns/coredns/plugin/pkg/upstream"
 	"github.com/mholt/caddy"
+	"github.com/miekg/dns"
 
 	"yunion.io/x/pkg/util/regutils"
 )
@@ -44,6 +45,10 @@ func setup(c *caddy.Controller) error {
 	if !regutils.MatchDomainName(rDNS.PrimaryZone) {
 		return fmt.Errorf("dns_domain %q invalid", rDNS.PrimaryZone)
 	}
+	if r := rDNS.PrimaryZone[len(rDNS.PrimaryZone)-1]; r != '.' {
+		rDNS.PrimaryZone += "."
+	}
+	rDNS.primaryZoneLabelCount = dns.CountLabel(rDNS.PrimaryZone)
 
 	err = rDNS.initDB(c)
 	if err != nil {

pkg/mcclient/README.md

@@ -1,2 +1,43 @@
-Yunion CMP API go library
-==========================
+Yunion OneCloud API go library
+===============================
+
+
+Sample code
+
+    :::golang
+    package main
+
+    import (
+        "context"
+        "fmt"
+
+        "yunion.io/x/onecloud/pkg/mcclient"
+        "yunion.io/x/onecloud/pkg/mcclient/modules"
+    )
+
+    func main() {
+        client := mcclient.NewClient("https://<onecloud_controller_ip>:30500/v3",
+            60,
+            true,
+            true,
+            "",
+            "")
+        token, err := client.Authenticate("sysadmin", "<password>", "Default", "system", "Default")
+        if err != nil {
+            panic(err)
+        }
+        s := client.NewSession(context.Background(),
+            "region0",
+            "",
+            "PublicURL",
+            token,
+            "")
+
+        result, err := modules.Servers.List(s, nil)
+        if err != nil {
+            panic(err)
+        }
+        fmt.Printf("%#v\n", result)
+    }
+
+

pkg/mcclient/modules/k8s/fed_rolebinding_cluster.go

@@ -29,7 +29,7 @@ func init() {
 		"federatedrolebindingclusters",
 		NewFedJointClusterCols(),
 		NewColumns(),
-		FederatedRoles,
+		FederatedRoleBindings,
 		KubeClusters,
 	)
 	modules.Register(&FederatedRoleBindingClusters)

pkg/mcclient/modules/mod_commonalert.go

@@ -21,7 +21,7 @@ func init() {
 
 func NewCommonAlertManager() *SCommonAlertManager {
 	man := NewMonitorV2Manager("commonalert", "commonalerts",
-		[]string{"id", "name", "level", "alert_type", "period", "recipients", "channel"},
+		[]string{"id", "name", "enabled", "level", "alert_type", "period", "recipients", "channel"},
 		[]string{})
 	return &SCommonAlertManager{
 		ResourceManager: &man,

pkg/mcclient/modules/mod_suggestsysrule.go

@@ -51,7 +51,7 @@ type SInfluxdbShemaManager struct {
 
 func NewSuggestSysRuleManager() *SSuggestSysRuleManager {
 	man := NewMonitorV2Manager("suggestsysrule", "suggestsysrules",
-		[]string{"id", "name", "type", "enabled", "setting"},
+		[]string{"id", "name", "type", "enabled", "period", "time_from", "setting"},
 		[]string{})
 	return &SSuggestSysRuleManager{
 		ResourceManager: &man,

pkg/mcclient/options/monitor/commonalert.go

@@ -0,0 +1,43 @@
+package monitor
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+type CommonAlertListOptions struct {
+	options.BaseListOptions
+	// 报警类型
+	AlertType string `help:"common alert type" choices:"normal|system"`
+	Level     string `help:"common alert notify level" choices:"normal|important|fatal"`
+}
+
+func (o *CommonAlertListOptions) Params() (jsonutils.JSONObject, error) {
+	return options.ListStructToParams(o)
+}
+
+type CommonAlertShowOptions struct {
+	ID string `help:"ID of alart " json:"-"`
+}
+
+func (o *CommonAlertShowOptions) Params() (jsonutils.JSONObject, error) {
+	return options.StructToParams(o)
+}
+
+func (o *CommonAlertShowOptions) GetId() string {
+	return o.ID
+}
+
+type CommonAlertDeleteOptions struct {
+	ID    string `help:"ID of alart"`
+	Force bool   `help:"force to delete alert"`
+}
+
+func (o *CommonAlertDeleteOptions) GetId() string {
+	return o.ID
+}
+
+func (o *CommonAlertDeleteOptions) Params() (jsonutils.JSONObject, error) {
+	return options.StructToParams(o)
+}

pkg/mcclient/options/monitor/suggestsysrule.go

@@ -27,10 +27,36 @@ type SuggestRuleListOptions struct {
 	options.BaseListOptions
 }
 
+func (o *SuggestRuleListOptions) Params() (jsonutils.JSONObject, error) {
+	return options.ListStructToParams(o)
+}
+
 type SuggestRuleShowOptions struct {
 	ID string `help:"ID or name of the alert" json:"-"`
 }
 
+func (o *SuggestRuleShowOptions) Params() (jsonutils.JSONObject, error) {
+	return options.StructToParams(o)
+}
+
+func (o *SuggestRuleShowOptions) GetId() string {
+	return o.ID
+}
+
+type SuggestRuleConfigOptions struct {
+	ID       string `help:"ID or name of the alert" json:"-"`
+	Period   string `help:"Period of suggest rule e.g. '5s', '1m'"`
+	TimeFrom string `help:"TimeFrom of suggest rule e.g. '24h'"`
+}
+
+func (o *SuggestRuleConfigOptions) Params() (jsonutils.JSONObject, error) {
+	return options.StructToParams(o)
+}
+
+func (o *SuggestRuleConfigOptions) GetId() string {
+	return o.ID
+}
+
 type SuggestSysRuleAlertSettingOptions struct {
 	Status string `help:"Status of eip_unused rule"`
 }
@@ -43,7 +69,7 @@ type SuggestRuleCreateOptions struct {
 	Period  string `help:"Period of suggest rule e.g. '5s', '1m'" default:"30s"`
 }
 
-func (opt SuggestRuleCreateOptions) Params() (jsonutils.JSONObject, error) {
+func (opt *SuggestRuleCreateOptions) Params() (jsonutils.JSONObject, error) {
 	input := new(monitor.SuggestSysRuleCreateInput)
 	input.Name = opt.Name
 	input.Period = opt.Period
@@ -119,5 +145,13 @@ func newSuggestSysAlertSetting(tp string) *monitor.SSuggestSysAlertSetting {
 }
 
 type SuggestRuleDeleteOptions struct {
-	ID []string `help:"ID of alert to delete"`
+	ID string `help:"ID of alert to delete"`
+}
+
+func (o *SuggestRuleDeleteOptions) GetId() string {
+	return o.ID
+}
+
+func (o *SuggestRuleDeleteOptions) Params() (jsonutils.JSONObject, error) {
+	return options.StructToParams(o)
 }

pkg/mcclient/options/network.go

@@ -40,6 +40,9 @@ type NetworkListOptions struct {
 	IsClassic   *bool `help:"search classic on-premise network"`
 
 	Status string `help:"filter by network status"`
+
+	GuestIpStart []string `help:"search by guest_ip_start"`
+	GuestIpEnd   []string `help:"search by guest_ip_end"`
 }
 
 func (opts *NetworkListOptions) GetContextId() string {

pkg/monitor/alerting/conditions/query.go

@@ -209,16 +209,16 @@ func (c *QueryCondition) NewEvalMatch(context *alerting.EvalContext, series tsdb
 	if len(queryKeyInfo) == 0 {
 		queryKeyInfo = evalMatch.Metric
 	}
-	msg := fmt.Sprintf("%s.%s %s %.4f", alertDetails.Measurement, alertDetails.Field,
-		alertDetails.Comparator, alertDetails.Threshold)
+	evalMatch.Unit = alertDetails.FieldDescription.Unit
+	msg := fmt.Sprintf("%s.%s %s %s", alertDetails.Measurement, alertDetails.Field,
+		alertDetails.Comparator, c.RationalizeValueFromUnit(alertDetails.Threshold, evalMatch.Unit, ""))
 	if len(context.Rule.Message) == 0 {
 		context.Rule.Message = msg
 	}
 	evalMatch.Condition = c.GenerateFormatCond(meta, queryKeyInfo).String()
 	evalMatch.Tags = c.filterTags(series.Tags, *alertDetails)
-	evalMatch.Unit = alertDetails.FieldDescription.Unit
 	evalMatch.Value = value
-	evalMatch.ValueStr = c.RationalizeValueFromUnit(*value, alertDetails.FieldDescription.Unit)
+	evalMatch.ValueStr = c.RationalizeValueFromUnit(*value, alertDetails.FieldDescription.Unit, alertDetails.FieldOpt)
 	evalMatch.MeasurementDesc = alertDetails.MeasurementDisplayName
 	evalMatch.FieldDesc = alertDetails.FieldDescription.DisplayName
 	return evalMatch, nil
@@ -226,13 +226,16 @@ func (c *QueryCondition) NewEvalMatch(context *alerting.EvalContext, series tsdb
 
 var fileSize = []string{"bps", "Bps", "byte"}
 
-func (c *QueryCondition) RationalizeValueFromUnit(value float64, unit string) string {
+func (c *QueryCondition) RationalizeValueFromUnit(value float64, unit string, opt string) string {
 	if utils.IsInStringArray(unit, fileSize) {
 		if unit == "byte" {
 			return (formatFileSize(value, unit, float64(1024)))
 		}
 		return formatFileSize(value, unit, float64(1000))
 	}
+	if unit == "%" && monitor.CommonAlertFieldOpt_Division == opt {
+		return fmt.Sprintf("%0.4f %s", value*100, unit)
+	}
 	return fmt.Sprintf("%0.4f %s", value, unit)
 }
 

pkg/monitor/dbinit/metric_dbinit.go

@@ -654,3 +654,6 @@ var MetricDescriptions = `
 	}
 ]
 `
+var MetricNeedDeleteDescriptions = []string{"rds_conn", "rds_cpu", "rds_mem", "rds_netio", "rds_disk", "dcs_cpu",
+	"dcs_mem", "dcs_netio", "dcs_conn", "dcs_instantopt", "dcs_cachekeys", "dcs_datamem", "oss_latency",
+	"oss_netio", "oss_req"}

pkg/monitor/models/commonalert.go

@@ -112,7 +112,7 @@ func (man *SCommonAlertManager) ValidateCreateData(
 			if query.Threshold == 0 {
 				return data, httperrors.NewInputParameterError("threshold is meaningless")
 			}
-			if strings.Contains(query.To, "now-") {
+			if strings.Contains(query.From, "now-") || strings.Contains(query.To, "now") {
 				query.To = "now"
 				query.From = "1h"
 			}
@@ -651,8 +651,9 @@ func (alert *SCommonAlert) ValidateUpdateData(
 			data.Set("frequency", jsonutils.NewInt(freqSpec))
 		}
 	}
-	if channel, _ := data.GetArray("channel"); len(channel) > 0 {
+	if recipients, _ := data.GetArray("recipients"); len(recipients) > 0 {
 		channelStr, _ := data.GetString("channel")
+		channel, _ := data.GetArray("channel")
 		if !strings.Contains(channelStr, monitor.DEFAULT_SEND_NOTIFY_CHANNEL) {
 			channels := jsonutils.NewArray()
 			channels.Add(channel...)
@@ -676,7 +677,7 @@ func (alert *SCommonAlert) ValidateUpdateData(
 			if query.Threshold == 0 {
 				return data, httperrors.NewInputParameterError("threshold is meaningless")
 			}
-			if strings.Contains(query.To, "now-") {
+			if strings.Contains(query.From, "now-") || strings.Contains(query.To, "now") {
 				query.To = "now"
 				query.From = "1h"
 			}

pkg/monitor/models/metric.go

@@ -75,6 +75,10 @@ func (man *SMetricMeasurementManager) ValidateCreateData(
 	ctx context.Context, userCred mcclient.TokenCredential,
 	ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject,
 	data monitor.MetricCreateInput) (monitor.MetricMeasurementCreateInput, error) {
+	enable := true
+	if data.Measurement.Enabled == nil {
+		data.Measurement.Enabled = &enable
+	}
 	return data.Measurement, nil
 }
 
@@ -96,11 +100,11 @@ func (measurement *SMetricMeasurement) CustomizeCreate(
 	for _, fieldInput := range input.MetricFields {
 		field, err := measurement.SaveMetricField(ctx, userCred, ownerId, fieldInput)
 		if err != nil {
-			return err
+			return errors.Wrap(err, "SMetricMeasurement CustomizeCreate to save field error")
 		}
 		err = measurement.attachMetricField(ctx, userCred, field)
 		if err != nil {
-			return err
+			return errors.Wrap(err, "attachMetricField error")
 		}
 	}
 	return nil
@@ -376,53 +380,6 @@ func (man *SMetricMeasurementManager) Run(ctx context.Context) error {
 	return nil
 }
 
-func (manager *SMetricMeasurementManager) initMeasurementDatabase(ctx context.Context) (err error) {
-	databases, err := DataSourceManager.GetDatabases()
-	if err != nil {
-		return err
-	}
-	databaseArr, err := databases.GetArray("databases")
-	if err != nil {
-		return err
-	}
-	databaseGroup, _ := errgroup.WithContext(ctx)
-	for dIndex, _ := range databaseArr {
-		databaseTmp := databaseArr[dIndex]
-		databaseStr, _ := databaseTmp.GetString()
-		databaseGroup.Go(manager.getMeasurementAsyn(ctx, databaseStr))
-	}
-	err = databaseGroup.Wait()
-	return
-}
-
-func (manager *SMetricMeasurementManager) getMeasurementAsyn(ctx context.Context, database string) func() error {
-	return func() error {
-		query := jsonutils.NewDict()
-		query.Add(jsonutils.NewString(database), "database")
-
-		measurements, err := DataSourceManager.GetMeasurementsWithOutTimeFilter(query, "", "")
-		if err != nil {
-			return err
-		}
-		measurementArr, err := measurements.GetArray("measurements")
-		if err != nil {
-			return err
-		}
-		metrics := make([]monitor.MetricCreateInput, 0)
-		for mIndex, _ := range measurementArr {
-			measurementStr, _ := measurementArr[mIndex].GetString("measurement")
-			metric := monitor.MetricCreateInput{}
-			metricMea := monitor.MetricMeasurementCreateInput{}
-			metricMea.Name = measurementStr
-			metricMea.Database = database
-			metric.Measurement = metricMea
-			metric.MetricFields = make([]monitor.MetricFieldCreateInput, 0)
-			metrics = append(metrics, metric)
-		}
-		return manager.initMetrics(ctx, metrics)
-	}
-}
-
 func (manager *SMetricMeasurementManager) initJsonMetricInfo(ctx context.Context) error {
 	metricDescriptions, err := jsonutils.ParseString(dbinit.MetricDescriptions)
 	if err != nil {
@@ -433,7 +390,31 @@ func (manager *SMetricMeasurementManager) initJsonMetricInfo(ctx context.Context
 	if err != nil {
 		return errors.Wrap(err, "SMetricMeasurementManager Unmarshal MetricDescriptionstr error")
 	}
-	return manager.initMetrics(ctx, metrics)
+	err = manager.initMetrics(ctx, metrics)
+	if err != nil {
+		return err
+	}
+	return manager.deleteUnusedMetricDescriptions()
+}
+
+func (manager *SMetricMeasurementManager) deleteUnusedMetricDescriptions() error {
+	metricMeasurements, err := manager.getMeasurementByName(dbinit.MetricNeedDeleteDescriptions...)
+	if err != nil {
+		return err
+	}
+	userCred := auth.AdminCredential()
+	for i, _ := range metricMeasurements {
+		err := (&metricMeasurements[i]).CustomizeDelete(context.Background(), userCred, jsonutils.NewDict(),
+			jsonutils.NewDict())
+		if err != nil {
+			return errors.Wrap(err, "init deleteUnusedMetricDescriptions error")
+		}
+		err = (&metricMeasurements[i]).Delete(context.Background(), userCred)
+		if err != nil {
+			return errors.Wrap(err, "init deleteUnusedMetricDescriptions error")
+		}
+	}
+	return nil
 }
 
 func (manager *SMetricMeasurementManager) initMetrics(ctx context.Context, metrics []monitor.MetricCreateInput) (err error) {
@@ -472,6 +453,7 @@ func (manager *SMetricMeasurementManager) initMeasurementAndFieldInfo(createInpu
 	if len(measurements) != 0 {
 		unInsertFields, updateFields, deleteFields = measurements[0].getInsertAndUpdateFields(userCred, createInput)
 	}
+
 	if len(measurements) == 0 {
 		_, err := db.DoCreate(manager, context.Background(), userCred, jsonutils.NewDict(),
 			jsonutils.Marshal(&createInput),

pkg/monitor/models/metric_field.go

@@ -6,6 +6,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/tristate"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
@@ -152,15 +153,23 @@ func (man *SMetricFieldManager) OrderByExtraFields(
 
 func (manager *SMetricFieldManager) SaveMetricField(ctx context.Context, userCred mcclient.TokenCredential,
 	ownerId mcclient.IIdentityProvider, fieldInput monitor.MetricFieldCreateInput) (*SMetricField, error) {
-	obj, err := db.DoCreate(manager, ctx, userCred, nil, fieldInput.JSON(&fieldInput), userCred)
-	if err != nil {
-		return nil, errors.Wrapf(err, "SaveMetricField error input: %s", fieldInput.JSON(&fieldInput))
+	field := new(SMetricField)
+	field.Name = fieldInput.Name
+	field.DisplayName = fieldInput.DisplayName
+	field.Description = fieldInput.Description
+	field.Unit = fieldInput.Unit
+	field.ValueType = fieldInput.ValueType
+	field.Enabled = tristate.True
+	field.SetModelManager(manager, field)
+	if err := manager.TableSpec().Insert(ctx, field); err != nil {
+		return nil, errors.Wrapf(err, "insert config %#v", field)
 	}
-	return obj.(*SMetricField), nil
+
+	return field, nil
 }
 
-func (man *SMetricFieldManager) GetField(id string) (*SMetricField, error) {
-	obj, err := man.FetchById(id)
+func (man *SMetricFieldManager) GetFieldByIdOrName(id string, userCred mcclient.TokenCredential) (*SMetricField, error) {
+	obj, err := man.FetchByIdOrName(userCred, id)
 	if err != nil {
 		if errors.Cause(err) == sql.ErrNoRows {
 			return nil, nil

pkg/monitor/models/metric_joint.go

@@ -5,6 +5,7 @@ import (
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
 )
 
 type SMetricManager struct {
@@ -51,7 +52,7 @@ func (metric *SMetric) DoSave(ctx context.Context) error {
 }
 
 func (self *SMetric) GetMetricField() (*SMetricField, error) {
-	return MetricFieldManager.GetField(self.FieldId)
+	return MetricFieldManager.GetFieldByIdOrName(self.FieldId, auth.AdminCredential())
 }
 
 func (joint *SMetric) Detach(ctx context.Context, userCred mcclient.TokenCredential) error {

pkg/monitor/models/suggestsysrule.go

@@ -99,6 +99,9 @@ func (rule *SSuggestSysRule) GetType() monitor.SuggestDriverType {
 //根据数据库中查询得到的信息进行适配转换，同时更新drivers中的内容
 func (rule *SSuggestSysRule) getSuggestSysAlertSetting() (*monitor.SSuggestSysAlertSetting, error) {
 	setting := new(monitor.SSuggestSysAlertSetting)
+	if rule.Setting == nil {
+		rule.Setting = jsonutils.NewDict()
+	}
 	err := rule.Setting.Unmarshal(setting)
 	if err != nil {
 		return nil, errors.Wrap(err, "SSuggestSysRule getSuggestSysAlertSetting error")
@@ -177,6 +180,8 @@ func (man *SSuggestSysRuleManager) ValidateCreateData(
 			if err := dri.ValidateSetting(data.Setting); err != nil {
 				return data, errors.Wrap(err, "validate setting error")
 			}
+		} else {
+			data.Setting = new(monitor.SSuggestSysAlertSetting)
 		}
 	}
 	return data, nil
@@ -306,6 +311,42 @@ func (self *SSuggestSysRule) PerformDisable(ctx context.Context, userCred mcclie
 	return nil, nil
 }
 
+func (self *SSuggestSysRule) AllowPerformConfig(ctx context.Context, userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return db.IsAdminAllowPerform(userCred, self, "config")
+}
+
+func (self *SSuggestSysRule) PerformConfig(ctx context.Context, userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	period, _ := data.GetString("period")
+	timeFrom, _ := data.GetString("time_from")
+	if len(period) != 0 {
+		period = parseDuration(period)
+		if _, err := time.ParseDuration(period); err != nil {
+			return data, httperrors.NewInputParameterError("Invalid period format: %s", period)
+		}
+	}
+	if len(timeFrom) != 0 {
+		timeFrom = parseDuration(timeFrom)
+		if _, err := time.ParseDuration(timeFrom); err != nil {
+			return data, httperrors.NewInputParameterError("Invalid time_from format: %s", timeFrom)
+		}
+	}
+	db.Update(self, func() error {
+		if len(period) != 0 {
+			self.Period = period
+		}
+		if len(timeFrom) != 0 {
+			self.TimeFrom = timeFrom
+
+		}
+		return nil
+	})
+	db.OpsLog.LogEvent(self, "modifyconfig", "", userCred)
+	self.updateCronjob()
+	return nil, nil
+}
+
 func (self *SSuggestSysRuleManager) AllowGetPropertyRuleType(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
 	return true
 }

pkg/monitor/tsdb/datasource.go

@@ -77,7 +77,7 @@ func (ds *DataSource) GetHttpClient() (*http.Client, error) {
 	}
 
 	return &http.Client{
-		Timeout:   30 * time.Second,
+		//Timeout:   30 * time.Second,
 		Transport: transport,
 	}, nil
 }

pkg/util/rbacutils/rbac.go

@@ -655,6 +655,7 @@ type sSimpleRbacIdentity struct {
 	domainId    string
 	projectName string
 	roleNames   []string
+	loginIp     string
 }
 
 func (id sSimpleRbacIdentity) GetProjectDomainId() string {
@@ -670,7 +671,7 @@ func (id sSimpleRbacIdentity) GetProjectName() string {
 }
 
 func (id sSimpleRbacIdentity) GetLoginIp() string {
-	return ""
+	return id.loginIp
 }
 
 func (id sSimpleRbacIdentity) GetTokenString() string {
@@ -678,10 +679,15 @@ func (id sSimpleRbacIdentity) GetTokenString() string {
 }
 
 func NewRbacIdentity(domainId, projectName string, roleNames []string) IRbacIdentity {
+	return NewRbacIdentity2(domainId, projectName, roleNames, "")
+}
+
+func NewRbacIdentity2(domainId, projectName string, roleNames []string, loginIp string) IRbacIdentity {
 	return sSimpleRbacIdentity{
 		domainId:    domainId,
 		projectName: projectName,
 		roleNames:   roleNames,
+		loginIp:     loginIp,
 	}
 }