fix: misc

cmd/climc/shell/quotas.go

@@ -37,8 +37,8 @@ type QuotaBaseOptions struct {
 
 func init() {
 	type QuotaOptions struct {
-		Tenant string `help:"Tenant name or ID"`
-		Domain string `help:"Domain name or ID"`
+		Tenant        string `help:"Tenant name or ID"`
+		ProjectDomain string `help:"Domain name or ID"`
 	}
 	R(&QuotaOptions{}, "quota", "Show quota for current user or tenant", func(s *mcclient.ClientSession, args *QuotaOptions) error {
 		params := jsonutils.Marshal(args)
@@ -60,8 +60,8 @@ func init() {
 	})
 
 	type QuotaSetOptions struct {
-		Tenant string `help:"Tenant name or ID to set quota" json:"tenant,omitempty"`
-		Domain string `help:"Domain name or ID to set quota" json:"domain,omitempty"`
+		Tenant        string `help:"Tenant name or ID to set quota" json:"tenant,omitempty"`
+		ProjectDomain string `help:"Domain name or ID to set quota" json:"domain,omitempty"`
 		QuotaBaseOptions
 	}
 	R(&QuotaSetOptions{}, "quota-set", "Set quota for tenant", func(s *mcclient.ClientSession, args *QuotaSetOptions) error {

pkg/cloudcommon/db/fetch.go

@@ -100,7 +100,7 @@ func FetchByName(manager IModelManager, userCred mcclient.IIdentityProvider, idS
 	if err != nil {
 		return nil, err
 	}
-	if count > 1 && userCred != nil {
+	if count > 0 && userCred != nil {
 		q = manager.FilterByOwner(q, userCred, manager.NamespaceScope())
 		q = manager.FilterBySystemAttributes(q, nil, nil, manager.ResourceScope())
 		count, err = q.CountWithError()
@@ -183,7 +183,7 @@ func fetchItemByName(manager IModelManager, ctx context.Context, userCred mcclie
 	if err != nil {
 		return nil, err
 	}
-	if count > 1 {
+	if count > 0 {
 		q = manager.FilterByOwner(q, userCred, manager.NamespaceScope())
 		q = manager.FilterBySystemAttributes(q, nil, nil, manager.ResourceScope())
 		count, err = q.CountWithError()

pkg/cloudcommon/db/quotas/errors.go

@@ -23,7 +23,7 @@ func (es *SOutOfQuotaErrors) Error() string {
 	qs := make([]string, len(es.errors))
 	for i := range es.errors {
 		e := es.errors[i]
-		qs = append(qs, e.Error())
+		qs[i] = e.Error()
 	}
 	return fmt.Sprintf("Out of quota: %s", strings.Join(qs, ", "))
 }

pkg/cloudcommon/db/quotas/handler.go

@@ -16,14 +16,15 @@ package quotas
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"net/http"
+	"strings"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
-	"database/sql"
-	"strings"
 	"yunion.io/x/onecloud/pkg/appctx"
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
@@ -81,18 +82,18 @@ func AddQuotaHandler(manager *SQuotaBaseManager, prefix string, app *appsrv.Appl
 	auth.Authenticate(checkQuotaHanlder), nil, "check_quota", nil)*/
 }
 
-func (manager *SQuotaBaseManager) queryQuota(ctx context.Context, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platforma []string) (*jsonutils.JSONDict, error) {
+func (manager *SQuotaBaseManager) queryQuota(ctx context.Context, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platforma []string) (*jsonutils.JSONDict, IQuota, error) {
 	ret := jsonutils.NewDict()
 
 	quota := manager.newQuota()
 	err := manager.GetQuota(ctx, scope, ownerId, platforma, quota)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	usage := manager.newQuota()
 	err = manager.usageStore.GetQuota(ctx, scope, ownerId, platforma, usage)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	if usage.IsEmpty() {
 		usageChan := make(chan IQuota)
@@ -104,14 +105,21 @@ func (manager *SQuotaBaseManager) queryQuota(ctx context.Context, scope rbacutil
 	pending := manager.newQuota()
 	err = manager.GetPendingUsage(ctx, scope, ownerId, nil, pending)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	ret.Update(quota.ToJSON(""))
 	ret.Update(usage.ToJSON("usage"))
 	ret.Update(pending.ToJSON("pending"))
 
-	return ret, nil
+	if scope == rbacutils.ScopeDomain {
+		total, err := manager.getDomainTotalQuota(ctx, ownerId.GetProjectDomainId(), nil)
+		if err == nil {
+			ret.Update(total.ToJSON("total"))
+		}
+	}
+
+	return ret, usage, nil
 }
 
 func (manager *SQuotaBaseManager) getQuotaHanlder(ctx context.Context, w http.ResponseWriter, r *http.Request) {
@@ -142,7 +150,7 @@ func (manager *SQuotaBaseManager) getQuotaHanlder(ctx context.Context, w http.Re
 		scope = rbacutils.ScopeProject
 	}
 
-	quota, err := manager.queryQuota(ctx, scope, ownerId, nil)
+	quota, _, err := manager.queryQuota(ctx, scope, ownerId, nil)
 
 	if err != nil {
 		httperrors.GeneralServerError(w, err)
@@ -226,6 +234,32 @@ func (manager *SQuotaBaseManager) setQuotaHanlder(ctx context.Context, w http.Re
 		return
 	}
 	oquota.Update(quota)
+
+	if scope == rbacutils.ScopeProject {
+		total, err := manager.getDomainTotalQuota(ctx, ownerId.GetProjectDomainId(), []string{ownerId.GetProjectId()})
+		if err != nil {
+			log.Errorf("get total quota fail %s", err)
+			httperrors.GeneralServerError(w, err)
+			return
+		}
+
+		domainQuota := manager.newQuota()
+		err = manager.GetQuota(ctx, rbacutils.ScopeDomain, ownerId, nil, domainQuota)
+		if err != nil {
+			log.Errorf("GetQuota for domain %s fail %s", ownerId.GetProjectDomainId(), err)
+			httperrors.GeneralServerError(w, err)
+			return
+		}
+
+		total.Add(oquota)
+		err = total.Exceed(quota, domainQuota)
+		if err != nil {
+			log.Errorf("project quota exeed domain quota: %s", err)
+			httperrors.GeneralServerError(w, fmt.Errorf("project quota exeed domain quota: %s", err))
+			return
+		}
+	}
+
 	err = manager.SetQuota(ctx, userCred, scope, ownerId, nil, oquota)
 	if err != nil {
 		log.Errorf("set quota fail %s", err)
@@ -299,6 +333,43 @@ func (manager *SQuotaBaseManager) listProjectQuotaHanlder(ctx context.Context, w
 	appsrv.SendJSON(w, rbody)
 }
 
+func (manager *SQuotaBaseManager) getDomainTotalQuota(ctx context.Context, targetDomainId string, excludes []string) (IQuota, error) {
+	q := manager.Query("domain_id", "tenant_id", "platform")
+	q = q.Equals("domain_id", targetDomainId)
+	q = q.IsNotEmpty("tenant_id")
+	if len(excludes) > 0 {
+		q = q.NotIn("tenant_id", excludes)
+	}
+	rows, err := q.Rows()
+	if err != nil && err != sql.ErrNoRows {
+		return nil, err
+	}
+	defer rows.Close()
+
+	ret := manager.newQuota()
+	for rows.Next() {
+		var domainId, projectId, platformStr string
+		err := rows.Scan(&domainId, &projectId, &platformStr)
+		if err != nil {
+			return nil, errors.Wrap(err, "scan")
+		}
+		scope := rbacutils.ScopeProject
+		owner := db.SOwnerId{
+			DomainId:  domainId,
+			ProjectId: projectId,
+		}
+		platform := strings.Split(platformStr, nameSeparator)
+
+		quota := manager.newQuota()
+		err = manager.GetQuota(ctx, scope, &owner, platform, quota)
+		if err != nil {
+			return nil, errors.Wrap(err, "GetQuota")
+		}
+		ret.Add(quota)
+	}
+	return ret, nil
+}
+
 func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, targetDomainId string) ([]jsonutils.JSONObject, error) {
 	q := manager.Query("domain_id", "tenant_id", "platform")
 	if len(targetDomainId) > 0 {
@@ -335,11 +406,14 @@ func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, targetDomainId
 			scope = rbacutils.ScopeDomain
 		}
 		platform := strings.Split(platformStr, nameSeparator)
-		quota, err := manager.queryQuota(ctx, scope, &owner, platform)
+		quota, _, err := manager.queryQuota(ctx, scope, &owner, platform)
 		if err != nil {
 			log.Errorf("query quota for %s fail %s", getMemoryStoreKey(scope, &owner, platform), err)
 			continue
 		}
+		// if usage.IsEmpty() {
+		// 	continue
+		// }
 		if len(projectId) > 0 {
 			quota.Set("tenant_id", jsonutils.NewString(projectId))
 			quota.Set("domain_id", jsonutils.NewString(domainId))

pkg/cloudcommon/db/quotas/models.go

@@ -11,6 +11,7 @@ import (
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/reflectutils"
 
+	identityapi "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
@@ -78,7 +79,7 @@ func (manager *SQuotaBaseManager) getQuotaInternal(ctx context.Context, scope rb
 	if err != nil && err != sql.ErrNoRows {
 		return err
 	} else if err == sql.ErrNoRows && manager.autoCreate {
-		quota.FetchSystemQuota()
+		quota.FetchSystemQuota(scope)
 		return manager.setQuotaInternal(ctx, nil, scope, ownerId, platform, quota)
 	}
 	return nil
@@ -121,30 +122,46 @@ func (manager *SQuotaBaseManager) InitializeData() error {
 	}
 
 	for i := range tenants {
-		ownerId := db.SOwnerId{
-			DomainId:  tenants[i].DomainId,
-			Domain:    tenants[i].Domain,
-			ProjectId: tenants[i].Id,
-			Project:   tenants[i].Name,
+		obj := tenants[i]
+		var scope rbacutils.TRbacScope
+		var ownerId mcclient.IIdentityProvider
+		if obj.DomainId == identityapi.KeystoneDomainRoot {
+			// domain
+			scope = rbacutils.ScopeDomain
+			ownerId = &db.SOwnerId{
+				DomainId: tenants[i].Id,
+				Domain:   tenants[i].Name,
+			}
+		} else {
+			// project
+			scope = rbacutils.ScopeProject
+			ownerId = &db.SOwnerId{
+				DomainId:  tenants[i].DomainId,
+				Domain:    tenants[i].Domain,
+				ProjectId: tenants[i].Id,
+				Project:   tenants[i].Name,
+			}
 		}
+
 		quota := manager.newQuota()
-		err := metaQuota.GetQuota(context.Background(), rbacutils.ScopeProject, &ownerId, quota)
+		err := metaQuota.GetQuota(context.Background(), scope, ownerId, quota)
 		if err != nil && err != sql.ErrNoRows {
 			log.Errorf("metaQuota.GetQuota error %s for %s", err, ownerId)
 			continue
 		}
-		if !quota.IsEmpty() {
-			baseQuota := SQuotaBase{}
-			baseQuota.DomainId = ownerId.DomainId
-			baseQuota.ProjectId = ownerId.ProjectId
-			baseQuota.SetModelManager(manager, quota.(db.IModel))
-			reflectutils.FillEmbededStructValue(reflect.Indirect(reflect.ValueOf(quota)), reflect.ValueOf(baseQuota))
+		if quota.IsEmpty() {
+			quota.FetchSystemQuota(scope)
+		}
+		baseQuota := SQuotaBase{}
+		baseQuota.DomainId = ownerId.GetProjectDomainId()
+		baseQuota.ProjectId = ownerId.GetProjectId()
+		baseQuota.SetModelManager(manager, quota.(db.IModel))
+		reflectutils.FillEmbededStructValue(reflect.Indirect(reflect.ValueOf(quota)), reflect.ValueOf(baseQuota))
 
-			err = manager.TableSpec().Insert(quota)
-			if err != nil {
-				log.Errorf("insert error %s", err)
-				continue
-			}
+		err = manager.TableSpec().Insert(quota)
+		if err != nil {
+			log.Errorf("insert error %s", err)
+			continue
 		}
 	}
 

pkg/cloudcommon/db/quotas/quotas.go

@@ -32,7 +32,7 @@ const (
 )
 
 type IQuota interface {
-	FetchSystemQuota()
+	FetchSystemQuota(scope rbacutils.TRbacScope)
 	FetchUsage(ctx context.Context, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string) error
 	Update(quota IQuota)
 	Add(quota IQuota)

pkg/cloudcommon/policy/policy.go

@@ -512,3 +512,15 @@ func (manager *SPolicyManager) AllPolicies() map[string][]string {
 	}
 	return ret
 }
+
+func (manager *SPolicyManager) RoleMatchPolicies(roleName string) []string {
+	ret := make([]string, 0)
+	for _, policies := range manager.policies {
+		for name, policy := range policies {
+			if policy.MatchRole(roleName) {
+				ret = append(ret, name)
+			}
+		}
+	}
+	return ret
+}

pkg/compute/models/quotas.go

@@ -65,66 +65,23 @@ type SQuota struct {
 	Snapshot       int
 }
 
-/*func (manager *SQuotaManager) InitializeData() error {
-	quotaCnt, err := manager.Query().CountWithError()
-	if err != nil {
-		return errors.Wrap(err, "SQuotaManager.CountWithError")
+func (self *SQuota) FetchSystemQuota(scope rbacutils.TRbacScope) {
+	base := 1
+	if scope == rbacutils.ScopeDomain {
+		base = 10
 	}
-	if quotaCnt > 0 {
-		// initlaized, quit
-		return nil
-	}
-
-	metaQuota := quotas.NewDBQuotaStore()
-
-	tenants := make([]db.STenant, 0)
-	err = db.TenantCacheManager.Query().All(&tenants)
-	if err != nil && err != sql.ErrNoRows {
-		return errors.Wrap(err, "Query")
-	}
-
-	for i := range tenants {
-		ownerId := db.SOwnerId{
-			DomainId:  tenants[i].DomainId,
-			Domain:    tenants[i].Domain,
-			ProjectId: tenants[i].Id,
-			Project:   tenants[i].Name,
-		}
-		quota := SQuota{}
-		err := metaQuota.GetQuota(context.Background(), rbacutils.ScopeProject, &ownerId, &quota)
-		if err != nil && err != sql.ErrNoRows {
-			log.Errorf("metaQuota.GetQuota error %s for %s", err, ownerId)
-			continue
-		}
-		if !quota.IsEmpty() {
-			quota.DomainId = ownerId.DomainId
-			quota.ProjectId = ownerId.ProjectId
-			quota.SetModelManager(manager, &quota)
-
-			err = manager.TableSpec().Insert(&quota)
-			if err != nil {
-				log.Errorf("insert error %s", err)
-				continue
-			}
-		}
-	}
-
-	return nil
-}*/
-
-func (self *SQuota) FetchSystemQuota() {
-	self.Cpu = options.Options.DefaultCpuQuota
-	self.Memory = options.Options.DefaultMemoryQuota
-	self.Storage = options.Options.DefaultStorageQuota
-	self.Port = options.Options.DefaultPortQuota
-	self.Eip = options.Options.DefaultEipQuota
-	self.Eport = options.Options.DefaultEportQuota
-	self.Bw = options.Options.DefaultBwQuota
-	self.Ebw = options.Options.DefaultEbwQuota
-	self.Group = options.Options.DefaultGroupQuota
-	self.Secgroup = options.Options.DefaultSecgroupQuota
-	self.IsolatedDevice = options.Options.DefaultIsolatedDeviceQuota
-	self.Snapshot = options.Options.DefaultSnapshotQuota
+	self.Cpu = options.Options.DefaultCpuQuota * base
+	self.Memory = options.Options.DefaultMemoryQuota * base
+	self.Storage = options.Options.DefaultStorageQuota * base
+	self.Port = options.Options.DefaultPortQuota * base
+	self.Eip = options.Options.DefaultEipQuota * base
+	self.Eport = options.Options.DefaultEportQuota * base
+	self.Bw = options.Options.DefaultBwQuota * base
+	self.Ebw = options.Options.DefaultEbwQuota * base
+	self.Group = options.Options.DefaultGroupQuota * base
+	self.Secgroup = options.Options.DefaultSecgroupQuota * base
+	self.IsolatedDevice = options.Options.DefaultIsolatedDeviceQuota * base
+	self.Snapshot = options.Options.DefaultSnapshotQuota * base
 }
 
 func (self *SQuota) FetchUsage(ctx context.Context, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, name []string) error {

pkg/image/models/quotas.go

@@ -51,8 +51,12 @@ type SQuota struct {
 	Image int
 }
 
-func (self *SQuota) FetchSystemQuota() {
-	self.Image = options.Options.DefaultImageQuota
+func (self *SQuota) FetchSystemQuota(scope rbacutils.TRbacScope) {
+	base := 1
+	if scope == rbacutils.ScopeDomain {
+		base = 10
+	}
+	self.Image = options.Options.DefaultImageQuota * base
 }
 
 func (self *SQuota) FetchUsage(ctx context.Context, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string) error {

pkg/keystone/models/assignments.go

@@ -265,6 +265,10 @@ func (manager *SAssignmentManager) projectRemoveUser(ctx context.Context, userCr
 	if project.IsAdminProject() && user.IsAdminUser() && role.IsSystemRole() {
 		return httperrors.NewForbiddenError("sysadmin is protected")
 	}
+	// prevent remove current user from current project
+	if project.Id == userCred.GetProjectId() && user.Id == userCred.GetUserId() {
+		return httperrors.NewForbiddenError("cannot remove current user from current project")
+	}
 	if project.DomainId != user.DomainId {
 		// if project.DomainId != api.DEFAULT_DOMAIN_ID {
 		//    return httperrors.NewInputParameterError("join user into project of default domain or identical domain")

pkg/keystone/models/domains.go

@@ -255,6 +255,19 @@ func (domain *SDomain) ValidateUpdateCondition(ctx context.Context) error {
 	return domain.SStandaloneResourceBase.ValidateUpdateCondition(ctx)
 }
 
+func (domain *SDomain) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	if domain.IsReadOnly() {
+		for _, k := range []string{
+			"name",
+		} {
+			if data.Contains(k) {
+				return nil, httperrors.NewForbiddenError("field %s is readonly", k)
+			}
+		}
+	}
+	return domain.SStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, data)
+}
+
 /*func (domain *SDomain) isReadOnly() bool {
 	if domain.GetDriver() == api.IdentityDriverSQL {
 		return false

pkg/keystone/models/groups.go

@@ -210,12 +210,26 @@ func (manager *SGroupManager) RegisterExternalGroup(ctx context.Context, idpId s
 }
 
 func (group *SGroup) ValidateUpdateCondition(ctx context.Context) error {
-	if group.IsReadOnly() {
-		return httperrors.NewForbiddenError("readonly")
-	}
+	// if group.IsReadOnly() {
+	// 	return httperrors.NewForbiddenError("readonly")
+	// }
 	return group.SIdentityBaseResource.ValidateUpdateCondition(ctx)
 }
 
+func (group *SGroup) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	if group.IsReadOnly() {
+		for _, k := range []string{
+			"name",
+			"displayname",
+		} {
+			if data.Contains(k) {
+				return nil, httperrors.NewForbiddenError("field %s is readonly", k)
+			}
+		}
+	}
+	return group.SIdentityBaseResource.ValidateUpdateData(ctx, userCred, query, data)
+}
+
 func (manager *SGroupManager) fetchGroupById(gid string) *SGroup {
 	obj, _ := GroupManager.FetchById(gid)
 	if obj != nil {

pkg/keystone/models/roles.go

@@ -213,6 +213,10 @@ func roleExtra(role *SRole, extra *jsonutils.JSONDict) *jsonutils.JSONDict {
 	extra.Add(jsonutils.NewInt(int64(grpCnt)), "group_count")
 	prjCnt, _ := role.GetProjectCount()
 	extra.Add(jsonutils.NewInt(int64(prjCnt)), "project_count")
+	policies := policy.PolicyManager.RoleMatchPolicies(role.Name)
+	if len(policies) > 0 {
+		extra.Add(jsonutils.NewStringArray(policies), "match_policies")
+	}
 	return extra
 }
 

pkg/keystone/models/users.go

@@ -355,13 +355,27 @@ func (user *SUser) ValidateUpdateData(ctx context.Context, userCred mcclient.Tok
 			return nil, httperrors.NewForbiddenError("cannot alter sysadmin user name")
 		}
 	}
+	if user.IsReadOnly() {
+		for _, k := range []string{
+			"name",
+			"enabled",
+			"displayname",
+			"email",
+			"mobile",
+			"pasword",
+		} {
+			if data.Contains(k) {
+				return nil, httperrors.NewForbiddenError("field %s is readonly", k)
+			}
+		}
+	}
 	return user.SEnabledIdentityBaseResource.ValidateUpdateData(ctx, userCred, query, data)
 }
 
 func (user *SUser) ValidateUpdateCondition(ctx context.Context) error {
-	if user.IsReadOnly() {
-		return httperrors.NewForbiddenError("readonly")
-	}
+	// if user.IsReadOnly() {
+	// 	return httperrors.NewForbiddenError("readonly")
+	// }
 	return user.SEnabledIdentityBaseResource.ValidateUpdateCondition(ctx)
 }
 

pkg/mcclient/modules/mod_quotas.go

@@ -33,7 +33,7 @@ func (this *QuotaManager) getURL(params jsonutils.JSONObject) string {
 		if len(tenant) > 0 {
 			url = fmt.Sprintf("%s/projects/%s", url, tenant)
 		} else {
-			domain, _ := params.GetString("domain")
+			domain := jsonutils.GetAnyString(params, []string{"domain", "project_domain"})
 			if len(domain) > 0 {
 				url = fmt.Sprintf("%s/domains/%s", url, domain)
 			}

pkg/util/rbacutils/rbac.go

@@ -576,6 +576,13 @@ func (policy *SRbacPolicy) Match(userCred IRbacIdentity) bool {
 	return false
 }
 
+func (policy *SRbacPolicy) MatchRole(roleName string) bool {
+	if len(policy.Roles) == 0 || contains(policy.Roles, roleName) {
+		return true
+	}
+	return false
+}
+
 func String2Scope(str string) TRbacScope {
 	return String2ScopeDefault(str, ScopeProject)
 }