支持过滤未关联某个安全组的虚机列表

cmd/climc/shell/secgroups.go

@@ -12,6 +12,7 @@ import (
 
 func init() {
 	type SecGroupsListOptions struct {
+		UnionSecgroup string `help:"Secgroup ID or Name, filter secgroups which rules equals with this secgroup"`
 		options.BaseListOptions
 	}
 
@@ -25,6 +26,9 @@ func init() {
 
 			}
 		}
+		if len(args.UnionSecgroup) > 0 {
+			params.Add(jsonutils.NewString(args.UnionSecgroup), "union_secgroup")
+		}
 		result, err := modules.SecGroups.List(s, params)
 		if err != nil {
 			return err

pkg/compute/models/guestsecgroups.go

@@ -65,6 +65,21 @@ func (manager *SGuestsecgroupManager) newGuestSecgroup(ctx context.Context, user
 	return &gs, manager.TableSpec().Insert(&gs)
 }
 
+func (manager *SGuestsecgroupManager) GetGuestSecgroups(guest *SGuest, secgroup *SSecurityGroup) ([]SGuestsecgroup, error) {
+	guestsecgroups := []SGuestsecgroup{}
+	q := manager.Query()
+	if guest != nil {
+		q = q.Equals("guest_id", guest.Id)
+	}
+	if secgroup != nil {
+		q = q.Equals("secgroup_id", secgroup.Id)
+	}
+	if err := db.FetchModelObjects(manager, q, &guestsecgroups); err != nil {
+		return nil, err
+	}
+	return guestsecgroups, nil
+}
+
 func (manager *SGuestsecgroupManager) DeleteGuestSecgroup(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, secgroup *SSecurityGroup) error {
 	gss := []SGuestsecgroup{}
 	q := manager.Query()

pkg/compute/models/secgroups.go

@@ -50,6 +50,32 @@ type SSecurityGroup struct {
 	IsDirty bool `nullable:"false" default:"false"` // Column(Boolean, nullable=False, default=False)
 }
 
+func (manager *SSecurityGroupManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
+	unionSecgroup, _ := query.GetString("union_secgroup")
+	if len(unionSecgroup) > 0 {
+		_secgroup, err := manager.FetchByIdOrName(userCred, unionSecgroup)
+		if err != nil {
+			return nil, err
+		}
+		secgroup := _secgroup.(*SSecurityGroup)
+		secgroup.SetModelManager(manager)
+		sq := manager.Query().NotEquals("id", secgroup.Id)
+		secgroups := []SSecurityGroup{}
+		err = db.FetchModelObjects(manager, sq, &secgroups)
+		if err != nil {
+			return nil, err
+		}
+		secgroupIds := []string{}
+		for i := 0; i < len(secgroups); i++ {
+			if equal := secgroup.isEqual(ctx, userCred, &secgroups[i]); equal {
+				secgroupIds = append(secgroupIds, secgroups[i].Id)
+			}
+		}
+		q = q.In("id", secgroupIds)
+	}
+	return q, nil
+}
+
 func (self *SSecurityGroup) GetGuestsQuery() *sqlchemy.SQuery {
 	guests := GuestManager.Query().SubQuery()
 	return guests.Query().Filter(
@@ -76,6 +102,14 @@ func (self *SSecurityGroup) GetGuests() []SGuest {
 	return guests
 }
 
+func (self *SSecurityGroup) GetSecgroupCacheQuery() *sqlchemy.SQuery {
+	return SecurityGroupCacheManager.Query().Equals("secgroup_id", self.Id)
+}
+
+func (self *SSecurityGroup) GetSecgroupCacheCount() int {
+	return self.GetSecgroupCacheQuery().Count()
+}
+
 func (self *SSecurityGroup) getDesc() jsonutils.JSONObject {
 	desc := jsonutils.NewDict()
 	desc.Add(jsonutils.NewString(self.Name), "name")
@@ -90,6 +124,7 @@ func (self *SSecurityGroup) GetExtraDetails(ctx context.Context, userCred mcclie
 		return nil, err
 	}
 	extra.Add(jsonutils.NewInt(int64(len(self.GetGuests()))), "guest_cnt")
+	extra.Add(jsonutils.NewInt(int64(self.GetSecgroupCacheCount())), "cache_cnt")
 	extra.Add(jsonutils.NewString(self.getSecurityRuleString("")), "rules")
 	extra.Add(jsonutils.NewString(self.getSecurityRuleString("in")), "in_rules")
 	extra.Add(jsonutils.NewString(self.getSecurityRuleString("out")), "out_rules")
@@ -99,6 +134,7 @@ func (self *SSecurityGroup) GetExtraDetails(ctx context.Context, userCred mcclie
 func (self *SSecurityGroup) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
 	extra := self.SSharableVirtualResourceBase.GetCustomizeColumns(ctx, userCred, query)
 	extra.Add(jsonutils.NewInt(int64(len(self.GetGuests()))), "guest_cnt")
+	extra.Add(jsonutils.NewInt(int64(self.GetSecgroupCacheCount())), "cache_cnt")
 	extra.Add(jsonutils.NewTimeString(self.CreatedAt), "created_at")
 	extra.Add(jsonutils.NewString(self.Description), "description")
 	extra.Add(jsonutils.NewString(self.getSecurityRuleString("in")), "in_rules")
@@ -252,6 +288,116 @@ func (self *SSecurityGroup) PerformClone(ctx context.Context, userCred mcclient.
 	return nil, nil
 }
 
+func (self *SSecurityGroup) AllowPerformUnion(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "union")
+}
+
+func (self *SSecurityGroup) PerformUnion(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	secgroupIds := jsonutils.GetQueryStringArray(data, "secgroups")
+	if len(secgroupIds) == 0 {
+		return nil, httperrors.NewMissingParameterError("secgroups")
+	}
+	secgroups := []*SSecurityGroup{}
+	for _, secgroupId := range secgroupIds {
+		_secgroup, err := SecurityGroupManager.FetchByIdOrName(userCred, secgroupId)
+		if err != nil {
+			return nil, httperrors.NewResourceNotFoundError("failed to find secgroup %s error: %v", secgroupId, err)
+		}
+		secgroup := _secgroup.(*SSecurityGroup)
+		secgroup.SetModelManager(SecurityGroupManager)
+		if equal := self.isEqual(ctx, userCred, secgroup); !equal {
+			return nil, httperrors.NewUnsupportOperationError("secgroup %s rules not equals %s rules", secgroup.Name, self.Name)
+		}
+		secgroups = append(secgroups, secgroup)
+	}
+
+	for i := 0; i < len(secgroups); i++ {
+		secgroup := secgroups[i]
+		if err := self.migrateSecurityGroupCache(secgroup); err != nil {
+			return nil, err
+		}
+
+		if err := self.migrateGuestSecurityGroup(secgroup); err != nil {
+			return nil, err
+		}
+	}
+	self.DoSync(ctx, userCred)
+	return nil, nil
+}
+
+func (self *SSecurityGroup) isEqual(ctx context.Context, userCred mcclient.TokenCredential, secgroup *SSecurityGroup) bool {
+	rules := self.GetSecRules("")
+	srs := secrules.SecurityGroupRuleSet{}
+	for i := 0; i < len(rules); i++ {
+		srs.AddRule(rules[i])
+	}
+
+	_rules := secgroup.GetSecRules("")
+	_srs := secrules.SecurityGroupRuleSet{}
+	for i := 0; i < len(_rules); i++ {
+		_srs.AddRule(_rules[i])
+	}
+
+	return srs.IsEqual(_srs)
+}
+
+func (self *SSecurityGroup) migrateSecurityGroupCache(secgroup *SSecurityGroup) error {
+	caches := []SSecurityGroupCache{}
+	q := SecurityGroupCacheManager.Query().Equals("secgroup_id", secgroup.Id)
+	err := db.FetchModelObjects(SecurityGroupCacheManager, q, &caches)
+	if err != nil {
+		return err
+	}
+	for i := 0; i < len(caches); i++ {
+		cache := caches[i]
+		_, err := db.Update(&cache, func() error {
+			cache.SecgroupId = self.Id
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (self *SSecurityGroup) migrateGuestSecurityGroup(secgroup *SSecurityGroup) error {
+	guests := secgroup.GetGuests()
+	for i := 0; i < len(guests); i++ {
+		guest := guests[i]
+		_, err := db.Update(&guest, func() error {
+			if guest.SecgrpId == secgroup.Id {
+				guest.SecgrpId = self.Id
+			}
+			if guest.AdminSecgrpId == secgroup.Id {
+				guest.AdminSecgrpId = self.Id
+			}
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	guestsecgroups, err := GuestsecgroupManager.GetGuestSecgroups(nil, secgroup)
+	if err != nil {
+		return err
+	}
+
+	for i := 0; i < len(guestsecgroups); i++ {
+		guestsecgroup := guestsecgroups[i]
+		_, err := db.Update(&guestsecgroup, func() error {
+			guestsecgroup.SecgroupId = self.Id
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func (manager *SSecurityGroupManager) getSecurityGroups() ([]SSecurityGroup, error) {
 	secgroups := make([]SSecurityGroup, 0)
 	q := manager.Query()
@@ -270,13 +416,13 @@ func (manager *SSecurityGroupManager) newFromCloudSecgroup(ctx context.Context,
 	secgroup.SetModelManager(manager)
 	secgroup.Name = db.GenerateName(manager, "", extSec.GetName())
 	secgroup.Description = extSec.GetDescription()
+	secgroup.ProjectId = userCred.GetProjectId()
 
 	if err := manager.TableSpec().Insert(&secgroup); err != nil {
 		return nil, err
 	}
 
 	db.OpsLog.LogEvent(&secgroup, db.ACT_CREATE, secgroup.GetShortDesc(ctx), userCred)
-	SyncCloudProject(userCred, &secgroup, provider.ProjectId, extSec, provider.Id)
 
 	return &secgroup, nil
 }
@@ -411,5 +557,16 @@ func (self *SSecurityGroup) Delete(ctx context.Context, userCred mcclient.TokenC
 }
 
 func (self *SSecurityGroup) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	rules := []SSecurityGroupRule{}
+	q := SecurityGroupRuleManager.Query().Equals("secgroup_id", self.Id)
+	if err := db.FetchModelObjects(SecurityGroupRuleManager, q, &rules); err != nil {
+		log.Errorf("failed to fetch secgroup %s rules error: %v", self.Name, err)
+		return err
+	}
+	for i := 0; i < len(rules); i++ {
+		if err := rules[i].Delete(ctx, userCred); err != nil {
+			return err
+		}
+	}
 	return self.SVirtualResourceBase.Delete(ctx, userCred)
 }

vendor/yunion.io/x/pkg/util/secrules/secrules.go

@@ -26,6 +26,18 @@ const (
 	SecurityRuleDeny  = TSecurityRuleAction("deny")
 )
 
+type TSecurityRuleRelation string
+
+const (
+	RELATION_INDEPENDENT = TSecurityRuleRelation("INDEPENDT")
+	RELATION_IDENTICAL   = TSecurityRuleRelation("IDENTICAL")
+	RELATION_SUBSET      = TSecurityRuleRelation("SUBSET")
+	RELATION_SUPERSET    = TSecurityRuleRelation("SUPERSET")
+	RELATION_NEXT_AHEAD  = TSecurityRuleRelation("NEXT_AHEAD")
+	RELATION_NEXT_AFTER  = TSecurityRuleRelation("NEXT_AFTER")
+	RELATION_OVERLAP     = TSecurityRuleRelation("OVERLAP")
+)
+
 type SecurityRule struct {
 	Priority    int // [1, 100]
 	Action      TSecurityRuleAction
@@ -167,6 +179,86 @@ func (rule *SecurityRule) IsWildMatch() bool {
 		rule.PortEnd == 0
 }
 
+func (rule SecurityRule) protoRelation(r SecurityRule) TSecurityRuleRelation {
+	if rule.Direction != r.Direction {
+		return RELATION_IDENTICAL
+	}
+	if rule.Protocol == r.Protocol {
+		if utils.IsInStringArray(rule.Protocol, []string{PROTO_ANY, PROTO_ICMP}) {
+			return RELATION_IDENTICAL
+		}
+		if rule.PortStart <= 0 && r.PortStart <= 0 {
+			return RELATION_IDENTICAL
+		}
+		if rule.PortStart > 0 && r.PortStart <= 0 {
+			return RELATION_SUBSET
+		}
+		if rule.PortStart <= 0 && r.PortStart > 0 {
+			return RELATION_SUPERSET
+		}
+		if rule.PortEnd+1 == r.PortStart {
+			return RELATION_NEXT_AHEAD
+		}
+		if rule.PortStart == r.PortEnd+1 {
+			return RELATION_NEXT_AFTER
+		}
+		if rule.PortEnd < r.PortStart || rule.PortStart > r.PortEnd {
+			return RELATION_INDEPENDENT
+		}
+		if rule.PortStart == r.PortStart && rule.PortEnd == r.PortEnd {
+			return RELATION_IDENTICAL
+		}
+		if rule.PortStart <= r.PortStart && rule.PortEnd >= r.PortEnd {
+			return RELATION_SUPERSET
+		}
+		if rule.PortStart >= r.PortStart && rule.PortEnd <= r.PortEnd {
+			return RELATION_SUBSET
+		}
+		return RELATION_OVERLAP
+	}
+	if rule.Protocol == PROTO_ANY {
+		return RELATION_SUPERSET
+	}
+	if r.Protocol == PROTO_ANY {
+		return RELATION_SUBSET
+	}
+	return RELATION_INDEPENDENT
+}
+
+func (rule SecurityRule) merge(r SecurityRule) (SecurityRule, error) {
+	if rule.getIPKey() != r.getIPKey() {
+		panic(fmt.Sprintf("rule %v ip addr not equal rule %v", rule, r))
+	}
+	if rule.Action != r.Action {
+		panic(fmt.Sprintf("rule %v action not equal rule %v", rule, r))
+	}
+	rel := rule.protoRelation(r)
+	switch rel {
+	case RELATION_NEXT_AHEAD:
+		rule.PortEnd = r.PortEnd
+		return rule, nil
+	case RELATION_NEXT_AFTER:
+		rule.PortStart = r.PortStart
+		return rule, nil
+	case RELATION_OVERLAP:
+		if rule.PortStart > r.PortStart {
+			rule.PortStart = r.PortStart
+		}
+		if rule.PortEnd < r.PortEnd {
+			rule.PortEnd = r.PortEnd
+		}
+		return rule, nil
+	}
+	return r, fmt.Errorf("Cannot merge")
+}
+
+func (rule SecurityRule) getIPKey() string {
+	if rule.IPNet == nil || rule.IPNet.String() == "0.0.0.0/0" {
+		return "0.0.0.0/0"
+	}
+	return rule.IPNet.String()
+}
+
 func (rule *SecurityRule) ParsePorts(seg string) error {
 	if len(seg) == 0 {
 		rule.Ports = []int{}

vendor/yunion.io/x/pkg/util/secrules/securityruleset.go

@@ -0,0 +1,202 @@
+package secrules
+
+import (
+	"bytes"
+	"sort"
+
+	"yunion.io/x/log"
+)
+
+type SecurityGroupSubSubRuleSet struct {
+	ipKey string
+	rules []SecurityRule
+}
+
+func (sssrs *SecurityGroupSubSubRuleSet) AddRule(rule SecurityRule) {
+	if sssrs.rules == nil {
+		sssrs.ipKey = rule.getIPKey()
+		sssrs.rules = []SecurityRule{rule}
+		return
+	}
+	if sssrs.ipKey != rule.getIPKey() {
+		panic("rule ip key not equal")
+	}
+	idx := 0
+	for idx < len(sssrs.rules) {
+		rel := sssrs.rules[idx].protoRelation(rule)
+		switch rel {
+		case RELATION_INDEPENDENT:
+			idx++
+		case RELATION_IDENTICAL, RELATION_SUPERSET:
+			return
+		case RELATION_SUBSET:
+			sssrs.rules = append(sssrs.rules[:idx], sssrs.rules[idx+1:]...)
+		case RELATION_NEXT_AHEAD, RELATION_NEXT_AFTER, RELATION_OVERLAP:
+			r, err := rule.merge(sssrs.rules[idx])
+			if err != nil {
+				log.Errorf("Merge failed?? %v", err)
+				continue
+			}
+			idx = 0
+			sssrs.rules[idx] = r
+		}
+	}
+	sssrs.rules = append(sssrs.rules, rule)
+}
+
+type SecurityGroupSubRuleSet struct {
+	allowRules *SecurityGroupSubSubRuleSet
+	denyRules  *SecurityGroupSubSubRuleSet
+}
+
+func (ssrs *SecurityGroupSubRuleSet) AddRule(rule SecurityRule) {
+	if rule.Action == SecurityRuleAllow {
+		if ssrs.allowRules == nil {
+			ssrs.allowRules = &SecurityGroupSubSubRuleSet{}
+		}
+		ssrs.allowRules.AddRule(rule)
+	} else {
+		if ssrs.denyRules == nil {
+			ssrs.denyRules = &SecurityGroupSubSubRuleSet{}
+		}
+		ssrs.denyRules.AddRule(rule)
+	}
+}
+
+func (ssrs *SecurityGroupSubRuleSet) isEqual(_ssrs *SecurityGroupSubRuleSet) bool {
+	allowRules := ssrs.GetAllowRules()
+	_allowRules := _ssrs.GetAllowRules()
+	if len(allowRules.rules) != len(_allowRules.rules) {
+		return false
+	}
+
+	sort.Slice(allowRules.rules, func(i, j int) bool {
+		return allowRules.rules[i].Priority < allowRules.rules[j].Priority
+	})
+
+	sort.Slice(_allowRules.rules, func(i, j int) bool {
+		return _allowRules.rules[i].Priority < _allowRules.rules[j].Priority
+	})
+	for i := 0; i < len(allowRules.rules); i++ {
+		if allowRules.rules[i].String() != _allowRules.rules[i].String() {
+			return false
+		}
+	}
+
+	denyRules := ssrs.GetDenyRules()
+	_denyRuels := _ssrs.GetDenyRules()
+	if len(denyRules.rules) != len(_denyRuels.rules) {
+		return false
+	}
+
+	sort.Slice(denyRules.rules, func(i, j int) bool {
+		return denyRules.rules[i].Priority < denyRules.rules[j].Priority
+	})
+
+	sort.Slice(_denyRuels.rules, func(i, j int) bool {
+		return _denyRuels.rules[i].Priority < _denyRuels.rules[j].Priority
+	})
+	for i := 0; i < len(denyRules.rules); i++ {
+		if denyRules.rules[i].String() != _denyRuels.rules[i].String() {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (ssrs *SecurityGroupSubRuleSet) GetAllowRules() SecurityGroupSubSubRuleSet {
+	if ssrs.allowRules == nil {
+		return SecurityGroupSubSubRuleSet{}
+	}
+	return *ssrs.allowRules
+}
+
+func (ssrs *SecurityGroupSubRuleSet) GetDenyRules() SecurityGroupSubSubRuleSet {
+	if ssrs.denyRules == nil {
+		return SecurityGroupSubSubRuleSet{}
+	}
+	return *ssrs.denyRules
+}
+
+type SecurityGroupRuleSet struct {
+	rules map[string]*SecurityGroupSubRuleSet
+}
+
+func (srs *SecurityGroupRuleSet) AddRule(rule SecurityRule) {
+	if srs.rules == nil {
+		srs.rules = map[string]*SecurityGroupSubRuleSet{}
+	}
+	key := rule.getIPKey()
+	if _, ok := srs.rules[key]; !ok {
+		srs.rules[key] = &SecurityGroupSubRuleSet{}
+	}
+	ssrs := srs.rules[key]
+	if len(rule.Ports) > 0 {
+		ports := rule.Ports
+		rule.Ports = []int{}
+		for i := 0; i < len(ports); i++ {
+			rule.PortStart = ports[i]
+			rule.PortEnd = ports[i]
+			ssrs.AddRule(rule)
+		}
+		return
+	}
+	ssrs.AddRule(rule)
+}
+
+func (srs *SecurityGroupRuleSet) GetSubRuleSet(key string) *SecurityGroupSubRuleSet {
+	if srs.rules == nil {
+		return nil
+	}
+	s, ok := srs.rules[key]
+	if !ok {
+		return nil
+	}
+	return s
+}
+
+func (srs *SecurityGroupRuleSet) GetAllowRules() []SecurityRule {
+	rules := []SecurityRule{}
+	for _, v := range srs.rules {
+		rules = append(rules, v.GetAllowRules().rules...)
+	}
+	return rules
+}
+
+func (srs *SecurityGroupRuleSet) GetDenyRules() []SecurityRule {
+	rules := []SecurityRule{}
+	for _, v := range srs.rules {
+		rules = append(rules, v.GetDenyRules().rules...)
+	}
+	return rules
+}
+
+func (srs *SecurityGroupRuleSet) IsEqual(src SecurityGroupRuleSet) bool {
+	if len(srs.rules) != len(src.rules) {
+		return false
+	}
+	for k, v := range srs.rules {
+		_v := src.GetSubRuleSet(k)
+		if _v == nil {
+			return false
+		}
+		if !v.isEqual(_v) {
+			return false
+		}
+	}
+	return true
+}
+
+func (srs *SecurityGroupRuleSet) String() string {
+	buf := bytes.Buffer{}
+	for _, r := range srs.GetAllowRules() {
+		buf.WriteString(r.String())
+		buf.WriteString(";")
+	}
+	for _, r := range srs.GetDenyRules() {
+		buf.WriteString(r.String())
+		buf.WriteString(";")
+	}
+	return buf.String()
+}

vendor/yunion.io/x/pkg/util/secrules/securityruleset_test.go

@@ -0,0 +1,39 @@
+package secrules
+
+import "testing"
+
+func TestRules(t *testing.T) {
+	rules := []string{"in:allow 192.168.0.1/32 tcp 80", "out:deny any", "in:allow any", "out:allow udp 3232-3000", "in:allow tcp"}
+	for _, r := range rules {
+		rule, err := ParseSecurityRule(r)
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Log(rule)
+	}
+
+	ruleSet := [][]string{
+		[]string{"in:allow any"},
+		[]string{"in:allow any", "in:deny tcp 25", "in:deny tcp 80"},
+		[]string{"in:allow tcp 80", "in:allow udp"},
+		[]string{"in:allow any", "in:allow tcp 80"},
+		[]string{"in:allow tcp 1000-1999", "in:allow tcp 2000", "in:allow tcp 2200"},
+		[]string{"in:allow tcp 1000-1999", "in:allow tcp 2000", "in:allow tcp 2200",
+			"in:deny tcp 500-1500", "in:deny tcp 2000-3000", "in:deny tcp 1888"},
+		[]string{"in:allow 192.168.0.0/16 tcp", "in:allow tcp 80",
+			"out:deny 192.168.0.0/16 tcp 80", "out:deny tcp 80", "in:allow any",
+			"in:allow 192.168.0.0/24 tcp", "out:deny 192.168.0.0/24 udp"},
+	}
+
+	for _, rs := range ruleSet {
+		srs := SecurityGroupRuleSet{}
+		for _, r := range rs {
+			rule, err := ParseSecurityRule(r)
+			if err != nil {
+				t.Fatalf("parse rule %s error: %v", r, err)
+			}
+			srs.AddRule(*rule)
+		}
+		t.Log(srs.String())
+	}
+}