diff --git a/cmd/climc/shell/secgroups.go b/cmd/climc/shell/secgroups.go index 565ac3bcdd..7fece12cd5 100644 --- a/cmd/climc/shell/secgroups.go +++ b/cmd/climc/shell/secgroups.go @@ -12,6 +12,7 @@ import ( func init() { type SecGroupsListOptions struct { + UnionSecgroup string `help:"Secgroup ID or Name, filter secgroups which rules equals with this secgroup"` options.BaseListOptions } @@ -25,6 +26,9 @@ func init() { } } + if len(args.UnionSecgroup) > 0 { + params.Add(jsonutils.NewString(args.UnionSecgroup), "union_secgroup") + } result, err := modules.SecGroups.List(s, params) if err != nil { return err diff --git a/pkg/compute/models/guestsecgroups.go b/pkg/compute/models/guestsecgroups.go index 954870d5d9..9c6c9b6423 100644 --- a/pkg/compute/models/guestsecgroups.go +++ b/pkg/compute/models/guestsecgroups.go @@ -65,6 +65,21 @@ func (manager *SGuestsecgroupManager) newGuestSecgroup(ctx context.Context, user return &gs, manager.TableSpec().Insert(&gs) } +func (manager *SGuestsecgroupManager) GetGuestSecgroups(guest *SGuest, secgroup *SSecurityGroup) ([]SGuestsecgroup, error) { + guestsecgroups := []SGuestsecgroup{} + q := manager.Query() + if guest != nil { + q = q.Equals("guest_id", guest.Id) + } + if secgroup != nil { + q = q.Equals("secgroup_id", secgroup.Id) + } + if err := db.FetchModelObjects(manager, q, &guestsecgroups); err != nil { + return nil, err + } + return guestsecgroups, nil +} + func (manager *SGuestsecgroupManager) DeleteGuestSecgroup(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, secgroup *SSecurityGroup) error { gss := []SGuestsecgroup{} q := manager.Query() diff --git a/pkg/compute/models/secgroups.go b/pkg/compute/models/secgroups.go index 637c2e8172..d3299ab08a 100644 --- a/pkg/compute/models/secgroups.go +++ b/pkg/compute/models/secgroups.go @@ -50,6 +50,32 @@ type SSecurityGroup struct { IsDirty bool `nullable:"false" default:"false"` // Column(Boolean, nullable=False, default=False) } +func (manager *SSecurityGroupManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { + unionSecgroup, _ := query.GetString("union_secgroup") + if len(unionSecgroup) > 0 { + _secgroup, err := manager.FetchByIdOrName(userCred, unionSecgroup) + if err != nil { + return nil, err + } + secgroup := _secgroup.(*SSecurityGroup) + secgroup.SetModelManager(manager) + sq := manager.Query().NotEquals("id", secgroup.Id) + secgroups := []SSecurityGroup{} + err = db.FetchModelObjects(manager, sq, &secgroups) + if err != nil { + return nil, err + } + secgroupIds := []string{} + for i := 0; i < len(secgroups); i++ { + if equal := secgroup.isEqual(ctx, userCred, &secgroups[i]); equal { + secgroupIds = append(secgroupIds, secgroups[i].Id) + } + } + q = q.In("id", secgroupIds) + } + return q, nil +} + func (self *SSecurityGroup) GetGuestsQuery() *sqlchemy.SQuery { guests := GuestManager.Query().SubQuery() return guests.Query().Filter( @@ -76,6 +102,14 @@ func (self *SSecurityGroup) GetGuests() []SGuest { return guests } +func (self *SSecurityGroup) GetSecgroupCacheQuery() *sqlchemy.SQuery { + return SecurityGroupCacheManager.Query().Equals("secgroup_id", self.Id) +} + +func (self *SSecurityGroup) GetSecgroupCacheCount() int { + return self.GetSecgroupCacheQuery().Count() +} + func (self *SSecurityGroup) getDesc() jsonutils.JSONObject { desc := jsonutils.NewDict() desc.Add(jsonutils.NewString(self.Name), "name") @@ -90,6 +124,7 @@ func (self *SSecurityGroup) GetExtraDetails(ctx context.Context, userCred mcclie return nil, err } extra.Add(jsonutils.NewInt(int64(len(self.GetGuests()))), "guest_cnt") + extra.Add(jsonutils.NewInt(int64(self.GetSecgroupCacheCount())), "cache_cnt") extra.Add(jsonutils.NewString(self.getSecurityRuleString("")), "rules") extra.Add(jsonutils.NewString(self.getSecurityRuleString("in")), "in_rules") extra.Add(jsonutils.NewString(self.getSecurityRuleString("out")), "out_rules") @@ -99,6 +134,7 @@ func (self *SSecurityGroup) GetExtraDetails(ctx context.Context, userCred mcclie func (self *SSecurityGroup) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { extra := self.SSharableVirtualResourceBase.GetCustomizeColumns(ctx, userCred, query) extra.Add(jsonutils.NewInt(int64(len(self.GetGuests()))), "guest_cnt") + extra.Add(jsonutils.NewInt(int64(self.GetSecgroupCacheCount())), "cache_cnt") extra.Add(jsonutils.NewTimeString(self.CreatedAt), "created_at") extra.Add(jsonutils.NewString(self.Description), "description") extra.Add(jsonutils.NewString(self.getSecurityRuleString("in")), "in_rules") @@ -252,6 +288,116 @@ func (self *SSecurityGroup) PerformClone(ctx context.Context, userCred mcclient. return nil, nil } +func (self *SSecurityGroup) AllowPerformUnion(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "union") +} + +func (self *SSecurityGroup) PerformUnion(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + secgroupIds := jsonutils.GetQueryStringArray(data, "secgroups") + if len(secgroupIds) == 0 { + return nil, httperrors.NewMissingParameterError("secgroups") + } + secgroups := []*SSecurityGroup{} + for _, secgroupId := range secgroupIds { + _secgroup, err := SecurityGroupManager.FetchByIdOrName(userCred, secgroupId) + if err != nil { + return nil, httperrors.NewResourceNotFoundError("failed to find secgroup %s error: %v", secgroupId, err) + } + secgroup := _secgroup.(*SSecurityGroup) + secgroup.SetModelManager(SecurityGroupManager) + if equal := self.isEqual(ctx, userCred, secgroup); !equal { + return nil, httperrors.NewUnsupportOperationError("secgroup %s rules not equals %s rules", secgroup.Name, self.Name) + } + secgroups = append(secgroups, secgroup) + } + + for i := 0; i < len(secgroups); i++ { + secgroup := secgroups[i] + if err := self.migrateSecurityGroupCache(secgroup); err != nil { + return nil, err + } + + if err := self.migrateGuestSecurityGroup(secgroup); err != nil { + return nil, err + } + } + self.DoSync(ctx, userCred) + return nil, nil +} + +func (self *SSecurityGroup) isEqual(ctx context.Context, userCred mcclient.TokenCredential, secgroup *SSecurityGroup) bool { + rules := self.GetSecRules("") + srs := secrules.SecurityGroupRuleSet{} + for i := 0; i < len(rules); i++ { + srs.AddRule(rules[i]) + } + + _rules := secgroup.GetSecRules("") + _srs := secrules.SecurityGroupRuleSet{} + for i := 0; i < len(_rules); i++ { + _srs.AddRule(_rules[i]) + } + + return srs.IsEqual(_srs) +} + +func (self *SSecurityGroup) migrateSecurityGroupCache(secgroup *SSecurityGroup) error { + caches := []SSecurityGroupCache{} + q := SecurityGroupCacheManager.Query().Equals("secgroup_id", secgroup.Id) + err := db.FetchModelObjects(SecurityGroupCacheManager, q, &caches) + if err != nil { + return err + } + for i := 0; i < len(caches); i++ { + cache := caches[i] + _, err := db.Update(&cache, func() error { + cache.SecgroupId = self.Id + return nil + }) + if err != nil { + return err + } + } + return nil +} + +func (self *SSecurityGroup) migrateGuestSecurityGroup(secgroup *SSecurityGroup) error { + guests := secgroup.GetGuests() + for i := 0; i < len(guests); i++ { + guest := guests[i] + _, err := db.Update(&guest, func() error { + if guest.SecgrpId == secgroup.Id { + guest.SecgrpId = self.Id + } + if guest.AdminSecgrpId == secgroup.Id { + guest.AdminSecgrpId = self.Id + } + return nil + }) + if err != nil { + return err + } + } + + guestsecgroups, err := GuestsecgroupManager.GetGuestSecgroups(nil, secgroup) + if err != nil { + return err + } + + for i := 0; i < len(guestsecgroups); i++ { + guestsecgroup := guestsecgroups[i] + _, err := db.Update(&guestsecgroup, func() error { + guestsecgroup.SecgroupId = self.Id + return nil + }) + if err != nil { + return err + } + } + + return nil +} + func (manager *SSecurityGroupManager) getSecurityGroups() ([]SSecurityGroup, error) { secgroups := make([]SSecurityGroup, 0) q := manager.Query() @@ -270,13 +416,13 @@ func (manager *SSecurityGroupManager) newFromCloudSecgroup(ctx context.Context, secgroup.SetModelManager(manager) secgroup.Name = db.GenerateName(manager, "", extSec.GetName()) secgroup.Description = extSec.GetDescription() + secgroup.ProjectId = userCred.GetProjectId() if err := manager.TableSpec().Insert(&secgroup); err != nil { return nil, err } db.OpsLog.LogEvent(&secgroup, db.ACT_CREATE, secgroup.GetShortDesc(ctx), userCred) - SyncCloudProject(userCred, &secgroup, provider.ProjectId, extSec, provider.Id) return &secgroup, nil } @@ -411,5 +557,16 @@ func (self *SSecurityGroup) Delete(ctx context.Context, userCred mcclient.TokenC } func (self *SSecurityGroup) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error { + rules := []SSecurityGroupRule{} + q := SecurityGroupRuleManager.Query().Equals("secgroup_id", self.Id) + if err := db.FetchModelObjects(SecurityGroupRuleManager, q, &rules); err != nil { + log.Errorf("failed to fetch secgroup %s rules error: %v", self.Name, err) + return err + } + for i := 0; i < len(rules); i++ { + if err := rules[i].Delete(ctx, userCred); err != nil { + return err + } + } return self.SVirtualResourceBase.Delete(ctx, userCred) } diff --git a/vendor/yunion.io/x/pkg/util/secrules/secrules.go b/vendor/yunion.io/x/pkg/util/secrules/secrules.go index 7770029a1e..589cd1c1a2 100644 --- a/vendor/yunion.io/x/pkg/util/secrules/secrules.go +++ b/vendor/yunion.io/x/pkg/util/secrules/secrules.go @@ -26,6 +26,18 @@ const ( SecurityRuleDeny = TSecurityRuleAction("deny") ) +type TSecurityRuleRelation string + +const ( + RELATION_INDEPENDENT = TSecurityRuleRelation("INDEPENDT") + RELATION_IDENTICAL = TSecurityRuleRelation("IDENTICAL") + RELATION_SUBSET = TSecurityRuleRelation("SUBSET") + RELATION_SUPERSET = TSecurityRuleRelation("SUPERSET") + RELATION_NEXT_AHEAD = TSecurityRuleRelation("NEXT_AHEAD") + RELATION_NEXT_AFTER = TSecurityRuleRelation("NEXT_AFTER") + RELATION_OVERLAP = TSecurityRuleRelation("OVERLAP") +) + type SecurityRule struct { Priority int // [1, 100] Action TSecurityRuleAction @@ -167,6 +179,86 @@ func (rule *SecurityRule) IsWildMatch() bool { rule.PortEnd == 0 } +func (rule SecurityRule) protoRelation(r SecurityRule) TSecurityRuleRelation { + if rule.Direction != r.Direction { + return RELATION_IDENTICAL + } + if rule.Protocol == r.Protocol { + if utils.IsInStringArray(rule.Protocol, []string{PROTO_ANY, PROTO_ICMP}) { + return RELATION_IDENTICAL + } + if rule.PortStart <= 0 && r.PortStart <= 0 { + return RELATION_IDENTICAL + } + if rule.PortStart > 0 && r.PortStart <= 0 { + return RELATION_SUBSET + } + if rule.PortStart <= 0 && r.PortStart > 0 { + return RELATION_SUPERSET + } + if rule.PortEnd+1 == r.PortStart { + return RELATION_NEXT_AHEAD + } + if rule.PortStart == r.PortEnd+1 { + return RELATION_NEXT_AFTER + } + if rule.PortEnd < r.PortStart || rule.PortStart > r.PortEnd { + return RELATION_INDEPENDENT + } + if rule.PortStart == r.PortStart && rule.PortEnd == r.PortEnd { + return RELATION_IDENTICAL + } + if rule.PortStart <= r.PortStart && rule.PortEnd >= r.PortEnd { + return RELATION_SUPERSET + } + if rule.PortStart >= r.PortStart && rule.PortEnd <= r.PortEnd { + return RELATION_SUBSET + } + return RELATION_OVERLAP + } + if rule.Protocol == PROTO_ANY { + return RELATION_SUPERSET + } + if r.Protocol == PROTO_ANY { + return RELATION_SUBSET + } + return RELATION_INDEPENDENT +} + +func (rule SecurityRule) merge(r SecurityRule) (SecurityRule, error) { + if rule.getIPKey() != r.getIPKey() { + panic(fmt.Sprintf("rule %v ip addr not equal rule %v", rule, r)) + } + if rule.Action != r.Action { + panic(fmt.Sprintf("rule %v action not equal rule %v", rule, r)) + } + rel := rule.protoRelation(r) + switch rel { + case RELATION_NEXT_AHEAD: + rule.PortEnd = r.PortEnd + return rule, nil + case RELATION_NEXT_AFTER: + rule.PortStart = r.PortStart + return rule, nil + case RELATION_OVERLAP: + if rule.PortStart > r.PortStart { + rule.PortStart = r.PortStart + } + if rule.PortEnd < r.PortEnd { + rule.PortEnd = r.PortEnd + } + return rule, nil + } + return r, fmt.Errorf("Cannot merge") +} + +func (rule SecurityRule) getIPKey() string { + if rule.IPNet == nil || rule.IPNet.String() == "0.0.0.0/0" { + return "0.0.0.0/0" + } + return rule.IPNet.String() +} + func (rule *SecurityRule) ParsePorts(seg string) error { if len(seg) == 0 { rule.Ports = []int{} diff --git a/vendor/yunion.io/x/pkg/util/secrules/securityruleset.go b/vendor/yunion.io/x/pkg/util/secrules/securityruleset.go new file mode 100644 index 0000000000..2dfa6a21c4 --- /dev/null +++ b/vendor/yunion.io/x/pkg/util/secrules/securityruleset.go @@ -0,0 +1,202 @@ +package secrules + +import ( + "bytes" + "sort" + + "yunion.io/x/log" +) + +type SecurityGroupSubSubRuleSet struct { + ipKey string + rules []SecurityRule +} + +func (sssrs *SecurityGroupSubSubRuleSet) AddRule(rule SecurityRule) { + if sssrs.rules == nil { + sssrs.ipKey = rule.getIPKey() + sssrs.rules = []SecurityRule{rule} + return + } + if sssrs.ipKey != rule.getIPKey() { + panic("rule ip key not equal") + } + idx := 0 + for idx < len(sssrs.rules) { + rel := sssrs.rules[idx].protoRelation(rule) + switch rel { + case RELATION_INDEPENDENT: + idx++ + case RELATION_IDENTICAL, RELATION_SUPERSET: + return + case RELATION_SUBSET: + sssrs.rules = append(sssrs.rules[:idx], sssrs.rules[idx+1:]...) + case RELATION_NEXT_AHEAD, RELATION_NEXT_AFTER, RELATION_OVERLAP: + r, err := rule.merge(sssrs.rules[idx]) + if err != nil { + log.Errorf("Merge failed?? %v", err) + continue + } + idx = 0 + sssrs.rules[idx] = r + } + } + sssrs.rules = append(sssrs.rules, rule) +} + +type SecurityGroupSubRuleSet struct { + allowRules *SecurityGroupSubSubRuleSet + denyRules *SecurityGroupSubSubRuleSet +} + +func (ssrs *SecurityGroupSubRuleSet) AddRule(rule SecurityRule) { + if rule.Action == SecurityRuleAllow { + if ssrs.allowRules == nil { + ssrs.allowRules = &SecurityGroupSubSubRuleSet{} + } + ssrs.allowRules.AddRule(rule) + } else { + if ssrs.denyRules == nil { + ssrs.denyRules = &SecurityGroupSubSubRuleSet{} + } + ssrs.denyRules.AddRule(rule) + } +} + +func (ssrs *SecurityGroupSubRuleSet) isEqual(_ssrs *SecurityGroupSubRuleSet) bool { + allowRules := ssrs.GetAllowRules() + _allowRules := _ssrs.GetAllowRules() + if len(allowRules.rules) != len(_allowRules.rules) { + return false + } + + sort.Slice(allowRules.rules, func(i, j int) bool { + return allowRules.rules[i].Priority < allowRules.rules[j].Priority + }) + + sort.Slice(_allowRules.rules, func(i, j int) bool { + return _allowRules.rules[i].Priority < _allowRules.rules[j].Priority + }) + for i := 0; i < len(allowRules.rules); i++ { + if allowRules.rules[i].String() != _allowRules.rules[i].String() { + return false + } + } + + denyRules := ssrs.GetDenyRules() + _denyRuels := _ssrs.GetDenyRules() + if len(denyRules.rules) != len(_denyRuels.rules) { + return false + } + + sort.Slice(denyRules.rules, func(i, j int) bool { + return denyRules.rules[i].Priority < denyRules.rules[j].Priority + }) + + sort.Slice(_denyRuels.rules, func(i, j int) bool { + return _denyRuels.rules[i].Priority < _denyRuels.rules[j].Priority + }) + for i := 0; i < len(denyRules.rules); i++ { + if denyRules.rules[i].String() != _denyRuels.rules[i].String() { + return false + } + } + + return true +} + +func (ssrs *SecurityGroupSubRuleSet) GetAllowRules() SecurityGroupSubSubRuleSet { + if ssrs.allowRules == nil { + return SecurityGroupSubSubRuleSet{} + } + return *ssrs.allowRules +} + +func (ssrs *SecurityGroupSubRuleSet) GetDenyRules() SecurityGroupSubSubRuleSet { + if ssrs.denyRules == nil { + return SecurityGroupSubSubRuleSet{} + } + return *ssrs.denyRules +} + +type SecurityGroupRuleSet struct { + rules map[string]*SecurityGroupSubRuleSet +} + +func (srs *SecurityGroupRuleSet) AddRule(rule SecurityRule) { + if srs.rules == nil { + srs.rules = map[string]*SecurityGroupSubRuleSet{} + } + key := rule.getIPKey() + if _, ok := srs.rules[key]; !ok { + srs.rules[key] = &SecurityGroupSubRuleSet{} + } + ssrs := srs.rules[key] + if len(rule.Ports) > 0 { + ports := rule.Ports + rule.Ports = []int{} + for i := 0; i < len(ports); i++ { + rule.PortStart = ports[i] + rule.PortEnd = ports[i] + ssrs.AddRule(rule) + } + return + } + ssrs.AddRule(rule) +} + +func (srs *SecurityGroupRuleSet) GetSubRuleSet(key string) *SecurityGroupSubRuleSet { + if srs.rules == nil { + return nil + } + s, ok := srs.rules[key] + if !ok { + return nil + } + return s +} + +func (srs *SecurityGroupRuleSet) GetAllowRules() []SecurityRule { + rules := []SecurityRule{} + for _, v := range srs.rules { + rules = append(rules, v.GetAllowRules().rules...) + } + return rules +} + +func (srs *SecurityGroupRuleSet) GetDenyRules() []SecurityRule { + rules := []SecurityRule{} + for _, v := range srs.rules { + rules = append(rules, v.GetDenyRules().rules...) + } + return rules +} + +func (srs *SecurityGroupRuleSet) IsEqual(src SecurityGroupRuleSet) bool { + if len(srs.rules) != len(src.rules) { + return false + } + for k, v := range srs.rules { + _v := src.GetSubRuleSet(k) + if _v == nil { + return false + } + if !v.isEqual(_v) { + return false + } + } + return true +} + +func (srs *SecurityGroupRuleSet) String() string { + buf := bytes.Buffer{} + for _, r := range srs.GetAllowRules() { + buf.WriteString(r.String()) + buf.WriteString(";") + } + for _, r := range srs.GetDenyRules() { + buf.WriteString(r.String()) + buf.WriteString(";") + } + return buf.String() +} diff --git a/vendor/yunion.io/x/pkg/util/secrules/securityruleset_test.go b/vendor/yunion.io/x/pkg/util/secrules/securityruleset_test.go new file mode 100644 index 0000000000..cfc9d95959 --- /dev/null +++ b/vendor/yunion.io/x/pkg/util/secrules/securityruleset_test.go @@ -0,0 +1,39 @@ +package secrules + +import "testing" + +func TestRules(t *testing.T) { + rules := []string{"in:allow 192.168.0.1/32 tcp 80", "out:deny any", "in:allow any", "out:allow udp 3232-3000", "in:allow tcp"} + for _, r := range rules { + rule, err := ParseSecurityRule(r) + if err != nil { + t.Fatal(err) + } + t.Log(rule) + } + + ruleSet := [][]string{ + []string{"in:allow any"}, + []string{"in:allow any", "in:deny tcp 25", "in:deny tcp 80"}, + []string{"in:allow tcp 80", "in:allow udp"}, + []string{"in:allow any", "in:allow tcp 80"}, + []string{"in:allow tcp 1000-1999", "in:allow tcp 2000", "in:allow tcp 2200"}, + []string{"in:allow tcp 1000-1999", "in:allow tcp 2000", "in:allow tcp 2200", + "in:deny tcp 500-1500", "in:deny tcp 2000-3000", "in:deny tcp 1888"}, + []string{"in:allow 192.168.0.0/16 tcp", "in:allow tcp 80", + "out:deny 192.168.0.0/16 tcp 80", "out:deny tcp 80", "in:allow any", + "in:allow 192.168.0.0/24 tcp", "out:deny 192.168.0.0/24 udp"}, + } + + for _, rs := range ruleSet { + srs := SecurityGroupRuleSet{} + for _, r := range rs { + rule, err := ParseSecurityRule(r) + if err != nil { + t.Fatalf("parse rule %s error: %v", r, err) + } + srs.AddRule(*rule) + } + t.Log(srs.String()) + } +}