From 8e39b7ccb44b2029d24eb8705884a040468a1124 Mon Sep 17 00:00:00 2001 From: Qu Xuan Date: Fri, 11 Sep 2020 14:19:05 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E9=BB=98=E8=AE=A4=E4=B8=8D=E8=87=AA?= =?UTF-8?q?=E5=8A=A8=E5=90=88=E5=B9=B6=E5=AE=89=E5=85=A8=E7=BB=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/climc/shell/compute/secgroups.go | 199 ++---------------- pkg/apis/compute/secgroup.go | 3 + pkg/compute/models/secgroupcache.go | 71 +++++-- pkg/compute/models/secgrouprules.go | 24 +-- pkg/compute/models/secgroups.go | 144 +++++++++---- pkg/compute/options/options.go | 2 + .../tasks/security_group_cache_delete_task.go | 38 +--- .../tasks/security_group_delete_task.go | 63 +++--- pkg/mcclient/options/base.go | 11 + pkg/mcclient/options/secgroup.go | 119 +++++++++++ 10 files changed, 361 insertions(+), 313 deletions(-) create mode 100644 pkg/mcclient/options/secgroup.go diff --git a/cmd/climc/shell/compute/secgroups.go b/cmd/climc/shell/compute/secgroups.go index 5402f62fcd..f1c74f50fb 100644 --- a/cmd/climc/shell/compute/secgroups.go +++ b/cmd/climc/shell/compute/secgroups.go @@ -15,194 +15,23 @@ package compute import ( - "fmt" - - "yunion.io/x/jsonutils" - - "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/cmd/climc/shell" "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/mcclient/options" ) func init() { - type SecGroupsListOptions struct { - Equals string `help:"Secgroup ID or Name, filter secgroups whose rules equals the specified one"` - Server string `help:"Filter secgroups bound to specified server"` - options.BaseListOptions - Ip string `help:"Filter secgroup by ip"` - Ports string `help:"Filter secgroup by ports"` - Direction string `help:"Filter secgroup by ports" choices:"all|in|out"` - } - - R(&SecGroupsListOptions{}, "secgroup-list", "List all security group", func(s *mcclient.ClientSession, args *SecGroupsListOptions) error { - params, err := options.ListStructToParams(args) - if err != nil { - return err - } - result, err := modules.SecGroups.List(s, params) - if err != nil { - return err - } - printList(result, modules.SecGroups.GetColumns(s)) - return nil - }) - - type SecGroupsCreateOptions struct { - NAME string `help:"Name of security group to create"` - RULES []string `help:"security rule to create"` - Desc string `help:"Description"` - } - - R(&SecGroupsCreateOptions{}, "secgroup-create", "Create a security group", func(s *mcclient.ClientSession, args *SecGroupsCreateOptions) error { - params := jsonutils.NewDict() - params.Add(jsonutils.NewString(args.NAME), "name") - if len(args.Desc) > 0 { - params.Add(jsonutils.NewString(args.Desc), "description") - } - for i, a := range args.RULES { - params.Add(jsonutils.NewString(a), fmt.Sprintf("rule.%d", i)) - } - secgroups, err := modules.SecGroups.Create(s, params) - if err != nil { - return err - } - printObject(secgroups) - return nil - - }) - - type SecGroupsUnionOptions struct { - ID string `help:"ID or Name of security group destination"` - SECGROUPS []string `help:"source IDs or Names of secgroup"` - } - - R(&SecGroupsUnionOptions{}, "secgroup-union", "Union secgroups to one secgroup", func(s *mcclient.ClientSession, args *SecGroupsUnionOptions) error { - params := jsonutils.NewDict() - secgroups := jsonutils.NewArray() - for i := 0; i < len(args.SECGROUPS); i++ { - secgroups.Add(jsonutils.NewString(args.SECGROUPS[i])) - } - params.Add(secgroups, "secgroups") - secgroup, err := modules.SecGroups.PerformAction(s, args.ID, "union", params) - if err != nil { - return err - } - printObject(secgroup) - return nil - - }) - - type SecGroupsDetailOptions struct { - ID string `help:"ID or Name of security group"` - } - R(&SecGroupsDetailOptions{}, "secgroup-show", "Show details of a security group", func(s *mcclient.ClientSession, args *SecGroupsDetailOptions) error { - result, err := modules.SecGroups.Get(s, args.ID, nil) - if err != nil { - return err - } - printObject(result) - return nil - }) - R(&SecGroupsDetailOptions{}, "secgroup-delete", "Delete a security group", func(s *mcclient.ClientSession, args *SecGroupsDetailOptions) error { - secgroups, err := modules.SecGroups.Delete(s, args.ID, nil) - if err != nil { - return err - } - printObject(secgroups) - return nil - }) - - R(&SecGroupsDetailOptions{}, "secgroup-public", "Make a security group publicly available", func(s *mcclient.ClientSession, args *SecGroupsDetailOptions) error { - result, err := modules.SecGroups.PerformAction(s, args.ID, "public", nil) - if err != nil { - return err - } - printObject(result) - return nil - }) - - R(&SecGroupsDetailOptions{}, "secgroup-private", "Make a security group private", func(s *mcclient.ClientSession, args *SecGroupsDetailOptions) error { - result, err := modules.SecGroups.PerformAction(s, args.ID, "private", nil) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type SecGroupsUpdateOptions struct { - ID string `help:"ID of security group"` - Name string `help:"Name of security group to update"` - Desc string `help:"Description of security groups"` - } - - R(&SecGroupsUpdateOptions{}, "secgroup-update", "Update details of a security group", func(s *mcclient.ClientSession, args *SecGroupsUpdateOptions) error { - params := jsonutils.NewDict() - if len(args.Name) > 0 { - params.Add(jsonutils.NewString(args.Name), "name") - } - if len(args.Desc) > 0 { - params.Add(jsonutils.NewString(args.Desc), "description") - } - secgroups, err := modules.SecGroups.Update(s, args.ID, params) - if err != nil { - return err - } - printObject(secgroups) - return nil - }) - - type SecGroupsAddRuleOptions struct { - ID string `help:"ID or Name of security group"` - DIRECTION string `help:"Direction of rule" choices:"in|out"` - PROTOCOL string `help:"Protocol of rule" choices:"any|tcp|udp|icmp"` - ACTION string `help:"Actin of rule" choices:"allow|deny"` - PRIORITY int `help:"Priority for rule, range 1 ~ 100"` - Cidr string `help:"IP or CIRD for rule"` - Description string `help:"Desciption for rule"` - Ports string `help:"Port for rule"` - } - - R(&SecGroupsAddRuleOptions{}, "secgroup-add-rule", "Add rule for a security group", func(s *mcclient.ClientSession, args *SecGroupsAddRuleOptions) error { - params, err := options.StructToParams(args) - secgroups, err := modules.SecGroups.PerformAction(s, args.ID, "add-rule", params) - if err != nil { - return err - } - printObject(secgroups) - return nil - }) - - type SecurityGroupCacheSecurityGroup struct { - ID string `help:"ID or Name of security group"` - VPC string `help:"ID or Name of vpc"` - Classic *bool `help:"Is classic vpc"` - } - - R(&SecurityGroupCacheSecurityGroup{}, "secgroup-cache-secgroup", "Cache security group for special vpc", func(s *mcclient.ClientSession, args *SecurityGroupCacheSecurityGroup) error { - params, err := options.StructToParams(args) - secgroups, err := modules.SecGroups.PerformAction(s, args.ID, "cache-secgroup", params) - if err != nil { - return err - } - printObject(secgroups) - return nil - }) - - type SecurityGroupUncacheSecurityGroup struct { - ID string `help:"ID or Name of security group"` - CACHE string `help:"ID of secgroup cache"` - } - - R(&SecurityGroupUncacheSecurityGroup{}, "secgroup-uncache-secgroup", "Unache special secgroup cache", func(s *mcclient.ClientSession, args *SecurityGroupUncacheSecurityGroup) error { - params := jsonutils.NewDict() - params.Add(jsonutils.NewString(args.CACHE), "secgroupcache") - secgroups, err := modules.SecGroups.PerformAction(s, args.ID, "uncache-secgroup", params) - if err != nil { - return err - } - printObject(secgroups) - return nil - }) - + cmd := shell.NewResourceCmd(&modules.SecGroups) + cmd.List(&options.SecgroupListOptions{}) + cmd.Create(&options.SecgroupCreateOptions{}) + cmd.Show(&options.SecgroupIdOptions{}) + cmd.Update(&options.BaseUpdateOptions{}) + cmd.Delete(&options.SecgroupIdOptions{}) + cmd.Perform("merge", &options.SecgroupMergeOptions{}) + cmd.Perform("public", &options.SecgroupIdOptions{}) + cmd.Perform("private", &options.SecgroupIdOptions{}) + cmd.Perform("add-rule", &options.SecgroupsAddRuleOptions{}) + cmd.Perform("cache-secgroup", &options.SecurityGroupCacheOptions{}) + cmd.Perform("uncache-secgroup", &options.SecurityGroupUncacheSecurityGroup{}) + cmd.Perform("purge", &options.SecgroupIdOptions{}) } diff --git a/pkg/apis/compute/secgroup.go b/pkg/apis/compute/secgroup.go index 3519a198b9..8e526d2bd7 100644 --- a/pkg/apis/compute/secgroup.go +++ b/pkg/apis/compute/secgroup.go @@ -257,3 +257,6 @@ type SecgroupMergeInput struct { // Deprecated Secgroups []string `json:"secgroup" yunion-deprecated-by:"secgroup_ids"` } + +type SecurityGroupPurgeInput struct { +} diff --git a/pkg/compute/models/secgroupcache.go b/pkg/compute/models/secgroupcache.go index 40f7dd096a..f5ed6c3202 100644 --- a/pkg/compute/models/secgroupcache.go +++ b/pkg/compute/models/secgroupcache.go @@ -355,6 +355,39 @@ func (self *SSecurityGroupCache) GetSecgroup() (*SSecurityGroup, error) { return model.(*SSecurityGroup), nil } +func (self *SSecurityGroupCache) syncWithCloudSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, ext cloudprovider.ICloudSecurityGroup) error { + _, err := db.Update(self, func() error { + self.Status = api.SECGROUP_CACHE_STATUS_READY + self.Name = ext.GetName() + self.Description = ext.GetDescription() + self.ExternalProjectId = ext.GetProjectId() + return nil + }) + if err != nil { + return errors.Wrapf(err, "db.Update") + } + secgroup, err := self.GetSecgroup() + if err != nil { + return errors.Wrapf(err, "GetSecurity") + } + cacheCount, err := secgroup.GetSecgroupCacheCount() + if err != nil { + return errors.Wrapf(err, "GetSecgroupCacheCount") + } + if cacheCount > 1 { + return nil + } + info, err := SecurityGroupManager.getRuleInfo(provider, ext) + if err != nil { + return errors.Wrapf(err, "getRuleInfo") + } + err = secgroup.SyncSecurityGroupRules(ctx, userCred, info) + if err != nil { + return errors.Wrapf(err, "SyncSecurityGroupRules") + } + return nil +} + func (manager *SSecurityGroupCacheManager) SyncSecurityGroupCaches(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, secgroups []cloudprovider.ICloudSecurityGroup, vpc *SVpc) ([]SSecurityGroup, []cloudprovider.ICloudSecurityGroup, compare.SyncResult) { lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred)) defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred)) @@ -415,25 +448,18 @@ func (manager *SSecurityGroupCacheManager) SyncSecurityGroupCaches(ctx context.C } for i := 0; i < len(commondb); i++ { - _, err = db.Update(&commondb[i], func() error { - commondb[i].Status = api.SECGROUP_CACHE_STATUS_READY - commondb[i].Name = commonext[i].GetName() - commondb[i].Description = commonext[i].GetDescription() - commondb[i].ExternalProjectId = commonext[i].GetProjectId() - return nil - }) + err = commondb[i].syncWithCloudSecurityGroup(ctx, userCred, provider, commonext[i]) if err != nil { - syncResult.UpdateError(err) - } else { - syncResult.Update() + syncResult.UpdateError(errors.Wrapf(err, "syncWithCloudSecurityGroup")) + continue } + syncResult.Update() } - //相同的不能同步, 原因: 多个平台的安全组可能共用一个本地安全组,下面仅仅是新加的安全组 for i := 0; i < len(added); i++ { secgroup, err := SecurityGroupManager.newFromCloudSecgroup(ctx, userCred, provider, added[i]) if err != nil { - syncResult.AddError(err) + syncResult.AddError(errors.Wrapf(err, "newFromCloudSecgroup")) continue } if secgroup.ProjectId != provider.ProjectId { @@ -450,7 +476,7 @@ func (manager *SSecurityGroupCacheManager) SyncSecurityGroupCaches(ctx context.C } cache, err := manager.NewCache(ctx, userCred, secgroup.Id, vpcId, vpc.CloudregionId, provider.Id, added[i].GetProjectId()) if err != nil { - syncResult.AddError(fmt.Errorf("failed to create secgroup cache for secgroup %s(%s) provider: %s: %s", secgroup.Name, secgroup.Name, provider.Name, err)) + syncResult.AddError(errors.Wrapf(err, "NewCache for secgroup %s provider %s", secgroup.Name, provider.Name)) continue } _, err = db.Update(cache, func() error { @@ -461,7 +487,7 @@ func (manager *SSecurityGroupCacheManager) SyncSecurityGroupCaches(ctx context.C return nil }) if err != nil { - syncResult.AddError(err) + syncResult.AddError(errors.Wrapf(err, "db.Update")) continue } localSecgroups = append(localSecgroups, *secgroup) @@ -560,3 +586,20 @@ func (manager *SSecurityGroupCacheManager) ListItemExportKeys(ctx context.Contex return q, nil } + +func (self *SSecurityGroupCache) GetISecurityGroup() (cloudprovider.ICloudSecurityGroup, error) { + if len(self.ExternalId) == 0 { + return nil, errors.Wrapf(cloudprovider.ErrNotFound, "empty externalId") + } + + manager := self.GetCloudprovider() + if manager == nil { + return nil, errors.Wrapf(cloudprovider.ErrNotFound, "failed to found manager") + } + + iRegion, err := self.GetIRegion() + if err != nil { + return nil, errors.Wrapf(err, "GetIRegion") + } + return iRegion.GetISecurityGroupById(self.ExternalId) +} diff --git a/pkg/compute/models/secgrouprules.go b/pkg/compute/models/secgrouprules.go index 003352b175..e714ea34e7 100644 --- a/pkg/compute/models/secgrouprules.go +++ b/pkg/compute/models/secgrouprules.go @@ -22,7 +22,6 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" - "yunion.io/x/pkg/util/compare" "yunion.io/x/pkg/util/regutils" "yunion.io/x/pkg/util/secrules" "yunion.io/x/pkg/util/stringutils" @@ -463,8 +462,7 @@ func (manager *SSecurityGroupRuleManager) getRulesBySecurityGroup(secgroup *SSec return rules, nil } -func (manager *SSecurityGroupRuleManager) SyncRules(ctx context.Context, userCred mcclient.TokenCredential, secgroup *SSecurityGroup, rules cloudprovider.SecurityRuleSet) compare.SyncResult { - syncResult := compare.SyncResult{} +func (self *SSecurityGroup) SyncRules(ctx context.Context, userCred mcclient.TokenCredential, rules cloudprovider.SecurityRuleSet) error { priority, prePriority := 10, 0 for i := 0; i < len(rules); i++ { // 这里避免了Rule规则优先级在 1-100之外的问题,ext.GetRules()不需要进行优先级转换 @@ -473,19 +471,17 @@ func (manager *SSecurityGroupRuleManager) SyncRules(ctx context.Context, userCre } prePriority = rules[i].Priority rules[i].Priority = priority - _, err := manager.newFromCloudSecurityGroup(ctx, userCred, rules[i], secgroup) + _, err := self.newFromCloudSecurityGroupRule(ctx, userCred, rules[i]) if err != nil { - syncResult.AddError(err) - continue + return errors.Wrapf(err, "newFromCloudSecurityGroupRule") } - syncResult.Add() } - return syncResult + return nil } -func (manager *SSecurityGroupRuleManager) newFromCloudSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, rule cloudprovider.SecurityRule, secgroup *SSecurityGroup) (*SSecurityGroupRule, error) { - lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred)) - defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred)) +func (self *SSecurityGroup) newFromCloudSecurityGroupRule(ctx context.Context, userCred mcclient.TokenCredential, rule cloudprovider.SecurityRule) (*SSecurityGroupRule, error) { + lockman.LockObject(ctx, self) + defer lockman.ReleaseObject(ctx, self) protocol := rule.Protocol if len(protocol) == 0 { @@ -506,11 +502,11 @@ func (manager *SSecurityGroupRuleManager) newFromCloudSecurityGroup(ctx context. Action: string(rule.Action), Description: rule.Description, } - secrule.SecgroupId = secgroup.Id + secrule.SecgroupId = self.Id - err := manager.TableSpec().Insert(ctx, secrule) + err := SecurityGroupRuleManager.TableSpec().Insert(ctx, secrule) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "SecurityGroupRuleManager.Insert") } return secrule, nil } diff --git a/pkg/compute/models/secgroups.go b/pkg/compute/models/secgroups.go index 605f83d3aa..0dbeff994d 100644 --- a/pkg/compute/models/secgroups.go +++ b/pkg/compute/models/secgroups.go @@ -37,6 +37,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/policy" "yunion.io/x/onecloud/pkg/cloudcommon/validators" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" @@ -539,6 +540,14 @@ func totalSecurityGroupCount(scope rbacutils.TRbacScope, ownerId mcclient.IIdent return q.CountWithError() } +func (self *SSecurityGroup) AllowPerformPurge(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "purge") +} + +func (self *SSecurityGroup) PerformPurge(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + return nil, self.StartDeleteSecurityGroupTask(ctx, userCred, true, "") +} + func (self *SSecurityGroup) AllowPerformUncacheSecgroup(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "uncache-secgroup") } @@ -814,7 +823,57 @@ func (manager *SSecurityGroupManager) getSecurityGroups() ([]SSecurityGroup, err } } -func (manager *SSecurityGroupManager) newFromCloudSecgroup(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, extSec cloudprovider.ICloudSecurityGroup) (*SSecurityGroup, error) { +func (self *SSecurityGroup) cleanRules(ctx context.Context, userCred mcclient.TokenCredential) error { + rules := []SSecurityGroupRule{} + q := SecurityGroupRuleManager.Query().Equals("secgroup_id", self.Id) + err := db.FetchModelObjects(SecurityGroupRuleManager, q, &rules) + if err != nil { + return errors.Wrapf(err, "db.FetchModelObjects") + } + for i := range rules { + err = rules[i].Delete(ctx, userCred) + if err != nil { + return errors.Wrapf(err, "DeleteRule(%s)", rules[i].Id) + } + } + return nil +} + +func (self *SSecurityGroup) SyncSecurityGroupRules(ctx context.Context, userCred mcclient.TokenCredential, info *sRuleInfo) error { + inRules := cloudprovider.AddDefaultRule(info.inRules, info.defaultInRule, "in:deny any", info.order, info.minPriority, info.maxPriority, info.onlyAllowRules) + cloudprovider.SortSecurityRule(inRules, info.order, info.onlyAllowRules) + outRules := cloudprovider.AddDefaultRule(info.outRules, info.defaultOutRule, "out:allow any", info.order, info.minPriority, info.maxPriority, info.onlyAllowRules) + cloudprovider.SortSecurityRule(outRules, info.order, info.onlyAllowRules) + + err := self.cleanRules(ctx, userCred) + if err != nil { + return errors.Wrapf(err, "cleanRules") + } + + err = self.SyncRules(ctx, userCred, inRules) + if err != nil { + return errors.Wrapf(err, "SyncInRules") + } + err = self.SyncRules(ctx, userCred, outRules) + if err != nil { + return errors.Wrapf(err, "SyncOutRules") + } + return nil +} + +type sRuleInfo struct { + rules []cloudprovider.SecurityRule + inRules []cloudprovider.SecurityRule + outRules []cloudprovider.SecurityRule + defaultInRule cloudprovider.SecurityRule + defaultOutRule cloudprovider.SecurityRule + order cloudprovider.TPriorityOrder + onlyAllowRules bool + maxPriority int + minPriority int +} + +func (manager *SSecurityGroupManager) getRuleInfo(provider *SCloudprovider, extSec cloudprovider.ICloudSecurityGroup) (*sRuleInfo, error) { regionDriver, err := provider.GetRegionDriver() if err != nil { return nil, errors.Wrap(err, "provider.GetRegionDriver") @@ -825,36 +884,48 @@ func (manager *SSecurityGroupManager) newFromCloudSecgroup(ctx context.Context, return nil, errors.Wrap(err, "extSec.GetRules") } - inRules := []cloudprovider.SecurityRule{} - outRules := []cloudprovider.SecurityRule{} + info := &sRuleInfo{ + rules: rules, + inRules: []cloudprovider.SecurityRule{}, + outRules: []cloudprovider.SecurityRule{}, + defaultInRule: regionDriver.GetDefaultSecurityGroupInRule(), + defaultOutRule: regionDriver.GetDefaultSecurityGroupOutRule(), + order: regionDriver.GetSecurityGroupRuleOrder(), + onlyAllowRules: regionDriver.IsOnlySupportAllowRules(), + maxPriority: regionDriver.GetSecurityGroupRuleMaxPriority(), + minPriority: regionDriver.GetSecurityGroupRuleMinPriority(), + } + for i := range rules { if rules[i].Direction == secrules.DIR_IN { - inRules = append(inRules, rules[i]) + info.inRules = append(info.inRules, rules[i]) } else { - outRules = append(outRules, rules[i]) + info.outRules = append(info.outRules, rules[i]) } } + return info, nil +} - maxPriority := regionDriver.GetSecurityGroupRuleMaxPriority() - minPriority := regionDriver.GetSecurityGroupRuleMinPriority() - - defaultInRule := regionDriver.GetDefaultSecurityGroupInRule() - defaultOutRule := regionDriver.GetDefaultSecurityGroupOutRule() - order := regionDriver.GetSecurityGroupRuleOrder() - onlyAllowRules := regionDriver.IsOnlySupportAllowRules() - - // 查询与provider在同域的安全组，比对寻找一个与云上安全组规则相同的安全组 - secgroups := []SSecurityGroup{} - q := manager.Query().Equals("domain_id", provider.DomainId) - err = db.FetchModelObjects(manager, q, &secgroups) +func (manager *SSecurityGroupManager) newFromCloudSecgroup(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, extSec cloudprovider.ICloudSecurityGroup) (*SSecurityGroup, error) { + info, err := manager.getRuleInfo(provider, extSec) if err != nil { - return nil, errors.Wrap(err, "db.FetchModelObjects") + return nil, errors.Wrapf(err, "getRuleInfo") } - for i := range secgroups { - localRules := secrules.SecurityRuleSet(secgroups[i].GetSecRules("")) - _, inAdds, outAdds, inDels, outDels := cloudprovider.CompareRules(minPriority, maxPriority, order, localRules, rules, defaultInRule, defaultOutRule, onlyAllowRules, false) - if len(inAdds) == 0 && len(outAdds) == 0 && len(inDels) == 0 && len(outDels) == 0 { - return &secgroups[i], nil + + if options.Options.EnableAutoMergeSecurityGroup { + // 查询与provider在同域的安全组，比对寻找一个与云上安全组规则相同的安全组 + secgroups := []SSecurityGroup{} + q := manager.Query().Equals("domain_id", provider.DomainId) + err = db.FetchModelObjects(manager, q, &secgroups) + if err != nil { + return nil, errors.Wrap(err, "db.FetchModelObjects") + } + for i := range secgroups { + localRules := secrules.SecurityRuleSet(secgroups[i].GetSecRules("")) + _, inAdds, outAdds, inDels, outDels := cloudprovider.CompareRules(info.minPriority, info.maxPriority, info.order, localRules, info.rules, info.defaultInRule, info.defaultOutRule, info.onlyAllowRules, false) + if len(inAdds) == 0 && len(outAdds) == 0 && len(inDels) == 0 && len(outDels) == 0 { + return &secgroups[i], nil + } } } @@ -863,31 +934,26 @@ func (manager *SSecurityGroupManager) newFromCloudSecgroup(ctx context.Context, secgroup := SSecurityGroup{} secgroup.SetModelManager(manager, &secgroup) - newName, err := db.GenerateName(manager, userCred, extSec.GetName()) + secgroup.Name, err = db.GenerateName(manager, userCred, extSec.GetName()) if err != nil { return nil, err } - secgroup.Name = newName secgroup.Description = extSec.GetDescription() secgroup.ProjectId = provider.ProjectId secgroup.DomainId = provider.DomainId - if err := manager.TableSpec().Insert(ctx, &secgroup); err != nil { - return nil, err + err = manager.TableSpec().Insert(ctx, &secgroup) + if err != nil { + return nil, errors.Wrapf(err, "Insert") } - //这里必须先同步下规则,不然下次对比此安全组规则为空 - inRules = cloudprovider.AddDefaultRule(inRules, defaultInRule, "in:deny any", order, minPriority, maxPriority, onlyAllowRules) - cloudprovider.SortSecurityRule(inRules, order, onlyAllowRules) - outRules = cloudprovider.AddDefaultRule(outRules, defaultOutRule, "out:allow any", order, minPriority, maxPriority, onlyAllowRules) - cloudprovider.SortSecurityRule(outRules, order, onlyAllowRules) - - SecurityGroupRuleManager.SyncRules(ctx, userCred, &secgroup, inRules) - SecurityGroupRuleManager.SyncRules(ctx, userCred, &secgroup, outRules) + err = secgroup.SyncSecurityGroupRules(ctx, userCred, info) + if err != nil { + return nil, errors.Wrapf(err, "SyncSecurityGroupRules") + } db.OpsLog.LogEvent(&secgroup, db.ACT_CREATE, secgroup.GetShortDesc(ctx), userCred) - return &secgroup, nil } @@ -1029,10 +1095,12 @@ func (self *SSecurityGroup) GetSecurityGroupCaches() ([]SSecurityGroupCache, err } func (self *SSecurityGroup) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { - return self.StartDeleteSecurityGroupTask(ctx, userCred, jsonutils.NewDict(), "") + return self.StartDeleteSecurityGroupTask(ctx, userCred, false, "") } -func (self *SSecurityGroup) StartDeleteSecurityGroupTask(ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, parentTaskId string) error { +func (self *SSecurityGroup) StartDeleteSecurityGroupTask(ctx context.Context, userCred mcclient.TokenCredential, isPurge bool, parentTaskId string) error { + params := jsonutils.NewDict() + params.Add(jsonutils.NewBool(isPurge), "purge") self.SetStatus(userCred, api.SECGROUP_STATUS_DELETING, "") task, err := taskman.TaskManager.NewTask(ctx, "SecurityGroupDeleteTask", self, userCred, params, parentTaskId, "", nil) if err != nil { diff --git a/pkg/compute/options/options.go b/pkg/compute/options/options.go index b49c357cc7..305ae89f36 100644 --- a/pkg/compute/options/options.go +++ b/pkg/compute/options/options.go @@ -145,6 +145,8 @@ type ComputeOptions struct { SASControllerOptions common_options.CommonOptions common_options.DBOptions + + EnableAutoMergeSecurityGroup bool `help:"Enable auto merge secgroup when sync security group from cloud, default False" default:"false"` } type SCapabilityOptions struct { diff --git a/pkg/compute/tasks/security_group_cache_delete_task.go b/pkg/compute/tasks/security_group_cache_delete_task.go index 38c0041b37..0af205979b 100644 --- a/pkg/compute/tasks/security_group_cache_delete_task.go +++ b/pkg/compute/tasks/security_group_cache_delete_task.go @@ -16,7 +16,6 @@ package tasks import ( "context" - "database/sql" "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" @@ -26,7 +25,6 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/onecloud/pkg/util/logclient" ) type SecurityGroupCacheDeleteTask struct { @@ -37,13 +35,9 @@ func init() { taskman.RegisterTask(SecurityGroupCacheDeleteTask{}) } -func (self *SecurityGroupCacheDeleteTask) taskFailed(ctx context.Context, cache *models.SSecurityGroupCache, err jsonutils.JSONObject) { - cache.SetStatus(self.UserCred, api.SECGROUP_CACHE_STATUS_DELETE_FAILED, err.String()) - secgroup, _ := cache.GetSecgroup() - if secgroup != nil { - logclient.AddActionLogWithStartable(self, secgroup, logclient.ACT_DELETE, err, self.UserCred, false) - } - self.SetStageFailed(ctx, err) +func (self *SecurityGroupCacheDeleteTask) taskFailed(ctx context.Context, cache *models.SSecurityGroupCache, err error) { + cache.SetStatus(self.UserCred, api.SECGROUP_CACHE_STATUS_DELETE_FAILED, err.Error()) + self.SetStageFailed(ctx, jsonutils.NewString(err.Error())) } func (self *SecurityGroupCacheDeleteTask) taskComplete(ctx context.Context, cache *models.SSecurityGroupCache) { @@ -54,38 +48,18 @@ func (self *SecurityGroupCacheDeleteTask) taskComplete(ctx context.Context, cach func (self *SecurityGroupCacheDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { cache := obj.(*models.SSecurityGroupCache) - if len(cache.ExternalId) == 0 { - self.taskComplete(ctx, cache) - return - } - - _, err := models.CloudproviderManager.FetchById(cache.ManagerId) - if err == sql.ErrNoRows { - self.taskComplete(ctx, cache) - return - } - - iRegion, err := cache.GetIRegion() + iSecgroup, err := cache.GetISecurityGroup() if err != nil { if errors.Cause(err) == cloudprovider.ErrNotFound { self.taskComplete(ctx, cache) return } - self.taskFailed(ctx, cache, jsonutils.NewString(errors.Wrap(err, "cache.GetIRegion").Error())) - return - } - iSecgroup, err := iRegion.GetISecurityGroupById(cache.ExternalId) - if err != nil { - if err == cloudprovider.ErrNotFound { - self.taskComplete(ctx, cache) - return - } - self.taskFailed(ctx, cache, jsonutils.NewString(errors.Wrap(err, "iRegion.GetIStoragecacheById").Error())) + self.taskFailed(ctx, cache, errors.Wrapf(err, "GetISecurityGroup")) return } err = iSecgroup.Delete() if err != nil { - self.taskFailed(ctx, cache, jsonutils.NewString(err.Error())) + self.taskFailed(ctx, cache, errors.Wrapf(err, "iSecgroup.Delete")) return } self.taskComplete(ctx, cache) diff --git a/pkg/compute/tasks/security_group_delete_task.go b/pkg/compute/tasks/security_group_delete_task.go index 14ca7c43d2..781db17b95 100644 --- a/pkg/compute/tasks/security_group_delete_task.go +++ b/pkg/compute/tasks/security_group_delete_task.go @@ -18,11 +18,14 @@ import ( "context" "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/util/logclient" ) type SecurityGroupDeleteTask struct { @@ -33,42 +36,42 @@ func init() { taskman.RegisterTask(SecurityGroupDeleteTask{}) } -func (self *SecurityGroupDeleteTask) getErrorCount() int64 { - count, _ := self.GetParams().Int("faild_count") - return count -} - -func (self *SecurityGroupDeleteTask) addErrorCount() { - count := self.getErrorCount() - count += 1 - self.GetParams().Set("failed_count", jsonutils.NewInt(count)) +func (self *SecurityGroupDeleteTask) taskFailed(ctx context.Context, secgroup *models.SSecurityGroup, err error) { + secgroup.SetStatus(self.UserCred, api.SECGROUP_STATUS_READY, "") + logclient.AddActionLogWithContext(ctx, secgroup, logclient.ACT_DELOCATE, err, self.UserCred, false) + self.SetStageFailed(ctx, jsonutils.Marshal(err)) } func (self *SecurityGroupDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { - self.SetStage("OnSecurityGroupUncacheComplete", nil) - self.OnSecurityGroupUncacheComplete(ctx, obj, data) -} - -func (self *SecurityGroupDeleteTask) OnSecurityGroupUncacheComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { secgroup := obj.(*models.SSecurityGroup) - secgroupCaches, err := secgroup.GetSecurityGroupCaches() + caches, err := secgroup.GetSecurityGroupCaches() if err != nil { - secgroup.SetStatus(self.UserCred, api.SECGROUP_STATUS_READY, "") - self.SetStageFailed(ctx, jsonutils.Marshal(err)) + self.taskFailed(ctx, secgroup, errors.Wrapf(err, "GetSecurityGroupCaches")) return } - errCount := self.getErrorCount() - if len(secgroupCaches) == int(errCount) { - if errCount == 0 { - secgroup.RealDelete(ctx, self.UserCred) - } - secgroup.SetStatus(self.UserCred, api.SECGROUP_STATUS_READY, "") - self.SetStageComplete(ctx, nil) - return - } - secgroupCaches[errCount].StartSecurityGroupCacheDeleteTask(ctx, self.UserCred, self.GetTaskId()) -} -func (self *SecurityGroupDeleteTask) OnSecurityGroupUncacheCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) { - self.addErrorCount() + isPurge := jsonutils.QueryBoolean(self.Params, "purge", false) + + for i := range caches { + if !isPurge { + iSecgroup, err := caches[i].GetISecurityGroup() + if err != nil { + if errors.Cause(err) == cloudprovider.ErrNotFound { + caches[i].RealDelete(ctx, self.GetUserCred()) + continue + } + self.taskFailed(ctx, secgroup, errors.Wrapf(err, "GetISecurityGroup for cache %s(%s)", caches[i].Name, caches[i].Id)) + return + } + err = iSecgroup.Delete() + if err != nil { + self.taskFailed(ctx, secgroup, errors.Wrapf(err, "iSecgroup.Delete")) + return + } + } + caches[i].RealDelete(ctx, self.GetUserCred()) + } + + secgroup.RealDelete(ctx, self.GetUserCred()) + self.SetStageComplete(ctx, nil) } diff --git a/pkg/mcclient/options/base.go b/pkg/mcclient/options/base.go index 5ef8030d98..9a1e7ac87e 100644 --- a/pkg/mcclient/options/base.go +++ b/pkg/mcclient/options/base.go @@ -337,6 +337,17 @@ func (opts *BaseUpdateOptions) GetId() string { return opts.ID } +func (opts *BaseUpdateOptions) Params() (jsonutils.JSONObject, error) { + params := jsonutils.NewDict() + if len(opts.Name) > 0 { + params.Add(jsonutils.NewString(opts.Name), "name") + } + if len(opts.Desc) > 0 { + params.Add(jsonutils.NewString(opts.Desc), "description") + } + return params, nil +} + type BasePublicOptions struct { ID string `help:"ID or name of resource" json:"-"` Scope string `help:"sharing scope" choices:"system|domain"` diff --git a/pkg/mcclient/options/secgroup.go b/pkg/mcclient/options/secgroup.go new file mode 100644 index 0000000000..f20af8b742 --- /dev/null +++ b/pkg/mcclient/options/secgroup.go @@ -0,0 +1,119 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package options + +import ( + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/secrules" +) + +type SecgroupListOptions struct { + BaseListOptions + + Equals string `help:"Secgroup ID or Name, filter secgroups whose rules equals the specified one"` + Server string `help:"Filter secgroups bound to specified server"` + Ip string `help:"Filter secgroup by ip"` + Ports string `help:"Filter secgroup by ports"` + Direction string `help:"Filter secgroup by ports" choices:"all|in|out"` +} + +func (opts *SecgroupListOptions) Params() (jsonutils.JSONObject, error) { + return ListStructToParams(opts) +} + +type SecgroupCreateOptions struct { + BaseCreateOptions + Rules []string `help:"security rule to create"` +} + +func (opts *SecgroupCreateOptions) Params() (jsonutils.JSONObject, error) { + params := jsonutils.Marshal(opts).(*jsonutils.JSONDict) + params.Remove("rules") + rules := []secrules.SecurityRule{} + for i, ruleStr := range opts.Rules { + rule, err := secrules.ParseSecurityRule(ruleStr) + if err != nil { + return nil, errors.Wrapf(err, "ParseSecurityRule(%s)", ruleStr) + } + rule.Priority = i + 1 + rules = append(rules, *rule) + } + if len(rules) > 0 { + params.Add(jsonutils.Marshal(rules), "rules") + } + return params, nil +} + +type SecgroupIdOptions struct { + ID string `help:"ID or Name of security group destination"` +} + +func (opts *SecgroupIdOptions) GetId() string { + return opts.ID +} + +func (opts *SecgroupIdOptions) Params() (jsonutils.JSONObject, error) { + return nil, nil +} + +type SecgroupMergeOptions struct { + SecgroupIdOptions + SECGROUPS []string `help:"source IDs or Names of secgroup"` +} + +func (opts *SecgroupMergeOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(map[string][]string{"secgruops": opts.SECGROUPS}), nil +} + +type SecgroupsAddRuleOptions struct { + SecgroupIdOptions + DIRECTION string `help:"Direction of rule" choices:"in|out"` + PROTOCOL string `help:"Protocol of rule" choices:"any|tcp|udp|icmp"` + ACTION string `help:"Actin of rule" choices:"allow|deny"` + PRIORITY int `help:"Priority for rule, range 1 ~ 100"` + Cidr string `help:"IP or CIRD for rule"` + Description string `help:"Desciption for rule"` + Ports string `help:"Port for rule"` +} + +func (opts *SecgroupsAddRuleOptions) Params() (jsonutils.JSONObject, error) { + params := jsonutils.Marshal(opts).(*jsonutils.JSONDict) + params.Remove("id") + return params, nil +} + +type SecurityGroupCacheOptions struct { + SecgroupIdOptions + VPC string `help:"ID or Name of vpc"` + Classic *bool `help:"Is classic vpc"` +} + +func (opts *SecurityGroupCacheOptions) Params() (jsonutils.JSONObject, error) { + params := jsonutils.Marshal(opts).(*jsonutils.JSONDict) + params.Remove("id") + return params, nil +} + +type SecurityGroupUncacheSecurityGroup struct { + SecgroupIdOptions + CACHE string `help:"ID of secgroup cache"` +} + +func (opts *SecurityGroupUncacheSecurityGroup) Params() (jsonutils.JSONObject, error) { + params := jsonutils.Marshal(opts).(*jsonutils.JSONDict) + params.Remove("id") + return params, nil +}