diff --git a/pkg/cloudid/models/cloudaccount.go b/pkg/cloudid/models/cloudaccount.go index 5ce59a256f..44066f6c3d 100644 --- a/pkg/cloudid/models/cloudaccount.go +++ b/pkg/cloudid/models/cloudaccount.go @@ -1057,7 +1057,7 @@ func (self *SCloudaccount) newCloudgroup(ctx context.Context, userCred mcclient. func (self *SCloudaccount) GetSystemPolicyByExternalId(id string) (*SCloudpolicy, error) { policies := []SCloudpolicy{} - q := CloudpolicyManager.Query().Equals("external_id", id).Equals("provider", self.Provider) + q := CloudpolicyManager.Query().Equals("external_id", id).Equals("provider", self.Provider).Equals("policy_type", api.CLOUD_POLICY_TYPE_SYSTEM) err := db.FetchModelObjects(CloudpolicyManager, q, &policies) if err != nil { return nil, errors.Wrapf(err, "db.FetchModelObjects") @@ -1162,7 +1162,7 @@ func (self *SCloudaccount) SyncCustomCloudpoliciesForCloud(ctx context.Context, for i := range providers { err = providers[i].SyncCustomCloudpoliciesForCloud(ctx, clouduser) if err != nil { - return errors.Wrapf(err, "SyncSystemCloudpoliciesForCloud for cloudprovider %s", providers[i].Name) + return errors.Wrapf(err, "SyncCustomCloudpoliciesForCloud for cloudprovider %s", providers[i].Name) } } return nil @@ -1171,7 +1171,7 @@ func (self *SCloudaccount) SyncCustomCloudpoliciesForCloud(ctx context.Context, policyIds := []string{} policies, err := clouduser.GetCustomCloudpolicies("") if err != nil { - return errors.Wrap(err, "GetSystemCloudpolicies") + return errors.Wrap(err, "GetCustomCloudpolicies") } for i := range policies { err = self.getOrCacheCustomCloudpolicy(ctx, "", policies[i].Id) @@ -1184,7 +1184,7 @@ func (self *SCloudaccount) SyncCustomCloudpoliciesForCloud(ctx context.Context, if !factory.IsSupportCreateCloudgroup() { policies, err = clouduser.GetCustomCloudgroupPolicies() if err != nil { - return errors.Wrap(err, "GetSystemCloudgroupPolicies") + return errors.Wrap(err, "GetCustomCloudgroupPolicies") } for i := range policies { err = self.getOrCacheCustomCloudpolicy(ctx, "", policies[i].Id) @@ -1195,9 +1195,12 @@ func (self *SCloudaccount) SyncCustomCloudpoliciesForCloud(ctx context.Context, } } - dbCaches, err := self.GetCloudpolicycaches(policyIds, "") - if err != nil { - return errors.Wrapf(err, "GetCloudpolicycaches") + dbCaches := []SCloudpolicycache{} + if len(policyIds) > 0 { + dbCaches, err = self.GetCloudpolicycaches(policyIds, "") + if err != nil { + return errors.Wrapf(err, "GetCloudpolicycaches") + } } iUser, err := clouduser.GetIClouduser() @@ -1207,7 +1210,7 @@ func (self *SCloudaccount) SyncCustomCloudpoliciesForCloud(ctx context.Context, iPolicies, err := iUser.GetICustomCloudpolicies() if err != nil { - return errors.Wrap(err, "GetISystemCloudpolicies") + return errors.Wrap(err, "GetICustomCloudpolicies") } added := make([]SCloudpolicycache, 0) diff --git a/pkg/cloudid/models/cloudgroupcaches.go b/pkg/cloudid/models/cloudgroupcaches.go index 7e82b7c30a..7c8328cf40 100644 --- a/pkg/cloudid/models/cloudgroupcaches.go +++ b/pkg/cloudid/models/cloudgroupcaches.go @@ -285,6 +285,10 @@ func (self *SCloudgroupcache) GetOrCreateICloudgroup(ctx context.Context, userCr if err != nil { return nil, errors.Wrapf(err, "SyncCustomCloudpoliciesForCloud") } + err = self.SyncCloudusersForCloud(ctx, userCred) + if err != nil { + return nil, errors.Wrapf(err, "SyncCloudusersForCloud") + } return iGroup, nil } diff --git a/pkg/cloudid/models/cloudprovider.go b/pkg/cloudid/models/cloudprovider.go index 691242ee5b..d8f12446f0 100644 --- a/pkg/cloudid/models/cloudprovider.go +++ b/pkg/cloudid/models/cloudprovider.go @@ -168,9 +168,12 @@ func (self *SCloudprovider) SyncCustomCloudpoliciesForCloud(ctx context.Context, } } - dbCaches, err := account.GetCloudpolicycaches(policyIds, self.Id) - if err != nil { - return errors.Wrapf(err, "GetCloudpolicycaches") + dbCaches := []SCloudpolicycache{} + if len(policyIds) > 0 { + dbCaches, err = account.GetCloudpolicycaches(policyIds, self.Id) + if err != nil { + return errors.Wrapf(err, "GetCloudpolicycaches") + } } provider, err := self.GetProvider() diff --git a/pkg/multicloud/aws/provider/provider.go b/pkg/multicloud/aws/provider/provider.go index 80eb78f983..06ff33db0b 100644 --- a/pkg/multicloud/aws/provider/provider.go +++ b/pkg/multicloud/aws/provider/provider.go @@ -209,7 +209,7 @@ func (self *SAwsProvider) GetISystemCloudpolicies() ([]cloudprovider.ICloudpolic } func (self *SAwsProvider) GetICustomCloudpolicies() ([]cloudprovider.ICloudpolicy, error) { - return self.client.GetISystemCloudpolicies() + return self.client.GetICustomCloudpolicies() } func (self *SAwsProvider) GetIClouduserByName(name string) (cloudprovider.IClouduser, error) { diff --git a/pkg/multicloud/azure/cloudpolicy.go b/pkg/multicloud/azure/cloudpolicy.go index 38acc0ce2b..25446a2fde 100644 --- a/pkg/multicloud/azure/cloudpolicy.go +++ b/pkg/multicloud/azure/cloudpolicy.go @@ -17,6 +17,7 @@ package azure import ( "fmt" "net/url" + "strings" "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" @@ -71,15 +72,22 @@ func (role *SCloudpolicy) Delete() error { return cloudprovider.ErrNotImplemented } -func (cli *SAzureClient) GetRoles(name string) ([]SCloudpolicy, error) { +func (cli *SAzureClient) GetRoles(name, policyType string) ([]SCloudpolicy, error) { ret := []SCloudpolicy{} subscriptionId, err := cli.getDefaultSubscriptionId() if err != nil { return nil, errors.Wrap(err, "getDefaultSubscriptionId") } params := url.Values{} + filter := []string{} if len(name) > 0 { - params.Set("$filter", fmt.Sprintf("roleName eq '%s'", name)) + filter = append(filter, fmt.Sprintf("roleName eq '%s'", name)) + } + if len(policyType) > 0 { + filter = append(filter, fmt.Sprintf("Type eq '%s'", policyType)) + } + if len(filter) > 0 { + params.Set("$filter", strings.Join(filter, " and ")) } resource := "providers/Microsoft.Authorization/roleDefinitions" if len(params) > 0 { @@ -92,8 +100,20 @@ func (cli *SAzureClient) GetRoles(name string) ([]SCloudpolicy, error) { return ret, nil } -func (cli *SAzureClient) GetICloudpolicies() ([]cloudprovider.ICloudpolicy, error) { - roles, err := cli.GetRoles("") +func (cli *SAzureClient) GetISystemCloudpolicies() ([]cloudprovider.ICloudpolicy, error) { + roles, err := cli.GetRoles("", "BuiltInRole") + if err != nil { + return nil, errors.Wrap(err, "GetRoles") + } + ret := []cloudprovider.ICloudpolicy{} + for i := range roles { + ret = append(ret, &roles[i]) + } + return ret, nil +} + +func (cli *SAzureClient) GetICustomCloudpolicies() ([]cloudprovider.ICloudpolicy, error) { + roles, err := cli.GetRoles("", "CustomRole") if err != nil { return nil, errors.Wrap(err, "GetRoles") } @@ -105,7 +125,7 @@ func (cli *SAzureClient) GetICloudpolicies() ([]cloudprovider.ICloudpolicy, erro } func (cli *SAzureClient) AssignPolicy(objectId, roleName, subscriptionId string) error { - roles, err := cli.GetRoles(roleName) + roles, err := cli.GetRoles(roleName, "") if err != nil { return errors.Wrapf(err, "GetRoles(%s)", roleName) } diff --git a/pkg/multicloud/azure/provider/provider.go b/pkg/multicloud/azure/provider/provider.go index 1b3f4fab9e..98bf53fcc0 100644 --- a/pkg/multicloud/azure/provider/provider.go +++ b/pkg/multicloud/azure/provider/provider.go @@ -261,7 +261,11 @@ func (self *SAzureProvider) GetEnrollmentAccounts() ([]cloudprovider.SEnrollment } func (self *SAzureProvider) GetISystemCloudpolicies() ([]cloudprovider.ICloudpolicy, error) { - return self.client.GetICloudpolicies() + return self.client.GetISystemCloudpolicies() +} + +func (self *SAzureProvider) GetICustomCloudpolicies() ([]cloudprovider.ICloudpolicy, error) { + return self.client.GetICustomCloudpolicies() } func (self *SAzureProvider) CreateSubscription(input cloudprovider.SubscriptionCreateInput) error { diff --git a/pkg/multicloud/azure/shell/cloudpolicy.go b/pkg/multicloud/azure/shell/cloudpolicy.go index 1ceea20179..54b26084af 100644 --- a/pkg/multicloud/azure/shell/cloudpolicy.go +++ b/pkg/multicloud/azure/shell/cloudpolicy.go @@ -21,10 +21,11 @@ import ( func init() { type CloudpolicyListOptions struct { - Name string + Name string + PolicyType string `choices:"CustomRole|BuiltInRole"` } shellutils.R(&CloudpolicyListOptions{}, "cloud-policy-list", "List cloudpolicies", func(cli *azure.SRegion, args *CloudpolicyListOptions) error { - roles, err := cli.GetClient().GetRoles(args.Name) + roles, err := cli.GetClient().GetRoles(args.Name, args.PolicyType) if err != nil { return err } diff --git a/pkg/multicloud/google/dbinstance.go b/pkg/multicloud/google/dbinstance.go index 9a376dc36e..3169d490ba 100644 --- a/pkg/multicloud/google/dbinstance.go +++ b/pkg/multicloud/google/dbinstance.go @@ -606,7 +606,7 @@ func (region *SRegion) CreateRds(name, engine, databaseVersion, category, instan rds := SDBInstance{region: region} err := region.rdsInsert("instances", jsonutils.Marshal(body), &rds) if err != nil { - if e, ok := errors.Cause(err).(*gError); ok && e.Code == 409 { //The instance or operation is not in an appropriate state to handle the request + if e, ok := errors.Cause(err).(*gError); ok && e.ErrorInfo.Code == 409 { //The instance or operation is not in an appropriate state to handle the request return nil, fmt.Errorf("the name %s is unavailable because it was used recently", name) } return nil, errors.Wrap(err, "rdsInsert") diff --git a/pkg/multicloud/google/google.go b/pkg/multicloud/google/google.go index 6fe3352e29..ade5db99cc 100644 --- a/pkg/multicloud/google/google.go +++ b/pkg/multicloud/google/google.go @@ -762,60 +762,82 @@ func rawRequest(client *http.Client, method httputils.THttpMethod, domain, apiVe return httputils.Request(client, context.Background(), method, resource, header, body, debug) } +/* + "error": { + "code": 400, + "message": "Request contains an invalid argument.", + "status": "INVALID_ARGUMENT", + "details": [ + { + "@type": "type.googleapis.com/google.cloudresourcemanager.v1.ProjectIamPolicyError", + "type": "SOLO_MUST_INVITE_OWNERS", + "member": "user:test", + "role": "roles/owner" + } + ] + } +*/ + type gError struct { - Code int - err error + ErrorInfo struct { + Code int + Message string + Status string + Details jsonutils.JSONObject + } `json:"error"` + Class string } func (g *gError) Error() string { - return g.err.Error() + return jsonutils.Marshal(g).String() } -func _jsonRequest(client *http.Client, method httputils.THttpMethod, url string, body jsonutils.JSONObject, debug bool) (jsonutils.JSONObject, error) { - var ( - retry bool = false - err error = nil - data jsonutils.JSONObject = nil - ) +func (g *gError) ParseErrorFromJsonResponse(statusCode int, body jsonutils.JSONObject) error { + if body != nil { + body.Unmarshal(g) + } + if g.ErrorInfo.Code == 0 { + g.ErrorInfo.Code = statusCode + } + if g.ErrorInfo.Details == nil { + g.ErrorInfo.Details = body + } + if len(g.Class) == 0 { + g.Class = http.StatusText(statusCode) + } + if statusCode == 404 { + return errors.Wrap(cloudprovider.ErrNotFound, g.Error()) + } + return g +} + +func _jsonRequest(cli *http.Client, method httputils.THttpMethod, url string, body jsonutils.JSONObject, debug bool) (jsonutils.JSONObject, error) { + client := httputils.NewJsonClient(cli) + req := httputils.NewJsonRequest(method, url, body) + var ge gError for i := 0; i < MAX_RETRY; i++ { - _, data, err = httputils.JSONRequest(client, context.Background(), method, url, nil, body, debug) - if err != nil { - if body != nil { - log.Errorf("%s %s params: %s error: %v", method, url, body.PrettyString(), err) - } else { - log.Errorf("%s %s error: %v", method, url, err) - } - for _, msg := range []string{ - "EOF", - "i/o timeout", - "TLS handshake timeout", - "connection reset by peer", - } { - if strings.Index(err.Error(), msg) >= 0 { - retry = true - break - } - } - if !retry { - break + _, data, err := client.Send(context.Background(), req, &ge, debug) + if err == nil { + return data, nil + } + if body != nil { + log.Errorf("%s %s params: %s error: %v", method, url, body.PrettyString(), err) + } else { + log.Errorf("%s %s error: %v", method, url, err) + } + for _, msg := range []string{ + "EOF", + "i/o timeout", + "TLS handshake timeout", + "connection reset by peer", + } { + if strings.Index(err.Error(), msg) >= 0 { + continue } } - if !retry { - break - } + return nil, &ge } - if err != nil { - ge := &gError{err: err} - e, ok := err.(*httputils.JSONClientError) - if ok { - if e.Code == 404 { - return nil, cloudprovider.ErrNotFound - } - ge.Code = e.Code - } - return nil, ge - } - return data, nil + return nil, &ge } func (self *SGoogleClient) GetRegion(regionId string) *SRegion { diff --git a/pkg/multicloud/qcloud/cam_user.go b/pkg/multicloud/qcloud/cam_user.go index f7b43736c3..7c9f533331 100644 --- a/pkg/multicloud/qcloud/cam_user.go +++ b/pkg/multicloud/qcloud/cam_user.go @@ -64,7 +64,27 @@ func (user *SUser) GetISystemCloudpolicies() ([]cloudprovider.ICloudpolicy, erro } func (user *SUser) GetICustomCloudpolicies() ([]cloudprovider.ICloudpolicy, error) { - return nil, cloudprovider.ErrNotImplemented + policies := []SPolicy{} + offset := 1 + for { + part, total, err := user.client.ListAttachedUserPolicies(user.GetGlobalId(), offset, 50) + if err != nil { + return nil, errors.Wrap(err, "GetClouduserPolicy") + } + policies = append(policies, part...) + if len(policies) >= total { + break + } + offset += 1 + } + ret := []cloudprovider.ICloudpolicy{} + for i := range policies { + if policies[i].PolicyType != "QCS" { + policies[i].client = user.client + ret = append(ret, &policies[i]) + } + } + return ret, nil } func (user *SUser) AttachSystemPolicy(policyId string) error {