diff --git a/cmd/ldapcli/shell/search.go b/cmd/ldapcli/shell/search.go index c460c4d38a..18b39b41e3 100644 --- a/cmd/ldapcli/shell/search.go +++ b/cmd/ldapcli/shell/search.go @@ -40,6 +40,8 @@ func init() { Search []string `help:"search conditions, in format of field:value"` Field []string `help:"retrieve field info"` Scope string `help:"query scope" choices:"one|sub" default:"sub"` + PageLimit uint32 `help:"page size" default:"100"` + Limit uint32 `help:"maximal output items"` } shellutils.R(&LdapSearchOptions{}, "search", "search ldap", func(cli *ldaputils.SLDAPClient, args *LdapSearchOptions) error { search := make(map[string]string) @@ -50,13 +52,14 @@ func init() { } search[s[:colonPos]] = s[(colonPos + 1):] } - entries, err := cli.Search(args.Base, args.Objectclass, search, "", args.Field, queryScope(args.Scope)) + total, err := cli.Search(args.Base, args.Objectclass, search, "", args.Field, queryScope(args.Scope), args.PageLimit, args.Limit, func(offset uint32, entry *ldap.Entry) error { + entry.PrettyPrint(2) + return nil + }) if err != nil { return err } - for _, entry := range entries { - entry.PrettyPrint(2) - } + fmt.Println("Total:", total) return nil }) diff --git a/pkg/keystone/driver/ldap/ldap.go b/pkg/keystone/driver/ldap/ldap.go index c0f354dc4c..6391af012f 100644 --- a/pkg/keystone/driver/ldap/ldap.go +++ b/pkg/keystone/driver/ldap/ldap.go @@ -253,14 +253,19 @@ func (self *SLDAPDriver) Authenticate(ctx context.Context, ident mcclient.SAuthe if err != nil { return nil, errors.Wrap(err, "IdmappingManager.FetchEntity for domain") } - entries, err := self.searchDomainEntries(cli, idMap.IdpEntityId) + var searchEntry *ldap.Entry + err = self.searchDomainEntries(cli, idMap.IdpEntityId, + func(entry *ldap.Entry) error { + searchEntry = entry + return ldaputils.StopSearch + }) if err != nil { return nil, errors.Wrap(err, "self.searchDomainEntries") } - if len(entries) == 0 { + if searchEntry == nil { return nil, errors.Error("fail to find domain DN") } - userTreeDN = entries[0].DN + userTreeDN = searchEntry.DN } else { userTreeDN = self.getUserTreeDN() } diff --git a/pkg/keystone/driver/ldap/sync.go b/pkg/keystone/driver/ldap/sync.go index 393a211e17..dca225abba 100644 --- a/pkg/keystone/driver/ldap/sync.go +++ b/pkg/keystone/driver/ldap/sync.go @@ -25,6 +25,7 @@ import ( api "yunion.io/x/onecloud/pkg/apis/identity" "yunion.io/x/onecloud/pkg/keystone/models" + "yunion.io/x/onecloud/pkg/keystone/options" "yunion.io/x/onecloud/pkg/util/ldaputils" ) @@ -83,32 +84,36 @@ func (self *SLDAPDriver) syncSingleDomain(ctx context.Context, cli *ldaputils.SL return nil } -func (self *SLDAPDriver) searchDomainEntries(cli *ldaputils.SLDAPClient, domainid string) ([]*ldap.Entry, error) { +func (self *SLDAPDriver) searchDomainEntries(cli *ldaputils.SLDAPClient, domainid string, entryFunc func(*ldap.Entry) error) error { attrMap := make(map[string]string) if len(domainid) > 0 { attrMap[self.ldapConfig.DomainIdAttribute] = domainid } - return cli.Search(self.getDomainTreeDN(), + _, err := cli.Search(self.getDomainTreeDN(), self.ldapConfig.DomainObjectclass, attrMap, self.ldapConfig.DomainFilter, self.domainAttributeList(), self.domainQueryScope(), + options.Options.LdapSearchPageSize, 0, + func(offset uint32, entry *ldap.Entry) error { + return entryFunc(entry) + }, ) + if err != nil { + return errors.Wrap(err, "Search") + } + return nil } func (self *SLDAPDriver) syncDomains(ctx context.Context, cli *ldaputils.SLDAPClient) error { - entries, err := self.searchDomainEntries(cli, "") - if err != nil { - return errors.Wrap(err, "searchLDAP") - } domainIds := make([]string, 0) - for i := range entries { - domainInfo := self.entry2Domain(entries[i]) + err := self.searchDomainEntries(cli, "", func(entry *ldap.Entry) error { + domainInfo := self.entry2Domain(entry) err := domainInfo.isValid() if err != nil { log.Errorf("invalid domainInfo: %s, skip", err) - continue + return nil } domain, err := self.syncDomainInfo(ctx, domainInfo) if err != nil { @@ -123,7 +128,12 @@ func (self *SLDAPDriver) syncDomains(ctx context.Context, cli *ldaputils.SLDAPCl if err != nil { return errors.Wrap(err, "syncGroups") } + return nil + }) + if err != nil { + return errors.Wrap(err, "searchDomainEntries") } + // remove any obsolete domains obsoleteDomainIds, err := models.IdmappingManager.FetchPublicIdsExcludes(self.IdpId, api.IdMappingEntityDomain, domainIds) if err != nil { @@ -171,37 +181,39 @@ func (self *SLDAPDriver) syncDomainInfo(ctx context.Context, info SDomainInfo) ( } func (self *SLDAPDriver) syncUsers(ctx context.Context, cli *ldaputils.SLDAPClient, domainId string, baseDN string) (map[string]string, error) { - entries, err := cli.Search(baseDN, + userIds := make([]string, 0) + userIdMap := make(map[string]string) + _, err := cli.Search(baseDN, self.ldapConfig.UserObjectclass, nil, self.ldapConfig.UserFilter, self.userAttributeList(), self.userQueryScope(), + options.Options.LdapSearchPageSize, 0, + func(offset uint32, entry *ldap.Entry) error { + userInfo := self.entry2User(entry) + err := userInfo.isValid() + if err != nil { + log.Debugf("userInfo is invalid: %s, skip", err) + return nil + } + userId, err := self.syncUserDB(ctx, userInfo, domainId) + if err != nil { + return errors.Wrap(err, "syncUserDB") + } + userIds = append(userIds, userId) + if self.ldapConfig.GroupMembersAreIds { + userIdMap[userInfo.Id] = userId + } else { + userIdMap[userInfo.DN] = userId + } + return nil + }, ) if err != nil { return nil, errors.Wrap(err, "searchLDAP") } - log.Debugf("syncUsers: ldapSearch entries: %#v", entries) - userIds := make([]string, 0) - userIdMap := make(map[string]string) - for i := range entries { - userInfo := self.entry2User(entries[i]) - err := userInfo.isValid() - if err != nil { - log.Debugf("userInfo is invalid: %s, skip", err) - continue - } - userId, err := self.syncUserDB(ctx, userInfo, domainId) - if err != nil { - return nil, errors.Wrap(err, "syncUserDB") - } - userIds = append(userIds, userId) - if self.ldapConfig.GroupMembersAreIds { - userIdMap[userInfo.Id] = userId - } else { - userIdMap[userInfo.DN] = userId - } - } + deleteUsers, err := models.UserManager.FetchUsersInDomain(domainId, userIds) if err != nil { return nil, errors.Wrap(err, "models.UserManager.FetchUserIdsInDomain") @@ -259,30 +271,33 @@ func (self *SLDAPDriver) syncUserDB(ctx context.Context, ui SUserInfo, domainId } func (self *SLDAPDriver) syncGroups(ctx context.Context, cli *ldaputils.SLDAPClient, domainId string, baseDN string, userIdMap map[string]string) error { - entries, err := cli.Search(baseDN, + groupIds := make([]string, 0) + _, err := cli.Search(baseDN, self.ldapConfig.GroupObjectclass, nil, self.ldapConfig.GroupFilter, self.groupAttributeList(), self.groupQueryScope(), + options.Options.LdapSearchPageSize, 0, + func(offset uint32, entry *ldap.Entry) error { + groupInfo := self.entry2Group(entry) + err := groupInfo.isValid() + if err != nil { + log.Errorf("invalid group info: %s, skip", err) + return nil + } + groupId, err := self.syncGroupDB(ctx, groupInfo, domainId, userIdMap) + if err != nil { + return errors.Wrap(err, "syncGroupDB") + } + groupIds = append(groupIds, groupId) + return nil + }, ) if err != nil { return errors.Wrap(err, "searchLDAP") } - groupIds := make([]string, 0) - for i := range entries { - groupInfo := self.entry2Group(entries[i]) - err := groupInfo.isValid() - if err != nil { - log.Errorf("invalid group info: %s, skip", err) - continue - } - groupId, err := self.syncGroupDB(ctx, groupInfo, domainId, userIdMap) - if err != nil { - return errors.Wrap(err, "syncGroupDB") - } - groupIds = append(groupIds, groupId) - } + deleteGroups, err := models.GroupManager.FetchGroupsInDomain(domainId, groupIds) if err != nil { return errors.Wrap(err, "models.GroupManager.FetchGroupsInDomain") diff --git a/pkg/keystone/options/options.go b/pkg/keystone/options/options.go index d7ec176166..f3b66cbe11 100644 --- a/pkg/keystone/options/options.go +++ b/pkg/keystone/options/options.go @@ -65,6 +65,8 @@ type SKeystoneOptions struct { ProjectDashboardPolicy string `help:"dashboard policy name for project view" default:"project-dashboard"` NoPolicyViolationCheck bool `help:"do not check policy violation when modify or assign policy" default:"false"` + + LdapSearchPageSize uint32 `help:"pagination size for LDAP search" default:"100"` } var ( diff --git a/pkg/util/ldaputils/ldaputils.go b/pkg/util/ldaputils/ldaputils.go index c800681515..2f889a6981 100644 --- a/pkg/util/ldaputils/ldaputils.go +++ b/pkg/util/ldaputils/ldaputils.go @@ -19,16 +19,16 @@ import ( "fmt" "strings" - "github.com/pkg/errors" "gopkg.in/ldap.v3" "yunion.io/x/log" + "yunion.io/x/pkg/errors" ) var ( - ErrUserNotFound = errors.New("not found") - ErrUserDuplicate = errors.New("user id duplicate") - ErrUserBadCredential = errors.New("bad credential") + ErrUserNotFound = errors.Error("not found") + ErrUserDuplicate = errors.Error("user id duplicate") + ErrUserBadCredential = errors.Error("bad credential") binaryAttributes = []string{ "objectGUID", @@ -61,7 +61,7 @@ func NewLDAPClient(url, account, password string, baseDN string, isDebug bool) * func (cli *SLDAPClient) Connect() error { conn, err := ldap.DialURL(cli.url) if err != nil { - return errors.WithMessage(err, "DiaURL") + return errors.Wrap(err, "DiaURL") } cli.conn = conn @@ -72,7 +72,7 @@ func (cli *SLDAPClient) bind() error { if len(cli.account) > 0 { err := cli.conn.Bind(cli.account, cli.password) if err != nil { - return errors.WithMessage(err, "Bind") + return errors.Wrap(err, "Bind") } } return nil @@ -88,26 +88,47 @@ func (cli *SLDAPClient) Close() { func (cli *SLDAPClient) Authenticate(baseDN string, objClass string, uidAttr string, uname string, passwd string, filter string, fields []string, queryScope int) (*ldap.Entry, error) { attrMap := make(map[string]string) attrMap[uidAttr] = uname - entries, err := cli.Search(baseDN, objClass, attrMap, filter, fields, queryScope) + var retEntry *ldap.Entry + total, err := cli.Search( + baseDN, objClass, attrMap, filter, fields, queryScope, + 2, 2, + func(offset uint32, entry *ldap.Entry) error { + retEntry = entry + return nil + }, + ) if err != nil { - return nil, errors.WithMessage(err, "Search") + return nil, errors.Wrap(err, "Search") } - if len(entries) == 0 { + if total == 0 { return nil, ErrUserNotFound } - if len(entries) > 1 { + if total > 1 { return nil, ErrUserDuplicate } defer cli.bind() - entry := entries[0] - err = cli.conn.Bind(entry.DN, passwd) + err = cli.conn.Bind(retEntry.DN, passwd) if err != nil { return nil, ErrUserBadCredential } - return entry, nil + return retEntry, nil } -func (cli *SLDAPClient) Search(base string, objClass string, condition map[string]string, filter string, fields []string, queryScope int) ([]*ldap.Entry, error) { +const ( + StopSearch = errors.Error("stop dap search") +) + +// support pagination +// reference: https://zerokspot.com/weblog/2018/03/07/paging-in-gopkg-ldap/ +func (cli *SLDAPClient) Search( + base string, objClass string, + condition map[string]string, + filter string, + fields []string, + queryScope int, + pageSize uint32, limit uint32, + entryFunc func(offset uint32, entry *ldap.Entry) error, +) (uint32, error) { searches := strings.Builder{} if len(condition) == 0 && len(objClass) == 0 { searches.WriteString("(objectClass=*)") @@ -142,19 +163,47 @@ func (cli *SLDAPClient) Search(base string, objClass string, condition map[strin log.Debugf("ldapSearch: %s", searchStr) - searchRequest := ldap.NewSearchRequest( - base, // The base dn to search - queryScope, ldap.NeverDerefAliases, 0, 0, false, - searchStr, - fields, // A list attributes to retrieve - nil, - ) - sr, err := cli.conn.Search(searchRequest) - if err != nil { - return nil, errors.Wrap(err, "Search") + offset := uint32(0) + paging := ldap.NewControlPaging(pageSize) + for { + searchRequest := ldap.NewSearchRequest( + base, // The base dn to search + queryScope, ldap.NeverDerefAliases, 0, 0, false, + searchStr, + fields, // A list attributes to retrieve + []ldap.Control{paging}, + ) + searchResult, err := cli.conn.Search(searchRequest) + if err != nil { + return offset, errors.Wrap(err, "Seearch") + } + pageTotal := len(searchResult.Entries) + for i := 0; i < pageTotal; i++ { + entry := searchResult.Entries[i] + err := entryFunc(offset, entry) + if err != nil && errors.Cause(err) != StopSearch { + return offset, errors.Wrapf(err, "process entry fail at %d-%d-%d", pageTotal, i, offset) + } + offset += 1 + if (err != nil && errors.Cause(err) == StopSearch) || (limit > 0 && offset >= limit) { + // stop, offset exceeds limit or receive StopSearch error + return offset, nil + } + } + + resultCtrl := ldap.FindControl(searchResult.Controls, paging.GetControlType()) + if resultCtrl == nil { + break + } + if pagingCtrl, ok := resultCtrl.(*ldap.ControlPaging); ok { + if len(pagingCtrl.Cookie) == 0 { + break + } + paging.SetCookie(pagingCtrl.Cookie) + } } - return sr.Entries, nil + return offset, nil } func isBinaryAttr(attrName string) bool {