opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-21 14:19:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #14916 from swordqiu/hotfix/qj-ldap-search-with-pagination

Browse Source 
fix(keystone): ldap search with pagination
This commit is contained in:
Zexi Li
2022-09-05 16:16:39 +08:00
committed by GitHub
parent 07d6ae5d5c c4343673e5
commit df4113bf1d
5 changed files with 152 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/ldapcli/shell/search.go
+7 -4
View File
@@ -40,6 +40,8 @@ func init() {
Search []string `help:"search conditions, in format of field:value"`
Field []string `help:"retrieve field info"`
Scope string `help:"query scope" choices:"one|sub" default:"sub"`
PageLimit uint32 `help:"page size" default:"100"`
Limit uint32 `help:"maximal output items"`
}
shellutils.R(&LdapSearchOptions{}, "search", "search ldap", func(cli *ldaputils.SLDAPClient, args *LdapSearchOptions) error {
search := make(map[string]string)
@@ -50,13 +52,14 @@ func init() {
}
search[s[:colonPos]] = s[(colonPos + 1):]
}
entries, err := cli.Search(args.Base, args.Objectclass, search, "", args.Field, queryScope(args.Scope))
total, err := cli.Search(args.Base, args.Objectclass, search, "", args.Field, queryScope(args.Scope), args.PageLimit, args.Limit, func(offset uint32, entry *ldap.Entry) error {
entry.PrettyPrint(2)
return nil
})
if err != nil {
return err
}
for _, entry := range entries {
entry.PrettyPrint(2)
}
fmt.Println("Total:", total)
return nil
})
pkg/keystone/driver/ldap/ldap.go
+8 -3
View File
@@ -253,14 +253,19 @@ func (self *SLDAPDriver) Authenticate(ctx context.Context, ident mcclient.SAuthe
if err != nil {
return nil, errors.Wrap(err, "IdmappingManager.FetchEntity for domain")
}
entries, err := self.searchDomainEntries(cli, idMap.IdpEntityId)
var searchEntry *ldap.Entry
err = self.searchDomainEntries(cli, idMap.IdpEntityId,
func(entry *ldap.Entry) error {
searchEntry = entry
return ldaputils.StopSearch
})
if err != nil {
return nil, errors.Wrap(err, "self.searchDomainEntries")
}
if len(entries) == 0 {
if searchEntry == nil {
return nil, errors.Error("fail to find domain DN")
}
userTreeDN = entries[0].DN
userTreeDN = searchEntry.DN
} else {
userTreeDN = self.getUserTreeDN()
}
pkg/keystone/driver/ldap/sync.go
+61 -46
View File
@@ -25,6 +25,7 @@ import (
api "yunion.io/x/onecloud/pkg/apis/identity"
"yunion.io/x/onecloud/pkg/keystone/models"
"yunion.io/x/onecloud/pkg/keystone/options"
"yunion.io/x/onecloud/pkg/util/ldaputils"
)
@@ -83,32 +84,36 @@ func (self *SLDAPDriver) syncSingleDomain(ctx context.Context, cli *ldaputils.SL
return nil
}
func (self *SLDAPDriver) searchDomainEntries(cli *ldaputils.SLDAPClient, domainid string) ([]*ldap.Entry, error) {
func (self *SLDAPDriver) searchDomainEntries(cli *ldaputils.SLDAPClient, domainid string, entryFunc func(*ldap.Entry) error) error {
attrMap := make(map[string]string)
if len(domainid) > 0 {
attrMap[self.ldapConfig.DomainIdAttribute] = domainid
}
return cli.Search(self.getDomainTreeDN(),
_, err := cli.Search(self.getDomainTreeDN(),
self.ldapConfig.DomainObjectclass,
attrMap,
self.ldapConfig.DomainFilter,
self.domainAttributeList(),
self.domainQueryScope(),
options.Options.LdapSearchPageSize, 0,
func(offset uint32, entry *ldap.Entry) error {
return entryFunc(entry)
},
)
if err != nil {
return errors.Wrap(err, "Search")
}
return nil
}
func (self *SLDAPDriver) syncDomains(ctx context.Context, cli *ldaputils.SLDAPClient) error {
entries, err := self.searchDomainEntries(cli, "")
if err != nil {
return errors.Wrap(err, "searchLDAP")
}
domainIds := make([]string, 0)
for i := range entries {
domainInfo := self.entry2Domain(entries[i])
err := self.searchDomainEntries(cli, "", func(entry *ldap.Entry) error {
domainInfo := self.entry2Domain(entry)
err := domainInfo.isValid()
if err != nil {
log.Errorf("invalid domainInfo: %s, skip", err)
continue
return nil
}
domain, err := self.syncDomainInfo(ctx, domainInfo)
if err != nil {
@@ -123,7 +128,12 @@ func (self *SLDAPDriver) syncDomains(ctx context.Context, cli *ldaputils.SLDAPCl
if err != nil {
return errors.Wrap(err, "syncGroups")
}
return nil
})
if err != nil {
return errors.Wrap(err, "searchDomainEntries")
}
// remove any obsolete domains
obsoleteDomainIds, err := models.IdmappingManager.FetchPublicIdsExcludes(self.IdpId, api.IdMappingEntityDomain, domainIds)
if err != nil {
@@ -171,37 +181,39 @@ func (self *SLDAPDriver) syncDomainInfo(ctx context.Context, info SDomainInfo) (
}
func (self *SLDAPDriver) syncUsers(ctx context.Context, cli *ldaputils.SLDAPClient, domainId string, baseDN string) (map[string]string, error) {
entries, err := cli.Search(baseDN,
userIds := make([]string, 0)
userIdMap := make(map[string]string)
_, err := cli.Search(baseDN,
self.ldapConfig.UserObjectclass,
nil,
self.ldapConfig.UserFilter,
self.userAttributeList(),
self.userQueryScope(),
options.Options.LdapSearchPageSize, 0,
func(offset uint32, entry *ldap.Entry) error {
userInfo := self.entry2User(entry)
err := userInfo.isValid()
if err != nil {
log.Debugf("userInfo is invalid: %s, skip", err)
return nil
}
userId, err := self.syncUserDB(ctx, userInfo, domainId)
if err != nil {
return errors.Wrap(err, "syncUserDB")
}
userIds = append(userIds, userId)
if self.ldapConfig.GroupMembersAreIds {
userIdMap[userInfo.Id] = userId
} else {
userIdMap[userInfo.DN] = userId
}
return nil
},
)
if err != nil {
return nil, errors.Wrap(err, "searchLDAP")
}
log.Debugf("syncUsers: ldapSearch entries: %#v", entries)
userIds := make([]string, 0)
userIdMap := make(map[string]string)
for i := range entries {
userInfo := self.entry2User(entries[i])
err := userInfo.isValid()
if err != nil {
log.Debugf("userInfo is invalid: %s, skip", err)
continue
}
userId, err := self.syncUserDB(ctx, userInfo, domainId)
if err != nil {
return nil, errors.Wrap(err, "syncUserDB")
}
userIds = append(userIds, userId)
if self.ldapConfig.GroupMembersAreIds {
userIdMap[userInfo.Id] = userId
} else {
userIdMap[userInfo.DN] = userId
}
}
deleteUsers, err := models.UserManager.FetchUsersInDomain(domainId, userIds)
if err != nil {
return nil, errors.Wrap(err, "models.UserManager.FetchUserIdsInDomain")
@@ -259,30 +271,33 @@ func (self *SLDAPDriver) syncUserDB(ctx context.Context, ui SUserInfo, domainId
}
func (self *SLDAPDriver) syncGroups(ctx context.Context, cli *ldaputils.SLDAPClient, domainId string, baseDN string, userIdMap map[string]string) error {
entries, err := cli.Search(baseDN,
groupIds := make([]string, 0)
_, err := cli.Search(baseDN,
self.ldapConfig.GroupObjectclass,
nil,
self.ldapConfig.GroupFilter,
self.groupAttributeList(),
self.groupQueryScope(),
options.Options.LdapSearchPageSize, 0,
func(offset uint32, entry *ldap.Entry) error {
groupInfo := self.entry2Group(entry)
err := groupInfo.isValid()
if err != nil {
log.Errorf("invalid group info: %s, skip", err)
return nil
}
groupId, err := self.syncGroupDB(ctx, groupInfo, domainId, userIdMap)
if err != nil {
return errors.Wrap(err, "syncGroupDB")
}
groupIds = append(groupIds, groupId)
return nil
},
)
if err != nil {
return errors.Wrap(err, "searchLDAP")
}
groupIds := make([]string, 0)
for i := range entries {
groupInfo := self.entry2Group(entries[i])
err := groupInfo.isValid()
if err != nil {
log.Errorf("invalid group info: %s, skip", err)
continue
}
groupId, err := self.syncGroupDB(ctx, groupInfo, domainId, userIdMap)
if err != nil {
return errors.Wrap(err, "syncGroupDB")
}
groupIds = append(groupIds, groupId)
}
deleteGroups, err := models.GroupManager.FetchGroupsInDomain(domainId, groupIds)
if err != nil {
return errors.Wrap(err, "models.GroupManager.FetchGroupsInDomain")
pkg/keystone/options/options.go
+2
View File
@@ -65,6 +65,8 @@ type SKeystoneOptions struct {
ProjectDashboardPolicy string `help:"dashboard policy name for project view" default:"project-dashboard"`
NoPolicyViolationCheck bool `help:"do not check policy violation when modify or assign policy" default:"false"`
LdapSearchPageSize uint32 `help:"pagination size for LDAP search" default:"100"`
}
var (
pkg/util/ldaputils/ldaputils.go
+74 -25
View File
@@ -19,16 +19,16 @@ import (
"fmt"
"strings"
"github.com/pkg/errors"
"gopkg.in/ldap.v3"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
)
var (
ErrUserNotFound = errors.New("not found")
ErrUserDuplicate = errors.New("user id duplicate")
ErrUserBadCredential = errors.New("bad credential")
ErrUserNotFound = errors.Error("not found")
ErrUserDuplicate = errors.Error("user id duplicate")
ErrUserBadCredential = errors.Error("bad credential")
binaryAttributes = []string{
"objectGUID",
@@ -61,7 +61,7 @@ func NewLDAPClient(url, account, password string, baseDN string, isDebug bool) *
func (cli *SLDAPClient) Connect() error {
conn, err := ldap.DialURL(cli.url)
if err != nil {
return errors.WithMessage(err, "DiaURL")
return errors.Wrap(err, "DiaURL")
}
cli.conn = conn
@@ -72,7 +72,7 @@ func (cli *SLDAPClient) bind() error {
if len(cli.account) > 0 {
err := cli.conn.Bind(cli.account, cli.password)
if err != nil {
return errors.WithMessage(err, "Bind")
return errors.Wrap(err, "Bind")
}
}
return nil
@@ -88,26 +88,47 @@ func (cli *SLDAPClient) Close() {
func (cli *SLDAPClient) Authenticate(baseDN string, objClass string, uidAttr string, uname string, passwd string, filter string, fields []string, queryScope int) (*ldap.Entry, error) {
attrMap := make(map[string]string)
attrMap[uidAttr] = uname
entries, err := cli.Search(baseDN, objClass, attrMap, filter, fields, queryScope)
var retEntry *ldap.Entry
total, err := cli.Search(
baseDN, objClass, attrMap, filter, fields, queryScope,
2, 2,
func(offset uint32, entry *ldap.Entry) error {
retEntry = entry
return nil
},
)
if err != nil {
return nil, errors.WithMessage(err, "Search")
return nil, errors.Wrap(err, "Search")
}
if len(entries) == 0 {
if total == 0 {
return nil, ErrUserNotFound
}
if len(entries) > 1 {
if total > 1 {
return nil, ErrUserDuplicate
}
defer cli.bind()
entry := entries[0]
err = cli.conn.Bind(entry.DN, passwd)
err = cli.conn.Bind(retEntry.DN, passwd)
if err != nil {
return nil, ErrUserBadCredential
}
return entry, nil
return retEntry, nil
}
func (cli *SLDAPClient) Search(base string, objClass string, condition map[string]string, filter string, fields []string, queryScope int) ([]*ldap.Entry, error) {
const (
StopSearch = errors.Error("stop dap search")
)
// support pagination
// reference: https://zerokspot.com/weblog/2018/03/07/paging-in-gopkg-ldap/
func (cli *SLDAPClient) Search(
base string, objClass string,
condition map[string]string,
filter string,
fields []string,
queryScope int,
pageSize uint32, limit uint32,
entryFunc func(offset uint32, entry *ldap.Entry) error,
) (uint32, error) {
searches := strings.Builder{}
if len(condition) == 0 && len(objClass) == 0 {
searches.WriteString("(objectClass=*)")
@@ -142,19 +163,47 @@ func (cli *SLDAPClient) Search(base string, objClass string, condition map[strin
log.Debugf("ldapSearch: %s", searchStr)
searchRequest := ldap.NewSearchRequest(
base, // The base dn to search
queryScope, ldap.NeverDerefAliases, 0, 0, false,
searchStr,
fields, // A list attributes to retrieve
nil,
)
sr, err := cli.conn.Search(searchRequest)
if err != nil {
return nil, errors.Wrap(err, "Search")
offset := uint32(0)
paging := ldap.NewControlPaging(pageSize)
for {
searchRequest := ldap.NewSearchRequest(
base, // The base dn to search
queryScope, ldap.NeverDerefAliases, 0, 0, false,
searchStr,
fields, // A list attributes to retrieve
[]ldap.Control{paging},
)
searchResult, err := cli.conn.Search(searchRequest)
if err != nil {
return offset, errors.Wrap(err, "Seearch")
}
pageTotal := len(searchResult.Entries)
for i := 0; i < pageTotal; i++ {
entry := searchResult.Entries[i]
err := entryFunc(offset, entry)
if err != nil && errors.Cause(err) != StopSearch {
return offset, errors.Wrapf(err, "process entry fail at %d-%d-%d", pageTotal, i, offset)
}
offset += 1
if (err != nil && errors.Cause(err) == StopSearch) || (limit > 0 && offset >= limit) {
// stop, offset exceeds limit or receive StopSearch error
return offset, nil
}
}
resultCtrl := ldap.FindControl(searchResult.Controls, paging.GetControlType())
if resultCtrl == nil {
break
}
if pagingCtrl, ok := resultCtrl.(*ldap.ControlPaging); ok {
if len(pagingCtrl.Cookie) == 0 {
break
}
paging.SetCookie(pagingCtrl.Cookie)
}
}
return sr.Entries, nil
return offset, nil
}
func isBinaryAttr(attrName string) bool {