Skip to content

Instantly share code, notes, and snippets.

@mattfahrner
Last active January 16, 2024 12:39
Show Gist options
  • Save mattfahrner/c228ead9c516fc322d3a to your computer and use it in GitHub Desktop.
Save mattfahrner/c228ead9c516fc322d3a to your computer and use it in GitHub Desktop.
This snippet allows you to do a Python LDAP search with paged controls. The latest version now supports Python "ldap" 2.4. Many thanks to Ilya Rumyantsev for doing the 2.4 legwork.
#! /usr/bin/python
import sys
import ldap
from ldap.controls import SimplePagedResultsControl
from distutils.version import LooseVersion
# Check if we're using the Python "ldap" 2.4 or greater API
LDAP24API = LooseVersion(ldap.__version__) >= LooseVersion('2.4')
# If you're talking to LDAP, you should be using LDAPS for security!
LDAPSERVER='ldaps://ldap.somecompany.com'
BASEDN='cn=users,dc=somecompany,dc=com'
LDAPUSER = 'uid=someuser,dc=somecompany,dc=com'
LDAPPASSWORD = 'somepassword'
PAGESIZE = 1000
ATTRLIST = ['uid', 'shadowLastChange', 'shadowMax', 'shadowExpire']
SEARCHFILTER='uid=*'
def create_controls(pagesize):
"""Create an LDAP control with a page size of "pagesize"."""
# Initialize the LDAP controls for paging. Note that we pass ''
# for the cookie because on first iteration, it starts out empty.
if LDAP24API:
return SimplePagedResultsControl(True, size=pagesize, cookie='')
else:
return SimplePagedResultsControl(ldap.LDAP_CONTROL_PAGE_OID, True,
(pagesize,''))
def get_pctrls(serverctrls):
"""Lookup an LDAP paged control object from the returned controls."""
# Look through the returned controls and find the page controls.
# This will also have our returned cookie which we need to make
# the next search request.
if LDAP24API:
return [c for c in serverctrls
if c.controlType == SimplePagedResultsControl.controlType]
else:
return [c for c in serverctrls
if c.controlType == ldap.LDAP_CONTROL_PAGE_OID]
def set_cookie(lc_object, pctrls, pagesize):
"""Push latest cookie back into the page control."""
if LDAP24API:
cookie = pctrls[0].cookie
lc_object.cookie = cookie
return cookie
else:
est, cookie = pctrls[0].controlValue
lc_object.controlValue = (pagesize,cookie)
return cookie
# This is essentially a placeholder callback function. You would do your real
# work inside of this. Really this should be all abstracted into a generator...
def process_entry(dn, attrs):
"""Process an entry. The two arguments passed are the DN and
a dictionary of attributes."""
print dn, attrs
# Ignore server side certificate errors (assumes using LDAPS and
# self-signed cert). Not necessary if not LDAPS or it's signed by
# a real CA.
ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_ALLOW)
# Don't follow referrals
ldap.set_option(ldap.OPT_REFERRALS, 0)
l = ldap.initialize(LDAPSERVER)
l.protocol_version = 3 # Paged results only apply to LDAP v3
try:
l.simple_bind_s(LDAPUSER, LDAPPASSWORD)
except ldap.LDAPError as e:
exit('LDAP bind failed: %s' % e)
# Create the page control to work from
lc = create_controls(PAGESIZE)
# Do searches until we run out of "pages" to get from
# the LDAP server.
while True:
# Send search request
try:
# If you leave out the ATTRLIST it'll return all attributes
# which you have permissions to access. You may want to adjust
# the scope level as well (perhaps "ldap.SCOPE_SUBTREE", but
# it can reduce performance if you don't need it).
msgid = l.search_ext(BASEDN, ldap.SCOPE_ONELEVEL, SEARCHFILTER,
ATTRLIST, serverctrls=[lc])
except ldap.LDAPError as e:
sys.exit('LDAP search failed: %s' % e)
# Pull the results from the search request
try:
rtype, rdata, rmsgid, serverctrls = l.result3(msgid)
except ldap.LDAPError as e:
sys.exit('Could not pull LDAP results: %s' % e)
# Each "rdata" is a tuple of the form (dn, attrs), where dn is
# a string containing the DN (distinguished name) of the entry,
# and attrs is a dictionary containing the attributes associated
# with the entry. The keys of attrs are strings, and the associated
# values are lists of strings.
for dn, attrs in rdata:
process_entry(dn, attrs)
# Get cookie for next request
pctrls = get_pctrls(serverctrls)
if not pctrls:
print >> sys.stderr, 'Warning: Server ignores RFC 2696 control.'
break
# Ok, we did find the page control, yank the cookie from it and
# insert it into the control for our next search. If however there
# is no cookie, we are done!
cookie = set_cookie(lc, pctrls, PAGESIZE)
if not cookie:
break
# Clean up
l.unbind()
# Done!
sys.exit(0)
#
# This code is in many ways a refactoring of code found at the following sites:
#
# http://www.novell.com/coolsolutions/tip/18274.html
# https://travelingfrontiers.wordpress.com/2013/04/05/how-to-use-python-ldap-paged-results-control-to-handle-large-ldap-searches
#
# More about this code can be found here:
#
# http://mattfahrner.com/2014/03/09/using-paged-controls-with-python-and-ldap/
#
import sys
import ldap
# If you're talking to LDAP, you should be using LDAPS for security!
LDAPSERVER='ldaps://ldap.somecompany.com'
BASEDN='cn=users,dc=somecompany,dc=com'
LDAPUSER = 'uid=someuser,dc=somecompany,dc=com'
LDAPPASSWORD = 'somepassword'
PAGESIZE = 1000
ATTRLIST = ['uid', 'shadowLastChange', 'shadowMax', 'shadowExpire']
SEARCHFILTER='uid=*'
# Ignore server side certificate errors (assumes using LDAPS and
# self-signed cert). Not necessary if not LDAPS or it's signed by
# a real CA.
ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_ALLOW)
# Don't follow referrals
ldap.set_option(ldap.OPT_REFERRALS, 0)
l = ldap.initialize(LDAPSERVER)
l.protocol_version = 3 # Paged results only apply to LDAP v3
try:
l.simple_bind_s(LDAPUSER, LDAPPASSWORD)
except ldap.LDAPError as e:
exit('LDAP bind failed: %s' % e)
# Initialize the LDAP controls for paging. Note that we pass ''
# for the cookie because on first iteration, it starts out empty.
lc = ldap.controls.SimplePagedResultsControl(ldap.LDAP_CONTROL_PAGE_OID, True,
(PAGESIZE,''))
# This is essentially a placeholder callback function. You would do your real
# work inside of this. Really this should be all abstracted into a generator...
def process_entry(dn, attrs):
"""Process an entry. The two arguments passed are the DN and
a dictionary of attributes."""
print dn, attrs
# Do searches until we run out of "pages" to get from
# the LDAP server.
while True:
# Send search request
try:
# If you leave out the ATTRLIST it'll return all attributes
# which you have permissions to access. You may want to adjust
# the scope level as well (perhaps "ldap.SCOPE_SUBTREE", but
# it can reduce performance if you don't need it).
msgid = l.search_ext(BASEDN, ldap.SCOPE_ONELEVEL, SEARCHFILTER,
ATTRLIST, serverctrls=[lc])
except ldap.LDAPError as e:
sys.exit('LDAP search failed: %s' % e)
# Pull the results from the search request
try:
rtype, rdata, rmsgid, serverctrls = l.result3(msgid)
except ldap.LDAPError as e:
sys.exit('Could not pull LDAP results: %s' % e)
# Each "rdata" is a tuple of the form (dn, attrs), where dn is
# a string containing the DN (distinguished name) of the entry,
# and attrs is a dictionary containing the attributes associated
# with the entry. The keys of attrs are strings, and the associated
# values are lists of strings.
for dn, attrs in rdata:
process_entry()
# Look through the returned controls and find the page controls.
# This will also have our returned cookie which we need to make
# the next search request.
pctrls = [
c for c in serverctrls if c.controlType == ldap.LDAP_CONTROL_PAGE_OID
]
if not pctrls:
print >> sys.stderr, 'Warning: Server ignores RFC 2696 control.'
break
# Ok, we did find the page control, yank the cookie from it and
# insert it into the control for our next search. If however there
# is no cookie, we are done!
est, cookie = pctrls[0].controlValue
if not cookie:
break
lc.controlValue = (page_size,cookie)
@ianrios
Copy link

ianrios commented Dec 13, 2018

Hello, I was wondering how I could print all results from the search. Currently I only get 3 printed values using process_entry(dn, attrs), but I have about 7K+ items so either this function is not doing what I want, or I am simply not understanding it. Thanks!

@ianrios
Copy link

ianrios commented Dec 13, 2018

Ahhhh I figured it out! I changed ldap.SCOPE_SUBTREE from ldap.SCOPE_ONELEVEL

@maxibillion
Copy link

Thank you for posting all in 1-place.

Using the 2.4 snippet version --- 389 DS (reportedly from docs supports RFC 2696 control.).

Receiving message: Could not pull LDAP results: {'desc': u'Administrative limit exceeded'} --- and seeing similar on the LDAP server logs: "RESULT err=11 tag=101 nentries=0 etime=0.0214126589 notes=P,A pr_idx=0 pr_cookie=-1"

Tearing my hair out on where potential flaw could be....is this a bug on the 389DS w.r.t cookie?

@whbogado
Copy link

whbogado commented Jan 20, 2019

@maxibillion Paged results cannot override the max result limit that is set in the server configuration, not only for 389DS but for any LDAP server. This is no bug, it is so by design, the limit is imposed on the total results requested, not on the page size. You are receiving this error because you have exceeded that limit.

The limit can be relaxed on a per user basis. For example, if you bind with the manager user (cn=Directory Manager) you can retrieve as many entries as you want.

LDAPUSER = 'cn=Directory Manager'
LDAPPASSWORD = 'managerpassword'

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment