Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add paging to ldap search #22

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

add paging to ldap search #22

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ed026bb
add paging to ldap search
TheoAndersen Mar 1, 2018
59ef5e7
add ldap protocol version as CLI option
TheoAndersen Mar 7, 2018
ffbcee3
fix missing p in ldap
TheoAndersen Apr 24, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
137 changes: 87 additions & 50 deletions sync_ldap_groups_to_svn_authz.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@
# This is the attribute of the group object that stores the group memberships.
#group_member_attribute = "member"

# This is the used LDAP protocol version
#ldap_protocol_version = 3

# This is the query/filter used to identify user objects.
#user_query = "objectClass=user"

Expand Down Expand Up @@ -115,6 +118,14 @@ def bind():

ldapobject = ldap.initialize(url)

if(ldap_protocol_version == 1):
ldapobject.protocol_version = ldap.VERSION1
elif(ldap_protocol_version == 2):
ldapobject.protocol_version = ldap.VERSION2
else:
ldapobject.protocol_version = ldap.VERSION3

ldapobject.set_option(ldap.OPT_REFERRALS, 0) #pagiing result only works without referrals enabled
ldapobject.bind_s(bind_dn, bind_password)

if verbose:
Expand All @@ -126,13 +137,14 @@ def bind():
return ldapobject

# bind()

def search_for_groups(ldapobject):
"""This function will search the LDAP directory for group definitions."""

groups = []
result_set = get_ldap_search_resultset(base_dn, group_query, ldapobject)


if (len(result_set) == 0):
if not silent:
sys.stderr.write("The group_query %s did not return any results.\n" % group_query)
Expand Down Expand Up @@ -179,17 +191,36 @@ def get_groups(ldapobject):

def get_ldap_search_resultset(base_dn, group_query, ldapobject, scope=ldap.SCOPE_SUBTREE):
"""This function will return a query result set."""
cookie = ''
page_size = 1000
criticality = False
result_set = []
result_id = ldapobject.search(base_dn, scope, group_query)

while 1:
result_type, result_data = ldapobject.result(result_id, 0)
if (result_type == ldap.RES_SEARCH_ENTRY):
first_pass = True
pg_ctrl = ldap.controls.SimplePagedResultsControl(criticality, page_size, cookie)

#run though <page_size> results at a time, to avoid exceeding ldap size limit
while first_pass or pg_ctrl.cookie:
first_pass = False
try:
msgid = ldapobject.search_ext(base_dn, scope, group_query, serverctrls=[pg_ctrl])
except:
sys.stderr.write("ERROR: \n" + str(sys.exc_info()[0]))
sys.exit(1)

while True:
result_data = None
result_type, result_data, msgid, serverctrls = ldapobject.result3(msgid, all=0)

if serverctrls is not None and len(serverctrls) > 0:
pg_ctrl.cookie = serverctrls[0].cookie

if (result_type == ldap.RES_SEARCH_ENTRY):
result_set.append(result_data)
elif (result_type == ldap.RES_SEARCH_RESULT):
break
elif (result_type == ldap.RES_SEARCH_RESULT):
break

return result_set
return result_set

# get_ldap_search_resultset()

Expand Down Expand Up @@ -229,7 +260,7 @@ def get_members_from_group(group, ldapobject):
else:
# Check to see if this member is really a group
mg = get_ldap_search_resultset(member, group_query, ldapobject)

if (len(mg) == 1):
# The member is a group
if followgroups:
Expand Down Expand Up @@ -287,11 +318,11 @@ def create_group_model(groups, ldapobject):

def get_dict_key_from_value(dict, value):
"""Returns the key of the dictionary entry with the matching value."""

for k, v in dict.iteritems():
if (v == value):
return k

return None

# get_dict_key_from_value()
Expand All @@ -303,19 +334,19 @@ def create_group_map(groups):
if groups:
for group in groups:
cn = simplify_name(group[1]['cn'][0])

if (not groupmap.has_key(cn)):
groupmap[cn] = group[0]
else:
if (not dups.has_key(cn)):
dups[cn] = 1
else:
index = dups[cn]

dups[cn] = (index + 1)

groupmap[cn + str(dups[cn])] = group[0]

return groupmap

# create_group_map()
Expand All @@ -339,21 +370,21 @@ def print_group_model(groups, memberships):
header = header_start + header_middle + header_end
footer = "### End generated content: " + application_name + " ###\n"
text_after_content = ""

file = None
filemode = None
tmp_fd, tmp_authz_path = tempfile.mkstemp()

if ((authz_path != None) and (authz_path != "None")):
if (os.path.exists(authz_path)):
filemode = os.stat(authz_path)
file = open(authz_path, 'r')
tmpfile = open(tmp_authz_path, 'w')

# Remove previous generated content
inside_content = False
before_content = True

for line in file.readlines(): # read from the existing file
if (inside_content): # currently between header and footer
if (line.find(footer) > -1): # footer found
Expand All @@ -368,14 +399,14 @@ def print_group_model(groups, memberships):
tmpfile.write(line) # found before the header: write directly
else:
text_after_content += line # found after the header, write to a temporary variable

file.close()
tmpfile.close()

if (os.path.exists(tmp_authz_path)):
cp = ConfigParser.ConfigParser()
cp.read(tmp_authz_path)

if (not cp.has_section("groups")):
tmpfile = open(tmp_authz_path, 'a')
tmpfile.write("[groups]\n")
Expand All @@ -385,32 +416,32 @@ def print_group_model(groups, memberships):
tmpfile = open(tmp_authz_path, 'a')
tmpfile.write("[groups]\n")
tmpfile.close()

needs_new_line = False

tmpfile = open(tmp_authz_path, 'r')
if (tmpfile.readlines()[-1].strip() != ''): # if the last line is not empty
needs_new_line = True # ask to insert a new empty line at the end
tmpfile.close()

tmpfile = open(tmp_authz_path, 'a')

if (needs_new_line):
tmpfile.write("\n")

tmpfile.write(header + "\n")

groupmap = create_group_map(groups)

if groups:
for i in range(len(groups)):
if (i != 0):
tmpfile.write("\n")

short_name = simplify_name(get_dict_key_from_value(groupmap, groups[i][0]))

tmpfile.write(short_name + " = ")

users = []
for j in range(len(memberships[i])):
user = None
Expand All @@ -422,7 +453,7 @@ def print_group_model(groups, memberships):
if not silent:
sys.stderr.write("[WARNING]: subgroup not in search scope: %s. This means " %
memberships[i][j].replace("GROUP:","") +
"you won't have all members in the SVN group: %s.\n" %
"you won't have all members in the SVN group: %s.\n" %
short_name)
else:
user = memberships[i][j]
Expand All @@ -431,22 +462,22 @@ def print_group_model(groups, memberships):
users.append(user)

tmpfile.write(", ".join(users))

generate_legend(tmpfile, groups)

tmpfile.write("\n" + footer)

tmpfile.write(text_after_content) # write back original content to file

tmpfile.close()

if authz_path:
if (os.path.exists(authz_path + ".bak")):
os.remove(authz_path + ".bak")

if (os.path.exists(authz_path)):
shutil.move(authz_path, authz_path + ".bak")

shutil.move(tmp_authz_path, authz_path)
os.chmod(authz_path, filemode.st_mode)
else:
Expand All @@ -470,14 +501,14 @@ def generate_legend(output, groups):
output.write("########### " + application_name +" (Legend) ##########\n")
output.write("###########################################################" +
"#####################\n")

groupmap = create_group_map(groups)

for group in groups:
short_name = simplify_name(get_dict_key_from_value(groupmap, group[0]))

output.write("### " + short_name + " = " + str(group[0]) + "\n")

output.write("###########################################################" +
"#####################\n")

Expand All @@ -493,14 +524,15 @@ def load_cli_properties(parser):
global group_query
global group_dns
global group_member_attribute
global ldap_protocol_version
global user_query
global userid_attribute
global followgroups
global authz_path
global keep_names
global silent
global verbose

global is_outfile_specified

(options, args) = parser.parse_args(args=None, values=None)
Expand All @@ -512,14 +544,15 @@ def load_cli_properties(parser):
group_query = options.group_query
group_dns = options.group_dns
group_member_attribute = options.group_member_attribute
ldap_protocol_version = options.ldap_protocol_version
user_query = options.user_query
userid_attribute = options.userid_attribute
followgroups = options.followgroups
authz_path = options.authz_path
keep_names = options.keep_names
silent = options.silent
verbose = options.verbose

is_outfile_specified = (authz_path != None) and (authz_path != "None")

# load_cli_properties()
Expand Down Expand Up @@ -563,6 +596,10 @@ def create_cli_parser():
"group memberships. " \
"[Example: member] " \
"[Default: %default]")
parser.add_option("-P", "--ldap-protocol-version",
dest="ldap_protocol_version", default="3",
help="The LDAP protocol version used. " \
"[Default: %default]")
parser.add_option("-u", "--user-query", dest="user_query",
default="objectClass=user",
help="The query/filter used to identify user objects. " \
Expand Down Expand Up @@ -617,7 +654,7 @@ def are_properties_set():
except:
# one of the variables may not exist (i.e. not defined at the start of the script)
return False

# bind_password is not checked since if not passed, the user will be prompted
# authz_path is not checked since it can be 'None' signifying stdout output

Expand Down Expand Up @@ -650,7 +687,7 @@ def get_unset_properties():

def main():
"""This function is the entry point for this script."""

parser = None

# If all necessary options are not properly set in the current script file
Expand All @@ -663,7 +700,7 @@ def main():
# if some properties are not set at this point, there is an error
if not are_properties_set():
sys.stderr.write("There is not enough information to proceed.\n")

for prop in get_unset_properties():
sys.stderr.write("'%s' was not passed\n" % prop)

Expand All @@ -688,9 +725,9 @@ def main():
sys.stderr.write("Could not connect to %s. Error: %s \n" % (url, error_message))
sys.exit(1)

try:
try:
if group_dns:
groups = get_groups(ldapobject)
groups = get_groups(ldapobject)
else:
groups = search_for_groups(ldapobject)
except ldap.LDAPError, error_message:
Expand All @@ -713,4 +750,4 @@ def main():
# main()

if __name__ == "__main__":
main()
main()