Skip to content

Commit

Permalink
project-usermap requested changes
Browse files Browse the repository at this point in the history
  • Loading branch information
@williamnswanson
williamnswanson committed Dec 29, 2023
1 parent 3fd7723 commit af0d2eb
Show file tree
Hide file tree
Showing 2 changed files with 59 additions and 102 deletions.
35 changes: 33 additions & 2 deletions comanage_scripts_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import json
import urllib.error
import urllib.request
from ldap3 import Server, Connection, ALL, ALL_ATTRIBUTES, SAFE_SYNC
from ldap3 import Server, Connection, ALL, SAFE_SYNC


MIN_TIMEOUT = 5
Expand Down Expand Up @@ -42,6 +42,14 @@ def mkauthstr(user, passwd):
return encodebytes(raw_authstr.encode()).decode().replace("\n", "")


def get_ldap_authtok(ldap_authfile):
if ldap_authfile is not None:
ldap_authtok = open(ldap_authfile).readline().rstrip("\n")
else:
raise PermissionError
return ldap_authtok


def mkrequest(method, target, data, endpoint, authstr, **kw):
url = os.path.join(endpoint, target)
if kw:
Expand Down Expand Up @@ -135,12 +143,35 @@ def get_ldap_groups(ldap_server, ldap_user, ldap_authtok):
ldap_group_osggids = set()
server = Server(ldap_server, get_info=ALL)
connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True)
_, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", "(cn=*)", attributes=ALL_ATTRIBUTES)
_, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", "(cn=*)", attributes=["gidNumber"])
for group in response:
ldap_group_osggids.add(group["attributes"]["gidNumber"])
return ldap_group_osggids


def get_ldap_group_members(ldap_gid, ldap_server, ldap_user, ldap_authtok):
ldap_group_members = set()
server = Server(ldap_server, get_info=ALL)
connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True)
_, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", f"(&(gidNumber={ldap_gid})(cn=*))", attributes=["hasMember"])
for group in response:
ldap_group_members.update(group["attributes"]["hasMember"])
return ldap_group_members


def get_ldap_active_users(ldap_server, ldap_user, ldap_authtok, filter_group_name=None):
ldap_active_users = set()
filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None
else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))")
server = Server(ldap_server, get_info=ALL)
connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True)
_, _, response, _ = connection.search("ou=people,o=OSG,o=CO,dc=cilogon,dc=org", filter_str, attributes=["employeeNumber"])
for person in response:
# the "employeeNumber" is the person's name in the first.last format
ldap_active_users.add(person["attributes"]["employeeNumber"])
return ldap_active_users


def identifier_from_list(id_list, id_type):
id_type_list = [id["Type"] for id in id_list]
try:
Expand Down
126 changes: 26 additions & 100 deletions osg-comanage-project-usermap.py
View file
Original file line number Diff line number Diff line change
@@ -1,63 +1,32 @@
#!/usr/bin/env python3

import re
import os
import sys
import json
import time
import getopt
import subprocess
import urllib.error
import urllib.request
import comanage_scripts_utils as utils


SCRIPT = os.path.basename(__file__)
ENDPOINT = "https://registry-test.cilogon.org/registry/"
LDAP_SERVER = "ldaps://ldap-test.cilogon.org"
LDAP_USER = "uid=registry_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org"
OSG_CO_ID = 8
MINTIMEOUT = 5
MAXTIMEOUT = 625
TIMEOUTMULTIPLE = 5
CACHE_FILENAME = "COmanage_Projects_cache.txt"
CACHE_LIFETIME_HOURS = 0.5

LDAP_AUTH_COMMAND = [
"awk", "/ldap_default_authtok/ {print $3}", "/etc/sssd/conf.d/0060_domain_CILOGON.ORG.conf",
]

LDAP_GROUP_MEMBERS_COMMAND = [
"ldapsearch",
"-H",
"ldaps://ldap.cilogon.org",
"-D",
"uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org",
"-w", "{auth}",
"-b",
"ou=groups,o=OSG,o=CO,dc=cilogon,dc=org",
"-s",
"one",
"(cn=*)",
]

LDAP_ACTIVE_USERS_COMMAND = [
"ldapsearch",
"-LLL",
"-H", "ldaps://ldap.cilogon.org",
"-D", "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org",
"-x",
"-w", "{auth}",
"-b", "ou=people,o=OSG,o=CO,dc=cilogon,dc=org",
"{filter}", "voPersonApplicationUID",
"|", "grep", "voPersonApplicationUID",
"|", "sort",
]

_usage = f"""\
usage: [PASS=...] {SCRIPT} [OPTIONS]
OPTIONS:
-u USER[:PASS] specify USER and optionally PASS on command line
-c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID})
-s LDAP_SERVER specify LDAP server to read data from
-l LDAP_USER specify LDAP user for reading data from LDAP server
-a ldap_authfile specify path to file to open and read LDAP authtok
-d passfd specify open fd to read PASS
-f passfile specify path to file to open and read PASS
-e ENDPOINT specify REST endpoint
Expand Down Expand Up @@ -87,6 +56,9 @@ class Options:
osg_co_id = OSG_CO_ID
outfile = None
authstr = None
ldap_server = LDAP_SERVER
ldap_user = LDAP_USER
ldap_authtok = None
filtergrp = None


Expand All @@ -110,14 +82,14 @@ def co_group_is_project(gid):


def get_co_group_osggid(gid):
resp_data = get_co_group_identifiers(gid)
data = get_datalist(resp_data, "Identifiers")
resp_data = utils.get_co_group_identifiers(gid, options.endpoint, options.authstr)
data = utils.get_datalist(resp_data, "Identifiers")
return list(filter(lambda x : x["Type"] == "osggid", data))[0]["Identifier"]


def get_co_group_osggid(gid):
resp_data = get_co_group_identifiers(gid)
data = get_datalist(resp_data, "Identifiers")
resp_data = utils.get_co_group_identifiers(gid, options.endpoint, options.authstr)
data = utils.get_datalist(resp_data, "Identifiers")
return list(filter(lambda x : x["Type"] == "osggid", data))[0]["Identifier"]


Expand Down Expand Up @@ -147,11 +119,15 @@ def parse_options(args):

passfd = None
passfile = None
ldap_authfile = None

for op, arg in ops:
if op == '-h': usage()
if op == '-u': options.user = arg
if op == '-c': options.osg_co_id = int(arg)
if op == '-s': options.ldap_server= arg
if op == '-l': options.ldap_user = arg
if op == '-a': ldap_authfile = arg
if op == '-d': passfd = int(arg)
if op == '-f': passfile = arg
if op == '-e': options.endpoint = arg
Expand All @@ -161,69 +137,20 @@ def parse_options(args):
try:
user, passwd = utils.getpw(options.user, passfd, passfile)
options.authstr = utils.mkauthstr(user, passwd)
options.ldap_authtok = utils.get_ldap_authtok(ldap_authfile)
except PermissionError:
usage("PASS required")


def get_ldap_group_members_data():
gidNumber_str = "gidNumber: "
gidNumber_regex = re.compile(gidNumber_str)
member_str = "hasMember: "
member_regex = re.compile(member_str)

auth_str = subprocess.run(
LDAP_AUTH_COMMAND,
stdout=subprocess.PIPE
).stdout.decode('utf-8').strip()

ldap_group_members_command = LDAP_GROUP_MEMBERS_COMMAND
ldap_group_members_command[LDAP_GROUP_MEMBERS_COMMAND.index("{auth}")] = auth_str

data_file = subprocess.run(
ldap_group_members_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n')

search_results = list(filter(
lambda x: not re.compile("#|dn:|cn:|objectClass:").match(x),
(line for line in data_file)))

search_results.reverse()

def get_ldap_group_members_dict():
group_data_dict = dict()
index = 0
while index < len(search_results) - 1:
while not gidNumber_regex.match(search_results[index]):
index += 1
gid = search_results[index].replace(gidNumber_str, "")
members_list = []
while search_results[index] != "":
if member_regex.match(search_results[index]):
members_list.append(search_results[index].replace(member_str, ""))
index += 1
group_data_dict[gid] = members_list
index += 1
for group_gid in utils.get_ldap_groups(options.ldap_server, options.ldap_user, options.ldap_authtok):
group_members = utils.get_ldap_group_members(group_gid, options.ldap_server, options.ldap_user, options.ldap_authtok)
group_data_dict[group_gid] = group_members

return group_data_dict


def get_ldap_active_users(filter_group_name):
auth_str = subprocess.run(
LDAP_AUTH_COMMAND,
stdout=subprocess.PIPE
).stdout.decode('utf-8').strip()

filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None
else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))")

ldap_active_users_command = LDAP_ACTIVE_USERS_COMMAND
ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{auth}")] = auth_str
ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{filter}")] = filter_str

active_users = subprocess.run(ldap_active_users_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n')
users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*")
else "" for line in active_users)
return users


def create_user_to_projects_map(project_to_user_map, active_users, osggids_to_names):
users_to_projects_map = dict()
for osggid in project_to_user_map:
Expand Down Expand Up @@ -256,26 +183,25 @@ def get_co_api_data():
for entry in entries:
osggid_name_pair = entry.split(":")
if len(osggid_name_pair) == 2:
project_osggids_to_name[osggid_name_pair[0]] = osggid_name_pair[1]
project_osggids_to_name[int(osggid_name_pair[0])] = osggid_name_pair[1].strip()
r.close()
else:
r.close()
raise OSError
except OSError:
with open(CACHE_FILENAME, "w") as w:
project_osggids_to_name = get_groups_data_from_api()
print(time.time(), file=w)
for osggid, name in project_osggids_to_name.items():
print(f"{osggid}:{name}", file=w)
finally:
if r:
r.close()

return project_osggids_to_name


def get_osguser_groups(filter_group_name=None):
project_osggids_to_name = get_co_api_data()
ldap_groups_members = get_ldap_group_members_data()
ldap_users = get_ldap_active_users(filter_group_name)
ldap_groups_members = get_ldap_group_members_dict()
ldap_users = utils.get_ldap_active_users(options.ldap_server, options.ldap_user, options.ldap_authtok, filter_group_name)

active_project_osggids = set(ldap_groups_members.keys()).intersection(set(project_osggids_to_name.keys()))
project_to_user_map = {
Expand Down

0 comments on commit af0d2eb

Please sign in to comment.