From 861bd5982a4cd29964545643f6a17e336e3ae6fb Mon Sep 17 00:00:00 2001 From: Kees Vegter Date: Fri, 4 Aug 2017 12:33:29 +0200 Subject: [PATCH] added apoc.load.ldap --- build.gradle | 4 +- docs/loadldap.adoc | 103 +++++++++ src/main/java/apoc/load/LDAPResult.java | 11 + src/main/java/apoc/load/LoadLdap.java | 256 ++++++++++++++++++++++ src/test/java/apoc/load/LoadLdapTest.java | 38 ++++ 5 files changed, 411 insertions(+), 1 deletion(-) create mode 100644 docs/loadldap.adoc create mode 100644 src/main/java/apoc/load/LDAPResult.java create mode 100644 src/main/java/apoc/load/LoadLdap.java create mode 100644 src/test/java/apoc/load/LoadLdapTest.java diff --git a/build.gradle b/build.gradle index 46e4e14645..aa6521f22c 100644 --- a/build.gradle +++ b/build.gradle @@ -48,7 +48,7 @@ ext { } repositories { - mavenLocal() +// mavenLocal() maven { url "https://m2.neo4j.org/content/repositories/snapshots" } mavenCentral() maven { url "http://oss.sonatype.org/content/repositories/snapshots/" } @@ -72,6 +72,8 @@ dependencies { compile 'com.jayway.jsonpath:json-path:2.2.0' compileOnly group: 'net.biville.florent', name: 'neo4j-sproc-compiler', version:'1.2' + compile 'com.novell.ldap:jldap:2009-10-07' + testCompile group: 'junit', name: 'junit', version:'4.12' testCompile group: 'org.hamcrest', name: 'hamcrest-library', version:'1.3' testCompile group: 'org.apache.derby', name: 'derby', version:'10.12.1.1' diff --git a/docs/loadldap.adoc b/docs/loadldap.adoc new file mode 100644 index 0000000000..bb034a58f0 --- /dev/null +++ b/docs/loadldap.adoc @@ -0,0 +1,103 @@ += Load LDAP + +With 'apoc.load.ldap' you can execute queries on any LDAP v3 enabled directory, the results are turned into a streams of entries. +The entries can then be used to update or create graph structures. + +Note this utility requires to have the link:https://mvnrepository.com/artifact/com.novell.ldap/jldap/2009-10-07[jldap] library to be placed the plugin directory. + +[separator=¦,opts=header,cols="1,1m,5"] +|=== +include::../build/generated-documentation/apoc.load.csv[lines=9..9] +|=== + +=== parameters +[options="header",cols="a,3m,a"] +|=== +|Parameter | Property | Description +|{connectionMap} | ldapHost | the ldapserver:port if port is omitted the default port 389 will be used +| | loginDN | This is the dn of the ldap server user who has read access on the ldap server +| | loginPW | This is the password used by the loginDN +|{searchMap} | searchBase | From this entry a search is executed +| | searchScope | SCOPE_ONE (one level) or + SCOPE_SUB (all sub levels) or + SCOPE_BASE (only the base node) +| | searchFilter | Place here a standard ldap search filter for example: (objectClass=*) means that the ldap entry must have an objectClass attribute. +| | attributes | optional. If omitted all the attributes of the entries will be returned. + When specified only the specified attributes will be returned. Regardless the attributes setting a returned entry will always have a "dn" property. +|=== + +==== load ldap example + +.Retrieve group member information from the ldap server +[source,cypher] +--- +call apoc.load.ldap({ldapHost : "ldap.forumsys.com", loginDN : "cn=read-only-admin,dc=example,dc=com", loginPW : "password"}, +{searchBase : "dc=example,dc=com",searchScope : "SCOPE_SUB" +,attributes : ["uniqueMember","cn","uid","objectClass"] +,searchFilter: "(&(objectClass=*)(uniqueMember=*))"}) yield entry +return entry.dn, entry.uniqueMember +--- +[options="header",cols="3m,a"] +|=== +| entry.dn | entry.uniqueMember | +| "ou=mathematicians,dc=example,dc=com" | ["uid=euclid,dc=example,dc=com", "uid=riemann,dc=example,dc=com", "uid=euler,dc=example,dc=com", "uid=gauss,dc=example,dc=com", "uid=test,dc=example,dc=com"] | +| "ou=scientists,dc=example,dc=com" | ["uid=einstein,dc=example,dc=com", "uid=galieleo,dc=example,dc=com", "uid=tesla,dc=example,dc=com", "uid=newton,dc=example,dc=com", "uid=training,dc=example,dc=com", "uid=jmacy,dc=example,dc=com"] | +| "ou=italians,ou=scientists,dc=example,dc=com" | "uid=tesla,dc=example,dc=com" | +| "ou=chemists,dc=example,dc=com" | ["uid=curie,dc=example,dc=com", "uid=boyle,dc=example,dc=com", "uid=nobel,dc=example,dc=com", "uid=pasteur,dc=example,dc=com"] | +|=== +--- + +.Retrieve group member information from the ldap server and create structure in Neo4j +[source,cypher] +--- +call apoc.load.ldap({ldapHost : "ldap.forumsys.com", loginDN : "cn=read-only-admin,dc=example,dc=com", loginPW : "password"}, +{searchBase : "dc=example,dc=com",searchScope : "SCOPE_SUB" +,attributes : ["uniqueMember","cn","uid","objectClass"] +,searchFilter: "(&(objectClass=*)(uniqueMember=*))"}) yield entry +merge (g:Group {dn : entry.dn}) +on create set g.cn = entry.cn +foreach (member in entry.uniqueMember | + merge (p:Person { dn : member }) + merge (p)-[:IS_MEMBER]->(g) +) +--- + + +=== credentials + +To protect credentials, you can configure aliases in `conf/neo4j.conf`: + +---- +apoc.loadldap.myldap.config=: +---- + + +==== example neo4j.conf: + +---- +apoc.loadldap.myldap.config=ldap.forumsys.com:389 cn=read-only-admin,dc=example,dc=com password +---- + +Then + +---- +call apoc.load.ldap({ldapHost : "ldap.forumsys.com", loginDN : "cn=read-only-admin,dc=example,dc=com", loginPW : "password"} +, {searchBase : "dc=example,dc=com" + ,searchScope : "SCOPE_SUB" + ,attributes : ["cn","uid","objectClass"] + ,searchFilter: "(&(objectClass=*))" + }) yield entry +return entry.dn, entry +---- + +becomes + +---- +call apoc.load.ldap("myldap" +,{searchBase : "dc=example,dc=com" + ,searchScope : "SCOPE_SUB" + ,attributes : ["cn","uid","objectClass"] + ,searchFilter: "(&(objectClass=*))" + }) yield entry +return entry.dn, entry +---- \ No newline at end of file diff --git a/src/main/java/apoc/load/LDAPResult.java b/src/main/java/apoc/load/LDAPResult.java new file mode 100644 index 0000000000..095beefac4 --- /dev/null +++ b/src/main/java/apoc/load/LDAPResult.java @@ -0,0 +1,11 @@ +package apoc.load; + +import java.util.Map; + +public class LDAPResult { + public final Map entry; + + public LDAPResult(Map entry) { + this.entry = entry; + } +} diff --git a/src/main/java/apoc/load/LoadLdap.java b/src/main/java/apoc/load/LoadLdap.java new file mode 100644 index 0000000000..c1d03c4c92 --- /dev/null +++ b/src/main/java/apoc/load/LoadLdap.java @@ -0,0 +1,256 @@ +package apoc.load; + +import apoc.ApocConfiguration; +import com.novell.ldap.*; +import org.neo4j.procedure.Description; +import org.neo4j.procedure.Mode; +import org.neo4j.procedure.Name; +import org.neo4j.procedure.Procedure; + +import java.io.UnsupportedEncodingException; +import java.util.*; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +public class LoadLdap { + + @Procedure(name = "apoc.load.ldap", mode = Mode.READ) + @Description("apoc.load.ldap(\"key\" or {connectionMap},{searchMap}) Load entries from an ldap source (yield entry)") + public Stream ldapQuery(@Name("connection") final Object conn, @Name("search") final Map search) { + + LDAPManager mgr = new LDAPManager(getConnectionMap(conn)); + + return mgr.executeSearch(search); + } + + public static Map getConnectionMap(Object conn) { + if (conn instanceof String) { + //String value = "ldap.forumsys.com cn=read-only-admin,dc=example,dc=com password"; + Object value = ApocConfiguration.get("loadldap").get(conn.toString() + ".config"); + // format + if (value == null) throw new RuntimeException("No apoc.loadldap."+conn+".config ldap access configuration specified"); + Map config = new HashMap<>(); + String[] sConf = ((String) value).split(" "); + config.put("ldapHost", sConf[0]); + config.put("loginDN", sConf[1]); + config.put("loginPW", sConf[2]); + + return config; + + } else { + return (Map ) conn; + } + } + + public static class LDAPManager { + private static final String LDAP_HOST_P = "ldapHost"; + private static final String LDAP_LOGIN_DN_P = "loginDN"; + private static final String LDAP_LOGIN_PW_P = "loginPW"; + private static final String SEARCH_BASE_P = "searchBase"; + private static final String SEARCH_SCOPE_P = "searchScope"; + private static final String SEARCH_FILTER_P = "searchFilter"; + private static final String SEARCH_ATTRIBUTES_P = "attributes"; + + private static final String SCOPE_BASE = "SCOPE_BASE"; + private static final String SCOPE_ONE = "SCOPE_ONE"; + private static final String SCOPE_SUB = "SCOPE_SUB"; + + private int ldapPort; + private int ldapVersion = LDAPConnection.LDAP_V3; + private String ldapHost; + private String loginDN; + private String password; + private LDAPConnection lc; + private List attributeList; + + public LDAPManager(Map connParms) { + + String sLdapHostPort = (String) connParms.get(LDAP_HOST_P); + if (sLdapHostPort.indexOf(":") > -1) { + this.ldapHost = sLdapHostPort.substring(0, sLdapHostPort.indexOf(":")); + this.ldapPort = Integer.parseInt(sLdapHostPort.substring(sLdapHostPort.indexOf(":") + 1)); + } else { + this.ldapHost = sLdapHostPort; + this.ldapPort = 389; // default + } + + this.loginDN = (String) connParms.get(LDAP_LOGIN_DN_P); + this.password = (String) connParms.get(LDAP_LOGIN_PW_P); + } + + public Stream executeSearch(Map search) { + try { + Iterator> supplier = new SearchResultsIterator(doSearch(search), attributeList); + Spliterator> spliterator = Spliterators.spliteratorUnknownSize(supplier, Spliterator.ORDERED); + return StreamSupport.stream(spliterator, false).map(LDAPResult::new).onClose(() -> closeIt(lc)); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + public LDAPSearchResults doSearch(Map search) { + // parse search parameters + String searchBase = (String) search.get(SEARCH_BASE_P); + String searchFilter = (String) search.get(SEARCH_FILTER_P); + String sScope = (String) search.get(SEARCH_SCOPE_P); + attributeList = (List) search.get(SEARCH_ATTRIBUTES_P); + if (attributeList == null) attributeList = new ArrayList<>(); + int searchScope = LDAPConnection.SCOPE_SUB; + if (sScope.equals(SCOPE_BASE)) { + searchScope = LDAPConnection.SCOPE_BASE; + } else if (sScope.equals(SCOPE_ONE)) { + searchScope = LDAPConnection.SCOPE_ONE; + } else if (sScope.equals(SCOPE_SUB)) { + searchScope = LDAPConnection.SCOPE_SUB; + } else { + throw new RuntimeException("Invalid scope:" + sScope + ". value scopes are SCOPE_BASE, SCOPE_ONE and SCOPE_SUB"); + } + // getting an ldap connection + try { + lc = getConnection(); + // execute query + LDAPSearchConstraints cons = new LDAPSearchConstraints(); + cons.setMaxResults(0); // no limit + LDAPSearchResults searchResults = null; + if (attributeList == null || attributeList.size() == 0) { + searchResults = lc.search(searchBase, searchScope, searchFilter, null, false, cons); + } else { + searchResults = lc.search(searchBase, searchScope, searchFilter, attributeList.toArray(new String[0]), false, cons); + } + return searchResults; + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + private LDAPEntry read(String dn) throws LDAPException, UnsupportedEncodingException { + if (dn == null) return null; + LDAPEntry r = null; + op("read start for dn: " + dn); + LDAPConnection lc = getConnection(); + r = lc.read(dn); + closeIt(lc); + // op( r.toString()); + op("read end"); + return r; + } + + private LDAPSchema getSchema() throws LDAPException, UnsupportedEncodingException { + LDAPSchema r = null; + LDAPConnection lc = getConnection(); + r = lc.fetchSchema(lc.getSchemaDN()); + closeIt(lc); + //op( r.toString()); + + return r; + } + + public static void closeIt(LDAPConnection lc) { + try { + lc.disconnect(); + } catch (Exception e) { + // ignore + e.printStackTrace(); + } + } + + private LDAPConnection getConnection() throws LDAPException, UnsupportedEncodingException { +// LDAPSocketFactory ssf; +// Security.addProvider(new com.sun.net.ssl.internal.ssl.Provider()); + // String path ="C:\\j2sdk1.4.2_09\\jre\\lib\\security\\cacerts"; + //op("the trustStore: " + System.getProperty("javax.net.ssl.trustStore")); + // System.setProperty("javax.net.ssl.trustStore", path); +// op(" reading the strustStore: " + System.getProperty("javax.net.ssl.trustStore")); +// ssf = new LDAPJSSESecureSocketFactory(); +// LDAPConnection.setSocketFactory(ssf); + + + LDAPConnection lc = new LDAPConnection(); + lc.connect(ldapHost, ldapPort); + + // bind to the server + lc.bind(ldapVersion, loginDN, password.getBytes("UTF8")); + // tbd + // LDAPConnection pooling here? + // + return lc; + } + + private void op(String s) { + System.out.println("LDAPManager:>" + s); + } + + } + private static class SearchResultsIterator implements Iterator> { + private final LDAPSearchResults lsr; + private final List attributes; + private Map map; + public SearchResultsIterator(LDAPSearchResults lsr, List attributes) { + this.lsr = lsr; + this.attributes = attributes; + this.map = get(); + } + + @Override + public boolean hasNext() { + return this.map != null; + } + + @Override + public Map next() { + Map current = this.map; + this.map = get(); + return current; + } + + public Map get() { + if (handleEndOfResults()) return null; + try { + Map entry = new LinkedHashMap<>(attributes.size() + 1); + LDAPEntry en = null; + en = lsr.next(); + entry.put("dn", en.getDN()); + if (attributes != null && attributes.size() > 0) { + for (int col = 0; col < attributes.size(); col++) { + Object val = readValue(en.getAttributeSet().getAttribute(attributes.get(col))); + if (val != null) entry.put(attributes.get(col),val ); + } + } else { + // make it dynamic + Iterator iter = en.getAttributeSet().iterator(); + while (iter.hasNext()) { + LDAPAttribute attr = iter.next(); + Object val = readValue(attr); + if (val != null) entry.put(attr.getName(), readValue(attr)); + } + } + //System.out.println("entry " + entry); + return entry; + + } catch (LDAPException e) { + e.printStackTrace(); + throw new RuntimeException("Error getting next ldap entry " + e.getLDAPErrorMessage()); + } + } + + private boolean handleEndOfResults() { + if (!lsr.hasMore()) { + return true; + } + return false; + } + private Object readValue(LDAPAttribute att) { + if (att == null) return null; + if (att.size() == 1) { + // single value + // for now everything is string + return att.getStringValue(); + } else { + return att.getStringValueArray(); + } + } + } + +} diff --git a/src/test/java/apoc/load/LoadLdapTest.java b/src/test/java/apoc/load/LoadLdapTest.java new file mode 100644 index 0000000000..9f4c695e5b --- /dev/null +++ b/src/test/java/apoc/load/LoadLdapTest.java @@ -0,0 +1,38 @@ +package apoc.load; + + +import com.novell.ldap.LDAPEntry; +import com.novell.ldap.LDAPSearchResults; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +public class LoadLdapTest { + + @Test + public void testLoadLDAP() throws Exception { + Map connParms = new HashMap<>(); + connParms.put("ldapHost", "ldap.forumsys.com"); + connParms.put("ldapPort", 389l); + connParms.put("loginDN", "cn=read-only-admin,dc=example,dc=com"); + connParms.put("loginPW", "password"); + LoadLdap.LDAPManager mgr = new LoadLdap.LDAPManager(LoadLdap.getConnectionMap(connParms)); + Map searchParms = new HashMap<>(); + searchParms.put("searchBase", "dc=example,dc=com"); + searchParms.put("searchScope", "SCOPE_ONE"); + searchParms.put("searchFilter", "(&(objectClass=*)(uid=training))"); + ArrayList ats = new ArrayList<>(); + ats.add("uid"); + searchParms.put("attributes", ats); + LDAPSearchResults results = mgr.doSearch(searchParms); + LDAPEntry le = results.next(); + assertEquals("uid=training,dc=example,dc=com", le.getDN()); + assertEquals("training", le.getAttribute("uid").getStringValue()); + } + +} +