From c5a5484d184f054ee7bea9d2970d6bc07363c279 Mon Sep 17 00:00:00 2001 From: Paxton Hare Date: Mon, 1 Dec 2014 08:51:50 -0500 Subject: [PATCH 1/4] #311 - adding support for external authentication (Ldap and such) --- deploy/lib/xquery/setup.xqy | 198 ++++++++++++++++++++++-- deploy/sample/ml-config.sample.xml | 16 ++ deploy/test/data/ml5-config.xml | 15 +- deploy/test/data/ml6-config.xml | 15 +- deploy/test/data/ml7-config-changed.xml | 12 ++ deploy/test/data/ml7-config.xml | 12 ++ 6 files changed, 257 insertions(+), 11 deletions(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index 1a7300d7..a30e6b40 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -134,7 +134,8 @@ declare variable $common-server-settings := pre-commit-trigger-depth pre-commit-trigger-limit collation - authentication + internal-security + authentication privilege concurrent-request-limit log-errors @@ -179,7 +180,7 @@ declare variable $http-server-settings := url-rewriter rewrite-resolves-globally static-expires - default-user + default-user ; @@ -295,7 +296,8 @@ declare function setup:get-rollback-config() element sec:privileges { map:get($roll-back, "privileges") - } + }, + map:get($roll-back, "external-security") } }; @@ -307,6 +309,7 @@ declare function setup:do-setup($import-config as element(configuration)) as ite setup:create-privileges($import-config), setup:create-roles($import-config), setup:create-users($import-config), + setup:create-external-security($import-config), setup:create-mimetypes($import-config), setup:create-groups($import-config), setup:create-forests($import-config), @@ -324,6 +327,14 @@ declare function setup:do-setup($import-config as element(configuration)) as ite } catch($ex) { + if ($ex/error:code = "ADMIN-INVALIDAUTHENTICATION") then + fn:concat(' + Either your authentication configuration is invalid or you + are trying to change from external authentication back to internal authentication. There is a bug + in MarkLogic''s Admin API which prevents going from external back to internal. + See http://docs.marklogic.com/guide/security/external-auth#id_63262 for more information on + configuring external authentication. ' ) + else (), xdmp:log($ex), setup:do-wipe(setup:get-rollback-config()), fn:concat($ex/err:format-string/text(), ' See MarkLogic Server error log for more details.') @@ -332,6 +343,7 @@ declare function setup:do-setup($import-config as element(configuration)) as ite declare function setup:do-wipe($import-config as element(configuration)) as item()* { + let $_ := xdmp:log(("wiping: ", $import-config)) (: remove scheduled tasks :) let $admin-config := admin:get-configuration() let $remove-tasks := @@ -432,7 +444,7 @@ declare function setup:do-wipe($import-config as element(configuration)) as item else ()', (xs:QName("amp"), $amp), - {xdmp:database("Security")} + {$default-security} ) } catch($ex) @@ -541,7 +553,7 @@ declare function setup:do-wipe($import-config as element(configuration)) as item sec:remove-user($user)', (xs:QName("user"), $user), - {xdmp:database("Security")} + {$default-security} ) } catch($ex) @@ -562,7 +574,7 @@ declare function setup:do-wipe($import-config as element(configuration)) as item sec:remove-role($role)', (xs:QName("role"), $role), - {xdmp:database("Security")} + {$default-security} ) } catch($ex) @@ -585,7 +597,7 @@ declare function setup:do-wipe($import-config as element(configuration)) as item (xs:QName("action"), $priv/sec:action, xs:QName("kind"), $priv/sec:kind), - {xdmp:database("Security")} + {$default-security} ) } catch($ex) @@ -595,6 +607,28 @@ declare function setup:do-wipe($import-config as element(configuration)) as item xdmp:rethrow() }, + (: remove external security :) + for $es in $import-config/sec:external-securities/sec:external-security + return + try + { + xdmp:eval( + 'import module namespace sec="http://marklogic.com/xdmp/security" at "/MarkLogic/security.xqy"; + declare variable $name as xs:string external; + + sec:remove-external-security($name)', + (xs:QName("name"), $es/sec:external-security-name), + + {$default-security} + ) + } + catch($ex) + { + if ($ex/error:code = "SEC-EXTERNALSECURITYDNE") then () + else + xdmp:rethrow() + }, + if ($restart-needed) then "note: restart required" else () @@ -3300,7 +3334,12 @@ declare function setup:configure-server( else fn:data(xdmp:value(fn:concat("$server-config/gr:", $setting, $setting-test))) let $min-version as xs:string? := $setting/@min-version - where (fn:exists($value)) + let $if := + if ($setting/@if) then + xdmp:value($setting/@if) + else + fn:true() + where ($if and fn:exists($value)) return if (fn:empty($min-version) or setup:at-least-version($min-version)) then xdmp:set($admin-config, @@ -3364,6 +3403,41 @@ declare function setup:configure-server( else $admin-config + let $admin-config := + let $external-security as xs:string? := $server-config/gr:external-security/@name + return + if ($external-security) then + try { + xdmp:eval(' + declare variable $admin-config external; + declare variable $server-id external; + declare variable $external-security external; + declare variable $server-config external; + + admin:appserver-set-external-security( + $admin-config, + $server-id, + $external-security, + xs:boolean($server-config/gr:internal-security), + fn:string($server-config/gr:authentication)) + ', + ( + xs:QName("admin-config"), $admin-config, + xs:QName("server-id"), $server-id, + xs:QName("external-security"), $external-security, + xs:QName("server-config"), $server-config + )) + } + catch($ex) { + if ($ex/error:code = "XDMP-UNDFUN" and fn:not(setup:at-least-version("7.0-0"))) then + (: If we're not using a recent enough version of ML, then the properties are irrelevant. :) + () + else + xdmp:rethrow() + } + else + $admin-config + let $module-locations := $server-config/gr:module-locations let $admin-config := if ($module-locations/*) then @@ -3739,6 +3813,75 @@ declare function setup:validate-privileges( setup:validation-fail(fn:concat("Missing privilege: ", $privilege-name)) }; +declare function setup:create-external-security( + $import-config as element(configuration)) +{ + let $eval-options := + + {$default-security} + + for $es in $import-config/sec:external-securities/sec:external-security + return + (: if it exists, don't recreate it :) + if (setup:get-external-securities($es/sec:external-security-name)) then () + else + ( + (: Wrapping this in xdmp:eval because it didn't exist until ML7 :) + try { + xdmp:eval( + 'import module namespace sec="http://marklogic.com/xdmp/security" at "/MarkLogic/security.xqy"; + declare variable $es as element(sec:external-security) external; + + sec:create-external-security( + $es/sec:external-security-name, + $es/sec:description, + $es/sec:authentication, + $es/sec:cache-timeout, + $es/sec:authorization, + $es/sec:ldap-server-uri, + $es/sec:ldap-base, + $es/sec:ldap-attribute, + $es/sec:ldap-default-user, + $es/sec:ldap-password)', + (xs:QName("es"), $es), + $eval-options) + } + catch($ex) { + if ($ex/error:code = "XDMP-UNDFUN" and fn:not(setup:at-least-version("7.0-0"))) then + (: If we're not using a recent enough version of ML, then the properties are irrelevant. :) + () + else + xdmp:rethrow() + }, + setup:add-rollback("external-security", $es) + ) +}; + +declare function setup:validate-external-security( + $import-config as element(configuration)) +{ + for $es in $import-config/sec:external-securities/sec:external-security + let $es-name as xs:string? := $es/sec:external-security-name + let $match := setup:get-external-securities($es-name) + return + if ($match) then + let $match-elements := $match/*[fn:not(fn:local-name(.) = 'external-security-id')] + let $all-match := + for $e in $match-elements + let $name := fn:node-name($e) + return + $es/*[fn:node-name(.) = $name] = $e + let $has-mismatch := $all-match = fn:false() + let $c1 := fn:count($es/*) + let $c2 := fn:count($match-elements) + return + if ($c1 ne $c2 or $has-mismatch) then + setup:validation-fail(fn:concat("Mismatched external-security ", $es-name)) + else () + else + setup:validation-fail(fn:concat("Missing external-security ", $es-name)) +}; + declare function setup:create-roles( $import-config as element(configuration)) { @@ -3898,7 +4041,6 @@ declare function setup:validate-roles( let $amps as element(sec:amp)* := $role/sec:amps/* let $match := setup:get-roles(())/sec:role[sec:role-name = $role-name] return - (: if the role exists, then update it :) if ($match) then if ($match/sec:role-name != $role-name or $match/sec:description != $description or @@ -4267,6 +4409,22 @@ declare function setup:get-appserver-default-user($server-config as element()) a else $default-user }; +declare function setup:get-appserver-internal-security($server-config as element()) as xs:boolean? +{ + if (setup:at-least-version("7.0-0")) then + ( + fn:data($server-config/gr:internal-security), + fn:not(setup:get-appserver-external-security($server-config)[fn:not(. = "")]), + fn:true() + )[1] + else () +}; + +declare function setup:get-appserver-external-security($server-config as element()) as xs:string? +{ + fn:data($server-config/gr:external-security/(@name|text())) +}; + declare function setup:get-ssl-certificate-template( $server-config as element()) as xs:unsignedLong @@ -4444,6 +4602,27 @@ declare function setup:get-roles($ids as xs:unsignedLong*) as element(sec:roles) } }; +declare function setup:get-external-securities($names as xs:string*) as element(sec:external-securities)* +{ + let $external-securities := + xdmp:eval( + 'import module namespace sec="http://marklogic.com/xdmp/security" at "/MarkLogic/security.xqy"; + fn:collection(sec:security-collection())/sec:external-security + ', + (), + + {$default-security} + ) + return + element sec:external-securities { + if ($names) then + $external-securities[sec:external-security-name = $names] + else + $external-securities + } +}; + + declare function setup:get-amps($ids as xs:unsignedLong*) as element(sec:amps)? { let $amps := xdmp:eval( @@ -4831,6 +5010,7 @@ declare function setup:validate-install($import-config as element(configuration) { try { + setup:validate-external-security($import-config), setup:validate-privileges($import-config), setup:validate-roles($import-config), setup:validate-users($import-config), diff --git a/deploy/sample/ml-config.sample.xml b/deploy/sample/ml-config.sample.xml index b027ed59..20694906 100644 --- a/deploy/sample/ml-config.sample.xml +++ b/deploy/sample/ml-config.sample.xml @@ -500,4 +500,20 @@ --> + diff --git a/deploy/test/data/ml5-config.xml b/deploy/test/data/ml5-config.xml index 958750e2..b9476ee0 100644 --- a/deploy/test/data/ml5-config.xml +++ b/deploy/test/data/ml5-config.xml @@ -697,4 +697,17 @@ text - \ No newline at end of file + + + test-external + a big test + ldap + 300 + ldap + ldap://dc1.mltest1.local:389 + CN=Users,DC=MLTEST1,DC=LOCAL + sAMAccountName + + + + diff --git a/deploy/test/data/ml6-config.xml b/deploy/test/data/ml6-config.xml index 431f62cb..5811de57 100644 --- a/deploy/test/data/ml6-config.xml +++ b/deploy/test/data/ml6-config.xml @@ -799,4 +799,17 @@ text - \ No newline at end of file + + + test-external + a big test + ldap + 300 + ldap + ldap://dc1.mltest1.local:389 + CN=Users,DC=MLTEST1,DC=LOCAL + sAMAccountName + + + + diff --git a/deploy/test/data/ml7-config-changed.xml b/deploy/test/data/ml7-config-changed.xml index 7999040b..7d19b5c2 100644 --- a/deploy/test/data/ml7-config-changed.xml +++ b/deploy/test/data/ml7-config-changed.xml @@ -1166,4 +1166,16 @@ text + + test-external + a big test + ldap + 300 + ldap + ldap://dc1.mltest1.local:389 + CN=Users,DC=MLTEST1,DC=LOCAL + sAMAccountName + + + diff --git a/deploy/test/data/ml7-config.xml b/deploy/test/data/ml7-config.xml index 0b20926d..89ae4434 100644 --- a/deploy/test/data/ml7-config.xml +++ b/deploy/test/data/ml7-config.xml @@ -1151,4 +1151,16 @@ text + + test-external + a big test + ldap + 300 + ldap + ldap://dc1.mltest1.local:389 + CN=Users,DC=MLTEST1,DC=LOCAL + sAMAccountName + + + From a5061a4a549e2417c245ed0ef1dc64572a6cf431 Mon Sep 17 00:00:00 2001 From: Paxton Hare Date: Mon, 1 Dec 2014 08:58:40 -0500 Subject: [PATCH 2/4] #311 - adding example for using external security with http server --- deploy/sample/ml-config.sample.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/deploy/sample/ml-config.sample.xml b/deploy/sample/ml-config.sample.xml index 20694906..f3a0c872 100644 --- a/deploy/sample/ml-config.sample.xml +++ b/deploy/sample/ml-config.sample.xml @@ -95,6 +95,13 @@ @ml.url-rewriter @ml.error-handler @ml.rewrite-resolves-globally + + @ml.test-appserver From 2930248a877b495d95fc748ce641d38027d397ff Mon Sep 17 00:00:00 2001 From: Paxton Hare Date: Mon, 1 Dec 2014 12:07:11 -0500 Subject: [PATCH 3/4] #311 - fixing some issues with external security --- deploy/lib/xquery/setup.xqy | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index a30e6b40..f2325abf 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -3328,20 +3328,21 @@ declare function setup:configure-server( "" else "[fn:string-length(fn:string(.)) > 0]" - let $value := - if ($setting/@value) then - xdmp:value($setting/@value) - else - fn:data(xdmp:value(fn:concat("$server-config/gr:", $setting, $setting-test))) let $min-version as xs:string? := $setting/@min-version + let $version-ok := fn:empty($min-version) or setup:at-least-version($min-version) let $if := if ($setting/@if) then xdmp:value($setting/@if) else fn:true() + let $value := + if ($setting/@value and $if and $version-ok) then + xdmp:value($setting/@value) + else + fn:data(xdmp:value(fn:concat("$server-config/gr:", $setting, $setting-test))) where ($if and fn:exists($value)) return - if (fn:empty($min-version) or setup:at-least-version($min-version)) then + if ($version-ok) then xdmp:set($admin-config, xdmp:value(fn:concat("admin:appserver-set-", $setting, "($admin-config, $server-id, $value)"))) else @@ -3409,6 +3410,10 @@ declare function setup:configure-server( if ($external-security) then try { xdmp:eval(' + import module namespace admin = "http://marklogic.com/xdmp/admin" at "/MarkLogic/admin.xqy"; + + declare namespace gr="http://marklogic.com/xdmp/group"; + declare variable $admin-config external; declare variable $server-id external; declare variable $external-security external; From f157d725fb5d9f93c2196a33cc2213a47389e13f Mon Sep 17 00:00:00 2001 From: Paxton Hare Date: Mon, 1 Dec 2014 13:56:08 -0500 Subject: [PATCH 4/4] #311 - fixing another bug. adding error messages --- deploy/lib/xquery/setup.xqy | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index f2325abf..e1905ff8 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -3435,8 +3435,9 @@ declare function setup:configure-server( } catch($ex) { if ($ex/error:code = "XDMP-UNDFUN" and fn:not(setup:at-least-version("7.0-0"))) then - (: If we're not using a recent enough version of ML, then the properties are irrelevant. :) - () + fn:error( + xs:QName("VERSION_NOT_SUPPORTED"), + fn:concat("MarkLogic ", xdmp:version(), " does not support external security. Use 7.0-0 or higher.")) else xdmp:rethrow() } @@ -3828,7 +3829,7 @@ declare function setup:create-external-security( for $es in $import-config/sec:external-securities/sec:external-security return (: if it exists, don't recreate it :) - if (setup:get-external-securities($es/sec:external-security-name)) then () + if (setup:get-external-securities($es/sec:external-security-name)/sec:external-security) then () else ( (: Wrapping this in xdmp:eval because it didn't exist until ML7 :) @@ -3853,8 +3854,9 @@ declare function setup:create-external-security( } catch($ex) { if ($ex/error:code = "XDMP-UNDFUN" and fn:not(setup:at-least-version("7.0-0"))) then - (: If we're not using a recent enough version of ML, then the properties are irrelevant. :) - () + fn:error( + xs:QName("VERSION_NOT_SUPPORTED"), + fn:concat("MarkLogic ", xdmp:version(), " does not support external security. Use 7.0-0 or higher.")) else xdmp:rethrow() },