From 93c0908442511a16788581a91071bfb089c37aca Mon Sep 17 00:00:00 2001 From: Richard Louapre Date: Thu, 17 Mar 2016 06:59:42 -0400 Subject: [PATCH 1/7] new apply-groups-setting-add function - #585 --- deploy/lib/xquery/setup.xqy | 53 +++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index dd571243..cce1246a 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -3153,13 +3153,16 @@ declare function setup:configure-groups($import-config as element(configuration) if ($setting/@value) then xdmp:value($setting/@value) else - fn:data(xdmp:value(fn:concat("$group-config/gr:", $setting, $setting-test))) + xdmp:value(fn:concat("$group-config/gr:", $setting, $setting-test)) let $min-version as xs:string? := $setting/@min-version where (fn:exists($value)) return if (fn:empty($min-version) or setup:at-least-version($min-version)) then xdmp:set($admin-config, - xdmp:value(fn:concat("admin:group-set-", $setting, "($admin-config, $group-id, $value)"))) + if ($setting/@function eq ="add") then + setup:apply-groups-setting-add($admin-config, $group-id, $setting, $value) + else + xdmp:value(fn:concat("admin:group-set-", $setting, "($admin-config, $group-id, fn:data($value))"))) else fn:error( xs:QName("VERSION_NOT_SUPPORTED"), @@ -3174,6 +3177,52 @@ declare function setup:configure-groups($import-config as element(configuration) ) }; +declare function setup:apply-groups-setting-add($admin-config as element(configuration), $group-id as xs:unsignedLong, $setting, $values as element()*) as element(configuration) { + if (fn:exists($values)) then + let $old-values := xdmp:value(fn:concat("admin:group-get-", $setting, "s", "($admin-config, $group-id)")) + let $admin-config := + (: First delete any values that matches the first child element, prefix must be unique :) + xdmp:value( + fn:concat( + "admin:group-delete-", $setting, "($admin-config, $group-id,", + "(", + fn:string-join( + ( + for $value in $values + let $old-value := $old-values[./node()[1] eq $value/node()[1]] + return + if (fn:exists($old-value)) then + fn:concat("admin:group-", $setting, "(", fn:string-join((for $child in $old-value/node() return fn:concat('"', $child, '"')), ","), ")") + else + () + ), + "," + ), + ")", + ")" + ) + ) + return + xdmp:value( + fn:concat( + "admin:group-add-", $setting, "($admin-config, $group-id,", + "(", + fn:string-join( + ( + for $value in $values + return fn:concat("admin:group-", $setting, "(", fn:string-join((for $child in $value/node() return fn:concat('"', $child, '"')), ","), ")") + ), + "," + ), + ")", + ")" + ) + ) + else + $admin-config +}; + + declare function setup:validate-groups-settings($import-config as element(configuration)) as item()* { let $admin-config := admin:get-configuration() From ce50ed1f2c4cab95b5b448171de3601dbedbf942 Mon Sep 17 00:00:00 2001 From: Richard Louapre Date: Fri, 18 Mar 2016 11:33:59 -0400 Subject: [PATCH 2/7] Add missing group settings for add functions --- deploy/lib/xquery/setup.xqy | 5 +++++ deploy/test/data/ml7-config.xml | 24 ++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index cce1246a..2e3e6cb0 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -110,6 +110,11 @@ declare variable $group-settings := + module-location + namespace + schema + trace-event + using-namespace ; declare variable $host-settings := diff --git a/deploy/test/data/ml7-config.xml b/deploy/test/data/ml7-config.xml index bb31be84..cfbdeed2 100644 --- a/deploy/test/data/ml7-config.xml +++ b/deploy/test/data/ml7-config.xml @@ -493,6 +493,26 @@ roxy-self-test + + + http://roxy/modules/dummy + /roxy/dummy.xqy + + + prefix1 + http://roxy/namespaces/prefix1 + + + http://roxy/schemas/dummy + /roxy/dummy.xsd + + + roxy-dummy-event + + + http://roxy/namespaces/prefix2 + + 2048 1 @@ -540,10 +560,10 @@ s3.amazonaws.com http none - + true - + true daily From fd828387882d281f87c94b641c5994c96c99bcb9 Mon Sep 17 00:00:00 2001 From: Richard Louapre Date: Fri, 18 Mar 2016 11:36:02 -0400 Subject: [PATCH 3/7] Fix syntax issue --- deploy/lib/xquery/setup.xqy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index 2e3e6cb0..042c464b 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -3164,7 +3164,7 @@ declare function setup:configure-groups($import-config as element(configuration) return if (fn:empty($min-version) or setup:at-least-version($min-version)) then xdmp:set($admin-config, - if ($setting/@function eq ="add") then + if ($setting/@function eq "add") then setup:apply-groups-setting-add($admin-config, $group-id, $setting, $value) else xdmp:value(fn:concat("admin:group-set-", $setting, "($admin-config, $group-id, fn:data($value))"))) From 482928563accc0c191a988a5cdb97f99a3de31f4 Mon Sep 17 00:00:00 2001 From: Richard Louapre Date: Fri, 18 Mar 2016 11:50:03 -0400 Subject: [PATCH 4/7] Fix test for single node cluster --- deploy/lib/xquery/setup.xqy | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index 042c464b..02e5c092 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -1110,17 +1110,16 @@ declare function setup:create-forests-from-config( let $hosts := admin:group-get-host-ids(admin:get-configuration(), $group-id) let $host-name as xs:string? := $forest-config/as:host-name[fn:string-length(fn:string(.)) > 0] let $host-id := if ($host-name) then xdmp:host($host-name) else $hosts[1] - let $hostnr := fn:index-of($hosts, $host-id) - let $replica-names as xs:string* := $forest-config/as:replica-names/as:replica-name[fn:string-length(fn:string(.)) > 0] - let $replicas := - $import-config/as:assignments/as:assignment[as:forest-name = $replica-names] return setup:create-forest( $forest-name, $data-directory, $host-id, if (fn:count($hosts) gt 1) then - setup:reassign-replicas($replicas, $hosts, $hostnr, $forest-name, 1, fn:false()) + let $hostnr := fn:index-of($hosts, $host-id) + let $replica-names as xs:string* := $forest-config/as:replica-names/as:replica-name[fn:string-length(fn:string(.)) > 0] + let $replicas := $import-config/as:assignments/as:assignment[as:forest-name = $replica-names] + return setup:reassign-replicas($replicas, $hosts, $hostnr, $forest-name, 1, fn:false()) else () ) From ccc0ea90b806ee687829c599aa48a3c8b8de2271 Mon Sep 17 00:00:00 2001 From: Richard Louapre Date: Sat, 19 Mar 2016 11:46:34 -0400 Subject: [PATCH 5/7] Add validation for group settings New function setup:validate-group-settings invoke by self-test --- deploy/lib/xquery/setup.xqy | 127 ++++++++++++++++++++++++++++-------- 1 file changed, 101 insertions(+), 26 deletions(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index 02e5c092..6f1b520a 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -3140,6 +3140,59 @@ declare function setup:validate-group( setup:validation-fail(fn:concat("Missing Group: ", $group)) }; +declare function setup:validate-group-settings( + $group-config as element(gr:group), + $setting as element() +) { + let $group := $group-config/gr:group-name/fn:string(.) + let $setting-name := $setting/fn:string(.) + let $function-name := fn:concat("group-get-", $setting-name, "s") + let $keys := fn:tokenize($setting/@keys, " ") + let $existing := + try + { + xdmp:eval( + fn:concat( + 'import module namespace admin = "http://marklogic.com/xdmp/admin" at "/MarkLogic/admin.xqy"; + declare variable $group external; + let $config := admin:get-configuration() + let $groupid := admin:group-get-id($config, $group) + return admin:', $function-name, '($config, $groupid)' + ), + ( + xs:QName("group"), $group + ) + ) + } + catch($ex) + { + if ($ex/error:code = "XDMP-UNDFUN") then () + else + xdmp:rethrow() + } + for $expected in xdmp:value(fn:concat("$group-config/gr:", $setting-name)) + return ( + if ($existing[setup:setting-equal($setting, ., $expected)]) then () + else ( + $keys ! setup:validation-fail(fn:string-join(("group setting mismatch", $setting-name, (., xdmp:value(fn:concat("$expected/gr:", .)), "=>", xdmp:value(fn:concat("$existing/gr:", .)))), " ")) + (:$keys ! fn:string-join(("group setting mismatch", $setting-name, (., xdmp:value(fn:concat("$expected/gr:", .)), "=>", xdmp:value(fn:concat("$existing/gr:", .)))), " "):) + ) + ) +}; + +declare function setup:setting-equal( + $setting as element(), + $expected as element(), + $existing as element() +) as xs:boolean { + let $keys := fn:tokenize($setting/@keys, " ") + return + if (fn:exists($keys)) then + (every $key in $keys satisfies xdmp:value(fn:concat("$expected/gr:", $key, "/fn:data(.)")) eq xdmp:value(fn:concat("$existing/gr:", $key, "/fn:data(.)"))) + else + fn:deep-equal($expected, $existing) +}; + declare function setup:configure-groups($import-config as element(configuration)) as item()* { let $admin-config := admin:get-configuration() @@ -3181,28 +3234,26 @@ declare function setup:configure-groups($import-config as element(configuration) ) }; -declare function setup:apply-groups-setting-add($admin-config as element(configuration), $group-id as xs:unsignedLong, $setting, $values as element()*) as element(configuration) { +declare function setup:apply-groups-setting-add( + $admin-config as element(configuration), + $group-id as xs:unsignedLong, + $setting as element(setting), + $values as element()* +) as element(configuration) { if (fn:exists($values)) then let $old-values := xdmp:value(fn:concat("admin:group-get-", $setting, "s", "($admin-config, $group-id)")) let $admin-config := - (: First delete any values that matches the first child element, prefix must be unique :) + (: First delete any values that matches the @keys first item, value must be unique :) xdmp:value( fn:concat( "admin:group-delete-", $setting, "($admin-config, $group-id,", - "(", - fn:string-join( - ( - for $value in $values - let $old-value := $old-values[./node()[1] eq $value/node()[1]] - return - if (fn:exists($old-value)) then - fn:concat("admin:group-", $setting, "(", fn:string-join((for $child in $old-value/node() return fn:concat('"', $child, '"')), ","), ")") - else - () - ), - "," + setup:get-groups-element-setting( + $admin-config, + $group-id, + $setting, + for $value in $values + return $old-values[./node()[1] eq $value/node()[fn:local-name() eq fn:tokenize($setting/@keys, " ")[1]]] ), - ")", ")" ) ) @@ -3210,15 +3261,7 @@ declare function setup:apply-groups-setting-add($admin-config as element(configu xdmp:value( fn:concat( "admin:group-add-", $setting, "($admin-config, $group-id,", - "(", - fn:string-join( - ( - for $value in $values - return fn:concat("admin:group-", $setting, "(", fn:string-join((for $child in $value/node() return fn:concat('"', $child, '"')), ","), ")") - ), - "," - ), - ")", + setup:get-groups-element-setting($admin-config, $group-id, $setting, $values), ")" ) ) @@ -3226,6 +3269,35 @@ declare function setup:apply-groups-setting-add($admin-config as element(configu $admin-config }; +declare function setup:get-groups-element-setting( + $admin-config as element(configuration), + $group-id as xs:unsignedLong, + $setting as element(setting), + $values as element()* +) as xs:string { + fn:concat( + "(", + fn:string-join( + ( + for $value in $values + return + fn:concat( + "admin:group-", + $setting, + "(", + fn:string-join( + for $key in fn:tokenize($setting/@keys, " ") + return fn:concat('"', xdmp:value(fn:concat("$value/gr:", $key)), '"'), + "," + ), + ")" + ) + ), + "," + ), + ")" + ) +}; declare function setup:validate-groups-settings($import-config as element(configuration)) as item()* { @@ -3245,11 +3317,14 @@ declare function setup:validate-groups-settings($import-config as element(config if ($setting/@value) then xdmp:value($setting/@value) else - fn:data(xdmp:value(fn:concat("$group-config/gr:", $setting, $setting-test))) + xdmp:value(fn:concat("$group-config/gr:", $setting, $setting-test)) let $min-version as xs:string? := $setting/@min-version where (fn:exists($expected)) return - if (fn:empty($min-version) or setup:at-least-version($min-version)) then + if (fn:exists($setting/@function)) then + setup:validate-group-settings($group-config, $setting) + else if (fn:empty($min-version) or setup:at-least-version($min-version)) then + let $expected := fn:data($expected) let $actual := xdmp:value(fn:concat("admin:group-get-", $setting, "($admin-config, $group-id)")) return if ($expected = $actual) then () From a3d1cb131e650c00a1c7a55524447f2242ad2b55 Mon Sep 17 00:00:00 2001 From: Richard Louapre Date: Sat, 19 Mar 2016 11:46:54 -0400 Subject: [PATCH 6/7] Fix tests --- deploy/test/data/ml7-config-changed.xml | 15 +++++++-------- deploy/test/data/ml7-config.xml | 4 ++-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/deploy/test/data/ml7-config-changed.xml b/deploy/test/data/ml7-config-changed.xml index ec91a251..30bb101b 100644 --- a/deploy/test/data/ml7-config-changed.xml +++ b/deploy/test/data/ml7-config-changed.xml @@ -490,10 +490,9 @@ @ml.odbc-server - + roxy-self-test - 2049 2 1025 @@ -520,19 +519,19 @@ info notice - hourly + daily 24 false true false - false + true ALL:!LOW:@STRENGTH2 0 false false - 2 + 60 8 31 91 @@ -540,13 +539,13 @@ s3.amazonaws.com2 http none - + false - + false - hourly + daily 24 both diff --git a/deploy/test/data/ml7-config.xml b/deploy/test/data/ml7-config.xml index cfbdeed2..91fe88d0 100644 --- a/deploy/test/data/ml7-config.xml +++ b/deploy/test/data/ml7-config.xml @@ -490,7 +490,7 @@ @ml.odbc-server - + roxy-self-test @@ -552,7 +552,7 @@ 0 true true - 1 + 60 7 30 90 From 16334b77f8f34559d55fbf15313a528b092e9ada Mon Sep 17 00:00:00 2001 From: Richard Louapre Date: Sat, 26 Mar 2016 10:56:49 -0400 Subject: [PATCH 7/7] Fix #591 --- deploy/lib/server_config.rb | 3 ++- .../test/data/ml7-properties/build.properties | 2 ++ deploy/test/test_server_config.rb | 20 +++++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index d1b87775..d3dc24a5 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -2518,9 +2518,10 @@ def ServerConfig.properties(prop_file_location = @@path) properties.merge!(ServerConfig.load_properties(env_properties_file, "ml.")) if File.exists? env_properties_file + properties = load_prop_from_args(properties) + properties = ServerConfig.substitute_properties(properties, properties, "ml.") - properties = load_prop_from_args(properties) end end diff --git a/deploy/test/data/ml7-properties/build.properties b/deploy/test/data/ml7-properties/build.properties index c356b335..51859382 100644 --- a/deploy/test/data/ml7-properties/build.properties +++ b/deploy/test/data/ml7-properties/build.properties @@ -147,3 +147,5 @@ local-server=localhost #cert-server= #prod-server= yoda-age= +dummy-port=9999 +dummy2-port=${dummy-port} diff --git a/deploy/test/test_server_config.rb b/deploy/test/test_server_config.rb index 9992c7b2..96d41854 100644 --- a/deploy/test/test_server_config.rb +++ b/deploy/test/test_server_config.rb @@ -188,6 +188,26 @@ end + # issue #591 + describe "override properties from command" do + + before do + ARGV << "--ml.dummy-port=8888" + # @logger = MiniTest::Mock.new + # ServerConfig.logger = @logger + @properties = ServerConfig.properties(File.expand_path("../data/ml7-properties/", __FILE__)) + end + + it "should load valid properites from a command" do + @properties['ml.dummy-port'].must_equal '8888' + @properties['ml.dummy2-port'].must_equal '8888' + end + + after do + ARGV.shift + ARGV.shift + end + end def with_stdin stdin = $stdin # remember $stdin