Issue 591: command-line properties

deploy/lib/server_config.rb

@@ -2518,9 +2518,10 @@ def ServerConfig.properties(prop_file_location = @@path)
 
     properties.merge!(ServerConfig.load_properties(env_properties_file, "ml.")) if File.exists? env_properties_file
 
+    properties = load_prop_from_args(properties)
+
     properties = ServerConfig.substitute_properties(properties, properties, "ml.")
 
-    properties = load_prop_from_args(properties)
   end
 
 end

deploy/lib/xquery/setup.xqy

@@ -110,6 +110,11 @@ declare variable $group-settings :=
     <!--TODO: setting>audit-role-restriction</setting>
     <setting>audit-uri-restriction</setting>
     <setting>audit-user-restriction</setting-->
+    <setting min-version="7.0-0" function="add" keys="namespace-uri location">module-location</setting>
+    <setting min-version="7.0-0" function="add" keys="prefix namespace-uri">namespace</setting>
+    <setting min-version="7.0-0" function="add" keys="namespace-uri schema-location">schema</setting>
+    <setting min-version="7.0-0" function="add" keys="event-id">trace-event</setting>
+    <setting min-version="7.0-0" function="add" keys="namespace-uri">using-namespace</setting>
   </settings>;
 
 declare variable $host-settings :=
@@ -1105,17 +1110,16 @@ declare function setup:create-forests-from-config(
   let $hosts := admin:group-get-host-ids(admin:get-configuration(), $group-id)
   let $host-name as xs:string? := $forest-config/as:host-name[fn:string-length(fn:string(.)) > 0]
   let $host-id := if ($host-name) then xdmp:host($host-name) else $hosts[1]
-  let $hostnr := fn:index-of($hosts, $host-id)
-  let $replica-names as xs:string* := $forest-config/as:replica-names/as:replica-name[fn:string-length(fn:string(.)) > 0]
-  let $replicas :=
-    $import-config/as:assignments/as:assignment[as:forest-name = $replica-names]
   return
     setup:create-forest(
       $forest-name,
       $data-directory,
       $host-id,
       if (fn:count($hosts) gt 1) then
-        setup:reassign-replicas($replicas, $hosts, $hostnr, $forest-name, 1, fn:false())
+        let $hostnr := fn:index-of($hosts, $host-id)
+        let $replica-names as xs:string* := $forest-config/as:replica-names/as:replica-name[fn:string-length(fn:string(.)) > 0]
+        let $replicas := $import-config/as:assignments/as:assignment[as:forest-name = $replica-names]
+        return setup:reassign-replicas($replicas, $hosts, $hostnr, $forest-name, 1, fn:false())
       else ()
     )
 
@@ -3136,6 +3140,59 @@ declare function setup:validate-group(
       setup:validation-fail(fn:concat("Missing Group: ", $group))
 };
 
+declare function setup:validate-group-settings(
+  $group-config as element(gr:group),
+  $setting as element()
+) {
+  let $group := $group-config/gr:group-name/fn:string(.)
+  let $setting-name := $setting/fn:string(.)
+  let $function-name := fn:concat("group-get-", $setting-name, "s")
+  let $keys := fn:tokenize($setting/@keys, " ")
+  let $existing :=
+    try
+    {
+      xdmp:eval(
+        fn:concat(
+          'import module namespace admin = "http://marklogic.com/xdmp/admin" at "/MarkLogic/admin.xqy";
+          declare variable $group external;
+          let $config := admin:get-configuration()
+          let $groupid := admin:group-get-id($config, $group)
+          return admin:', $function-name, '($config, $groupid)'
+        ),
+        (
+          xs:QName("group"), $group
+        )
+      )
+    }
+    catch($ex)
+    {
+      if ($ex/error:code = "XDMP-UNDFUN") then ()
+      else
+        xdmp:rethrow()
+    }
+  for $expected in xdmp:value(fn:concat("$group-config/gr:", $setting-name))
+  return (
+    if ($existing[setup:setting-equal($setting, ., $expected)]) then ()
+    else (
+      $keys ! setup:validation-fail(fn:string-join(("group setting mismatch", $setting-name, (., xdmp:value(fn:concat("$expected/gr:", .)), "=>", xdmp:value(fn:concat("$existing/gr:", .)))), " "))
+      (:$keys ! fn:string-join(("group setting mismatch", $setting-name, (., xdmp:value(fn:concat("$expected/gr:", .)), "=>", xdmp:value(fn:concat("$existing/gr:", .)))), " "):)
+    )
+  )
+};
+
+declare function setup:setting-equal(
+  $setting as element(),
+  $expected as element(),
+  $existing as element()
+) as xs:boolean {
+  let $keys := fn:tokenize($setting/@keys, " ")
+  return
+    if (fn:exists($keys)) then
+      (every $key in $keys satisfies xdmp:value(fn:concat("$expected/gr:", $key, "/fn:data(.)")) eq xdmp:value(fn:concat("$existing/gr:", $key, "/fn:data(.)")))
+    else
+      fn:deep-equal($expected, $existing)
+};
+
 declare function setup:configure-groups($import-config as element(configuration)) as item()*
 {
   let $admin-config := admin:get-configuration()
@@ -3153,13 +3210,16 @@ declare function setup:configure-groups($import-config as element(configuration)
       if ($setting/@value) then
         xdmp:value($setting/@value)
       else
-        fn:data(xdmp:value(fn:concat("$group-config/gr:", $setting, $setting-test)))
+        xdmp:value(fn:concat("$group-config/gr:", $setting, $setting-test))
     let $min-version as xs:string? := $setting/@min-version
     where (fn:exists($value))
     return
       if (fn:empty($min-version) or setup:at-least-version($min-version)) then
         xdmp:set($admin-config,
-          xdmp:value(fn:concat("admin:group-set-", $setting, "($admin-config, $group-id, $value)")))
+          if ($setting/@function eq "add") then
+            setup:apply-groups-setting-add($admin-config, $group-id, $setting, $value)
+          else
+            xdmp:value(fn:concat("admin:group-set-", $setting, "($admin-config, $group-id, fn:data($value))")))
       else
         fn:error(
           xs:QName("VERSION_NOT_SUPPORTED"),
@@ -3174,6 +3234,71 @@ declare function setup:configure-groups($import-config as element(configuration)
   )
 };
 
+declare function setup:apply-groups-setting-add(
+  $admin-config as element(configuration),
+  $group-id as xs:unsignedLong,
+  $setting as element(setting),
+  $values as element()*
+) as element(configuration) {
+  if (fn:exists($values)) then
+    let $old-values := xdmp:value(fn:concat("admin:group-get-", $setting, "s", "($admin-config, $group-id)"))
+    let $admin-config :=
+      (: First delete any values that matches the @keys first item, value must be unique :)
+      xdmp:value(
+        fn:concat(
+          "admin:group-delete-", $setting, "($admin-config, $group-id,",
+          setup:get-groups-element-setting(
+            $admin-config,
+            $group-id,
+            $setting,
+            for $value in $values
+            return $old-values[./node()[1] eq $value/node()[fn:local-name() eq fn:tokenize($setting/@keys, " ")[1]]]
+          ),
+          ")"
+        )
+      )
+    return
+      xdmp:value(
+        fn:concat(
+          "admin:group-add-", $setting, "($admin-config, $group-id,",
+          setup:get-groups-element-setting($admin-config, $group-id, $setting, $values),
+          ")"
+        )
+      )
+  else
+    $admin-config
+};
+
+declare function setup:get-groups-element-setting(
+  $admin-config as element(configuration),
+  $group-id as xs:unsignedLong,
+  $setting as element(setting),
+  $values as element()*
+) as xs:string {
+    fn:concat(
+      "(",
+      fn:string-join(
+        (
+          for $value in $values
+          return
+            fn:concat(
+              "admin:group-",
+              $setting,
+              "(",
+              fn:string-join(
+                for $key in fn:tokenize($setting/@keys, " ")
+                return fn:concat('"', xdmp:value(fn:concat("$value/gr:", $key)), '"'),
+                ","
+              ),
+              ")"
+            )
+        ),
+        ","
+      ),
+      ")"
+    )
+};
+
 declare function setup:validate-groups-settings($import-config as element(configuration)) as item()*
 {
   let $admin-config := admin:get-configuration()
@@ -3192,11 +3317,14 @@ declare function setup:validate-groups-settings($import-config as element(config
     if ($setting/@value) then
       xdmp:value($setting/@value)
     else
-      fn:data(xdmp:value(fn:concat("$group-config/gr:", $setting, $setting-test)))
+      xdmp:value(fn:concat("$group-config/gr:", $setting, $setting-test))
   let $min-version as xs:string? := $setting/@min-version
   where (fn:exists($expected))
   return
-    if (fn:empty($min-version) or setup:at-least-version($min-version)) then
+    if (fn:exists($setting/@function)) then
+      setup:validate-group-settings($group-config, $setting)
+    else if (fn:empty($min-version) or setup:at-least-version($min-version)) then
+      let $expected := fn:data($expected)
       let $actual := xdmp:value(fn:concat("admin:group-get-", $setting, "($admin-config, $group-id)"))
       return
         if ($expected = $actual) then ()

deploy/test/data/ml7-config-changed.xml

@@ -490,10 +490,9 @@
   <odbc-servers xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://marklogic.com/xdmp/group" xsi:schemaLocation="http://marklogic.com/xdmp/group group.xsd">
     @ml.odbc-server
   </odbc-servers>
-  <groups>
+  <groups xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://marklogic.com/xdmp/group" xsi:schemaLocation="http://marklogic.com/xdmp/group group.xsd">
     <group>
       <group-name>roxy-self-test</group-name>
-
       <list-cache-size>2049</list-cache-size>
       <list-cache-partitions>2</list-cache-partitions>
       <compressed-tree-cache-size>1025</compressed-tree-cache-size>
@@ -520,33 +519,33 @@
 
       <system-log-level>info</system-log-level>
       <file-log-level>notice</file-log-level>
-      <rotate-log-files>hourly</rotate-log-files>
+      <rotate-log-files>daily</rotate-log-files>
       <keep-log-files>24</keep-log-files>
 
       <failover-enable>false</failover-enable>
       <xdqp-ssl-enabled>true</xdqp-ssl-enabled>
       <xdqp-ssl-allow-sslv3>false</xdqp-ssl-allow-sslv3>
-      <xdqp-ssl-allow-tls>false</xdqp-ssl-allow-tls>
+      <xdqp-ssl-allow-tls>true</xdqp-ssl-allow-tls>
       <xdqp-ssl-ciphers>ALL:!LOW:@STRENGTH2</xdqp-ssl-ciphers>
 
       <background-io-limit>0</background-io-limit>
       <metering-enabled>false</metering-enabled>
       <performance-metering-enabled>false</performance-metering-enabled>
-      <performance-metering-period>2</performance-metering-period>
+      <performance-metering-period>60</performance-metering-period>
       <performance-metering-retain-raw>8</performance-metering-retain-raw>
       <performance-metering-retain-hourly>31</performance-metering-retain-hourly>
       <performance-metering-retain-daily>91</performance-metering-retain-daily>
 
       <s3-domain>s3.amazonaws.com2</s3-domain>
       <s3-protocol>http</s3-protocol>
       <s3-server-side-encryption>none</s3-server-side-encryption>
-      
+
       <!-- Diagnostics -->
       <trace-events-activated>false</trace-events-activated>
-      
+
       <!-- Auditing -->
       <audit-enabled>false</audit-enabled>
-      <rotate-audit-files>hourly</rotate-audit-files>
+      <rotate-audit-files>daily</rotate-audit-files>
       <keep-audit-files>24</keep-audit-files>
       <audit-outcome-restriction>both</audit-outcome-restriction>
     </group>

deploy/test/data/ml7-config.xml

@@ -490,9 +490,29 @@
   <odbc-servers xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://marklogic.com/xdmp/group" xsi:schemaLocation="http://marklogic.com/xdmp/group group.xsd">
     @ml.odbc-server
   </odbc-servers>
-  <groups>
+  <groups xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://marklogic.com/xdmp/group" xsi:schemaLocation="http://marklogic.com/xdmp/group group.xsd">
     <group>
       <group-name>roxy-self-test</group-name>
+      <!-- Begin - issue-585 -->
+      <module-location>
+        <namespace-uri>http://roxy/modules/dummy</namespace-uri>
+        <location>/roxy/dummy.xqy</location>
+      </module-location>
+      <namespace>
+        <prefix>prefix1</prefix>
+        <namespace-uri>http://roxy/namespaces/prefix1</namespace-uri>
+      </namespace>
+      <schema>
+        <namespace-uri>http://roxy/schemas/dummy</namespace-uri>
+        <schema-location>/roxy/dummy.xsd</schema-location>
+      </schema>
+      <trace-event>
+        <event-id>roxy-dummy-event</event-id>
+      </trace-event>
+      <using-namespace>
+        <namespace-uri>http://roxy/namespaces/prefix2</namespace-uri>
+      </using-namespace>
+      <!-- End - issue-585 -->
 
       <list-cache-size>2048</list-cache-size>
       <list-cache-partitions>1</list-cache-partitions>
@@ -532,18 +552,18 @@
       <background-io-limit>0</background-io-limit>
       <metering-enabled>true</metering-enabled>
       <performance-metering-enabled>true</performance-metering-enabled>
-      <performance-metering-period>1</performance-metering-period>
+      <performance-metering-period>60</performance-metering-period>
       <performance-metering-retain-raw>7</performance-metering-retain-raw>
       <performance-metering-retain-hourly>30</performance-metering-retain-hourly>
       <performance-metering-retain-daily>90</performance-metering-retain-daily>
 
       <s3-domain>s3.amazonaws.com</s3-domain>
       <s3-protocol>http</s3-protocol>
       <s3-server-side-encryption>none</s3-server-side-encryption>
-      
+
       <!-- Diagnostics -->
       <trace-events-activated>true</trace-events-activated>
-      
+
       <!-- Auditing -->
       <audit-enabled>true</audit-enabled>
       <rotate-audit-files>daily</rotate-audit-files>

deploy/test/data/ml7-properties/build.properties

@@ -147,3 +147,5 @@ local-server=localhost
 #cert-server=
 #prod-server=
 yoda-age=
+dummy-port=9999
+dummy2-port=${dummy-port}

deploy/test/test_server_config.rb

@@ -188,6 +188,26 @@
 
   end
 
+  # issue #591
+  describe "override properties from command" do
+
+    before do
+      ARGV << "--ml.dummy-port=8888"
+      # @logger = MiniTest::Mock.new
+      # ServerConfig.logger = @logger
+      @properties = ServerConfig.properties(File.expand_path("../data/ml7-properties/", __FILE__))
+    end
+
+    it "should load valid properites from a command" do
+      @properties['ml.dummy-port'].must_equal '8888'
+      @properties['ml.dummy2-port'].must_equal '8888'
+    end
+
+    after do
+      ARGV.shift
+      ARGV.shift
+    end
+  end
 
   def with_stdin
     stdin = $stdin             # remember $stdin