DHFPROD-1754 + other issues (#1792)

* Remove hardcoded "data-hub-MODULES" and pretty printing json * Remove 'DeployHubAmpsCommand' and its gradle plugin task * Remove references to 'DeployHubAmpsCommand' and not uninstall amps * Change 'CMASettings'to handle nightly and remove 'CMASettings' from HubAppDeployer * Replace 'mlDeploySecurity' with 'hubDeploySecurity' as amps are scaffolded
Marklogic-retired · Jan 26, 2019 · 04d62cf · 04d62cf
1 parent 1454d2e
commit 04d62cf
Show file tree

Hide file tree

Showing 77 changed files with 519 additions and 238 deletions.
diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/HubAppDeployer.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/HubAppDeployer.java
@@ -54,7 +54,6 @@ public HubAppDeployer(ManageClient manageClient, AdminManager adminManager, HubD
     @Override
     public void deploy(AppConfig appConfig) {
         this.completed = 0;
-        CMASettings.getInstance().setCmaSettings(appConfig);
         onStatusChange(0, "Installing...");
         super.deploy(appConfig);
         onStatusChange(100, "Installation Complete");
@@ -83,7 +82,6 @@ protected void executeCommand(Command command, CommandContext context) {
     @Override
     public void undeploy(AppConfig appConfig) {
         this.completed = 0;
-        CMASettings.getInstance().setCmaSettings(appConfig);
         onStatusChange(0, "Uninstalling...");
         super.undeploy(appConfig);
         onStatusChange(100, "Installation Complete");

diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/DeployHubAmpsCommand.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/DeployHubAmpsCommand.java
diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/util/CMASettings.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/util/CMASettings.java
@@ -2,6 +2,12 @@
 
 import com.marklogic.appdeployer.AppConfig;
 import com.marklogic.hub.impl.Versions;
+import org.apache.commons.lang3.StringUtils;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.GregorianCalendar;
 
 public class CMASettings {
     private String mlVersion = null;
@@ -18,10 +24,37 @@ public void setCmaSettings(AppConfig appConfig) {
         if (mlVersion == null) {
             this.mlVersion = new Versions(appConfig).getMarkLogicVersion();
         }
-        if (mlVersion.matches("^[9]\\.0-([56789]|[0-9]{2,})(\\.\\d+)?")) {
-            appConfig.setDeployForestsWithCma(true);
-            appConfig.setDeployPrivilegesWithCma(true);
-            if (mlVersion.matches("^[9]\\.0-([6789]|[0-9]{2,})(\\.\\d+)?")) {
+
+        int major = Integer.parseInt(mlVersion.replaceAll("([^.]+)\\..*", "$1"));
+
+        boolean isNightly = mlVersion.matches("[^-]+-(\\d{4})(\\d{2})(\\d{2})");
+        if (major == 9) {
+            String alteredString = mlVersion.replaceAll("[^\\d]+", "");
+            if (alteredString.length() < 4) {
+                alteredString = StringUtils.rightPad(alteredString, 4, "0");
+            }
+            int ver = Integer.parseInt(alteredString.substring(0, 4));
+            if (!isNightly && ver >= 9050 ) {
+                appConfig.setDeployForestsWithCma(true);
+                appConfig.setDeployPrivilegesWithCma(true);
+            }
+            if (!isNightly && ver >= 9060 ) {
+                appConfig.setDeployAmpsWithCma(true);
+            }
+        }
+        //Setting all true for nightly build after 07/01/2018
+        if (isNightly) {
+            String dateString = mlVersion.replaceAll("[^-]+-(\\d{4})(\\d{2})(\\d{2})", "$1-$2-$3");
+            Date minDate = new GregorianCalendar(2018, 6, 1).getTime();
+            Date date = null;
+            try {
+                date = new SimpleDateFormat("y-M-d").parse(dateString);
+            } catch (ParseException e) {
+                throw  new RuntimeException("Unable to set CMA settings for nightly build");
+            }
+            if (date.after(minDate)) {
+                appConfig.setDeployForestsWithCma(true);
+                appConfig.setDeployPrivilegesWithCma(true);
                 appConfig.setDeployAmpsWithCma(true);
             }
         }

diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/impl/DataHubImpl.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/impl/DataHubImpl.java
@@ -42,6 +42,7 @@
 import com.marklogic.hub.*;
 import com.marklogic.hub.deploy.HubAppDeployer;
 import com.marklogic.hub.deploy.commands.*;
+import com.marklogic.hub.deploy.util.CMASettings;
 import com.marklogic.hub.deploy.util.HubDeployStatusListener;
 import com.marklogic.hub.error.*;
 import com.marklogic.mgmt.ManageClient;
@@ -92,9 +93,6 @@ public class DataHubImpl implements DataHub {
 
     @Autowired
     private LoadUserArtifactsCommand loadUserArtifactsCommand;
-
-    @Autowired
-    private DeployHubAmpsCommand deployHubAmpsCommand;
 
     @Autowired
     private Versions versions;
@@ -494,10 +492,12 @@ public void install(HubDeployStatusListener listener) {
                 throw new DataHubConfigurationException(e);
             }
         }
+        AppConfig appConfig = hubConfig.getAppConfig();
+        CMASettings.getInstance().setCmaSettings(appConfig);
 
         HubAppDeployer finalDeployer = new HubAppDeployer(getManageClient(), getAdminManager(), listener, hubConfig.newStagingClient());
         finalDeployer.setCommands(buildListOfCommands());
-        finalDeployer.deploy(hubConfig.getAppConfig());
+        finalDeployer.deploy(appConfig);
     }
 
     /**
@@ -552,10 +552,16 @@ public void uninstall() {
     @Override
     public void uninstall(HubDeployStatusListener listener) {
         logger.warn("Uninstalling the Data Hub and Final Databases/Servers from MarkLogic");
+        List<Command> commandMap = buildListOfCommands();
+        //Remove this line if CMA supports uninstalling amps
+        commandMap.removeIf(command -> command instanceof DeployAmpsCommand);
+
+        AppConfig appConfig = hubConfig.getAppConfig();
+        CMASettings.getInstance().setCmaSettings(appConfig);
 
         HubAppDeployer finalDeployer = new HubAppDeployer(getManageClient(), getAdminManager(), listener, hubConfig.newStagingClient());
-        finalDeployer.setCommands(buildListOfCommands());
-        finalDeployer.undeploy(hubConfig.getAppConfig());
+        finalDeployer.setCommands(commandMap);
+        finalDeployer.undeploy(appConfig);
     }
 
     private void runInDatabase(String query, String databaseName) {
@@ -574,14 +580,6 @@ private void runInDatabase(String query, String databaseName) {
     public Map<String, List<Command>> buildCommandMap() {
         Map<String, List<Command>> commandMap = new CommandMapBuilder().buildCommandMap();
 
-        /**
-         * This kept separate from mlSecurityCommands because hub amps are stored in the DHF jar, while the commands
-         * in the mlSecurityCommands list deploy resources defined by users.
-         */
-        List<Command> hubAmpsCommands = new ArrayList<>();
-        hubAmpsCommands.add(deployHubAmpsCommand);
-        commandMap.put("mlHubAmpsCommand", hubAmpsCommands);
-
         updateDatabaseCommandList(commandMap);
 
         updateServerCommandList(commandMap);

diff --git a/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-1.json b/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-1.json
@@ -1 +1,7 @@
-{"namespace":"", "local-name":"addResponseHeader", "document-uri":"/data-hub/4/rest-api/lib/endpoint-util.sjs", "modules-database":"data-hub-MODULES", "role":["rest-reader-internal"]}
+{
+  "namespace" : "",
+  "local-name" : "addResponseHeader",
+  "document-uri" : "/data-hub/4/rest-api/lib/endpoint-util.sjs",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "rest-reader-internal" ]
+}
diff --git a/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-10.json b/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-10.json
@@ -1 +1,7 @@
-{"namespace":"http://marklogic.com/rest-api/lib/endpoint-util", "local-name":"get-mimetypes", "document-uri":"/data-hub/4/rest-api/lib/endpoint-util.xqy", "modules-database":"data-hub-MODULES", "role":["rest-reader-internal"]}
+{
+  "namespace" : "http://marklogic.com/rest-api/lib/endpoint-util",
+  "local-name" : "get-mimetypes",
+  "document-uri" : "/data-hub/4/rest-api/lib/endpoint-util.xqy",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "rest-reader-internal" ]
+}
diff --git a/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-11.json b/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-11.json
@@ -1 +1,7 @@
-{"namespace":"http://marklogic.com/rest-api/lib/endpoint-util", "local-name":"get-server-field", "document-uri":"/data-hub/4/rest-api/lib/endpoint-util.xqy", "modules-database":"data-hub-MODULES", "role":["rest-reader-internal"]}
+{
+  "namespace" : "http://marklogic.com/rest-api/lib/endpoint-util",
+  "local-name" : "get-server-field",
+  "document-uri" : "/data-hub/4/rest-api/lib/endpoint-util.xqy",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "rest-reader-internal" ]
+}
diff --git a/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-12.json b/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-12.json
@@ -1 +1,7 @@
-{"namespace":"http://marklogic.com/rest-api/lib/endpoint-util", "local-name":"invoke-module", "document-uri":"/data-hub/4/rest-api/lib/endpoint-util.xqy", "modules-database":"data-hub-MODULES", "role":["rest-reader-internal"]}
+{
+  "namespace" : "http://marklogic.com/rest-api/lib/endpoint-util",
+  "local-name" : "invoke-module",
+  "document-uri" : "/data-hub/4/rest-api/lib/endpoint-util.xqy",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "rest-reader-internal" ]
+}
diff --git a/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-13.json b/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-13.json
@@ -1 +1,7 @@
-{"namespace":"http://marklogic.com/rest-api/lib/endpoint-util", "local-name":"set-server-field", "document-uri":"/data-hub/4/rest-api/lib/endpoint-util.xqy", "modules-database":"data-hub-MODULES", "role":["rest-reader-internal"]}
+{
+  "namespace" : "http://marklogic.com/rest-api/lib/endpoint-util",
+  "local-name" : "set-server-field",
+  "document-uri" : "/data-hub/4/rest-api/lib/endpoint-util.xqy",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "rest-reader-internal" ]
+}
diff --git a/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-14.json b/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-14.json
@@ -1 +1,7 @@
-{"namespace":"http://marklogic.com/rest-api/lib/endpoint-util", "local-name":"xslt-invoke", "document-uri":"/data-hub/4/rest-api/lib/endpoint-util.xqy", "modules-database":"data-hub-MODULES", "role":["rest-reader-internal"]}
+{
+  "namespace" : "http://marklogic.com/rest-api/lib/endpoint-util",
+  "local-name" : "xslt-invoke",
+  "document-uri" : "/data-hub/4/rest-api/lib/endpoint-util.xqy",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "rest-reader-internal" ]
+}
diff --git a/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-15.json b/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-15.json
@@ -1 +1,7 @@
-{"namespace":"http://marklogic.com/rest-api/lib/endpoint-util", "local-name":"lookup-role-ids", "document-uri":"/data-hub/4/rest-api/lib/endpoint-util.xqy", "modules-database":"data-hub-MODULES", "role":["rest-reader-internal"]}
+{
+  "namespace" : "http://marklogic.com/rest-api/lib/endpoint-util",
+  "local-name" : "lookup-role-ids",
+  "document-uri" : "/data-hub/4/rest-api/lib/endpoint-util.xqy",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "rest-reader-internal" ]
+}
diff --git a/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-16.json b/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-16.json
@@ -1 +1,7 @@
-{"namespace":"http://marklogic.com/rest-api/models/document-model-common", "local-name":"read-collections", "document-uri":"/data-hub/4/rest-api/models/document-model-common.xqy", "modules-database":"data-hub-MODULES", "role":["rest-reader-internal"]}
+{
+  "namespace" : "http://marklogic.com/rest-api/models/document-model-common",
+  "local-name" : "read-collections",
+  "document-uri" : "/data-hub/4/rest-api/models/document-model-common.xqy",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "rest-reader-internal" ]
+}
diff --git a/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-17.json b/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-17.json
@@ -1 +1,7 @@
-{"namespace":"http://marklogic.com/rest-api/models/document-model-common", "local-name":"read-permissions", "document-uri":"/data-hub/4/rest-api/models/document-model-common.xqy", "modules-database":"data-hub-MODULES", "role":["rest-reader-internal"]}
+{
+  "namespace" : "http://marklogic.com/rest-api/models/document-model-common",
+  "local-name" : "read-permissions",
+  "document-uri" : "/data-hub/4/rest-api/models/document-model-common.xqy",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "rest-reader-internal" ]
+}
diff --git a/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-18.json b/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-18.json
@@ -1 +1,7 @@
-{"namespace":"http://marklogic.com/rest-api/models/document-model-common", "local-name":"read-properties", "document-uri":"/data-hub/4/rest-api/models/document-model-common.xqy", "modules-database":"data-hub-MODULES", "role":["rest-reader-internal"]}
+{
+  "namespace" : "http://marklogic.com/rest-api/models/document-model-common",
+  "local-name" : "read-properties",
+  "document-uri" : "/data-hub/4/rest-api/models/document-model-common.xqy",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "rest-reader-internal" ]
+}
diff --git a/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-19.json b/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-19.json
@@ -1 +1,7 @@
-{"namespace":"http://marklogic.com/rest-api/models/document-model-common", "local-name":"read-quality", "document-uri":"/data-hub/4/rest-api/models/document-model-common.xqy", "modules-database":"data-hub-MODULES", "role":["rest-reader-internal"]}
+{
+  "namespace" : "http://marklogic.com/rest-api/models/document-model-common",
+  "local-name" : "read-quality",
+  "document-uri" : "/data-hub/4/rest-api/models/document-model-common.xqy",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "rest-reader-internal" ]
+}
diff --git a/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-2.json b/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-2.json
@@ -1 +1,7 @@
-{"namespace":"http://marklogic.com/rest-api/lib/db-util", "local-name":"access-config", "document-uri":"/data-hub/4/rest-api/lib/db-util.xqy", "modules-database":"data-hub-MODULES", "role":["rest-reader-internal"]}
+{
+  "namespace" : "http://marklogic.com/rest-api/lib/db-util",
+  "local-name" : "access-config",
+  "document-uri" : "/data-hub/4/rest-api/lib/db-util.xqy",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "rest-reader-internal" ]
+}
diff --git a/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-20.json b/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-20.json
@@ -1 +1,7 @@
-{"namespace":"http://marklogic.com/rest-api/models/document-model-common", "local-name":"lookup-role-names", "document-uri":"/data-hub/4/rest-api/models/document-model-common.xqy", "modules-database":"data-hub-MODULES", "role":["rest-reader-internal"]}
+{
+  "namespace" : "http://marklogic.com/rest-api/models/document-model-common",
+  "local-name" : "lookup-role-names",
+  "document-uri" : "/data-hub/4/rest-api/models/document-model-common.xqy",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "rest-reader-internal" ]
+}
diff --git a/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-21.json b/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-21.json
@@ -1 +1,7 @@
-{"namespace":"http://marklogic.com/rest-api/models/document-model-query", "local-name":"get", "document-uri":"/data-hub/4/rest-api/models/document-model-query.xqy", "modules-database":"data-hub-MODULES", "role":["rest-reader-internal"]}
+{
+  "namespace" : "http://marklogic.com/rest-api/models/document-model-query",
+  "local-name" : "get",
+  "document-uri" : "/data-hub/4/rest-api/models/document-model-query.xqy",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "rest-reader-internal" ]
+}
diff --git a/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-22.json b/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-22.json
@@ -1 +1,7 @@
-{"namespace":"http://marklogic.com/rest-api/models/document-model-query-get", "local-name":"get", "document-uri":"/data-hub/4/rest-api/models/document-model-query-get.xqy", "modules-database":"data-hub-MODULES", "role":["rest-reader-internal"]}
+{
+  "namespace" : "http://marklogic.com/rest-api/models/document-model-query-get",
+  "local-name" : "get",
+  "document-uri" : "/data-hub/4/rest-api/models/document-model-query-get.xqy",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "rest-reader-internal" ]
+}
diff --git a/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-23.json b/marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/dhf-amp-23.json
@@ -1 +1,7 @@
-{"namespace":"http://marklogic.com/rest-api/models/document-model-query-head", "local-name":"head", "document-uri":"/data-hub/4/rest-api/models/document-model-query-head.xqy", "modules-database":"data-hub-MODULES", "role":["rest-reader-internal"]}
+{
+  "namespace" : "http://marklogic.com/rest-api/models/document-model-query-head",
+  "local-name" : "head",
+  "document-uri" : "/data-hub/4/rest-api/models/document-model-query-head.xqy",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "rest-reader-internal" ]
+}