DHFPROD-1693 Use CMA with ml-gradle 3.11

marklogic-data-hub/build.gradle

@@ -3,7 +3,7 @@ plugins {
     id 'java'
     id 'maven-publish'
     id 'com.jfrog.bintray' version '1.7.2'
-    id 'com.marklogic.ml-gradle' version '3.8.2'
+    id 'com.marklogic.ml-gradle' version '3.11.0'
     id 'com.moowork.node' version '1.1.1'
     id 'org.springframework.boot' version '2.0.6.RELEASE'
     id "io.spring.dependency-management" version "1.0.5.RELEASE"
@@ -27,8 +27,8 @@ ext.junitPlatformVersion = '1.3.1'
 ext.junitJupiterVersion  = '5.3.1'
 
 dependencies {
-    compile 'com.marklogic:marklogic-client-api:4.1.1'
-    compile('com.marklogic:ml-app-deployer:3.10.1'){
+    compile 'com.marklogic:marklogic-client-api:4.1.2'
+    compile('com.marklogic:ml-app-deployer:3.11.0'){
         exclude group: 'org.springframework', module: 'spring-context'
     }
     compile group: 'org.springframework.boot', name: 'spring-boot-starter', version: '2.0.6.RELEASE'

marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/HubAppDeployer.java

@@ -23,6 +23,7 @@
 import com.marklogic.client.DatabaseClient;
 import com.marklogic.client.FailedRequestException;
 import com.marklogic.client.eval.ServerEvaluationCall;
+import com.marklogic.hub.deploy.util.CMASettings;
 import com.marklogic.hub.deploy.util.HubDeployStatusListener;
 import com.marklogic.mgmt.ManageClient;
 import com.marklogic.mgmt.admin.AdminManager;
@@ -38,6 +39,7 @@ public class HubAppDeployer extends SimpleAppDeployer {
     // this is for the telemetry hook to use mlUsername/mlPassword
     private DatabaseClient databaseClient;
 
+    private String mlVersion = null;
     // Keeps track of completion percentage
     private int completed = 0;
 
@@ -52,7 +54,7 @@ public HubAppDeployer(ManageClient manageClient, AdminManager adminManager, HubD
     @Override
     public void deploy(AppConfig appConfig) {
         this.completed = 0;
-
+        CMASettings.getInstance().setCmaSettings(appConfig);
         onStatusChange(0, "Installing...");
         super.deploy(appConfig);
         onStatusChange(100, "Installation Complete");
@@ -81,7 +83,7 @@ protected void executeCommand(Command command, CommandContext context) {
     @Override
     public void undeploy(AppConfig appConfig) {
         this.completed = 0;
-
+        CMASettings.getInstance().setCmaSettings(appConfig);
         onStatusChange(0, "Uninstalling...");
         super.undeploy(appConfig);
         onStatusChange(100, "Installation Complete");

marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/DeployHubAmpsCommand.java

@@ -17,22 +17,13 @@
 
 import com.marklogic.appdeployer.command.CommandContext;
 import com.marklogic.appdeployer.command.security.DeployAmpsCommand;
-import com.marklogic.client.DatabaseClient;
-import com.marklogic.client.DatabaseClientFactory;
-import com.marklogic.client.eval.ServerEvaluationCall;
 import com.marklogic.hub.HubConfig;
-import com.marklogic.hub.error.DataHubConfigurationException;
+import com.marklogic.hub.deploy.util.CMASettings;
 import com.marklogic.hub.impl.Versions;
-import com.marklogic.mgmt.ManageClient;
-import com.marklogic.mgmt.ManageConfig;
-import org.apache.commons.io.IOUtils;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.io.ClassPathResource;
 import org.springframework.stereotype.Component;
 
 import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
 
 @Component
 public class DeployHubAmpsCommand extends DeployAmpsCommand {
@@ -63,96 +54,28 @@ public DeployHubAmpsCommand(HubConfig hubConfig) {
      */
     @Override
     public void execute(CommandContext context) {
-
-        // this is a place to optimize -- is there a way to get
-        // server versions without an http call?
-        String serverVersion = versions.getMarkLogicVersion();
-
-
-        logger.info("Choosing amp installation based on server version " + serverVersion);
-
-        if (serverVersion.matches("^[789]\\.0-[1234]\\.\\d+")) {
-            throw new DataHubConfigurationException("DHF " + hubConfig.getDHFVersion() +" cannot deploy security to server version " + serverVersion);
-        }
-        if (serverVersion.startsWith("9.0-5")) {
-            logger.info("Using non-SSL-compatible method for 9.0-5 servers, for demos only");
-            String modulesDatabaseName = hubConfig.getAppConfig().getModulesDatabaseName();
-            ManageConfig manageConfig = context.getManageClient().getManageConfig();
-            String securityUsername = manageConfig.getSecurityUsername();
-            String securityPassword = manageConfig.getSecurityPassword();
-            DatabaseClient installerClient = DatabaseClientFactory.newClient(
-                hubConfig.getHost(),
-                8000,
-                "Security",
-                new DatabaseClientFactory.DigestAuthContext(securityUsername, securityPassword)
-            );
-            //new AmpsInstaller(securityStagingClient).installAmps(modulesDatabaseName);
-            ServerEvaluationCall call = installerClient.newServerEval();
-            try (InputStream is = new ClassPathResource("installer-util/install-amps.xqy").getInputStream()) {
-                String ampCall = IOUtils.toString(is, "utf-8");
-                is.close();
-                ampCall = ampCall.replaceAll("data-hub-MODULES", modulesDatabaseName);
-                call.xquery(ampCall);
-                call.eval();
-            } catch (IOException e) {
-                throw new DataHubConfigurationException(e);
-            }
-        } else {
-            logger.info("Using CMA for servers starting with 9.0-6");
-            String modulesDatabaseName = hubConfig.getAppConfig().getModulesDatabaseName();
-            ManageClient manageClient = context.getManageClient();
-
-            try (InputStream is = new ClassPathResource("hub-internal-config/configurations/amps.json").getInputStream()) {
-                String payload = IOUtils.toString(is, "utf-8");
-                payload = payload.replaceAll("data-hub-MODULES", modulesDatabaseName);
-                manageClient.postJsonAsSecurityUser("/manage/v3", payload);
-            } catch (IOException e) {
-                throw new DataHubConfigurationException(e);
-            }
-        }
+        CMASettings.getInstance().setCmaSettings(context.getAppConfig());
+        super.execute(context);
     }
 
     @Override
     public void undo(CommandContext context) {
         // this is a place to optimize -- is there a way to get
         // server versions without an http call?
-        String serverVersion = versions.getMarkLogicVersion();
-        logger.info("Choosing amp uninstall based on server version " + serverVersion);
-
-        if (serverVersion.startsWith("9.0-5")) {
-            logger.info("Using non-SSL-compatable method for 9.0-5 servers");
-            String modulesDatabaseName = hubConfig.getAppConfig().getModulesDatabaseName();
-            ManageConfig manageConfig = context.getManageClient().getManageConfig();
-            String securityUsername = manageConfig.getSecurityUsername();
-            String securityPassword = manageConfig.getSecurityPassword();
-            DatabaseClient installerClient = DatabaseClientFactory.newClient(
-                hubConfig.getHost(),
-                8000,
-                "Security",
-                new DatabaseClientFactory.DigestAuthContext(securityUsername, securityPassword)
-            );
-            //new AmpsInstaller(securityStagingClient).unInstallAmps(modulesDatabaseName);
-            ServerEvaluationCall call = installerClient.newServerEval();
-            try (InputStream is = new ClassPathResource("installer-util/uninstall-amps.xqy").getInputStream()) {
-                String ampCall = IOUtils.toString(is, "utf-8");
-                is.close();
-                ampCall = ampCall.replaceAll("data-hub-MODULES", modulesDatabaseName);
-                call.xquery(ampCall);
-                call.eval();
-            } catch (IOException e) {
-                throw new DataHubConfigurationException(e);
-            }
+        String hubVersion = versions.getHubVersion();
+        CMASettings.getInstance().setCmaSettings(context.getAppConfig());
+        if (hubVersion.contains("-SNAPSHOT")) {
+            logger.warn("Amps from uninstalled data hub framework to remain, but are disabled.");
         }
         else {
-            // only on 9.0-5 are amps uninstalled at all.
-            logger.warn("Amps from uninstalled data hub framework to remain, but are disabled.");
+            logger.info("Uninstalling amps from data hub framework.");
+            super.execute(context);
         }
     }
 
     @Override
     protected File[] getResourceDirs(CommandContext context) {
-        return new File[] {
-        };
+        return super.getResourceDirs(context);
     }
 
 

marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/util/CMASettings.java

@@ -0,0 +1,29 @@
+package com.marklogic.hub.deploy.util;
+
+import com.marklogic.appdeployer.AppConfig;
+import com.marklogic.hub.impl.Versions;
+
+public class CMASettings {
+    private String mlVersion = null;
+    private static CMASettings cmaSettings = null;
+
+    public static synchronized CMASettings getInstance() {
+        if (cmaSettings == null) {
+            cmaSettings = new CMASettings();
+        }
+        return cmaSettings;
+    }
+
+    public void setCmaSettings(AppConfig appConfig) {
+        if (mlVersion == null) {
+            this.mlVersion = new Versions(appConfig).getMarkLogicVersion();
+        }
+        if (mlVersion.matches("^[9]\\.0-([56789]|[0-9]{2,})(\\.\\d+)?")) {
+            appConfig.setDeployForestsWithCma(true);
+            appConfig.setDeployPrivilegesWithCma(true);
+            if (mlVersion.matches("^[9]\\.0-([6789]|[0-9]{2,})(\\.\\d+)?")) {
+                appConfig.setDeployAmpsWithCma(true);
+            }
+        }
+    }
+}

marklogic-data-hub/src/main/java/com/marklogic/hub/impl/HubProjectImpl.java

@@ -27,6 +27,8 @@
 import org.apache.commons.io.IOUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
+import org.springframework.core.io.Resource;
 import org.springframework.stereotype.Component;
 
 import java.io.File;
@@ -219,6 +221,7 @@ public void createProject(String projectDirString) {
         Path userSecurityDir = getUserSecurityDir();
         Path rolesDir = hubSecurityDir.resolve("roles");
         Path usersDir = hubSecurityDir.resolve("users");
+        Path ampsDir = hubSecurityDir.resolve("amps");
         Path privilegesDir = hubSecurityDir.resolve("privileges");
 
         Path userRolesDir = userSecurityDir.resolve("roles");
@@ -233,6 +236,20 @@ public void createProject(String projectDirString) {
         userUsersDir.toFile().mkdirs();
         userPrivilegesDir.toFile().mkdirs();
 
+        PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
+
+        // Ant-style path matching
+        Resource[] resources = new Resource[0];
+        try {
+            resources = resolver.getResources("classpath:hub-internal-config/security/amps/*.json");
+            for (Resource resource : resources) {
+                InputStream is = resource.getInputStream();
+                FileUtil.copy(is, ampsDir.resolve(resource.getFilename()).toFile());
+            }
+        } catch (IOException e) {
+            logger.error("Failed to load amp resource", e);
+        }
+
         writeResourceFile("hub-internal-config/security/roles/data-hub-role.json", rolesDir.resolve("data-hub-role.json"), true);
         writeResourceFile("hub-internal-config/security/users/data-hub-user.json", usersDir.resolve("data-hub-user.json"), true);
         writeResourceFile("hub-internal-config/security/roles/hub-admin-role.json", rolesDir.resolve("hub-admin-role.json"), true);

marklogic-data-hub/src/main/java/com/marklogic/hub/impl/Versions.java

@@ -15,6 +15,7 @@
  */
 package com.marklogic.hub.impl;
 
+import com.marklogic.appdeployer.AppConfig;
 import com.marklogic.client.DatabaseClient;
 import com.marklogic.client.eval.EvalResultIterator;
 import com.marklogic.client.eval.ServerEvaluationCall;
@@ -31,6 +32,7 @@ public class Versions extends ResourceManager {
     private static final String NAME = "ml:hubversion";
 
     DatabaseClient stagingClient;
+    private AppConfig appConfig;
 
     @Autowired
     private HubConfig hubConfig;
@@ -42,10 +44,20 @@ public Versions() {
     /**
      * Needed for the Gradle tasks.
      *
-     * @param hubConfig
+     * @param hubConfig HubConfig
      */
     public Versions(HubConfig hubConfig) {
         this.hubConfig = hubConfig;
+        this.appConfig = hubConfig.getAppConfig();
+    }
+
+    /**
+     * Needed for the Gradle tasks.
+     *
+     * @param appConfig AppConfig
+     */
+    public Versions(AppConfig appConfig) {
+        this.appConfig = appConfig;
     }
 
     public void setupClient() {
@@ -74,8 +86,11 @@ public String getHubVersion() {
     }
 
     public String getMarkLogicVersion() {
+        if (this.appConfig == null) {
+            this.appConfig = hubConfig.getAppConfig();
+        }
         // this call specifically needs to access marklogic without a known database
-        ServerEvaluationCall eval = hubConfig.getAppConfig().newAppServicesDatabaseClient(null).newServerEval();
+        ServerEvaluationCall eval = appConfig.newAppServicesDatabaseClient(null).newServerEval();
         String xqy = "xdmp:version()";
         EvalResultIterator result = eval.xquery(xqy).eval();
         if (result.hasNext()) {