Skip to content

Commit

Permalink
DHFPROD-1693 Use CMA with ml-gradle 3.11 (#1764)
Browse files Browse the repository at this point in the history 
* DHFPROD-1693 Use CMA with ml-gradle 3.11

* DHFPROD-1693 Leave amps when SNAPSHOT and set CMA as part of pre-install
  • Loading branch information
@ryanjdew @aebadirad
ryanjdew authored and aebadirad committed Jan 23, 2019
1 parent f2b2398 commit 8aa609d
Show file tree
Hide file tree
Showing 77 changed files with 150 additions and 856 deletions.
6 changes: 3 additions & 3 deletions marklogic-data-hub/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ plugins {
id 'java'
id 'maven-publish'
id 'com.jfrog.bintray' version '1.7.2'
id 'com.marklogic.ml-gradle' version '3.8.2'
id 'com.marklogic.ml-gradle' version '3.11.0'
id 'com.moowork.node' version '1.1.1'
id 'org.springframework.boot' version '2.0.6.RELEASE'
id "io.spring.dependency-management" version "1.0.5.RELEASE"
Expand All @@ -27,8 +27,8 @@ ext.junitPlatformVersion = '1.3.1'
ext.junitJupiterVersion = '5.3.1'

dependencies {
compile 'com.marklogic:marklogic-client-api:4.1.1'
compile('com.marklogic:ml-app-deployer:3.10.1'){
compile 'com.marklogic:marklogic-client-api:4.1.2'
compile('com.marklogic:ml-app-deployer:3.11.0'){
exclude group: 'org.springframework', module: 'spring-context'
}
compile group: 'org.springframework.boot', name: 'spring-boot-starter', version: '2.0.6.RELEASE'
Expand Down
6 changes: 4 additions & 2 deletions marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/HubAppDeployer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import com.marklogic.client.DatabaseClient;
import com.marklogic.client.FailedRequestException;
import com.marklogic.client.eval.ServerEvaluationCall;
import com.marklogic.hub.deploy.util.CMASettings;
import com.marklogic.hub.deploy.util.HubDeployStatusListener;
import com.marklogic.mgmt.ManageClient;
import com.marklogic.mgmt.admin.AdminManager;
Expand All @@ -38,6 +39,7 @@ public class HubAppDeployer extends SimpleAppDeployer {
// this is for the telemetry hook to use mlUsername/mlPassword
private DatabaseClient databaseClient;

private String mlVersion = null;
// Keeps track of completion percentage
private int completed = 0;

Expand All @@ -52,7 +54,7 @@ public HubAppDeployer(ManageClient manageClient, AdminManager adminManager, HubD
@Override
public void deploy(AppConfig appConfig) {
this.completed = 0;

CMASettings.getInstance().setCmaSettings(appConfig);
onStatusChange(0, "Installing...");
super.deploy(appConfig);
onStatusChange(100, "Installation Complete");
Expand Down Expand Up @@ -81,7 +83,7 @@ protected void executeCommand(Command command, CommandContext context) {
@Override
public void undeploy(AppConfig appConfig) {
this.completed = 0;

CMASettings.getInstance().setCmaSettings(appConfig);
onStatusChange(0, "Uninstalling...");
super.undeploy(appConfig);
onStatusChange(100, "Installation Complete");
Expand Down
97 changes: 10 additions & 87 deletions marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/DeployHubAmpsCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,22 +17,13 @@

import com.marklogic.appdeployer.command.CommandContext;
import com.marklogic.appdeployer.command.security.DeployAmpsCommand;
import com.marklogic.client.DatabaseClient;
import com.marklogic.client.DatabaseClientFactory;
import com.marklogic.client.eval.ServerEvaluationCall;
import com.marklogic.hub.HubConfig;
import com.marklogic.hub.error.DataHubConfigurationException;
import com.marklogic.hub.deploy.util.CMASettings;
import com.marklogic.hub.impl.Versions;
import com.marklogic.mgmt.ManageClient;
import com.marklogic.mgmt.ManageConfig;
import org.apache.commons.io.IOUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Component;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;

@Component
public class DeployHubAmpsCommand extends DeployAmpsCommand {
Expand Down Expand Up @@ -63,96 +54,28 @@ public DeployHubAmpsCommand(HubConfig hubConfig) {
*/
@Override
public void execute(CommandContext context) {

// this is a place to optimize -- is there a way to get
// server versions without an http call?
String serverVersion = versions.getMarkLogicVersion();


logger.info("Choosing amp installation based on server version " + serverVersion);

if (serverVersion.matches("^[789]\\.0-[1234]\\.\\d+")) {
throw new DataHubConfigurationException("DHF " + hubConfig.getDHFVersion() +" cannot deploy security to server version " + serverVersion);
}
if (serverVersion.startsWith("9.0-5")) {
logger.info("Using non-SSL-compatible method for 9.0-5 servers, for demos only");
String modulesDatabaseName = hubConfig.getAppConfig().getModulesDatabaseName();
ManageConfig manageConfig = context.getManageClient().getManageConfig();
String securityUsername = manageConfig.getSecurityUsername();
String securityPassword = manageConfig.getSecurityPassword();
DatabaseClient installerClient = DatabaseClientFactory.newClient(
hubConfig.getHost(),
8000,
"Security",
new DatabaseClientFactory.DigestAuthContext(securityUsername, securityPassword)
);
//new AmpsInstaller(securityStagingClient).installAmps(modulesDatabaseName);
ServerEvaluationCall call = installerClient.newServerEval();
try (InputStream is = new ClassPathResource("installer-util/install-amps.xqy").getInputStream()) {
String ampCall = IOUtils.toString(is, "utf-8");
is.close();
ampCall = ampCall.replaceAll("data-hub-MODULES", modulesDatabaseName);
call.xquery(ampCall);
call.eval();
} catch (IOException e) {
throw new DataHubConfigurationException(e);
}
} else {
logger.info("Using CMA for servers starting with 9.0-6");
String modulesDatabaseName = hubConfig.getAppConfig().getModulesDatabaseName();
ManageClient manageClient = context.getManageClient();

try (InputStream is = new ClassPathResource("hub-internal-config/configurations/amps.json").getInputStream()) {
String payload = IOUtils.toString(is, "utf-8");
payload = payload.replaceAll("data-hub-MODULES", modulesDatabaseName);
manageClient.postJsonAsSecurityUser("/manage/v3", payload);
} catch (IOException e) {
throw new DataHubConfigurationException(e);
}
}
CMASettings.getInstance().setCmaSettings(context.getAppConfig());
super.execute(context);
}

@Override
public void undo(CommandContext context) {
// this is a place to optimize -- is there a way to get
// server versions without an http call?
String serverVersion = versions.getMarkLogicVersion();
logger.info("Choosing amp uninstall based on server version " + serverVersion);

if (serverVersion.startsWith("9.0-5")) {
logger.info("Using non-SSL-compatable method for 9.0-5 servers");
String modulesDatabaseName = hubConfig.getAppConfig().getModulesDatabaseName();
ManageConfig manageConfig = context.getManageClient().getManageConfig();
String securityUsername = manageConfig.getSecurityUsername();
String securityPassword = manageConfig.getSecurityPassword();
DatabaseClient installerClient = DatabaseClientFactory.newClient(
hubConfig.getHost(),
8000,
"Security",
new DatabaseClientFactory.DigestAuthContext(securityUsername, securityPassword)
);
//new AmpsInstaller(securityStagingClient).unInstallAmps(modulesDatabaseName);
ServerEvaluationCall call = installerClient.newServerEval();
try (InputStream is = new ClassPathResource("installer-util/uninstall-amps.xqy").getInputStream()) {
String ampCall = IOUtils.toString(is, "utf-8");
is.close();
ampCall = ampCall.replaceAll("data-hub-MODULES", modulesDatabaseName);
call.xquery(ampCall);
call.eval();
} catch (IOException e) {
throw new DataHubConfigurationException(e);
}
String hubVersion = versions.getHubVersion();
CMASettings.getInstance().setCmaSettings(context.getAppConfig());
if (hubVersion.contains("-SNAPSHOT")) {
logger.warn("Amps from uninstalled data hub framework to remain, but are disabled.");
}
else {
// only on 9.0-5 are amps uninstalled at all.
logger.warn("Amps from uninstalled data hub framework to remain, but are disabled.");
logger.info("Uninstalling amps from data hub framework.");
super.execute(context);
}
}

@Override
protected File[] getResourceDirs(CommandContext context) {
return new File[] {
};
return super.getResourceDirs(context);
}


Expand Down
29 changes: 29 additions & 0 deletions marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/util/CMASettings.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package com.marklogic.hub.deploy.util;

import com.marklogic.appdeployer.AppConfig;
import com.marklogic.hub.impl.Versions;

public class CMASettings {
private String mlVersion = null;
private static CMASettings cmaSettings = null;

public static synchronized CMASettings getInstance() {
if (cmaSettings == null) {
cmaSettings = new CMASettings();
}
return cmaSettings;
}

public void setCmaSettings(AppConfig appConfig) {
if (mlVersion == null) {
this.mlVersion = new Versions(appConfig).getMarkLogicVersion();
}
if (mlVersion.matches("^[9]\\.0-([56789]|[0-9]{2,})(\\.\\d+)?")) {
appConfig.setDeployForestsWithCma(true);
appConfig.setDeployPrivilegesWithCma(true);
if (mlVersion.matches("^[9]\\.0-([6789]|[0-9]{2,})(\\.\\d+)?")) {
appConfig.setDeployAmpsWithCma(true);
}
}
}
}
17 changes: 17 additions & 0 deletions marklogic-data-hub/src/main/java/com/marklogic/hub/impl/HubProjectImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Component;

import java.io.File;
Expand Down Expand Up @@ -219,6 +221,7 @@ public void createProject(String projectDirString) {
Path userSecurityDir = getUserSecurityDir();
Path rolesDir = hubSecurityDir.resolve("roles");
Path usersDir = hubSecurityDir.resolve("users");
Path ampsDir = hubSecurityDir.resolve("amps");
Path privilegesDir = hubSecurityDir.resolve("privileges");

Path userRolesDir = userSecurityDir.resolve("roles");
Expand All @@ -233,6 +236,20 @@ public void createProject(String projectDirString) {
userUsersDir.toFile().mkdirs();
userPrivilegesDir.toFile().mkdirs();

PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();

// Ant-style path matching
Resource[] resources = new Resource[0];
try {
resources = resolver.getResources("classpath:hub-internal-config/security/amps/*.json");
for (Resource resource : resources) {
InputStream is = resource.getInputStream();
FileUtil.copy(is, ampsDir.resolve(resource.getFilename()).toFile());
}
} catch (IOException e) {
logger.error("Failed to load amp resource", e);
}

writeResourceFile("hub-internal-config/security/roles/data-hub-role.json", rolesDir.resolve("data-hub-role.json"), true);
writeResourceFile("hub-internal-config/security/users/data-hub-user.json", usersDir.resolve("data-hub-user.json"), true);
writeResourceFile("hub-internal-config/security/roles/hub-admin-role.json", rolesDir.resolve("hub-admin-role.json"), true);
Expand Down
19 changes: 17 additions & 2 deletions marklogic-data-hub/src/main/java/com/marklogic/hub/impl/Versions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/
package com.marklogic.hub.impl;

import com.marklogic.appdeployer.AppConfig;
import com.marklogic.client.DatabaseClient;
import com.marklogic.client.eval.EvalResultIterator;
import com.marklogic.client.eval.ServerEvaluationCall;
Expand All @@ -31,6 +32,7 @@ public class Versions extends ResourceManager {
private static final String NAME = "ml:hubversion";

DatabaseClient stagingClient;
private AppConfig appConfig;

@Autowired
private HubConfig hubConfig;
Expand All @@ -42,10 +44,20 @@ public Versions() {
/**
* Needed for the Gradle tasks.
*
* @param hubConfig
* @param hubConfig HubConfig
*/
public Versions(HubConfig hubConfig) {
this.hubConfig = hubConfig;
this.appConfig = hubConfig.getAppConfig();
}

/**
* Needed for the Gradle tasks.
*
* @param appConfig AppConfig
*/
public Versions(AppConfig appConfig) {
this.appConfig = appConfig;
}

public void setupClient() {
Expand Down Expand Up @@ -74,8 +86,11 @@ public String getHubVersion() {
}

public String getMarkLogicVersion() {
if (this.appConfig == null) {
this.appConfig = hubConfig.getAppConfig();
}
// this call specifically needs to access marklogic without a known database
ServerEvaluationCall eval = hubConfig.getAppConfig().newAppServicesDatabaseClient(null).newServerEval();
ServerEvaluationCall eval = appConfig.newAppServicesDatabaseClient(null).newServerEval();
String xqy = "xdmp:version()";
EvalResultIterator result = eval.xquery(xqy).eval();
if (result.hasNext()) {
Expand Down
Loading

0 comments on commit 8aa609d

Please sign in to comment.