164 - Load Data Improvements

examples/gradle-advanced/build.gradle

@@ -10,7 +10,7 @@ plugins {
     id 'eclipse'
     id 'idea'
     id 'net.saliman.properties' version '1.4.5'
-    id 'com.marklogic.ml-data-hub-plugin' version '1.0.0-beta.1'
+    id 'com.marklogic.ml-data-hub-plugin' version '1.0.0-beta.2'
     id 'com.marklogic.ml-gradle' version '2.1.0'
 }
 

examples/hr-hub/plugins/entities/Employee/conformance/conform-acme-tech/headers/headers.sjs

@@ -5,7 +5,7 @@
  * @param content  - the output of your content plugin
  * @param options  - an object containing options. Options are sent from Java
  *
- * @return - an array of header objects
+ * @return - an object of headers
  */
 function createHeaders(id, content, options) {
   var latest = xs.date('1900-01-01');
@@ -20,17 +20,11 @@ function createHeaders(id, content, options) {
     }
   }
 
-  return [
-    {
-      employeeId: content.id
-    },
-    {
-      hireDate: xs.date(xdmp.parseDateTime('[M01]/[D01]/[Y0001]', content.hireDate))
-    },
-    {
-      salary: xs.int(salary)
-    }
-  ];
+  return {
+    employeeId: content.id,
+    hireDate: xs.date(xdmp.parseDateTime('[M01]/[D01]/[Y0001]', content.hireDate)),
+    salary: xs.int(salary)
+  };
 }
 
 module.exports = {

examples/hr-hub/plugins/entities/Employee/conformance/conform-global-corp/headers/headers.sjs

@@ -5,20 +5,14 @@
  * @param content  - the output of your content plugin
  * @param options  - an object containing options. Options are sent from Java
  *
- * @return - an array of header objects
+ * @return - an object of headers
  */
 function createHeaders(id, content, options) {
-  return [
-    {
-      employeeId: content.emp_id
-    },
-    {
-      hireDate: xs.date(xdmp.parseDateTime('[M01]/[D01]/[Y0001]', content.hire_date))
-    },
-    {
-      salary: xs.int(content.base_salary) + xs.int(content.bonus)
-    }
-  ];
+  return {
+    employeeId: content.emp_id,
+    hireDate: xs.date(xdmp.parseDateTime('[M01]/[D01]/[Y0001]', content.hire_date)),
+    salary: xs.int(content.base_salary) + xs.int(content.bonus)
+  };
 }
 
 module.exports = {

examples/hr-hub/plugins/entities/Employee/input/load-acme-tech/headers/headers.sjs

@@ -5,10 +5,10 @@
  * @param content  - the output of your content plugin
  * @param options  - an object containing options. Options are sent from Java
  *
- * @return - an array of header objects
+ * @return - an object of headers
  */
 function createHeaders(id, content, options) {
-  return [];
+  return {};
 }
 
 module.exports = {

examples/hr-hub/plugins/entities/Employee/input/load-global-corp/headers/headers.sjs

@@ -5,10 +5,10 @@
  * @param content  - the output of your content plugin
  * @param options  - an object containing options. Options are sent from Java
  *
- * @return - an array of header objects
+ * @return - an object of headers
  */
 function createHeaders(id, content, options) {
-  return [];
+  return {};
 }
 
 module.exports = {

marklogic-data-hub/build.gradle

@@ -5,6 +5,7 @@ plugins {
     id 'idea'
     id 'maven-publish'
     id 'com.jfrog.bintray' version '1.6'
+    id 'com.marklogic.ml-gradle' version '2.1.0'
 }
 
 repositories {
@@ -121,3 +122,70 @@ bintray {
         issueTrackerUrl = 'https://github.com/marklogic/marklogic-data-hub/issues'
     }
 }
+
+ext {
+    // mlAppConfig is an instance of com.marklogic.appdeployer.AppConfig
+    mlAppConfig {
+        // override some default values with our values
+        modulesDatabaseName = mlModulesDbName
+        triggersDatabaseName = mlTriggersDbName
+        schemasDatabaseName = mlSchemasDbName
+        restPort = Integer.parseInt(mlStagingPort)
+
+        // Configure custom tokens for our json files
+        customTokens.put("%%STAGING_SERVER_NAME%%", mlStagingAppserverName)
+        customTokens.put("%%STAGING_SERVER_PORT%%", mlStagingPort)
+        customTokens.put("%%STAGING_DB_NAME%%", mlStagingDbName)
+
+        customTokens.put("%%FINAL_SERVER_NAME%%", mlFinalAppserverName)
+        customTokens.put("%%FINAL_SERVER_PORT%%", mlFinalPort)
+        customTokens.put("%%FINAL_DB_NAME%%", mlFinalDbName)
+
+        customTokens.put("%%TRACE_SERVER_NAME%%", mlTraceAppserverName)
+        customTokens.put("%%TRACE_SERVER_PORT%%", mlTracePort)
+        customTokens.put("%%TRACE_DB_NAME%%", mlTraceDbName)
+
+        customTokens.put("%%MODULES_DB_NAME%%", mlModulesDbName)
+
+        modulePaths = ["marklogic-data-hub/src/main/resources/ml-modules"]
+    }
+}
+
+ext {
+    // don't create the REST Api. We will do it manually
+    mlAppDeployer.commands.remove(mlAppDeployer.getCommand("DeployRestApiServersCommand"))
+    mlAppDeployer.commands.remove(mlAppDeployer.getCommand("UpdateRestApiServersCommand"))
+
+    // remove the original deploy content database command
+    // as we do not need it.
+    def deployDbCmd = mlAppDeployer.getCommand("DeployContentDatabasesCommand")
+    mlAppDeployer.commands.remove(deployDbCmd)
+
+    // install the staging database
+    def stagingDbCommand = new com.marklogic.appdeployer.command.databases.DeployDatabaseCommand("staging-database.json")
+    stagingDbCommand.setForestsPerHost(Integer.parseInt(mlStagingForestsPerHost));
+    mlAppDeployer.commands.add(stagingDbCommand)
+    mlDatabaseCommands.add(stagingDbCommand)
+
+    // install the final database
+    def finalDbCommand = new com.marklogic.appdeployer.command.databases.DeployDatabaseCommand("final-database.json")
+    finalDbCommand.setForestsPerHost(Integer.parseInt(mlFinalForestsPerHost));
+    mlAppDeployer.commands.add(finalDbCommand)
+    mlDatabaseCommands.add(finalDbCommand)
+
+    // install the trace database
+    def traceDbCommand = new com.marklogic.appdeployer.command.databases.DeployDatabaseCommand("trace-database.json")
+    traceDbCommand.setForestsPerHost(Integer.parseInt(mlTraceForestsPerHost));
+    mlAppDeployer.commands.add(traceDbCommand)
+    mlDatabaseCommands.add(traceDbCommand)
+
+    // install the modules database
+    def modulesDbCommand = new com.marklogic.appdeployer.command.databases.DeployDatabaseCommand("modules-database.json")
+    mlAppDeployer.commands.add(modulesDbCommand)
+    mlDatabaseCommands.add(modulesDbCommand)
+
+    // temp workaround for ml-gradle issue #78
+    // https://github.com/rjrudin/ml-gradle/issues/78
+    def lmc = mlAppDeployer.getCommand("LoadModulesCommand")
+    lmc.setModulesLoader(new com.marklogic.client.modulesloader.impl.DefaultModulesLoader(mlAppConfig.newXccAssetLoader()))
+}

marklogic-data-hub/gradle.properties

@@ -1,12 +1,33 @@
-mlAppDeployerDependency=com.marklogic:ml-app-deployer:+
 mlConfigDir=marklogic-data-hub/src/main/resources/ml-config
-publishUrl=file:../marklogic-data-hub/releases
-mlStagingRestPort=8010
-mlFinalRestPort=8011
-mlTraceRestPort=8012
-mlUsername=admin
-mlPassword=admin
+
 mlHost=localhost
 mlAppName=data-hub
+
+mlUsername=admin
+mlPassword=admin
 auth=digest
+
+mlStagingAppserverName=data-hub-STAGING
+mlStagingPort=8010
+mlStagingDbName=data-hub-STAGING
+mlStagingForestsPerHost=4
+
+mlFinalAppserverName=data-hub-FINAL
+mlFinalPort=8011
+mlFinalDbName=data-hub-FINAL
+mlFinalForestsPerHost=4
+
+mlTraceAppserverName=data-hub-TRACING
+mlTracePort=8012
+mlTraceDbName=data-hub-TRACING
+mlTraceForestsPerHost=1
+
+mlModulesDbName=data-hub-MODULES
+mlTriggersDbName=data-hub-TRIGGERS
+mlSchemasDbName=data-hub-SCHEMAS
+
+hubModulesPath=exmaples/hr-hub/plugins
+
 version=1.0.0-beta.2
+
+

marklogic-data-hub/src/main/java/com/marklogic/hub/DataHub.java

@@ -58,6 +58,7 @@
 import com.marklogic.hub.commands.DeployModulesDatabaseCommand;
 import com.marklogic.hub.commands.DeployRestApiCommand;
 import com.marklogic.hub.commands.LoadModulesCommand;
+import com.marklogic.hub.commands.UpdateRestApiServersCommand;
 import com.marklogic.hub.util.HubFileFilter;
 import com.marklogic.hub.util.HubModulesLoader;
 import com.marklogic.mgmt.ManageClient;
@@ -315,15 +316,17 @@ private List<Command> getCommands(AppConfig config) {
         dbCommands.add(tracingDb);
 
         dbCommands.add(new DeployModulesDatabaseCommand(hubConfig.modulesDbName));
-        dbCommands.add(new DeployTriggersDatabaseCommand());
-        dbCommands.add(new DeploySchemasDatabaseCommand());
         commands.addAll(dbCommands);
 
         // App Servers
         commands.add(new DeployRestApiCommand(hubConfig.stagingHttpName, hubConfig.stagingPort));
         commands.add(new DeployRestApiCommand(hubConfig.finalHttpName, hubConfig.finalPort));
         commands.add(new DeployRestApiCommand(hubConfig.tracingHttpName, hubConfig.tracePort));
 
+        commands.add(new UpdateRestApiServersCommand(hubConfig.stagingHttpName));
+        commands.add(new UpdateRestApiServersCommand(hubConfig.finalHttpName));
+        commands.add(new UpdateRestApiServersCommand(hubConfig.tracingHttpName));
+
         // Modules
         commands.add(new LoadModulesCommand());
 

marklogic-data-hub/src/main/java/com/marklogic/hub/Debugging.java

@@ -0,0 +1,52 @@
+package com.marklogic.hub;
+
+import com.marklogic.client.DatabaseClient;
+import com.marklogic.client.extensions.ResourceManager;
+import com.marklogic.client.extensions.ResourceServices.ServiceResult;
+import com.marklogic.client.extensions.ResourceServices.ServiceResultIterator;
+import com.marklogic.client.io.StringHandle;
+import com.marklogic.client.util.RequestParameters;
+
+public class Debugging extends ResourceManager {
+    private static final String NAME = "debug";
+
+    public Debugging(DatabaseClient client) {
+        super();
+        client.init(NAME, this);
+    }
+
+    /**
+     * Enables tracing
+     */
+    public void enable() {
+        RequestParameters params = new RequestParameters();
+        params.add("enable", "true");
+        this.getServices().post(params, new StringHandle("{}"));
+    }
+
+    /**
+     * Disables tracing
+     */
+    public void disable() {
+        RequestParameters params = new RequestParameters();
+        params.add("enable", "false");
+        this.getServices().post(params, new StringHandle("{}"));
+    }
+
+    /**
+     * Determines if the hub has tracing enabled or not
+     *
+     * @return - true if enabled, false otherwise
+     */
+    public boolean isEnabled() {
+        RequestParameters params = new RequestParameters();
+        ServiceResultIterator resultItr = this.getServices().get(params);
+        if (resultItr == null || ! resultItr.hasNext()) {
+            return false;
+        }
+        ServiceResult res = resultItr.next();
+        StringHandle handle = new StringHandle();
+        String enabled = res.getContent(handle).get();
+        return Boolean.parseBoolean(enabled);
+    }
+}

marklogic-data-hub/src/main/java/com/marklogic/hub/HubConfig.java

@@ -59,7 +59,7 @@ public class HubConfig {
 
     public String tracingDbName = DEFAULT_TRACING_NAME;
     public String tracingHttpName = DEFAULT_TRACING_NAME;
-    public int tracingForestsPerHost = DEFAULT_FORESTS_PER_HOST;
+    public int tracingForestsPerHost = 1;
     public Integer tracePort = DEFAULT_TRACE_PORT;
 
     public String modulesDbName = DEFAULT_MODULES_DB_NAME;

marklogic-data-hub/src/main/java/com/marklogic/hub/commands/UpdateRestApiServersCommand.java

@@ -0,0 +1,53 @@
+package com.marklogic.hub.commands;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.marklogic.appdeployer.AppConfig;
+import com.marklogic.appdeployer.command.AbstractCommand;
+import com.marklogic.appdeployer.command.CommandContext;
+import com.marklogic.appdeployer.command.SortOrderConstants;
+import com.marklogic.mgmt.appservers.ServerManager;
+
+/**
+ * Command for updating an existing REST API server that was presumably created via /v1/rest-apis.
+ */
+public class UpdateRestApiServersCommand extends AbstractCommand {
+
+    private String serverName;
+    public UpdateRestApiServersCommand(String serverName) {
+        this.serverName = serverName;
+        setExecuteSortOrder(SortOrderConstants.UPDATE_REST_API_SERVERS);
+    }
+
+    /**
+     * This uses a different file than that of creating a REST API, as the payload for /v1/rest-apis differs from that
+     * of the /manage/v2/servers endpoint.
+     */
+    @Override
+    public void execute(CommandContext context) {
+        AppConfig appConfig = context.getAppConfig();
+
+        ServerManager mgr = new ServerManager(context.getManageClient(), appConfig.getGroupName());
+
+        String json = buildRestApiJson(appConfig);
+        mgr.save(json);
+    }
+
+    private String buildRestApiJson(AppConfig config) {
+        ObjectMapper m = new ObjectMapper();
+        ObjectNode node = m.createObjectNode();
+        node.put("server-name", serverName);
+        node.put("error-handler", "/com.marklogic.hub/error-handler.xqy");
+
+        try {
+            String json = m.writer(new DefaultPrettyPrinter()).writeValueAsString(node);
+            return json;
+        } catch (JsonProcessingException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+
+}

marklogic-data-hub/src/main/resources/ml-config/databases/final-database.json

@@ -1,5 +1,9 @@
 {
-  "database-name": "data-hub-FINAL",
+  "database-name": "%%FINAL_DB_NAME%%",
+  "range-element-index": [],
+  "schema-database": "%%SCHEMAS_DATABASE%%",
+  "triggers-database": "%%TRIGGERS_DATABASE%%",
   "triple-index": true,
-  "collection-lexicon":true
+  "collection-lexicon": true,
+  "uri-lexicon": true
 }

marklogic-data-hub/src/main/resources/ml-config/databases/modules-database.json

@@ -0,0 +1,5 @@
+{
+  "database-name": "%%MODULES_DB_NAME%%",
+  "collection-lexicon": true,
+  "uri-lexicon": true
+}

marklogic-data-hub/src/main/resources/ml-config/databases/schemas-database.json

@@ -0,0 +1,3 @@
+{
+  "database-name": "%%SCHEMAS_DATABASE%%"
+}

marklogic-data-hub/src/main/resources/ml-config/databases/staging-database.json

@@ -1,5 +1,9 @@
 {
-  "database-name": "data-hub-STAGING",
+  "database-name": "%%STAGING_DB_NAME%%",
+  "range-element-index": [],
+  "schema-database": "%%SCHEMAS_DATABASE%%",
+  "triggers-database": "%%TRIGGERS_DATABASE%%",
   "triple-index": true,
-  "collection-lexicon":true
+  "collection-lexicon": true,
+  "uri-lexicon": true
 }