Feature/sjs perf

dependencies.md

@@ -16,7 +16,7 @@
 - ml-app-deployer 3.4 -> https://github.com/marklogic-community/ml-app-deployer
 - ml-javaclient-util 3.3.1 -> https://github.com/marklogic-community/ml-javaclient-util
 - marklogic-client-api 4.0.3.1 -> https://github.com/marklogic/java-client-api
-- mlcp-util 0.2.0 -> https://github.com/marklogic-community/mlcp-util
+- mlcp-util 0.9.0 -> https://github.com/marklogic-community/mlcp-util
 - apache commons io 2.4 -> https://commons.apache.org/proper/commons-io/
 - apache commons text 1.1 -> https://commons.apache.org/proper/commons-text/
 - apache commons csv 1.4 -> https://commons.apache.org/proper/commons-csv/

examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-acme-tech/main.sjs

@@ -1,11 +1,12 @@
-// dhf.xqy exposes helper functions to make your life easier
+// dhf.sjs exposes helper functions to make your life easier
 // See documentation at:
 // https://github.com/marklogic/marklogic-data-hub/wiki/dhf-lib
-const dhf = require('/MarkLogic/data-hub-framework/dhf.xqy');
+const dhf = require('/MarkLogic/data-hub-framework/dhf.sjs');
 
 const contentPlugin = require('./content/content.sjs');
 const headersPlugin = require('./headers/headers.sjs');
 const triplesPlugin = require('./triples/triples.sjs');
+const writerPlugin = require('./writer.sjs');
 
 /*
  * Plugin Entry point
@@ -34,7 +35,7 @@ function main(id, options) {
 
   // writers must be invoked this way.
   // see: https://github.com/marklogic-community/marklogic-data-hub/wiki/dhf-lib#run-writer
-  dhf.runWriter(xdmp.function(null, './writer/writer.sjs'), id, envelope, options);
+  dhf.runWriter(writerPlugin, id, envelope, options);
 }
 
 module.exports = {

examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-global-corp/main.sjs

@@ -1,11 +1,12 @@
-// dhf.xqy exposes helper functions to make your life easier
+// dhf.sjs exposes helper functions to make your life easier
 // See documentation at:
 // https://github.com/marklogic/marklogic-data-hub/wiki/dhf-lib
-const dhf = require('/MarkLogic/data-hub-framework/dhf.xqy');
+const dhf = require('/MarkLogic/data-hub-framework/dhf.sjs');
 
 const contentPlugin = require('./content/content.sjs');
 const headersPlugin = require('./headers/headers.sjs');
 const triplesPlugin = require('./triples/triples.sjs');
+const writerPlugin = require('./writer.sjs');
 
 /*
  * Plugin Entry point
@@ -34,7 +35,7 @@ function main(id, options) {
 
   // writers must be invoked this way.
   // see: https://github.com/marklogic-community/marklogic-data-hub/wiki/dhf-lib#run-writer
-  dhf.runWriter(xdmp.function(null, './writer/writer.sjs'), id, envelope, options);
+  dhf.runWriter(writerPlugin, id, envelope, options);
 }
 
 module.exports = {

examples/hr-hub/plugins/entities/Employee/input/load-acme-tech/main.sjs

@@ -1,7 +1,7 @@
-// dhf.xqy exposes helper functions to make your life easier
+// dhf.sjs exposes helper functions to make your life easier
 // See documentation at:
 // https://github.com/marklogic/marklogic-data-hub/wiki/dhf-lib
-const dhf = require('/MarkLogic/data-hub-framework/dhf.xqy');
+const dhf = require('/MarkLogic/data-hub-framework/dhf.sjs');
 
 const contentPlugin = require('./content/content.sjs');
 const headersPlugin = require('./headers/headers.sjs');

examples/hr-hub/plugins/entities/Employee/input/load-global-corp/main.sjs

@@ -1,7 +1,7 @@
-// dhf.xqy exposes helper functions to make your life easier
+// dhf.sjs exposes helper functions to make your life easier
 // See documentation at:
 // https://github.com/marklogic/marklogic-data-hub/wiki/dhf-lib
-const dhf = require('/MarkLogic/data-hub-framework/dhf.xqy');
+const dhf = require('/MarkLogic/data-hub-framework/dhf.sjs');
 
 const contentPlugin = require('./content/content.sjs');
 const headersPlugin = require('./headers/headers.sjs');

examples/load-binaries/plugins/entities/Guides/input/LoadAsJson/main.sjs

@@ -1,7 +1,7 @@
-// dhf.xqy exposes helper functions to make your life easier
+// dhf.sjs exposes helper functions to make your life easier
 // See documentation at:
 // https://github.com/marklogic/marklogic-data-hub/wiki/dhf-lib
-const dhf = require('/MarkLogic/data-hub-framework/dhf.xqy');
+const dhf = require('/MarkLogic/data-hub-framework/dhf.sjs');
 
 const contentPlugin = require('./content.sjs');
 const headersPlugin = require('./headers.sjs');

examples/online-store/plugins/entities/Order/harmonize/Harmonize Orders/main.sjs

@@ -1,11 +1,12 @@
-// dhf.xqy exposes helper functions to make your life easier
+// dhf.sjs exposes helper functions to make your life easier
 // See documentation at:
 // https://github.com/marklogic/marklogic-data-hub/wiki/dhf-lib
-const dhf = require('/MarkLogic/data-hub-framework/dhf.xqy');
+const dhf = require('/MarkLogic/data-hub-framework/dhf.sjs');
 
 const contentPlugin = require('./content/content.sjs');
 const headersPlugin = require('./headers/headers.sjs');
 const triplesPlugin = require('./triples/triples.sjs');
+const writerPlugin = require('./writer.sjs');
 
 /*
  * Plugin Entry point
@@ -34,7 +35,7 @@ function main(id, options) {
 
   // writers must be invoked this way.
   // see: https://github.com/marklogic-community/marklogic-data-hub/wiki/dhf-lib#run-writer
-  dhf.runWriter(xdmp.function(null, './writer/writer.sjs'), id, envelope, options);
+  dhf.runWriter(writerPlugin, id, envelope, options);
 }
 
 module.exports = {

examples/online-store/plugins/entities/Order/input/Load Orders/main.sjs

@@ -1,7 +1,7 @@
-// dhf.xqy exposes helper functions to make your life easier
+// dhf.sjs exposes helper functions to make your life easier
 // See documentation at:
 // https://github.com/marklogic/marklogic-data-hub/wiki/dhf-lib
-const dhf = require('/MarkLogic/data-hub-framework/dhf.xqy');
+const dhf = require('/MarkLogic/data-hub-framework/dhf.sjs');
 
 const contentPlugin = require('./content/content.sjs');
 const headersPlugin = require('./headers/headers.sjs');

examples/online-store/plugins/entities/Product/harmonize/Harmonize Products/main.sjs

@@ -1,11 +1,12 @@
-// dhf.xqy exposes helper functions to make your life easier
+// dhf.sjs exposes helper functions to make your life easier
 // See documentation at:
 // https://github.com/marklogic/marklogic-data-hub/wiki/dhf-lib
-const dhf = require('/MarkLogic/data-hub-framework/dhf.xqy');
+const dhf = require('/MarkLogic/data-hub-framework/dhf.sjs');
 
 const contentPlugin = require('./content/content.sjs');
 const headersPlugin = require('./headers/headers.sjs');
 const triplesPlugin = require('./triples/triples.sjs');
+const writerPlugin = require('./writer.sjs');
 
 /*
  * Plugin Entry point
@@ -34,7 +35,7 @@ function main(id, options) {
 
   // writers must be invoked this way.
   // see: https://github.com/marklogic-community/marklogic-data-hub/wiki/dhf-lib#run-writer
-  dhf.runWriter(xdmp.function(null, './writer/writer.sjs'), id, envelope, options);
+  dhf.runWriter(writerPlugin, id, envelope, options);
 }
 
 module.exports = {

examples/online-store/plugins/entities/Product/input/Load Products/main.sjs

@@ -1,7 +1,7 @@
-// dhf.xqy exposes helper functions to make your life easier
+// dhf.sjs exposes helper functions to make your life easier
 // See documentation at:
 // https://github.com/marklogic/marklogic-data-hub/wiki/dhf-lib
-const dhf = require('/MarkLogic/data-hub-framework/dhf.xqy');
+const dhf = require('/MarkLogic/data-hub-framework/dhf.sjs');
 
 const contentPlugin = require('./content/content.sjs');
 const headersPlugin = require('./headers/headers.sjs');

examples/performance-sample/build.gradle

@@ -175,7 +175,7 @@ task prepareMlcpLog() {
   ].join("\n")
 }
 
-task loadSjsJson(type: com.marklogic.gradle.task.MlcpTask) {
+task loadSjsJson(type: MlcpTask) {
   mustRunAfter createSjsJsonInputFlow
   mustRunAfter mlLoadModules
   mustRunAfter getInputData
@@ -194,13 +194,14 @@ task loadSjsJson(type: com.marklogic.gradle.task.MlcpTask) {
   output_uri_prefix = "/city/"
   output_uri_suffix = ".json"
   output_collections = "input-json"
-  transform_module = "/com.marklogic.hub/mlcp-flow-transform.xqy"
-  transform_namespace = "http://marklogic.com/data-hub/mlcp-flow-transform"
+  transform_module = "/MarkLogic/data-hub-framework/transforms/mlcp-flow-transform.sjs"
+  transform_function = "transform"
   transform_param =  "entity-name=input-json,flow-name=input-sjs-json"
+  modules_root = "/"
   thread_count = 4
 }
 
-task loadXqyJson(type: com.marklogic.gradle.task.MlcpTask) {
+task loadXqyJson(type: MlcpTask) {
   mustRunAfter createXqyJsonInputFlow
   mustRunAfter mlLoadModules
   mustRunAfter getInputData
@@ -219,13 +220,14 @@ task loadXqyJson(type: com.marklogic.gradle.task.MlcpTask) {
   output_uri_prefix = "/city/"
   output_uri_suffix = ".json"
   output_collections = "input-json"
-  transform_module = "/com.marklogic.hub/mlcp-flow-transform.xqy"
+  transform_module = "/MarkLogic/data-hub-framework/transforms/mlcp-flow-transform.xqy"
   transform_namespace = "http://marklogic.com/data-hub/mlcp-flow-transform"
   transform_param =  "entity-name=input-json,flow-name=input-xqy-json"
+  modules_root = "/"
   thread_count = 4
 }
 
-task loadSjsXml(type: com.marklogic.gradle.task.MlcpTask) {
+task loadSjsXml(type: MlcpTask) {
   mustRunAfter createSjsXmlInputFlow
   mustRunAfter mlLoadModules
   mustRunAfter getInputData
@@ -244,13 +246,14 @@ task loadSjsXml(type: com.marklogic.gradle.task.MlcpTask) {
   output_uri_prefix = "/city/"
   output_uri_suffix = ".xml"
   output_collections = "input-xml"
-  transform_module = "/com.marklogic.hub/mlcp-flow-transform.xqy"
-  transform_namespace = "http://marklogic.com/data-hub/mlcp-flow-transform"
+  transform_module = "/MarkLogic/data-hub-framework/transforms/mlcp-flow-transform.sjs"
+  transform_function = "transform"
   transform_param =  "entity-name=input-xml,flow-name=input-sjs-xml"
+  modules_root = "/"
   thread_count = 4
 }
 
-task loadXqyXml(type: com.marklogic.gradle.task.MlcpTask) {
+task loadXqyXml(type: MlcpTask) {
   mustRunAfter createXqyXmlInputFlow
   mustRunAfter mlLoadModules
   mustRunAfter getInputData
@@ -269,9 +272,10 @@ task loadXqyXml(type: com.marklogic.gradle.task.MlcpTask) {
   output_uri_prefix = "/city/"
   output_uri_suffix = ".xml"
   output_collections = "input-xml"
-  transform_module = "/com.marklogic.hub/mlcp-flow-transform.xqy"
+  transform_module = "/MarkLogic/data-hub-framework/transforms/mlcp-flow-transform.xqy"
   transform_namespace = "http://marklogic.com/data-hub/mlcp-flow-transform"
   transform_param =  "entity-name=input-xml,flow-name=input-xqy-xml"
+  modules_root = "/"
   thread_count = 4
 }
 
@@ -417,3 +421,114 @@ class TimingsListener implements TaskExecutionListener, BuildListener {
 if (project.hasProperty('profile')) {
   gradle.addListener new TimingsListener()
 }
+
+// This example needs mlcp-util 0.9.0 or greater
+// right now we are relying on an older version of
+// ml-gradle which relies on 0.2.0 of mlcp-util
+// redefining this class here is a stop-gap until
+// we update stuff
+import com.marklogic.client.DatabaseClient
+import com.marklogic.client.io.FileHandle
+import com.marklogic.contentpump.bean.MlcpBean
+import org.gradle.api.logging.Logger
+import org.gradle.api.logging.Logging
+import org.gradle.api.tasks.JavaExec
+import org.gradle.api.tasks.TaskAction
+
+import com.marklogic.appdeployer.AppConfig
+class MlcpTask extends JavaExec {
+
+    @Delegate
+    MlcpBean mlcpBean = new MlcpBean();
+
+    // Set this to define a URI in your content database for mlcp output to be written to as a text document
+    String logOutputUri
+
+    Logger getLogger() {
+        return Logging.getLogger(MlcpTask.class)
+    }
+
+    @TaskAction
+    @Override
+    void exec() {
+        setMain("com.marklogic.contentpump.ContentPump")
+        AppConfig config = getProject().property("mlAppConfig")
+
+        List<String> newArgs = new ArrayList<>()
+        newArgs.add(command)
+
+        mlcpBean.properties.each { prop, val ->
+            def propVal
+            if (val) {
+                switch (prop) {
+                    case "host":
+                        propVal = (val ? val : config.getHost())
+                        break
+                    case "port":
+                        propVal = (val ? val : 8000)
+                        break
+                    case "username":
+                        propVal = (val ? val : config.getRestAdminUsername())
+                        break
+                    case ["class", "logger", "command", "password"]:
+                        // skip for now
+                        return
+                    case "additionalOptions":
+                        // Not supported by this task; use JavaExec's args instead
+                        return
+                    default:
+                        propVal = val
+                        break
+                }
+
+                newArgs.add("-" + prop);
+                newArgs.add(String.valueOf(propVal));
+            }
+        }
+
+        // Ensure connection arguments are present, but not if a COPY
+        boolean isCopy = "COPY".equals(command)
+        if (!isCopy) {
+            if (!newArgs.contains("-host")) {
+                newArgs.add("-host")
+                newArgs.add(config.getHost())
+            }
+            if (!newArgs.contains("-port")) {
+                newArgs.add("-port")
+                newArgs.add("8000")
+            }
+            if (!newArgs.contains("-username")) {
+                newArgs.add("-username")
+                newArgs.add(config.getRestAdminUsername())
+            }
+        }
+
+        // Include any args that a user has configured via the args parameter of the Gradle task
+        newArgs.addAll(getArgs())
+
+        println "mlcp arguments, excluding password: " + newArgs
+
+        if (!isCopy) {
+            newArgs.add("-password")
+            newArgs.add(password ? password : config.getRestAdminPassword())
+        }
+
+        setArgs(newArgs)
+
+        File logOutputFile = null
+        if (logOutputUri) {
+            println "Will write mlcp log output to URI: " + logOutputUri
+            logOutputFile = new File(getProject().getBuildDir(), "mlcp-log-output-" + System.currentTimeMillis() + ".txt")
+            setStandardOutput(logOutputFile.newOutputStream())
+        }
+
+        super.exec()
+
+        if (logOutputFile != null) {
+            AppConfig appConfig = project.property("mlAppConfig")
+            DatabaseClient client = appConfig.newDatabaseClient()
+            client.newDocumentManager().write(logOutputUri, new FileHandle(logOutputFile))
+            println "Wrote mlcp log output to URI: " + logOutputUri
+        }
+    }
+}

marklogic-data-hub/build.gradle

@@ -25,7 +25,7 @@ ext.junitJupiterVersion  = '5.0.0-RC3'
 
 dependencies {
     compile 'com.marklogic:marklogic-client-api:4.0.3.1'
-    compile 'com.marklogic:mlcp-util:0.3.0'
+    compile 'com.marklogic:mlcp-util:0.9.0'
     compile 'com.marklogic:ml-app-deployer:3.4.0'
     compile 'commons-io:commons-io:2.4'
     compile 'org.apache.commons:commons-text:1.1'
@@ -48,7 +48,7 @@ dependencies {
     testCompile 'org.hamcrest:hamcrest-junit:2.0.0.0'
     testCompile 'org.easymock:easymock:3.4'
     testCompile 'ch.qos.logback:logback-classic:1.1.11'
-    testCompile("com.marklogic:mlcp-util:0.3.0")
+    testCompile("com.marklogic:mlcp-util:0.9.0")
     testCompile("com.marklogic:mlcp:9.0.3") {
       exclude group: 'org.apache.avro', module: 'avro-tools'
     }

marklogic-data-hub/src/main/java/com/marklogic/hub/flow/impl/FlowRunnerImpl.java

@@ -345,8 +345,6 @@ private String jsonToString(JsonNode node) {
 
     class FlowResource extends ResourceManager {
 
-        static final public String NAME = "ml:flow";
-
         private DatabaseClient srcClient;
         private String targetDatabase;
         private Flow flow;
@@ -356,7 +354,7 @@ public FlowResource(DatabaseClient srcClient, String targetDatabase, Flow flow)
             this.flow = flow;
             this.srcClient = srcClient;
             this.targetDatabase = targetDatabase;
-            this.srcClient.init(NAME, this);
+            this.srcClient.init(flow.getCodeFormat().equals(CodeFormat.JAVASCRIPT) ? "ml:sjsFlow" : "ml:flow", this);
         }
 
         public RunFlowResponse run(String jobId, String[] items) {