3.x develop fast forward

examples/README.md

@@ -3,9 +3,11 @@
 This folder contains working examples of various DHF usage scenarios.
 
 1. [barebones](https://github.com/marklogic/marklogic-data-hub/tree/master/examples/barebones) - an example of the minimum configuration necessary to run a Gradle based Data Hub
+1. [custom-tokens](https://github.com/marklogic/marklogic-data-hub/tree/master/examples/custom-tokens) - an example where gradle environment properties are used in substitutions of configuration files that is based off of the barebones example
 1. [healthcare](https://github.com/marklogic/marklogic-data-hub/tree/master/examples/healthcare) - an example of a Healthcare 360 Data Hub
 1. [hr-hub](https://github.com/marklogic/marklogic-data-hub/tree/master/examples/hr-hub) - an example used for our 1.x tutorial. This example harmonizes data from various HR systems
 1. [load-binaries](https://github.com/marklogic/marklogic-data-hub/tree/master/examples/load-binaries) - an example of how to ingest binaries via an MLCP Input Flow
 1. [online-store](https://github.com/marklogic/marklogic-data-hub/tree/master/examples/online-store) - the example we use for our [Tutorial](https://marklogic.github.io/marklogic-data-hub/tutorial/)
+1. [single-step-ingest](https://github.com/marklogic/marklogic-data-hub/tree/master/examples/single-step-ingest) - an example of a custom REST endpoint that a user can call that will ingest a document into the STAGING database and then harmonize the same document immediately after the document was ingested
 1. [spring-batch](https://github.com/marklogic/marklogic-data-hub/tree/master/examples/spring-batch) - an example of how to load relational data into a Data Hub using Spring Batch
 1. [ssl](https://github.com/marklogic/marklogic-data-hub/tree/master/examples/ssl) - an example of how to configure your Data Hub to use SSL for added security

examples/custom-tokens/README.md

@@ -0,0 +1,57 @@
+# Custom Tokens Example
+
+This example demonstrates how to utilize gradle environment properties that are substituted in configuraton files within the MarkLogic deployment process.  We are basing this example off of the [barebones example](https://github.com/marklogic/marklogic-data-hub/tree/master/examples/barebones).
+
+This example utilizes gradle environment properties that are substituted in the configuration files for a user defined database and application server.  You can see the environment properties that are defined at the bottom of the `gradle.properties` file.
+
+Inside of the `gradle.properties` file, you can see the following information:
+
+```
+# Custom properties defined here
+TEST_DATABASE_NAME=custom-tokens-test-database
+TEST_SERVER_NAME=custom-tokens-test-database-server
+TEST_SERVER_PORT=8014
+TEST_TRACE_AUTH=digest
+```
+
+To utilize the custom tokens, then you will need to refer to them as `%%TEST_DATABASE_NAME%%` if you want to reference the `TEST_DATABASE_NAME` name token within your gradle deployment.  You can change the default token prefix and suffix from "%%" by utilizing the following tokens in your `gradle.properties` file by setting the appropriate prefix and suffix values according:
+
+```
+mlTokenPrefix=
+mlTokenSuffix=
+```
+For more information regarding this, then you can refer to the [ml-gradle wiki](https://github.com/marklogic-community/ml-gradle/wiki/Configuring-resources)
+
+You can see that we are referencing the new custom tokens within the `custom-tokens-test-server.json` file.  We are utilizing the four (4) custom tokens that we defined in our properties file which are the following: `%%TEST_SERVER_NAME%%`, `%%TEST_SERVER_PORT%%`, `%%TEST_DATABASE_NAME%%`, and `%%TEST_TRACE_AUTH%%`.
+
+```json
+{
+  "server-name": "%%TEST_SERVER_NAME%%",
+  "server-type": "http",
+  "root": "/",
+  "group-name": "%%GROUP%%",
+  "port": "%%TEST_SERVER_PORT%%",
+  "modules-database": "%%mlModulesDbName%%",
+  "content-database": "%%TEST_DATABASE_NAME%%",
+  "authentication": "%%TEST_TRACE_AUTH%%",
+  "default-error-format": "json",
+  "error-handler": "/MarkLogic/rest-api/error-handler.xqy",
+  "url-rewriter": "/MarkLogic/rest-api/rewriter.xml",
+  "rewrite-resolves-globally": true
+}
+```
+
+Next, Initialize your DHF app:
+
+```bash
+gradle hubInit
+```
+
+Then Bootstrap your DHF app with the user defined database and application that utilized custom tokens:
+
+```bash
+gradle mlDeploy
+```
+Once the deployment is complete, then you can login to the admin console and see the new test database and test database application server that were created with the values that were specified from our properties file.  Now that you've mastered utilizing custom tokens for your gradle deployment, you can continue on with your DHF app development.
+
+For a complete list of gradle tasks, check here: [https://github.com/marklogic/marklogic-data-hub/wiki/Gradle-Tasks](https://github.com/marklogic/marklogic-data-hub/wiki/Gradle-Tasks)

examples/custom-tokens/build.gradle

@@ -0,0 +1,11 @@
+plugins {
+    // this plugin lets you create properties files
+    // for multiple environments... like dev, qa, prod
+    id 'net.saliman.properties' version '1.4.6'
+
+    // this is the data hub framework gradle plugin
+    // it includes ml-gradle. This plugin is what lets you
+    // run DHF (Data Hub Framework) tasks from the
+    // command line
+    id 'com.marklogic.ml-data-hub' version '2.0.4'
+}

examples/custom-tokens/gradle.properties

@@ -0,0 +1,89 @@
+# These settings are used by the Data Hub Framework when
+# communicating with MarkLogic.
+# The values in this file are meant as project-wide settings.
+# You can override these properties for a specific environment
+# by creating a gradle-{environment}.properties file.
+# For example, to create a properties file for your prod environment create a file
+# named gradle-prod.properties.
+#
+# ....
+mlHost=localhost
+
+# Your MarkLogic Username and Password
+mlUsername=admin
+mlPassword=admin
+
+# If specified, the manage username/password combo is used with the ML Management REST API for managing application
+# resources; this user must have the manage-admin and rest-admin roles.
+#
+# If these are not set, then mlUsername/mlPassword is used for managing application resources.
+# mlManageUsername=
+# mlManagePassword=
+
+# If specified, the admin username/password combo is used with the ML Management REST API for creating users and roles. This
+# user must have the manage-admin or admin role. A good practice is to use your admin account here to create app-specific
+# users and roles, which can then be used as mlManageUsername/mlManagePassword and mlUsername/mlPassword.
+#
+# These properties are also used for connecting to the admin application on port 8001 - e.g. for initializing ML and for
+# waiting for ML to restart.
+#
+# If these properties are not set, then mlUsername/mlPassword will be used.
+# mlAdminUsername=
+# mlAdminPassword=
+
+# If specified, these values can override where the DHF thinks
+# MarkLogic default ports are at. You would only use this if you
+# have changed the ports on which MarkLogic listens
+#
+# mlAppServicesPort=8000
+# mlAdminPort=8001
+# mlManagePort=8002
+
+mlStagingAppserverName=data-hub-STAGING
+mlStagingPort=8010
+mlStagingDbName=data-hub-STAGING
+mlStagingForestsPerHost=4
+mlStagingAuth=digest
+
+mlFinalAppserverName=data-hub-FINAL
+mlFinalPort=8011
+mlFinalDbName=data-hub-FINAL
+mlFinalForestsPerHost=4
+mlFinalAuth=digest
+
+mlTraceAppserverName=data-hub-TRACING
+mlTracePort=8012
+mlTraceDbName=data-hub-TRACING
+mlTraceForestsPerHost=1
+mlTraceAuth=digest
+
+mlJobAppserverName=data-hub-JOBS
+mlJobPort=8013
+mlJobDbName=data-hub-JOBS
+mlJobForestsPerHost=1
+mlJobAuth=digest
+
+mlModulesDbName=data-hub-MODULES
+mlModulesForestsPerHost=1
+
+mlTriggersDbName=data-hub-TRIGGERS
+mlTriggersForestsPerHost=1
+
+mlSchemasDbName=data-hub-SCHEMAS
+mlSchemasForestsPerHost=1
+
+# You can override this to specify an alternate folder for your
+# custom forest info. Defaults to user-config/forests/
+# mlCustomForestPath=forests
+
+# The name of the Role to create for Hub Access
+mlHubUserRole=data-hub-role
+mlHubUserName=data-hub-user
+# this password is autogenerated for you via the 'gradle hubInit' task
+mlHubUserPassword=b$I7'3Ya|&;Ohw.ZzsDY
+
+# Custom properties defined here
+TEST_DATABASE_NAME=custom-tokens-test-database
+TEST_SERVER_NAME=custom-tokens-test-database-server
+TEST_SERVER_PORT=8014
+TEST_TRACE_AUTH=digest

examples/custom-tokens/user-config/databases/test-database.json

@@ -0,0 +1,4 @@
+{
+  "database-name": "%%TEST_DATABASE_NAME%%"
+}
+

examples/custom-tokens/user-config/servers/test-server.json

@@ -0,0 +1,15 @@
+{
+  "server-name": "%%TEST_SERVER_NAME%%",
+  "server-type": "http",
+  "root": "/",
+  "group-name": "%%GROUP%%",
+  "port": "%%TEST_SERVER_PORT%%",
+  "modules-database": "%%mlModulesDbName%%",
+  "content-database": "%%TEST_DATABASE_NAME%%",
+  "authentication": "%%TEST_TRACE_AUTH%%",
+  "default-error-format": "json",
+  "error-handler": "/MarkLogic/rest-api/error-handler.xqy",
+  "url-rewriter": "/MarkLogic/rest-api/rewriter.xml",
+  "rewrite-resolves-globally": true
+}
+

examples/data-integration-tests/build.gradle

@@ -36,7 +36,7 @@ dependencies {
     // on the Data Hub Framework library
     compile 'com.marklogic:marklogic-data-hub:2.0.3'
     compile 'com.marklogic:marklogic-xcc:9.0.4'
-    compile 'com.marklogic:marklogic-client-api:4.0.3.1'
+    compile 'com.marklogic:marklogic-client-api:4.0.4'
     // prototyping with kotlin tests
     compile 'io.github.microutils:kotlin-logging:1.4.6'
     compile "org.jetbrains.kotlin:kotlin-stdlib"

examples/online-store/plugins/entities/Product/harmonize/Harmonize Products/content/content.sjs

@@ -41,7 +41,6 @@ function extractInstanceProduct(source) {
   let attachments = source;
 
   let sku = xs.string(source.sku || source.SKU);
-  let title = xs.string(source.title);
   let price = xs.decimal(source.price);
 
   // return the instance object
@@ -50,7 +49,6 @@ function extractInstanceProduct(source) {
     '$type': 'Product',
     '$version': '0.0.1',
     'sku': sku,
-    'title': title,
     'price': price
   }
 };

marklogic-data-hub/build.gradle

@@ -25,7 +25,7 @@ ext.junitPlatformVersion = '1.0.0-RC3'
 ext.junitJupiterVersion  = '5.0.0-RC3'
 
 dependencies {
-    compile 'com.marklogic:marklogic-client-api:4.0.3.1'
+    compile 'com.marklogic:marklogic-client-api:4.0.4'
     compile 'com.marklogic:mlcp-util:0.9.0'
     compile 'com.marklogic:ml-app-deployer:3.6.1'
     compile 'com.marklogic:marklogic-data-movement-components:1.0'

marklogic-data-hub/src/main/java/com/marklogic/hub/impl/DataHubImpl.java

@@ -347,7 +347,7 @@ public List<Command> getCommandList() {
     }
 
     private void runInDatabase(String query, String databaseName) {
-        ServerEvaluationCall eval = hubConfig.newStagingClient().newServerEval();
+        ServerEvaluationCall eval = hubConfig.newModulesDbClient().newServerEval();
         String xqy =
             "xdmp:invoke-function(function() {" +
                 query +
@@ -506,7 +506,7 @@ private Map<Integer, String> getServerPortsInUse() {
     }
 
     //DataHubUpgrader stuff
-    public static String MIN_UPGRADE_VERSION = "1.1.3";
+    public static String MIN_UPGRADE_VERSION = "2.0.0";
 
     @Override public boolean upgradeHub() throws CantUpgradeException {
         return upgradeHub(null);
@@ -545,6 +545,8 @@ private Map<Integer, String> getServerPortsInUse() {
                 updatedFlows.addAll(flows);
             }
 
+            runInDatabase("cts:uris(\"\", (), cts:and-not-query(cts:collection-query(\"hub-core-module\"), cts:document-query((\"/com.marklogic.hub/config.sjs\", \"/com.marklogic.hub/config.xqy\")))) ! xdmp:document-delete(.)", hubConfig.getDbName(DatabaseKind.MODULES));
+
             if (isHubInstalled) {
                 // install hub modules into MarkLogic
                 this.install();

marklogic-data-hub/src/main/java/com/marklogic/hub/impl/HubConfigImpl.java

@@ -1014,7 +1014,7 @@ public DatabaseClient newModulesDbClient() {
 
         // this lets debug builds work from an IDE
         if (version.equals("${project.version}")) {
-            version = "0.1.2";
+            version = "3.0.0";
         }
         return version;
     }

marklogic-data-hub/src/main/java/com/marklogic/hub/scaffold/impl/ScaffoldingImpl.java

@@ -280,6 +280,7 @@ public boolean update2xFlow(String entityName, String flowName, FlowType flowTyp
 
         Path flowDir = getFlowDir(entityName, flowName, flowType);
         Path mainPath = flowDir.resolve("main.sjs");
+        Path xqyMainPath = flowDir.resolve("main.xqy");
         if (mainPath.toFile().exists()) {
             try {
                 String mainFile = FileUtils.readFileToString(mainPath.toFile());
@@ -309,6 +310,22 @@ public boolean update2xFlow(String entityName, String flowName, FlowType flowTyp
                 throw new RuntimeException(e);
             }
         }
+
+        if (mainPath.toFile().exists() || xqyMainPath.toFile().exists()) {
+            if (xqyMainPath.toFile().exists()) {
+                mainPath = xqyMainPath;
+            }
+            try {
+                String mainFile = FileUtils.readFileToString(mainPath.toFile());
+                mainFile = mainFile.replaceFirst("com\\.marklogic\\.hub", "MarkLogic/data-hub-framework");
+                FileOutputStream fileOutputStream = new FileOutputStream(mainPath.toFile());
+                IOUtils.write(mainFile, fileOutputStream);
+                fileOutputStream.close();
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }
+
         return updated;
     }
 

ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/UpdateHubTaskTest.groovy

@@ -29,6 +29,7 @@ class UpdateHubTaskTest extends BaseTest {
     def setupSpec() {
         createGradleFiles()
         runTask('hubInit')
+        println(runTask('mlDeploy', '-i').getOutput())
     }
 
     def "no updates needed"() {

quick-start/e2e/page-objects/flows/flows.ts

@@ -182,6 +182,10 @@ export class FlowPage extends AppPage {
     return element(by.css(`mdl-switch[id="${name}"]`));
   }
 
+  mlcpInput(name: string) {
+    return element(by.css(`input[name="${name}"]`));
+  }
+
   get toast() {
     return element(by.css('mdl-snackbar-component'));
   }
@@ -252,6 +256,11 @@ export class FlowPage extends AppPage {
     browser.actions().mouseMove(this.menuItem(dataFormat)).perform();
     this.menuItem(dataFormat).click();
 
+    // set output uri suffix
+    this.mlcpInput('output_uri_suffix').clear();
+    // verify that uri can contain character &
+    this.mlcpInput('output_uri_suffix').sendKeys('?doc=yes&type=foo');
+
     browser.wait(EC.elementToBeClickable(this.mlcpSection(' Delimited Text Options')));
     this.mlcpSection(' Delimited Text Options').click();
     browser.wait(EC.visibilityOf(this.mlcpSection(' Delimited Text Options')));

quick-start/e2e/specs/auth/authenticated.ts

@@ -62,7 +62,6 @@ export default function(tmpDir) {
       expect(loginPage.stagingAppserverNameLabel.isPresent()).toBe(true);
       expect(loginPage.stagingAppserverName.getAttribute('value')).toEqual('data-hub-STAGING');
       expect(loginPage.modulesDbName.getAttribute('value')).toEqual('data-hub-MODULES');
-      expect(loginPage.dataHubName.getAttribute('value')).toEqual('');
       expect(loginPage.dataHubName.getAttribute('value')).toEqual('data-hub');
       browser.driver.sleep(3000);
       expect(loginPage.projectDirTab.isDisplayed()).toBe(false);