Fix for generating TDE templates for DHF style entities with references

marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommand.java

@@ -23,10 +23,14 @@
 import com.marklogic.client.ext.es.EntityServicesManager;
 import com.marklogic.client.ext.es.GeneratedCode;
 import com.marklogic.hub.HubConfig;
+import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
 import java.nio.file.Path;
 import java.util.*;
 import java.util.function.Function;
@@ -70,7 +74,24 @@ public void execute(CommandContext context) {
                     entityNameFileMap.keySet(), hubConfig.getAppConfig().getSchemasPath());
 
                 for (File f : entityNameFileMap.values()) {
-                    GeneratedCode code = loadModelDefinition(request, f, mgr);
+                    File esModel;
+                    try {
+                        //Write the ES model to a temp file
+                        esModel = File.createTempFile("es-", f.getName());
+                        FileUtils.writeStringToFile(esModel, generateModel(f));
+                    } catch (IOException e) {
+                        throw new RuntimeException("Unable to generate ES model");
+                    }
+
+                    GeneratedCode code;
+                    try {
+                        code = loadModelDefinition(request, esModel, mgr);
+                    } catch (RuntimeException e) {
+                        throw new RuntimeException("Unable to read model definition from file: " + f.getAbsolutePath(), e);
+                    }
+                    finally {
+                        FileUtils.deleteQuietly(esModel);
+                    }
                     generateExtractionTemplate(appConfig, code);
                 }
 
@@ -83,6 +104,14 @@ public void execute(CommandContext context) {
 
     }
 
+    //Method to obtain es-style model
+    private String generateModel(File f) {
+        String xquery = "import module namespace hent = \"http://marklogic.com/data-hub/hub-entities\"\n" +
+            "at \"/data-hub/4/impl/hub-entities.xqy\";\n" +
+            String.format("hent:get-model(\"%s\")", extactEntityNameFromFilename(f.getName()).get());
+        return  hubConfig.newStagingClient().newServerEval().xquery(xquery).eval().next().getString();
+    }
+
     public String getEntityNames() {
         return entityNames;
     }

ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/GenerateTDETemplateFromEntityTaskTest.groovy

@@ -0,0 +1,65 @@
+package com.marklogic.gradle.task
+
+import com.marklogic.hub.HubConfig
+import org.apache.commons.io.FileUtils
+import org.gradle.testkit.runner.UnexpectedBuildFailure
+
+import java.nio.file.Path
+import java.nio.file.Paths
+
+import static org.gradle.testkit.runner.TaskOutcome.SUCCESS
+
+class GenerateTDETemplateFromEntityTaskTest extends BaseTest {
+    def setupSpec() {
+        createGradleFiles()
+        runTask('hubInit')
+        clearDatabases(HubConfig.DEFAULT_STAGING_NAME, HubConfig.DEFAULT_FINAL_NAME, HubConfig.DEFAULT_JOB_NAME);
+    }
+
+    def "GenerateTDETEmplates"() {
+        given:
+        def pluginDir = Paths.get(hubConfig().hubProject.projectDirString).resolve("plugins")
+
+        //DHF style nested entities (references in separate file)
+        def entitiesDir1 = pluginDir.resolve("entities").resolve("Order")
+        def entitiesDir2 = pluginDir.resolve("entities").resolve("Item")
+        def entitiesDir3 = pluginDir.resolve("entities").resolve("Customer")
+
+        entitiesDir1.toFile().mkdirs()
+        entitiesDir2.toFile().mkdirs()
+        entitiesDir3.toFile().mkdirs()
+
+        //ES style nested entities(references in the same file)
+
+        def entitiesDir4 = pluginDir.resolve("entities").resolve("Entity1")
+        entitiesDir4.toFile().mkdirs()
+
+        //DHF style without references
+
+        def entitiesDir5 = pluginDir.resolve("entities").resolve("e2eentity")
+        entitiesDir5.toFile().mkdirs()
+
+        FileUtils.copyFile(new File("src/test/resources/tde-template/Order.entity.json"), entitiesDir1.resolve('Order.entity.json').toFile())
+        FileUtils.copyFile(new File("src/test/resources/tde-template/Item.entity.json"), entitiesDir2.resolve('Item.entity.json').toFile())
+        FileUtils.copyFile(new File("src/test/resources/tde-template/Customer.entity.json"), entitiesDir3.resolve('Customer.entity.json').toFile())
+
+        FileUtils.copyFile(new File("src/test/resources/tde-template/Order1.entity.json"), entitiesDir4.resolve('Order1.entity.json').toFile())
+        FileUtils.copyFile(new File("src/test/resources/tde-template/e2eentity.entity.json"), entitiesDir5.resolve('e2eentity.entity.json').toFile())
+
+        runTask("hubDeployUserArtifacts")
+
+        when:
+        def result = runTask('hubGenerateTDETemplates', '-PentityNames=Order,Order1,e2eentity,nonexistentEntity')
+
+        then:
+        notThrown(UnexpectedBuildFailure)
+        result.task(":hubGenerateTDETemplates").outcome == SUCCESS
+
+        Path tdePath = Paths.get(testProjectDir.root.toString(), "src", "main", "ml-schemas", "tde")
+        tdePath.resolve("Order-1.0.0.tdex").toFile().exists() == true
+        tdePath.resolve("Order1-1.0.0.tdex").toFile().exists() == true
+        tdePath.resolve("e2eentity-0.0.1.tdex").toFile().exists() == true
+        tdePath.resolve("nonexistentEntity-0.0.1.tdex").toFile().exists() == false
+
+    }
+}

ml-data-hub-plugin/src/test/resources/tde-template/Customer.entity.json

@@ -0,0 +1,23 @@
+{
+  "info": {
+    "title": "Customer",
+    "description": "DHF Customer",
+    "version": "1.0.0"
+  },
+  "definitions": {
+    "Customer": {
+      "properties": {
+        "id": {
+          "datatype": "int"
+        },
+        "name": {
+          "datatype": "string"
+        }
+      },
+      "required": ["id", "name"],
+      "primaryKey": "id",
+      "pii": ["name"],
+      "pathRangeIndex": ["id"]
+    }
+  }
+}

ml-data-hub-plugin/src/test/resources/tde-template/Item.entity.json

@@ -0,0 +1,29 @@
+{
+  "info": {
+    "title": "Item",
+    "description": "DHF Item",
+    "version": "1.0.0"
+  },
+  "definitions": {
+    "Item": {
+      "properties": {
+        "id": {
+          "datatype": "int"
+        },
+        "name": {
+          "datatype": "string"
+        },
+        "description": {
+          "datatype": "string"
+        },
+        "rating": {
+          "datatype": "float"
+        }
+      },
+      "required": ["id", "name"],
+      "primaryKey": "id",
+      "pathRangeIndex": ["id", "rating"],
+      "wordLexicon": ["description"]
+    }
+  }
+}

ml-data-hub-plugin/src/test/resources/tde-template/Order.entity.json

@@ -0,0 +1,68 @@
+{
+  "info": {
+    "title": "Order",
+    "description": "DHF Order",
+    "version": "1.0.0"
+  },
+  "definitions": {
+    "Order": {
+      "properties": {
+        "id": {
+          "datatype": "int"
+        },
+        "purchasedItems": {
+          "datatype": "array",
+          "items": {
+            "$ref": "#/definitions/Item"
+          }
+        },
+        "customer": {
+          "$ref": "#/definitions/Customer"
+        },
+        "transactionDateTime": {
+          "datatype": "dateTime"
+        },
+        "totalCost": {
+          "datatype": "double"
+        }
+      },
+      "required": ["id", "transactionDateTime", "totalCost"],
+      "primaryKey": "id",
+      "pathRangeIndex": ["id", "totalCost"]
+    },
+    "Customer": {
+      "properties": {
+        "id": {
+          "datatype": "int"
+        },
+        "name": {
+          "datatype": "string"
+        }
+      },
+      "required": ["id", "name"],
+      "primaryKey": "id",
+      "pii": ["name"],
+      "pathRangeIndex": ["id"]
+    },
+    "Item": {
+      "properties": {
+        "id": {
+          "datatype": "int"
+        },
+        "name": {
+          "datatype": "string"
+        },
+        "description": {
+          "datatype": "string"
+        },
+        "rating": {
+          "datatype": "float"
+        }
+      },
+      "required": ["id", "name"],
+      "primaryKey": "id",
+      "pathRangeIndex": ["id", "rating"],
+      "wordLexicon": ["description"]
+    }
+  }
+}

ml-data-hub-plugin/src/test/resources/tde-template/Order1.entity.json

@@ -0,0 +1,90 @@
+{
+  "info": {
+    "title": "Order1",
+    "description": "DHF Order1",
+    "version": "1.0.0"
+  },
+  "definitions": {
+    "Order1": {
+      "properties": {
+        "id": {
+          "datatype": "int"
+        },
+        "purchasedItems": {
+          "datatype": "array",
+          "items": {
+            "$ref": "#/definitions/Item1"
+          }
+        },
+        "customer": {
+          "$ref": "#/definitions/Customer1"
+        },
+        "transactionDateTime": {
+          "datatype": "dateTime"
+        },
+        "totalCost": {
+          "datatype": "double"
+        }
+      },
+      "required": [
+        "id",
+        "transactionDateTime",
+        "totalCost"
+      ],
+      "primaryKey": "id",
+      "pathRangeIndex": [
+        "id",
+        "totalCost"
+      ]
+    },
+    "Customer1": {
+      "properties": {
+        "id": {
+          "datatype": "int"
+        },
+        "name": {
+          "datatype": "string"
+        }
+      },
+      "required": [
+        "id",
+        "name"
+      ],
+      "primaryKey": "id",
+      "pii": [
+        "name"
+      ],
+      "pathRangeIndex": [
+        "id"
+      ]
+    },
+    "Item1": {
+      "properties": {
+        "id": {
+          "datatype": "int"
+        },
+        "name": {
+          "datatype": "string"
+        },
+        "description": {
+          "datatype": "string"
+        },
+        "rating": {
+          "datatype": "float"
+        }
+      },
+      "required": [
+        "id",
+        "name"
+      ],
+      "primaryKey": "id",
+      "pathRangeIndex": [
+        "id",
+        "rating"
+      ],
+      "wordLexicon": [
+        "description"
+      ]
+    }
+  }
+}

ml-data-hub-plugin/src/test/resources/tde-template/e2eentity.entity.json

@@ -0,0 +1,28 @@
+{
+  "info" : {
+    "title" : "e2eentity",
+    "version" : "0.0.1"
+  },
+  "definitions" : {
+    "e2eentity" : {
+      "required" : [ ],
+      "primaryKey": "id",
+      "rangeIndex" : [ "name" ],
+      "elementRangeIndex": ["id"],
+      "wordLexicon" : [ ],
+      "properties" : {
+        "id": {
+          "datatype": "string",
+          "collation" : "http://marklogic.com/collation/codepoint"
+        },
+        "name" : {
+          "datatype" : "string",
+          "collation" : "http://marklogic.com/collation/codepoint"
+        },
+        "salary": {
+          "datatype": "decimal"
+        }
+      }
+    }
+  }
+}