diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommand.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommand.java index 55ce151de7..3b35f0ee34 100644 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommand.java +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommand.java @@ -23,10 +23,14 @@ import com.marklogic.client.ext.es.EntityServicesManager; import com.marklogic.client.ext.es.GeneratedCode; import com.marklogic.hub.HubConfig; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; import java.nio.file.Path; import java.util.*; import java.util.function.Function; @@ -70,7 +74,24 @@ public void execute(CommandContext context) { entityNameFileMap.keySet(), hubConfig.getAppConfig().getSchemasPath()); for (File f : entityNameFileMap.values()) { - GeneratedCode code = loadModelDefinition(request, f, mgr); + File esModel; + try { + //Write the ES model to a temp file + esModel = File.createTempFile("es-", f.getName()); + FileUtils.writeStringToFile(esModel, generateModel(f)); + } catch (IOException e) { + throw new RuntimeException("Unable to generate ES model"); + } + + GeneratedCode code; + try { + code = loadModelDefinition(request, esModel, mgr); + } catch (RuntimeException e) { + throw new RuntimeException("Unable to read model definition from file: " + f.getAbsolutePath(), e); + } + finally { + FileUtils.deleteQuietly(esModel); + } generateExtractionTemplate(appConfig, code); } @@ -83,6 +104,14 @@ public void execute(CommandContext context) { } + //Method to obtain es-style model + private String generateModel(File f) { + String xquery = "import module namespace hent = \"http://marklogic.com/data-hub/hub-entities\"\n" + + "at \"/data-hub/4/impl/hub-entities.xqy\";\n" + + String.format("hent:get-model(\"%s\")", extactEntityNameFromFilename(f.getName()).get()); + return hubConfig.newStagingClient().newServerEval().xquery(xquery).eval().next().getString(); + } + public String getEntityNames() { return entityNames; } diff --git a/ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/GenerateTDETemplateFromEntityTaskTest.groovy b/ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/GenerateTDETemplateFromEntityTaskTest.groovy new file mode 100644 index 0000000000..aa1033838d --- /dev/null +++ b/ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/GenerateTDETemplateFromEntityTaskTest.groovy @@ -0,0 +1,65 @@ +package com.marklogic.gradle.task + +import com.marklogic.hub.HubConfig +import org.apache.commons.io.FileUtils +import org.gradle.testkit.runner.UnexpectedBuildFailure + +import java.nio.file.Path +import java.nio.file.Paths + +import static org.gradle.testkit.runner.TaskOutcome.SUCCESS + +class GenerateTDETemplateFromEntityTaskTest extends BaseTest { + def setupSpec() { + createGradleFiles() + runTask('hubInit') + clearDatabases(HubConfig.DEFAULT_STAGING_NAME, HubConfig.DEFAULT_FINAL_NAME, HubConfig.DEFAULT_JOB_NAME); + } + + def "GenerateTDETEmplates"() { + given: + def pluginDir = Paths.get(hubConfig().hubProject.projectDirString).resolve("plugins") + + //DHF style nested entities (references in separate file) + def entitiesDir1 = pluginDir.resolve("entities").resolve("Order") + def entitiesDir2 = pluginDir.resolve("entities").resolve("Item") + def entitiesDir3 = pluginDir.resolve("entities").resolve("Customer") + + entitiesDir1.toFile().mkdirs() + entitiesDir2.toFile().mkdirs() + entitiesDir3.toFile().mkdirs() + + //ES style nested entities(references in the same file) + + def entitiesDir4 = pluginDir.resolve("entities").resolve("Entity1") + entitiesDir4.toFile().mkdirs() + + //DHF style without references + + def entitiesDir5 = pluginDir.resolve("entities").resolve("e2eentity") + entitiesDir5.toFile().mkdirs() + + FileUtils.copyFile(new File("src/test/resources/tde-template/Order.entity.json"), entitiesDir1.resolve('Order.entity.json').toFile()) + FileUtils.copyFile(new File("src/test/resources/tde-template/Item.entity.json"), entitiesDir2.resolve('Item.entity.json').toFile()) + FileUtils.copyFile(new File("src/test/resources/tde-template/Customer.entity.json"), entitiesDir3.resolve('Customer.entity.json').toFile()) + + FileUtils.copyFile(new File("src/test/resources/tde-template/Order1.entity.json"), entitiesDir4.resolve('Order1.entity.json').toFile()) + FileUtils.copyFile(new File("src/test/resources/tde-template/e2eentity.entity.json"), entitiesDir5.resolve('e2eentity.entity.json').toFile()) + + runTask("hubDeployUserArtifacts") + + when: + def result = runTask('hubGenerateTDETemplates', '-PentityNames=Order,Order1,e2eentity,nonexistentEntity') + + then: + notThrown(UnexpectedBuildFailure) + result.task(":hubGenerateTDETemplates").outcome == SUCCESS + + Path tdePath = Paths.get(testProjectDir.root.toString(), "src", "main", "ml-schemas", "tde") + tdePath.resolve("Order-1.0.0.tdex").toFile().exists() == true + tdePath.resolve("Order1-1.0.0.tdex").toFile().exists() == true + tdePath.resolve("e2eentity-0.0.1.tdex").toFile().exists() == true + tdePath.resolve("nonexistentEntity-0.0.1.tdex").toFile().exists() == false + + } +} diff --git a/ml-data-hub-plugin/src/test/resources/tde-template/Customer.entity.json b/ml-data-hub-plugin/src/test/resources/tde-template/Customer.entity.json new file mode 100644 index 0000000000..02bf9b5dad --- /dev/null +++ b/ml-data-hub-plugin/src/test/resources/tde-template/Customer.entity.json @@ -0,0 +1,23 @@ +{ + "info": { + "title": "Customer", + "description": "DHF Customer", + "version": "1.0.0" + }, + "definitions": { + "Customer": { + "properties": { + "id": { + "datatype": "int" + }, + "name": { + "datatype": "string" + } + }, + "required": ["id", "name"], + "primaryKey": "id", + "pii": ["name"], + "pathRangeIndex": ["id"] + } + } +} diff --git a/ml-data-hub-plugin/src/test/resources/tde-template/Item.entity.json b/ml-data-hub-plugin/src/test/resources/tde-template/Item.entity.json new file mode 100644 index 0000000000..171e9d31c2 --- /dev/null +++ b/ml-data-hub-plugin/src/test/resources/tde-template/Item.entity.json @@ -0,0 +1,29 @@ +{ + "info": { + "title": "Item", + "description": "DHF Item", + "version": "1.0.0" + }, + "definitions": { + "Item": { + "properties": { + "id": { + "datatype": "int" + }, + "name": { + "datatype": "string" + }, + "description": { + "datatype": "string" + }, + "rating": { + "datatype": "float" + } + }, + "required": ["id", "name"], + "primaryKey": "id", + "pathRangeIndex": ["id", "rating"], + "wordLexicon": ["description"] + } + } +} diff --git a/ml-data-hub-plugin/src/test/resources/tde-template/Order.entity.json b/ml-data-hub-plugin/src/test/resources/tde-template/Order.entity.json new file mode 100644 index 0000000000..d9e822297b --- /dev/null +++ b/ml-data-hub-plugin/src/test/resources/tde-template/Order.entity.json @@ -0,0 +1,68 @@ +{ + "info": { + "title": "Order", + "description": "DHF Order", + "version": "1.0.0" + }, + "definitions": { + "Order": { + "properties": { + "id": { + "datatype": "int" + }, + "purchasedItems": { + "datatype": "array", + "items": { + "$ref": "#/definitions/Item" + } + }, + "customer": { + "$ref": "#/definitions/Customer" + }, + "transactionDateTime": { + "datatype": "dateTime" + }, + "totalCost": { + "datatype": "double" + } + }, + "required": ["id", "transactionDateTime", "totalCost"], + "primaryKey": "id", + "pathRangeIndex": ["id", "totalCost"] + }, + "Customer": { + "properties": { + "id": { + "datatype": "int" + }, + "name": { + "datatype": "string" + } + }, + "required": ["id", "name"], + "primaryKey": "id", + "pii": ["name"], + "pathRangeIndex": ["id"] + }, + "Item": { + "properties": { + "id": { + "datatype": "int" + }, + "name": { + "datatype": "string" + }, + "description": { + "datatype": "string" + }, + "rating": { + "datatype": "float" + } + }, + "required": ["id", "name"], + "primaryKey": "id", + "pathRangeIndex": ["id", "rating"], + "wordLexicon": ["description"] + } + } +} diff --git a/ml-data-hub-plugin/src/test/resources/tde-template/Order1.entity.json b/ml-data-hub-plugin/src/test/resources/tde-template/Order1.entity.json new file mode 100644 index 0000000000..9a7adfb075 --- /dev/null +++ b/ml-data-hub-plugin/src/test/resources/tde-template/Order1.entity.json @@ -0,0 +1,90 @@ +{ + "info": { + "title": "Order1", + "description": "DHF Order1", + "version": "1.0.0" + }, + "definitions": { + "Order1": { + "properties": { + "id": { + "datatype": "int" + }, + "purchasedItems": { + "datatype": "array", + "items": { + "$ref": "#/definitions/Item1" + } + }, + "customer": { + "$ref": "#/definitions/Customer1" + }, + "transactionDateTime": { + "datatype": "dateTime" + }, + "totalCost": { + "datatype": "double" + } + }, + "required": [ + "id", + "transactionDateTime", + "totalCost" + ], + "primaryKey": "id", + "pathRangeIndex": [ + "id", + "totalCost" + ] + }, + "Customer1": { + "properties": { + "id": { + "datatype": "int" + }, + "name": { + "datatype": "string" + } + }, + "required": [ + "id", + "name" + ], + "primaryKey": "id", + "pii": [ + "name" + ], + "pathRangeIndex": [ + "id" + ] + }, + "Item1": { + "properties": { + "id": { + "datatype": "int" + }, + "name": { + "datatype": "string" + }, + "description": { + "datatype": "string" + }, + "rating": { + "datatype": "float" + } + }, + "required": [ + "id", + "name" + ], + "primaryKey": "id", + "pathRangeIndex": [ + "id", + "rating" + ], + "wordLexicon": [ + "description" + ] + } + } +} diff --git a/ml-data-hub-plugin/src/test/resources/tde-template/e2eentity.entity.json b/ml-data-hub-plugin/src/test/resources/tde-template/e2eentity.entity.json new file mode 100644 index 0000000000..eb752f63b4 --- /dev/null +++ b/ml-data-hub-plugin/src/test/resources/tde-template/e2eentity.entity.json @@ -0,0 +1,28 @@ +{ + "info" : { + "title" : "e2eentity", + "version" : "0.0.1" + }, + "definitions" : { + "e2eentity" : { + "required" : [ ], + "primaryKey": "id", + "rangeIndex" : [ "name" ], + "elementRangeIndex": ["id"], + "wordLexicon" : [ ], + "properties" : { + "id": { + "datatype": "string", + "collation" : "http://marklogic.com/collation/codepoint" + }, + "name" : { + "datatype" : "string", + "collation" : "http://marklogic.com/collation/codepoint" + }, + "salary": { + "datatype": "decimal" + } + } + } + } +}