From f0e62339b2434a6cf676f7e84b7fa5925f9ddd1e Mon Sep 17 00:00:00 2001 From: Dermot Smyth Date: Fri, 26 Jan 2018 12:19:57 +0100 Subject: [PATCH 1/3] #551 Create gradle command to generate a TDE Template --- .../GenerateHubTDETemplateCommand.java | 148 ++++++++++++++++ .../GenerateHubTDETemplateCommandTest.java | 160 ++++++++++++++++++ .../generate-tde-template/myfirst.entity.json | 28 +++ .../mysecond.entity.json | 28 +++ .../com/marklogic/gradle/DataHubPlugin.groovy | 3 + .../GenerateTDETemplateFromEntityTask.groovy | 22 +++ 6 files changed, 389 insertions(+) create mode 100644 marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommand.java create mode 100644 marklogic-data-hub/src/test/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommandTest.java create mode 100644 marklogic-data-hub/src/test/resources/scaffolding-test/generate-tde-template/myfirst.entity.json create mode 100644 marklogic-data-hub/src/test/resources/scaffolding-test/generate-tde-template/mysecond.entity.json create mode 100644 ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/GenerateTDETemplateFromEntityTask.groovy diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommand.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommand.java new file mode 100644 index 0000000000..25db67e7d5 --- /dev/null +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommand.java @@ -0,0 +1,148 @@ +package com.marklogic.hub.deploy.commands; + +import com.marklogic.appdeployer.AppConfig; +import com.marklogic.appdeployer.command.CommandContext; +import com.marklogic.appdeployer.command.es.GenerateModelArtifactsCommand; +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.ext.es.CodeGenerationRequest; +import com.marklogic.client.ext.es.EntityServicesManager; +import com.marklogic.client.ext.es.GeneratedCode; +import com.marklogic.hub.HubConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.file.Path; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; +import static java.util.stream.Collectors.*; + + +public class GenerateHubTDETemplateCommand extends GenerateModelArtifactsCommand { + private static final String ENTITY_FILE_EXTENSION = ".entity.json"; + protected final Logger logger = LoggerFactory.getLogger(this.getClass()); + + private HubConfig hubConfig; + + private String entityNames; + + public GenerateHubTDETemplateCommand(HubConfig hubConfig) { + this.hubConfig = hubConfig; + } + + @Override + public void execute(CommandContext context) { + AppConfig appConfig = context.getAppConfig(); + DatabaseClient client = appConfig.newDatabaseClient(); + EntityServicesManager mgr = new EntityServicesManager(client); + + CodeGenerationRequest request = createCodeGenerationRequest(); + + List entityFiles = findEntityFiles(); + + if (!entityFiles.isEmpty()) { + //create map of entity name -> entity definition file + Map entityNameFileMap = createEntityNameFileMap(entityFiles); + + logger.debug("Found the following entities->files: {} " + entityNameFileMap); + + filterEntities(entityNameFileMap); + + if (!entityNameFileMap.isEmpty()) { + logger.warn("About to generate a template for the following entities: {} into directory {} ", + entityNameFileMap.keySet(), hubConfig.getAppConfig().getSchemasPath()); + + for (File f : entityNameFileMap.values()) { + GeneratedCode code = loadModelDefinition(request, f, mgr); + generateExtractionTemplate(appConfig, code); + } + + } + + } else { + logger.info("No data hub entity files found under {} or its sub-directories.", + hubConfig.getHubEntitiesDir()); + } + + } + + public String getEntityNames() { + return entityNames; + } + + public void setEntityNames(String entityNames) { + this.entityNames = entityNames; + } + + protected void filterEntities(Map entityNameFileMap) { + Set entityNameFileMapKeys = entityNameFileMap.keySet(); + + //filter on entityNames parameter if specified + if (entityNames!=null&&!entityNames.isEmpty()) { + List entityNamesAsList = Arrays.asList(entityNames.split(",")); + logger.info("Entities specified for TDE Generation: {} " + entityNamesAsList); + + //this will only keep keys in the map that are also in the entityNamesAsList + entityNameFileMapKeys.retainAll(entityNamesAsList); + + if (entityNameFileMapKeys.isEmpty()) { + logger.warn("No entities files found under {} or its sub-directories with the entity name(s) {}", hubConfig.getHubEntitiesDir(),entityNamesAsList); + } + } + } + + protected static Map createEntityNameFileMap(List entityFiles) { + if (entityFiles==null) { + return Collections.emptyMap(); + } + return entityFiles.stream().collect( + toMap(extractEntityNameFunction(),Function.identity())); + } + + protected List findEntityFiles() { + List entities = new ArrayList<>(); + Path entitiesPath = hubConfig.getHubEntitiesDir(); + File[] entityDirectories = entitiesPath.toFile().listFiles(pathname -> pathname.isDirectory() && !pathname.isHidden()); + List entityNames; + if (entityDirectories != null) { + entityNames = Arrays.stream(entityDirectories) + .map(file -> file.getName()) + .collect(Collectors.toList()); + for (String entityName : entityNames) { + File[] entityDefs = entitiesPath.resolve(entityName).toFile().listFiles((dir, name) -> name.endsWith(ENTITY_FILE_EXTENSION)); + if (entityDefs!=null) { + entities.addAll(Arrays.asList(entityDefs)); + } + } + } + return entities; + } + + protected static Optional extactEntityNameFromFilename(String filename) { + if (filename==null || filename.trim().isEmpty()) { + return Optional.of(null); + } + int index = filename.indexOf(ENTITY_FILE_EXTENSION); + if (index<0) { + //not found + return Optional.of(null); + } + return Optional.of(filename.substring(0,index)); + } + + private static Function extractEntityNameFunction() { + Function fileName = File::getName; + return fileName.andThen(name -> extactEntityNameFromFilename(name).get()); + } + + private static final CodeGenerationRequest createCodeGenerationRequest() { + CodeGenerationRequest request = new CodeGenerationRequest(); + request.setGenerateExtractionTemplate(true); + request.setGenerateDatabaseProperties(false); + request.setGenerateInstanceConverter(false); + request.setGenerateSchema(false); + request.setGenerateSearchOptions(false); + return request; + } +} diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommandTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommandTest.java new file mode 100644 index 0000000000..58f087aecf --- /dev/null +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommandTest.java @@ -0,0 +1,160 @@ +package com.marklogic.hub.deploy.commands; + +import com.marklogic.appdeployer.command.CommandContext; +import com.marklogic.hub.HubConfig; +import com.marklogic.hub.HubTestBase; +import com.marklogic.hub.scaffold.Scaffolding; +import com.marklogic.hub.util.FileUtil; +import org.custommonkey.xmlunit.XMLUnit; +import org.easymock.EasyMock; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.xml.sax.SAXException; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class GenerateHubTDETemplateCommandTest extends HubTestBase { + static Path projectPath = Paths.get(PROJECT_PATH).toAbsolutePath(); + private static File projectDir = projectPath.toFile(); + private static final String RESOURCES_DIR = "scaffolding-test/generate-tde-template/"; + + GenerateHubTDETemplateCommand GenerateHubTDETemplateCommand; + + @Before + public void setup() { + GenerateHubTDETemplateCommand = new GenerateHubTDETemplateCommand(getHubConfig()); + deleteProjectDir(); + } + + + @Before + public void clearDirs() { + deleteProjectDir(); + createProjectDir(); + } + + @AfterClass + public static void teardown() throws IOException { + deleteProjectDir(); + } + + private void installEntity(String entityName) { + Scaffolding scaffolding = new Scaffolding(projectDir.toString(), finalClient); + Path entityDir = scaffolding.getEntityDir(entityName); + entityDir.toFile().mkdirs(); + assertTrue(entityDir.toFile().exists()); + FileUtil.copy(getResourceStream(RESOURCES_DIR + entityName + ".entity.json"), entityDir.resolve(entityName + ".entity.json").toFile()); + } + + @Test + public void testFindEntityFilesNoEntityFiles() { + List entityFiles = GenerateHubTDETemplateCommand.findEntityFiles(); + assertEquals("Expected to find no entity files",0,entityFiles.size()); + } + + @Test + public void testFindEntityFilesOneEntityFiles() { + installEntity("myfirst"); + List entityFiles = GenerateHubTDETemplateCommand.findEntityFiles(); + assertEquals("Expected to find one entity file",1,entityFiles.size()); + } + + @Test + public void testFindEntityFilesTwoEntityFiles() { + installEntity("myfirst"); + installEntity("mysecond"); + List entityFiles = GenerateHubTDETemplateCommand.findEntityFiles(); + assertEquals("Expected to find two entity files",2,entityFiles.size()); + } + + @Test + public void testCreateEntityNameFileMapWithNoEntityFiles() { + Map entityNameFileMap = GenerateHubTDETemplateCommand.createEntityNameFileMap(null); + assertEquals("Expected to find no entity files",0,entityNameFileMap.size()); + + entityNameFileMap = GenerateHubTDETemplateCommand.createEntityNameFileMap(new ArrayList<>()); + assertEquals("Expected to find no entity files",0,entityNameFileMap.size()); + } + + @Test + public void testCreateEntityNameFileMapWithTwoEntityFiles() { + installEntity("myfirst"); + installEntity("mysecond"); + List entityFiles = GenerateHubTDETemplateCommand.findEntityFiles(); + + Map entityNameFileMap = GenerateHubTDETemplateCommand.createEntityNameFileMap(entityFiles); + assertTrue("Does not contain myfirst entity",entityNameFileMap.containsKey("myfirst")); + assertTrue("Does not contain mysecond entity",entityNameFileMap.containsKey("mysecond")); + + //assertEquals("Expected to find no entity files",2,entityNameFileMap.size()); + } + + @Test + public void testFilterSingleEntityWithTwoEntityFiles() { + installEntity("myfirst"); + installEntity("mysecond"); + GenerateHubTDETemplateCommand.setEntityNames("myfirst"); + + List entityFiles = GenerateHubTDETemplateCommand.findEntityFiles(); + + Map entityNameFileMap = GenerateHubTDETemplateCommand.createEntityNameFileMap(entityFiles); + assertTrue("Does not contain myfirst entity",entityNameFileMap.containsKey("myfirst")); + assertTrue("Does not contain mysecond entity",entityNameFileMap.containsKey("mysecond")); + + GenerateHubTDETemplateCommand.filterEntities(entityNameFileMap); + assertTrue("Does not contain myfirst entity",entityNameFileMap.containsKey("myfirst")); + assertFalse("Does contain mysecond entity",entityNameFileMap.containsKey("mysecond")); + } + + @Test + public void testFilterTwoEntityWithTwoEntityFiles() { + installEntity("myfirst"); + installEntity("mysecond"); + GenerateHubTDETemplateCommand.setEntityNames("myfirst,mysecond"); + + List entityFiles = GenerateHubTDETemplateCommand.findEntityFiles(); + + Map entityNameFileMap = GenerateHubTDETemplateCommand.createEntityNameFileMap(entityFiles); + assertTrue("Does not contain myfirst entity",entityNameFileMap.containsKey("myfirst")); + assertTrue("Does not contain mysecond entity",entityNameFileMap.containsKey("mysecond")); + + GenerateHubTDETemplateCommand.filterEntities(entityNameFileMap); + assertTrue("Does not contain myfirst entity",entityNameFileMap.containsKey("myfirst")); + assertTrue("Does not contain mysecond entity",entityNameFileMap.containsKey("mysecond")); + } + + @Test + public void testFilterIncorrectEntities() { + installEntity("myfirst"); + installEntity("mysecond"); + GenerateHubTDETemplateCommand.setEntityNames("XCXZ,ZXCXZC"); + + List entityFiles = GenerateHubTDETemplateCommand.findEntityFiles(); + + Map entityNameFileMap = GenerateHubTDETemplateCommand.createEntityNameFileMap(entityFiles); + assertTrue("Does not contain myfirst entity",entityNameFileMap.containsKey("myfirst")); + assertTrue("Does not contain mysecond entity",entityNameFileMap.containsKey("mysecond")); + + GenerateHubTDETemplateCommand.filterEntities(entityNameFileMap); + assertFalse("Does contain myfirst entity",entityNameFileMap.containsKey("myfirst")); + assertFalse("Does contain mysecond entity",entityNameFileMap.containsKey("mysecond")); + } + + @Test + public void testExtactEntityNameFromFilename() { + assertEquals("Could not extract entity ABC", "ABC",GenerateHubTDETemplateCommand.extactEntityNameFromFilename("ABC.entity.json").get()); + } +} diff --git a/marklogic-data-hub/src/test/resources/scaffolding-test/generate-tde-template/myfirst.entity.json b/marklogic-data-hub/src/test/resources/scaffolding-test/generate-tde-template/myfirst.entity.json new file mode 100644 index 0000000000..457a5c6168 --- /dev/null +++ b/marklogic-data-hub/src/test/resources/scaffolding-test/generate-tde-template/myfirst.entity.json @@ -0,0 +1,28 @@ +{ + "info" : { + "title" : "myfirst", + "version" : "0.0.1" + }, + "definitions" : { + "Employee" : { + "required" : [ ], + "primaryKey": "id", + "rangeIndex" : [ "name" ], + "elementRangeIndex": ["salary"], + "wordLexicon" : [ ], + "properties" : { + "id": { + "datatype": "string", + "collation" : "http://marklogic.com/collation/codepoint" + }, + "name" : { + "datatype" : "string", + "collation" : "http://marklogic.com/collation/codepoint" + }, + "salary": { + "datatype": "decimal" + } + } + } + } +} diff --git a/marklogic-data-hub/src/test/resources/scaffolding-test/generate-tde-template/mysecond.entity.json b/marklogic-data-hub/src/test/resources/scaffolding-test/generate-tde-template/mysecond.entity.json new file mode 100644 index 0000000000..42fe0319dc --- /dev/null +++ b/marklogic-data-hub/src/test/resources/scaffolding-test/generate-tde-template/mysecond.entity.json @@ -0,0 +1,28 @@ +{ + "info" : { + "title" : "mysecond", + "version" : "0.0.1" + }, + "definitions" : { + "Employee" : { + "required" : [ ], + "primaryKey": "id", + "rangeIndex" : [ "name" ], + "elementRangeIndex": ["salary"], + "wordLexicon" : [ ], + "properties" : { + "id": { + "datatype": "string", + "collation" : "http://marklogic.com/collation/codepoint" + }, + "name" : { + "datatype" : "string", + "collation" : "http://marklogic.com/collation/codepoint" + }, + "salary": { + "datatype": "decimal" + } + } + } + } +} diff --git a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/DataHubPlugin.groovy b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/DataHubPlugin.groovy index 0d2dfd804c..912be3cd39 100644 --- a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/DataHubPlugin.groovy +++ b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/DataHubPlugin.groovy @@ -51,6 +51,9 @@ class DataHubPlugin implements Plugin { project.task("hubCreateEntity", group: scaffoldGroup, type: CreateEntityTask) project.task("hubCreateHarmonizeFlow", group: scaffoldGroup, type: CreateHarmonizeFlowTask) project.task("hubCreateInputFlow", group: scaffoldGroup, type: CreateInputFlowTask) + project.task("hubGenerateTDETemplates", group: scaffoldGroup, type: GenerateTDETemplateFromEntityTask, + description: "Generates TDE Templates from the entity definition files. It is possible to only generate TDE templates" + + " for specific entities by setting the (comma separated) project property 'entityNames'. E.g. -PentityNames=Entity1,Entity2") project.tasks.replace("mlLoadModules", DeployUserModulesTask) project.tasks.replace("mlWatch", HubWatchTask) diff --git a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/GenerateTDETemplateFromEntityTask.groovy b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/GenerateTDETemplateFromEntityTask.groovy new file mode 100644 index 0000000000..b9604710fc --- /dev/null +++ b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/GenerateTDETemplateFromEntityTask.groovy @@ -0,0 +1,22 @@ +package com.marklogic.gradle.task + +import com.marklogic.hub.deploy.commands.GenerateHubTDETemplateCommand +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.TaskAction + +class GenerateTDETemplateFromEntityTask extends HubTask { + + @Input + public String entityNames + + @TaskAction + void generateTDETEmplates() { + def cmd = new GenerateHubTDETemplateCommand(getHubConfig()) + if (entityNames == null) { + entityNames = project.hasProperty("entityNames") ? project.property("entityNames") : null + } + cmd.setEntityNames(entityNames) + cmd.execute(getCommandContext()) + } + +} From 0aed60d34737e61d2af877f368d80818596ee8b7 Mon Sep 17 00:00:00 2001 From: Dermot Smyth Date: Mon, 29 Jan 2018 10:14:35 +0100 Subject: [PATCH 2/3] upgraded gradle and ml-app-deployer to 3.4.0 --- marklogic-data-hub/build.gradle | 5 +++-- ml-data-hub-plugin/build.gradle | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/marklogic-data-hub/build.gradle b/marklogic-data-hub/build.gradle index 6fa70b9a1a..ae37bb1c55 100644 --- a/marklogic-data-hub/build.gradle +++ b/marklogic-data-hub/build.gradle @@ -3,7 +3,7 @@ plugins { id 'java' id 'maven-publish' id 'com.jfrog.bintray' version '1.7.2' - id 'com.marklogic.ml-gradle' version '3.3.1' + id 'com.marklogic.ml-gradle' version '3.4.0' id 'com.moowork.node' version '1.1.1' } @@ -13,6 +13,7 @@ repositories { maven { url 'https://developer.marklogic.com/maven2/' } } + group = 'com.marklogic' sourceCompatibility = 1.8 @@ -26,7 +27,7 @@ ext.junitJupiterVersion = '5.0.0-RC3' dependencies { compile 'com.marklogic:marklogic-client-api:4.0.2' compile 'com.marklogic:mlcp-util:0.3.0' - compile 'com.marklogic:ml-app-deployer:3.3.1' + compile 'com.marklogic:ml-app-deployer:3.4.0' compile 'commons-io:commons-io:2.4' compile 'org.apache.commons:commons-text:1.1' diff --git a/ml-data-hub-plugin/build.gradle b/ml-data-hub-plugin/build.gradle index adb61c586c..d4c718e297 100644 --- a/ml-data-hub-plugin/build.gradle +++ b/ml-data-hub-plugin/build.gradle @@ -30,7 +30,7 @@ repositories { dependencies { compile gradleApi() compile project(':marklogic-data-hub') - compile ('com.marklogic:ml-gradle:3.3.1') + compile ('com.marklogic:ml-gradle:3.4.0') testCompile localGroovy() testCompile gradleTestKit() testCompile 'xmlunit:xmlunit:1.3' From 4b0d95cab0ac023b0058a1ee2eb6eb5ee2bf135c Mon Sep 17 00:00:00 2001 From: Dermot Smyth Date: Mon, 5 Feb 2018 21:07:22 +0100 Subject: [PATCH 3/3] removed whitespace --- marklogic-data-hub/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/marklogic-data-hub/build.gradle b/marklogic-data-hub/build.gradle index abc8144790..2d390696a2 100644 --- a/marklogic-data-hub/build.gradle +++ b/marklogic-data-hub/build.gradle @@ -13,7 +13,6 @@ repositories { maven { url 'https://developer.marklogic.com/maven2/' } } - group = 'com.marklogic' sourceCompatibility = 1.8