diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommand.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommand.java new file mode 100644 index 0000000000..25db67e7d5 --- /dev/null +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommand.java @@ -0,0 +1,148 @@ +package com.marklogic.hub.deploy.commands; + +import com.marklogic.appdeployer.AppConfig; +import com.marklogic.appdeployer.command.CommandContext; +import com.marklogic.appdeployer.command.es.GenerateModelArtifactsCommand; +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.ext.es.CodeGenerationRequest; +import com.marklogic.client.ext.es.EntityServicesManager; +import com.marklogic.client.ext.es.GeneratedCode; +import com.marklogic.hub.HubConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.file.Path; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; +import static java.util.stream.Collectors.*; + + +public class GenerateHubTDETemplateCommand extends GenerateModelArtifactsCommand { + private static final String ENTITY_FILE_EXTENSION = ".entity.json"; + protected final Logger logger = LoggerFactory.getLogger(this.getClass()); + + private HubConfig hubConfig; + + private String entityNames; + + public GenerateHubTDETemplateCommand(HubConfig hubConfig) { + this.hubConfig = hubConfig; + } + + @Override + public void execute(CommandContext context) { + AppConfig appConfig = context.getAppConfig(); + DatabaseClient client = appConfig.newDatabaseClient(); + EntityServicesManager mgr = new EntityServicesManager(client); + + CodeGenerationRequest request = createCodeGenerationRequest(); + + List entityFiles = findEntityFiles(); + + if (!entityFiles.isEmpty()) { + //create map of entity name -> entity definition file + Map entityNameFileMap = createEntityNameFileMap(entityFiles); + + logger.debug("Found the following entities->files: {} " + entityNameFileMap); + + filterEntities(entityNameFileMap); + + if (!entityNameFileMap.isEmpty()) { + logger.warn("About to generate a template for the following entities: {} into directory {} ", + entityNameFileMap.keySet(), hubConfig.getAppConfig().getSchemasPath()); + + for (File f : entityNameFileMap.values()) { + GeneratedCode code = loadModelDefinition(request, f, mgr); + generateExtractionTemplate(appConfig, code); + } + + } + + } else { + logger.info("No data hub entity files found under {} or its sub-directories.", + hubConfig.getHubEntitiesDir()); + } + + } + + public String getEntityNames() { + return entityNames; + } + + public void setEntityNames(String entityNames) { + this.entityNames = entityNames; + } + + protected void filterEntities(Map entityNameFileMap) { + Set entityNameFileMapKeys = entityNameFileMap.keySet(); + + //filter on entityNames parameter if specified + if (entityNames!=null&&!entityNames.isEmpty()) { + List entityNamesAsList = Arrays.asList(entityNames.split(",")); + logger.info("Entities specified for TDE Generation: {} " + entityNamesAsList); + + //this will only keep keys in the map that are also in the entityNamesAsList + entityNameFileMapKeys.retainAll(entityNamesAsList); + + if (entityNameFileMapKeys.isEmpty()) { + logger.warn("No entities files found under {} or its sub-directories with the entity name(s) {}", hubConfig.getHubEntitiesDir(),entityNamesAsList); + } + } + } + + protected static Map createEntityNameFileMap(List entityFiles) { + if (entityFiles==null) { + return Collections.emptyMap(); + } + return entityFiles.stream().collect( + toMap(extractEntityNameFunction(),Function.identity())); + } + + protected List findEntityFiles() { + List entities = new ArrayList<>(); + Path entitiesPath = hubConfig.getHubEntitiesDir(); + File[] entityDirectories = entitiesPath.toFile().listFiles(pathname -> pathname.isDirectory() && !pathname.isHidden()); + List entityNames; + if (entityDirectories != null) { + entityNames = Arrays.stream(entityDirectories) + .map(file -> file.getName()) + .collect(Collectors.toList()); + for (String entityName : entityNames) { + File[] entityDefs = entitiesPath.resolve(entityName).toFile().listFiles((dir, name) -> name.endsWith(ENTITY_FILE_EXTENSION)); + if (entityDefs!=null) { + entities.addAll(Arrays.asList(entityDefs)); + } + } + } + return entities; + } + + protected static Optional extactEntityNameFromFilename(String filename) { + if (filename==null || filename.trim().isEmpty()) { + return Optional.of(null); + } + int index = filename.indexOf(ENTITY_FILE_EXTENSION); + if (index<0) { + //not found + return Optional.of(null); + } + return Optional.of(filename.substring(0,index)); + } + + private static Function extractEntityNameFunction() { + Function fileName = File::getName; + return fileName.andThen(name -> extactEntityNameFromFilename(name).get()); + } + + private static final CodeGenerationRequest createCodeGenerationRequest() { + CodeGenerationRequest request = new CodeGenerationRequest(); + request.setGenerateExtractionTemplate(true); + request.setGenerateDatabaseProperties(false); + request.setGenerateInstanceConverter(false); + request.setGenerateSchema(false); + request.setGenerateSearchOptions(false); + return request; + } +} diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommandTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommandTest.java new file mode 100644 index 0000000000..58f087aecf --- /dev/null +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommandTest.java @@ -0,0 +1,160 @@ +package com.marklogic.hub.deploy.commands; + +import com.marklogic.appdeployer.command.CommandContext; +import com.marklogic.hub.HubConfig; +import com.marklogic.hub.HubTestBase; +import com.marklogic.hub.scaffold.Scaffolding; +import com.marklogic.hub.util.FileUtil; +import org.custommonkey.xmlunit.XMLUnit; +import org.easymock.EasyMock; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.xml.sax.SAXException; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class GenerateHubTDETemplateCommandTest extends HubTestBase { + static Path projectPath = Paths.get(PROJECT_PATH).toAbsolutePath(); + private static File projectDir = projectPath.toFile(); + private static final String RESOURCES_DIR = "scaffolding-test/generate-tde-template/"; + + GenerateHubTDETemplateCommand GenerateHubTDETemplateCommand; + + @Before + public void setup() { + GenerateHubTDETemplateCommand = new GenerateHubTDETemplateCommand(getHubConfig()); + deleteProjectDir(); + } + + + @Before + public void clearDirs() { + deleteProjectDir(); + createProjectDir(); + } + + @AfterClass + public static void teardown() throws IOException { + deleteProjectDir(); + } + + private void installEntity(String entityName) { + Scaffolding scaffolding = new Scaffolding(projectDir.toString(), finalClient); + Path entityDir = scaffolding.getEntityDir(entityName); + entityDir.toFile().mkdirs(); + assertTrue(entityDir.toFile().exists()); + FileUtil.copy(getResourceStream(RESOURCES_DIR + entityName + ".entity.json"), entityDir.resolve(entityName + ".entity.json").toFile()); + } + + @Test + public void testFindEntityFilesNoEntityFiles() { + List entityFiles = GenerateHubTDETemplateCommand.findEntityFiles(); + assertEquals("Expected to find no entity files",0,entityFiles.size()); + } + + @Test + public void testFindEntityFilesOneEntityFiles() { + installEntity("myfirst"); + List entityFiles = GenerateHubTDETemplateCommand.findEntityFiles(); + assertEquals("Expected to find one entity file",1,entityFiles.size()); + } + + @Test + public void testFindEntityFilesTwoEntityFiles() { + installEntity("myfirst"); + installEntity("mysecond"); + List entityFiles = GenerateHubTDETemplateCommand.findEntityFiles(); + assertEquals("Expected to find two entity files",2,entityFiles.size()); + } + + @Test + public void testCreateEntityNameFileMapWithNoEntityFiles() { + Map entityNameFileMap = GenerateHubTDETemplateCommand.createEntityNameFileMap(null); + assertEquals("Expected to find no entity files",0,entityNameFileMap.size()); + + entityNameFileMap = GenerateHubTDETemplateCommand.createEntityNameFileMap(new ArrayList<>()); + assertEquals("Expected to find no entity files",0,entityNameFileMap.size()); + } + + @Test + public void testCreateEntityNameFileMapWithTwoEntityFiles() { + installEntity("myfirst"); + installEntity("mysecond"); + List entityFiles = GenerateHubTDETemplateCommand.findEntityFiles(); + + Map entityNameFileMap = GenerateHubTDETemplateCommand.createEntityNameFileMap(entityFiles); + assertTrue("Does not contain myfirst entity",entityNameFileMap.containsKey("myfirst")); + assertTrue("Does not contain mysecond entity",entityNameFileMap.containsKey("mysecond")); + + //assertEquals("Expected to find no entity files",2,entityNameFileMap.size()); + } + + @Test + public void testFilterSingleEntityWithTwoEntityFiles() { + installEntity("myfirst"); + installEntity("mysecond"); + GenerateHubTDETemplateCommand.setEntityNames("myfirst"); + + List entityFiles = GenerateHubTDETemplateCommand.findEntityFiles(); + + Map entityNameFileMap = GenerateHubTDETemplateCommand.createEntityNameFileMap(entityFiles); + assertTrue("Does not contain myfirst entity",entityNameFileMap.containsKey("myfirst")); + assertTrue("Does not contain mysecond entity",entityNameFileMap.containsKey("mysecond")); + + GenerateHubTDETemplateCommand.filterEntities(entityNameFileMap); + assertTrue("Does not contain myfirst entity",entityNameFileMap.containsKey("myfirst")); + assertFalse("Does contain mysecond entity",entityNameFileMap.containsKey("mysecond")); + } + + @Test + public void testFilterTwoEntityWithTwoEntityFiles() { + installEntity("myfirst"); + installEntity("mysecond"); + GenerateHubTDETemplateCommand.setEntityNames("myfirst,mysecond"); + + List entityFiles = GenerateHubTDETemplateCommand.findEntityFiles(); + + Map entityNameFileMap = GenerateHubTDETemplateCommand.createEntityNameFileMap(entityFiles); + assertTrue("Does not contain myfirst entity",entityNameFileMap.containsKey("myfirst")); + assertTrue("Does not contain mysecond entity",entityNameFileMap.containsKey("mysecond")); + + GenerateHubTDETemplateCommand.filterEntities(entityNameFileMap); + assertTrue("Does not contain myfirst entity",entityNameFileMap.containsKey("myfirst")); + assertTrue("Does not contain mysecond entity",entityNameFileMap.containsKey("mysecond")); + } + + @Test + public void testFilterIncorrectEntities() { + installEntity("myfirst"); + installEntity("mysecond"); + GenerateHubTDETemplateCommand.setEntityNames("XCXZ,ZXCXZC"); + + List entityFiles = GenerateHubTDETemplateCommand.findEntityFiles(); + + Map entityNameFileMap = GenerateHubTDETemplateCommand.createEntityNameFileMap(entityFiles); + assertTrue("Does not contain myfirst entity",entityNameFileMap.containsKey("myfirst")); + assertTrue("Does not contain mysecond entity",entityNameFileMap.containsKey("mysecond")); + + GenerateHubTDETemplateCommand.filterEntities(entityNameFileMap); + assertFalse("Does contain myfirst entity",entityNameFileMap.containsKey("myfirst")); + assertFalse("Does contain mysecond entity",entityNameFileMap.containsKey("mysecond")); + } + + @Test + public void testExtactEntityNameFromFilename() { + assertEquals("Could not extract entity ABC", "ABC",GenerateHubTDETemplateCommand.extactEntityNameFromFilename("ABC.entity.json").get()); + } +} diff --git a/marklogic-data-hub/src/test/resources/scaffolding-test/generate-tde-template/myfirst.entity.json b/marklogic-data-hub/src/test/resources/scaffolding-test/generate-tde-template/myfirst.entity.json new file mode 100644 index 0000000000..457a5c6168 --- /dev/null +++ b/marklogic-data-hub/src/test/resources/scaffolding-test/generate-tde-template/myfirst.entity.json @@ -0,0 +1,28 @@ +{ + "info" : { + "title" : "myfirst", + "version" : "0.0.1" + }, + "definitions" : { + "Employee" : { + "required" : [ ], + "primaryKey": "id", + "rangeIndex" : [ "name" ], + "elementRangeIndex": ["salary"], + "wordLexicon" : [ ], + "properties" : { + "id": { + "datatype": "string", + "collation" : "http://marklogic.com/collation/codepoint" + }, + "name" : { + "datatype" : "string", + "collation" : "http://marklogic.com/collation/codepoint" + }, + "salary": { + "datatype": "decimal" + } + } + } + } +} diff --git a/marklogic-data-hub/src/test/resources/scaffolding-test/generate-tde-template/mysecond.entity.json b/marklogic-data-hub/src/test/resources/scaffolding-test/generate-tde-template/mysecond.entity.json new file mode 100644 index 0000000000..42fe0319dc --- /dev/null +++ b/marklogic-data-hub/src/test/resources/scaffolding-test/generate-tde-template/mysecond.entity.json @@ -0,0 +1,28 @@ +{ + "info" : { + "title" : "mysecond", + "version" : "0.0.1" + }, + "definitions" : { + "Employee" : { + "required" : [ ], + "primaryKey": "id", + "rangeIndex" : [ "name" ], + "elementRangeIndex": ["salary"], + "wordLexicon" : [ ], + "properties" : { + "id": { + "datatype": "string", + "collation" : "http://marklogic.com/collation/codepoint" + }, + "name" : { + "datatype" : "string", + "collation" : "http://marklogic.com/collation/codepoint" + }, + "salary": { + "datatype": "decimal" + } + } + } + } +} diff --git a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/DataHubPlugin.groovy b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/DataHubPlugin.groovy index 693602f220..0f2aacd556 100644 --- a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/DataHubPlugin.groovy +++ b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/DataHubPlugin.groovy @@ -51,6 +51,9 @@ class DataHubPlugin implements Plugin { project.task("hubCreateEntity", group: scaffoldGroup, type: CreateEntityTask) project.task("hubCreateHarmonizeFlow", group: scaffoldGroup, type: CreateHarmonizeFlowTask) project.task("hubCreateInputFlow", group: scaffoldGroup, type: CreateInputFlowTask) + project.task("hubGenerateTDETemplates", group: scaffoldGroup, type: GenerateTDETemplateFromEntityTask, + description: "Generates TDE Templates from the entity definition files. It is possible to only generate TDE templates" + + " for specific entities by setting the (comma separated) project property 'entityNames'. E.g. -PentityNames=Entity1,Entity2") project.tasks.replace("mlLoadModules", DeployUserModulesTask) project.tasks.replace("mlWatch", HubWatchTask) diff --git a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/GenerateTDETemplateFromEntityTask.groovy b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/GenerateTDETemplateFromEntityTask.groovy new file mode 100644 index 0000000000..b9604710fc --- /dev/null +++ b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/GenerateTDETemplateFromEntityTask.groovy @@ -0,0 +1,22 @@ +package com.marklogic.gradle.task + +import com.marklogic.hub.deploy.commands.GenerateHubTDETemplateCommand +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.TaskAction + +class GenerateTDETemplateFromEntityTask extends HubTask { + + @Input + public String entityNames + + @TaskAction + void generateTDETEmplates() { + def cmd = new GenerateHubTDETemplateCommand(getHubConfig()) + if (entityNames == null) { + entityNames = project.hasProperty("entityNames") ? project.property("entityNames") : null + } + cmd.setEntityNames(entityNames) + cmd.execute(getCommandContext()) + } + +}