Merge pull request #685 from derms/feature/#551_Generate_TDE_Template

#551 Create gradle command to generate a TDE Template

marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommand.java

@@ -0,0 +1,148 @@
+package com.marklogic.hub.deploy.commands;
+
+import com.marklogic.appdeployer.AppConfig;
+import com.marklogic.appdeployer.command.CommandContext;
+import com.marklogic.appdeployer.command.es.GenerateModelArtifactsCommand;
+import com.marklogic.client.DatabaseClient;
+import com.marklogic.client.ext.es.CodeGenerationRequest;
+import com.marklogic.client.ext.es.EntityServicesManager;
+import com.marklogic.client.ext.es.GeneratedCode;
+import com.marklogic.hub.HubConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import static java.util.stream.Collectors.*;
+
+
+public class GenerateHubTDETemplateCommand extends GenerateModelArtifactsCommand {
+    private static final String ENTITY_FILE_EXTENSION = ".entity.json";
+    protected final Logger logger = LoggerFactory.getLogger(this.getClass());
+
+    private HubConfig hubConfig;
+
+    private String entityNames;
+
+    public GenerateHubTDETemplateCommand(HubConfig hubConfig) {
+        this.hubConfig = hubConfig;
+    }
+
+    @Override
+    public void execute(CommandContext context) {
+        AppConfig appConfig = context.getAppConfig();
+        DatabaseClient client = appConfig.newDatabaseClient();
+        EntityServicesManager mgr = new EntityServicesManager(client);
+
+        CodeGenerationRequest request = createCodeGenerationRequest();
+
+        List<File> entityFiles  = findEntityFiles();
+
+        if (!entityFiles.isEmpty()) {
+            //create map of entity name -> entity definition file
+            Map<String,File> entityNameFileMap = createEntityNameFileMap(entityFiles);
+
+            logger.debug("Found the following entities->files: {} " + entityNameFileMap);
+
+            filterEntities(entityNameFileMap);
+
+            if (!entityNameFileMap.isEmpty()) {
+                logger.warn("About to generate a template for the following entities: {} into directory {} ",
+                    entityNameFileMap.keySet(), hubConfig.getAppConfig().getSchemasPath());
+
+                for (File f : entityNameFileMap.values()) {
+                    GeneratedCode code = loadModelDefinition(request, f, mgr);
+                    generateExtractionTemplate(appConfig, code);
+                }
+
+            }
+
+        } else {
+            logger.info("No data hub entity files found under {} or its sub-directories.",
+                hubConfig.getHubEntitiesDir());
+        }
+
+    }
+
+    public String getEntityNames() {
+        return entityNames;
+    }
+
+    public void setEntityNames(String entityNames) {
+        this.entityNames = entityNames;
+    }
+
+    protected void filterEntities(Map<String,File> entityNameFileMap) {
+        Set<String> entityNameFileMapKeys = entityNameFileMap.keySet();
+
+        //filter on entityNames parameter if specified
+        if (entityNames!=null&&!entityNames.isEmpty()) {
+            List<String> entityNamesAsList = Arrays.asList(entityNames.split(","));
+            logger.info("Entities specified for TDE Generation: {} " + entityNamesAsList);
+
+            //this will only keep keys in the map that are also in the entityNamesAsList
+            entityNameFileMapKeys.retainAll(entityNamesAsList);
+
+            if (entityNameFileMapKeys.isEmpty()) {
+                logger.warn("No entities files found under {} or its sub-directories with the entity name(s) {}", hubConfig.getHubEntitiesDir(),entityNamesAsList);
+            }
+        }
+    }
+
+    protected static Map<String,File> createEntityNameFileMap(List<File> entityFiles) {
+        if (entityFiles==null) {
+            return Collections.emptyMap();
+        }
+        return entityFiles.stream().collect(
+            toMap(extractEntityNameFunction(),Function.identity()));
+    }
+
+    protected List<File> findEntityFiles() {
+        List<File> entities = new ArrayList<>();
+        Path entitiesPath = hubConfig.getHubEntitiesDir();
+        File[] entityDirectories = entitiesPath.toFile().listFiles(pathname -> pathname.isDirectory() && !pathname.isHidden());
+        List<String> entityNames;
+        if (entityDirectories != null) {
+            entityNames = Arrays.stream(entityDirectories)
+                .map(file -> file.getName())
+                .collect(Collectors.toList());
+            for (String entityName : entityNames) {
+                File[] entityDefs = entitiesPath.resolve(entityName).toFile().listFiles((dir, name) -> name.endsWith(ENTITY_FILE_EXTENSION));
+                if (entityDefs!=null) {
+                    entities.addAll(Arrays.asList(entityDefs));
+                }
+            }
+        }
+        return entities;
+    }
+
+    protected static Optional<String> extactEntityNameFromFilename(String filename) {
+        if (filename==null || filename.trim().isEmpty()) {
+            return Optional.of(null);
+        }
+        int index = filename.indexOf(ENTITY_FILE_EXTENSION);
+        if (index<0) {
+            //not found
+            return Optional.of(null);
+        }
+        return Optional.of(filename.substring(0,index));
+    }
+
+    private static Function<File, String> extractEntityNameFunction() {
+        Function<File, String> fileName = File::getName;
+        return fileName.andThen(name -> extactEntityNameFromFilename(name).get());
+    }
+
+    private static final CodeGenerationRequest createCodeGenerationRequest() {
+        CodeGenerationRequest request = new CodeGenerationRequest();
+        request.setGenerateExtractionTemplate(true);
+        request.setGenerateDatabaseProperties(false);
+        request.setGenerateInstanceConverter(false);
+        request.setGenerateSchema(false);
+        request.setGenerateSearchOptions(false);
+        return request;
+    }
+}

marklogic-data-hub/src/test/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommandTest.java

@@ -0,0 +1,160 @@
+package com.marklogic.hub.deploy.commands;
+
+import com.marklogic.appdeployer.command.CommandContext;
+import com.marklogic.hub.HubConfig;
+import com.marklogic.hub.HubTestBase;
+import com.marklogic.hub.scaffold.Scaffolding;
+import com.marklogic.hub.util.FileUtil;
+import org.custommonkey.xmlunit.XMLUnit;
+import org.easymock.EasyMock;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.jupiter.api.BeforeEach;
+import org.xml.sax.SAXException;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class GenerateHubTDETemplateCommandTest extends HubTestBase  {
+    static Path projectPath = Paths.get(PROJECT_PATH).toAbsolutePath();
+    private static File projectDir = projectPath.toFile();
+    private static final String RESOURCES_DIR = "scaffolding-test/generate-tde-template/";
+
+    GenerateHubTDETemplateCommand GenerateHubTDETemplateCommand;
+
+    @Before
+    public void setup() {
+        GenerateHubTDETemplateCommand = new GenerateHubTDETemplateCommand(getHubConfig());
+        deleteProjectDir();
+    }
+
+
+    @Before
+    public void clearDirs() {
+        deleteProjectDir();
+        createProjectDir();
+    }
+
+    @AfterClass
+    public static void teardown() throws IOException {
+        deleteProjectDir();
+    }
+
+    private void installEntity(String entityName) {
+        Scaffolding scaffolding = new Scaffolding(projectDir.toString(), finalClient);
+        Path entityDir = scaffolding.getEntityDir(entityName);
+        entityDir.toFile().mkdirs();
+        assertTrue(entityDir.toFile().exists());
+        FileUtil.copy(getResourceStream(RESOURCES_DIR + entityName + ".entity.json"), entityDir.resolve(entityName + ".entity.json").toFile());
+    }
+
+    @Test
+    public void testFindEntityFilesNoEntityFiles()  {
+        List<File> entityFiles = GenerateHubTDETemplateCommand.findEntityFiles();
+        assertEquals("Expected to find no entity files",0,entityFiles.size());
+    }
+
+    @Test
+    public void testFindEntityFilesOneEntityFiles()  {
+        installEntity("myfirst");
+        List<File> entityFiles = GenerateHubTDETemplateCommand.findEntityFiles();
+        assertEquals("Expected to find one entity file",1,entityFiles.size());
+    }
+
+    @Test
+    public void testFindEntityFilesTwoEntityFiles()  {
+        installEntity("myfirst");
+        installEntity("mysecond");
+        List<File> entityFiles = GenerateHubTDETemplateCommand.findEntityFiles();
+        assertEquals("Expected to find two entity files",2,entityFiles.size());
+    }
+
+    @Test
+    public void testCreateEntityNameFileMapWithNoEntityFiles()  {
+        Map<String,File> entityNameFileMap = GenerateHubTDETemplateCommand.createEntityNameFileMap(null);
+        assertEquals("Expected to find no entity files",0,entityNameFileMap.size());
+
+        entityNameFileMap = GenerateHubTDETemplateCommand.createEntityNameFileMap(new ArrayList<>());
+        assertEquals("Expected to find no entity files",0,entityNameFileMap.size());
+    }
+
+    @Test
+    public void testCreateEntityNameFileMapWithTwoEntityFiles()  {
+        installEntity("myfirst");
+        installEntity("mysecond");
+        List<File> entityFiles = GenerateHubTDETemplateCommand.findEntityFiles();
+
+        Map<String,File> entityNameFileMap = GenerateHubTDETemplateCommand.createEntityNameFileMap(entityFiles);
+        assertTrue("Does not contain myfirst entity",entityNameFileMap.containsKey("myfirst"));
+        assertTrue("Does not contain mysecond entity",entityNameFileMap.containsKey("mysecond"));
+
+        //assertEquals("Expected to find no entity files",2,entityNameFileMap.size());
+    }
+
+    @Test
+    public void testFilterSingleEntityWithTwoEntityFiles() {
+        installEntity("myfirst");
+        installEntity("mysecond");
+        GenerateHubTDETemplateCommand.setEntityNames("myfirst");
+
+        List<File> entityFiles = GenerateHubTDETemplateCommand.findEntityFiles();
+
+        Map<String,File> entityNameFileMap = GenerateHubTDETemplateCommand.createEntityNameFileMap(entityFiles);
+        assertTrue("Does not contain myfirst entity",entityNameFileMap.containsKey("myfirst"));
+        assertTrue("Does not contain mysecond entity",entityNameFileMap.containsKey("mysecond"));
+
+        GenerateHubTDETemplateCommand.filterEntities(entityNameFileMap);
+        assertTrue("Does not contain myfirst entity",entityNameFileMap.containsKey("myfirst"));
+        assertFalse("Does contain mysecond entity",entityNameFileMap.containsKey("mysecond"));
+    }
+
+    @Test
+    public void testFilterTwoEntityWithTwoEntityFiles()  {
+        installEntity("myfirst");
+        installEntity("mysecond");
+        GenerateHubTDETemplateCommand.setEntityNames("myfirst,mysecond");
+
+        List<File> entityFiles = GenerateHubTDETemplateCommand.findEntityFiles();
+
+        Map<String,File> entityNameFileMap = GenerateHubTDETemplateCommand.createEntityNameFileMap(entityFiles);
+        assertTrue("Does not contain myfirst entity",entityNameFileMap.containsKey("myfirst"));
+        assertTrue("Does not contain mysecond entity",entityNameFileMap.containsKey("mysecond"));
+
+        GenerateHubTDETemplateCommand.filterEntities(entityNameFileMap);
+        assertTrue("Does not contain myfirst entity",entityNameFileMap.containsKey("myfirst"));
+        assertTrue("Does not contain mysecond entity",entityNameFileMap.containsKey("mysecond"));
+    }
+
+    @Test
+    public void testFilterIncorrectEntities()  {
+        installEntity("myfirst");
+        installEntity("mysecond");
+        GenerateHubTDETemplateCommand.setEntityNames("XCXZ,ZXCXZC");
+
+        List<File> entityFiles = GenerateHubTDETemplateCommand.findEntityFiles();
+
+        Map<String,File> entityNameFileMap = GenerateHubTDETemplateCommand.createEntityNameFileMap(entityFiles);
+        assertTrue("Does not contain myfirst entity",entityNameFileMap.containsKey("myfirst"));
+        assertTrue("Does not contain mysecond entity",entityNameFileMap.containsKey("mysecond"));
+
+        GenerateHubTDETemplateCommand.filterEntities(entityNameFileMap);
+        assertFalse("Does contain myfirst entity",entityNameFileMap.containsKey("myfirst"));
+        assertFalse("Does contain mysecond entity",entityNameFileMap.containsKey("mysecond"));
+    }
+
+    @Test
+    public void testExtactEntityNameFromFilename()  {
+        assertEquals("Could not extract entity ABC", "ABC",GenerateHubTDETemplateCommand.extactEntityNameFromFilename("ABC.entity.json").get());
+    }
+}

marklogic-data-hub/src/test/resources/scaffolding-test/generate-tde-template/myfirst.entity.json

@@ -0,0 +1,28 @@
+{
+  "info" : {
+    "title" : "myfirst",
+    "version" : "0.0.1"
+  },
+  "definitions" : {
+    "Employee" : {
+      "required" : [ ],
+      "primaryKey": "id",
+      "rangeIndex" : [ "name" ],
+      "elementRangeIndex": ["salary"],
+      "wordLexicon" : [ ],
+      "properties" : {
+        "id": {
+          "datatype": "string",
+          "collation" : "http://marklogic.com/collation/codepoint"
+        },
+        "name" : {
+          "datatype" : "string",
+          "collation" : "http://marklogic.com/collation/codepoint"
+        },
+        "salary": {
+          "datatype": "decimal"
+        }
+      }
+    }
+  }
+}

marklogic-data-hub/src/test/resources/scaffolding-test/generate-tde-template/mysecond.entity.json

@@ -0,0 +1,28 @@
+{
+  "info" : {
+    "title" : "mysecond",
+    "version" : "0.0.1"
+  },
+  "definitions" : {
+    "Employee" : {
+      "required" : [ ],
+      "primaryKey": "id",
+      "rangeIndex" : [ "name" ],
+      "elementRangeIndex": ["salary"],
+      "wordLexicon" : [ ],
+      "properties" : {
+        "id": {
+          "datatype": "string",
+          "collation" : "http://marklogic.com/collation/codepoint"
+        },
+        "name" : {
+          "datatype" : "string",
+          "collation" : "http://marklogic.com/collation/codepoint"
+        },
+        "salary": {
+          "datatype": "decimal"
+        }
+      }
+    }
+  }
+}

ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/DataHubPlugin.groovy

@@ -51,6 +51,9 @@ class DataHubPlugin implements Plugin<Project> {
         project.task("hubCreateEntity", group: scaffoldGroup, type: CreateEntityTask)
         project.task("hubCreateHarmonizeFlow", group: scaffoldGroup, type: CreateHarmonizeFlowTask)
         project.task("hubCreateInputFlow", group: scaffoldGroup, type: CreateInputFlowTask)
+        project.task("hubGenerateTDETemplates", group: scaffoldGroup, type: GenerateTDETemplateFromEntityTask,
+            description: "Generates TDE Templates from the entity definition files. It is possible to only generate TDE templates" +
+                " for specific entities by setting the (comma separated) project property 'entityNames'. E.g. -PentityNames=Entity1,Entity2")
 
         project.tasks.replace("mlLoadModules", DeployUserModulesTask)
         project.tasks.replace("mlWatch", HubWatchTask)