Dhfprod 1788 - develop PR

marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/GenerateHubTDETemplateCommand.java

@@ -21,16 +21,16 @@
 import com.marklogic.client.DatabaseClient;
 import com.marklogic.client.eval.EvalResultIterator;
 import com.marklogic.client.ext.es.CodeGenerationRequest;
-import com.marklogic.client.ext.es.EntityServicesManager;
 import com.marklogic.client.ext.es.GeneratedCode;
+import com.marklogic.hub.DatabaseKind;
 import com.marklogic.hub.HubConfig;
+import com.marklogic.hub.es.EntityServicesManager;
 import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.util.FileCopyUtils;
 
 import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.*;
@@ -45,11 +45,14 @@ public class GenerateHubTDETemplateCommand extends GenerateModelArtifactsCommand
     protected final Logger logger = LoggerFactory.getLogger(this.getClass());
 
     private HubConfig hubConfig;
+    private Path userFinalSchemasTDEs;
 
     private String entityNames;
+    private List<String> entityNamesList = Collections.EMPTY_LIST;
 
     public GenerateHubTDETemplateCommand(HubConfig hubConfig) {
         this.hubConfig = hubConfig;
+        this.userFinalSchemasTDEs = hubConfig.getHubProject().getUserDatabaseDir().resolve(hubConfig.getDbName(DatabaseKind.FINAL_SCHEMAS)).resolve("schemas").resolve("tde");
     }
 
     @Override
@@ -68,25 +71,27 @@ public void execute(CommandContext context) {
 
             logger.debug("Found the following entities->files: {} " + entityNameFileMap);
 
-            filterEntities(entityNameFileMap);
+            //filterEntities(entityNameFileMap);
 
             if (!entityNameFileMap.isEmpty()) {
                 logger.warn("About to generate a template for the following entities: {} into directory {} ",
-                    entityNameFileMap.keySet(), hubConfig.getAppConfig().getSchemasPath());
-
+                    this.entityNamesList.isEmpty() ? entityNameFileMap.keySet() : this.entityNamesList, hubConfig.getAppConfig().getSchemasPath());
+                List<GeneratedCode> generatedCodes = new ArrayList<>();
                 for (File f : entityNameFileMap.values()) {
                     File esModel;
+                    String modelName;
                     try {
                         //Write the ES model to a temp file
                         String tempDir = System.getProperty("java.io.tmpdir");
                         String fileName = f.getName();
+                        modelName = EntityServicesManager.extractEntityNameFromURI(fileName).get();
                         esModel = new File(tempDir, fileName);
                         String modelString = generateModel(f);
                         if(modelString == null) {
                             logger.warn(f.getName() + " is not deployed to the database");
                             continue;
                         }
-                        FileUtils.writeStringToFile(esModel, generateModel(f));
+                        FileUtils.writeStringToFile(esModel, modelString);
                     } catch (IOException e) {
                         throw new RuntimeException("Unable to generate ES model");
                     }
@@ -100,9 +105,13 @@ public void execute(CommandContext context) {
                     finally {
                         FileUtils.deleteQuietly(esModel);
                     }
+                    if (this.entityNamesList.isEmpty() || this.entityNamesList.contains(modelName)) {
+                        generatedCodes.add(code);
+                    }
+                }
+                for (GeneratedCode code: generatedCodes) {
                     generateExtractionTemplate(appConfig, code);
                 }
-
             }
 
         } else {
@@ -116,7 +125,7 @@ public void execute(CommandContext context) {
     private String generateModel(File f) {
         String xquery = "import module namespace hent = \"http://marklogic.com/data-hub/hub-entities\"\n" +
             "at \"/data-hub/4/impl/hub-entities.xqy\";\n" +
-            String.format("hent:get-model(\"%s\")", extactEntityNameFromFilename(f.getName()).get());
+            String.format("hent:get-model(\"%s\")", extractEntityNameFromFilename(f.getName()).get());
         EvalResultIterator resp = hubConfig.newStagingClient().newServerEval().xquery(xquery).eval();
         if (resp.hasNext()) {
             return resp.next().getString();
@@ -130,6 +139,9 @@ public String getEntityNames() {
 
     public void setEntityNames(String entityNames) {
         this.entityNames = entityNames;
+        if (entityNames != null) {
+            this.entityNamesList = Arrays.asList(entityNames.split(","));
+        }
     }
 
     protected void filterEntities(Map<String,File> entityNameFileMap) {
@@ -176,21 +188,43 @@ protected List<File> findEntityFiles() {
         return entities;
     }
 
-    protected static Optional<String> extactEntityNameFromFilename(String filename) {
-        if (filename==null || filename.trim().isEmpty()) {
-            return Optional.of(null);
-        }
-        int index = filename.indexOf(ENTITY_FILE_EXTENSION);
-        if (index<0) {
-            //not found
-            return Optional.of(null);
+    // Overriding to insert schemas into the final DB schemas folder.
+    @Override
+    protected void generateExtractionTemplate(AppConfig appConfig, GeneratedCode code) {
+        String template = code.getExtractionTemplate();
+        if (template != null) {
+            File dir = userFinalSchemasTDEs.toFile();
+            dir.mkdirs();
+            File out = new File(dir, code.getTitle() + "-" + code.getVersion() + ".tdex");
+            String logMessage = "Wrote extraction template to: ";
+            if (out.exists()) {
+                if (!fileHasDifferentContent(out, template)) {
+                    if (logger.isInfoEnabled()) {
+                        logger.info("Extraction template matches file, so not modifying: " + out.getAbsolutePath());
+                    }
+                    return;
+                }
+                out = new File(dir, code.getTitle() + "-" + code.getVersion() + "-GENERATED.tdex");
+                logMessage = "Extraction template does not match existing file, so writing to: ";
+            }
+            try {
+                FileCopyUtils.copy(template.getBytes(), out);
+                if (logger.isInfoEnabled()) {
+                    logger.info(logMessage + out.getAbsolutePath());
+                }
+            } catch (IOException e) {
+                throw new RuntimeException("Unable to write extraction template to file: " + out.getAbsolutePath(), e);
+            }
         }
-        return Optional.of(filename.substring(0,index));
+    }
+
+    protected static Optional<String> extractEntityNameFromFilename(String filename) {
+        return EntityServicesManager.extractEntityNameFromURI(filename);
     }
 
     private static Function<File, String> extractEntityNameFunction() {
         Function<File, String> fileName = File::getName;
-        return fileName.andThen(name -> extactEntityNameFromFilename(name).get());
+        return fileName.andThen(name -> extractEntityNameFromFilename(name).get());
     }
 
     private static final CodeGenerationRequest createCodeGenerationRequest() {

marklogic-data-hub/src/main/java/com/marklogic/hub/es/EntityServicesManager.java

@@ -0,0 +1,47 @@
+package com.marklogic.hub.es;
+
+import com.marklogic.client.DatabaseClient;
+
+import java.util.Optional;
+
+/*
+ * This class overrides the original so we can fix the TDE templates to work properly.
+ */
+public class EntityServicesManager extends com.marklogic.client.ext.es.EntityServicesManager {
+    protected DatabaseClient client;
+    private static final String ENTITY_FILE_EXTENSION = ".entity.json";
+
+    public EntityServicesManager(DatabaseClient client) {
+        super(client);
+        this.client = client;
+    }
+
+    @Override
+    protected String generateCode(String modelUri, String functionName) {
+        if ("extraction-template-generate".equals(functionName)) {
+            String xquery = "import module namespace es = \"http://marklogic.com/entity-services\" at \"/MarkLogic/entity-services/entity-services.xqy\"; \n" +
+                "import module namespace hent = \"http://marklogic.com/data-hub/hub-entities\" at \"/data-hub/4/impl/hub-entities.xqy\";\n" +
+                "declare variable $entity-title external; \n" +
+                "hent:dump-tde(json:to-array(es:model-validate(hent:get-model($entity-title))))";
+            return client.newServerEval().xquery(xquery).addVariable("entity-title", extractEntityNameFromURI(modelUri).get()).eval().next().getString();
+        } else {
+            return super.generateCode(modelUri, functionName);
+        }
+    }
+
+    public static Optional<String> extractEntityNameFromURI(String filename) {
+        if (filename==null || filename.trim().isEmpty()) {
+            return Optional.of(null);
+        }
+        int pathIndex = filename.lastIndexOf("/");
+        if (pathIndex >= 0) {
+            filename = filename.substring(pathIndex + 1);
+        }
+        int index = filename.indexOf(ENTITY_FILE_EXTENSION);
+        if (index<0) {
+            //not found
+            return Optional.of(null);
+        }
+        return Optional.of(filename.substring(0,index));
+    }
+}

marklogic-data-hub/src/main/java/com/marklogic/hub/impl/EntityManagerImpl.java

@@ -400,7 +400,6 @@ public String getIndexes(List<JsonNode> entities) {
         }
     }
 
-
     @Override
     public boolean savePii() {
         try {

marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/impl/hub-entities.xqy

@@ -19,6 +19,8 @@ module namespace hent = "http://marklogic.com/data-hub/hub-entities";
 
 import module namespace es = "http://marklogic.com/entity-services"
   at "/MarkLogic/entity-services/entity-services.xqy";
+import module namespace tde = "http://marklogic.com/xdmp/tde"
+  at "/MarkLogic/tde.xqy";
 
 declare namespace search = "http://marklogic.com/appservices/search";
 
@@ -37,7 +39,7 @@ declare function hent:get-model($entity-name as xs:string, $used-models as xs:st
     let $_ := fn:collection($ENTITY-MODEL-COLLECTION)[lower-case(info/title) = lower-case($entity-name)]
     return
       if (fn:count($_) > 1) then
-        fn:collection($ENTITY-MODEL-COLLECTION)[info/title = $entity-name]
+        fn:head(fn:collection($ENTITY-MODEL-COLLECTION)[info/title = $entity-name])
       else
         $_
   where fn:exists($model)
@@ -156,3 +158,145 @@ declare function hent:dump-indexes($entities as json:array)
   return
     xdmp:to-json($o)
 };
+
+declare variable $generated-primary-key-column as xs:string := "DataHubGeneratedPrimaryKey";
+declare variable $generated-primary-key-expression as xs:string := "xdmp:node-uri(.) || '#' || fn:position()";
+
+declare function hent:dump-tde($entities as json:array)
+{
+  let $entity-values as map:map* := json:array-values($entities)
+  let $uber-model := hent:uber-model($entity-values ! xdmp:to-json(.)/object-node())
+  let $_set-info := $uber-model => map:put("info", fn:head($entity-values) => map:get("info"))
+  let $uber-definitions := $uber-model => map:get("definitions")
+  (: Primary keys are required for each definition in an entity. If the primary key is missing, we'll help out by using the doc URI and position as the primary key. :)
+  let $_set-primary-keys-for-TDE :=
+    for $definition-type in map:keys($uber-definitions)
+    let $definition := $uber-definitions => map:get($definition-type)
+    where fn:empty($definition => map:get("primaryKey"))
+    return (
+      $definition => map:put("primaryKey", $generated-primary-key-column),
+      $definition => map:get("properties") => map:put($generated-primary-key-column, map:entry("datatype", "string"))
+    )
+  let $entity-model-contexts := map:keys($uber-definitions) ! ("./" || .)
+  return hent:fix-tde(es:extraction-template-generate($uber-model), $entity-model-contexts, $uber-definitions)
+};
+
+declare variable $default-nullable as element(tde:nullable) := element tde:nullable {fn:true()};
+declare variable $default-invalid-values as element(tde:invalid-values) := element tde:invalid-values {"ignore"};
+(:
+  this method doctors the TDE output from ES
+:)
+declare function hent:fix-tde($nodes as node()*, $entity-model-contexts as xs:string*, $uber-definitions as map:map)
+{
+  for $n in $nodes
+  return
+    typeswitch($n)
+      case document-node() return
+        document {
+          hent:fix-tde($n/node(), $entity-model-contexts, $uber-definitions)
+        }
+      case element(tde:nullable) return
+        $default-nullable
+      case element(tde:invalid-values) return
+        $default-invalid-values
+      case element(tde:val) return
+        element { fn:node-name($n) } {
+          $n/namespace::node(),
+          let $col-name := fn:string($n/../tde:name)
+          return
+            if (fn:ends-with($col-name, $generated-primary-key-column)) then
+              $generated-primary-key-expression
+            else if (fn:starts-with($n, $col-name)) then
+              let $parts := fn:tokenize($n, "/")
+              let $entity-definition := $uber-definitions => map:get(fn:string($parts[2]))
+              return
+                if (fn:exists($entity-definition)) then
+                  let $primary-key := $entity-definition => map:get("primaryKey")
+                  return
+                    if ($primary-key = $generated-primary-key-column) then
+                      $generated-primary-key-expression
+                    else
+                      fn:string($n) || "/" || $primary-key
+                else
+                  hent:fix-tde($n/node(), $entity-model-contexts, $uber-definitions)
+            else
+              hent:fix-tde($n/node(), $entity-model-contexts, $uber-definitions)
+        }
+      case element(tde:context) return
+        element { fn:node-name($n) } {
+          $n/namespace::node(),
+          if ($n = $entity-model-contexts) then
+            fn:replace(fn:string($n),"^\./", ".//")
+          else
+            $n/node()
+        }
+      case element(tde:column) return
+        element { fn:node-name($n) } {
+          $n/namespace::node(),
+          $n/@*,
+          hent:fix-tde($n/* except $n/(tde:nullable|tde:invalid-values), $entity-model-contexts, $uber-definitions),
+          $default-nullable,
+          $default-invalid-values
+        }
+      case element(tde:subject)|element(tde:predicate)|element(tde:object) return
+        element { fn:node-name($n) } {
+          $n/namespace::node(),
+          hent:fix-tde($n/* except $n/tde:invalid-values, $entity-model-contexts, $uber-definitions),
+          $default-invalid-values
+        }
+      case element(tde:template) return
+        element { fn:node-name($n) } {
+          $n/namespace::node(),
+          $n/@*,
+          let $context-item :=  fn:replace(fn:string($n/tde:context), "\./", "")
+          let $parent-context-item := fn:replace(fn:string($n/../../tde:context), "\./", "")
+          let $is-join-template := $n/tde:rows/tde:row/tde:view-name = $parent-context-item  || "_" || $context-item
+          let $rows := $n/tde:rows/tde:row
+          return
+            if ($is-join-template) then (
+              hent:fix-tde($n/tde:context, $entity-model-contexts, $uber-definitions),
+              element tde:rows {
+                element tde:row {
+                  $rows/(tde:schema-name|tde:view-name|tde:view-layout),
+                  element tde:columns {
+                    let $join-prefix := $context-item || "_"
+                    for $column in $rows/tde:columns/tde:column
+                    return
+                      element tde:column {
+                        $column/@*,
+                        if (fn:starts-with($column/tde:name, $join-prefix)) then (
+                          hent:fix-tde($column/(tde:name|tde:scalar-type), $entity-model-contexts, $uber-definitions),
+                          let $tde-val := fn:string($column/tde:val)
+                          let $primary-key := $uber-definitions => map:get($tde-val) => map:get("primaryKey")
+                          return
+                            element tde:val {
+                              if ($primary-key = $generated-primary-key-column) then
+                                $generated-primary-key-expression
+                              else
+                                $tde-val || "/" || $primary-key
+                            },
+                          $default-nullable,
+                          $default-invalid-values,
+                          hent:fix-tde($column/(tde:default|tde:reindexing|tde:collation), $entity-model-contexts, $uber-definitions)
+                        ) else
+                          hent:fix-tde($column/node(), $entity-model-contexts, $uber-definitions)
+                      }
+                  }
+                }
+              }
+            ) else
+              hent:fix-tde($n/node(), $entity-model-contexts, $uber-definitions)
+        }
+      case element() return
+        element { fn:node-name($n) } {
+          $n/namespace::node(),
+          hent:fix-tde(($n/@*, $n/node()), $entity-model-contexts, $uber-definitions)
+        }
+      case text() return
+        fn:replace(
+          fn:replace($n, "^\.\./(.+)$", "(../$1|parent::array-node()/../$1)"),
+          "\./" || $generated-primary-key-column,
+          $generated-primary-key-expression
+        )
+      default return $n
+};

marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/triggers/entity-model-delete-trigger.xqy

@@ -12,7 +12,9 @@ declare variable $TDE-COLLECTION as xs:string := "http://marklogic.com/entity-se
 declare variable $trgr:uri as xs:string external;
 
 let $entity-def := fn:doc($trgr:uri)
-let $tde-uri := $trgr:uri || ".tde.xml"
+let $entity-title := $entity-def/info/title
+let $entity-version := $entity-def/info/version
+let $tde-uri := "/tde/" || $entity-title || "-" || $entity-version || ".tdex"
 return (
   xdmp:invoke-function(
     function() {