Feature/dhfprod 1703 Integration Test PR (#1797)

* DHFPROD-1703 Initial refactor for new flow * DHFPROD-1740, Create 5.x FlowManager and refactor 4.x FlowManager to LegacyFlowManager (#1791) * Refactoring the 4.x FlowManager as LegacyFlowManager * Creating 5.x FlowManager, its implementation and unit test * Fixing the interface docs * Deleting project dir before tests * DHFPROD-156 Gradle task for new flow creation * DHFPROD-1703 fix tests * Cleanup test code * Make cleanup method static
Marklogic-retired · Jan 28, 2019 · 659dddc · 659dddc
1 parent 3b46908
commit 659dddc
Show file tree

Hide file tree

Showing 74 changed files with 3,693 additions and 3,046 deletions.
diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/ApplicationConfig.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/ApplicationConfig.java
@@ -11,7 +11,7 @@
  * This application configuration is an entry point to using the DHF from a set property
  */
 @Configuration
-@ComponentScan(basePackages = {"com.marklogic.hub.impl", "com.marklogic.hub.deploy.commands"})
+@ComponentScan(basePackages = {"com.marklogic.hub.impl", "com.marklogic.hub.legacy.impl", "com.marklogic.hub.deploy.commands"})
 @EnableAutoConfiguration
 public class ApplicationConfig {
 

diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/FlowManager.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/FlowManager.java
@@ -16,100 +16,79 @@
 
 package com.marklogic.hub;
 
+import com.fasterxml.jackson.databind.JsonNode;
+import com.marklogic.hub.error.DataHubProjectException;
 import com.marklogic.hub.flow.Flow;
-import com.marklogic.hub.flow.FlowRunner;
-import com.marklogic.hub.flow.FlowType;
-import com.marklogic.hub.flow.impl.FlowImpl;
-import org.w3c.dom.Element;
 
-import java.nio.file.Path;
 import java.util.List;
 
 /**
- * Manages existing flows and creates flow runners to execute flows.
+ * Manages CRUD operations for flows
  */
 public interface FlowManager {
 
     /**
-     * Turns an XML document into a flow
-     * @param doc - the xml document representing a flow
-     * @return a Flow instance
+     * String value for the flow file extension
      */
-    static Flow flowFromXml(Element doc) {
-        return FlowImpl.fromXml(doc);
-    }
+    String FLOW_FILE_EXTENSION = ".flow.json";
 
     /**
-     * retrieves a list of all the flows on the local files systems
-     * @return a list of Flows
+     * Retrieves a named flow
+     * @param flowName - name of the flow
+     * @return a flow object
      */
-    List<Flow> getLocalFlows();
+    Flow getFlow(String flowName);
 
     /**
-     * retrieves a list of all the flows on the local files systems
-     * @param entityName - string name of the entity for the flow
-     * @return a list of Flows
+     * Returns a flow based on the provided name as JSON string
+     * @param flowName - name of the flow
+     * @return string json representation of the flow object
      */
-    List<Flow> getLocalFlowsForEntity(String entityName);
+    String getFlowAsJSON(String flowName);
 
     /**
-     * retrieves a list of all the flows on the local files systems
-     * @param entityName - string name of the entity for the flow
-     * @param flowType - the FlowType enum, eg: ingest or harmonize
-     * @return a list of Flows
+     * Retrieves a list of flows installed on the MarkLogic server
+     * @return - a list of all flows
      */
-    List<Flow> getLocalFlowsForEntity(String entityName, FlowType flowType);
+    List<Flow> getFlows();
 
     /**
-     * Obtains a flow from a property file
-     * @param propertiesFile - the Path to the property file
-     * @return - a flow object
+     * Retrieves a list of names of flows installed on the MarkLogic server
+     * @return - a list of names of all flows
      */
-    Flow getFlowFromProperties(Path propertiesFile);
+    List<String> getFlowNames();
 
     /**
-     * Retrieves a list of flows installed on the MarkLogic server
-     * @param entityName - the entity from which to fetch the flows
-     * @return - a list of flows for the given entity
+     * Creates a flow
+     * @param flowName - name of the flow
      */
-    List<Flow> getFlows(String entityName);
+    Flow createFlow(String flowName);
 
     /**
-     * Retrieves a named flow from a given entity
-     *
-     * @param entityName - the entity that the flow belongs to
-     * @param flowName - the name of the flow to get
-     * @return the flow
+     * Creates a flow from a given JSON string
+     * @param  json - string representation of the flow
+     * @return - a Flow object
      */
-    Flow getFlow(String entityName, String flowName);
+    Flow createFlowFromJSON(String json);
 
     /**
-     * Retrieves a named flow from a given entity
-     *
-     * @param entityName - the entity that the flow belongs to
-     * @param flowName - the name of the flow to get
-     * @param flowType - the type of flow (ingest/harmonize)
-     * @return the flow
+     * Creates a flow from a given JsonNode
+     * @param json - JsonNode representation of the flow
+     * @return - a Flow object
      */
-    Flow getFlow(String entityName, String flowName, FlowType flowType);
+    Flow createFlowFromJSON(JsonNode json);
 
     /**
-     * Updates the indexes in the database based on the project
-     * @return - a list of names for all the flows that are legacy
+     * Deletes a flow
+     * @param flowName - name of the flow
      */
-
-    List<String> getLegacyFlows();
+    void deleteFlow(String flowName);
 
     /**
-     * Sets the version that the legacy flow is to be updated from
-     * @param fromVersion - string representation of DHF version
-     * @return a list of updated flow names that were updated
+     * Saves a flow to disk
+     * @param flow - the flow object to be saved
      */
-    List<String> updateLegacyFlows(String fromVersion);
+    void saveFlow(Flow flow);
+
 
-    /**
-     * Creates and returns a new FlowRunner object using the FlowManager's hubconfig
-     * @return FlowRunner object with current hubconfig already set
-     */
-    FlowRunner newFlowRunner();
 }
diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/HubConfig.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/HubConfig.java
@@ -471,6 +471,13 @@ public interface HubConfig {
      */
     Path getEntityDatabaseDir();
 
+    /**
+     * Gets the path for the flows directory
+     *
+     * @return the path for the flows directory
+     */
+    Path getFlowsDir();
+
     /**
      * Returns the current AppConfig object attached to the HubConfig
      * @return Returns current AppConfig object set for HubConfig

diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/HubProject.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/HubProject.java
@@ -149,6 +149,13 @@ public interface HubProject {
      */
     Path getEntityDatabaseDir();
 
+    /**
+     * Gets the path for the flows directory
+     *
+     * @return the path for the flows directory
+     */
+    Path getFlowsDir();
+
     /**
      * Gets the path for the hub staging modules
      *

diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/collector/Collector.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/collector/Collector.java
@@ -17,7 +17,7 @@
 
 import com.marklogic.client.DatabaseClient;
 import com.marklogic.hub.HubConfig;
-import com.marklogic.hub.flow.CodeFormat;
+import com.marklogic.hub.legacy.flow.CodeFormat;
 
 import java.util.Map;
 

diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/collector/impl/CollectorImpl.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/collector/impl/CollectorImpl.java
@@ -23,7 +23,7 @@
 import com.marklogic.hub.HubConfig;
 import com.marklogic.hub.collector.Collector;
 import com.marklogic.hub.collector.DiskQueue;
-import com.marklogic.hub.flow.CodeFormat;
+import com.marklogic.hub.legacy.flow.CodeFormat;
 import com.marklogic.hub.impl.HubConfigImpl;
 import com.marklogic.rest.util.MgmtResponseErrorHandler;
 import org.apache.http.auth.AuthScope;

diff --git a/...ogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadUserModulesCommand.java b/...ogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadUserModulesCommand.java
@@ -30,11 +30,11 @@
 import com.marklogic.client.io.Format;
 import com.marklogic.client.io.StringHandle;
 import com.marklogic.hub.EntityManager;
-import com.marklogic.hub.FlowManager;
+import com.marklogic.hub.legacy.LegacyFlowManager;
 import com.marklogic.hub.HubConfig;
 import com.marklogic.hub.deploy.util.HubFileFilter;
 import com.marklogic.hub.error.LegacyFlowsException;
-import com.marklogic.hub.flow.Flow;
+import com.marklogic.hub.legacy.flow.Flow;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 import org.springframework.stereotype.Component;
@@ -63,7 +63,7 @@ public class LoadUserModulesCommand extends LoadModulesCommand {
     private EntityManager entityManager;
 
     @Autowired
-    private FlowManager flowManager;
+    private LegacyFlowManager flowManager;
 
     private DocumentPermissionsParser documentPermissionsParser = new DefaultDocumentPermissionsParser();
     private ThreadPoolTaskExecutor threadPoolTaskExecutor;

diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/flow/Flow.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/flow/Flow.java
@@ -15,127 +15,35 @@
  */
 package com.marklogic.hub.flow;
 
-import com.marklogic.hub.collector.Collector;
-import com.marklogic.hub.main.MainPlugin;
+import com.fasterxml.jackson.databind.JsonNode;
 
-import java.util.Properties;
-
-/**
- * Manages the creation and configuration of flow objects
- */
 public interface Flow {
-
-    /**
-     * Sets the entity name for the flow
-     * @param entityName the string name of the entity to use in the flow
-     */
-    void setEntityName(String entityName);
-
-    /**
-     * Returns the name of the entity that's been set for the flow in strong form
-     * @return entity name in string form
-     */
-    String getEntityName();
-
-    /**
-     * Sets the name of the flow
-     * @param name - string representing the name of the flow
-     */
-    void setName(String name);
-
     /**
-     * Gets the name of the flow
-     * @return the name of the flow in string form
+     * Returns the name of the flow
+     *
+     * @return a flow name
      */
     String getName();
 
     /**
-     * Sets the mapping name used to generate the flow
-     * @param mappingName - string representing the mapping name for the flow
-     */
-    void setMappingName(String mappingName);
-
-    /**
-     * Gets the mapping name used for the flow
-     * @return the mapping name of the flow in string form
-     */
-    String getMappingName();
-
-    /**
-     * Sets the type of the flow
-     * @param type - FlowType enum for harmonize or input
-     */
-    void setType(FlowType type);
-
-    /**
-     * Gets the FlowType enum for the flow
-     * @return FlowType of ingest or harmonize
-     */
-    FlowType getType();
-
-    /**
-     * Sets the DataFormat for the flow
-     * @param dataFormat - DataFormat enum of json or xml
-     */
-    void setDataFormat(DataFormat dataFormat);
-
-    /**
-     * Returns the DataForm enum of the flow
-     * @return DataForm enum of json or xml
-     */
-    DataFormat getDataFormat();
-
-    /**
-     * Sets the CodeFormat enum of the flow
-     * @param codeFormat enum of sjs or xqy
-     */
-    void setCodeFormat(CodeFormat codeFormat);
-
-    /**
-     * Returns the CodeFormat enum of the flow
-     * @return CodeFormat enum of sjs or xqy
+     * Sets the name of the flow
+     *
+     * @param flowName - a flow name
      */
-    CodeFormat getCodeFormat();
+    void setName(String flowName);
 
     /**
-     * Serializes the flow into an xml string
-     * @return a serialized xml string of the flow
+     * Serializes the flow as a json string
+     *
+     * @return the serialized JSON string
      */
     String serialize();
 
     /**
-     * Creates a properties object representing the flow
-     * @return a Properties object representation of the flow
-     */
-    Properties toProperties();
-
-    /**
-     * Gets the DbPath (uri) for the flow in string form
-     * @return a uri path as a string
-     */
-    String getFlowDbPath();
-
-    /**
-     * Gets the collector to be used for the flow
-     * @return the collector object that is used for the flow
-     */
-    Collector getCollector();
-
-    /**
-     * Sets the collector to be used for the flow
-     * @param collector the collector to be used for the flow
-     */
-    void setCollector(Collector collector);
-
-    /**
-     * Gets the main plugin module that the flow is set to use
-     * @return MainPlugin object
-     */
-    MainPlugin getMain();
-
-    /**
-     * Sets the main plugin module for the flow
-     * @param main - a MainPlugin object that defines the main plugin to be used
+     * Deserialize a json response and applies it to this flow
+     *
+     * @param json - the JsonNode you want deserialize
+     * @return this mapping
      */
-    void setMain(MainPlugin main);
+    Flow deserialize(JsonNode json);
 }