From bc696002c4936c27139833df65a5f91443970fa1 Mon Sep 17 00:00:00 2001 From: Akshay Sonvane Date: Mon, 25 Feb 2019 11:08:21 -0800 Subject: [PATCH 1/3] Get resource as stream to work in both jar file and via IDE --- .../commands/LoadHubArtifactsCommand.java | 109 ++++++------------ 1 file changed, 37 insertions(+), 72 deletions(-) diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadHubArtifactsCommand.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadHubArtifactsCommand.java index 771a417e8a..c0da8d7a87 100644 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadHubArtifactsCommand.java +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadHubArtifactsCommand.java @@ -22,10 +22,7 @@ import com.marklogic.client.DatabaseClient; import com.marklogic.client.document.DocumentWriteSet; import com.marklogic.client.document.JSONDocumentManager; -import com.marklogic.client.ext.modulesloader.Modules; -import com.marklogic.client.ext.modulesloader.impl.FlowDefModulesFinder; import com.marklogic.client.ext.modulesloader.impl.PropertiesModuleManager; -import com.marklogic.client.ext.modulesloader.impl.StepDefModulesFinder; import com.marklogic.client.ext.util.DefaultDocumentPermissionsParser; import com.marklogic.client.ext.util.DocumentPermissionsParser; import com.marklogic.client.io.DocumentMetadataHandle; @@ -33,16 +30,12 @@ import com.marklogic.hub.HubConfig; import org.apache.commons.io.IOUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.Resource; import org.springframework.stereotype.Component; import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.nio.file.*; -import java.nio.file.attribute.BasicFileAttributes; import java.util.Date; -import java.util.Objects; /** * Loads user artifacts like mappings and entities. This will be deployed after triggers @@ -93,12 +86,6 @@ public void execute(CommandContext context) { DatabaseClient stagingClient = hubConfig.newStagingClient(); DatabaseClient finalClient = hubConfig.newFinalClient(); - String baseDir = Objects.requireNonNull(getClass().getClassLoader().getResource("hub-internal-artifacts")) - .getFile(); - Path basePath = Paths.get(baseDir); - Path flowPath = basePath.resolve("flows"); - Path stepPath = basePath.resolve("steps"); - JSONDocumentManager finalDocMgr = finalClient.newJSONDocumentManager(); JSONDocumentManager stagingDocMgr = stagingClient.newJSONDocumentManager(); @@ -110,75 +97,53 @@ public void execute(CommandContext context) { PropertiesModuleManager propertiesModuleManager = getModulesManager(); - try { + File defaultFlowFile = new File("/flows/default-flow.flow.json"); + File defaultStepIngestFile = new File("/steps/ingest/marklogic/default-ingest.step.json"); + File defaultStepMappingFile = new File("/steps/mapping/marklogic/default-mapping.step.json"); - // let's do steps - if (stepPath.toFile().exists()) { - Files.walkFileTree(stepPath, new SimpleFileVisitor() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - Modules modules = new StepDefModulesFinder().findModules(dir.toString()); - DocumentMetadataHandle meta = new DocumentMetadataHandle(); - meta.getCollections().add("http://marklogic.com/data-hub/step"); - documentPermissionsParser.parsePermissions(hubConfig.getModulePermissions(), meta.getPermissions()); - for (Resource r : modules.getAssets()) { - if (forceLoad || propertiesModuleManager.hasFileBeenModifiedSinceLastLoaded(r.getFile())) { - InputStream inputStream = r.getInputStream(); - StringHandle handle = new StringHandle(IOUtils.toString(inputStream)); - inputStream.close(); - stagingStepDocumentWriteSet.add("/steps/" + r.getFile().getParentFile().getParentFile().getName() + "/" + r.getFile().getParentFile().getName() + "/" + r.getFilename(), meta, handle); - finalStepDocumentWriteSet.add("/steps/" + r.getFile().getParentFile().getParentFile().getName() + "/" + r.getFile().getParentFile().getName() + "/" + r.getFilename(), meta, handle); - propertiesModuleManager.saveLastLoadedTimestamp(r.getFile(), new Date()); - } - } - return FileVisitResult.CONTINUE; - } - }); - } + // let's do flows + InputStream inputStream = getClass().getResourceAsStream("/hub-internal-artifacts/flows/default-flow.flow.json"); + addToWriteSet(inputStream, "http://marklogic.com/data-hub/flow", defaultFlowFile, stagingFlowDocumentWriteSet, finalFlowDocumentWriteSet, propertiesModuleManager); + // let's do steps + inputStream = getClass().getResourceAsStream("/hub-internal-artifacts/steps/ingest/marklogic/default-ingest.step.json"); + addToWriteSet(inputStream, "http://marklogic.com/data-hub/step", defaultStepIngestFile, stagingStepDocumentWriteSet, finalStepDocumentWriteSet, propertiesModuleManager); - // let's do flows - if (flowPath.toFile().exists()) { - Files.walkFileTree(flowPath, new SimpleFileVisitor() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - Modules modules = new FlowDefModulesFinder().findModules(dir.toString()); - DocumentMetadataHandle meta = new DocumentMetadataHandle(); - meta.getCollections().add("http://marklogic.com/data-hub/flow"); - documentPermissionsParser.parsePermissions(hubConfig.getModulePermissions(), meta.getPermissions()); - for (Resource r : modules.getAssets()) { - if (forceLoad || propertiesModuleManager.hasFileBeenModifiedSinceLastLoaded(r.getFile())) { - InputStream inputStream = r.getInputStream(); - StringHandle handle = new StringHandle(IOUtils.toString(inputStream)); - inputStream.close(); - stagingFlowDocumentWriteSet.add("/flows/" + r.getFilename(), meta, handle); - finalFlowDocumentWriteSet.add("/flows/" + r.getFilename(), meta, handle); - propertiesModuleManager.saveLastLoadedTimestamp(r.getFile(), new Date()); - } - } - return FileVisitResult.CONTINUE; - } - }); - } - - if (stagingStepDocumentWriteSet.size() > 0) { - stagingDocMgr.write(stagingStepDocumentWriteSet); - finalDocMgr.write(stagingStepDocumentWriteSet); - } - if (stagingFlowDocumentWriteSet.size() > 0) { - stagingDocMgr.write(stagingFlowDocumentWriteSet); - finalDocMgr.write(stagingFlowDocumentWriteSet); - } + inputStream = getClass().getResourceAsStream("/hub-internal-artifacts/steps/mapping/marklogic/default-mapping.step.json"); + addToWriteSet(inputStream, "http://marklogic.com/data-hub/step", defaultStepMappingFile, stagingStepDocumentWriteSet, finalStepDocumentWriteSet, propertiesModuleManager); + if (stagingStepDocumentWriteSet.size() > 0) { + stagingDocMgr.write(stagingStepDocumentWriteSet); + finalDocMgr.write(stagingStepDocumentWriteSet); } - catch (IOException e) { - e.printStackTrace(); - //throw new RuntimeException(e); + if (stagingFlowDocumentWriteSet.size() > 0) { + stagingDocMgr.write(stagingFlowDocumentWriteSet); + finalDocMgr.write(stagingFlowDocumentWriteSet); } + } public void setHubConfig(HubConfig hubConfig) { this.hubConfig = hubConfig; } + + private void addToWriteSet(InputStream inputStream, String docCollection, File file, DocumentWriteSet stagingDocumentWriteSet, DocumentWriteSet finalDocumentWriteSet, PropertiesModuleManager propertiesModuleManager) { + try { + StringHandle handle = new StringHandle(IOUtils.toString(inputStream)); + inputStream.close(); + + DocumentMetadataHandle meta = new DocumentMetadataHandle(); + meta.getCollections().add(docCollection); + + if (forceLoad || propertiesModuleManager.hasFileBeenModifiedSinceLastLoaded(file)) { + stagingDocumentWriteSet.add(file.getPath(), meta, handle); + finalDocumentWriteSet.add(file.getPath(), meta, handle); + propertiesModuleManager.saveLastLoadedTimestamp(file, new Date()); + } + } + catch (IOException e) { + e.printStackTrace(); + } + } } From ce5be131f59ae611cbbf2744395e0a7ccfbe0536 Mon Sep 17 00:00:00 2001 From: Akshay Sonvane Date: Mon, 25 Feb 2019 11:08:57 -0800 Subject: [PATCH 2/3] Add LoadHubArtifactsCommand to deploy default steps and flows --- .../com/marklogic/gradle/DataHubPlugin.groovy | 7 ++++ .../gradle/task/DeployHubArtifactsTask.groovy | 32 +++++++++++++++++++ .../com/marklogic/gradle/task/HubTask.groovy | 6 ++++ 3 files changed, 45 insertions(+) create mode 100644 ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/DeployHubArtifactsTask.groovy diff --git a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/DataHubPlugin.groovy b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/DataHubPlugin.groovy index caaf14a76f..7e80b1f7b4 100644 --- a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/DataHubPlugin.groovy +++ b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/DataHubPlugin.groovy @@ -22,6 +22,7 @@ import com.marklogic.appdeployer.impl.SimpleAppDeployer import com.marklogic.gradle.task.* import com.marklogic.hub.ApplicationConfig import com.marklogic.hub.deploy.commands.GeneratePiiCommand +import com.marklogic.hub.deploy.commands.LoadHubArtifactsCommand import com.marklogic.hub.deploy.commands.LoadHubModulesCommand import com.marklogic.hub.deploy.commands.LoadUserArtifactsCommand import com.marklogic.hub.deploy.commands.LoadUserModulesCommand @@ -43,6 +44,7 @@ class DataHubPlugin implements Plugin { private HubProjectImpl hubProject private HubConfigImpl hubConfig private LoadHubModulesCommand loadHubModulesCommand + private LoadHubArtifactsCommand loadHubArtifactsCommand private LoadUserModulesCommand loadUserModulesCommand private LoadUserArtifactsCommand loadUserArtifactsCommand private MappingManagerImpl mappingManager @@ -129,6 +131,9 @@ class DataHubPlugin implements Plugin { description: "Installs user modules from the plugins and src/main/entity-config directories.") .finalizedBy(["hubDeployUserArtifacts"]) + project.task("hubDeployArtifacts", group: deployGroup, type: DeployHubArtifactsTask, + description: "Installs hub artifacts such as default mappings and default flows.") + project.task("hubDeployUserArtifacts", group: deployGroup, type: DeployUserArtifactsTask, description: "Installs user artifacts such as entities and mappings.") @@ -170,6 +175,7 @@ class DataHubPlugin implements Plugin { dataHub = ctx.getBean(DataHubImpl.class) scaffolding = ctx.getBean(ScaffoldingImpl.class) loadHubModulesCommand = ctx.getBean(LoadHubModulesCommand.class) + loadHubArtifactsCommand = ctx.getBean(LoadHubArtifactsCommand.class) loadUserModulesCommand = ctx.getBean(LoadUserModulesCommand.class) loadUserArtifactsCommand = ctx.getBean(LoadUserArtifactsCommand.class) mappingManager = ctx.getBean(MappingManagerImpl.class) @@ -224,6 +230,7 @@ class DataHubPlugin implements Plugin { project.extensions.add("dataHub", dataHub) project.extensions.add("scaffolding", scaffolding) project.extensions.add("loadHubModulesCommand", loadHubModulesCommand) + project.extensions.add("loadHubArtifactsCommand", loadHubArtifactsCommand) project.extensions.add("loadUserModulesCommand", loadUserModulesCommand) project.extensions.add("loadUserArtifactsCommand", loadUserArtifactsCommand) project.extensions.add("mappingManager", mappingManager) diff --git a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/DeployHubArtifactsTask.groovy b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/DeployHubArtifactsTask.groovy new file mode 100644 index 0000000000..eaa0cf2063 --- /dev/null +++ b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/DeployHubArtifactsTask.groovy @@ -0,0 +1,32 @@ +/* + * Copyright 2012-2018 MarkLogic Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.marklogic.gradle.task + +import org.gradle.api.tasks.TaskAction + +class DeployHubArtifactsTask extends HubTask { + + @TaskAction + void deployHubArtifacts() { + def cmd = getLoadHubArtifactsCommand() + cmd.setForceLoad(true) + + cmd.execute(getCommandContext()) + + } +} diff --git a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/HubTask.groovy b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/HubTask.groovy index ebaf858558..852e986482 100644 --- a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/HubTask.groovy +++ b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/HubTask.groovy @@ -23,6 +23,7 @@ import com.marklogic.appdeployer.command.CommandContext import com.marklogic.client.DatabaseClient import com.marklogic.hub.* import com.marklogic.hub.deploy.commands.GeneratePiiCommand +import com.marklogic.hub.deploy.commands.LoadHubArtifactsCommand import com.marklogic.hub.deploy.commands.LoadHubModulesCommand import com.marklogic.hub.deploy.commands.LoadUserArtifactsCommand import com.marklogic.hub.deploy.commands.LoadUserModulesCommand @@ -61,6 +62,11 @@ abstract class HubTask extends DefaultTask { getProject().property("loadHubModulesCommand") } + @Internal + LoadHubArtifactsCommand getLoadHubArtifactsCommand() { + getProject().property("loadHubArtifactsCommand") + } + @Internal LoadUserModulesCommand getLoadUserModulesCommand() { getProject().property("loadUserModulesCommand") From e6d54f47d54720014b76fbaf2381724141e935d6 Mon Sep 17 00:00:00 2001 From: Akshay Sonvane Date: Mon, 25 Feb 2019 15:36:16 -0800 Subject: [PATCH 3/3] Adds capability to iterate over all dirs under flows and steps --- .../commands/LoadHubArtifactsCommand.java | 80 ++++++++++--------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadHubArtifactsCommand.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadHubArtifactsCommand.java index c0da8d7a87..b7e2159290 100644 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadHubArtifactsCommand.java +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadHubArtifactsCommand.java @@ -15,7 +15,6 @@ */ package com.marklogic.hub.deploy.commands; -import com.marklogic.appdeployer.AppConfig; import com.marklogic.appdeployer.command.AbstractCommand; import com.marklogic.appdeployer.command.CommandContext; import com.marklogic.appdeployer.command.SortOrderConstants; @@ -23,13 +22,14 @@ import com.marklogic.client.document.DocumentWriteSet; import com.marklogic.client.document.JSONDocumentManager; import com.marklogic.client.ext.modulesloader.impl.PropertiesModuleManager; -import com.marklogic.client.ext.util.DefaultDocumentPermissionsParser; -import com.marklogic.client.ext.util.DocumentPermissionsParser; import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.io.StringHandle; import com.marklogic.hub.HubConfig; import org.apache.commons.io.IOUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.Resource; +import org.springframework.core.io.support.PathMatchingResourcePatternResolver; +import org.springframework.core.io.support.ResourcePatternResolver; import org.springframework.stereotype.Component; import java.io.File; @@ -46,8 +46,6 @@ public class LoadHubArtifactsCommand extends AbstractCommand { @Autowired private HubConfig hubConfig; - private DocumentPermissionsParser documentPermissionsParser = new DefaultDocumentPermissionsParser(); - public void setForceLoad(boolean forceLoad) { this.forceLoad = forceLoad; } @@ -81,8 +79,6 @@ private PropertiesModuleManager getModulesManager() { @Override public void execute(CommandContext context) { - AppConfig config = context.getAppConfig(); - DatabaseClient stagingClient = hubConfig.newStagingClient(); DatabaseClient finalClient = hubConfig.newFinalClient(); @@ -96,21 +92,49 @@ public void execute(CommandContext context) { PropertiesModuleManager propertiesModuleManager = getModulesManager(); + ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(getClass().getClassLoader()); + Resource[] resources = null; - File defaultFlowFile = new File("/flows/default-flow.flow.json"); - File defaultStepIngestFile = new File("/steps/ingest/marklogic/default-ingest.step.json"); - File defaultStepMappingFile = new File("/steps/mapping/marklogic/default-mapping.step.json"); + try { - // let's do flows - InputStream inputStream = getClass().getResourceAsStream("/hub-internal-artifacts/flows/default-flow.flow.json"); - addToWriteSet(inputStream, "http://marklogic.com/data-hub/flow", defaultFlowFile, stagingFlowDocumentWriteSet, finalFlowDocumentWriteSet, propertiesModuleManager); + // lets do flows + resources = resolver.getResources("classpath*:/hub-internal-artifacts/flows/**/*.flow.json"); + for (Resource r : resources) { + File flowFile = new File("hub-internal-artifacts/flows/" + r.getFilename()); + InputStream inputStream = r.getInputStream(); + StringHandle handle = new StringHandle(IOUtils.toString(inputStream)); + inputStream.close(); + DocumentMetadataHandle meta = new DocumentMetadataHandle(); + meta.getCollections().add("http://marklogic.com/data-hub/flow"); + + if (forceLoad || propertiesModuleManager.hasFileBeenModifiedSinceLastLoaded(flowFile)) { + stagingFlowDocumentWriteSet.add("/flows/" + flowFile.getName(), meta, handle); + finalFlowDocumentWriteSet.add("/flows/" + flowFile.getName(), meta, handle); + propertiesModuleManager.saveLastLoadedTimestamp(flowFile, new Date()); + } + } - // let's do steps - inputStream = getClass().getResourceAsStream("/hub-internal-artifacts/steps/ingest/marklogic/default-ingest.step.json"); - addToWriteSet(inputStream, "http://marklogic.com/data-hub/step", defaultStepIngestFile, stagingStepDocumentWriteSet, finalStepDocumentWriteSet, propertiesModuleManager); - inputStream = getClass().getResourceAsStream("/hub-internal-artifacts/steps/mapping/marklogic/default-mapping.step.json"); - addToWriteSet(inputStream, "http://marklogic.com/data-hub/step", defaultStepMappingFile, stagingStepDocumentWriteSet, finalStepDocumentWriteSet, propertiesModuleManager); + // lets do steps + resources = resolver.getResources("classpath*:/hub-internal-artifacts/steps/**/*.step.json"); + for (Resource r : resources) { + File flowFile = new File("hub-internal-artifacts/steps/" + r.getURL().getPath().substring(r.getURL().getPath().indexOf("hub-internal-artifacts/steps/"))); + InputStream inputStream = r.getInputStream(); + StringHandle handle = new StringHandle(IOUtils.toString(inputStream)); + inputStream.close(); + DocumentMetadataHandle meta = new DocumentMetadataHandle(); + meta.getCollections().add("http://marklogic.com/data-hub/step"); + + if (forceLoad || propertiesModuleManager.hasFileBeenModifiedSinceLastLoaded(flowFile)) { + stagingStepDocumentWriteSet.add("/steps/" + flowFile.getParentFile().getParentFile().getName() + "/" + flowFile.getParentFile().getName() + "/" + flowFile.getName(), meta, handle); + finalStepDocumentWriteSet.add("/steps/" + flowFile.getParentFile().getParentFile().getName() + "/" + flowFile.getParentFile().getName() + "/" + flowFile.getName(), meta, handle); + propertiesModuleManager.saveLastLoadedTimestamp(flowFile, new Date()); + } + } + } + catch (IOException e) { + e.printStackTrace(); + } if (stagingStepDocumentWriteSet.size() > 0) { stagingDocMgr.write(stagingStepDocumentWriteSet); @@ -126,24 +150,4 @@ public void execute(CommandContext context) { public void setHubConfig(HubConfig hubConfig) { this.hubConfig = hubConfig; } - - - private void addToWriteSet(InputStream inputStream, String docCollection, File file, DocumentWriteSet stagingDocumentWriteSet, DocumentWriteSet finalDocumentWriteSet, PropertiesModuleManager propertiesModuleManager) { - try { - StringHandle handle = new StringHandle(IOUtils.toString(inputStream)); - inputStream.close(); - - DocumentMetadataHandle meta = new DocumentMetadataHandle(); - meta.getCollections().add(docCollection); - - if (forceLoad || propertiesModuleManager.hasFileBeenModifiedSinceLastLoaded(file)) { - stagingDocumentWriteSet.add(file.getPath(), meta, handle); - finalDocumentWriteSet.add(file.getPath(), meta, handle); - propertiesModuleManager.saveLastLoadedTimestamp(file, new Date()); - } - } - catch (IOException e) { - e.printStackTrace(); - } - } }