From 4c34948276b4760852620b1337344f2e12380f2b Mon Sep 17 00:00:00 2001 From: ssambasu Date: Wed, 21 Mar 2018 13:10:17 -0700 Subject: [PATCH 1/4] Fix for DHFPROD-412 (Invalidating field cache before every test run). --- .../java/com/marklogic/hub/HubTestBase.java | 2 +- .../java/com/marklogic/hub/TracingTest.java | 59 +++++++++++++++---- 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/HubTestBase.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/HubTestBase.java index d21fd8afa8..d5e73260ed 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/HubTestBase.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/HubTestBase.java @@ -282,7 +282,7 @@ private static void init() { initialized = true; } - private static DatabaseClient getClient(String host, int port, String dbName, String user,String password, Authentication authMethod) throws Exception { + protected static DatabaseClient getClient(String host, int port, String dbName, String user,String password, Authentication authMethod) throws Exception { if(isCertAuth()) { certContext = createSSLContext(getResourceFile("ssl/client-cert.p12")); return DatabaseClientFactory.newClient(host, port, dbName, new DatabaseClientFactory.CertificateAuthContext(certContext,SSLHostnameVerifier.ANY)); diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/TracingTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/TracingTest.java index caf29fc92f..81b510cd3e 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/TracingTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/TracingTest.java @@ -15,7 +15,29 @@ */ package com.marklogic.hub; +import static org.custommonkey.xmlunit.XMLAssert.assertXMLEqual; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import org.custommonkey.xmlunit.XMLUnit; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.w3c.dom.Document; + import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.marklogic.client.DatabaseClient; import com.marklogic.client.document.DocumentRecord; import com.marklogic.client.io.DOMHandle; import com.marklogic.client.io.JacksonHandle; @@ -24,22 +46,13 @@ import com.marklogic.client.query.StructuredQueryBuilder; import com.marklogic.hub.flow.Flow; import com.marklogic.hub.flow.FlowRunner; -import org.custommonkey.xmlunit.XMLUnit; -import org.junit.*; -import org.w3c.dom.Document; - -import java.io.IOException; -import java.net.URISyntaxException; -import java.net.URL; -import java.nio.file.Paths; - -import static org.custommonkey.xmlunit.XMLAssert.assertXMLEqual; -import static org.junit.Assert.*; +import com.marklogic.hub.impl.HubConfigImpl; +import com.marklogic.mgmt.ManageClient; public class TracingTest extends HubTestBase { private static final String BINARY_HEX_ENCODED = "89504E470D0A1A0A0000000D494844520000000D0000001308060000004B378797000000017352474200AECE1CE900000006624B474400FF00FF00FFA0BDA793000000097048597300000B1300000B1301009A9C180000000774494D4507DA0811012332D018A204000002AF4944415428CF9592CB6E1C4500454F555757573FC6F3308E9F40AC286C909122082B36ACF317EC59B2E4A7F8098422A17889402436A3D8C61ECFABBBA7BBEBC502943D573AFB239D2BBEFFE1C7D7428849AA539565D9814A9298EA340A04CE39A15325807BEBBCCB8D591963BE534AABAF46D98CAA2C383898B13FDB23CD6054569455C5FE74C2D9C95159B73B16CB3521C673A9303C7966397AEEE9D47BFE5AFC815639C664E42623D50A21252A4970CED1B4BB4E4A34C5F192A75F248CCEEFF8FDFE35432BF13E60AD63182C2104AC7338EF8921A02C35B7BF1E53FF2610EA635E9C5E90950E44865209B9314829D15A93E7062913A9BC77D44D438C91E9788CD18675BDC5C788F39EC7E58AF5664BBBEB98DFDC320CB6550021044288586769BB0E1F5204026B2D006591635D40A72926334A299532998CD199444A49A14B9E1C8D1915157B7B15FBFB5366D329FD60B9B9BB67D7F7A954B1E2F0C50D2F5FC1F4E22DBF5CFF44B791F8E0E9074BDDB48410E8FB9EAEEFB1838DCAC78EF575C9ADD30CED8CD3598E5001008440FE479224A4A98288508E8E873FA7EC6E2DB3E909CF8FF791DA12A2012211E807CB765BF3F0B0A06E765E451FB1B1C14618A2E261E5C8BB9CB6ED596D362C168FBCBB9E7F501DAC8B4A2A8949A648AFC06AC6E50C530ACABCA42C4BAAAAE4ECF888C13996EB2D314425534A0E2F167CFEADA338FB9BABD51B525191E739459E331E555455894E53ACB5F4D646E93DE8D23239481145CD7CF1166F81183FDCE7DF8E81444A887150CBE68EAB9F3FE5FE32A0D353BEFEE44BCC28A0544296698A3C474A496E0C7BE3316F2E2FBF519F3D3B6F04B2CC32C5A82CF00CDC3F3CD2343BD6DB86F56643BBDBD1B41D8BE5AADF6EEB97EAE4A3C92BA1D2A70879E87C98EEFABE68BA906CB60D4A253ED5BABB9EBF5F393BDCD59BF595526A2E628CFCDFFD03C6146669F7B691AB0000000049454E44AE426082"; - + private static final List clients = new ArrayList(); @BeforeClass public static void setup() throws IOException, URISyntaxException { XMLUnit.setIgnoreWhitespace(true); @@ -52,14 +65,34 @@ public static void setup() throws IOException, URISyntaxException { URL url = TracingTest.class.getClassLoader().getResource("tracing-test"); String path = Paths.get(url.toURI()).toFile().getAbsolutePath(); - installUserModules(getHubConfig(path), true); + + ManageClient manageClient = ((HubConfigImpl)getHubConfig()).getManageClient(); + String resp = manageClient.getJson("/manage/v2/hosts?format=json"); + JsonNode actualObj = new ObjectMapper().readTree(resp); + JsonNode nameNode = actualObj.path("host-default-list").path("list-items"); + List hosts = nameNode.findValuesAsText("nameref"); + hosts.forEach(serverHost -> + { + try { + clients.add(getClient(serverHost, stagingPort, HubConfig.DEFAULT_STAGING_NAME, user, password, stagingAuthMethod)); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + }); } @Before public void beforeEach() { clearDatabases(HubConfig.DEFAULT_JOB_NAME, HubConfig.DEFAULT_TRACE_NAME, HubConfig.DEFAULT_FINAL_NAME); Tracing.create(stagingClient).disable(); + clients.forEach(client -> + { + client.newServerEval().xquery("xquery version \"1.0-ml\";\n" + + "import module namespace hul = \"http://marklogic.com/data-hub/hub-utils-lib\" at \"/MarkLogic/data-hub-framework/impl/hub-utils-lib.xqy\";\n" + + "hul:invalidate-field-cache(\"tracing-enabled\")").eval(); + }); } @AfterClass From fb2472316faa1c864f4c02b7566f75bbfb2bd28c Mon Sep 17 00:00:00 2001 From: ssambasu Date: Wed, 21 Mar 2018 19:51:13 -0700 Subject: [PATCH 2/4] Fixing tests failing after sjs support --- .../src/test/java/com/marklogic/hub/EndToEndFlowTests.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndFlowTests.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndFlowTests.java index 344b167811..e324b776cd 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndFlowTests.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndFlowTests.java @@ -933,7 +933,7 @@ private void testInputFlowViaMlcp(String prefix, String fileSuffix, DatabaseClie String optionString; JsonNode mlcpOptions; try { - optionString = toJsonString(options).replace("\"", "\\\""); + optionString = toJsonString(options).replace("\"", "\\\\\\\""); String optionsJson = "{" + "\"input_file_path\":\"" + inputPath.replace("\\", "\\\\\\\\") + "\"," + @@ -1127,8 +1127,9 @@ private void testInputFlowViaDMSDK(String prefix, String fileSuffix, CodeFormat assertEquals(0, finalCount); assertEquals(0, tracingCount); assertEquals(0, jobsCount); - - ServerTransform serverTransform = new ServerTransform("ml:inputFlow"); + + String transform = codeFormat.equals(CodeFormat.JAVASCRIPT) ? "ml:sjsInputFlow" : "ml:inputFlow"; + ServerTransform serverTransform = new ServerTransform(transform); if (passJobId) { serverTransform.addParameter("job-id", UUID.randomUUID().toString()); } From 2503a4b876bd124b1e13e937d76853021f764b65 Mon Sep 17 00:00:00 2001 From: ssambasu Date: Thu, 22 Mar 2018 14:00:11 -0700 Subject: [PATCH 3/4] Fixing mlcp options and adding cleanup to tests --- .../com/marklogic/hub/EndToEndFlowTests.java | 77 ++++++++++++------- .../com/marklogic/hub/ScaffoldingTest.java | 5 ++ .../java/com/marklogic/hub/TracingTest.java | 2 + 3 files changed, 58 insertions(+), 26 deletions(-) diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndFlowTests.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndFlowTests.java index e324b776cd..7221956643 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndFlowTests.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndFlowTests.java @@ -1,21 +1,24 @@ package com.marklogic.hub; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.marklogic.client.DatabaseClient; -import com.marklogic.client.FailedRequestException; -import com.marklogic.client.datamovement.DataMovementManager; -import com.marklogic.client.datamovement.JobTicket; -import com.marklogic.client.datamovement.WriteBatcher; -import com.marklogic.client.datamovement.WriteEvent; -import com.marklogic.client.document.GenericDocumentManager; -import com.marklogic.client.document.ServerTransform; -import com.marklogic.client.io.*; -import com.marklogic.hub.flow.*; -import com.marklogic.hub.scaffold.Scaffolding; -import com.marklogic.hub.util.FileUtil; -import com.marklogic.hub.util.MlcpRunner; -import com.marklogic.hub.validate.EntitiesValidator; +import static org.custommonkey.xmlunit.XMLAssert.assertXMLEqual; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.Vector; +import java.util.concurrent.TimeUnit; + import org.apache.commons.io.FileUtils; import org.custommonkey.xmlunit.XMLUnit; import org.junit.jupiter.api.AfterAll; @@ -30,15 +33,31 @@ import org.skyscreamer.jsonassert.JSONCompareResult; import org.w3c.dom.Document; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.*; -import java.util.concurrent.TimeUnit; - -import static org.custommonkey.xmlunit.XMLAssert.assertXMLEqual; -import static org.junit.jupiter.api.Assertions.*; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.FailedRequestException; +import com.marklogic.client.datamovement.DataMovementManager; +import com.marklogic.client.datamovement.JobTicket; +import com.marklogic.client.datamovement.WriteBatcher; +import com.marklogic.client.document.GenericDocumentManager; +import com.marklogic.client.document.ServerTransform; +import com.marklogic.client.io.DOMHandle; +import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.io.FileHandle; +import com.marklogic.client.io.Format; +import com.marklogic.client.io.JacksonHandle; +import com.marklogic.client.io.StringHandle; +import com.marklogic.hub.flow.CodeFormat; +import com.marklogic.hub.flow.DataFormat; +import com.marklogic.hub.flow.Flow; +import com.marklogic.hub.flow.FlowBuilder; +import com.marklogic.hub.flow.FlowRunner; +import com.marklogic.hub.flow.FlowType; +import com.marklogic.hub.scaffold.Scaffolding; +import com.marklogic.hub.util.FileUtil; +import com.marklogic.hub.util.MlcpRunner; +import com.marklogic.hub.validate.EntitiesValidator; interface CreateFlowListener { void onFlowCreated(CodeFormat codeFormat, DataFormat dataFormat, FlowType flowType, String srcDir, Path flowDir, boolean useEs); @@ -930,10 +949,16 @@ private void testInputFlowViaMlcp(String prefix, String fileSuffix, DatabaseClie Flow flow = flowManager.getFlow(ENTITY, flowName, FlowType.INPUT); String inputPath = getResourceFile("e2e-test/input/input" + fileSuffix + "." + dataFormat.toString()).getAbsolutePath(); String basePath = getResourceFile("e2e-test/input").getAbsolutePath(); + String OS = System.getProperty("os.name").toLowerCase(); String optionString; JsonNode mlcpOptions; try { - optionString = toJsonString(options).replace("\"", "\\\\\\\""); + if (OS.indexOf("win") >= 0) { + optionString = toJsonString(options).replace("\"", "\\\\\\\""); + } + else { + optionString = toJsonString(options).replace("\"", "\\\""); + } String optionsJson = "{" + "\"input_file_path\":\"" + inputPath.replace("\\", "\\\\\\\\") + "\"," + diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/ScaffoldingTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/ScaffoldingTest.java index c300363568..7391be6aef 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/ScaffoldingTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/ScaffoldingTest.java @@ -62,6 +62,11 @@ public static void setup() throws IOException { isMl9 = getMlMajorVersion() == 9; } + + @AfterAll + public static void teardownFinal() { + uninstallHub(); + } @AfterEach public void teardown() throws IOException { diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/TracingTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/TracingTest.java index 81b510cd3e..89a5c1306c 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/TracingTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/TracingTest.java @@ -99,6 +99,8 @@ public void beforeEach() { public static void teardown() { clearDatabases(HubConfig.DEFAULT_JOB_NAME, HubConfig.DEFAULT_TRACE_NAME, HubConfig.DEFAULT_FINAL_NAME); Tracing.create(stagingClient).disable(); + disableDebugging(); + uninstallHub(); } @Test From ab3ebf593ee68d484c776189cbe4ee83cd7b3534 Mon Sep 17 00:00:00 2001 From: ssambasu Date: Thu, 22 Mar 2018 15:53:51 -0700 Subject: [PATCH 4/4] Adding cleanup to the tests --- .../src/test/java/com/marklogic/hub/DebugLibTest.java | 8 ++++++++ .../test/java/com/marklogic/hub/EndToEndFlowTests.java | 2 +- .../test/java/com/marklogic/hub/EntityManagerTest.java | 1 + .../com/marklogic/hub/collector/StreamCollectorTest.java | 1 + .../test/java/com/marklogic/hub/flow/FlowRunnerTest.java | 1 + 5 files changed, 12 insertions(+), 1 deletion(-) diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/DebugLibTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/DebugLibTest.java index 005a20b6e2..5fa8f35cec 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/DebugLibTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/DebugLibTest.java @@ -9,6 +9,8 @@ import com.marklogic.hub.flow.DataFormat; import com.marklogic.hub.flow.FlowType; import com.marklogic.hub.scaffold.Scaffolding; + +import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -34,6 +36,12 @@ public static void setup() { installUserModules(getHubConfig(), true); } + + @AfterClass + public static void teardown() { + uninstallHub(); + deleteProjectDir(); + } // testing https://github.com/marklogic-community/marklogic-data-hub/issues/516 // when debugging is enable the debug-lib explodes if http post body is multi-part diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndFlowTests.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndFlowTests.java index 7221956643..ea323d7d42 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndFlowTests.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndFlowTests.java @@ -123,7 +123,7 @@ public class EndToEndFlowTests extends HubTestBase { @BeforeAll public static void setup() { XMLUnit.setIgnoreWhitespace(true); - + uninstallHub(); deleteProjectDir(); if(isSslRun() || isCertAuth()) { sslSetup(); diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/EntityManagerTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/EntityManagerTest.java index 5e9c011cc4..86e480a979 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/EntityManagerTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/EntityManagerTest.java @@ -58,6 +58,7 @@ public void clearDbs() { @AfterClass public static void teardown() { + uninstallHub(); deleteProjectDir(); } diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/collector/StreamCollectorTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/collector/StreamCollectorTest.java index 513544cdef..53b81a05fe 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/collector/StreamCollectorTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/collector/StreamCollectorTest.java @@ -116,6 +116,7 @@ public void setupFiles() { @AfterClass public static void teardown() { uninstallHub(); + deleteProjectDir(); } @Test diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/flow/FlowRunnerTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/flow/FlowRunnerTest.java index d78781b88c..382133db49 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/flow/FlowRunnerTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/flow/FlowRunnerTest.java @@ -70,6 +70,7 @@ public static void setup() throws IOException { @AfterClass public static void teardown() { + uninstallHub(); deleteProjectDir(); }