Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix for DHFPROD-412, other failing tests #828

Merged
merged 4 commits into from
Mar 23, 2018
Merged

Fix for DHFPROD-412, other failing tests #828

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4c34948
Fix for DHFPROD-412 (Invalidating field cache before every test run).
srinathgit Mar 21, 2018
fb24723
Fixing tests failing after sjs support
srinathgit Mar 22, 2018
2503a4b
Fixing mlcp options and adding cleanup to tests
srinathgit Mar 22, 2018
ab3ebf5
Adding cleanup to the tests
srinathgit Mar 22, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions marklogic-data-hub/src/test/java/com/marklogic/hub/DebugLibTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
import com.marklogic.hub.flow.DataFormat;
import com.marklogic.hub.flow.FlowType;
import com.marklogic.hub.scaffold.Scaffolding;

import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
Expand All @@ -34,6 +36,12 @@ public static void setup() {

installUserModules(getHubConfig(), true);
}

@AfterClass
public static void teardown() {
uninstallHub();
deleteProjectDir();
}

// testing https://github.com/marklogic-community/marklogic-data-hub/issues/516
// when debugging is enable the debug-lib explodes if http post body is multi-part
Expand Down
84 changes: 55 additions & 29 deletions marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndFlowTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,21 +1,24 @@
package com.marklogic.hub;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.marklogic.client.DatabaseClient;
import com.marklogic.client.FailedRequestException;
import com.marklogic.client.datamovement.DataMovementManager;
import com.marklogic.client.datamovement.JobTicket;
import com.marklogic.client.datamovement.WriteBatcher;
import com.marklogic.client.datamovement.WriteEvent;
import com.marklogic.client.document.GenericDocumentManager;
import com.marklogic.client.document.ServerTransform;
import com.marklogic.client.io.*;
import com.marklogic.hub.flow.*;
import com.marklogic.hub.scaffold.Scaffolding;
import com.marklogic.hub.util.FileUtil;
import com.marklogic.hub.util.MlcpRunner;
import com.marklogic.hub.validate.EntitiesValidator;
import static org.custommonkey.xmlunit.XMLAssert.assertXMLEqual;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;

import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.Vector;
import java.util.concurrent.TimeUnit;

import org.apache.commons.io.FileUtils;
import org.custommonkey.xmlunit.XMLUnit;
import org.junit.jupiter.api.AfterAll;
Expand All @@ -30,15 +33,31 @@
import org.skyscreamer.jsonassert.JSONCompareResult;
import org.w3c.dom.Document;

import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.TimeUnit;

import static org.custommonkey.xmlunit.XMLAssert.assertXMLEqual;
import static org.junit.jupiter.api.Assertions.*;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.marklogic.client.DatabaseClient;
import com.marklogic.client.FailedRequestException;
import com.marklogic.client.datamovement.DataMovementManager;
import com.marklogic.client.datamovement.JobTicket;
import com.marklogic.client.datamovement.WriteBatcher;
import com.marklogic.client.document.GenericDocumentManager;
import com.marklogic.client.document.ServerTransform;
import com.marklogic.client.io.DOMHandle;
import com.marklogic.client.io.DocumentMetadataHandle;
import com.marklogic.client.io.FileHandle;
import com.marklogic.client.io.Format;
import com.marklogic.client.io.JacksonHandle;
import com.marklogic.client.io.StringHandle;
import com.marklogic.hub.flow.CodeFormat;
import com.marklogic.hub.flow.DataFormat;
import com.marklogic.hub.flow.Flow;
import com.marklogic.hub.flow.FlowBuilder;
import com.marklogic.hub.flow.FlowRunner;
import com.marklogic.hub.flow.FlowType;
import com.marklogic.hub.scaffold.Scaffolding;
import com.marklogic.hub.util.FileUtil;
import com.marklogic.hub.util.MlcpRunner;
import com.marklogic.hub.validate.EntitiesValidator;

interface CreateFlowListener {
void onFlowCreated(CodeFormat codeFormat, DataFormat dataFormat, FlowType flowType, String srcDir, Path flowDir, boolean useEs);
Expand Down Expand Up @@ -104,7 +123,7 @@ public class EndToEndFlowTests extends HubTestBase {
@BeforeAll
public static void setup() {
XMLUnit.setIgnoreWhitespace(true);

uninstallHub();
deleteProjectDir();
if(isSslRun() || isCertAuth()) {
sslSetup();
Expand Down Expand Up @@ -930,10 +949,16 @@ private void testInputFlowViaMlcp(String prefix, String fileSuffix, DatabaseClie
Flow flow = flowManager.getFlow(ENTITY, flowName, FlowType.INPUT);
String inputPath = getResourceFile("e2e-test/input/input" + fileSuffix + "." + dataFormat.toString()).getAbsolutePath();
String basePath = getResourceFile("e2e-test/input").getAbsolutePath();
String OS = System.getProperty("os.name").toLowerCase();
String optionString;
JsonNode mlcpOptions;
try {
optionString = toJsonString(options).replace("\"", "\\\"");
if (OS.indexOf("win") >= 0) {
optionString = toJsonString(options).replace("\"", "\\\\\\\"");
}
else {
optionString = toJsonString(options).replace("\"", "\\\"");
}
String optionsJson =
"{" +
"\"input_file_path\":\"" + inputPath.replace("\\", "\\\\\\\\") + "\"," +
Expand Down Expand Up @@ -1127,8 +1152,9 @@ private void testInputFlowViaDMSDK(String prefix, String fileSuffix, CodeFormat
assertEquals(0, finalCount);
assertEquals(0, tracingCount);
assertEquals(0, jobsCount);

ServerTransform serverTransform = new ServerTransform("ml:inputFlow");

String transform = codeFormat.equals(CodeFormat.JAVASCRIPT) ? "ml:sjsInputFlow" : "ml:inputFlow";
ServerTransform serverTransform = new ServerTransform(transform);
if (passJobId) {
serverTransform.addParameter("job-id", UUID.randomUUID().toString());
}
Expand Down
1 change: 1 addition & 0 deletions marklogic-data-hub/src/test/java/com/marklogic/hub/EntityManagerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ public void clearDbs() {

@AfterClass
public static void teardown() {
uninstallHub();
deleteProjectDir();
}

Expand Down
2 changes: 1 addition & 1 deletion marklogic-data-hub/src/test/java/com/marklogic/hub/HubTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,7 @@ private static void init() {
initialized = true;
}

private static DatabaseClient getClient(String host, int port, String dbName, String user,String password, Authentication authMethod) throws Exception {
protected static DatabaseClient getClient(String host, int port, String dbName, String user,String password, Authentication authMethod) throws Exception {
if(isCertAuth()) {
certContext = createSSLContext(getResourceFile("ssl/client-cert.p12"));
return DatabaseClientFactory.newClient(host, port, dbName, new DatabaseClientFactory.CertificateAuthContext(certContext,SSLHostnameVerifier.ANY));
Expand Down
5 changes: 5 additions & 0 deletions marklogic-data-hub/src/test/java/com/marklogic/hub/ScaffoldingTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,11 @@ public static void setup() throws IOException {
isMl9 = getMlMajorVersion() == 9;

}

@AfterAll
public static void teardownFinal() {
uninstallHub();
}

@AfterEach
public void teardown() throws IOException {
Expand Down
61 changes: 48 additions & 13 deletions marklogic-data-hub/src/test/java/com/marklogic/hub/TracingTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,29 @@
*/
package com.marklogic.hub;

import static org.custommonkey.xmlunit.XMLAssert.assertXMLEqual;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;

import org.custommonkey.xmlunit.XMLUnit;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.w3c.dom.Document;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.marklogic.client.DatabaseClient;
import com.marklogic.client.document.DocumentRecord;
import com.marklogic.client.io.DOMHandle;
import com.marklogic.client.io.JacksonHandle;
Expand All @@ -24,22 +46,13 @@
import com.marklogic.client.query.StructuredQueryBuilder;
import com.marklogic.hub.flow.Flow;
import com.marklogic.hub.flow.FlowRunner;
import org.custommonkey.xmlunit.XMLUnit;
import org.junit.*;
import org.w3c.dom.Document;

import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Paths;

import static org.custommonkey.xmlunit.XMLAssert.assertXMLEqual;
import static org.junit.Assert.*;
import com.marklogic.hub.impl.HubConfigImpl;
import com.marklogic.mgmt.ManageClient;

public class TracingTest extends HubTestBase {

private static final String BINARY_HEX_ENCODED = "89504E470D0A1A0A0000000D494844520000000D0000001308060000004B378797000000017352474200AECE1CE900000006624B474400FF00FF00FFA0BDA793000000097048597300000B1300000B1301009A9C180000000774494D4507DA0811012332D018A204000002AF4944415428CF9592CB6E1C4500454F555757573FC6F3308E9F40AC286C909122082B36ACF317EC59B2E4A7F8098422A17889402436A3D8C61ECFABBBA7BBEBC502943D573AFB239D2BBEFFE1C7D7428849AA539565D9814A9298EA340A04CE39A15325807BEBBCCB8D591963BE534AABAF46D98CAA2C383898B13FDB23CD6054569455C5FE74C2D9C95159B73B16CB3521C673A9303C7966397AEEE9D47BFE5AFC815639C664E42623D50A21252A4970CED1B4BB4E4A34C5F192A75F248CCEEFF8FDFE35432BF13E60AD63182C2104AC7338EF8921A02C35B7BF1E53FF2610EA635E9C5E90950E44865209B9314829D15A93E7062913A9BC77D44D438C91E9788CD18675BDC5C788F39EC7E58AF5664BBBEB98DFDC320CB6550021044288586769BB0E1F5204026B2D006591635D40A72926334A299532998CD199444A49A14B9E1C8D1915157B7B15FBFB5366D329FD60B9B9BB67D7F7A954B1E2F0C50D2F5FC1F4E22DBF5CFF44B791F8E0E9074BDDB48410E8FB9EAEEFB1838DCAC78EF575C9ADD30CED8CD3598E5001008440FE479224A4A98288508E8E873FA7EC6E2DB3E909CF8FF791DA12A2012211E807CB765BF3F0B0A06E765E451FB1B1C14618A2E261E5C8BB9CB6ED596D362C168FBCBB9E7F501DAC8B4A2A8949A648AFC06AC6E50C530ACABCA42C4BAAAAE4ECF888C13996EB2D314425534A0E2F167CFEADA338FB9BABD51B525191E739459E331E555455894E53ACB5F4D646E93DE8D23239481145CD7CF1166F81183FDCE7DF8E81444A887150CBE68EAB9F3FE5FE32A0D353BEFEE44BCC28A0544296698A3C474A496E0C7BE3316F2E2FBF519F3D3B6F04B2CC32C5A82CF00CDC3F3CD2343BD6DB86F56643BBDBD1B41D8BE5AADF6EEB97EAE4A3C92BA1D2A70879E87C98EEFABE68BA906CB60D4A253ED5BABB9EBF5F393BDCD59BF595526A2E628CFCDFFD03C6146669F7B691AB0000000049454E44AE426082";

private static final List<DatabaseClient> clients = new ArrayList<DatabaseClient>();
@BeforeClass
public static void setup() throws IOException, URISyntaxException {
XMLUnit.setIgnoreWhitespace(true);
Expand All @@ -52,20 +65,42 @@ public static void setup() throws IOException, URISyntaxException {

URL url = TracingTest.class.getClassLoader().getResource("tracing-test");
String path = Paths.get(url.toURI()).toFile().getAbsolutePath();

installUserModules(getHubConfig(path), true);

ManageClient manageClient = ((HubConfigImpl)getHubConfig()).getManageClient();
String resp = manageClient.getJson("/manage/v2/hosts?format=json");
JsonNode actualObj = new ObjectMapper().readTree(resp);
JsonNode nameNode = actualObj.path("host-default-list").path("list-items");
List<String> hosts = nameNode.findValuesAsText("nameref");
hosts.forEach(serverHost ->
{
try {
clients.add(getClient(serverHost, stagingPort, HubConfig.DEFAULT_STAGING_NAME, user, password, stagingAuthMethod));
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
});
}

@Before
public void beforeEach() {
clearDatabases(HubConfig.DEFAULT_JOB_NAME, HubConfig.DEFAULT_TRACE_NAME, HubConfig.DEFAULT_FINAL_NAME);
Tracing.create(stagingClient).disable();
clients.forEach(client ->
{
client.newServerEval().xquery("xquery version \"1.0-ml\";\n" +
"import module namespace hul = \"http://marklogic.com/data-hub/hub-utils-lib\" at \"/MarkLogic/data-hub-framework/impl/hub-utils-lib.xqy\";\n" +
"hul:invalidate-field-cache(\"tracing-enabled\")").eval();
});
}

@AfterClass
public static void teardown() {
clearDatabases(HubConfig.DEFAULT_JOB_NAME, HubConfig.DEFAULT_TRACE_NAME, HubConfig.DEFAULT_FINAL_NAME);
Tracing.create(stagingClient).disable();
disableDebugging();
uninstallHub();
}

@Test
Expand Down
1 change: 1 addition & 0 deletions marklogic-data-hub/src/test/java/com/marklogic/hub/collector/StreamCollectorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ public void setupFiles() {
@AfterClass
public static void teardown() {
uninstallHub();
deleteProjectDir();
}

@Test
Expand Down
1 change: 1 addition & 0 deletions marklogic-data-hub/src/test/java/com/marklogic/hub/flow/FlowRunnerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ public static void setup() throws IOException {

@AfterClass
public static void teardown() {
uninstallHub();
deleteProjectDir();
}

Expand Down