diff --git a/extended/src/test/java/apoc/StartupExtendedTest.java b/extended/src/test/java/apoc/StartupExtendedTest.java new file mode 100644 index 0000000000..1841d4e90b --- /dev/null +++ b/extended/src/test/java/apoc/StartupExtendedTest.java @@ -0,0 +1,136 @@ +package apoc; + +import apoc.util.ExtendedTestContainerUtil; +import apoc.util.Neo4jContainerExtension; +import apoc.util.TestContainerUtil; +import apoc.util.TestContainerUtil.Neo4jVersion; +import org.apache.commons.io.FileUtils; +import org.junit.Test; +import org.neo4j.driver.Session; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static apoc.util.TestContainerUtil.ApocPackage.CORE; +import static apoc.util.TestContainerUtil.ApocPackage.EXTENDED; +import static apoc.util.TestContainerUtil.createDB; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/* + This test is just to verify if the APOC procedures and functions are correctly deployed into a Neo4j instance without any startup issue. + */ +public class StartupExtendedTest { + private static final String APOC_HELP_QUERY = "CALL apoc.help('') YIELD core, type, name WHERE core = $core and type = $type RETURN name"; + private static final List EXPECTED_EXTENDED_NAMES; + + static { + // retrieve every extended procedure and function via the extended.txt file + final File extendedFile = new File(TestContainerUtil.extendedDir, "src/main/resources/extended.txt"); + try { + EXPECTED_EXTENDED_NAMES = FileUtils.readLines(extendedFile, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Test + public void checkCoreAndFullWithExtraDependenciesJars() { + // we check that with apoc-extended, apoc-core jar and all extra-dependencies jars every procedure/function is detected + startContainerSessionWithExtraDeps((version) -> createDB(version, List.of(CORE, EXTENDED), true), + session -> { + checkCoreProcsAndFuncsExistence(session); + + // all full procedures and functions are present, also the ones which require extra-deps, e.g. the apoc.export.xls.* + final List actualExtNames = getNames(session, APOC_HELP_QUERY, + Map.of("core", false, "type", "function") ); + final List functionExtNames = getNames(session, APOC_HELP_QUERY, + Map.of("core", false, "type", "procedure") ); + + actualExtNames.addAll(functionExtNames); + + assertEquals(sorted(EXPECTED_EXTENDED_NAMES), sorted(actualExtNames)); + }); + } + + @Test + public void checkExtendedWithExtraDependenciesJars() { + // we check that with apoc-extended jar and all extra-dependencies jars every procedure/function is detected + startContainerSessionWithExtraDeps((version) -> createDB(version, List.of(EXTENDED), true), + session -> { + // all full procedures and functions are present, also the ones which require extra-deps, e.g. the apoc.export.xls.* + final List actualExtNames = getNames(session, "SHOW PROCEDURES YIELD name WHERE name STARTS WITH 'apoc.' RETURN name"); + final List functionExtNames = getNames(session, "SHOW FUNCTIONS YIELD name WHERE name STARTS WITH 'apoc.' RETURN name"); + + actualExtNames.addAll(functionExtNames); + + assertEquals(sorted(EXPECTED_EXTENDED_NAMES), sorted(actualExtNames)); + }); + } + + @Test + public void checkCoreWithExtraDependenciesJars() { + // we check that with apoc-core jar and all extra-dependencies jars every procedure/function is detected + startContainerSessionWithExtraDeps((version) -> createDB(version, List.of(CORE), true), + this::checkCoreProcsAndFuncsExistence); + } + + private void startContainerSessionWithExtraDeps(Function neo4jContainerCreation, + Consumer sessionConsumer) { + for (var version: Neo4jVersion.values()) { + + try (final Neo4jContainerExtension neo4jContainer = neo4jContainerCreation.apply(version)) { + // add extra-deps before starting it + ExtendedTestContainerUtil.addExtraDependencies(); + neo4jContainer.start(); + assertTrue("Neo4j Instance should be up-and-running", neo4jContainer.isRunning()); + + final Session session = neo4jContainer.getSession(); + + sessionConsumer.accept(session); + } catch (Exception ex) { + // if Testcontainers wasn't able to retrieve the docker image we ignore the test + if (TestContainerUtil.isDockerImageAvailable(ex)) { + ex.printStackTrace(); + fail("Should not have thrown exception when trying to start Neo4j: " + ex); + } else { + fail("The docker image could not be loaded. Check whether it's available locally / in the CI. Exception:" + ex); + } + } + } + } + + private void checkCoreProcsAndFuncsExistence(Session session) { + final List functionNames = getNames(session, APOC_HELP_QUERY, + Map.of("core", true, "type", "function") ); + + final List procedureNames = getNames(session, APOC_HELP_QUERY, + Map.of("core", true, "type", "procedure") ); + + assertEquals(sorted(ApocSignatures.PROCEDURES), procedureNames); + assertEquals(sorted(ApocSignatures.FUNCTIONS), functionNames); + } + + private static List getNames(Session session, String query, Map params) { + return session.run(query, params) + .list(i -> i.get("name").asString()); + } + + private static List getNames(Session session, String query) { + return getNames(session, query, Collections.emptyMap()); + } + + private List sorted(List signatures) { + return signatures.stream() + .sorted() + .collect(Collectors.toList()); + } +} \ No newline at end of file diff --git a/extended/src/test/java/apoc/util/ExtendedTestContainerUtil.java b/extended/src/test/java/apoc/util/ExtendedTestContainerUtil.java index 74342d2777..8c85a4085e 100644 --- a/extended/src/test/java/apoc/util/ExtendedTestContainerUtil.java +++ b/extended/src/test/java/apoc/util/ExtendedTestContainerUtil.java @@ -1,11 +1,18 @@ package apoc.util; +import java.io.File; import java.time.Duration; import java.util.List; import java.util.Map; import java.util.function.Consumer; + +import org.apache.commons.io.filefilter.IOFileFilter; +import org.apache.commons.io.filefilter.WildcardFileFilter; import org.neo4j.driver.Session; +import static apoc.util.TestContainerUtil.copyFilesToPlugin; +import static apoc.util.TestContainerUtil.executeGradleTasks; + public class ExtendedTestContainerUtil { public static TestcontainersCausalCluster createEnterpriseCluster( List apocPackages, int numOfCoreInstances, int numberOfReadReplica, Map neo4jConfig, Map envSettings) { @@ -19,4 +26,16 @@ public static T singleResultFirstColumn(Session session, String cypher) { public static void testCallInReadTransaction(Session session, String call, Consumer> consumer) { TestContainerUtil.testCallInReadTransaction(session, call, null, consumer); } + + public static void addExtraDependencies() { + File extraDepsDir = new File(TestContainerUtil.baseDir, "extra-dependencies"); + // build the extra-dependencies + executeGradleTasks(extraDepsDir, "buildDependencies"); + + // add all extra deps to the plugin docker folder + final File directory = new File(extraDepsDir, "build/allJars"); + final IOFileFilter instance = new WildcardFileFilter("*-all.jar"); + copyFilesToPlugin(directory, instance, TestContainerUtil.pluginsFolder); + } + } diff --git a/test-startup/src/test/java/StartupTest.java b/test-startup/src/test/java/StartupTest.java deleted file mode 100644 index 0cb5c32634..0000000000 --- a/test-startup/src/test/java/StartupTest.java +++ /dev/null @@ -1,86 +0,0 @@ -import apoc.ApocSignatures; -import apoc.util.Neo4jContainerExtension; -import apoc.util.TestContainerUtil; -import apoc.util.TestUtil; -import apoc.util.TestContainerUtil.Neo4jVersion; -import apoc.util.TestContainerUtil.ApocPackage; -import org.junit.Test; - -import org.neo4j.driver.Session; - -import java.util.List; -import java.util.stream.Collectors; - -import static apoc.util.TestContainerUtil.createDB; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/* - This test is just to verify if the APOC procedures and functions are correctly deployed into a Neo4j instance without any startup issue. - If you don't have docker installed it will fail, and you can simply ignore it. - */ -public class StartupTest { - - @Test - public void check_basic_deployment() { - for (var version: Neo4jVersion.values()) { - try (Neo4jContainerExtension neo4jContainer = createDB(version, List.of(ApocPackage.CORE), !TestUtil.isRunningInCI()) - .withNeo4jConfig("dbms.transaction.timeout", "60s")) { - - neo4jContainer.start(); - assertTrue("Neo4j Instance should be up-and-running", neo4jContainer.isRunning()); - - Session session = neo4jContainer.getSession(); - int procedureCount = session.run("SHOW PROCEDURES YIELD name WHERE name STARTS WITH 'apoc' RETURN count(*) AS count").peek().get("count").asInt(); - int functionCount = session.run("SHOW FUNCTIONS YIELD name WHERE name STARTS WITH 'apoc' RETURN count(*) AS count").peek().get("count").asInt(); - int coreCount = session.run("CALL apoc.help('') YIELD core WHERE core = true RETURN count(*) AS count").peek().get("count").asInt(); - - assertTrue(procedureCount > 0); - assertTrue(functionCount > 0); - assertTrue(coreCount > 0); - } catch (Exception ex) { - // if Testcontainers wasn't able to retrieve the docker image we ignore the test - if (TestContainerUtil.isDockerImageAvailable(ex)) { - ex.printStackTrace(); - fail("Should not have thrown exception when trying to start Neo4j: " + ex); - } else if (!TestUtil.isRunningInCI()) { - fail( "The docker image " + TestContainerUtil.neo4jEnterpriseDockerImageVersion + " should be available in the CI. Exception:" + ex); - } - } - } - } - - @Test - public void compare_with_sources() { - for (var version: Neo4jVersion.values()) { - try (Neo4jContainerExtension neo4jContainer = createDB(version, List.of(ApocPackage.CORE), !TestUtil.isRunningInCI())) { - neo4jContainer.start(); - - assertTrue("Neo4j Instance should be up-and-running", neo4jContainer.isRunning()); - - try (Session session = neo4jContainer.getSession()) { - final List functionNames = session.run("CALL apoc.help('') YIELD core, type, name WHERE core = true and type = 'function' RETURN name") - .list(record -> record.get("name").asString()); - final List procedureNames = session.run("CALL apoc.help('') YIELD core, type, name WHERE core = true and type = 'procedure' RETURN name") - .list(record -> record.get("name").asString()); - - - assertEquals(sorted(ApocSignatures.PROCEDURES), procedureNames); - assertEquals(sorted(ApocSignatures.FUNCTIONS), functionNames); - } - } catch (Exception ex) { - if (TestContainerUtil.isDockerImageAvailable(ex)) { - ex.printStackTrace(); - fail("Should not have thrown exception when trying to start Neo4j: " + ex); - } else { - fail( "The docker image " + TestContainerUtil.neo4jEnterpriseDockerImageVersion + " should be available in the CI. Exception:" + ex); - } - } - } - } - - private List sorted(List signatures) { - return signatures.stream().sorted().collect(Collectors.toList()); - } -} diff --git a/test-utils/src/main/java/apoc/util/TestContainerUtil.java b/test-utils/src/main/java/apoc/util/TestContainerUtil.java index 6996e51866..43c7f522bf 100644 --- a/test-utils/src/main/java/apoc/util/TestContainerUtil.java +++ b/test-utils/src/main/java/apoc/util/TestContainerUtil.java @@ -2,6 +2,8 @@ import com.github.dockerjava.api.exception.NotFoundException; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.IOFileFilter; +import org.apache.commons.io.filefilter.TrueFileFilter; import org.apache.commons.io.filefilter.WildcardFileFilter; import org.apache.commons.lang3.exception.ExceptionUtils; import org.gradle.tooling.BuildLauncher; @@ -46,6 +48,7 @@ public enum ApocPackage { private TestContainerUtil() {} + private static File pluginsFolder; private static File baseDir = Paths.get("..").toFile(); private static File coreDir = new File(baseDir, "core"); private static File extendedDir = new File(baseDir, "extended"); @@ -68,8 +71,24 @@ public static Neo4jContainerExtension createEnterpriseDB(List apocP public static Neo4jContainerExtension createCommunityDB(List apocPackages, boolean withLogging) { return createNeo4jContainer(apocPackages, withLogging, Neo4jVersion.COMMUNITY); } + + private static void addExtraDependencies() { + final File projectRootDir = Paths.get("..").toFile(); + File extraDepsDir = new File(projectRootDir, "extra-dependencies"); + // build the extra-dependencies + executeGradleTasks(extraDepsDir, "buildDependencies"); + + // add all extra deps to the plugin docker folder + final File directory = new File(extraDepsDir, "build/allJars"); + final IOFileFilter instance = TrueFileFilter.TRUE; + copyFilesToPlugin(directory, instance); + } + + public static Neo4jContainerExtension createEnterpriseDB(File baseDir, boolean withLogging, Neo4jVersion version) { + return createEnterpriseDB(baseDir, withLogging, version, false); + } - private static Neo4jContainerExtension createNeo4jContainer(List apocPackages, boolean withLogging, Neo4jVersion version) { + private static Neo4jContainerExtension createNeo4jContainer(List apocPackages, boolean withLogging, Neo4jVersion version, boolean withExtraDeps) { String dockerImage; if (version == Neo4jVersion.ENTERPRISE) { dockerImage = neo4jEnterpriseDockerImageVersion; @@ -88,6 +107,7 @@ private static Neo4jContainerExtension createNeo4jContainer(List ap File importFolder = new File("import"); importFolder.mkdirs(); // use a separate folder for mounting plugins jar - build/libs might contain other jars as well. + pluginsFolder = new File(baseDir, "build/plugins"); pluginsFolder.mkdirs(); String canonicalPath = null; @@ -97,23 +117,20 @@ private static Neo4jContainerExtension createNeo4jContainer(List ap e.printStackTrace(); } - for (ApocPackage apocPackage: apocPackages) { - if (apocPackage == ApocPackage.CORE) { - projectDir = coreDir; - } else { - projectDir = extendedDir; - } - - executeGradleTasks(projectDir, "shadowJar"); - - Collection files = FileUtils.listFiles(new File(projectDir, "build/libs"), new WildcardFileFilter(Arrays.asList("*-extended.jar", "*-core.jar")), null); - for (File file: files) { - try { - FileUtils.copyFileToDirectory(file, pluginsFolder); - } catch (IOException e) { - throw new RuntimeException(e); - } - } + final File directory = new File(baseDir, "build/libs"); + final IOFileFilter fileFilter = new WildcardFileFilter(Arrays.asList("*-extended.jar", "*-core.jar")); + + copyFilesToPlugin(directory, fileFilter); + + if (withExtraDeps) { + addExtraDependencies(); + } + + String canonicalPath = null; + try { + canonicalPath = importFolder.getCanonicalPath(); + } catch (IOException e) { + e.printStackTrace(); } System.out.println("neo4jDockerImageVersion = " + dockerImage); @@ -156,6 +173,17 @@ private static Neo4jContainerExtension createNeo4jContainer(List ap return neo4jContainer.withWaitForNeo4jDatabaseReady(password, version); } + private static void copyFilesToPlugin(File directory, IOFileFilter instance) { + Collection files = FileUtils.listFiles(directory, instance, null); + for (File file: files) { + try { + FileUtils.copyFileToDirectory(file, pluginsFolder); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + public static void executeGradleTasks(File baseDir, String... tasks) { try (ProjectConnection connection = GradleConnector.newConnector() .forProjectDirectory(baseDir)