[TOQo1n9M] apoc-hadoop dependency is conflicting (#3450)

extended/src/test/java/apoc/StartupExtendedTest.java

@@ -0,0 +1,136 @@
+package apoc;
+
+import apoc.util.ExtendedTestContainerUtil;
+import apoc.util.Neo4jContainerExtension;
+import apoc.util.TestContainerUtil;
+import apoc.util.TestContainerUtil.Neo4jVersion;
+import org.apache.commons.io.FileUtils;
+import org.junit.Test;
+import org.neo4j.driver.Session;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static apoc.util.TestContainerUtil.ApocPackage.CORE;
+import static apoc.util.TestContainerUtil.ApocPackage.EXTENDED;
+import static apoc.util.TestContainerUtil.createDB;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/*
+ This test is just to verify if the APOC procedures and functions are correctly deployed into a Neo4j instance without any startup issue.
+ */
+public class StartupExtendedTest {
+    private static final String APOC_HELP_QUERY = "CALL apoc.help('') YIELD core, type, name WHERE core = $core and type = $type RETURN name";
+    private static final List<String> EXPECTED_EXTENDED_NAMES;
+
+    static {
+        // retrieve every extended procedure and function via the extended.txt file
+        final File extendedFile = new File(TestContainerUtil.extendedDir, "src/main/resources/extended.txt");
+        try {
+            EXPECTED_EXTENDED_NAMES = FileUtils.readLines(extendedFile, StandardCharsets.UTF_8);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Test
+    public void checkCoreAndFullWithExtraDependenciesJars() {
+        // we check that with apoc-extended, apoc-core jar and all extra-dependencies jars every procedure/function is detected
+        startContainerSessionWithExtraDeps((version) -> createDB(version, List.of(CORE, EXTENDED), true),
+                session -> {
+                    checkCoreProcsAndFuncsExistence(session);
+
+                    // all full procedures and functions are present, also the ones which require extra-deps, e.g. the apoc.export.xls.*
+                    final List<String> actualExtNames = getNames(session, APOC_HELP_QUERY,
+                            Map.of("core", false, "type", "function") );
+                    final List<String> functionExtNames = getNames(session, APOC_HELP_QUERY,
+                            Map.of("core", false, "type", "procedure") );
+
+                    actualExtNames.addAll(functionExtNames);
+
+                    assertEquals(sorted(EXPECTED_EXTENDED_NAMES), sorted(actualExtNames));
+                });
+    }
+
+    @Test
+    public void checkExtendedWithExtraDependenciesJars() {
+        // we check that with apoc-extended jar and all extra-dependencies jars every procedure/function is detected
+        startContainerSessionWithExtraDeps((version) -> createDB(version, List.of(EXTENDED), true),
+                session -> {
+                    // all full procedures and functions are present, also the ones which require extra-deps, e.g. the apoc.export.xls.*
+                    final List<String> actualExtNames = getNames(session, "SHOW PROCEDURES YIELD name WHERE name STARTS WITH 'apoc.' RETURN name");
+                    final List<String> functionExtNames = getNames(session, "SHOW FUNCTIONS YIELD name WHERE name STARTS WITH 'apoc.' RETURN name");
+
+                    actualExtNames.addAll(functionExtNames);
+
+                    assertEquals(sorted(EXPECTED_EXTENDED_NAMES), sorted(actualExtNames));
+                });
+    }
+
+    @Test
+    public void checkCoreWithExtraDependenciesJars() {
+        // we check that with apoc-core jar and all extra-dependencies jars every procedure/function is detected
+        startContainerSessionWithExtraDeps((version) -> createDB(version, List.of(CORE), true),
+                this::checkCoreProcsAndFuncsExistence);
+    }
+
+    private void startContainerSessionWithExtraDeps(Function<Neo4jVersion, Neo4jContainerExtension> neo4jContainerCreation,
+                                                    Consumer<Session> sessionConsumer) {
+        for (var version: Neo4jVersion.values()) {
+
+            try (final Neo4jContainerExtension neo4jContainer = neo4jContainerCreation.apply(version)) {
+                // add extra-deps before starting it
+                ExtendedTestContainerUtil.addExtraDependencies();
+                neo4jContainer.start();
+                assertTrue("Neo4j Instance should be up-and-running", neo4jContainer.isRunning());
+
+                final Session session = neo4jContainer.getSession();
+
+                sessionConsumer.accept(session);
+            } catch (Exception ex) {
+                // if Testcontainers wasn't able to retrieve the docker image we ignore the test
+                if (TestContainerUtil.isDockerImageAvailable(ex)) {
+                    ex.printStackTrace();
+                    fail("Should not have thrown exception when trying to start Neo4j: " + ex);
+                } else {
+                    fail("The docker image could not be loaded. Check whether it's available locally / in the CI. Exception:" + ex);
+                }
+            }
+        }
+    }
+
+    private void checkCoreProcsAndFuncsExistence(Session session) {
+        final List<String> functionNames = getNames(session, APOC_HELP_QUERY,
+                Map.of("core", true, "type", "function") );
+
+        final List<String> procedureNames = getNames(session, APOC_HELP_QUERY,
+                Map.of("core", true, "type", "procedure") );
+
+        assertEquals(sorted(ApocSignatures.PROCEDURES), procedureNames);
+        assertEquals(sorted(ApocSignatures.FUNCTIONS), functionNames);
+    }
+
+    private static List<String> getNames(Session session, String query, Map<String, Object> params) {
+        return session.run(query, params)
+                .list(i -> i.get("name").asString());
+    }
+
+    private static List<String> getNames(Session session, String query) {
+        return getNames(session, query, Collections.emptyMap());
+    }
+
+    private List<String> sorted(List<String> signatures) {
+        return signatures.stream()
+                .sorted()
+                .collect(Collectors.toList());
+    }
+}

extended/src/test/java/apoc/util/ExtendedTestContainerUtil.java

@@ -1,11 +1,18 @@
 package apoc.util;
 
+import java.io.File;
 import java.time.Duration;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Consumer;
+
+import org.apache.commons.io.filefilter.IOFileFilter;
+import org.apache.commons.io.filefilter.WildcardFileFilter;
 import org.neo4j.driver.Session;
 
+import static apoc.util.TestContainerUtil.copyFilesToPlugin;
+import static apoc.util.TestContainerUtil.executeGradleTasks;
+
 public class ExtendedTestContainerUtil
 {
     public static TestcontainersCausalCluster createEnterpriseCluster( List<TestContainerUtil.ApocPackage> apocPackages, int numOfCoreInstances, int numberOfReadReplica, Map<String, Object> neo4jConfig, Map<String, String> envSettings) {
@@ -19,4 +26,16 @@ public static <T> T singleResultFirstColumn(Session session, String cypher) {
     public static void testCallInReadTransaction(Session session, String call, Consumer<Map<String, Object>> consumer) {
         TestContainerUtil.testCallInReadTransaction(session, call, null, consumer);
     }
+
+    public static void addExtraDependencies() {
+        File extraDepsDir = new File(TestContainerUtil.baseDir, "extra-dependencies");
+        // build the extra-dependencies
+        executeGradleTasks(extraDepsDir, "buildDependencies");
+
+        // add all extra deps to the plugin docker folder
+        final File directory = new File(extraDepsDir, "build/allJars");
+        final IOFileFilter instance = new WildcardFileFilter("*-all.jar");
+        copyFilesToPlugin(directory, instance, TestContainerUtil.pluginsFolder);
+    }
+
 }

test-utils/src/main/java/apoc/util/TestContainerUtil.java

@@ -2,6 +2,8 @@
 
 import com.github.dockerjava.api.exception.NotFoundException;
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.IOFileFilter;
+import org.apache.commons.io.filefilter.TrueFileFilter;
 import org.apache.commons.io.filefilter.WildcardFileFilter;
 import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.gradle.tooling.BuildLauncher;
@@ -46,6 +48,7 @@ public enum ApocPackage {
 
     private TestContainerUtil() {}
 
+    private static File pluginsFolder;
     private static File baseDir = Paths.get("..").toFile();
     private static File coreDir = new File(baseDir, "core");
     private static File extendedDir = new File(baseDir, "extended");
@@ -68,8 +71,24 @@ public static Neo4jContainerExtension createEnterpriseDB(List<ApocPackage> apocP
     public static Neo4jContainerExtension createCommunityDB(List<ApocPackage> apocPackages, boolean withLogging) {
         return createNeo4jContainer(apocPackages, withLogging, Neo4jVersion.COMMUNITY);
     }
+
+    private static void addExtraDependencies() {
+        final File projectRootDir = Paths.get("..").toFile();
+        File extraDepsDir = new File(projectRootDir, "extra-dependencies");
+        // build the extra-dependencies
+        executeGradleTasks(extraDepsDir, "buildDependencies");
+
+        // add all extra deps to the plugin docker folder
+        final File directory = new File(extraDepsDir, "build/allJars");
+        final IOFileFilter instance = TrueFileFilter.TRUE;
+        copyFilesToPlugin(directory, instance);
+    }
+
+    public static Neo4jContainerExtension createEnterpriseDB(File baseDir, boolean withLogging, Neo4jVersion version)  {
+        return createEnterpriseDB(baseDir, withLogging, version, false);
+    }
 
-    private static Neo4jContainerExtension createNeo4jContainer(List<ApocPackage> apocPackages, boolean withLogging, Neo4jVersion version) {
+    private static Neo4jContainerExtension createNeo4jContainer(List<ApocPackage> apocPackages, boolean withLogging, Neo4jVersion version, boolean withExtraDeps)  {
         String dockerImage;
         if (version == Neo4jVersion.ENTERPRISE) {
             dockerImage = neo4jEnterpriseDockerImageVersion;
@@ -88,6 +107,7 @@ private static Neo4jContainerExtension createNeo4jContainer(List<ApocPackage> ap
         File importFolder = new File("import");
         importFolder.mkdirs();
         // use a separate folder for mounting plugins jar - build/libs might contain other jars as well.
+        pluginsFolder = new File(baseDir, "build/plugins");
         pluginsFolder.mkdirs();
         String canonicalPath = null;
 
@@ -97,23 +117,20 @@ private static Neo4jContainerExtension createNeo4jContainer(List<ApocPackage> ap
             e.printStackTrace();
         }
 
-        for (ApocPackage apocPackage: apocPackages) {
-            if (apocPackage == ApocPackage.CORE) {
-                projectDir = coreDir;
-            } else {
-                projectDir = extendedDir;
-            }
-
-            executeGradleTasks(projectDir, "shadowJar");
-
-            Collection<File> files = FileUtils.listFiles(new File(projectDir, "build/libs"), new WildcardFileFilter(Arrays.asList("*-extended.jar", "*-core.jar")), null);
-            for (File file: files) {
-                try {
-                    FileUtils.copyFileToDirectory(file, pluginsFolder);
-                } catch (IOException e) {
-                    throw new RuntimeException(e);
-                }
-            }
+        final File directory = new File(baseDir, "build/libs");
+        final IOFileFilter fileFilter = new WildcardFileFilter(Arrays.asList("*-extended.jar", "*-core.jar"));
+
+        copyFilesToPlugin(directory, fileFilter);
+
+        if (withExtraDeps) {
+            addExtraDependencies();
+        }
+
+        String canonicalPath = null;
+        try {
+            canonicalPath = importFolder.getCanonicalPath();
+        } catch (IOException e) {
+            e.printStackTrace();
         }
 
         System.out.println("neo4jDockerImageVersion = " + dockerImage);
@@ -156,6 +173,17 @@ private static Neo4jContainerExtension createNeo4jContainer(List<ApocPackage> ap
         return neo4jContainer.withWaitForNeo4jDatabaseReady(password, version);
     }
 
+    private static void copyFilesToPlugin(File directory, IOFileFilter instance) {
+        Collection<File> files = FileUtils.listFiles(directory, instance, null);
+        for (File file: files) {
+            try {
+                FileUtils.copyFileToDirectory(file, pluginsFolder);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
     public static void executeGradleTasks(File baseDir, String... tasks) {
         try (ProjectConnection connection = GradleConnector.newConnector()
                 .forProjectDirectory(baseDir)