Fixes #4207: Integration Tests for Load Procedures with Cloud Object …

…Storage (#4226) (#4299)

* Fixes #4207: Integration Tests for Load Procedures with Cloud Object Storage

* added google cloud tests

* wip - adding other procs

* cleanup

* added jsonParams proc

* fix tests

* fix extended-it tests

* cleanup

* fix NoFileFound errors

* fixed tests due to missing apoc config

* added ignored export tests

extended-it/build.gradle

@@ -28,6 +28,8 @@ dependencies {
     }
 
     implementation project(":extended")
+
+    testImplementation group: 'com.google.cloud', name: 'google-cloud-storage', version: '2.26.1'
     testImplementation project(':extended').sourceSets.main.allJava
 
     testImplementation group: 'us.fatehi', name: 'schemacrawler-mysql', version: '16.20.8'
@@ -55,7 +57,9 @@ dependencies {
     testImplementation group: 'org.testcontainers', name: 'chromadb', version: '1.20.2'
     testImplementation group: 'org.testcontainers', name: 'weaviate', version: '1.20.2'
     testImplementation group: 'org.testcontainers', name: 'milvus', version: '1.20.2'
-
+    testImplementation group: 'org.apache.poi', name: 'poi', version: '5.1.0'
+    testImplementation group: 'org.apache.poi', name: 'poi-ooxml', version: '5.1.0'
+    testImplementation 'com.azure:azure-storage-blob:12.22.0'
     configurations.all {
         exclude group: 'org.slf4j', module: 'slf4j-nop'
         exclude group: 'ch.qos.logback', module: 'logback-classic'

extended-it/src/test/java/apoc/azure/ArrowAzureStorageTest.java

@@ -0,0 +1,62 @@
+package apoc.azure;
+
+import apoc.export.arrow.ArrowTestUtil;
+import apoc.util.s3.S3BaseTest;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+import org.neo4j.configuration.GraphDatabaseInternalSettings;
+import org.neo4j.test.rule.DbmsRule;
+import org.neo4j.test.rule.ImpermanentDbmsRule;
+
+import java.util.Map;
+
+import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED;
+import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED;
+import static apoc.ApocConfig.apocConfig;
+import static apoc.export.arrow.ArrowTestUtil.initDbCommon;
+import static apoc.export.arrow.ArrowTestUtil.testImportCommon;
+import static apoc.export.arrow.ArrowTestUtil.testLoadArrow;
+
+@Ignore("This test won't work until the Azure Storage files will be correctly handled via FileUtils, placed in APOC Core")
+public class ArrowAzureStorageTest extends AzureStorageBaseTest {
+
+    @Rule
+    public DbmsRule db = new ImpermanentDbmsRule()
+            .withSetting(GraphDatabaseInternalSettings.enable_experimental_cypher_versions, true);
+
+    @Before
+    public void beforeClass() {
+        initDbCommon(db);
+        apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true);
+        apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true);
+    }
+
+    @Test
+    public void testFileRoundtripWithLoadArrow() {
+        String url = putToAzureStorageAndGetUrl("test_all.arrow");
+
+        String file = db.executeTransactionally("CYPHER 25 CALL apoc.export.arrow.all($url) YIELD file",
+                Map.of("url", url),
+                ArrowTestUtil::extractFileName);
+
+        // check that the exported file is correct
+        final String query = "CYPHER 25 CALL apoc.load.arrow($file, {})";
+        testLoadArrow(db, query, Map.of("file", file));
+    }
+
+
+    @Test
+    public void testFileRoundtripWithImportArrow() {
+        db.executeTransactionally("CREATE (:Another {foo:1, listInt: [1,2]}), (:Another {bar:'Sam'})");
+
+        String url = putToAzureStorageAndGetUrl("test_all_import.arrow");
+        String file = db.executeTransactionally("CYPHER 25 CALL apoc.export.arrow.all($url) YIELD file",
+                Map.of("url", url),
+                ArrowTestUtil::extractFileName);
+
+        // check that the exported file is correct
+        testImportCommon(db, file, ArrowTestUtil.MAPPING_ALL);
+    }
+}

extended-it/src/test/java/apoc/azure/AzureStorageBaseTest.java

@@ -0,0 +1,74 @@
+package apoc.azure;
+
+import com.azure.core.util.Context;
+import com.azure.storage.blob.BlobClient;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.BlobContainerClientBuilder;
+import com.azure.storage.blob.sas.BlobSasPermission;
+import com.azure.storage.blob.sas.BlobServiceSasSignatureValues;
+import org.apache.commons.io.FileUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.utility.DockerImageName;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.time.OffsetDateTime;
+import java.util.UUID;
+
+public class AzureStorageBaseTest {
+
+    public static GenericContainer<?> azuriteContainer;
+    public static BlobContainerClient containerClient;
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        DockerImageName azuriteImg = DockerImageName.parse("mcr.microsoft.com/azure-storage/azurite");
+        azuriteContainer = new GenericContainer<>(azuriteImg)
+                .withExposedPorts(10000);
+
+        azuriteContainer.start();
+
+        var accountName = "devstoreaccount1";
+        var accountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
+        var blobEndpoint = "http://%s:%d/%s".formatted(azuriteContainer.getHost(), azuriteContainer.getMappedPort(10000), accountName);
+        var connectionString = "DefaultEndpointsProtocol=http;AccountName=%s;AccountKey=%s;BlobEndpoint=%s;"
+                .formatted(accountName, accountKey, blobEndpoint);
+
+        containerClient = new BlobContainerClientBuilder()
+                .connectionString(connectionString)
+                .containerName("test-container")
+                .buildClient();
+        containerClient.create();
+    }    
+
+    @AfterClass
+    public static void teardown() {
+        azuriteContainer.close();
+    }
+
+    public static String putToAzureStorageAndGetUrl(String url) {
+        try {
+            File file = new File(url);
+            byte[] content = FileUtils.readFileToByteArray(file);
+
+            var blobClient = getBlobClient(content);
+            BlobSasPermission permission = new BlobSasPermission().setReadPermission(true);
+            OffsetDateTime expiryTime = OffsetDateTime.now().plusHours(1);
+            String sasToken = blobClient.generateSas(new BlobServiceSasSignatureValues(expiryTime, permission), new Context("Azure-Storage-Log-String-To-Sign", "true"));
+            return blobClient.getBlobUrl() + "?" + sasToken;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static BlobClient getBlobClient(byte[] content) {
+        var blobName = "blob-" + UUID.randomUUID();
+        var blobClient = containerClient.getBlobClient(blobName);
+        blobClient.upload(new ByteArrayInputStream(content));
+        return blobClient;
+    }
+
+}

extended-it/src/test/java/apoc/azure/ImportAzureStorageTest.java

@@ -0,0 +1,51 @@
+package apoc.azure;
+
+import apoc.load.Gexf;
+import apoc.util.TestUtil;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.neo4j.test.rule.DbmsRule;
+import org.neo4j.test.rule.ImpermanentDbmsRule;
+
+import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED;
+import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED;
+import static apoc.ApocConfig.apocConfig;
+import static apoc.export.arrow.ArrowTestUtil.MAPPING_ALL;
+import static apoc.export.arrow.ArrowTestUtil.initDbCommon;
+import static apoc.export.arrow.ArrowTestUtil.createNodesForImportTests;
+import static apoc.export.arrow.ArrowTestUtil.testImportCommon;
+import static apoc.util.ExtendedITUtil.EXTENDED_RESOURCES_PATH;
+import static apoc.util.GexfTestUtil.testImportGexfCommon;
+
+public class ImportAzureStorageTest extends AzureStorageBaseTest {
+
+    @Rule
+    public DbmsRule db = new ImpermanentDbmsRule();
+
+    @Before
+    public void beforeClass() {
+        apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true);
+        apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true);
+    }
+
+    @Test
+    public void testImportArrow() {
+        initDbCommon(db);
+        createNodesForImportTests(db);
+
+        String fileWithPath = EXTENDED_RESOURCES_PATH + "test_all.arrow";
+        String url = putToAzureStorageAndGetUrl(fileWithPath);
+
+        testImportCommon(db, url, MAPPING_ALL);
+    }
+
+    @Test
+    public void testImportGexf() {
+        TestUtil.registerProcedure(db, Gexf.class);
+
+        String filename = EXTENDED_RESOURCES_PATH + "gexf/data.gexf";
+        String url = putToAzureStorageAndGetUrl(filename);
+        testImportGexfCommon(db, url);
+    }
+}

extended-it/src/test/java/apoc/azure/LoadAzureStorageTest.java

@@ -0,0 +1,74 @@
+package apoc.azure;
+
+import apoc.load.LoadCsv;
+import apoc.load.LoadDirectory;
+import apoc.load.LoadHtml;
+import apoc.load.LoadJsonExtended;
+import apoc.load.Xml;
+import apoc.load.xls.LoadXls;
+import apoc.util.TestUtil;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.neo4j.configuration.GraphDatabaseInternalSettings;
+import org.neo4j.test.rule.DbmsRule;
+import org.neo4j.test.rule.ImpermanentDbmsRule;
+
+import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED;
+import static apoc.ApocConfig.APOC_IMPORT_FILE_USE_NEO4J_CONFIG;
+import static apoc.ApocConfig.apocConfig;
+import static apoc.load.LoadCsvTest.commonTestLoadCsv;
+import static apoc.load.LoadHtmlTest.testLoadHtmlWithGetLinksCommon;
+import static apoc.load.xls.LoadXlsTest.testLoadXlsCommon;
+import static apoc.util.ExtendedITUtil.EXTENDED_RESOURCES_PATH;
+import static apoc.util.ExtendedITUtil.testLoadJsonCommon;
+import static apoc.util.ExtendedITUtil.testLoadXmlCommon;
+
+
+public class LoadAzureStorageTest extends AzureStorageBaseTest {
+
+    @ClassRule
+    public static DbmsRule db = new ImpermanentDbmsRule()
+            .withSetting(GraphDatabaseInternalSettings.enable_experimental_cypher_versions, true);
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        AzureStorageBaseTest.setUp();
+
+        TestUtil.registerProcedure(db, LoadCsv.class, LoadDirectory.class, LoadJsonExtended.class, LoadHtml.class, LoadXls.class, Xml.class);
+        apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true);
+        apocConfig().setProperty(APOC_IMPORT_FILE_USE_NEO4J_CONFIG, false);
+    }
+
+
+    @Test
+    public void testLoadCsv() {
+        String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "test.csv");
+        commonTestLoadCsv(db, url);
+    }
+
+    @Test
+    public void testLoadJson() {
+        String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "map.json");
+        testLoadJsonCommon(db, url);
+    }
+
+    @Test
+    public void testLoadXml() {
+        String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "xml/books.xml");
+        testLoadXmlCommon(db, url);
+    }
+
+    @Test
+    public void testLoadXls() {
+        String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "load_test.xlsx");
+        testLoadXlsCommon(db, url);
+    }
+
+    @Test
+    public void testLoadHtml() {
+        String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "wikipedia.html");
+        testLoadHtmlWithGetLinksCommon(db, url);
+    }
+
+}

extended-it/src/test/java/apoc/azure/ParquetAzureStorageTest.java

@@ -0,0 +1,72 @@
+package apoc.azure;
+
+import apoc.export.parquet.ParquetTestUtil;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.neo4j.test.rule.DbmsRule;
+import org.neo4j.test.rule.ImpermanentDbmsRule;
+
+import java.util.Map;
+
+import static apoc.export.parquet.ParquetTest.MAPPING_ALL;
+import static apoc.export.parquet.ParquetTestUtil.beforeClassCommon;
+import static apoc.export.parquet.ParquetTestUtil.beforeCommon;
+import static apoc.export.parquet.ParquetTestUtil.testImportAllCommon;
+import static apoc.util.ExtendedITUtil.EXTENDED_RESOURCES_PATH;
+import static apoc.util.GoogleCloudStorageContainerExtension.gcsUrl;
+import static apoc.util.TestUtil.testResult;
+
+public class ParquetAzureStorageTest extends AzureStorageBaseTest {
+
+    private final String EXPORT_FILENAME = "test_all.parquet";
+
+    @ClassRule
+    public static DbmsRule db = new ImpermanentDbmsRule();
+
+    @BeforeClass
+    public static void beforeClass() {
+        beforeClassCommon(db);
+    }
+
+    @Before
+    public void before() {
+        beforeCommon(db);
+    }
+
+    @Test
+    @Ignore("This test won't work until the Azure Storage files will be correctly handled via FileUtils, placed in APOC Core")
+    public void testFileRoundtripParquetAll() {
+        // given - when
+        String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + EXPORT_FILENAME);
+        String file = db.executeTransactionally("CALL apoc.export.parquet.all($url) YIELD file",
+                Map.of("url", url),
+                ParquetTestUtil::extractFileName);
+
+        // then
+        final String query = "CALL apoc.load.parquet($file, $config) YIELD value " +
+                             "RETURN value";
+
+        testResult(db, query, Map.of("file", file,  "config", MAPPING_ALL),
+                ParquetTestUtil::roundtripLoadAllAssertions);
+    }
+    @Test
+    public void testLoadParquet() {
+        String query = "CALL apoc.load.parquet($url, $config) YIELD value " +
+                       "RETURN value";
+
+        String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + EXPORT_FILENAME);
+        testResult(db, query, Map.of("url", url,  "config", MAPPING_ALL),
+                ParquetTestUtil::roundtripLoadAllAssertions);
+    }
+
+    @Test
+    public void testImportParquet() {
+        String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + EXPORT_FILENAME);
+
+        Map<String, Object> params = Map.of("file", url, "config", MAPPING_ALL);
+        testImportAllCommon(db, params);
+    }
+}