Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[EdwwgWgR] Stream schema results in batches of batchSize #3620

Merged
merged 1 commit into from
Jun 14, 2023
Merged

[EdwwgWgR] Stream schema results in batches of batchSize #3620

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 37 additions & 38 deletions core-it/src/test/java/apoc/core/it/ExportCypherS3Test.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import org.neo4j.test.rule.DbmsRule;
import org.neo4j.test.rule.ImpermanentDbmsRule;

import java.io.IOException;
import java.util.Map;

import static apoc.export.cypher.ExportCypherTest.ExportCypherResults.*;
Expand All @@ -48,13 +47,13 @@ public class ExportCypherS3Test extends S3BaseTest {
public TestName testName = new TestName();

@Before
public void setUp() throws Exception {
public void setUp() {
ExportCypherTestUtils.setUp( db, testName);
}

// -- Whole file test -- //
@Test
public void testExportAllCypherDefault() throws Exception {
public void testExportAllCypherDefault() {
String fileName = "all.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{useOptimizations: { type: 'none'}, format: 'neo4j-shell'})",
Expand All @@ -64,7 +63,7 @@ public void testExportAllCypherDefault() throws Exception {
}

@Test
public void testExportAllCypherForCypherShell() throws Exception {
public void testExportAllCypherForCypherShell() {
String fileName = "all.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,$config)",
Expand All @@ -74,7 +73,7 @@ public void testExportAllCypherForCypherShell() throws Exception {
}

@Test
public void testExportQueryCypherForNeo4j() throws Exception {
public void testExportQueryCypherForNeo4j() {
String fileName = "all.cypher";
String s3Url = s3Container.getUrl(fileName);
String query = "MATCH (n) OPTIONAL MATCH p = (n)-[r]-(m) RETURN n,r,m";
Expand All @@ -85,7 +84,7 @@ public void testExportQueryCypherForNeo4j() throws Exception {
}

@Test
public void testExportGraphCypher() throws Exception {
public void testExportGraphCypher() {
String fileName = "graph.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " +
Expand All @@ -99,7 +98,7 @@ public void testExportGraphCypher() throws Exception {

// -- Separate files tests -- //
@Test
public void testExportAllCypherNodes() throws Exception {
public void testExportAllCypherNodes() {
String fileName = "all.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,$exportConfig)",
Expand All @@ -109,7 +108,7 @@ public void testExportAllCypherNodes() throws Exception {
}

@Test
public void testExportAllCypherRelationships() throws Exception {
public void testExportAllCypherRelationships() {
String fileName = "all.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,$exportConfig)",
Expand All @@ -119,7 +118,7 @@ public void testExportAllCypherRelationships() throws Exception {
}

@Test
public void testExportAllCypherSchema() throws Exception {
public void testExportAllCypherSchema() {
String fileName = "all.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,$exportConfig)",
Expand All @@ -129,7 +128,7 @@ public void testExportAllCypherSchema() throws Exception {
}

@Test
public void testExportAllCypherCleanUp() throws Exception {
public void testExportAllCypherCleanUp() {
String fileName = "all.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,$exportConfig)",
Expand All @@ -139,7 +138,7 @@ public void testExportAllCypherCleanUp() throws Exception {
}

@Test
public void testExportGraphCypherNodes() throws Exception {
public void testExportGraphCypherNodes() {
String fileName = "graph.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " +
Expand All @@ -152,7 +151,7 @@ public void testExportGraphCypherNodes() throws Exception {
}

@Test
public void testExportGraphCypherRelationships() throws Exception {
public void testExportGraphCypherRelationships() {
String fileName = "graph.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " +
Expand All @@ -165,7 +164,7 @@ public void testExportGraphCypherRelationships() throws Exception {
}

@Test
public void testExportGraphCypherSchema() throws Exception {
public void testExportGraphCypherSchema() {
String fileName = "graph.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " +
Expand All @@ -178,7 +177,7 @@ public void testExportGraphCypherSchema() throws Exception {
}

@Test
public void testExportGraphCypherCleanUp() throws Exception {
public void testExportGraphCypherCleanUp() {
String fileName = "graph.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " +
Expand All @@ -201,7 +200,7 @@ private void assertResults(String fileName, Map<String, Object> r, final String
}

@Test
public void testExportQueryCypherPlainFormat() throws Exception {
public void testExportQueryCypherPlainFormat() {
String fileName = "all.cypher";
String s3Url = s3Container.getUrl(fileName);
String query = "MATCH (n) OPTIONAL MATCH p = (n)-[r]-(m) RETURN n,r,m";
Expand All @@ -212,7 +211,7 @@ public void testExportQueryCypherPlainFormat() throws Exception {
}

@Test
public void testExportQueryCypherFormatUpdateAll() throws Exception {
public void testExportQueryCypherFormatUpdateAll() {
String fileName = "all.cypher";
String s3Url = s3Container.getUrl(fileName);
String query = "MATCH (n) OPTIONAL MATCH p = (n)-[r]-(m) RETURN n,r,m";
Expand All @@ -223,7 +222,7 @@ public void testExportQueryCypherFormatUpdateAll() throws Exception {
}

@Test
public void testExportQueryCypherFormatAddStructure() throws Exception {
public void testExportQueryCypherFormatAddStructure() {
String fileName = "all.cypher";
String s3Url = s3Container.getUrl(fileName);
String query = "MATCH (n) OPTIONAL MATCH p = (n)-[r]-(m) RETURN n,r,m";
Expand All @@ -234,7 +233,7 @@ public void testExportQueryCypherFormatAddStructure() throws Exception {
}

@Test
public void testExportQueryCypherFormatUpdateStructure() throws Exception {
public void testExportQueryCypherFormatUpdateStructure() {
String fileName = "all.cypher";
String s3Url = s3Container.getUrl(fileName);
String query = "MATCH (n) OPTIONAL MATCH p = (n)-[r]-(m) RETURN n,r,m";
Expand All @@ -245,7 +244,7 @@ public void testExportQueryCypherFormatUpdateStructure() throws Exception {
}

@Test
public void testExportSchemaCypher() throws Exception {
public void testExportSchemaCypher() {
String fileName = "onlySchema.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.cypher.schema($s3,$exportConfig)", map("s3", s3Url, "exportConfig", exportConfig), (r) -> {
Expand All @@ -254,7 +253,7 @@ public void testExportSchemaCypher() throws Exception {
}

@Test
public void testExportSchemaCypherShell() throws Exception {
public void testExportSchemaCypherShell() {
String fileName = "onlySchema.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.cypher.schema($s3,$exportConfig)",
Expand All @@ -265,7 +264,7 @@ public void testExportSchemaCypherShell() throws Exception {
}

@Test
public void testExportCypherNodePoint() throws IOException {
public void testExportCypherNodePoint() {
db.executeTransactionally("CREATE (f:Test {name:'foo'," +
"place2d:point({ x: 2.3, y: 4.5 })," +
"place3d1:point({ x: 2.3, y: 4.5 , z: 1.2})})" +
Expand All @@ -282,7 +281,7 @@ public void testExportCypherNodePoint() throws IOException {
}

@Test
public void testExportCypherNodeDate() throws IOException {
public void testExportCypherNodeDate() {
db.executeTransactionally("CREATE (f:Test {name:'foo', " +
"date:date('2018-10-30'), " +
"datetime:datetime('2018-10-30T12:50:35.556+0100'), " +
Expand All @@ -300,7 +299,7 @@ public void testExportCypherNodeDate() throws IOException {
}

@Test
public void testExportCypherNodeTime() throws IOException {
public void testExportCypherNodeTime() {
db.executeTransactionally("CREATE (f:Test {name:'foo', " +
"local:localtime('12:50:35.556')," +
"t:time('125035.556+0100')})" +
Expand All @@ -317,7 +316,7 @@ public void testExportCypherNodeTime() throws IOException {
}

@Test
public void testExportCypherNodeDuration() throws IOException {
public void testExportCypherNodeDuration() {
db.executeTransactionally("CREATE (f:Test {name:'foo', " +
"duration:duration('P5M1.5D')})" +
"-[:FRIEND_OF {duration:duration('P5M1.5D')}]->" +
Expand All @@ -333,7 +332,7 @@ public void testExportCypherNodeDuration() throws IOException {
}

@Test
public void testExportWithAscendingLabels() throws IOException {
public void testExportWithAscendingLabels() {
db.executeTransactionally("CREATE (f:User:User1:User0:User12 {name:'Alan'})");
String fileName = "ascendingLabels.cypher";
String s3Url = s3Container.getUrl(fileName);
Expand All @@ -346,7 +345,7 @@ public void testExportWithAscendingLabels() throws IOException {
}

@Test
public void testExportAllCypherDefaultWithUnwindBatchSizeOptimized() throws Exception {
public void testExportAllCypherDefaultWithUnwindBatchSizeOptimized() {
String fileName = "allDefaultOptimized.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{useOptimizations: { type: 'unwind_batch', unwindBatchSize: 2}, format: 'neo4j-shell'})", map("s3", s3Url),
Expand All @@ -355,7 +354,7 @@ public void testExportAllCypherDefaultWithUnwindBatchSizeOptimized() throws Exce
}

@Test
public void testExportAllCypherDefaultOptimized() throws Exception {
public void testExportAllCypherDefaultOptimized() {
String fileName = "allDefaultOptimized.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3, $exportConfig)", map("s3", s3Url, "exportConfig", map("format", "neo4j-shell")),
Expand All @@ -364,7 +363,7 @@ public void testExportAllCypherDefaultOptimized() throws Exception {
}

@Test
public void testExportAllCypherDefaultSeparatedFilesOptimized() throws Exception {
public void testExportAllCypherDefaultSeparatedFilesOptimized() {
String fileName = "allDefaultOptimized.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3, $exportConfig)",
Expand All @@ -377,7 +376,7 @@ public void testExportAllCypherDefaultSeparatedFilesOptimized() throws Exception
}

@Test
public void testExportAllCypherCypherShellWithUnwindBatchSizeOptimized() throws Exception {
public void testExportAllCypherCypherShellWithUnwindBatchSizeOptimized() {
String fileName = "allCypherShellOptimized.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{format:'cypher-shell', useOptimizations: {type: 'unwind_batch'}})",
Expand All @@ -387,7 +386,7 @@ public void testExportAllCypherCypherShellWithUnwindBatchSizeOptimized() throws
}

@Test
public void testExportAllCypherCypherShellOptimized() throws Exception {
public void testExportAllCypherCypherShellOptimized() {
String fileName = "allCypherShellOptimized.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{format:'cypher-shell'})",
Expand All @@ -397,7 +396,7 @@ public void testExportAllCypherCypherShellOptimized() throws Exception {
}

@Test
public void testExportAllCypherPlainWithUnwindBatchSizeOptimized() throws Exception {
public void testExportAllCypherPlainWithUnwindBatchSizeOptimized() {
String fileName = "allPlainOptimized.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{format:'plain', useOptimizations: { type: 'unwind_batch', unwindBatchSize: 2}})",
Expand All @@ -407,7 +406,7 @@ public void testExportAllCypherPlainWithUnwindBatchSizeOptimized() throws Except
}

@Test
public void testExportAllCypherPlainAddStructureWithUnwindBatchSizeOptimized() throws Exception {
public void testExportAllCypherPlainAddStructureWithUnwindBatchSizeOptimized() {
String fileName = "allPlainAddStructureOptimized.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{format:'plain', cypherFormat: 'addStructure', useOptimizations: { type: 'unwind_batch', unwindBatchSize: 2}})",
Expand All @@ -416,7 +415,7 @@ public void testExportAllCypherPlainAddStructureWithUnwindBatchSizeOptimized() t
}

@Test
public void testExportAllCypherPlainUpdateStructureWithUnwindBatchSizeOptimized() throws Exception {
public void testExportAllCypherPlainUpdateStructureWithUnwindBatchSizeOptimized() {
String fileName = "allPlainUpdateStructureOptimized.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{format:'plain', cypherFormat: 'updateStructure', useOptimizations: { type: 'unwind_batch', unwindBatchSize: 2}})",
Expand All @@ -432,7 +431,7 @@ public void testExportAllCypherPlainUpdateStructureWithUnwindBatchSizeOptimized(
}

@Test
public void testExportAllCypherPlainUpdateAllWithUnwindBatchSizeOptimized() throws Exception {
public void testExportAllCypherPlainUpdateAllWithUnwindBatchSizeOptimized() {
String fileName = "allPlainUpdateAllOptimized.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{format:'plain', cypherFormat: 'updateAll', useOptimizations: { type: 'unwind_batch', unwindBatchSize: 2}})",
Expand All @@ -441,7 +440,7 @@ public void testExportAllCypherPlainUpdateAllWithUnwindBatchSizeOptimized() thro
}

@Test
public void testExportQueryCypherShellWithUnwindBatchSizeWithBatchSizeOptimized() throws Exception {
public void testExportQueryCypherShellWithUnwindBatchSizeWithBatchSizeOptimized() {
String fileName = "allPlainOptimized.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{format:'cypher-shell', useOptimizations: { type: 'unwind_batch', unwindBatchSize: 2}, batchSize: 2})",
Expand All @@ -451,7 +450,7 @@ public void testExportQueryCypherShellWithUnwindBatchSizeWithBatchSizeOptimized(
}

@Test
public void testExportQueryCypherShellWithUnwindBatchSizeWithBatchSizeOddDataset() throws Exception {
public void testExportQueryCypherShellWithUnwindBatchSizeWithBatchSizeOddDataset() {
String fileName = "allPlainOdd.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{format:'cypher-shell', useOptimizations: { type: 'unwind_batch', unwindBatchSize: 2}, batchSize: 2})",
Expand All @@ -460,7 +459,7 @@ public void testExportQueryCypherShellWithUnwindBatchSizeWithBatchSizeOddDataset
}

@Test
public void testExportQueryCypherShellUnwindBatchParamsWithOddDataset() throws Exception {
public void testExportQueryCypherShellUnwindBatchParamsWithOddDataset() {
String fileName = "allPlainOdd.cypher";
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{format:'cypher-shell', useOptimizations: { type: 'unwind_batch_params', unwindBatchSize: 2}, batchSize:2})",
Expand All @@ -470,7 +469,7 @@ public void testExportQueryCypherShellUnwindBatchParamsWithOddDataset() throws E
}

@Test
public void testExportQueryCypherShellUnwindBatchParamsWithOddBatchSizeOddDataset() throws Exception {
public void testExportQueryCypherShellUnwindBatchParamsWithOddBatchSizeOddDataset() {
db.executeTransactionally("CREATE (:Bar {name:'bar3',age:35}), (:Bar {name:'bar4',age:36})");
String fileName = "allPlainOddNew.cypher";
String s3Url = s3Container.getUrl(fileName);
Expand Down
Loading