Skip to content

Commit

Permalink
[G9u7OAg9] Ignore check for optimizations if the format is CSV (#3857)
Browse files Browse the repository at this point in the history
  • Loading branch information
@gem-neo4j
gem-neo4j authored Dec 6, 2023
1 parent 9f7b8aa commit 0d50b37
Show file tree
Hide file tree
Showing 5 changed files with 45 additions and 35 deletions.
9 changes: 5 additions & 4 deletions core/src/main/java/apoc/export/csv/ExportCSV.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import apoc.export.cypher.ExportFileManager;
import apoc.export.cypher.FileManagerFactory;
import apoc.export.util.ExportConfig;
import apoc.export.util.ExportFormat;
import apoc.export.util.ExportUtils;
import apoc.export.util.NodesAndRelsSubGraph;
import apoc.export.util.ProgressReporter;
Expand Down Expand Up @@ -74,13 +75,13 @@ public ExportCSV() {
@Description("apoc.export.csv.all(file,config) - exports whole database as csv to the provided file")
public Stream<ProgressInfo> all(@Name("file") String fileName, @Name("config") Map<String, Object> config) throws Exception {
String source = String.format("database: nodes(%d), rels(%d)", Util.nodeCount(tx), Util.relCount(tx));
return exportCsv(fileName, source, new DatabaseSubGraph(tx), new ExportConfig(config));
return exportCsv(fileName, source, new DatabaseSubGraph(tx), new ExportConfig(config, ExportFormat.CSV));
}

@Procedure
@Description("apoc.export.csv.data(nodes,rels,file,config) - exports given nodes and relationships as csv to the provided file")
public Stream<ProgressInfo> data(@Name("nodes") List<Node> nodes, @Name("rels") List<Relationship> rels, @Name("file") String fileName, @Name("config") Map<String, Object> config) throws Exception {
ExportConfig exportConfig = new ExportConfig(config);
ExportConfig exportConfig = new ExportConfig(config, ExportFormat.CSV);
preventBulkImport(exportConfig);
String source = String.format("data: nodes(%d), rels(%d)", nodes.size(), rels.size());
return exportCsv(fileName, source, new NodesAndRelsSubGraph(tx, nodes, rels), exportConfig);
Expand All @@ -91,13 +92,13 @@ public Stream<ProgressInfo> graph(@Name("graph") Map<String,Object> graph, @Name
Collection<Node> nodes = (Collection<Node>) graph.get("nodes");
Collection<Relationship> rels = (Collection<Relationship>) graph.get("relationships");
String source = String.format("graph: nodes(%d), rels(%d)", nodes.size(), rels.size());
return exportCsv(fileName, source, new NodesAndRelsSubGraph(tx, nodes, rels), new ExportConfig(config));
return exportCsv(fileName, source, new NodesAndRelsSubGraph(tx, nodes, rels), new ExportConfig(config, ExportFormat.CSV));
}

@Procedure
@Description("apoc.export.csv.query(query,file,{config,...,params:{params}}) - exports results from the cypher statement as csv to the provided file")
public Stream<ProgressInfo> query(@Name("query") String query, @Name("file") String fileName, @Name("config") Map<String, Object> config) throws Exception {
ExportConfig exportConfig = new ExportConfig(config);
ExportConfig exportConfig = new ExportConfig(config, ExportFormat.CSV);
preventBulkImport(exportConfig);
Map<String,Object> params = config == null ? Collections.emptyMap() : (Map<String,Object>)config.getOrDefault("params", Collections.emptyMap());
Result result = tx.execute(query,params);
Expand Down
11 changes: 8 additions & 3 deletions core/src/main/java/apoc/export/util/ExportConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,11 @@ public boolean isMultipleRelationshipsWithType() {
return multipleRelationshipsWithType;
}

public ExportConfig(Map<String,Object> config) {
public ExportConfig(Map<String, Object> config) {
this(config, ExportFormat.CYPHER_SHELL);
}

public ExportConfig(Map<String, Object> config, ExportFormat exportFormat) {
super(config);
config = config != null ? config : Collections.emptyMap();
this.silent = toBoolean(config.getOrDefault("silent",false));
Expand All @@ -138,7 +142,7 @@ public ExportConfig(Map<String,Object> config) {
this.nodesOfRelationships = toBoolean(config.get("nodesOfRelationships"));
this.bulkImport = toBoolean(config.get("bulkImport"));
this.separateHeader = toBoolean(config.get("separateHeader"));
this.format = ExportFormat.fromString((String) config.getOrDefault("format", "cypher-shell"));
this.format = ExportFormat.fromString((String) config.getOrDefault("format", exportFormat.getFormat()));
this.cypherFormat = CypherFormat.fromString((String) config.getOrDefault("cypherFormat", "create"));
this.config = config;
this.streamStatements = toBoolean(config.get("streamStatements")) || toBoolean(config.get("stream"));
Expand All @@ -162,7 +166,8 @@ private void validate() {
if (OptimizationType.UNWIND_BATCH_PARAMS.equals(this.optimizationType) && !ExportFormat.CYPHER_SHELL.equals(this.format)) {
throw new RuntimeException("`useOptimizations: 'UNWIND_BATCH_PARAMS'` can be used only in combination with `format: 'CYPHER_SHELL' but got [format:`" + this.format + "]");
}
if (!OptimizationType.NONE.equals(this.optimizationType) && this.unwindBatchSize > this.batchSize) {
if (!OptimizationType.NONE.equals(this.optimizationType) && this.unwindBatchSize > this.batchSize
&& !ExportFormat.CSV.equals(this.format)) {
throw new RuntimeException("`unwindBatchSize` must be <= `batchSize`, but got [unwindBatchSize:" + unwindBatchSize + ", batchSize:" + batchSize + "]");
}
}
Expand Down
6 changes: 5 additions & 1 deletion core/src/main/java/apoc/export/util/ExportFormat.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,9 @@ public enum ExportFormat {

GEPHI("gephi", "", "", "", ""),

TINKERPOP("tinkerpop", "", "", "", "");
TINKERPOP("tinkerpop", "", "", "", ""),

CSV("csv", "", "", "", "");

private final String format;

Expand All @@ -50,6 +51,9 @@ public enum ExportFormat {

private String schemaAwait;

public String getFormat() {
return format;
}
ExportFormat(String format, String commit, String begin, String schemaAwait, String indexAwait) {
this.format = format;
this.begin = begin;
Expand Down
52 changes: 26 additions & 26 deletions core/src/test/java/apoc/export/csv/ExportCsvTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ private String readFile(String fileName, Charset charset, CompressionAlgo compre
}

@Test
public void testExportInvalidQuoteValue() throws Exception {
public void testExportInvalidQuoteValue() {
try {
String fileName = "all.csv";
TestUtil.testCall(db, "CALL apoc.export.csv.all($file,{quotes: 'Invalid'})",
Expand Down Expand Up @@ -218,7 +218,7 @@ public void testCsvRoundTrip() {
}

@Test
public void testExportAllCsv() throws Exception {
public void testExportAllCsv() {
String fileName = "all.csv";
testExportCsvAllCommon(fileName);
}
Expand Down Expand Up @@ -265,7 +265,7 @@ public void testExportAllCsvWithSample() throws IOException {
}

@Test
public void testExportAllCsvWithQuotes() throws Exception {
public void testExportAllCsvWithQuotes() {
String fileName = "all.csv";
TestUtil.testCall(db, "CALL apoc.export.csv.all($file,{quotes: true})",
map("file", fileName),
Expand All @@ -274,7 +274,7 @@ public void testExportAllCsvWithQuotes() throws Exception {
}

@Test
public void testExportAllCsvWithoutQuotes() throws Exception {
public void testExportAllCsvWithoutQuotes() {
String fileName = "all.csv";
TestUtil.testCall(db, "CALL apoc.export.csv.all($file,{quotes: 'none'})",
map("file", fileName),
Expand All @@ -283,7 +283,7 @@ public void testExportAllCsvWithoutQuotes() throws Exception {
}

@Test
public void testExportAllCsvNeededQuotes() throws Exception {
public void testExportAllCsvNeededQuotes() {
String fileName = "all.csv";
TestUtil.testCall(db, "CALL apoc.export.csv.all($file,{quotes: 'ifNeeded'})",
map("file", fileName),
Expand All @@ -292,7 +292,7 @@ public void testExportAllCsvNeededQuotes() throws Exception {
}

@Test
public void testExportGraphCsv() throws Exception {
public void testExportGraphCsv() {
String fileName = "graph.csv";
TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " +
"CALL apoc.export.csv.graph(graph, $file,{quotes: 'none'}) " +
Expand All @@ -303,7 +303,7 @@ public void testExportGraphCsv() throws Exception {
}

@Test
public void testExportGraphCsvWithoutQuotes() throws Exception {
public void testExportGraphCsvWithoutQuotes() {
String fileName = "graph.csv";
TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " +
"CALL apoc.export.csv.graph(graph, $file,null) " +
Expand All @@ -314,7 +314,7 @@ public void testExportGraphCsvWithoutQuotes() throws Exception {
}

@Test
public void testExportQueryCsv() throws Exception {
public void testExportQueryCsv() {
String fileName = "query.csv";
String query = "MATCH (u:User) return u.age, u.name, u.male, u.kids, labels(u)";
TestUtil.testCall(db, "CALL apoc.export.csv.query($query,$file,null)",
Expand All @@ -329,7 +329,7 @@ public void testExportQueryCsv() throws Exception {
}

@Test
public void testExportQueryCsvWithoutQuotes() throws Exception {
public void testExportQueryCsvWithoutQuotes() {
String fileName = "query.csv";
String query = "MATCH (u:User) return u.age, u.name, u.male, u.kids, labels(u)";
TestUtil.testCall(db, "CALL apoc.export.csv.query($query,$file,{quotes: false})",
Expand Down Expand Up @@ -370,7 +370,7 @@ public void testExportCsvAdminOperationErrorMessage() {


@Test
public void testExportQueryNodesCsv() throws Exception {
public void testExportQueryNodesCsv() {
String fileName = "query_nodes.csv";
String query = "MATCH (u:User) return u";
TestUtil.testCall(db, "CALL apoc.export.csv.query($query,$file,null)",
Expand All @@ -385,7 +385,7 @@ public void testExportQueryNodesCsv() throws Exception {
}

@Test
public void testExportQueryNodesCsvParams() throws Exception {
public void testExportQueryNodesCsvParams() {
String fileName = "query_nodes.csv";
String query = "MATCH (u:User) WHERE u.age > $age return u";
TestUtil.testCall(db, "CALL apoc.export.csv.query($query,$file,{params:{age:10}})", map("file", fileName,"query",query),
Expand Down Expand Up @@ -422,15 +422,15 @@ private void assertCsvCommon(String fileName, Map<String, Object> r) {
assertTrue("Should get time greater than 0",((long) r.get("time")) >= 0);
}

@Test public void testExportAllCsvStreaming() throws Exception {
String statement = "CALL apoc.export.csv.all(null,{stream:true,batchSize:2,useOptimizations:{unwindBatchSize:2}})";
@Test public void testExportAllCsvStreaming() {
String statement = "CALL apoc.export.csv.all(null,{stream:true,batchSize:2})";
assertExportStreaming(statement, NONE);
}

@Test
public void testExportAllCsvStreamingCompressed() throws Exception {
public void testExportAllCsvStreamingCompressed() {
final CompressionAlgo algo = GZIP;
String statement = "CALL apoc.export.csv.all(null, {compression: '" + algo.name() + "',stream:true,batchSize:2,useOptimizations:{unwindBatchSize:2}})";
String statement = "CALL apoc.export.csv.all(null, {compression: '" + algo.name() + "',stream:true,batchSize:2})";
assertExportStreaming(statement, algo);
}

Expand Down Expand Up @@ -480,18 +480,18 @@ private void assertExportStreaming(String statement, CompressionAlgo algo) {
assertEquals(EXPECTED, sb.toString());
}

@Test public void testCypherCsvStreaming() throws Exception {
@Test public void testCypherCsvStreaming() {
String query = "MATCH (u:User) return u.age, u.name, u.male, u.kids, labels(u)";
StringBuilder sb = new StringBuilder();
testResult(db, "CALL apoc.export.csv.query($query,null,{stream:true,batchSize:2, useOptimizations:{unwindBatchSize:2}})", map("query",query),
testResult(db, "CALL apoc.export.csv.query($query,null,{stream:true,batchSize:2})", map("query",query),
getAndCheckStreamingMetadataQueryMatchUsers(sb));
assertEquals(EXPECTED_QUERY, sb.toString());
}

@Test public void testCypherCsvStreamingWithoutQuotes() throws Exception {
@Test public void testCypherCsvStreamingWithoutQuotes() {
String query = "MATCH (u:User) return u.age, u.name, u.male, u.kids, labels(u)";
StringBuilder sb = new StringBuilder();
testResult(db, "CALL apoc.export.csv.query($query,null,{quotes: false, stream:true,batchSize:2, useOptimizations:{unwindBatchSize:2}})", map("query",query),
testResult(db, "CALL apoc.export.csv.query($query,null,{quotes: false, stream:true,batchSize:2})", map("query",query),
getAndCheckStreamingMetadataQueryMatchUsers(sb));

assertEquals(EXPECTED_QUERY_WITHOUT_QUOTES, sb.toString());
Expand Down Expand Up @@ -524,35 +524,35 @@ private Consumer<Result> getAndCheckStreamingMetadataQueryMatchUsers(StringBuild
};
}

@Test public void testCypherCsvStreamingWithAlwaysQuotes() throws Exception {
@Test public void testCypherCsvStreamingWithAlwaysQuotes() {
String query = "MATCH (a:Address) return a.name, a.city, a.street, labels(a)";
StringBuilder sb = new StringBuilder();
testResult(db, "CALL apoc.export.csv.query($query,null,{quotes: 'always', stream:true,batchSize:2, useOptimizations:{unwindBatchSize:2}})", map("query",query),
testResult(db, "CALL apoc.export.csv.query($query,null,{quotes: 'always', stream:true,batchSize:2})", map("query",query),
getAndCheckStreamingMetadataQueryMatchAddress(sb));

assertEquals(EXPECTED_QUERY_QUOTES_ALWAYS, sb.toString());
}

@Test public void testCypherCsvStreamingWithNeededQuotes() throws Exception {
@Test public void testCypherCsvStreamingWithNeededQuotes() {
String query = "MATCH (a:Address) return a.name, a.city, a.street, labels(a)";
StringBuilder sb = new StringBuilder();
testResult(db, "CALL apoc.export.csv.query($query,null,{quotes: 'ifNeeded', stream:true,batchSize:2, useOptimizations:{unwindBatchSize:2}})", map("query",query),
testResult(db, "CALL apoc.export.csv.query($query,null,{quotes: 'ifNeeded', stream:true,batchSize:2})", map("query",query),
getAndCheckStreamingMetadataQueryMatchAddress(sb));

assertEquals(EXPECTED_QUERY_QUOTES_NEEDED, sb.toString());
}

@Test public void testCypherCsvStreamingWithNoneQuotes() throws Exception {
@Test public void testCypherCsvStreamingWithNoneQuotes() {
String query = "MATCH (a:Address) return a.name, a.city, a.street, labels(a)";
StringBuilder sb = new StringBuilder();
testResult(db, "CALL apoc.export.csv.query($query,null,{quotes: 'none', stream:true,batchSize:2, useOptimizations:{unwindBatchSize:2}})", map("query",query),
testResult(db, "CALL apoc.export.csv.query($query,null,{quotes: 'none', stream:true,batchSize:2})", map("query",query),
getAndCheckStreamingMetadataQueryMatchAddress(sb));

assertEquals(EXPECTED_QUERY_QUOTES_NONE, sb.toString());
}

@Test
public void testExportQueryCsvIssue1188() throws Exception {
public void testExportQueryCsvIssue1188() {
String copyright = "\n" +
"(c) 2018 Hovsepian, Albanese, et al. \"\"ASCB(r),\"\" \"\"The American Society for Cell Biology(r),\"\" and \"\"Molecular Biology of the Cell(r)\"\" are registered trademarks of The American Society for Cell Biology.\n" +
"2018\n" +
Expand Down
2 changes: 1 addition & 1 deletion docs/asciidoc/modules/ROOT/pages/overview/apoc.export/compression.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ Moreover, you can use it with a file split into batches:

[source,cypher]
----
CALL apoc.export.csv.all(null, {compression: 'DEFLATE',stream:true,batchSize:2,useOptimizations:{unwindBatchSize:2}})
CALL apoc.export.csv.all(null, {compression: 'DEFLATE',stream:true,batchSize:2})
YIELD data RETURN data
----

Expand Down

0 comments on commit 0d50b37

Please sign in to comment.