Skip to content

Commit

Permalink
Fixes neo4j-contrib#607 - add an failOnError:false option to load.j…
Browse files Browse the repository at this point in the history 
…son and friends
  • Loading branch information
DanielBerton committed Sep 28, 2017
1 parent 9e1b6ed commit 915a0ee
Show file tree
Hide file tree
Showing 9 changed files with 114 additions and 47 deletions.
4 changes: 2 additions & 2 deletions docs/loadxml.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ To make these datastructures available to Cypher, you can use `apoc.load.xml`.
It takes a file or http URL and parses the XML into a map datastructure.

NOTE: in previous releases we've had `apoc.load.xmlSimple`. This is now deprecated and got superseeded by
`apoc.load.xml(url, true)`.
`apoc.load.xml(url, [xPath], [config], true)`.Simple XML Format

See the following usage-examples for the procedures.

Expand Down Expand Up @@ -59,7 +59,7 @@ Here is the example file from above loaded with `apoc.load.xmlSimple`

[source,cypher]
----
call apoc.load.xml("https://raw.githubusercontent.com/neo4j-contrib/neo4j-apoc-procedures/master/src/test/resources/books.xml", true)
call apoc.load.xml("https://raw.githubusercontent.com/neo4j-contrib/neo4j-apoc-procedures/master/src/test/resources/books.xml", '', {}, true)
----

[source,javascript]
Expand Down
7 changes: 5 additions & 2 deletions docs/overview.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -293,13 +293,16 @@ Moreover, if 'apoc.import.file.use_neo4j_config' is enabled the procedures verif
reading the two configuration parameters `dbms.security.allow_csv_import_from_file_urls` and `dbms.directories.import` respectively.
[cols="1m,5"]
|===
| CALL apoc.load.json('http://example.com/map.json') YIELD value as person CREATE (p:Person) SET p = person | load from JSON URL (e.g. web-api) to import JSON as stream of values if the JSON was an array or a single value if it was a map
| CALL apoc.load.xml('http://example.com/test.xml') YIELD value as doc CREATE (p:Person) SET p.name = doc.name | load from XML URL (e.g. web-api) to import XML as single nested map with attributes and `_type`, `_text` and `_children` fields.
| CALL apoc.load.json('http://example.com/map.json', [path], [config]) YIELD value as person CREATE (p:Person) SET p = person | load from JSON URL (e.g. web-api) to import JSON as stream of values if the JSON was an array or a single value if it was a map
| CALL apoc.load.xml('http://example.com/test.xml', ['xPath'], [config]) YIELD value as doc CREATE (p:Person) SET p.name = doc.name | load from XML URL (e.g. web-api) to import XML as single nested map with attributes and `_type`, `_text` and `_children` fields.
| CALL apoc.load.xmlSimple('http://example.com/test.xml') YIELD value as doc CREATE (p:Person) SET p.name = doc.name | load from XML URL (e.g. web-api) to import XML as single nested map with attributes and `_type`, `_text` fields and `_<childtype>` collections per child-element-type.
| CALL apoc.load.csv('url',{sep:";"}) YIELD lineNo, list, map | load CSV fom URL as stream of values +
config contains any of: `{skip:1,limit:5,header:false,sep:'TAB',ignore:['tmp'],arraySep:';',mapping:{years:{type:'int',arraySep:'-',array:false,name:'age',ignore:false}}`
|===

.failOnError

Adding on config the parameter `failOnError:false` (by default `true`), in case of error the procedure don't fail but just return zero rows.
// end::xml[]

== Interacting with Elastic Search
Expand Down
7 changes: 6 additions & 1 deletion src/main/java/apoc/load/LoadCsv.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ public class LoadCsv {
@Procedure
@Description("apoc.load.csv('url',{config}) YIELD lineNo, list, map - load CSV fom URL as stream of values,\n config contains any of: {skip:1,limit:5,header:false,sep:'TAB',ignore:['tmp'],arraySep:';',mapping:{years:{type:'int',arraySep:'-',array:false,name:'age',ignore:false}}")
public Stream<CSVResult> csv(@Name("url") String url, @Name("config") Map<String, Object> config) {
boolean failOnError = booleanValue(config, "failOnError", true);
try {
CountingReader reader = FileUtils.readerFor(url);

Expand All @@ -47,7 +48,11 @@ public Stream<CSVResult> csv(@Name("url") String url, @Name("config") Map<String
boolean checkIgnore = !ignore.isEmpty() || mappings.values().stream().anyMatch( m -> m.ignore);
return StreamSupport.stream(new CSVSpliterator(csv, header, url, skip, limit, checkIgnore,mappings), false);
} catch (IOException e) {
throw new RuntimeException("Can't read CSV from URL " + cleanUrl(url), e);

if(!failOnError)
return Stream.of(new CSVResult(new String[0], new String[0], 0, true, Collections.emptyMap()));
else
throw new RuntimeException("Can't read CSV from URL " + cleanUrl(url), e);
}
}

Expand Down
36 changes: 22 additions & 14 deletions src/main/java/apoc/load/LoadJson.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ public class LoadJson {
@Procedure
@Description("apoc.load.jsonArray('url') YIELD value - load array from JSON URL (e.g. web-api) to import JSON as stream of values")
public Stream<ObjectResult> jsonArray(@Name("url") String url, @Name(value = "path",defaultValue = "") String path) {
return JsonUtil.loadJson(url, null, null, path)
return JsonUtil.loadJson(url, null, null, path, true)
.flatMap((value) -> {
if (value instanceof List) {
List list = (List) value;
Expand All @@ -38,25 +38,27 @@ public Stream<ObjectResult> jsonArray(@Name("url") String url, @Name(value = "pa
}

@Procedure
@Description("apoc.load.json('url') YIELD value - import JSON as stream of values if the JSON was an array or a single value if it was a map")
public Stream<MapResult> json(@Name("url") String url, @Name(value = "path",defaultValue = "") String path) {
return jsonParams(url,null,null, path);
@Description("apoc.load.json('url',path, config) YIELD value - import JSON as stream of values if the JSON was an array or a single value if it was a map")
public Stream<MapResult> json(@Name("url") String url, @Name(value = "path",defaultValue = "") String path, @Name(value = "config",defaultValue = "{}") Map<String, Object> config) {
return jsonParams(url,null,null, path, config);
}

@SuppressWarnings("unchecked")
@Procedure
@Description("apoc.load.jsonParams('url',{header:value},payload) YIELD value - load from JSON URL (e.g. web-api) while sending headers / payload to import JSON as stream of values if the JSON was an array or a single value if it was a map")
public Stream<MapResult> jsonParams(@Name("url") String url, @Name("headers") Map<String,Object> headers, @Name("payload") String payload, @Name(value = "path",defaultValue = "") String path) {
return loadJsonStream(url, headers, payload, path);
@Description("apoc.load.jsonParams('url',{header:value},payload, config) YIELD value - load from JSON URL (e.g. web-api) while sending headers / payload to import JSON as stream of values if the JSON was an array or a single value if it was a map")
public Stream<MapResult> jsonParams(@Name("url") String url, @Name("headers") Map<String,Object> headers, @Name("payload") String payload, @Name(value = "path",defaultValue = "") String path, @Name(value = "config",defaultValue = "{}") Map<String, Object> config) {
if (config == null) config = Collections.emptyMap();
boolean failOnError = (boolean) config.getOrDefault("failOnError", true);
return loadJsonStream(url, headers, payload, path, failOnError);
}

public static Stream<MapResult> loadJsonStream(@Name("url") String url, @Name("headers") Map<String, Object> headers, @Name("payload") String payload) {
return loadJsonStream(url, headers, payload, "" );
return loadJsonStream(url, headers, payload, "", true);
}
public static Stream<MapResult> loadJsonStream(@Name("url") String url, @Name("headers") Map<String, Object> headers, @Name("payload") String payload, String path) {
public static Stream<MapResult> loadJsonStream(@Name("url") String url, @Name("headers") Map<String, Object> headers, @Name("payload") String payload, String path, boolean failOnError) {
headers = null != headers ? headers : new HashMap<>();
headers.putAll(extractCredentialsIfNeeded(url));
Stream<Object> stream = JsonUtil.loadJson(url,headers,payload, path);
headers.putAll(extractCredentialsIfNeeded(url, failOnError));
Stream<Object> stream = JsonUtil.loadJson(url,headers,payload, path, failOnError);
return stream.flatMap((value) -> {
if (value instanceof Map) {
return Stream.of(new MapResult((Map) value));
Expand All @@ -67,11 +69,14 @@ public static Stream<MapResult> loadJsonStream(@Name("url") String url, @Name("h
return ((List) value).stream().map((v) -> new MapResult((Map) v));
return Stream.of(new MapResult(Collections.singletonMap("result",value)));
}
throw new RuntimeException("Incompatible Type " + (value == null ? "null" : value.getClass()));
if(!failOnError)
throw new RuntimeException("Incompatible Type " + (value == null ? "null" : value.getClass()));
else
return Stream.of(new MapResult(Collections.emptyMap()));
});
}

private static Map<String, Object> extractCredentialsIfNeeded(String url) {
private static Map<String, Object> extractCredentialsIfNeeded(String url, boolean failOnError) {
try {
URI uri = new URI(url);
String authInfo = uri.getUserInfo();
Expand All @@ -84,7 +89,10 @@ private static Map<String, Object> extractCredentialsIfNeeded(String url) {
}

} catch (Exception e) {
throw new RuntimeException(e);
if(!failOnError)
return Collections.emptyMap();
else
throw new RuntimeException(e);
}

return Collections.emptyMap();
Expand Down
63 changes: 39 additions & 24 deletions src/main/java/apoc/load/Xml.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathFactory;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URL;
import java.net.URLConnection;
Expand All @@ -37,9 +38,9 @@ public class Xml {
public GraphDatabaseService db;

@Procedure
@Description("apoc.load.xml('http://example.com/test.xml', 'xPath', false) YIELD value as doc CREATE (p:Person) SET p.name = doc.name load from XML URL (e.g. web-api) to import XML as single nested map with attributes and _type, _text and _childrenx fields.")
public Stream<MapResult> xml(@Name("url") String url, @Name(value = "path", defaultValue = "/") String path, @Name(value = "simple", defaultValue = "false") boolean simpleMode) throws Exception {
return xmlXpathToMapResult(url, simpleMode, path);
@Description("apoc.load.xml('http://example.com/test.xml', 'xPath',config, false) YIELD value as doc CREATE (p:Person) SET p.name = doc.name load from XML URL (e.g. web-api) to import XML as single nested map with attributes and _type, _text and _childrenx fields.")
public Stream<MapResult> xml(@Name("url") String url, @Name(value = "path", defaultValue = "/") String path, @Name(value = "config",defaultValue = "{}") Map<String, Object> config, @Name(value = "simple", defaultValue = "false") boolean simpleMode) throws Exception {
return xmlXpathToMapResult(url, simpleMode, path ,config);
}

@Procedure(deprecatedBy = "apoc.load.xml")
Expand All @@ -49,35 +50,49 @@ public Stream<MapResult> xmlSimple(@Name("url") String url) throws Exception {
return xmlToMapResult(url, true);
}

private Stream<MapResult> xmlXpathToMapResult(@Name("url") String url, boolean simpleMode, String path) throws Exception {

DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
documentBuilderFactory.setNamespaceAware(true);
documentBuilderFactory.setIgnoringElementContentWhitespace(true);
DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
private Stream<MapResult> xmlXpathToMapResult(@Name("url") String url, boolean simpleMode, String path, Map<String, Object> config) throws Exception {
if (config == null) config = Collections.emptyMap();
boolean failOnError = (boolean) config.getOrDefault("failOnError", true);
List<MapResult> result = new ArrayList<>();
try {
DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
documentBuilderFactory.setNamespaceAware(true);
documentBuilderFactory.setIgnoringElementContentWhitespace(true);
DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();

FileUtils.checkReadAllowed(url);
URLConnection urlConnection = new URL(url).openConnection();
Document doc = documentBuilder.parse(urlConnection.getInputStream());
FileUtils.checkReadAllowed(url);
URLConnection urlConnection = new URL(url).openConnection();
Document doc = documentBuilder.parse(urlConnection.getInputStream());

XPathFactory xPathFactory = XPathFactory.newInstance();
XPathFactory xPathFactory = XPathFactory.newInstance();

XPath xPath = xPathFactory.newXPath();
XPath xPath = xPathFactory.newXPath();

path = StringUtils.isEmpty(path) ? "/" : path;
XPathExpression xPathExpression = xPath.compile(path);
NodeList nodeList = (NodeList) xPathExpression.evaluate(doc, XPathConstants.NODESET);
path = StringUtils.isEmpty(path) ? "/" : path;
XPathExpression xPathExpression = xPath.compile(path);
NodeList nodeList = (NodeList) xPathExpression.evaluate(doc, XPathConstants.NODESET);

List<MapResult> result = new ArrayList<>();
for (int i = 0; i < nodeList.getLength(); i++) {
final Deque<Map<String, Object>> stack = new LinkedList<>();

for (int i = 0; i < nodeList.getLength(); i++) {
final Deque<Map<String, Object>> stack = new LinkedList<>();

handleNode(stack, nodeList.item(i), simpleMode);
for (int index = 0; index < stack.size(); index++) {
result.add(new MapResult(stack.pollFirst()));
handleNode(stack, nodeList.item(i), simpleMode);
for (int index = 0; index < stack.size(); index++) {
result.add(new MapResult(stack.pollFirst()));
}
}
}
catch (FileNotFoundException e){
if(!failOnError)
return Stream.of(new MapResult(Collections.emptyMap()));
else
throw new FileNotFoundException(e.getMessage());
}
catch (Exception e){
if(!failOnError)
return Stream.of(new MapResult(Collections.emptyMap()));
else
throw new Exception(e);
}
return result.stream();
}

Expand Down
11 changes: 7 additions & 4 deletions src/main/java/apoc/util/JsonUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,9 @@ public void close() throws IOException {
}

public static Stream<Object> loadJson(String url, Map<String,Object> headers, String payload) {
return loadJson(url,headers,payload,"");
return loadJson(url,headers,payload,"", true);
}
public static Stream<Object> loadJson(String url, Map<String,Object> headers, String payload, String path) {
public static Stream<Object> loadJson(String url, Map<String,Object> headers, String payload, String path, boolean failOnError) {
try {
FileUtils.checkReadAllowed(url);
InputStream input = Util.openInputStream(url, headers, payload);
Expand All @@ -66,12 +66,15 @@ public static Stream<Object> loadJson(String url, Map<String,Object> headers, St
return (path==null||path.isEmpty()) ? stream : stream.map((value) -> JsonPath.parse(value,JSON_PATH_CONFIG).read(path));
} catch (IOException e) {
String u = Util.cleanUrl(url);
throw new RuntimeException("Can't read url " + u + " as json: "+e.getMessage(), e);
if(!failOnError)
return Stream.of();
else
throw new RuntimeException("Can't read url " + u + " as json: "+e.getMessage(), e);
}
}

public static Stream<Object> loadJson(@Name("url") String url) {
return loadJson(url,null,null,"");
return loadJson(url,null,null,"", true);
}

public static <T> T parse(String json, String path, Class<T> type) {
Expand Down
15 changes: 15 additions & 0 deletions src/test/java/apoc/load/LoadCsvTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,15 @@
import org.neo4j.test.TestGraphDatabaseFactory;

import java.net.URL;
import java.util.Collections;
import java.util.Map;

import static apoc.util.MapUtil.map;
import static apoc.util.TestUtil.testCall;
import static apoc.util.TestUtil.testResult;
import static java.util.Arrays.asList;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;

public class LoadCsvTest {

Expand Down Expand Up @@ -133,4 +135,17 @@ public void testLoadCsvByUrlRedirect() throws Exception {
assertEquals(false, r.hasNext());
});
}

@Test
public void testLoadCsvNoFailOnError() throws Exception {
String url = "test.csv";
testResult(db, "CALL apoc.load.csv({url},{failOnError:false})", map("url",url), // 'file:test.csv'
(r) -> {
Map<String, Object> row = r.next();
assertEquals(0L, row.get("lineNo"));
assertEquals(Collections.emptyList(), row.get("list"));
assertEquals(Collections.emptyMap(), row.get("map"));
assertEquals(false, r.hasNext());
});
}
}
8 changes: 8 additions & 0 deletions src/test/java/apoc/load/LoadJsonTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,4 +100,12 @@ public class LoadJsonTest {
assertEquals(true, value.containsKey("nodes"));
});
}

@Test public void testLoadJsonNoFailOnError() throws Exception {
String url = "file.json";
testResult(db, "CALL apoc.load.json({url},null, {failOnError:false})",map("url", url), // 'file:map.json' YIELD value RETURN value
(row) -> {
assertFalse(row.hasNext());
});
}
}
10 changes: 10 additions & 0 deletions src/test/java/apoc/load/XmlTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.stream.IntStream;

import static apoc.util.MapUtil.map;
Expand Down Expand Up @@ -223,4 +224,13 @@ public void testLoadXmlXpathBooKsFromGenre () {
assertEquals(false, r.hasNext());
});
}

@Test
public void testLoadXmlNoFailOnError () {
testCall(db, "CALL apoc.load.xml('file:src/test/resources/books.xm', '', {failOnError:false}) yield value as result",
(r) -> {
Map resultMap = (Map) r.get("result");
assertEquals(Collections.emptyMap(), resultMap);
});
}
}

0 comments on commit 915a0ee

Please sign in to comment.