diff --git a/core/src/main/java/apoc/export/csv/CsvPropertyConverter.java b/core/src/main/java/apoc/export/csv/CsvPropertyConverter.java index 434c7413bd..4a7cc52c35 100644 --- a/core/src/main/java/apoc/export/csv/CsvPropertyConverter.java +++ b/core/src/main/java/apoc/export/csv/CsvPropertyConverter.java @@ -61,7 +61,7 @@ public static boolean addPropertyToGraphEntity( return true; } - static Object[] getPrototypeFor(String type) { + public static Object[] getPrototypeFor(String type) { switch (type) { case "INT": case "LONG": diff --git a/docs/asciidoc/modules/ROOT/pages/overview/apoc.convert/apoc.convert.fromYaml.adoc b/docs/asciidoc/modules/ROOT/pages/overview/apoc.convert/apoc.convert.fromYaml.adoc new file mode 100644 index 0000000000..3c5b0ad15f --- /dev/null +++ b/docs/asciidoc/modules/ROOT/pages/overview/apoc.convert/apoc.convert.fromYaml.adoc @@ -0,0 +1,37 @@ += apoc.convert.fromYaml +:description: This section contains reference documentation for the apoc.convert.fromYaml function. + +label:function[] label:apoc-extended[] + +[.emphasis] +apoc.convert.fromYaml(value, $config) - Deserializes the YAML string to Neo4j value + +== Signature + +[source] +---- +apoc.convert.fromYaml(value :: STRING, config = {} :: MAP) :: ANY +---- + +== Input parameters +[.procedures, opts=header] +|=== +| Name | Type | Default +|value|STRING|null +|config|MAP|{} +|=== + +== Config parameters +include::partial$usage/config/apoc.convert.fromYaml.adoc[] + +[[yaml-dependencies]] +=== Install dependencies +Note that to use this function, you have to install additional dependencies +which can be downloaded https://github.com/neo4j-contrib/neo4j-apoc-procedures/releases/download/{apoc-release}/apoc-yaml-dependencies-{apoc-release}-all.jar[from this link]. + + +[[usage-apoc.convert.fromYaml]] +== Usage Examples +include::partial$usage/apoc.convert.fromYaml.adoc[] + + diff --git a/docs/asciidoc/modules/ROOT/pages/overview/apoc.convert/apoc.convert.toYaml.adoc b/docs/asciidoc/modules/ROOT/pages/overview/apoc.convert/apoc.convert.toYaml.adoc new file mode 100644 index 0000000000..196b966c13 --- /dev/null +++ b/docs/asciidoc/modules/ROOT/pages/overview/apoc.convert/apoc.convert.toYaml.adoc @@ -0,0 +1,36 @@ += apoc.convert.fromYaml +:description: This section contains reference documentation for the apoc.convert.fromYaml function. + +label:function[] label:apoc-full[] + +[.emphasis] +apoc.convert.fromYaml(value, $config) - Deserializes the YAML string to Neo4j value + +== Signature + +[source] +---- +apoc.convert.fromYaml(value :: STRING, config = {} :: MAP) :: ANY +---- + +== Input parameters +[.procedures, opts=header] +|=== +| Name | Type | Default +|value|STRING|null +|config|MAP|{} +|=== + +== Config parameters +include::partial$usage/config/apoc.convert.fromYaml.adoc[] + +[[yaml-dependencies]] +=== Install dependencies +Note that to use this function, you have to install additional dependencies +which can be downloaded https://github.com/neo4j-contrib/neo4j-apoc-procedures/releases/download/{apoc-release}/apoc-yaml-dependencies-{apoc-release}-all.jar[from this link]. + + +[[usage-apoc.convert.fromYaml]] +== Usage Examples +include::partial$usage/apoc.convert.fromYaml.adoc[] + diff --git a/docs/asciidoc/modules/ROOT/pages/overview/apoc.convert/index.adoc b/docs/asciidoc/modules/ROOT/pages/overview/apoc.convert/index.adoc index 33c38fcb91..7a3bf6545f 100644 --- a/docs/asciidoc/modules/ROOT/pages/overview/apoc.convert/index.adoc +++ b/docs/asciidoc/modules/ROOT/pages/overview/apoc.convert/index.adoc @@ -118,5 +118,10 @@ apoc.convert.toString(value) \| tries it's best to convert the value to a string apoc.convert.toStringList(value) \| tries it's best to convert the value to a list of strings |label:function[] |label:apoc-core[] +|xref::overview/apoc.convert/apoc.convert.fromYaml.adoc[apoc.convert.fromYaml icon:book[]] + +apoc.convert.fromYaml(value, $config) - Deserializes the YAML string to Neo4j value +|label:function[] +|label:apoc-full[] |=== diff --git a/docs/asciidoc/modules/ROOT/partials/generated-documentation/documentation.adoc b/docs/asciidoc/modules/ROOT/partials/generated-documentation/documentation.adoc index b2b7b1105d..83eef173b8 100644 --- a/docs/asciidoc/modules/ROOT/partials/generated-documentation/documentation.adoc +++ b/docs/asciidoc/modules/ROOT/partials/generated-documentation/documentation.adoc @@ -676,6 +676,13 @@ apoc.convert.toString(value) \| tries it's best to convert the value to a string apoc.convert.toStringList(value) \| tries it's best to convert the value to a list of strings |label:function[] |label:apoc-core[] + +|xref::overview/apoc.convert/apoc.convert.fromYaml.adoc[apoc.convert.fromYaml icon:book[]] + +apoc.convert.fromYaml(value, $config) - Deserializes the YAML string to Neo4j value +|label:function[] +|label:apoc-full[] + |=== == xref::overview/apoc.couchbase/index.adoc[] diff --git a/docs/asciidoc/modules/ROOT/partials/generated-documentation/nav.adoc b/docs/asciidoc/modules/ROOT/partials/generated-documentation/nav.adoc index 5c1eb11b53..cf67a0aef6 100644 --- a/docs/asciidoc/modules/ROOT/partials/generated-documentation/nav.adoc +++ b/docs/asciidoc/modules/ROOT/partials/generated-documentation/nav.adoc @@ -129,6 +129,8 @@ This file is generated by DocsTest, so don't change it! *** xref::overview/apoc.convert/apoc.convert.toSortedJsonMap.adoc[] *** xref::overview/apoc.convert/apoc.convert.toString.adoc[] *** xref::overview/apoc.convert/apoc.convert.toStringList.adoc[] +*** xref::overview/apoc.convert/apoc.convert.toYaml.adoc[] +*** xref::overview/apoc.convert/apoc.convert.fromYaml.adoc[] ** xref::overview/apoc.couchbase/index.adoc[] *** xref::overview/apoc.couchbase/apoc.couchbase.append.adoc[] *** xref::overview/apoc.couchbase/apoc.couchbase.exists.adoc[] diff --git a/docs/asciidoc/modules/ROOT/partials/usage/apoc.convert.fromYaml.adoc b/docs/asciidoc/modules/ROOT/partials/usage/apoc.convert.fromYaml.adoc new file mode 100644 index 0000000000..bd4f26dc20 --- /dev/null +++ b/docs/asciidoc/modules/ROOT/partials/usage/apoc.convert.fromYaml.adoc @@ -0,0 +1,309 @@ +We can convert any YAML string to list/map of nodes/rels + +.Convert YAML string to map +[source,cypher] +---- +RETURN apoc.convert.fromYaml("a: 42 +b: foo") AS value +---- + +.Results +[opts="header",cols="1"] +|=== +| value +a| +[source,json] +---- +{ + "b": "foo", + "a": 42 +} +---- +|=== + + +.Convert YAML string to map, with custom mapping +[source,cypher] +---- +RETURN apoc.convert.fromYaml("a: 42 +b: foo", {mapping: {a: "Long"} }) AS value +---- + +.Results +[opts="header",cols="1"] +|=== +| value +a| +[source,json] +{ + "b": "foo", + "a": 42 +} +|=== + +.Convert YAML string to map with custom features +[source,cypher] +---- +RETURN apoc.convert.fromYaml("a: 42 +b: foo", {enable: ['MINIMIZE_QUOTES'], disable: ['WRITE_DOC_START_MARKER']}) AS value +---- + +.Results +[opts="header",cols="1"] +|=== +| value +a| +[source,json] +{ + "b": "foo", + "a": 42 +} +|=== + + +.Convert YAML string to list +[source,cypher] +---- +RETURN apoc.convert.fromYaml("--- +- 1 +- 2 +- 3") as value +---- + +.Results +[opts="header",cols="1"] +|=== +| value +a| +[source,json] +---- +[1, 2, 3] +|=== + + +.Convert YAML string to list and mapping type Long +[source,cypher] +---- +RETURN apoc.convert.fromYaml("--- +- 1 +- 2 +- 3", +{mapping: {_: "Long"}}) as value +---- + +.Results +[opts="header",cols="1"] +|=== +| value +a| +[source,json] +---- +[1, 2, 3] +|=== + +.Convert from YAML string to map +[source,cypher] +---- +RETURN apoc.convert.fromYaml("--- + a: 42 + b: \"foo\" + c: + - 1 + - 2 + - 3") as value +---- + +.Results +[opts="header",cols="1"] +|=== +| value +a| +[source,json] +---- +{ + "b": "foo", + "c": [ + 1, + 2, + 3 + ], + "a": 42 +} +|=== + +.Convert from YAML string to node +[source,cypher] +---- +RETURN apoc.convert.fromYaml("--- +id: \"\" +type: \"node\" +labels: +- \"Test\" +properties: +foo: 7") as value +---- + +.Results +[opts="header",cols="1"] +|=== +| value +a| +[source,json] +---- +{ + "id": "", + "labels": [ + "Test" + ], + "properties": { + "foo": 7 + }, + "type": "node" +} +|=== + +.Convert from YAML string to map with null values +[source,cypher] +---- +RETURN apoc.convert.fromYaml("--- +a: null +b: \"myString\" +c: +- 1 +- \"2\" +- null") as value +---- + +.Results +[opts="header",cols="1"] +|=== +| value +a| +[source,json] +---- +{ + "b": "myString", + "c": [ + 1, + "2", + null + ], + "a": null +} +|=== + +.Convert from YAML string to map of nodes +[source,cypher] +---- +RETURN apoc.convert.fromYaml("--- +one: + id: \"8d3a6b87-39ad-4482-9ce7-5684fe79fc57\" + type: \"node\" + labels: + - \"Test\" + properties: + foo: 7 +two: + id: \"3fc16aeb-629f-4181-97d2-a25b22b28b75\" + type: \"node\" + labels: + - \"Test\" + properties: + bar: 9 +") as value +---- + +.Results +[opts="header",cols="1"] +|=== +| value +a| +[source,json] +---- +{ + "two": { + "id": "3fc16aeb-629f-4181-97d2-a25b22b28b75", + "labels": [ + "Test" + ], + "properties": null, + "bar": 9, + "type": "node" + }, + "one": { + "id": "8d3a6b87-39ad-4482-9ce7-5684fe79fc57", + "labels": [ + "Test" + ], + "foo": 7, + "properties": null, + "type": "node" + } +} +|=== + +.Convert from YAML string to relationship +[source,cypher] +---- +RETURN apoc.convert.fromYaml("--- +id: \"94996be1-7200-48c2-81e8-479f28bba84d\" +type: \"relationship\" +label: \"KNOWS\" +start: + id: \"8d3a6b87-39ad-4482-9ce7-5684fe79fc57\" + type: \"node\" + labels: + - \"User\" + properties: + name: \"Adam\" +end: + id: \"3fc16aeb-629f-4181-97d2-a25b22b28b75\" + type: \"node\" + labels: + - \"User\" + properties: + name: \"Jim\" + age: 42 +properties: + bffSince: \"P5M1DT12H\" + since: 1993.1 +") as value +---- + +.Results +[opts="header",cols="1"] +|=== +| value +a| +[source,json] +---- +{ + "id": "94996be1-7200-48c2-81e8-479f28bba84d", + "start": { + "id": "8d3a6b87-39ad-4482-9ce7-5684fe79fc57", + "name": "Adam", + "labels": [ + "User" + ], + "properties": null, + "type": "node" + }, + "label": "KNOWS", + "properties": { + "bffSince": "P5M1DT12H", + "since": 1993.1 + }, + "type": "relationship", + "end": { + "id": "3fc16aeb-629f-4181-97d2-a25b22b28b75", + "labels": [ + "User" + ], + "properties": { + "name": "Jim", + "age": 42 + }, + "type": "node" + } +} +|=== \ No newline at end of file diff --git a/docs/asciidoc/modules/ROOT/partials/usage/config/apoc.convert.fromYaml.adoc b/docs/asciidoc/modules/ROOT/partials/usage/config/apoc.convert.fromYaml.adoc new file mode 100644 index 0000000000..96abe219c2 --- /dev/null +++ b/docs/asciidoc/modules/ROOT/partials/usage/config/apoc.convert.fromYaml.adoc @@ -0,0 +1,17 @@ +The procedure support the following config parameters: + +.Config parameters +[opts=header, cols="1,1,1,5"] +|=== +| name | type | default | description +| disable | list of strings | empty list | To disable one or more configurations, enabled by default, of the library used under the hood. + See https://www.javadoc.io/doc/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/latest/com/fasterxml/jackson/dataformat/yaml/YAMLGenerator.Feature.html[here]. +| enable | list of strings | empty list | To enable one or more configurations of the library used under the hood. + See https://www.javadoc.io/doc/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/latest/com/fasterxml/jackson/dataformat/yaml/YAMLGenerator.Feature.html[here]. +| mapping | map of strings | empty map | to map complex YAMLs. + +In order to read complex types not supported by FasterXML Jackson, like Long... +we can use the mapping config to convert to the desired data type. +For example, if we have a YAML `a: 42 b: foo` , we can define a map `{mapping: {a: "Long"} }` + +|=== \ No newline at end of file diff --git a/full/src/main/java/apoc/convert/ConvertExtended.java b/full/src/main/java/apoc/convert/ConvertExtended.java new file mode 100644 index 0000000000..52f8aabca0 --- /dev/null +++ b/full/src/main/java/apoc/convert/ConvertExtended.java @@ -0,0 +1,169 @@ +package apoc.convert; + +import apoc.Extended; +import apoc.util.JsonUtil; +import apoc.util.MissingDependencyException; + +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Path; +import org.neo4j.graphdb.Relationship; +import org.neo4j.procedure.Description; +import org.neo4j.procedure.Name; +import org.neo4j.procedure.UserFunction; + +import java.math.BigInteger; +import java.time.Duration; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.temporal.TemporalAccessor; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.LongStream; +import java.util.stream.Stream; + +import static apoc.export.csv.CsvPropertyConverter.getPrototypeFor; +import static apoc.export.cypher.formatter.CypherFormatterUtils.formatProperties; +import static apoc.export.cypher.formatter.CypherFormatterUtils.formatToString; +import static apoc.util.Util.getAllQueryProcs; + +@Extended +public class ConvertExtended { + public static final String MAPPING_KEY = "mapping"; + + + @UserFunction("apoc.convert.fromYaml") + @Description("apoc.convert.fromYaml(value, $config) - Deserializes the YAML string to Neo4j value") + public Object fromYaml(@Name("value") String value, @Name(value = "config", defaultValue = "{}") Map config) throws Exception { + try { + Object parse = parse(value, config); + Ma mappingConf = (Map) config.getOrDefault(MAPPING_KEY, Map.of()); + return toValidYamlValue(parse, null, mappingConf, true); + } catch (NoClassDefFoundError e) { + throw new MissingDependencyException(YAML_MISSING_DEPS_ERROR); + } + } + + + + public static Object parse(String value, Map config) throws JsonProcessingException { + YAMLFactory factory = new YAMLFactory(); + + List enable = (List) config.getOrDefault("enable", List.of()); + List disable = (List) config.getOrDefault("disable", List.of()); + enable.forEach(name -> factory.enable(YAMLGenerator.Feature.valueOf(name))); + disable.forEach(name -> factory.disable(YAMLGenerator.Feature.valueOf(name))); + + ObjectMapper objectMapper = new ObjectMapper(factory); + objectMapper.registerModule(YAML_MODULE); + + return objectMapper.readValue(value, Object.class); + } + + public static Object toValidYamlValue(Object object, String field, Map mapping, boolean start) { + Object fieldName = field == null ? null : mapping.get(field); + + if (fieldName == null) { + fieldName = start ? mapping.get("_") : null; + } + + if (object instanceof Collection) { + return ((Collection) object).stream() + .map(i -> toValidYamlValue(i, field, mapping, start)) + .collect(Collectors.toList()) + .toArray(i -> new Object[] {}); + } + if (object instanceof Map) { + return ((Map) object).entrySet().stream() + .collect( + HashMap::new, // workaround for https://bugs.openjdk.java.net/browse/JDK-8148463 + (mapAccumulator, entry) -> + mapAccumulator.put(entry.getKey(), toValidYamlValue(entry.getValue(), entry.getKey(), mapping, false)), + HashMap::putAll); + } + + if (object != null && fieldName != null) { + return convertValue(object.toString(), fieldName.toString()); + } + + return getNeo4jValue(object); + } + + private static Object getNeo4jValue(Object object) { + try { + // we test if is a valid Neo4j type + Values.of(object); + return object; + } catch (Exception e) { + // otherwise we try to coerce it + return object.toString(); + } + } + + /** + * In case of complex type non-readable from Parquet, i.e. Duration, Point, List of Neo4j Types... + * we can use the `mapping: {keyToConvert: valueTypeName}` config to convert them. + * For example `mapping: {myPropertyKey: "DateArray"}` + */ + private static Object convertValue(String value, String typeName) { + switch (typeName) { + // {"crs":"wgs-84-3d","latitude":13.1,"longitude":33.46789,"height":100.0} + case "Point": + return getPointValue(value); + case "LocalDateTime": + return LocalDateTimeValue.parse(value).asObjectCopy(); + case "LocalTime": + return LocalTimeValue.parse(value).asObjectCopy(); + case "DateTime": + return DateTimeValue.parse(value, () -> ZoneId.of("Z")).asObjectCopy(); + case "Time": + return TimeValue.parse(value, () -> ZoneId.of("Z")).asObjectCopy(); + case "Date": + return DateValue.parse(value).asObjectCopy(); + case "Duration": + return DurationValue.parse(value); + case "Char": + return value.charAt(0); + case "Byte": + return value.getBytes(); + case "Double": + return Double.parseDouble(value); + case "Float": + return Float.parseFloat(value); + case "Short": + return Short.parseShort(value); + case "Int": + return Integer.parseInt(value); + case "Long": + return Long.parseLong(value); + case "Node", "Relationship": + return JsonUtil.parse(value, null, Map.class); + case "NO_VALUE": + return null; + default: + // If ends with "Array", for example StringArray + if (typeName.endsWith("Array")) { + value = StringUtils.removeStart(value, "["); + value = StringUtils.removeEnd(value, "]"); + String array = typeName.replace("Array", ""); + final Object[] prototype = getPrototypeFor( array.toUpperCase() ); + return Arrays.stream(value.split(",")) + .map(item -> convertValue(StringUtils.trim(item), array)) + .collect(Collectors.toList()) + .toArray(prototype); + } + return value; + } + } +} diff --git a/full/src/main/resources/extended.txt b/full/src/main/resources/extended.txt index f52194c086..6a335870b1 100644 --- a/full/src/main/resources/extended.txt +++ b/full/src/main/resources/extended.txt @@ -8,6 +8,8 @@ apoc.bolt.load.fromLocal apoc.cluster.graph apoc.config.list apoc.config.map +apoc.convert.toYaml +apoc.convert.fromYaml apoc.couchbase.append apoc.couchbase.exists apoc.couchbase.get