Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for protobuf serialisation of Arrow Map type #5359

Merged
merged 3 commits into from
Feb 26, 2023
Merged

Add support for protobuf serialisation of Arrow Map type #5359

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7e52344
Add protobuf serialisation support for Arrow Map type
ahmedriza Feb 21, 2023
bab7d7e
Merge branch 'apache:main' into proto-maps
ahmedriza Feb 21, 2023
ca353ec
Add protobuf serialisation support for Arrow Map type
ahmedriza Feb 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions datafusion/proto/proto/datafusion.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -743,6 +743,11 @@ message Struct{
repeated Field sub_field_types = 1;
}

message Map {
Field field_type = 1;
bool keys_sorted = 2;
}

enum UnionMode{
sparse = 0;
dense = 1;
Expand Down Expand Up @@ -894,6 +899,7 @@ message ArrowType{
Struct STRUCT = 28;
Union UNION = 29;
Dictionary DICTIONARY = 30;
Map MAP = 33;
}
}

Expand Down
123 changes: 123 additions & 0 deletions datafusion/proto/src/generated/pbjson.rs
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 11 additions & 1 deletion datafusion/proto/src/generated/prost.rs
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 6 additions & 0 deletions datafusion/proto/src/logical_plan/from_proto.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -351,6 +351,12 @@ impl TryFrom<&protobuf::arrow_type::ArrowTypeEnum> for DataType {
let value_datatype = dict.as_ref().value.as_deref().required("value")?;
DataType::Dictionary(Box::new(key_datatype), Box::new(value_datatype))
}
arrow_type::ArrowTypeEnum::Map(map) => {
let field: Field =
map.as_ref().field_type.as_deref().required("field_type")?;
let keys_sorted = map.keys_sorted;
DataType::Map(Box::new(field), keys_sorted)
}
})
}
}
Expand Down
11 changes: 11 additions & 0 deletions datafusion/proto/src/logical_plan/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2212,6 +2212,17 @@ mod roundtrip_tests {
4,
)),
),
DataType::Map(
new_box_field(
"entries",
DataType::Struct(vec![
Field::new("keys", DataType::Utf8, false),
Field::new("values", DataType::Int32, true),
]),
true,
),
false,
),
];

for test_case in test_cases.into_iter() {
Expand Down
9 changes: 6 additions & 3 deletions datafusion/proto/src/logical_plan/to_proto.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,9 +218,12 @@ impl TryFrom<&DataType> for protobuf::arrow_type::ArrowTypeEnum {
DataType::Decimal256(_, _) => {
return Err(Error::General("Proto serialization error: The Decimal256 data type is not yet supported".to_owned()))
}
DataType::Map(_, _) => {
return Err(Error::General(
"Proto serialization error: The Map data type is not yet supported".to_owned()
DataType::Map(field, sorted) => {
Self::Map(Box::new(
protobuf::Map {
field_type: Some(Box::new(field.as_ref().try_into()?)),
keys_sorted: *sorted,
}
))
}
DataType::RunEndEncoded(_, _) => {
Expand Down