Bump arrow2

Cargo.toml

@@ -38,5 +38,5 @@ lto = true
 codegen-units = 1
 
 [patch.crates-io]
-arrow2 = { git = "https://github.com/jorgecarleitao/arrow2.git", rev = "v0.10.0" }
-parquet2 = { git = "https://github.com/jorgecarleitao/parquet2.git", rev = "v0.10.1" }
+arrow2 = { git = "https://github.com/jorgecarleitao/arrow2.git", rev = "v0.12.0" }
+parquet2 = { git = "https://github.com/jorgecarleitao/parquet2.git", rev = "v0.13.2" }

ballista-examples/Cargo.toml

@@ -31,8 +31,8 @@ rust-version = "1.59"
 [dependencies]
 datafusion = { path = "../datafusion" }
 ballista = { path = "../ballista/rust/client", version = "0.6.0"}
-prost = "0.9"
-tonic = "0.6"
+prost = "0.10"
+tonic = "0.7"
 tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot"] }
 futures = "0.3"
 num_cpus = "1.13.0"

ballista/rust/core/Cargo.toml

@@ -35,19 +35,19 @@ async-trait = "0.1.41"
 futures = "0.3"
 hashbrown = "0.12"
 log = "0.4"
-prost = "0.9"
-prost-types = "0.9"
+prost = "0.10"
+prost-types = "0.10"
 serde = {version = "1", features = ["derive"]}
 sqlparser = "0.15"
 tokio = "1.0"
-tonic = "0.6"
+tonic = "0.7"
 uuid = { version = "0.8", features = ["v4"] }
 chrono = { version = "0.4", default-features = false }
 clap = { version = "3", features = ["derive", "cargo"] }
 parse_arg = "0.1.3"
 
-arrow-format = { version = "0.4", features = ["flight-data", "flight-service"] }
-arrow = { package = "arrow2", version="0.10", features = ["io_ipc", "io_flight"] }
+arrow-format = { version = "0.6", features = ["flight-data", "flight-service"] }
+arrow = { package = "arrow2", version="0.12", features = ["io_ipc", "io_flight"] }
 
 datafusion = { path = "../../../datafusion", version = "7.0.0" }
 datafusion-proto = { path = "../../../datafusion-proto", version = "7.0.0" }
@@ -58,4 +58,4 @@ parking_lot = "0.12"
 tempfile = "3"
 
 [build-dependencies]
-tonic-build = { version = "0.6" }
+tonic-build = { version = "0.7" }

ballista/rust/core/src/client.rs

@@ -34,7 +34,7 @@ use arrow_format::flight::data::{FlightData, Ticket};
 use arrow_format::flight::service::flight_service_client::FlightServiceClient;
 use datafusion::arrow::{
     datatypes::SchemaRef,
-    error::{ArrowError, Result as ArrowResult},
+    error::{Error as ArrowError, Result as ArrowResult},
 };
 use datafusion::field_util::SchemaExt;
 use datafusion::physical_plan::RecordBatchStream;

ballista/rust/core/src/error.rs

@@ -23,7 +23,7 @@ use std::{
     io, result,
 };
 
-use datafusion::arrow::error::ArrowError;
+use datafusion::arrow::error::Error as ArrowError;
 use datafusion::error::DataFusionError;
 use sqlparser::parser;
 

ballista/rust/core/src/memory_stream.rs

@@ -1 +1,16 @@
-
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.

ballista/rust/executor/Cargo.toml

@@ -29,8 +29,8 @@ edition = "2018"
 snmalloc = ["snmalloc-rs"]
 
 [dependencies]
-arrow-format = { version = "0.4", features = ["flight-data", "flight-service"] }
-arrow = { package = "arrow2", version="0.10", features = ["io_ipc"] }
+arrow-format = { version = "0.6", features = ["flight-data", "flight-service"] }
+arrow = { package = "arrow2", version="0.12", features = ["io_ipc"] }
 anyhow = "1"
 async-trait = "0.1.41"
 ballista-core = { path = "../core", version = "0.6.0" }
@@ -43,7 +43,7 @@ snmalloc-rs = {version = "0.2", optional = true}
 tempfile = "3"
 tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "parking_lot"] }
 tokio-stream = { version = "0.1", features = ["net"] }
-tonic = "0.6"
+tonic = "0.7"
 uuid = { version = "0.8", features = ["v4"] }
 hyper = "0.14.4"
 parking_lot = "0.12"

ballista/rust/executor/src/execution_loop.rs

@@ -62,16 +62,14 @@ pub async fn poll_loop<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan>
         // to avoid going in sleep mode between polling
         let mut active_job = false;
 
-        let poll_work_result: anyhow::Result<
-            tonic::Response<PollWorkResult>,
-            tonic::Status,
-        > = scheduler
-            .poll_work(PollWorkParams {
-                metadata: Some(executor.metadata.clone()),
-                can_accept_task: available_tasks_slots.load(Ordering::SeqCst) > 0,
-                task_status,
-            })
-            .await;
+        let poll_work_result: Result<tonic::Response<PollWorkResult>, tonic::Status> =
+            scheduler
+                .poll_work(PollWorkParams {
+                    metadata: Some(executor.metadata.clone()),
+                    can_accept_task: available_tasks_slots.load(Ordering::SeqCst) > 0,
+                    task_status,
+                })
+                .await;
 
         let task_status_sender = task_status_sender.clone();
 

ballista/rust/executor/src/flight_service.rs

@@ -35,7 +35,7 @@ use arrow_format::flight::data::{
 };
 use arrow_format::flight::service::flight_service_server::FlightService;
 use datafusion::arrow::{
-    error::ArrowError, io::ipc::read::FileReader, io::ipc::write::WriteOptions,
+    error::Error as ArrowError, io::ipc::read::FileReader, io::ipc::write::WriteOptions,
 };
 use futures::{Stream, StreamExt};
 use log::{info, warn};

ballista/rust/scheduler/Cargo.toml

@@ -44,13 +44,13 @@ http-body = "0.4"
 hyper = "0.14.4"
 log = "0.4"
 parse_arg = "0.1.3"
-prost = "0.9"
+prost = "0.10"
 rand = "0.8"
 serde = {version = "1", features = ["derive"]}
 sled_package = { package = "sled", version = "0.34", optional = true }
 tokio = { version = "1.0", features = ["full"] }
 tokio-stream = { version = "0.1", features = ["net"], optional = true }
-tonic = "0.6"
+tonic = "0.7"
 tower = { version = "0.4" }
 warp = "0.3"
 parking_lot = "0.12"
@@ -62,7 +62,7 @@ uuid = { version = "0.8", features = ["v4"] }
 
 [build-dependencies]
 configure_me_codegen = "0.4.1"
-tonic-build = { version = "0.6" }
+tonic-build = { version = "0.7" }
 
 [package.metadata.configure_me.bin]
 scheduler = "scheduler_config_spec.toml"

benchmarks/Cargo.toml

@@ -32,7 +32,8 @@ simd = ["datafusion/simd"]
 snmalloc = ["snmalloc-rs"]
 
 [dependencies]
-arrow = { package = "arrow2", version="0.10", features = ["io_csv", "io_json", "io_parquet", "io_parquet_compression", "io_ipc", "io_print", "ahash", "compute_merge_sort", "compute", "regex"] }
+arrow = { package = "arrow2", version="0.12", features = ["io_csv", "io_json", "io_parquet", "io_parquet_compression", "io_ipc", "io_print", "ahash", "compute_merge_sort", "compute", "regex"] }
+parquet2 = "0.13"
 datafusion = { path = "../datafusion" }
 ballista = { path = "../ballista/rust/client" }
 structopt = { version = "0.3", default-features = false }

benchmarks/src/bin/tpch.rs

@@ -19,6 +19,7 @@
 
 use arrow::array::ArrayRef;
 use arrow::chunk::Chunk;
+use datafusion::arrow::io::print;
 use futures::future::join_all;
 use rand::prelude::*;
 use std::ops::Div;
@@ -31,8 +32,6 @@ use std::{
     time::{Instant, SystemTime},
 };
 
-use datafusion::arrow::io::print;
-
 use datafusion::datasource::{
     listing::{ListingOptions, ListingTable},
     object_store::local::LocalFileSystem,
@@ -52,7 +51,8 @@ use datafusion::{
     datasource::file_format::parquet::ParquetFormat, record_batch::RecordBatch,
 };
 
-use arrow::io::parquet::write::{Compression, Version, WriteOptions};
+use arrow::io::parquet::write::Version;
+use parquet2::compression::CompressionOptions;
 use ballista::prelude::{
     BallistaConfig, BallistaContext, BALLISTA_DEFAULT_SHUFFLE_PARTITIONS,
 };
@@ -665,26 +665,26 @@ async fn convert_tbl(opt: ConvertOpt) -> Result<()> {
         match opt.file_format.as_str() {
             "csv" => ctx.write_csv(csv, output_path).await?,
             "parquet" => {
+
                 let compression = match opt.compression.as_str() {
-                    "none" => Compression::Uncompressed,
-                    "snappy" => Compression::Snappy,
-                    "brotli" => Compression::Brotli,
-                    "gzip" => Compression::Gzip,
-                    "lz4" => Compression::Lz4,
-                    "lz0" => Compression::Lzo,
-                    "zstd" => Compression::Zstd,
+                    "none" => CompressionOptions::Uncompressed,
+                    "snappy" => CompressionOptions::Snappy,
+                    "brotli" => CompressionOptions::Brotli(None),
+                    "gzip" => CompressionOptions::Gzip(None),
+                    "lz4" => CompressionOptions::Lz4,
+                    "lz0" => CompressionOptions::Lzo,
+                    "zstd" => CompressionOptions::Zstd(None),
                     other => {
                         return Err(DataFusionError::NotImplemented(format!(
                             "Invalid compression format: {}",
                             other
                         )))
                     }
                 };
-
-                let options = WriteOptions {
-                    compression,
+                let options = arrow::io::parquet::write::WriteOptions {
                     write_statistics: false,
                     version: Version::V1,
+                    compression,
                 };
                 ctx.write_parquet(csv, output_path, options).await?
             }

datafusion-cli/Cargo.toml

@@ -32,7 +32,7 @@ clap = { version = "3", features = ["derive", "cargo"] }
 rustyline = "9.0"
 tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot"] }
 datafusion = { path = "../datafusion", version = "7.0.0" }
-arrow = { package = "arrow2", version="0.10", features = ["io_print"] }
+arrow = { package = "arrow2", version="0.12", features = ["io_print"] }
 ballista = { path = "../ballista/rust/client", version = "0.6.0", optional=true }
 env_logger = "0.9"
 mimalloc = { version = "*", default-features = false }

datafusion-common/Cargo.toml

@@ -37,8 +37,8 @@ pyarrow = ["pyo3"]
 jit = ["cranelift-module"]
 
 [dependencies]
-arrow = { package = "arrow2", version = "0.10", default-features = false }
-parquet = { package = "parquet2", version = "0.10", default_features = false, features = ["stream"], optional = true }
+arrow = { package = "arrow2", version = "0.12.0", default-features = false }
+parquet = { package = "parquet2", version = "0.13", default_features = false, optional = true }
 pyo3 = { version = "0.16", optional = true }
 sqlparser = "0.15"
 ordered-float = "2.10"

datafusion-common/src/error.rs

@@ -22,13 +22,13 @@ use std::fmt::{Display, Formatter};
 use std::io;
 use std::result;
 
-use arrow::error::ArrowError;
+use arrow::error::Error as ArrowError;
 #[cfg(feature = "avro")]
 use avro_rs::Error as AvroError;
 #[cfg(feature = "jit")]
 use cranelift_module::ModuleError;
 #[cfg(feature = "parquet")]
-use parquet::error::ParquetError;
+use parquet::error::Error as ParquetError;
 use sqlparser::parser::ParserError;
 
 /// Result type for operations that could result in an [DataFusionError]
@@ -181,7 +181,7 @@ impl error::Error for DataFusionError {}
 #[cfg(test)]
 mod test {
     use crate::error::DataFusionError;
-    use arrow::error::ArrowError;
+    use arrow::error::Error as ArrowError;
 
     #[test]
     fn arrow_error_to_datafusion() {

datafusion-common/src/field_util.rs

@@ -19,7 +19,7 @@
 
 use arrow::array::{ArrayRef, StructArray};
 use arrow::datatypes::{DataType, Field, Metadata, Schema};
-use arrow::error::ArrowError;
+use arrow::error::Error as ArrowError;
 use std::borrow::Borrow;
 use std::collections::BTreeMap;
 

datafusion-common/src/record_batch.rs

@@ -23,7 +23,7 @@ use arrow::array::*;
 use arrow::chunk::Chunk;
 use arrow::compute::filter::{build_filter, filter};
 use arrow::datatypes::*;
-use arrow::error::{ArrowError, Result};
+use arrow::error::{Error as ArrowError, Result};
 
 /// A two-dimensional dataset with a number of
 /// columns ([`Array`]) and rows and defined [`Schema`](crate::datatypes::Schema).

datafusion-examples/Cargo.toml

@@ -34,11 +34,11 @@ path = "examples/avro_sql.rs"
 required-features = ["datafusion/avro"]
 
 [dev-dependencies]
-arrow-format = { version = "0.4", features = ["flight-service", "flight-data"] }
-arrow = { package = "arrow2", version="0.10", features = ["io_ipc", "io_flight"] }
+arrow-format = { version = "0.6", features = ["flight-service", "flight-data"] }
+arrow = { package = "arrow2", version="0.12", features = ["io_ipc", "io_flight"] }
 datafusion = { path = "../datafusion" }
-prost = "0.9"
-tonic = "0.6"
+prost = "0.10"
+tonic = "0.7"
 tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot"] }
 futures = "0.3"
 num_cpus = "1.13.0"

datafusion-expr/Cargo.toml

@@ -36,6 +36,6 @@ path = "src/lib.rs"
 
 [dependencies]
 datafusion-common = { path = "../datafusion-common", version = "7.0.0" }
-arrow = { package = "arrow2", version = "0.10", default-features = false }
+arrow = { package = "arrow2", version = "0.12", default-features = false }
 sqlparser = "0.15"
 ahash = { version = "0.7", default-features = false }

datafusion-physical-expr/Cargo.toml

@@ -41,7 +41,7 @@ unicode_expressions = ["unicode-segmentation"]
 [dependencies]
 datafusion-common = { path = "../datafusion-common", version = "7.0.0" }
 datafusion-expr = { path = "../datafusion-expr", version = "7.0.0" }
-arrow = { package = "arrow2", version = "0.10" }
+arrow = { package = "arrow2", version = "0.12" }
 paste = "^1.0"
 ahash = { version = "0.7", default-features = false }
 ordered-float = "2.10"

datafusion-physical-expr/src/arrow_temporal_util.rs

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use arrow::error::{ArrowError, Result};
+use arrow::error::{Error as ArrowError, Result};
 use chrono::{prelude::*, LocalResult};
 
 /// Accepts a string in RFC3339 / ISO8601 standard format and some

datafusion-physical-expr/src/expressions/binary.rs

@@ -936,7 +936,7 @@ mod tests {
     use crate::expressions::{col, lit};
     use crate::test_util::create_decimal_array;
     use arrow::datatypes::{Field, SchemaRef};
-    use arrow::error::ArrowError;
+    use arrow::error::Error as ArrowError;
     use datafusion_common::field_util::SchemaExt;
 
     // TODO add iter for decimal array

datafusion-physical-expr/src/regex_expressions.rs

@@ -28,7 +28,7 @@ use std::any::type_name;
 use std::sync::Arc;
 
 use arrow::array::*;
-use arrow::error::ArrowError;
+use arrow::error::Error as ArrowError;
 
 use datafusion_common::{DataFusionError, Result};
 

datafusion-physical-expr/src/test_util.rs

@@ -1,3 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 use arrow::datatypes::DataType;
 
 #[cfg(test)]

datafusion-proto/Cargo.toml

@@ -36,7 +36,7 @@ path = "src/lib.rs"
 
 [dependencies]
 datafusion = { path = "../datafusion", version = "7.0.0" }
-prost = "0.9"
+prost = "0.10"
 
 [build-dependencies]
-tonic-build = { version = "0.6" }
+tonic-build = { version = "0.7" }