Update to arrow/parquet 53.0.0, tonic, prost, object_store, pyo3

Cargo.toml

@@ -164,3 +164,20 @@ large_futures = "warn"
 [workspace.lints.rust]
 unexpected_cfgs = { level = "warn", check-cfg = ["cfg(tarpaulin)"] }
 unused_imports = "deny"
+
+
+## Temporary arrow-rs patch until 53.0.0 is released
+
+[patch.crates-io]
+arrow = { git = "https://github.com/apache/arrow-rs.git", rev = "042d725888358c73cd2a0d58868ea5c4bad778f7" }
+arrow-array = { git = "https://github.com/apache/arrow-rs.git", rev = "042d725888358c73cd2a0d58868ea5c4bad778f7" }
+arrow-buffer = { git = "https://github.com/apache/arrow-rs.git", rev = "042d725888358c73cd2a0d58868ea5c4bad778f7" }
+arrow-cast = { git = "https://github.com/apache/arrow-rs.git", rev = "042d725888358c73cd2a0d58868ea5c4bad778f7" }
+arrow-data = { git = "https://github.com/apache/arrow-rs.git", rev = "042d725888358c73cd2a0d58868ea5c4bad778f7" }
+arrow-ipc = { git = "https://github.com/apache/arrow-rs.git", rev = "042d725888358c73cd2a0d58868ea5c4bad778f7" }
+arrow-schema = { git = "https://github.com/apache/arrow-rs.git", rev = "042d725888358c73cd2a0d58868ea5c4bad778f7" }
+arrow-select = { git = "https://github.com/apache/arrow-rs.git", rev = "042d725888358c73cd2a0d58868ea5c4bad778f7" }
+arrow-string = { git = "https://github.com/apache/arrow-rs.git", rev = "042d725888358c73cd2a0d58868ea5c4bad778f7" }
+arrow-ord = { git = "https://github.com/apache/arrow-rs.git", rev = "042d725888358c73cd2a0d58868ea5c4bad778f7" }
+arrow-flight = { git = "https://github.com/apache/arrow-rs.git", rev = "042d725888358c73cd2a0d58868ea5c4bad778f7" }
+parquet = { git = "https://github.com/apache/arrow-rs.git", rev = "042d725888358c73cd2a0d58868ea5c4bad778f7" }

datafusion-examples/Cargo.toml

@@ -72,14 +72,14 @@ log = { workspace = true }
 mimalloc = { version = "0.1", default-features = false }
 num_cpus = { workspace = true }
 object_store = { workspace = true, features = ["aws", "http"] }
-prost = { version = "0.12", default-features = false }
-prost-derive = { version = "0.13", default-features = false }
+prost = { version = "0.13.1", default-features = false }
+prost-derive = { version = "0.13.1", default-features = false }
 serde = { version = "1.0.136", features = ["derive"] }
 serde_json = { workspace = true }
 tempfile = { workspace = true }
 test-utils = { path = "../test-utils" }
 tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot"] }
-tonic = "0.11"
+tonic = "0.12.1"
 url = { workspace = true }
 uuid = "1.7"
 

datafusion/common/Cargo.toml

@@ -61,7 +61,7 @@ num_cpus = { workspace = true }
 object_store = { workspace = true, optional = true }
 parquet = { workspace = true, optional = true, default-features = true }
 paste = "1.0.15"
-pyo3 = { version = "0.21.0", optional = true }
+pyo3 = { version = "0.22.0", optional = true }
 sqlparser = { workspace = true }
 
 [target.'cfg(target_family = "wasm")'.dependencies]

datafusion/core/src/datasource/file_format/parquet.rs

@@ -1999,7 +1999,7 @@ mod tests {
 
         // test result in int_col
         let int_col_index = page_index.get(4).unwrap();
-        let int_col_offset = offset_index.get(4).unwrap();
+        let int_col_offset = offset_index.get(4).unwrap().page_locations();
 
         // 325 pages in int_col
         assert_eq!(int_col_offset.len(), 325);

datafusion/core/src/datasource/physical_plan/parquet/page_filter.rs

@@ -392,13 +392,16 @@ impl<'a> PagesPruningStatistics<'a> {
             trace!("No page offsets for row group {row_group_index}, skipping");
             return None;
         };
-        let Some(page_offsets) = row_group_page_offsets.get(parquet_column_index) else {
+        let Some(offset_index_metadata) =
+            row_group_page_offsets.get(parquet_column_index)
+        else {
             trace!(
                 "No page offsets for column {:?} in row group {row_group_index}, skipping",
                 converter.arrow_field()
             );
             return None;
         };
+        let page_offsets = offset_index_metadata.page_locations();
 
         Some(Self {
             row_group_index,

datafusion/core/src/datasource/physical_plan/parquet/row_group_filter.rs

@@ -487,11 +487,23 @@ mod tests {
         let schema_descr = get_test_schema_descr(vec![field]);
         let rgm1 = get_row_group_meta_data(
             &schema_descr,
-            vec![ParquetStatistics::int32(Some(1), Some(10), None, 0, false)],
+            vec![ParquetStatistics::int32(
+                Some(1),
+                Some(10),
+                None,
+                Some(0),
+                false,
+            )],
         );
         let rgm2 = get_row_group_meta_data(
             &schema_descr,
-            vec![ParquetStatistics::int32(Some(11), Some(20), None, 0, false)],
+            vec![ParquetStatistics::int32(
+                Some(11),
+                Some(20),
+                None,
+                Some(0),
+                false,
+            )],
         );
 
         let metrics = parquet_file_metrics();
@@ -520,11 +532,11 @@ mod tests {
         let schema_descr = get_test_schema_descr(vec![field]);
         let rgm1 = get_row_group_meta_data(
             &schema_descr,
-            vec![ParquetStatistics::int32(None, None, None, 0, false)],
+            vec![ParquetStatistics::int32(None, None, None, Some(0), false)],
         );
         let rgm2 = get_row_group_meta_data(
             &schema_descr,
-            vec![ParquetStatistics::int32(Some(11), Some(20), None, 0, false)],
+            vec![ParquetStatistics::int32(Some(11), Some(20), None, Some(0), false)],
         );
         let metrics = parquet_file_metrics();
         // missing statistics for first row group mean that the result from the predicate expression
@@ -560,15 +572,15 @@ mod tests {
         let rgm1 = get_row_group_meta_data(
             &schema_descr,
             vec![
-                ParquetStatistics::int32(Some(1), Some(10), None, 0, false),
-                ParquetStatistics::int32(Some(1), Some(10), None, 0, false),
+                ParquetStatistics::int32(Some(1), Some(10), None, Some(0), false),
+                ParquetStatistics::int32(Some(1), Some(10), None, Some(0), false),
             ],
         );
         let rgm2 = get_row_group_meta_data(
             &schema_descr,
             vec![
-                ParquetStatistics::int32(Some(11), Some(20), None, 0, false),
-                ParquetStatistics::int32(Some(11), Some(20), None, 0, false),
+                ParquetStatistics::int32(Some(11), Some(20), None, Some(0), false),
+                ParquetStatistics::int32(Some(11), Some(20), None, Some(0), false),
             ],
         );
 
@@ -633,16 +645,16 @@ mod tests {
         let rgm1 = get_row_group_meta_data(
             &schema_descr,
             vec![
-                ParquetStatistics::int32(Some(-10), Some(-1), None, 0, false), // c2
-                ParquetStatistics::int32(Some(1), Some(10), None, 0, false),
+                ParquetStatistics::int32(Some(-10), Some(-1), None, Some(0), false), // c2
+                ParquetStatistics::int32(Some(1), Some(10), None, Some(0), false),
             ],
         );
         // rg1 has c2 greater than zero, c1 less than zero
         let rgm2 = get_row_group_meta_data(
             &schema_descr,
             vec![
-                ParquetStatistics::int32(Some(1), Some(10), None, 0, false),
-                ParquetStatistics::int32(Some(-10), Some(-1), None, 0, false),
+                ParquetStatistics::int32(Some(1), Some(10), None, Some(0), false),
+                ParquetStatistics::int32(Some(-10), Some(-1), None, Some(0), false),
             ],
         );
 
@@ -669,15 +681,15 @@ mod tests {
         let rgm1 = get_row_group_meta_data(
             &schema_descr,
             vec![
-                ParquetStatistics::int32(Some(1), Some(10), None, 0, false),
-                ParquetStatistics::boolean(Some(false), Some(true), None, 0, false),
+                ParquetStatistics::int32(Some(1), Some(10), None, Some(0), false),
+                ParquetStatistics::boolean(Some(false), Some(true), None, Some(0), false),
             ],
         );
         let rgm2 = get_row_group_meta_data(
             &schema_descr,
             vec![
-                ParquetStatistics::int32(Some(11), Some(20), None, 0, false),
-                ParquetStatistics::boolean(Some(false), Some(true), None, 1, false),
+                ParquetStatistics::int32(Some(11), Some(20), None, Some(0), false),
+                ParquetStatistics::boolean(Some(false), Some(true), None, Some(1), false),
             ],
         );
         vec![rgm1, rgm2]
@@ -775,21 +787,21 @@ mod tests {
                 Some(100),
                 Some(600),
                 None,
-                0,
+                Some(0),
                 false,
             )],
         );
         let rgm2 = get_row_group_meta_data(
             &schema_descr,
             // [0.1, 0.2]
             // c1 > 5, this row group will not be included in the results.
-            vec![ParquetStatistics::int32(Some(10), Some(20), None, 0, false)],
+            vec![ParquetStatistics::int32(Some(10), Some(20), None, Some(0), false)],
         );
         let rgm3 = get_row_group_meta_data(
             &schema_descr,
             // [1, None]
             // c1 > 5, this row group can not be filtered out, so will be included in the results.
-            vec![ParquetStatistics::int32(Some(100), None, None, 0, false)],
+            vec![ParquetStatistics::int32(Some(100), None, None, Some(0), false)],
         );
         let metrics = parquet_file_metrics();
         let mut row_groups = RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(3));
@@ -837,27 +849,27 @@ mod tests {
                 Some(100),
                 Some(600),
                 None,
-                0,
+                Some(0),
                 false,
             )],
         );
         let rgm2 = get_row_group_meta_data(
             &schema_descr,
             // [10, 20]
             // c1 > 5, this row group will be included in the results.
-            vec![ParquetStatistics::int32(Some(10), Some(20), None, 0, false)],
+            vec![ParquetStatistics::int32(Some(10), Some(20), None, Some(0), false)],
         );
         let rgm3 = get_row_group_meta_data(
             &schema_descr,
             // [0, 2]
             // c1 > 5, this row group will not be included in the results.
-            vec![ParquetStatistics::int32(Some(0), Some(2), None, 0, false)],
+            vec![ParquetStatistics::int32(Some(0), Some(2), None, Some(0), false)],
         );
         let rgm4 = get_row_group_meta_data(
             &schema_descr,
             // [None, 2]
             // c1 > 5, this row group can not be filtered out, so will be included in the results.
-            vec![ParquetStatistics::int32(None, Some(2), None, 0, false)],
+            vec![ParquetStatistics::int32(None, Some(2), None, Some(0), false)],
         );
         let metrics = parquet_file_metrics();
         let mut row_groups = RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(4));
@@ -896,19 +908,19 @@ mod tests {
                 Some(600),
                 Some(800),
                 None,
-                0,
+                Some(0),
                 false,
             )],
         );
         let rgm2 = get_row_group_meta_data(
             &schema_descr,
             // [0.1, 0.2]
-            vec![ParquetStatistics::int64(Some(10), Some(20), None, 0, false)],
+            vec![ParquetStatistics::int64(Some(10), Some(20), None, Some(0), false)],
         );
         let rgm3 = get_row_group_meta_data(
             &schema_descr,
             // [0.1, 0.2]
-            vec![ParquetStatistics::int64(None, None, None, 0, false)],
+            vec![ParquetStatistics::int64(None, None, None, Some(0), false)],
         );
         let metrics = parquet_file_metrics();
         let mut row_groups = RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(3));
@@ -957,7 +969,7 @@ mod tests {
                     8000i128.to_be_bytes().to_vec(),
                 ))),
                 None,
-                0,
+                Some(0),
                 false,
             )],
         );
@@ -973,15 +985,15 @@ mod tests {
                     20000i128.to_be_bytes().to_vec(),
                 ))),
                 None,
-                0,
+                Some(0),
                 false,
             )],
         );
 
         let rgm3 = get_row_group_meta_data(
             &schema_descr,
             vec![ParquetStatistics::fixed_len_byte_array(
-                None, None, None, 0, false,
+                None, None, None, Some(0), false,
             )],
         );
         let metrics = parquet_file_metrics();
@@ -1027,7 +1039,7 @@ mod tests {
                 // 80.00
                 Some(ByteArray::from(8000i128.to_be_bytes().to_vec())),
                 None,
-                0,
+                Some(0),
                 false,
             )],
         );
@@ -1039,13 +1051,13 @@ mod tests {
                 // 200.00
                 Some(ByteArray::from(20000i128.to_be_bytes().to_vec())),
                 None,
-                0,
+                Some(0),
                 false,
             )],
         );
         let rgm3 = get_row_group_meta_data(
             &schema_descr,
-            vec![ParquetStatistics::byte_array(None, None, None, 0, false)],
+            vec![ParquetStatistics::byte_array(None, None, None, Some(0), false)],
         );
         let metrics = parquet_file_metrics();
         let mut row_groups = RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(3));

datafusion/functions/src/regex/regexpreplace.rs

@@ -401,8 +401,7 @@ fn _regexp_replace_static_pattern_replace<T: OffsetSizeTrait>(
         DataType::Utf8View => {
             let string_view_array = as_string_view_array(&args[0])?;
 
-            let mut builder = StringViewBuilder::with_capacity(string_view_array.len())
-                .with_block_size(1024 * 1024 * 2);
+            let mut builder = StringViewBuilder::with_capacity(string_view_array.len());
 
             for val in string_view_array.iter() {
                 if let Some(val) = val {

datafusion/physical-expr-common/src/binary_view_map.rs

@@ -149,7 +149,7 @@ where
             output_type,
             map: hashbrown::raw::RawTable::with_capacity(INITIAL_MAP_CAPACITY),
             map_size: 0,
-            builder: GenericByteViewBuilder::new().with_block_size(2 * 1024 * 1024),
+            builder: GenericByteViewBuilder::new(),
             random_state: RandomState::new(),
             hashes_buffer: vec![],
             null: None,

datafusion/physical-plan/src/coalesce_batches.rs

@@ -535,7 +535,7 @@ fn gc_string_view_batch(batch: &RecordBatch) -> RecordBatch {
                 // See https://github.com/apache/arrow-rs/issues/6094 for more details.
                 let mut builder = StringViewBuilder::with_capacity(s.len());
                 if ideal_buffer_size > 0 {
-                    builder = builder.with_block_size(ideal_buffer_size as u32);
+                    builder = builder.with_fixed_block_size(ideal_buffer_size as u32);
                 }
 
                 for v in s.iter() {
@@ -856,7 +856,8 @@ mod tests {
     impl StringViewTest {
         /// Create a `StringViewArray` with the parameters specified in this struct
         fn build(self) -> StringViewArray {
-            let mut builder = StringViewBuilder::with_capacity(100).with_block_size(8192);
+            let mut builder =
+                StringViewBuilder::with_capacity(100).with_fixed_block_size(8192);
             loop {
                 for &v in self.strings.iter() {
                     builder.append_option(v);

datafusion/proto-common/Cargo.toml

@@ -44,8 +44,8 @@ arrow = { workspace = true }
 chrono = { workspace = true }
 datafusion-common = { workspace = true }
 object_store = { workspace = true }
-pbjson = { version = "0.6.0", optional = true }
-prost = "0.12.0"
+pbjson = { version = "0.7.0", optional = true }
+prost = "0.13.0"
 serde = { version = "1.0", optional = true }
 serde_json = { workspace = true, optional = true }
 

datafusion/proto-common/gen/Cargo.toml

@@ -34,5 +34,5 @@ workspace = true
 
 [dependencies]
 # Pin these dependencies so that the generated output is deterministic
-pbjson-build = "=0.6.2"
-prost-build = "=0.12.6"
+pbjson-build = "=0.7.0"
+prost-build = "=0.13.1"

datafusion/proto/Cargo.toml

@@ -52,8 +52,8 @@ datafusion-common = { workspace = true, default-features = true }
 datafusion-expr = { workspace = true }
 datafusion-proto-common = { workspace = true }
 object_store = { workspace = true }
-pbjson = { version = "0.6.0", optional = true }
-prost = "0.12.0"
+pbjson = { version = "0.7.0", optional = true }
+prost = "0.13.0"
 serde = { version = "1.0", optional = true }
 serde_json = { workspace = true, optional = true }
 

datafusion/proto/gen/Cargo.toml

@@ -34,5 +34,5 @@ workspace = true
 
 [dependencies]
 # Pin these dependencies so that the generated output is deterministic
-pbjson-build = "=0.6.2"
-prost-build = "=0.12.6"
+pbjson-build = "=0.7.0"
+prost-build = "=0.13.1"