Move protoc generation to binary crate (#5718)

.github/workflows/rust.yml

@@ -320,6 +320,20 @@ jobs:
       - name: Run datafusion-common tests
         run: cargo test -p datafusion-common --features=pyarrow
 
+  vendor:
+    name: Verify Vendored Code
+    runs-on: ubuntu-latest
+    container:
+      image: amd64/rust
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+      - name: Run gen
+        run: ./regen.sh
+        working-directory: ./datafusion/proto
+      - name: Verify workspace clean (if this fails, run ./datafusion/proto/regen.sh and check in results)
+        run: git diff --exit-code
 
   check-fmt:
     name: Check cargo fmt

Cargo.toml

@@ -26,6 +26,7 @@ members = [
     "datafusion/optimizer",
     "datafusion/physical-expr",
     "datafusion/proto",
+    "datafusion/proto/gen",
     "datafusion/row",
     "datafusion/sql",
     "datafusion/substrait",

datafusion/proto/Cargo.toml

@@ -54,8 +54,3 @@ serde_json = { version = "1.0", optional = true }
 [dev-dependencies]
 doc-comment = "0.3"
 tokio = "1.18"
-
-[build-dependencies]
-# Pin these dependencies so that the generated output is deterministic
-pbjson-build = { version = "=0.5.1" }
-prost-build = { version = "=0.11.7" }

datafusion/proto/README.md

@@ -94,4 +94,15 @@ async fn main() -> Result<()> {
 
 ```
 
+## Generated Code
+
+The prost/tonic code can be generated by running, which in turn invokes the Rust binary located in [gen](./gen)
+
+This is necessary after modifying the protobuf definitions or altering the dependencies of [gen](./gen), and requires a
+valid installation of [protoc](https://github.com/protocolbuffers/protobuf#protocol-compiler-installation).
+
+```bash
+./regen.sh
+```
+
 [df]: https://crates.io/crates/datafusion

datafusion/proto/gen/Cargo.toml

@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "gen"
+description = "Code generation for proto"
+version = "0.1.0"
+edition = "2021"
+rust-version = "1.62"
+authors = ["Apache Arrow <[email protected]>"]
+homepage = "https://github.com/apache/arrow-datafusion"
+repository = "https://github.com/apache/arrow-datafusion"
+license = "Apache-2.0"
+publish = false
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+# Pin these dependencies so that the generated output is deterministic
+pbjson-build = { version = "=0.5.1" }
+prost-build = { version = "=0.11.7" }

datafusion/proto/build.rs → datafusion/proto/gen/src/main.rs

@@ -15,51 +15,40 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use std::path::{Path, PathBuf};
+use std::path::Path;
 
 type Error = Box<dyn std::error::Error>;
 type Result<T, E = Error> = std::result::Result<T, E>;
 
 fn main() -> Result<(), String> {
-    // for use in docker build where file changes can be wonky
-    println!("cargo:rerun-if-env-changed=FORCE_REBUILD");
+    let proto_dir = Path::new("proto");
+    let proto_path = Path::new("proto/datafusion.proto");
 
-    // We don't include the proto files in releases so that downstreams
-    // do not need to have PROTOC included
-    if Path::new("proto/datafusion.proto").exists() {
-        println!("cargo:rerun-if-changed=proto/datafusion.proto");
-        build()?
-    }
-
-    Ok(())
-}
-
-fn build() -> Result<(), String> {
-    let out: PathBuf = std::env::var("OUT_DIR")
-        .expect("Cannot find OUT_DIR environment variable")
-        .into();
-    let descriptor_path = out.join("proto_descriptor.bin");
+    // proto definitions has to be there
+    let descriptor_path = proto_dir.join("proto_descriptor.bin");
 
     prost_build::Config::new()
         .file_descriptor_set_path(&descriptor_path)
+        .out_dir("src")
         .compile_well_known_types()
         .extern_path(".google.protobuf", "::pbjson_types")
-        .compile_protos(&["proto/datafusion.proto"], &["proto"])
+        .compile_protos(&[proto_path], &["proto"])
         .map_err(|e| format!("protobuf compilation failed: {e}"))?;
 
     let descriptor_set = std::fs::read(&descriptor_path)
         .unwrap_or_else(|e| panic!("Cannot read {:?}: {}", &descriptor_path, e));
 
     pbjson_build::Builder::new()
+        .out_dir("src")
         .register_descriptors(&descriptor_set)
         .unwrap_or_else(|e| {
             panic!("Cannot register descriptors {:?}: {}", &descriptor_set, e)
         })
         .build(&[".datafusion"])
         .map_err(|e| format!("pbjson compilation failed: {e}"))?;
 
-    let prost = out.join("datafusion.rs");
-    let pbjson = out.join("datafusion.serde.rs");
+    let prost = Path::new("src/datafusion.rs");
+    let pbjson = Path::new("src/datafusion.serde.rs");
 
     std::fs::copy(prost, "src/generated/prost.rs").unwrap();
     std::fs::copy(pbjson, "src/generated/pbjson.rs").unwrap();

datafusion/proto/regen.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+cd $SCRIPT_DIR && cargo run --manifest-path gen/Cargo.toml