Don't make users install protoc

.github/workflows/rust.yml

@@ -81,16 +81,6 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
-      - name: Install protobuf compiler
-        shell: bash
-        run: |
-          mkdir -p $HOME/d/protoc
-          cd $HOME/d/protoc
-          export PROTO_ZIP="protoc-21.4-linux-x86_64.zip"
-          curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
-          unzip $PROTO_ZIP
-          export PATH=$PATH:$HOME/d/protoc/bin
-          protoc --version
       - name: Cache Cargo
         uses: actions/cache@v3
         with:
@@ -103,11 +93,9 @@ jobs:
           rust-version: stable
       - name: Run tests
         run: |
-          export PATH=$PATH:$HOME/d/protoc/bin
           cargo test --features avro,jit,scheduler,json
       - name: Run examples
         run: |
-          export PATH=$PATH:$HOME/d/protoc/bin
           # test datafusion-sql examples
           cargo run --example sql
           # test datafusion-examples
@@ -190,16 +178,6 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
-      - name: Install protobuf compiler
-        shell: bash
-        run: |
-          mkdir -p $HOME/d/protoc
-          cd $HOME/d/protoc
-          export PROTO_ZIP="protoc-21.4-win64.zip"
-          curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
-          unzip $PROTO_ZIP
-          export PATH=$PATH:$HOME/d/protoc/bin
-          protoc.exe --version
       # TODO: this won't cache anything, which is expensive. Setup this action
       # with a OS-dependent path.
       - name: Setup Rust toolchain
@@ -210,7 +188,6 @@ jobs:
       - name: Run tests
         shell: bash
         run: |
-          export PATH=$PATH:$HOME/d/protoc/bin
           cargo test
         env:
           # do not produce debug symbols to keep memory usage down
@@ -223,17 +200,6 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
-      - name: Install protobuf compiler
-        shell: bash
-        run: |
-          mkdir -p $HOME/d/protoc
-          cd $HOME/d/protoc
-          export PROTO_ZIP="protoc-21.4-osx-x86_64.zip"
-          curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
-          unzip $PROTO_ZIP
-          echo "$HOME/d/protoc/bin" >> $GITHUB_PATH
-          export PATH=$PATH:$HOME/d/protoc/bin
-          protoc --version
       # TODO: this won't cache anything, which is expensive. Setup this action
       # with a OS-dependent path.
       - name: Setup Rust toolchain
@@ -312,16 +278,6 @@ jobs:
   #     - uses: actions/checkout@v3
   #       with:
   #         submodules: true
-  #     - name: Install protobuf compiler
-  #       shell: bash
-  #       run: |
-  #         mkdir -p $HOME/d/protoc
-  #         cd $HOME/d/protoc
-  #         export PROTO_ZIP="protoc-21.4-linux-x86_64.zip"
-  #         curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
-  #         unzip $PROTO_ZIP
-  #         export PATH=$PATH:$HOME/d/protoc/bin
-  #         protoc --version
   #     - name: Setup Rust toolchain
   #       run: |
   #         rustup toolchain install stable
@@ -335,7 +291,6 @@ jobs:
   #         key: cargo-coverage-cache3-
   #     - name: Run coverage
   #       run: |
-  #         export PATH=$PATH:$HOME/d/protoc/bin
   #         rustup toolchain install stable
   #         rustup default stable
   #         cargo install --version 0.20.1 cargo-tarpaulin

datafusion/proto/Cargo.toml

@@ -58,3 +58,6 @@ tokio = "1.18"
 [build-dependencies]
 pbjson-build = { version = "0.5", optional = true }
 prost-build = { version = "0.11.1" }
+reqwest = "0.11.12"
+tokio = "1.18"
+zip-extract = "0.1.1"

datafusion/proto/build.rs

@@ -18,24 +18,54 @@
 type Error = Box<dyn std::error::Error>;
 type Result<T, E = Error> = std::result::Result<T, E>;
 
-fn main() -> Result<(), String> {
+#[tokio::main]
+async fn main() -> Result<(), String> {
     // for use in docker build where file changes can be wonky
     println!("cargo:rerun-if-env-changed=FORCE_REBUILD");
     println!("cargo:rerun-if-changed=proto/datafusion.proto");
 
-    build()?;
+    build().await?;
 
     Ok(())
 }
 
-fn build() -> Result<(), String> {
+async fn build() -> Result<(), String> {
     use std::io::Write;
 
     let out = std::path::PathBuf::from(
         std::env::var("OUT_DIR").expect("Cannot find OUT_DIR environment variable"),
     );
     let descriptor_path = out.join("proto_descriptor.bin");
 
+    // compute protoc distribution URL
+    let host = std::env::var("HOST").expect("HOST not specified!");
+    let (proto_platform, suffix) = match host.as_str() {
+        "todo" => ("linux-aarch_64", ""), // TODO: arm
+        "x86_64-unknown-linux-gnu" => ("linux-x86_64", ""),
+        "x86_64-pc-windows-msvc" => ("win64", ".exe"),
+        "x86_64-apple-darwin" => ("osx-x86_64", ""),
+        _ => panic!("No protobuf found for OS type: {}", host),
+    };
+    let proto_base = "https://github.com/protocolbuffers/protobuf/releases/download";
+    let proto_ver = "21.8";
+    let proto_url =
+        format!("{proto_base}/v{proto_ver}/protoc-{proto_ver}-{proto_platform}.zip");
+
+    // Download protoc binary
+    if std::env::var("PROTOC").is_err() {
+        let target_dir = out.join("protoc");
+        if !target_dir.exists() {
+            let archive = reqwest::get(proto_url)
+                .await
+                .expect("Can't download protoc");
+            let archive = archive.bytes().await.expect("Can't download protoc");
+            let archive = std::io::Cursor::new(archive);
+            zip_extract::extract(archive, &target_dir, true)
+                .expect("Can't extract protoc");
+        }
+        std::env::set_var("PROTOC", out.join(format!("protoc/bin/protoc{suffix}")));
+    }
+
     prost_build::Config::new()
         .file_descriptor_set_path(&descriptor_path)
         .compile_well_known_types()