Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(proto-compiler): download protobuf definitions via https instead of git #18

Merged
merged 14 commits into from
Mar 31, 2023
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/actions/deps/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ inputs:
runs:
using: composite
steps:
- uses: actions/checkout@v3
with:
submodules: true
- uses: dtolnay/rust-toolchain@master
name: Install Rust toolchain
with:
Expand Down
4 changes: 4 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
[submodule "tenderdash"]
path = tenderdash
url = https://github.com/dashpay/tenderdash
branch = v0.12.0-dev.1
1 change: 0 additions & 1 deletion proto-compiler/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ publish = false
[dependencies]
walkdir = { version = "2.3" }
prost-build = { version = "0.11" }
git2 = { version = "0.16" }
tempfile = { version = "3.2.0" }
subtle-encoding = { version = "0.5" }
regex = { "version" = "1.7.1" }
186 changes: 39 additions & 147 deletions proto-compiler/src/functions.rs
Original file line number Diff line number Diff line change
@@ -1,167 +1,59 @@
use std::{
env,
fs::{copy, create_dir_all, read_to_string, remove_dir_all, File},
io::Write,
io::{self, Write},
path::{Path, PathBuf},
process::ExitStatus,
Fixed Show fixed Hide fixed
Fixed Show fixed Hide fixed
};

use git2::{
build::{CheckoutBuilder, RepoBuilder},
AutotagOption, Commit, FetchOptions, Oid, Reference, Repository,
};
use subtle_encoding::hex;
use walkdir::WalkDir;

use crate::constants::DEFAULT_TENDERDASH_COMMITISH;

/// Clone or open+fetch a repository and check out a specific commitish
/// In case of an existing repository, the origin remote will be set to `url`.
pub fn fetch_commitish(dir: &Path, url: &str, commitish: &str) {
let mut dotgit = dir.to_path_buf();
dotgit.push(".git");
let repo = if dotgit.is_dir() {
fetch_existing(dir, url)
} else {
clone_new(dir, url)
};
checkout_commitish(&repo, commitish)
}

fn clone_new(dir: &Path, url: &str) -> Repository {
/// Check out a specific commitish of the tenderdash repository.
///
/// As this tool is mainly used by build.rs script, we rely
/// on cargo to decide wherther or not to call it. It means
/// we will not be called too frequently, so the fetch will
/// not happen too often.
pub fn fetch_commitish(workspace_dir: &Path, url: &str, commitish: &str) {
println!(
" [info] => Cloning {} into {} folder",
url,
dir.to_string_lossy()
);

let mut fo = FetchOptions::new();
fo.download_tags(AutotagOption::All);
fo.update_fetchhead(true);

let mut builder = RepoBuilder::new();
builder.fetch_options(fo);

builder.clone(url, dir).unwrap()
}

fn fetch_existing(dir: &Path, url: &str) -> Repository {
println!(
" [info] => Fetching from {} into existing {} folder",
url,
dir.to_string_lossy()
);
let repo = Repository::open(dir).unwrap();

let mut fo = git2::FetchOptions::new();
fo.download_tags(git2::AutotagOption::All);
fo.update_fetchhead(true);

let mut remote = repo
.find_remote("origin")
.unwrap_or_else(|_| repo.remote("origin", url).unwrap());
if remote.url().is_none() || remote.url().unwrap() != url {
repo.remote_set_url("origin", url).unwrap();
}
println!(" [info] => Fetching repo using remote `origin`");
let specs: &[&str] = &[];
remote.fetch(specs, Some(&mut fo), None).unwrap();

let stats = remote.stats();
if stats.local_objects() > 0 {
println!(
" [info] => Received {}/{} objects in {} bytes (used {} local objects)",
stats.indexed_objects(),
stats.total_objects(),
stats.received_bytes(),
stats.local_objects()
);
} else {
println!(
" [info] => Received {}/{} objects in {} bytes",
stats.indexed_objects(),
stats.total_objects(),
stats.received_bytes()
);
}

Repository::open(dir).unwrap()
}

fn checkout_commitish(repo: &Repository, commitish: &str) {
let (reference, commit) = find_reference_or_commit(repo, commitish);

println!(
" [info] => Checking out repo in detached HEAD mode:\n \
[info] => id: {},\n \
[info] => author: {},\n \
[info] => committer: {},\n \
[info] => summary: {}",
commit.id(),
commit.author(),
commit.committer(),
commit.summary().unwrap_or(""),
workspace_dir.join("tenderdash").to_string_lossy()
);

match reference {
None => repo.set_head_detached(commit.id()).unwrap(),
Some(reference) => {
println!(" [info] => name: {}", reference.shorthand().unwrap());
repo.set_head(reference.name().unwrap()).unwrap();
},
std::env::set_current_dir(workspace_dir).expect("cannot change directory to root dir");
let dir = "tenderdash";
// We use `git` executable as we need --depth option, not supported by git2
// crate
let output = std::process::Command::new("git")
.arg("submodule")
.arg("set-branch")
.arg("--branch")
.arg(commitish)
.arg(dir)
.output()
.expect("cannot select git branch/tag");

if !output.status.success() {
io::stdout().write_all(&output.stdout).unwrap();
io::stderr().write_all(&output.stderr).unwrap();
panic!("git submodule set-branch failed: {}", output.status);
}

let mut checkout_options = CheckoutBuilder::new();
checkout_options
.force()
.remove_untracked(true)
.remove_ignored(true)
.use_theirs(true);
repo.checkout_head(Some(&mut checkout_options)).unwrap();
}

fn find_reference_or_commit<'a>(
repo: &'a Repository,
commitish: &str,
) -> (Option<Reference<'a>>, Commit<'a>) {
let mut tried_origin = false; // we tried adding 'origin/' to the commitish

let mut try_reference = repo.resolve_reference_from_short_name(commitish);
if try_reference.is_err() {
// Local branch might be missing, try the remote branch
try_reference = repo.resolve_reference_from_short_name(&format!("origin/{commitish}"));
tried_origin = true;
if try_reference.is_err() {
// Remote branch not found, last chance: try as a commit ID
// Note: Oid::from_str() currently does an incorrect conversion and cuts the
// second half of the ID. We are falling back on Oid::from_bytes()
// for now.
let commitish_vec = hex::decode(commitish).unwrap_or_else(|_| {
hex::decode_upper(commitish).expect(
"TENDERDASH_COMMITISH refers to non-existing or invalid git branch/tag/commit",
)
});
return (
None,
repo.find_commit(Oid::from_bytes(commitish_vec.as_slice()).unwrap())
.unwrap(),
);
}
}

let mut reference = try_reference.unwrap();
if reference.is_branch() {
if tried_origin {
panic!("[error] => local branch names with 'origin/' prefix not supported");
}
try_reference = repo.resolve_reference_from_short_name(&format!("origin/{commitish}"));
reference = try_reference.unwrap();
if reference.is_branch() {
panic!("[error] => local branch names with 'origin/' prefix not supported");
}
let output = std::process::Command::new("git")
.arg("submodule")
.arg("update")
.arg("--depth=1")
.arg("--force")
.arg(dir)
.output()
.expect("cannot update git branch");
if !output.status.success() {
io::stdout().write_all(&output.stdout).unwrap();
io::stderr().write_all(&output.stderr).unwrap();
panic!("git submodule update failed: {}", output.status);
}

let commit = reference.peel_to_commit().unwrap();
(Some(reference), commit)
}

/// Copy generated files to target folder
Expand Down
24 changes: 8 additions & 16 deletions proto-compiler/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
use std::{
env::{self, var},
path::PathBuf,
};
use std::{env::var, path::PathBuf};

use tempfile::tempdir;

Expand All @@ -21,6 +18,8 @@ use constants::{CUSTOM_FIELD_ATTRIBUTES, CUSTOM_TYPE_ATTRIBUTES, TENDERDASH_REPO
/// ../proto/src/tenderdash.rs
pub fn proto_compile() {
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let workspace = root.join("..");

let tenderdash_lib_target = root
.join("..")
.join("proto")
Expand All @@ -32,23 +31,16 @@ pub fn proto_compile() {
.or_else(|_| tempdir().map(|d| d.into_path()))
.unwrap();

let cargo_target_dir = match env::var("CARGO_TARGET_DIR") {
Ok(s) => PathBuf::from(s),
Err(_) => root.join("..").join("target"),
};
let tenderdash_dir = PathBuf::from(var("TENDERDASH_DIR").unwrap_or_else(|_| {
cargo_target_dir
.join("tenderdash")
.to_str()
.unwrap()
.to_string()
}));
let tenderdash_dir = PathBuf::from(
var("TENDERDASH_DIR")
.unwrap_or_else(|_| workspace.join("tenderdash").to_str().unwrap().to_string()),
);

let thirdparty_dir = root.join("third_party");

let commitish = tenderdash_commitish();
println!("[info] => Fetching {TENDERDASH_REPO} at {commitish} into {tenderdash_dir:?}");
fetch_commitish(&PathBuf::from(&tenderdash_dir), TENDERDASH_REPO, &commitish); // This panics if it fails.
fetch_commitish(&PathBuf::from(&workspace), TENDERDASH_REPO, &commitish); // This panics if it fails.

let proto_paths = vec![tenderdash_dir.join("proto").join("tendermint").join("abci")];
let proto_includes_paths = vec![tenderdash_dir.join("proto"), thirdparty_dir];
Expand Down
6 changes: 6 additions & 0 deletions proto/tests/unit.rs
Original file line number Diff line number Diff line change
Expand Up @@ -133,3 +133,9 @@ pub fn test_response_exception_from() {
"string"
);
}

#[test]
pub fn test_tenderdash_version() {
let version = env!("CARGO_PKG_VERSION");
assert_eq!(version, tenderdash_proto::meta::TENDERDASH_VERSION)
}
1 change: 1 addition & 0 deletions tenderdash
Submodule tenderdash added at 1c355d