diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index dd952922..89da0e52 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -10,7 +10,7 @@ jobs: security_audit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Cache cargo bin uses: actions/cache@v1 with: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 41130bea..27309427 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,7 +22,7 @@ jobs: tenderdash: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: ./.github/actions/deps - name: Build source code shell: bash diff --git a/.github/workflows/rust-clippy.yml b/.github/workflows/rust-clippy.yml index fec392e3..1a50584c 100644 --- a/.github/workflows/rust-clippy.yml +++ b/.github/workflows/rust-clippy.yml @@ -25,8 +25,7 @@ jobs: actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status steps: - name: Checkout code - uses: actions/checkout@v2 - + uses: actions/checkout@v3 - name: Install Rust toolchain and deps uses: ./.github/actions/deps diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 79f8314a..4c03c7d3 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -23,7 +23,7 @@ jobs: fmt: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: ./.github/actions/deps - name: Check code formatting shell: bash @@ -32,7 +32,7 @@ jobs: docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: ./.github/actions/deps - name: Check documentation generation shell: bash diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fd7368df..206af614 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,8 +28,7 @@ jobs: RUST_TEST_TIME_INTEGRATION: "3000,6000" RUST_TEST_TIME_DOCTEST: "3000,6000" steps: - - uses: actions/checkout@v2 - + - uses: actions/checkout@v3 - uses: ./.github/actions/deps with: toolchain: nightly diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..e69de29b diff --git a/proto-compiler/Cargo.toml b/proto-compiler/Cargo.toml index a2945851..2af58944 100644 --- a/proto-compiler/Cargo.toml +++ b/proto-compiler/Cargo.toml @@ -11,7 +11,9 @@ publish = false [dependencies] walkdir = { version = "2.3" } prost-build = { version = "0.11" } -git2 = { version = "0.16" } tempfile = { version = "3.2.0" } subtle-encoding = { version = "0.5" } regex = { "version" = "1.7.1" } +reqwest = { "version" = "0.11.16", features = ["blocking"] } +zip = { version = "0.6.4", default-features = false, features = ["deflate"] } +fs_extra = { version = "1.3.0" } diff --git a/proto-compiler/src/functions.rs b/proto-compiler/src/functions.rs index 80b232d6..658b5c2e 100644 --- a/proto-compiler/src/functions.rs +++ b/proto-compiler/src/functions.rs @@ -5,163 +5,87 @@ use std::{ path::{Path, PathBuf}, }; -use git2::{ - build::{CheckoutBuilder, RepoBuilder}, - AutotagOption, Commit, FetchOptions, Oid, Reference, Repository, -}; -use subtle_encoding::hex; use walkdir::WalkDir; use crate::constants::DEFAULT_TENDERDASH_COMMITISH; -/// Clone or open+fetch a repository and check out a specific commitish -/// In case of an existing repository, the origin remote will be set to `url`. -pub fn fetch_commitish(dir: &Path, url: &str, commitish: &str) { - let mut dotgit = dir.to_path_buf(); - dotgit.push(".git"); - let repo = if dotgit.is_dir() { - fetch_existing(dir, url) - } else { - clone_new(dir, url) - }; - checkout_commitish(&repo, commitish) -} - -fn clone_new(dir: &Path, url: &str) -> Repository { - println!( - " [info] => Cloning {} into {} folder", - url, - dir.to_string_lossy() - ); - - let mut fo = FetchOptions::new(); - fo.download_tags(AutotagOption::All); - fo.update_fetchhead(true); +/// Check out a specific commitish of the tenderdash repository. +/// +/// As this tool is mainly used by build.rs script, we rely +/// on cargo to decide wherther or not to call it. It means +/// we will not be called too frequently, so the fetch will +/// not happen too often. +pub fn fetch_commitish(tenderdash_dir: &Path, cache_dir: &Path, url: &str, commitish: &str) { + let url = format!("{url}/archive/{commitish}.zip"); - let mut builder = RepoBuilder::new(); - builder.fetch_options(fo); - - builder.clone(url, dir).unwrap() -} - -fn fetch_existing(dir: &Path, url: &str) -> Repository { println!( - " [info] => Fetching from {} into existing {} folder", + " [info] => Downloading and extracting {} into {}", url, - dir.to_string_lossy() + tenderdash_dir.to_string_lossy() ); - let repo = Repository::open(dir).unwrap(); - - let mut fo = git2::FetchOptions::new(); - fo.download_tags(git2::AutotagOption::All); - fo.update_fetchhead(true); - let mut remote = repo - .find_remote("origin") - .unwrap_or_else(|_| repo.remote("origin", url).unwrap()); - if remote.url().is_none() || remote.url().unwrap() != url { - repo.remote_set_url("origin", url).unwrap(); - } - println!(" [info] => Fetching repo using remote `origin`"); - let specs: &[&str] = &[]; - remote.fetch(specs, Some(&mut fo), None).unwrap(); - - let stats = remote.stats(); - if stats.local_objects() > 0 { - println!( - " [info] => Received {}/{} objects in {} bytes (used {} local objects)", - stats.indexed_objects(), - stats.total_objects(), - stats.received_bytes(), - stats.local_objects() - ); - } else { - println!( - " [info] => Received {}/{} objects in {} bytes", - stats.indexed_objects(), - stats.total_objects(), - stats.received_bytes() - ); + // ensure cache dir exists + if !cache_dir.is_dir() { + std::fs::create_dir_all(cache_dir).expect("cannot create cache directory"); } - Repository::open(dir).unwrap() -} + let archive_file = cache_dir.join(format!("tenderdash-{}.zip", commitish)); + // Unzip Tenderdash sources to tmpdir and move to target/tenderdash + let tmpdir = tempfile::tempdir().expect("cannot create temporary dir to extract archive"); + download_and_unzip(&url, archive_file.as_path(), tmpdir.path()); -fn checkout_commitish(repo: &Repository, commitish: &str) { - let (reference, commit) = find_reference_or_commit(repo, commitish); + // Downloaded zip contains subdirectory like tenderdash-0.12.0-dev.1. We need to + // move its contents to target/tederdash, so that we get correct paths like + // target/tenderdash/version/version.go + let src_dir = find_subdir(tmpdir.path(), "tenderdash-"); - println!( - " [info] => Checking out repo in detached HEAD mode:\n \ - [info] => id: {},\n \ - [info] => author: {},\n \ - [info] => committer: {},\n \ - [info] => summary: {}", - commit.id(), - commit.author(), - commit.committer(), - commit.summary().unwrap_or(""), - ); + let options = fs_extra::dir::CopyOptions::new().content_only(true); - match reference { - None => repo.set_head_detached(commit.id()).unwrap(), - Some(reference) => { - println!(" [info] => name: {}", reference.shorthand().unwrap()); - repo.set_head(reference.name().unwrap()).unwrap(); - }, - } - - let mut checkout_options = CheckoutBuilder::new(); - checkout_options - .force() - .remove_untracked(true) - .remove_ignored(true) - .use_theirs(true); - repo.checkout_head(Some(&mut checkout_options)).unwrap(); + fs_extra::dir::create(tenderdash_dir, true).expect("cannot create destination directory"); + fs_extra::dir::move_dir(src_dir, tenderdash_dir, &options) + .expect("cannot move tenderdash directory"); } -fn find_reference_or_commit<'a>( - repo: &'a Repository, - commitish: &str, -) -> (Option>, Commit<'a>) { - let mut tried_origin = false; // we tried adding 'origin/' to the commitish - - let mut try_reference = repo.resolve_reference_from_short_name(commitish); - if try_reference.is_err() { - // Local branch might be missing, try the remote branch - try_reference = repo.resolve_reference_from_short_name(&format!("origin/{commitish}")); - tried_origin = true; - if try_reference.is_err() { - // Remote branch not found, last chance: try as a commit ID - // Note: Oid::from_str() currently does an incorrect conversion and cuts the - // second half of the ID. We are falling back on Oid::from_bytes() - // for now. - let commitish_vec = hex::decode(commitish).unwrap_or_else(|_| { - hex::decode_upper(commitish).expect( - "TENDERDASH_COMMITISH refers to non-existing or invalid git branch/tag/commit", - ) - }); - return ( - None, - repo.find_commit(Oid::from_bytes(commitish_vec.as_slice()).unwrap()) - .unwrap(), - ); +/// Download file from URL and unzip it to `dest_dir` +fn download_and_unzip(url: &str, archive_file: &Path, dest_dir: &Path) { + // We download only if the file does not exist + if !archive_file.is_file() { + let mut file = File::create(archive_file).expect("cannot create file"); + + let mut rb = reqwest::blocking::get(url).expect("cannot download archive"); + if !rb.status().is_success() { + panic!( + "cannot download tenderdash sources from {url}: {:?}", + rb.status() + ) } + rb.copy_to(&mut file).expect("cannot save downloaded data"); + file.flush().expect("flush of archive file failed"); } - let mut reference = try_reference.unwrap(); - if reference.is_branch() { - if tried_origin { - panic!("[error] => local branch names with 'origin/' prefix not supported"); - } - try_reference = repo.resolve_reference_from_short_name(&format!("origin/{commitish}")); - reference = try_reference.unwrap(); - if reference.is_branch() { - panic!("[error] => local branch names with 'origin/' prefix not supported"); - } - } + let file = File::open(archive_file).expect("cannot open downloaded zip"); + let mut archive = zip::ZipArchive::new(&file).expect("cannot open zip archive"); - let commit = reference.peel_to_commit().unwrap(); - (Some(reference), commit) + archive.extract(dest_dir).expect("cannot extract archive"); +} +/// Find a subdirectory of a parent path which has provided name prefix +fn find_subdir(parent: &Path, name_prefix: &str) -> PathBuf { + let dir_content = fs_extra::dir::get_dir_content(parent).expect("cannot ls tmp dir"); + let mut src_dir = String::new(); + for directory in dir_content.directories { + let directory = Path::new(&directory) + .file_name() + .expect("cannot extract dir name"); + println!("{:?}", directory); + if directory.to_string_lossy().starts_with(name_prefix) { + src_dir = directory.to_string_lossy().into(); + break; + }; + } + if src_dir.is_empty() { + panic!("cannot find extracted Tenderdash sources") + } + parent.join(src_dir) } /// Copy generated files to target folder diff --git a/proto-compiler/src/lib.rs b/proto-compiler/src/lib.rs index cedd6d27..9a8048fc 100644 --- a/proto-compiler/src/lib.rs +++ b/proto-compiler/src/lib.rs @@ -1,7 +1,4 @@ -use std::{ - env::{self, var}, - path::PathBuf, -}; +use std::{env::var, path::PathBuf}; use tempfile::tempdir; @@ -21,18 +18,21 @@ use constants::{CUSTOM_FIELD_ATTRIBUTES, CUSTOM_TYPE_ATTRIBUTES, TENDERDASH_REPO /// ../proto/src/tenderdash.rs pub fn proto_compile() { let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let tenderdash_lib_target = root .join("..") .join("proto") .join("src") .join("tenderdash.rs"); + let target_dir = root.join("..").join("proto").join("src").join("prost"); + let out_dir = var("OUT_DIR") .map(PathBuf::from) .or_else(|_| tempdir().map(|d| d.into_path())) .unwrap(); - let cargo_target_dir = match env::var("CARGO_TARGET_DIR") { + let cargo_target_dir = match std::env::var("CARGO_TARGET_DIR") { Ok(s) => PathBuf::from(s), Err(_) => root.join("..").join("target"), }; @@ -48,7 +48,12 @@ pub fn proto_compile() { let commitish = tenderdash_commitish(); println!("[info] => Fetching {TENDERDASH_REPO} at {commitish} into {tenderdash_dir:?}"); - fetch_commitish(&PathBuf::from(&tenderdash_dir), TENDERDASH_REPO, &commitish); // This panics if it fails. + fetch_commitish( + &PathBuf::from(&tenderdash_dir), + &cargo_target_dir, + TENDERDASH_REPO, + &commitish, + ); // This panics if it fails. let proto_paths = vec![tenderdash_dir.join("proto").join("tendermint").join("abci")]; let proto_includes_paths = vec![tenderdash_dir.join("proto"), thirdparty_dir]; diff --git a/proto/tests/unit.rs b/proto/tests/unit.rs index f820160b..8ebd5929 100644 --- a/proto/tests/unit.rs +++ b/proto/tests/unit.rs @@ -133,3 +133,9 @@ pub fn test_response_exception_from() { "string" ); } + +#[test] +pub fn test_tenderdash_version() { + let version = env!("CARGO_PKG_VERSION"); + assert_eq!(version, tenderdash_proto::meta::TENDERDASH_VERSION) +}