Add integration tests with dapptools repos (gakonst#160)

* ci: rename to unit-tests

* ci: add integration testing workflow

* feat: add readme and basic scripts

* test: add drai

* pin cache to git sha

* integration-tests: add geb

* integration-tests: add https://github.com/hexonaut/guni-lev/

* integration-tests: add https://github.com/Rari-Capital/solmate

* integration-tests: add https://github.com/Arachnid/solidity-stringutils

* integration-tests: add https://github.com/rari-capital/vaults

* integration-tests: add https://github.com/makerdao/multicall

* integration-tests: add https://github.com/gakonst/lootloose

* ci: enable all repos

* chore: load path to binary directly

* exit when any command fails in test.sh

* fix: ignore errors for certain repos with failing tests

.github/workflows/integration-tests.yml

@@ -0,0 +1,64 @@
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+name: e2e tests
+
+env:
+  ETH_RPC_URL: https://eth-mainnet.alchemyapi.io/v2/Lc7oIGYeL_QvInzI0Wiu_pOZZDEKBrdf
+  FORK_BLOCK: 13633752
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # TODO: Can we automatically generate this matrix via `ls` and `fromJSON`?
+        # https://docs.github.com/en/actions/learn-github-actions/expressions#fromjson
+        repo: ["lootloose", "solmate", "vaults", "geb", "solidity-stringutils", "multicall", "guni-lev", "drai"]
+
+    steps:
+      # clone
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      # some deps require node
+      - uses: actions/setup-node@v2
+      # install rust
+      - name: Install stable toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          components: rustfmt, clippy
+      # ..with caching
+      - uses: Swatinem/rust-cache@v1
+        with:
+          cache-on-failure: true
+
+      - name: Get git sha
+        id: vars
+        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+
+      # try to load `forge` from cache
+      - name: Cache forge
+        id: cache-forge
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-forge
+        with:
+          path: ./target/debug/forge
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ steps.vars.outputs.sha_short }}
+
+      # if couldn't get, install forge
+      - name: Install forge
+        if: steps.cache-forge.outputs.cache-hit != 'true'
+        run: cargo build --bin forge
+
+      - name: Test ${{ matrix.repo }}
+        run: ./test.sh ${{ matrix.repo }}
+        working-directory:
+          ./integration-tests

.gitmodules

@@ -0,0 +1,24 @@
+[submodule "integration-tests/testdata/drai"]
+	path = integration-tests/testdata/drai
+	url = [email protected]:mds1/drai.git
+[submodule "integration-tests/testdata/geb"]
+	path = integration-tests/testdata/geb
+	url = https://github.com/reflexer-labs/geb
+[submodule "integration-tests/testdata/guni-lev"]
+	path = integration-tests/testdata/guni-lev
+	url = https://github.com/hexonaut/guni-lev/
+[submodule "integration-tests/testdata/solmate"]
+	path = integration-tests/testdata/solmate
+	url = https://github.com/Rari-Capital/solmate
+[submodule "integration-tests/testdata/solidity-stringutils"]
+	path = integration-tests/testdata/solidity-stringutils
+	url = https://github.com/Arachnid/solidity-stringutils
+[submodule "integration-tests/testdata/vaults"]
+	path = integration-tests/testdata/vaults
+	url = https://github.com/rari-capital/vaults
+[submodule "integration-tests/testdata/multicall"]
+	path = integration-tests/testdata/multicall
+	url = https://github.com/makerdao/multicall
+[submodule "integration-tests/testdata/lootloose"]
+	path = integration-tests/testdata/lootloose
+	url = https://github.com/gakonst/lootloose

cli/src/forge.rs

@@ -42,6 +42,7 @@ fn main() -> eyre::Result<()> {
             sender,
             ffi,
             verbosity,
+            allow_failure,
         } => {
             // Setup the fuzzer
             // TODO: Add CLI Options to modify the persistence
@@ -107,7 +108,7 @@ fn main() -> eyre::Result<()> {
                         ffi,
                     );
 
-                    test(builder, project, evm, pattern, json, verbosity)?;
+                    test(builder, project, evm, pattern, json, verbosity, allow_failure)?;
                 }
                 #[cfg(feature = "evmodin-evm")]
                 EvmType::EvmOdin => {
@@ -121,7 +122,7 @@ fn main() -> eyre::Result<()> {
                     let host = env.evmodin_state();
 
                     let evm = EvmOdin::new(host, env.gas_limit, revision, NoopTracer);
-                    test(builder, project, evm, pattern, json, verbosity)?;
+                    test(builder, project, evm, pattern, json, verbosity, allow_failure)?;
                 }
             }
         }
@@ -241,6 +242,7 @@ fn test<A: ArtifactOutput + 'static, S: Clone, E: evm_adapters::Evm<S>>(
     pattern: Regex,
     json: bool,
     verbosity: u8,
+    allow_failure: bool,
 ) -> eyre::Result<HashMap<String, HashMap<String, forge::TestResult>>> {
     let mut runner = builder.build(project, evm)?;
 
@@ -315,6 +317,9 @@ fn test<A: ArtifactOutput + 'static, S: Clone, E: evm_adapters::Evm<S>>(
         }
     }
 
+    if allow_failure {
+        exit_code = 0;
+    }
     std::process::exit(exit_code);
 }
 

cli/src/forge_opts.rs

@@ -78,6 +78,13 @@ pub enum Subcommands {
 
         #[structopt(help = "verbosity of 'forge test' output (0-3)", long, default_value = "0")]
         verbosity: u8,
+
+        #[structopt(
+            help = "if set to true, the process will exit with an exit code = 0, even if the tests fail",
+            long,
+            env = "FORGE_ALLOW_FAILURE"
+        )]
+        allow_failure: bool,
     },
     #[structopt(about = "build your smart contracts")]
     #[structopt(alias = "b")]

integration-tests/README.md

@@ -0,0 +1,25 @@
+# Benchmarks & Integration Tests
+
+## Motivation
+
+This sub-project is used for integration testing
+[forge](https://github.com/gakonst/foundry/) with common dapptools repositories,
+to ensure that it's compatible with the test cases in them, e.g. usage of HEVM
+cheatcodes, proper forking mode integration, fuzzing etc.
+
+It is also used for getting quick performance benchmarks for Forge.
+
+## How to run?
+
+1. Make sure forge & dapptools are installed
+1. Run `./test.sh $REPO_NAME`, e.g. `./test.sh LootLoose`.
+
+## Repositories Included
+
+See the submodules linked within the [`testdata/`](./testdata) folder.
+
+## Adding a new repository
+
+We use git submodules (I know, I know submodules are not great, feel free to
+recommend a working alternative), you can add a new one via:
+`./add_test.sh $URL`

integration-tests/add_test.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+# Installs a new dapptools test repository to use
+
+TESTDATA=testdata
+REPO=$1
+
+cd $TESTDATA
+git submodule add $REPO
+
+git commit -m "integration-tests: add $REPO"

integration-tests/test.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+set +x
+set -e
+
+
+# TODO: Add logic for running with all
+REPO=$1
+TESTDATA=testdata
+
+ALLOWED_FAILURE_REPOS=("geb" "drai" "guni-lev")
+if [[ " ${ALLOWED_FAILURE_REPOS[*]} " =~ " ${REPO} " ]]; then
+    export FORGE_ALLOW_FAILURE=1
+fi
+
+DIR=`pwd`
+
+function runTests() {
+    cd $TESTDATA/$1
+
+    # run any installation step if needed
+    make install || true
+
+    # update the deps
+    $DIR/../target/debug/forge update
+    # always have the ffi flag turned on
+    $DIR/../target/debug/forge test --ffi
+
+    cd -
+}
+
+runTests $REPO