diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 000000000..ae78a17b3 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,64 @@ +on: + push: + branches: + - master + pull_request: + +name: e2e tests + +env: + ETH_RPC_URL: https://eth-mainnet.alchemyapi.io/v2/Lc7oIGYeL_QvInzI0Wiu_pOZZDEKBrdf + FORK_BLOCK: 13633752 + +jobs: + tests: + runs-on: ubuntu-latest + strategy: + matrix: + # TODO: Can we automatically generate this matrix via `ls` and `fromJSON`? + # https://docs.github.com/en/actions/learn-github-actions/expressions#fromjson + repo: ["lootloose", "solmate", "vaults", "geb", "solidity-stringutils", "multicall", "guni-lev", "drai"] + + steps: + # clone + - uses: actions/checkout@v2 + with: + submodules: recursive + # some deps require node + - uses: actions/setup-node@v2 + # install rust + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + components: rustfmt, clippy + # ..with caching + - uses: Swatinem/rust-cache@v1 + with: + cache-on-failure: true + + - name: Get git sha + id: vars + run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + + # try to load `forge` from cache + - name: Cache forge + id: cache-forge + uses: actions/cache@v2 + env: + cache-name: cache-forge + with: + path: ./target/debug/forge + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ steps.vars.outputs.sha_short }} + + # if couldn't get, install forge + - name: Install forge + if: steps.cache-forge.outputs.cache-hit != 'true' + run: cargo build --bin forge + + - name: Test ${{ matrix.repo }} + run: ./test.sh ${{ matrix.repo }} + working-directory: + ./integration-tests diff --git a/.github/workflows/ci.yml b/.github/workflows/unit-tests.yml similarity index 100% rename from .github/workflows/ci.yml rename to .github/workflows/unit-tests.yml diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..8c8b31b5c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,24 @@ +[submodule "integration-tests/testdata/drai"] + path = integration-tests/testdata/drai + url = git@github.com:mds1/drai.git +[submodule "integration-tests/testdata/geb"] + path = integration-tests/testdata/geb + url = https://github.com/reflexer-labs/geb +[submodule "integration-tests/testdata/guni-lev"] + path = integration-tests/testdata/guni-lev + url = https://github.com/hexonaut/guni-lev/ +[submodule "integration-tests/testdata/solmate"] + path = integration-tests/testdata/solmate + url = https://github.com/Rari-Capital/solmate +[submodule "integration-tests/testdata/solidity-stringutils"] + path = integration-tests/testdata/solidity-stringutils + url = https://github.com/Arachnid/solidity-stringutils +[submodule "integration-tests/testdata/vaults"] + path = integration-tests/testdata/vaults + url = https://github.com/rari-capital/vaults +[submodule "integration-tests/testdata/multicall"] + path = integration-tests/testdata/multicall + url = https://github.com/makerdao/multicall +[submodule "integration-tests/testdata/lootloose"] + path = integration-tests/testdata/lootloose + url = https://github.com/gakonst/lootloose diff --git a/cli/src/forge.rs b/cli/src/forge.rs index 042ef2d3a..6f4500071 100644 --- a/cli/src/forge.rs +++ b/cli/src/forge.rs @@ -42,6 +42,7 @@ fn main() -> eyre::Result<()> { sender, ffi, verbosity, + allow_failure, } => { // Setup the fuzzer // TODO: Add CLI Options to modify the persistence @@ -107,7 +108,7 @@ fn main() -> eyre::Result<()> { ffi, ); - test(builder, project, evm, pattern, json, verbosity)?; + test(builder, project, evm, pattern, json, verbosity, allow_failure)?; } #[cfg(feature = "evmodin-evm")] EvmType::EvmOdin => { @@ -121,7 +122,7 @@ fn main() -> eyre::Result<()> { let host = env.evmodin_state(); let evm = EvmOdin::new(host, env.gas_limit, revision, NoopTracer); - test(builder, project, evm, pattern, json, verbosity)?; + test(builder, project, evm, pattern, json, verbosity, allow_failure)?; } } } @@ -241,6 +242,7 @@ fn test>( pattern: Regex, json: bool, verbosity: u8, + allow_failure: bool, ) -> eyre::Result>> { let mut runner = builder.build(project, evm)?; @@ -315,6 +317,9 @@ fn test>( } } + if allow_failure { + exit_code = 0; + } std::process::exit(exit_code); } diff --git a/cli/src/forge_opts.rs b/cli/src/forge_opts.rs index 88df61e6d..89794c715 100644 --- a/cli/src/forge_opts.rs +++ b/cli/src/forge_opts.rs @@ -78,6 +78,13 @@ pub enum Subcommands { #[structopt(help = "verbosity of 'forge test' output (0-3)", long, default_value = "0")] verbosity: u8, + + #[structopt( + help = "if set to true, the process will exit with an exit code = 0, even if the tests fail", + long, + env = "FORGE_ALLOW_FAILURE" + )] + allow_failure: bool, }, #[structopt(about = "build your smart contracts")] #[structopt(alias = "b")] diff --git a/integration-tests/README.md b/integration-tests/README.md new file mode 100644 index 000000000..ac386ae28 --- /dev/null +++ b/integration-tests/README.md @@ -0,0 +1,25 @@ +# Benchmarks & Integration Tests + +## Motivation + +This sub-project is used for integration testing +[forge](https://github.com/gakonst/foundry/) with common dapptools repositories, +to ensure that it's compatible with the test cases in them, e.g. usage of HEVM +cheatcodes, proper forking mode integration, fuzzing etc. + +It is also used for getting quick performance benchmarks for Forge. + +## How to run? + +1. Make sure forge & dapptools are installed +1. Run `./test.sh $REPO_NAME`, e.g. `./test.sh LootLoose`. + +## Repositories Included + +See the submodules linked within the [`testdata/`](./testdata) folder. + +## Adding a new repository + +We use git submodules (I know, I know submodules are not great, feel free to +recommend a working alternative), you can add a new one via: +`./add_test.sh $URL` diff --git a/integration-tests/add_test.sh b/integration-tests/add_test.sh new file mode 100755 index 000000000..527df1d24 --- /dev/null +++ b/integration-tests/add_test.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Installs a new dapptools test repository to use + +TESTDATA=testdata +REPO=$1 + +cd $TESTDATA +git submodule add $REPO + +git commit -m "integration-tests: add $REPO" diff --git a/integration-tests/test.sh b/integration-tests/test.sh new file mode 100755 index 000000000..4cc63e791 --- /dev/null +++ b/integration-tests/test.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set +x +set -e + + +# TODO: Add logic for running with all +REPO=$1 +TESTDATA=testdata + +ALLOWED_FAILURE_REPOS=("geb" "drai" "guni-lev") +if [[ " ${ALLOWED_FAILURE_REPOS[*]} " =~ " ${REPO} " ]]; then + export FORGE_ALLOW_FAILURE=1 +fi + +DIR=`pwd` + +function runTests() { + cd $TESTDATA/$1 + + # run any installation step if needed + make install || true + + # update the deps + $DIR/../target/debug/forge update + # always have the ffi flag turned on + $DIR/../target/debug/forge test --ffi + + cd - +} + +runTests $REPO diff --git a/integration-tests/testdata/drai b/integration-tests/testdata/drai new file mode 160000 index 000000000..f31ce4fb1 --- /dev/null +++ b/integration-tests/testdata/drai @@ -0,0 +1 @@ +Subproject commit f31ce4fb15bbb06c94eefea2a3a43384c75b95cf diff --git a/integration-tests/testdata/geb b/integration-tests/testdata/geb new file mode 160000 index 000000000..50aa78bd6 --- /dev/null +++ b/integration-tests/testdata/geb @@ -0,0 +1 @@ +Subproject commit 50aa78bd61ccd57fd2d5e9d7ab0f0ce75c231fef diff --git a/integration-tests/testdata/guni-lev b/integration-tests/testdata/guni-lev new file mode 160000 index 000000000..e333f4200 --- /dev/null +++ b/integration-tests/testdata/guni-lev @@ -0,0 +1 @@ +Subproject commit e333f4200538ede70ebe7357608401df94657a65 diff --git a/integration-tests/testdata/lootloose b/integration-tests/testdata/lootloose new file mode 160000 index 000000000..7b639efe9 --- /dev/null +++ b/integration-tests/testdata/lootloose @@ -0,0 +1 @@ +Subproject commit 7b639efe97836155a6a6fc626bf1018d4f8b2495 diff --git a/integration-tests/testdata/multicall b/integration-tests/testdata/multicall new file mode 160000 index 000000000..1e1b44362 --- /dev/null +++ b/integration-tests/testdata/multicall @@ -0,0 +1 @@ +Subproject commit 1e1b44362640820bef92d0ccf5eeee25d9b41474 diff --git a/integration-tests/testdata/solidity-stringutils b/integration-tests/testdata/solidity-stringutils new file mode 160000 index 000000000..01e955c1d --- /dev/null +++ b/integration-tests/testdata/solidity-stringutils @@ -0,0 +1 @@ +Subproject commit 01e955c1d60fe8ac6c7a40f208d3b64758fae40c diff --git a/integration-tests/testdata/solmate b/integration-tests/testdata/solmate new file mode 160000 index 000000000..938b2d692 --- /dev/null +++ b/integration-tests/testdata/solmate @@ -0,0 +1 @@ +Subproject commit 938b2d6925e24de7542bce4b08f3f03988a82245 diff --git a/integration-tests/testdata/vaults b/integration-tests/testdata/vaults new file mode 160000 index 000000000..3d92dbb11 --- /dev/null +++ b/integration-tests/testdata/vaults @@ -0,0 +1 @@ +Subproject commit 3d92dbb11ec119c3275dfa5ea6b9bda5dd1fb3aa