Expand forge test --match interface (gakonst#388)

* Expanded match interface into options for filtering by test name, contract name, and their inverse

* Added forge::TestFilter impl for Filter && replaced Regex with Filter

* Added TestFilter trait

* Replaced Regex with TestFilter

* Replaced Regex with TestFilter

* Cleaned up warnings

* Formatting

* Add conflicts for old match flag and new expaded match flags

* Added TestFilter impl to test_helpers

* Replaced Regex in tests with impl TestFilter

cli/src/cmd/test.rs

@@ -10,26 +10,88 @@ use crate::{
 };
 use ansi_term::Colour;
 use ethers::solc::{ArtifactOutput, Project};
-use forge::MultiContractRunnerBuilder;
-use regex::Regex;
+use forge::{MultiContractRunnerBuilder, TestFilter};
 use std::collections::BTreeMap;
-use structopt::StructOpt;
+use structopt::{clap::AppSettings, StructOpt};
 
 #[derive(Debug, Clone, StructOpt)]
+pub struct Filter {
+    #[structopt(
+        long = "--match",
+        short = "-m",
+        help = "only run test methods matching regex (deprecated, see --match-test, --match-contract)"
+    )]
+    pattern: Option<regex::Regex>,
+
+    #[structopt(
+        long = "--match-test",
+        help = "only run test methods matching regex",
+        conflicts_with = "pattern"
+    )]
+    test_pattern: Option<regex::Regex>,
+
+    #[structopt(
+        long = "--no-match-test",
+        help = "only run test methods not matching regex",
+        conflicts_with = "pattern"
+    )]
+    test_pattern_inverse: Option<regex::Regex>,
+
+    #[structopt(
+        long = "--match-contract",
+        help = "only run test methods in contracts matching regex",
+        conflicts_with = "pattern"
+    )]
+    contract_pattern: Option<regex::Regex>,
+
+    #[structopt(
+        long = "--no-match-contract",
+        help = "only run test methods in contracts not matching regex",
+        conflicts_with = "pattern"
+    )]
+    contract_pattern_inverse: Option<regex::Regex>,
+}
+
+impl TestFilter for Filter {
+    fn matches_test(&self, test_name: &str) -> bool {
+        let mut ok = true;
+        // Handle the deprecated option match
+        if let Some(re) = &self.pattern {
+            ok &= re.is_match(test_name);
+        }
+        if let Some(re) = &self.test_pattern {
+            ok &= re.is_match(test_name);
+        }
+        if let Some(re) = &self.test_pattern_inverse {
+            ok &= !re.is_match(test_name);
+        }
+        ok
+    }
+
+    fn matches_contract(&self, contract_name: &str) -> bool {
+        let mut ok = true;
+        if let Some(re) = &self.contract_pattern {
+            ok &= re.is_match(contract_name);
+        }
+        if let Some(re) = &self.contract_pattern_inverse {
+            ok &= !re.is_match(contract_name);
+        }
+        ok
+    }
+}
+
+#[derive(Debug, Clone, StructOpt)]
+// This is required to group Filter options in help output
+#[structopt(global_settings = &[AppSettings::DeriveDisplayOrder])]
 pub struct TestArgs {
     #[structopt(help = "print the test results in json format", long, short)]
     json: bool,
 
     #[structopt(flatten)]
     evm_opts: EvmOpts,
 
-    #[structopt(
-        long = "--match",
-        short = "-m",
-        help = "only run test methods matching regex",
-        default_value = ".*"
-    )]
-    pattern: regex::Regex,
+    #[structopt(flatten)]
+    filter: Filter,
 
     #[structopt(flatten)]
     opts: BuildArgs,
@@ -46,7 +108,7 @@ impl Cmd for TestArgs {
     type Output = TestOutcome;
 
     fn run(self) -> eyre::Result<Self::Output> {
-        let TestArgs { opts, evm_opts, json, pattern, allow_failure } = self;
+        let TestArgs { opts, evm_opts, json, filter, allow_failure } = self;
         // Setup the fuzzer
         // TODO: Add CLI Options to modify the persistence
         let cfg = proptest::test_runner::Config { failure_persistence: None, ..Default::default() };
@@ -68,7 +130,7 @@ impl Cmd for TestArgs {
                 let mut cfg = utils::sputnik_cfg(opts.compiler.evm_version);
                 let vicinity = evm_opts.vicinity()?;
                 let evm = utils::sputnik_helpers::evm(&evm_opts, &mut cfg, &vicinity)?;
-                test(builder, project, evm, pattern, json, evm_opts.verbosity, allow_failure)
+                test(builder, project, evm, filter, json, evm_opts.verbosity, allow_failure)
             }
             #[cfg(feature = "evmodin-evm")]
             EvmType::EvmOdin => {
@@ -82,7 +144,7 @@ impl Cmd for TestArgs {
                 let host = evm_opts.env.evmodin_state();
 
                 let evm = EvmOdin::new(host, evm_opts.env.gas_limit, revision, NoopTracer);
-                test(builder, project, evm, pattern, json, evm_opts.verbosity, allow_failure)
+                test(builder, project, evm, filter, json, evm_opts.verbosity, allow_failure)
             }
         }
     }
@@ -163,14 +225,14 @@ fn test<A: ArtifactOutput + 'static, S: Clone, E: evm_adapters::Evm<S>>(
     builder: MultiContractRunnerBuilder,
     project: Project<A>,
     evm: E,
-    pattern: Regex,
+    filter: Filter,
     json: bool,
     verbosity: u8,
     allow_failure: bool,
 ) -> eyre::Result<TestOutcome> {
     let mut runner = builder.build(project, evm)?;
 
-    let results = runner.test(pattern)?;
+    let results = runner.test(&filter)?;
 
     if json {
         let res = serde_json::to_string(&results)?;

forge/src/lib.rs

@@ -4,12 +4,19 @@ pub use runner::{ContractRunner, TestKind, TestKindGas, TestResult};
 mod multi_runner;
 pub use multi_runner::{MultiContractRunner, MultiContractRunnerBuilder};
 
+pub trait TestFilter {
+    fn matches_test(&self, test_name: &str) -> bool;
+    fn matches_contract(&self, contract_name: &str) -> bool;
+}
+
 #[cfg(test)]
 pub mod test_helpers {
+    use super::*;
     use ethers::{
         prelude::Lazy,
         solc::{CompilerOutput, Project, ProjectPathsConfig},
     };
+    use regex::Regex;
 
     pub static COMPILED: Lazy<CompilerOutput> = Lazy::new(|| {
         // NB: should we add a test-helper function that makes creating these
@@ -19,4 +26,28 @@ pub mod test_helpers {
         let project = Project::builder().paths(paths).ephemeral().no_artifacts().build().unwrap();
         project.compile().unwrap().output()
     });
+
+    pub struct Filter {
+        test_regex: Regex,
+        contract_regex: Regex,
+    }
+
+    impl Filter {
+        pub fn new(test_pattern: &str, contract_pattern: &str) -> Self {
+            return Filter {
+                test_regex: Regex::new(test_pattern).unwrap(),
+                contract_regex: Regex::new(contract_pattern).unwrap(),
+            }
+        }
+    }
+
+    impl TestFilter for Filter {
+        fn matches_test(&self, test_name: &str) -> bool {
+            self.test_regex.is_match(test_name)
+        }
+
+        fn matches_contract(&self, contract_name: &str) -> bool {
+            self.contract_regex.is_match(contract_name)
+        }
+    }
 }

forge/src/multi_runner.rs

@@ -1,4 +1,5 @@
-use crate::{runner::TestResult, ContractRunner};
+use crate::{runner::TestResult, ContractRunner, TestFilter};
+
 use ethers::solc::Artifact;
 
 use evm_adapters::Evm;
@@ -11,7 +12,6 @@ use ethers::{
 };
 
 use proptest::test_runner::TestRunner;
-use regex::Regex;
 
 use eyre::Result;
 use std::{collections::BTreeMap, marker::PhantomData};
@@ -140,16 +140,17 @@ where
 {
     pub fn test(
         &mut self,
-        pattern: Regex,
+        filter: &impl TestFilter,
     ) -> Result<BTreeMap<String, BTreeMap<String, TestResult>>> {
         // TODO: Convert to iterator, ideally parallel one?
         let contracts = std::mem::take(&mut self.contracts);
 
         let init_state: S = self.evm.state().clone();
         let results = contracts
             .iter()
+            .filter(|(name, _)| filter.matches_contract(name))
             .map(|(name, (abi, address, logs))| {
-                let result = self.run_tests(name, abi, *address, logs, &pattern, &init_state)?;
+                let result = self.run_tests(name, abi, *address, logs, filter, &init_state)?;
                 Ok((name.clone(), result))
             })
             .filter_map(|x: Result<_>| x.ok())
@@ -174,18 +175,19 @@ where
         contract: &Abi,
         address: Address,
         init_logs: &[String],
-        pattern: &Regex,
+        filter: &impl TestFilter,
         init_state: &S,
     ) -> Result<BTreeMap<String, TestResult>> {
         let mut runner =
             ContractRunner::new(&mut self.evm, contract, address, self.sender, init_logs);
-        runner.run_tests(pattern, self.fuzzer.as_mut(), init_state, Some(&self.known_contracts))
+        runner.run_tests(filter, self.fuzzer.as_mut(), init_state, Some(&self.known_contracts))
     }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::test_helpers::Filter;
     use ethers::solc::ProjectPathsConfig;
     use std::path::PathBuf;
 
@@ -211,7 +213,7 @@ mod tests {
 
     fn test_multi_runner<S: Clone, E: Evm<S>>(evm: E) {
         let mut runner = runner(evm);
-        let results = runner.test(Regex::new(".*").unwrap()).unwrap();
+        let results = runner.test(&Filter::new(".*", ".*")).unwrap();
 
         // 6 contracts being built
         assert_eq!(results.keys().len(), 5);
@@ -221,7 +223,7 @@ mod tests {
         }
 
         // can also filter
-        let only_gm = runner.test(Regex::new("testGm.*").unwrap()).unwrap();
+        let only_gm = runner.test(&Filter::new("testGm.*", ".*")).unwrap();
         assert_eq!(only_gm.len(), 1);
 
         assert_eq!(only_gm["GmTest"].len(), 1);
@@ -238,7 +240,7 @@ mod tests {
             let evm = vm();
 
             let mut runner = runner(evm);
-            let results = runner.test(Regex::new(".*").unwrap()).unwrap();
+            let results = runner.test(&Filter::new(".*", ".*")).unwrap();
 
             let reasons = results["DebugLogsTest"]
                 .iter()

forge/src/runner.rs

@@ -1,3 +1,5 @@
+use crate::TestFilter;
+
 use ethers::{
     abi::{Abi, Function, Token},
     types::{Address, Bytes},
@@ -9,7 +11,6 @@ use evm_adapters::{
     Evm, EvmError,
 };
 use eyre::{Context, Result};
-use regex::Regex;
 use std::{collections::BTreeMap, fmt, marker::PhantomData, time::Instant};
 
 use proptest::test_runner::{TestError, TestRunner};
@@ -171,7 +172,7 @@ impl<'a, S: Clone, E: Evm<S>> ContractRunner<'a, S, E> {
     /// Runs all tests for a contract whose names match the provided regular expression
     pub fn run_tests(
         &mut self,
-        regex: &Regex,
+        filter: &impl TestFilter,
         fuzzer: Option<&mut TestRunner>,
         init_state: &S,
         known_contracts: Option<&BTreeMap<String, (Abi, Vec<u8>)>>,
@@ -184,7 +185,7 @@ impl<'a, S: Clone, E: Evm<S>> ContractRunner<'a, S, E> {
             .functions()
             .into_iter()
             .filter(|func| func.name.starts_with("test"))
-            .filter(|func| regex.is_match(&func.name))
+            .filter(|func| filter.matches_test(&func.name))
             .collect::<Vec<_>>();
 
         // run all unit tests
@@ -396,13 +397,11 @@ impl<'a, S: Clone, E: Evm<S>> ContractRunner<'a, S, E> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test_helpers::COMPILED;
+    use crate::test_helpers::{Filter, COMPILED};
     use ethers::solc::artifacts::CompactContractRef;
     use evm_adapters::sputnik::helpers::vm;
 
     mod sputnik {
-        use std::str::FromStr;
-
         use foundry_utils::get_func;
         use proptest::test_runner::Config as FuzzConfig;
 
@@ -432,12 +431,7 @@ mod tests {
             cfg.failure_persistence = None;
             let mut fuzzer = TestRunner::new(cfg);
             let results = runner
-                .run_tests(
-                    &Regex::from_str("testGreeting").unwrap(),
-                    Some(&mut fuzzer),
-                    &init_state,
-                    None,
-                )
+                .run_tests(&Filter::new("testGreeting", ".*"), Some(&mut fuzzer), &init_state, None)
                 .unwrap();
             assert!(results["testGreeting()"].success);
             assert!(results["testGreeting(string)"].success);
@@ -461,12 +455,7 @@ mod tests {
             cfg.failure_persistence = None;
             let mut fuzzer = TestRunner::new(cfg);
             let results = runner
-                .run_tests(
-                    &Regex::from_str("testFuzz.*").unwrap(),
-                    Some(&mut fuzzer),
-                    &init_state,
-                    None,
-                )
+                .run_tests(&Filter::new("testFuzz.*", ".*"), Some(&mut fuzzer), &init_state, None)
                 .unwrap();
             for (_, res) in results {
                 assert!(!res.success);
@@ -565,7 +554,7 @@ mod tests {
         let mut runner =
             ContractRunner::new(&mut evm, compiled.abi.as_ref().unwrap(), addr, None, &[]);
 
-        let res = runner.run_tests(&".*".parse().unwrap(), None, &init_state, None).unwrap();
+        let res = runner.run_tests(&Filter::new(".*", ".*"), None, &init_state, None).unwrap();
         assert!(!res.is_empty());
         assert!(res.iter().all(|(_, result)| result.success));
     }