add: evaluator tool

evaluator/README.md

@@ -0,0 +1,101 @@
+# Patch Evaluator
+
+A tool for evaluating smart contract patches by running test suites and analyzing results.
+
+## Prerequisites
+
+- Python 3.8 or higher
+- pip (Python package installer)
+
+## Installation
+
+1. Clone the repository:
+```bash
+git clone <repository-url>
+cd evaluator
+```
+
+2. Create and activate a virtual environment (recommended):
+```bash
+python -m venv .venv
+source .venv/bin/activate
+```
+
+3. Install the package in development mode:
+```bash
+pip install -e ".[dev]"
+```
+
+## Configuration
+
+The tool uses several configuration settings that can be modified in `src/config.py`:
+
+- `BASE_DIR`: Base directory for hardhat project
+- `LOG_LEVEL`: Logging verbosity (default: "ERROR")
+- `DEFAULT_BACKUP_SUFFIX`: Suffix for backup files (default: ".bak")
+
+## Usage
+
+The evaluator can be run from the command line with the following arguments:
+
+```bash
+python src/main.py \
+    --format <solidity|bytecode> \
+    --patch <path-to-patch-file> \
+    --contract-file <path-to-contract> \
+    --main-contract <contract-name>
+```
+
+### Required Arguments
+
+- `--format`: The format of the patch file (choices: 'solidity' or 'bytecode')
+- `--patch`: Path to the patch file that will be evaluated
+- `--contract-file`: Path to the original smart contract file
+- `--main-contract`: Name of the main contract to be patched
+
+### Example
+
+```bash
+python src/main.py \
+    --format solidity \
+    --patch ./patches/fix.sol \
+    --contract-file ./contracts/vulnerable.sol \
+    --main-contract VulnerableContract
+```
+
+## Output
+
+The tool will output evaluation results including:
+- Contract and patch file information
+- Total number of tests run
+- Number of passed tests
+- Sanity check results
+- Details of any test failures
+
+Example output:
+```
+Evaluation Results:
+Contract File: ./contracts/vulnerable.sol
+Patch File: ./patches/fix.sol
+Total Tests: 10
+Passed Tests: 8
+Sanity Success: True
+Sanity Failures: 0
+
+Exploit Test Failures:
+- Test case #3: Invalid state after transfer
+- Test case #7: Reentrancy vulnerability still present
+```
+
+## Development
+
+To run tests:
+```bash
+pytest
+```
+
+To run tests with coverage:
+```bash
+pytest --cov=src
+```
+

evaluator/requirements.txt

@@ -0,0 +1,2 @@
+pytest>=7.0.0
+pytest-cov>=4.0.0 

evaluator/setup.py

@@ -0,0 +1,19 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="evaluator",
+    version="0.1",
+    package_dir={"": "src"},
+    packages=find_packages(where="src"),
+    python_requires=">=3.8",
+    install_requires=[],  # Core dependencies (none currently needed)
+    extras_require={
+        "dev": [
+            "pytest>=7.0.0",
+            "pytest-cov>=4.0.0",
+        ],
+    },
+    description="A tool for evaluating patches",
+    author="Monica Jin",
+    author_email="[email protected]",
+)

evaluator/src/config.py

@@ -0,0 +1,3 @@
+DEFAULT_BACKUP_SUFFIX = ".bak"
+BASE_DIR = "../smartbugs-curated/0.4.x"
+LOG_LEVEL = "ERROR"

evaluator/src/constants.py

@@ -0,0 +1,5 @@
+from enum import Enum
+
+class PatchFormat(Enum):
+    SOLIDITY_PATCH = "solidity"
+    BYTECODE_PATCH = "bytecode"

evaluator/src/core/file_manager.py

@@ -0,0 +1,50 @@
+import os
+import shutil
+from config import DEFAULT_BACKUP_SUFFIX
+from exceptions import PatchEvaluatorError
+
+class FileManager:
+    def __init__(self, base_directory: str):
+        self.base_directory = base_directory
+        self.backup_directory = os.path.join(base_directory, "backups")
+        os.makedirs(self.backup_directory, exist_ok=True)
+
+    def read_file(self, path: str, absolute: bool = False) -> str:
+        full_path = os.path.join(self.base_directory, path) if not absolute else path
+        try:
+            with open(full_path, 'r') as f:
+                return f.read()
+        except IOError as e:
+            raise PatchEvaluatorError(f"Failed to read file {path}: {str(e)}")
+
+    def write_file(self, path: str, content: str, absolute: bool = False):
+        full_path = os.path.join(self.base_directory, path) if not absolute else path
+        try:
+            os.makedirs(os.path.dirname(full_path), exist_ok=True)
+            with open(full_path, 'w') as f:
+                f.write(content)
+        except IOError as e:
+            raise PatchEvaluatorError(f"Failed to write file {path}: {str(e)}")
+
+    def backup(self, path: str):
+        source = os.path.join(self.base_directory, path)
+        print(f"Backing up file {source} to {self.backup_directory}")
+        backup = f"{os.path.join(self.backup_directory, path)}{DEFAULT_BACKUP_SUFFIX}"
+        os.makedirs(os.path.dirname(backup), exist_ok=True)
+        try:
+            shutil.copy2(source, backup)
+        except IOError as e:
+            raise PatchEvaluatorError(f"Failed to backup file {path}: {str(e)}")
+
+    def restore(self, path: str):
+        source = os.path.join(self.base_directory, path)
+        backup = f"{os.path.join(self.backup_directory, path)}{DEFAULT_BACKUP_SUFFIX}"
+        try:
+            if os.path.exists(backup):
+                shutil.move(backup, source)
+        except IOError as e:
+            raise PatchEvaluatorError(f"Failed to restore file {path}: {str(e)}")
+
+    def remove_backup(self):
+        if os.path.exists(self.backup_directory):
+            shutil.rmtree(self.backup_directory)

evaluator/src/core/patch_evaluator.py

@@ -0,0 +1,48 @@
+from models.patch import Patch
+from models.test_result import TestResult
+from core.file_manager import FileManager
+from core.strategy_factory import PatchStrategyFactory
+from testing.hardhat_runner import HardhatTestRunner
+import logging
+
+class PatchEvaluator:
+    def __init__(self, base_directory: str):
+        self.file_manager = FileManager(base_directory)
+        self.patch_factory = PatchStrategyFactory()
+        self.test_runner = HardhatTestRunner(base_directory)
+        self.logger = logging.getLogger(__name__)
+
+    def evaluate_patch(self, patch: Patch) -> TestResult:
+        strategy = self.patch_factory.create_strategy(patch)
+        self.logger.info(f"Evaluating patch: {patch.path} for contract: {patch.contract_file}")
+
+        try:
+            contract_path = strategy.contract_path(patch)
+            self.logger.debug(f"Backing up contract at: {contract_path}")
+            self.file_manager.backup(contract_path)
+
+            self.logger.info("Applying patch...")
+            strategy.apply(patch, self.file_manager)
+
+            self.logger.info("Running tests...")
+            test_result = self.test_runner.run_tests(patch, strategy)
+
+            self.logger.debug("Restoring original contract")
+            self.file_manager.restore(contract_path)
+
+            self.logger.info(f"Evaluation complete. Passed tests: {test_result.passed_tests}/{test_result.total_tests}")
+            return test_result
+
+        except Exception as e:
+            self.logger.error(f"Error during patch evaluation: {str(e)}")
+            self.file_manager.restore(strategy.contract_path(patch))
+            return TestResult(
+                failures_sanity=list(e),
+                failures_exploits=[0],
+                total_tests=0,
+                passed_tests=0,
+                sanity_success=0,
+                sanity_failures=0
+            )
+        finally:
+            self.file_manager.remove_backup()

evaluator/src/core/strategy_factory.py

@@ -0,0 +1,23 @@
+from typing import Dict
+from constants import PatchFormat
+from strategies.base import PatchStrategy
+from strategies.bytecode_strategy import BytecodePatchStrategy
+from strategies.solidity_strategy import SolidityPatchStrategy
+from exceptions import UnsupportedPatchFormatError
+from models.patch import Patch
+
+class PatchStrategyFactory:
+    def __init__(self):
+        self.strategies: Dict[PatchFormat, PatchStrategy] = {
+            PatchFormat.SOLIDITY_PATCH: SolidityPatchStrategy(),
+            PatchFormat.BYTECODE_PATCH: BytecodePatchStrategy()
+        }
+
+    def create_strategy(self, patch: Patch) -> PatchStrategy:
+        strategy = self.strategies.get(patch.format)
+        if not strategy:
+            raise UnsupportedPatchFormatError(f"No strategy available for format: {patch.format}")
+        return strategy
+
+    def register_strategy(self, format: PatchFormat, strategy: PatchStrategy):
+        self.strategies[format] = strategy

evaluator/src/exceptions.py

@@ -0,0 +1,15 @@
+class PatchEvaluatorError(Exception):
+    """Base exception for patch evaluator."""
+    pass
+
+class PatchValidationError(PatchEvaluatorError):
+    """Raised when patch validation fails."""
+    pass
+
+class PatchApplicationError(PatchEvaluatorError):
+    """Raised when patch application fails."""
+    pass
+
+class UnsupportedPatchFormatError(PatchEvaluatorError):
+    """Raised when patch format is not supported."""
+    pass

evaluator/src/main.py

@@ -0,0 +1,64 @@
+import argparse
+from core.patch_evaluator import PatchEvaluator
+from models.patch import Patch
+from constants import PatchFormat
+from config import BASE_DIR, LOG_LEVEL
+import logging
+
+def setup_logging():
+    logging.basicConfig(
+        level=LOG_LEVEL,
+        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+        datefmt='%Y-%m-%d %H:%M:%S'
+    )
+
+def load_patch(patch_path: str, contract_file: str, main_contract: str, format: str) -> Patch:
+    return Patch(patch_path, contract_file, main_contract, PatchFormat(format))
+
+def main():
+    setup_logging()
+    logger = logging.getLogger(__name__)
+
+    parser = argparse.ArgumentParser(description='Evaluate smart contract patches')
+    parser.add_argument('--format', required=True, help='Patch format', choices=['solidity', 'bytecode'])
+    parser.add_argument('--patch', required=True, help='Path to patch file')
+    parser.add_argument('--contract-file', required=True, help='Contract file to patch')
+    parser.add_argument('--main-contract', required=True, help='Main contract to patch')
+
+    args = parser.parse_args()
+
+    try:
+        logger.info(f"Loading patch from {args.patch}")
+        patch = load_patch(args.patch, args.contract_file, args.main_contract, args.format)
+
+        logger.info("Initializing patch evaluator")
+        evaluator = PatchEvaluator(BASE_DIR)
+
+        logger.info("Starting patch evaluation")
+        result = evaluator.evaluate_patch(patch)
+
+        # Print results
+        print("\nEvaluation Results:")
+        print(f"Contract File: {result.contract}")
+        print(f"Patch File: {result.patch_path}")
+        print(f"Total Tests: {result.total_tests}")
+        print(f"Passed Tests: {result.passed_tests}")
+        print(f"Sanity Success: {result.sanity_success}")
+        print(f"Sanity Failures: {result.sanity_failures}")
+
+        if result.failed_sanity_results:
+            print("\nSanity Test Failures:")
+            for failure in result.failed_sanity_results:
+                print(f"- {failure}")
+
+        if result.failed_results:
+            print("\nExploit Test Failures:")
+            for failure in result.failed_results:
+                print(f"- {failure}")
+
+    except Exception as e:
+        logger.error(f"Fatal error: {str(e)}")
+        exit(1)
+
+if __name__ == "__main__":
+    main()

evaluator/src/models/patch.py

@@ -0,0 +1,17 @@
+from dataclasses import dataclass
+from constants import PatchFormat
+
+@dataclass
+class Patch:
+    path: str
+    contract_file: str
+    main: str
+    format: PatchFormat
+    def get_contract_file(self) -> str:
+        return self.contract_file
+
+    def get_path(self) -> str:
+        return self.path
+
+    def get_main(self) -> str:
+        return self.main