Merge branch 'main' into #268-fix-readme-missing-kwarg

.github/workflows/codecov.yml

@@ -1,18 +1,35 @@
 name: codecov
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+
 jobs:
   run:
     runs-on: ubuntu-latest
     steps:
-      - name: Fetch
-        uses: actions/checkout@master
-      - name: Test
-        run: |
-          pip install -r requirements_test.txt
-          pytest tests -v --cov-report term --cov-report html:htmlcov --cov-report xml --cov=quantus
+      - uses: actions/checkout@v3
+
+      - name: Setup python 3.10
+        uses: actions/setup-python@v4
+        with:
+          cache: 'pip'
+          python-version: "3.10"
+
+      - name: Install tox-gh
+        run: pip install tox-gh
+
+      - name: Setup test environment
+        run: tox run -e coverage --notest
+
+      - name: Measure coverage.
+        run: tox run -e coverage
+
       - name: Upload
-        uses:  codecov/codecov-action@v3
+        uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./coverage.xml, coverage.xml
-
+          files: ./coverage.xml, coverage.xml

.github/workflows/lint.yml

@@ -0,0 +1,25 @@
+name: Lint
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    name: Lint
+    steps:
+      - name: Check out source repository
+        uses: actions/checkout@v3
+      - name: Set up Python environment
+        uses: actions/setup-python@v4
+        with:
+          cache: 'pip'
+          python-version: "3.10"
+      - name: Install tox-gh
+        run: pip install tox-gh
+      - name: Run flake8
+        run: tox run -e lint
+      - name: Run mypy
+        run: tox run -e type

.github/workflows/python-package.yml

@@ -1,42 +1,37 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-
 name: Python package
 
 on:
-  #push:
-    #branches: [ main ]
+  # Run tests every night, to catch potentially breaking changes in dependencies.
+  schedule:
+    - cron: '0 0 * * *'
+  push:
+    branches:
+      - main
   pull_request:
-    branches: [ main ]
+  workflow_dispatch:
+
+
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
+    name: Python Package
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
-
+        python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install flake8 pytest mypy==0.982
-        if [ -f requirements_test.txt ]; then pip install -r requirements_test.txt; fi
-    - name: Lint
-      run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-        # run mypy
-        mypy quantus
-    - name: Test with pytest
-      run: |
-        pytest -s -v
+      - uses: actions/checkout@v3
+      - name: Setup python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          cache: 'pip'
+          python-version: ${{ matrix.python-version }}
+      - name: Install tox-gh
+        run: pip install tox-gh
+      - name: Setup test environment
+        run: tox run --notest
+      - name: Test with PyTest
+        run: tox run
+      - name: Test building package
+        run: tox run -e build

.gitignore

@@ -38,8 +38,8 @@ build/
 *sample_data/
 tutorials/invited_talks/
 
-# PyCharm project files
+# pyCharm project files
 .idea
 
-# Tox files.
+# tox files.
 .tox

CONTRIBUTING.md

@@ -48,8 +48,13 @@ Make sure to install the latest version of Quantus from the main branch.
 ```bash
 git clone https://github.com/understandable-machine-intelligence-lab/Quantus.git
 cd quantus
-pip install -r requirements_test.txt
-pip install -e .
+```
+
+Tox will provision dev environment with editable installation for you.
+```bash
+python3 -m pip install tox
+python3 -m tox devenv
+source venv/bin/activate
 ```
 
 ### Branching
@@ -63,52 +68,67 @@ We use [flake8](https://pypi.org/project/flake8/) for quick style checks and [bl
 
 ### Unit Tests
 
-Tests are written using [pytest](https://github.com/pytest-dev/pytest) and executed together with [codecov](https://github.com/codecov/codecov-action) for coverage reports.
-To perform the tests, execute the following (make sure pytest is installed):
-```bash
-pytest
+Tests are written using [pytest](https://github.com/pytest-dev/pytest) and executed together
+with [codecov](https://github.com/codecov/codecov-action) for coverage reports.
+We use [tox](https://tox.wiki/en/latest/) for test automation. For complete list of CLI commands, please refer
+to [tox - CLI interface](https://tox.wiki/en/latest/cli_interface.html).
+To perform the tests for all supported python versions execute the following CLI command (a re-intall of tox is necessary):
+
+```shell
+python3 -m pip install tox
+python3 -m tox run
 ```
 
+... alternatively, to get additionally coverage details, run:
 
-... optionally, you could split test execution between multiple CPU cores using [pytest-xdist](https://github.com/pytest-dev/pytest-xdist)
 ```bash
-pytest tests -n auto
+python3 -m tox run -e coverage
 ```
 
-... alternatively, to get additionaly coverage details, run:
-```bash
-pytest --cov=. --cov-report term-missing
-```
+It is possible to limit the scope of testing to specific sections of the codebase, for example, only test the
+Faithfulness metrics using python3.9 (make sure the python versions match in your environment):
 
-It is possible to limit the scope of testing to specific sections of the codebase, for example, only test the Faithfulness metrics:
 ```bash
-pytest -m faithfulness -s
+python3 -m tox run -e py39 -- -m faithfulness -s
 ```
+
 For a complete overview of the possible testing scopes, please refer to `pytest.ini`.
 
 ### Documentation
 
 Make sure to add docstrings to every class, method and function that you add to the codebase. The docstring should include a description of all parameters and returns. Use the existing documentation as an example.
-TODO: Automatic docstring generation.
 
 ### Before You Create a Pull Request
 
 Before creating a PR, double-check that the following tasks are completed:
 
 - Make sure that the latest version of the code from the `main` branch is merged into your working branch.
 - Run `black` to format source code:
+
 ```bash
-black quantus/INSERT_YOUR_FILE_NAME.py
+black quantus/*/INSERT_YOUR_FILE_NAME.py
 ```
+
 - Run `flake8` for quick style checks, e.g.:
+
 ```bash
-flake8 quantus/INSERT_YOUR_FILE_NAME.py
+flake8 quantus/*/INSERT_YOUR_FILE_NAME.py
+```
+
+- Create a unit test for new functionality and add under `tests/` folder, add `@pytest.mark` with fitting category.
+- If newly added test cases include a new category of `@pytest.mark` then add that category with description
+  to `pytest.ini`
+- Make sure all unit tests pass for all supported python version by running:
+
+```shell
+python3 -m tox run
 ```
-- Create `pytests` for new functionality (if needed) and add under `tests/` folder
-- If the `pytests` include a new category of `@pytest.mark` then add that category with description to `pytest.ini`
-- Make sure all unit tests are passed and the coverage level is maintained (we aim at ~100% code coverage for Quantus):
+
+- Generally, every change should be covered with respective test-case, we aim at ~100% code coverage in Quantus, you can
+  verify it by running:
+
 ```bash
-pytest tests -v --cov-report term --cov-report html:htmlcov --cov-report xml --cov=quantus
+python3 -m tox run -e coverage
 ```
 
 ### Pull Requests
@@ -137,6 +157,7 @@ See a more detailed description of those in [README](https://github.com/understa
 Identify which category your metric belongs to and create a Python file for your metric class in the respective folder in `quantus/metrics`.
 
 Add the metric to the `__init__.py` file in the respective folder.
+
 ### Metric Class
 Every metric class inherits from the base `Metric` class: `quantus/metrics/base.py`. Importantly, Faithfulness and Robustness inherit not from the `Metric` class directly, but rather from its child `PerturbationMetric`.
 

README.md

@@ -184,15 +184,6 @@ For TensorFlow, please run:
 pip install "quantus[tensorflow]"
 ```
 
-Alternatively, you can simply install Quantus with [requirements.txt](https://github.com/understandable-machine-intelligence-lab/Quantus/blob/main/requirements.txt).
-Note that this installation requires that either [PyTorch](https://pytorch.org/) or [TensorFlow](https://www.TensorFlow.org) are already installed on your machine.
-
-```setup
-pip install -r requirements.txt
-```
-
-For a more in-depth guide on how to install Quantus, please read more [here](https://quantus.readthedocs.io/en/latest/getting_started/installation.html). This includes instructions for how to install a desired deep learning framework such as PyTorch or TensorFlow together with Quantus.
-
 ### Package requirements
 
 The package requirements are as follows:
@@ -201,7 +192,10 @@ python>=3.7.0
 torch>=1.11.0
 tensorflow>=2.5.0
 ```
-Please note that the exact [PyTorch](https://pytorch.org/) and/ or [TensorFlow](https://www.TensorFlow.org) versions to be installed depends on your Python version (3.7-3.11) and platform (`darwin`, `linux`, …). See `requirements_test.txt` to retrieve the exact versions of [PyTorch](https://pytorch.org/) and/ or [TensorFlow](https://www.TensorFlow.org).
+
+Please note that the exact [PyTorch](https://pytorch.org/) and/ or [TensorFlow](https://www.TensorFlow.org) versions
+to be installed depends on your Python version (3.7-3.11) and platform (`darwin`, `linux`, …).
+See `[project.optional-dependencies]` section in the `pyproject.toml` file.
 
 ## Getting started
 

docs/source/docs_dev/CONTRIBUTING.md

@@ -48,8 +48,10 @@ Make sure to install the latest version of Quantus from the main branch.
 ```bash
 git clone https://github.com/understandable-machine-intelligence-lab/Quantus.git
 cd quantus
-pip install -r requirements_test.txt
-pip install -e .
+# Tox will provision dev environment with editable installation for you.
+python3 -m pip install tox
+python3 -m tox devenv
+source venv/bin/activate
 ```
 
 ### Branching
@@ -63,20 +65,29 @@ We use [flake8](https://pypi.org/project/flake8/) for quick style checks and [bl
 
 ### Unit Tests
 
-Tests are written using [pytest](https://github.com/pytest-dev/pytest) and executed together with [codecov](https://github.com/codecov/codecov-action) for coverage reports.
-To perform the tests, execute the following (make sure pytest is installed):
-```bash
-pytest
+Tests are written using [pytest](https://github.com/pytest-dev/pytest) and executed together
+with [codecov](https://github.com/codecov/codecov-action) for coverage reports.
+We use [tox](https://tox.wiki/en/latest/) for test automation. For complete list of CLI commands, please refer
+to [tox - CLI interface](https://tox.wiki/en/latest/cli_interface.html).
+To perform the tests for all supported python versions execute the following CLI command:
+
+```shell
+python3 -m tox run
 ```
-... alternatively, to get additionaly coverage details, run:
+
+... alternatively, to get additionally coverage details, run:
+
 ```bash
-pytest --cov=. --cov-report term-missing
+python3 -m tox run -e coverage
 ```
 
-It is possible to limit the scope of testing to specific sections of the codebase, for example, only test the Faithfulness metrics:
+It is possible to limit the scope of testing to specific sections of the codebase, for example, only test the
+Faithfulness metrics using python3.10:
+
 ```bash
-pytest -m faithfulness -s
+python3 -m tox run -e py310 -- -m faithfulness -s
 ```
+
 For a complete overview of the possible testing scopes, please refer to `pytest.ini`.
 
 ### Documentation
@@ -90,18 +101,31 @@ Before creating a PR, double-check that the following tasks are completed:
 
 - Make sure that the latest version of the code from the `main` branch is merged into your working branch.
 - Run `black` to format source code:
+
 ```bash
 black quantus/INSERT_YOUR_FILE_NAME.py
 ```
+
 - Run `flake8` for quick style checks, e.g.:
+
 ```bash
 flake8 quantus/INSERT_YOUR_FILE_NAME.py
 ```
-- Create `pytests` for new functionality (if needed) and add under `tests/` folder
-- If the `pytests` include a new category of `@pytest.mark` then add that category with description to `pytest.ini`
-- Make sure all unit tests are passed and the coverage level is maintained (we aim at ~100% code coverage for Quantus):
+
+- Create a unit test for new functionality and add under `tests/` folder, add `@pytest.mark` with fitting category.
+- If newly added test cases include a new category of `@pytest.mark` then add that category with description
+  to `pytest.ini`
+- Make sure all unit tests pass for all supported python version by running:
+
+```shell
+python3 -m tox run
+```
+
+- Generally, every change should be covered with respective test-case, we aim at ~100% code coverage in Quantus, you can
+  verify it by running:
+
 ```bash
-pytest tests -v --cov-report term --cov-report html:htmlcov --cov-report xml --cov=quantus
+python3 -m tox run -e coverage
 ```
 
 ### Pull Requests
@@ -174,4 +198,4 @@ Please note that by contributing to the project you agree that it will be licens
 ## Questions
 
 If you have any developer-related questions, please [open an issue](https://github.com/understandable-machine-intelligence-lab/Quantus/issues/new/choose)
-or write us at [[email protected]](mailto:[email protected]).
+or write us at [[email protected]](mailto:[email protected]).