From 2ab458817799d4ec609add8d11d541cf89156245 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Wed, 7 Jun 2023 18:45:55 +0200 Subject: [PATCH 01/34] wip --- .coveragerc | 15 -- .github/workflows/codecov.yml | 47 ++++- .github/workflows/lint.yml | 42 +++++ .github/workflows/python-package.yml | 57 +++--- .gitignore | 3 + MANIFEST.in | 5 - README.md | 57 +++--- mypy.ini | 18 -- pyproject.toml | 268 +++++++++++++++++++++++++++ pytest.ini | 24 --- quantus.toml | 6 - requirements.txt | 8 - requirements_test.txt | 24 --- setup.py | 74 -------- tox.ini | 49 +++++ 15 files changed, 467 insertions(+), 230 deletions(-) delete mode 100644 .coveragerc create mode 100644 .github/workflows/lint.yml delete mode 100644 MANIFEST.in delete mode 100644 mypy.ini create mode 100644 pyproject.toml delete mode 100644 pytest.ini delete mode 100644 quantus.toml delete mode 100644 requirements.txt delete mode 100644 requirements_test.txt delete mode 100644 setup.py create mode 100644 tox.ini diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 2001db9b2..000000000 --- a/.coveragerc +++ /dev/null @@ -1,15 +0,0 @@ -[run] -source = quantus -omit = - /tutorials/* - /tests/* - quantus/metrics/base.py - quantus/helpers/asserts.py - quantus/helpers/warn_func.py - quantus/helpers/model_interface.py - quantus/helpers/models.py - quantus/helpers/constants.py - quantus/helpers/plotting.py - -[report] -ignore_errors = True diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index b52a7df6c..80237627c 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -1,18 +1,45 @@ name: codecov -on: [push, pull_request] +on: + push: + # Run coverage only when pushed to main + branches: + - main + # and changes are in source or testing code. + paths: + - quantus + - tests + # Run coverage on any PR modifying source or testing code. + pull_request: + paths: + - quantus + - tests + # Allow manual trigger. + workflow_dispatch: + + jobs: run: runs-on: ubuntu-latest steps: - - name: Fetch - uses: actions/checkout@master - - name: Test - run: | - pip install -r requirements_test.txt - pytest tests -v --cov-report term --cov-report html:htmlcov --cov-report xml --cov=quantus + - uses: actions/checkout@v3 + + - name: Setup python 3.10 + uses: actions/setup-python@v4 + with: + cache: 'pip' + python-version: "3.10" + + - name: Install tox-gh + run: pip install tox-gh + + - name: Setup test environment + run: tox run -e coverage --notest + + - name: Measure coverage. + run: tox run -e coverage + - name: Upload - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml, coverage.xml - + files: ./coverage.xml, coverage.xml \ No newline at end of file diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..b8d983fb1 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,42 @@ +name: Lint + +on: + push: + # Run only when pushed to main + branches: + - main + # and changes are in source or testing code. + paths: + - quantus + - tests + # Run on any PR modifying source or testing code. + pull_request: + paths: + - quantus + - tests + # Allow manual trigger. + workflow_dispatch: + + +jobs: + lint: + runs-on: ubuntu-latest + name: Lint + steps: + - name: Check out source repository + uses: actions/checkout@v3 + - name: Set up Python environment + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install tox-gh + run: pip install tox-gh + - name: Run flake8 + run: tox run -e lint + - name: Run mypy + run: tox run -e mypy + - name: Verify black code-style + uses: psf/black@stable + with: + options: "--check --diff --color" + jupyter: true \ No newline at end of file diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 3f46b41b3..d6296983f 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -4,10 +4,22 @@ name: Python package on: - #push: - #branches: [ main ] + push: + # Run only when pushed to main + branches: + - main + # and changes are in source or testing code. + paths: + - quantus + - tests + # Run on any PR modifying source or testing code. pull_request: - branches: [ main ] + paths: + - quantus + - tests + # Allow manual trigger. + workflow_dispatch: + jobs: build: @@ -16,27 +28,22 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: [ "3.8", "3.9", "3.10" ] steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install flake8 pytest mypy==0.982 - if [ -f requirements_test.txt ]; then pip install -r requirements_test.txt; fi - - name: Lint - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - # run mypy - mypy quantus - - name: Test with pytest - run: | - pytest -s -v + - uses: actions/checkout@v3 + + - name: Setup python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + cache: 'pip' + python-version: ${{ matrix.python-version }} + + - name: Install tox-gh + run: pip install tox-gh + + - name: Setup test environment + run: tox run --notest + + - name: Test with PyTest + run: tox run --skip-pkg-install \ No newline at end of file diff --git a/.gitignore b/.gitignore index d9a4d1488..201e5246f 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,6 @@ tutorials/invited_talks/ # PyCharm project files .idea + +# Tox isolated environmet +.tox \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 14737e6b0..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -include requirements.txt -include requirements_test.txt -include quantus/* -global-exclude *.pyc -recursive-exclude . "__pycache__" diff --git a/README.md b/README.md index bc8c1de0b..8d5c0644f 100644 --- a/README.md +++ b/README.md @@ -218,29 +218,44 @@ Let's first load the data and model. In this example, a pre-trained LeNet availa for the purpose of this tutorial is loaded, but generally, you might use any Pytorch (or TensorFlow) model instead. To follow this example, one needs to have quantus and torch installed, by e.g., `pip install 'quantus[torch]'`. ```python -import quantus -from quantus.helpers.model.models import LeNet +from collections import OrderedDict +import urllib.request import torch -import torchvision -from torchvision import transforms - -# Enable GPU. +# Download model weights and batch of sample data. +urllib.request.urlretrieve( + "https://raw.github.com/understandable-machine-intelligence-lab/Quantus/main/tests/assets/mnist", + "/tmp/lenet_mnist_weights.pickle" +) +urllib.request.urlretrieve( + "https://raw.github.com/understandable-machine-intelligence-lab/Quantus/main/tests/assets/mnist_x", + "/tmp/mnist_x_batch.pt" +) +urllib.request.urlretrieve( + "https://raw.github.com/understandable-machine-intelligence-lab/Quantus/main/tests/assets/mnist_y", + "/tmp/mnist_y_batch.pt" +) +# Enable GPU if available. device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - -# Load a pre-trained LeNet classification model (architecture at quantus/helpers/models). -model = LeNet() -if device.type == "cpu": - model.load_state_dict(torch.load("tests/assets/mnist", map_location=torch.device('cpu'))) -else: - model.load_state_dict(torch.load("tests/assets/mnist")) - -# Load datasets and make loaders. -test_set = torchvision.datasets.MNIST(root='./sample_data', download=True, transforms=transforms.Compose([transforms.ToTensor()])) -test_loader = torch.utils.data.DataLoader(test_set, batch_size=24) - -# Load a batch of inputs and outputs to use for XAI evaluation. -x_batch, y_batch = iter(test_loader).next() -x_batch, y_batch = x_batch.cpu().numpy(), y_batch.cpu().numpy() +# Define LeNet model. +model = torch.nn.Sequential(OrderedDict([ + ("conv_1", torch.nn.Conv2d(1, 6, 5)), + ("relu_1", torch.nn.ReLU(),), + ("max_pool_1", torch.nn.MaxPool2d(2, 2),), + ("conv_2", torch.nn.Conv2d(6, 16, 5)), + ("relu_2", torch.nn.ReLU(),), + ("max_pool_2", torch.nn.MaxPool2d(2, 2),), + ("flatten", torch.nn.Flatten(),), + ("fc_1", torch.nn.Linear(256, 120)), + ("relu_3", torch.nn.ReLU(),), + ("fc_2", torch.nn.Linear(120, 84),), + ("relu_4", torch.nn.ReLU(),), + ("fc_3", torch.nn.Linear(84, 10),), +])) +# Load weights. +model.load_state_dict(torch.load("/tmp/lenet_mnist_weights.pickle", map_location=device)) +# Load a batch of inputs and labels to use for XAI evaluation. +x_batch = torch.load('/tmp/mnist_x_batch.pt').to(device) +y_batch = torch.load('/tmp/mnist_y_batch.pt').to(device) ``` diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 09c5feae7..000000000 --- a/mypy.ini +++ /dev/null @@ -1,18 +0,0 @@ -# Global options: -# https://mypy.readthedocs.io/en/stable/config_file.html#config-file-import-discovery-global - -# Instructions to run mypy. Go to library root, then run: -# > mypy quantus - -[mypy] -warn_return_any = False -warn_unused_configs = True -mypy_path = "quantus/" -ignore_missing_imports = True -no_site_packages = True -show_none_errors = False -ignore_errors = False - -[mypy-quantus.*] -disallow_untyped_defs = False -disable_error_code = misc, index, arg-type diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..d558db2b2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,268 @@ +[project] +# This is the name of your project. The first time you publish this +# package, this name will be registered for you. It will determine how +# users can install this project, e.g.: +# +# $ pip install sampleproject +# +# And where it will live on PyPI: https://pypi.org/project/sampleproject/ +# +# There are some restrictions on what makes a valid project name +# specification here: +# https://packaging.python.org/specifications/core-metadata/#name +name = "quantus" # Required + +# Versions should comply with PEP 440: +# https://www.python.org/dev/peps/pep-0440/ +# +# For a discussion on single-sourcing the version, see +# https://packaging.python.org/guides/single-sourcing-package-version/ +version = "0.4.0" # Required + +# This is a one-line description or tagline of what your project does. This +# corresponds to the "Summary" metadata field: +# https://packaging.python.org/specifications/core-metadata/#summary +description = "A metrics toolkit to evaluate neural network explanations." # Optional + +# This is an optional longer description of your project that represents +# the body of text which users will see when they visit PyPI. +# +# Often, this is the same as your README, so you can just read it in from +# that file directly (as we have already done above) +# +# This field corresponds to the "Description" metadata field: +# https://packaging.python.org/specifications/core-metadata/#description-optional +readme = "README.md" # Optional + +# Specify which Python versions you support. In contrast to the +# 'Programming Language' classifiers above, 'pip install' will check this +# and refuse to install the project if the version does not match. See +# https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires +requires-python = ">=3.7" + +# This is either text indicating the license for the distribution, or a file +# that contains the license +# https://packaging.python.org/en/latest/specifications/core-metadata/#license +license = { file = "LICENSE" } + +# This field adds keywords for your project which will appear on the +# project page. What does your project relate to? +# +# Note that this is a list of additional keywords, separated +# by commas, to be used to assist searching for the distribution in a +# larger catalog. +keywords = ["explainable ai", "xai", "machine learning", "deep learning"] # Optional + +# This should be your name or the name of the organization who originally +# authored the project, and a valid email address corresponding to the name +# listed. +authors = [ + { name = "Anna Hedstrom", email = "hedstroem.anna@gmail.com" } # Optional +] + +# This should be your name or the names of the organization who currently +# maintains the project, and a valid email address corresponding to the name +# listed. +maintainers = [ + { name = "Anna Hedstrom", email = "hedstroem.anna@gmail.com" } # Optional +] + +# Classifiers help users find your project by categorizing it. +# +# For a list of valid classifiers, see https://pypi.org/classifiers/ +classifiers = [# Optional + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + "Development Status :: 5 - Production/Stable", + # Indicate who your project is intended for + "Intended Audience :: Education", + "Intended Audience :: Science/Research", + "Intended Audience :: Developers", + # Pick your license as you wish + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Topic :: Scientific :: Explainable Artificial Intelligence", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] + +# This field lists other packages that your project depends on to run. +# Any package you put here will be installed by pip when your project is +# installed, so they must be valid existing projects. +# +# For an analysis of this field vs pip's requirements files see: +# https://packaging.python.org/discussions/install-requires-vs-requirements/ +dependencies = [ + "numpy>=1.19.5", + "opencv-python>=4.5.5.62", + "scikit-image>=0.19.3", + "scikit-learn>=0.24.2", + "scipy>=1.7.3", + "tqdm>=4.62.3", + "matplotlib>=3.3.4" +] + +# List additional groups of dependencies here (e.g. development +# dependencies). Users will be able to install these using the "extras" +# syntax, for example: +# +# $ pip install sampleproject[dev] +# +# Similar to `dependencies` above, these must be valid existing +# projects. +[project.optional-dependencies] # Optional +tests = [ + "captum>=0.6.0", + "coverage>=7.2.3", + "flake8<=4.0.1; python_version == '3.7'", + "flake8>=6.0.0; python_version > '3.7'", + "pandas<=1.3.3; python_version == '3.7'", + "pandas>=2.0.1; python_version > '3.7'", + "pytest>=7.3.1", + "pytest-cov>=4.0.0", + "pytest-lazy-fixture>=0.6.3", + "pytest-mock==3.10.0", + "pytest_xdist", + "tf-explain>=0.3.1", + "zennit>=0.4.5; python_version >= '3.7'", + "tensorflow>=2.5.0; python_version == '3.7'", + "tensorflow>=2.12.0; sys_platform != 'darwin' and python_version > '3.7'", + "tensorflow_macos>=2.9.0; sys_platform == 'darwin' and python_version > '3.7'", + "torch<=1.9.0; python_version == '3.7'", + "torch>=1.13.1; sys_platform != 'linux' and python_version > '3.7'", + "torch>=1.13.1, <2.0.0; sys_platform == 'linux' and python_version > '3.7' and python_version <= '3.10'", + "torch>=2.0.0; sys_platform == 'linux' and python_version >= '3.11'", + "torchvision<=0.12.0; python_version == '3.7'", + "torchvision>=0.15.1; sys_platform != 'linux' and python_version > '3.7'", + "torchvision>=0.14.0, <0.15.1; sys_platform == 'linux' and python_version > '3.7' and python_version <= '3.10'", + "torchvision>=0.15.1; sys_platform == 'linux' and python_version >= '3.11'" +] +torch = [ + "torch<=1.11.0; python_version == '3.7'", + "torch>=1.13.1; sys_platform != 'linux' and python_version > '3.7'", + "torch>=1.13.1, <2.0.0; sys_platform == 'linux' and python_version > '3.7' and python_version <= '3.10'", + "torch>=2.0.0; sys_platform == 'linux' and python_version >= '3.11'", + "torchvision<=0.12.0; python_version == '3.7'", + "torchvision>=0.15.1; sys_platform != 'linux' and python_version > '3.7'", + "torchvision>=0.14.0, <0.15.1; sys_platform == 'linux' and python_version > '3.7' and python_version <= '3.10'", + "torchvision>=0.15.1; sys_platform == 'linux' and python_version >= '3.11'" +] +tensorflow = [ + "tensorflow>=2.5.0; python_version == '3.7'", + "tensorflow>=2.12.0; sys_platform != 'darwin' and python_version > '3.7'", + "tensorflow_macos>=2.12.0; sys_platform == 'darwin' and python_version > '3.7'", +] +captum = [ + "quantus[torch]", + "captum>=0.6.0" +] +tf_explain = [ + "quantus[tensorflow]", + "tf-explain>=0.3.1" +] +zennit = [ + "quantus[torch]", + "zennit>=0.5.1" +] +full = [ + "quantus[captum,tf_explain,zennit]" +] + +# List URLs that are relevant to your project +# +# This field corresponds to the "Project-URL" and "Home-Page" metadata fields: +# https://packaging.python.org/specifications/core-metadata/#project-url-multiple-use +# https://packaging.python.org/specifications/core-metadata/#home-page-optional +# +# Examples listed include a pattern for specifying where the package tracks +# issues, where the source is hosted, where to say thanks to the package +# maintainers, and where to support the project financially. The key is +# what's used to render the link text on PyPI. +[project.urls] # Optional +"Homepage" = "https://github.com/understandable-machine-intelligence-lab/Quantus" +#"Bug Reports" = "https://github.com/pypa/sampleproject/issues" +#"Funding" = "https://donate.pypi.org" +#"Say Thanks!" = "http://saythanks.io/to/example" +#"Source" = "https://github.com/pypa/sampleproject/" + +# The following would provide a command line executable called `sample` +# which executes the function `main` from this package when invoked. +#[project.scripts] # Optional +#sample = "sample:main" + +[build-system] +# These are the assumed default build requirements from pip: +# https://pip.pypa.io/en/stable/reference/pip/#pep-517-and-518-support +requires = ["flit-core >= 3.4"] +build-backend = "flit_core.buildapi" + + +# Global options: +# https://mypy.readthedocs.io/en/stable/config_file.html#config-file-import-discovery-global + +# Instructions to run mypy. Go to library root, then run: +# > mypy quantus + +[tool.mypy] +warn_return_any = false +warn_unused_configs = true +mypy_path = "quantus/" +ignore_missing_imports = true +no_site_packages = true +show_none_errors = false +plugins = ["numpy.typing.mypy_plugin"] + + +[[tool.mypy.overrides]] +module = "quantus.*" +disallow_untyped_defs = false +disable_error_code = ["misc", "index", "arg-type"] + +# PyTest confoguration, for all options refer to https://docs.pytest.org/en/7.2.x/reference/customize.html +[tool.pytest.ini_options] +# addopts = ["-s -v -n auto"] +markers = [ + "loss_func: loss_func tests.", + "similar_func: similar_func tests.", + "perturb_func: perturb_func tests.", + "normalise_func: normalise_func tests.", + "norm_func: norm_func tests.", + "explain_func: explain_func tests.", + "evaluate_func: evaluate tests.", + "utils: utils tests.", + "fixes: fixing tests.", + "pytorch_model: pytorch model interface tests.", + "tf_model: tensorflow model interface tests.", + "randomisation: randomisation metrics tests.", + "faithfulness: faithfulness metrics tests.", + "robustness: robustness metrics tests.", + "localisation: localisation metrics tests.", + "complexity: complexity metrics tests.", + "axiomatic: axiomatic metrics tests." +] +filterwarnings = [ + "error", + "ignore::UserWarning", + "ignore::DeprecationWarning" +] + +# Coverage configuration, for all options refer to https://coverage.readthedocs.io/en/latest/config.html +# Note, we need to run python 3.11 or have coverage[toml] installed. +[tool.coverage.run] +source = ["quantus.*"] +omit = [ + "/tutorials/*", + "/tests/*", + "quantus/metrics/base.py", + "quantus/helpers/asserts.py", + "quantus/helpers/warn_func.py", + "quantus/helpers/model_interface.py", + "quantus/helpers/models.py", + "quantus/helpers/constants.py", + "quantus/helpers/plotting.py" +] + +[tool.coverage.report] +ignore_errors = true diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 00a92ec87..000000000 --- a/pytest.ini +++ /dev/null @@ -1,24 +0,0 @@ -[pytest] -markers = - loss_func: loss_func tests. - similar_func: similar_func tests. - perturb_func: perturb_func tests. - normalise_func: normalise_func tests. - norm_func: norm_func tests. - explain_func: explain_func tests. - evaluate_func: evaluate tests. - utils: utils tests. - fixes: fixing tests. - pytorch_model: pytorch model interface tests. - tf_model: tensorflow model interface tests. - randomisation: randomisation metrics tests. - faithfulness: faithfulness metrics tests. - robustness: robustness metrics tests. - localisation: localisation metrics tests. - complexity: complexity metrics tests. - axiomatic: axiomatic metrics tests. - -filterwarnings = - error - ignore::UserWarning - ignore::DeprecationWarning diff --git a/quantus.toml b/quantus.toml deleted file mode 100644 index b5a3c468d..000000000 --- a/quantus.toml +++ /dev/null @@ -1,6 +0,0 @@ -[build-system] -requires = [ - "setuptools>=42", - "wheel" -] -build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 0f9df44cd..000000000 --- a/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -cachetools>=5.3.0 -matplotlib>=3.3.4 -numpy>=1.19.5 -opencv-python>=4.5.5.62 -scikit-image>=0.19.3 -scikit-learn>=0.24.2 -scipy>=1.7.3 -tqdm>=4.62.3 \ No newline at end of file diff --git a/requirements_test.txt b/requirements_test.txt deleted file mode 100644 index acd4f09f8..000000000 --- a/requirements_test.txt +++ /dev/null @@ -1,24 +0,0 @@ --r requirements.txt -captum>=0.6.0 -coverage>=7.2.3 -flake8<=4.0.1; python_version == '3.7' -flake8>=6.0.0; python_version > '3.7' -pandas<=1.3.3; python_version == '3.7' -pandas>=2.0.1; python_version > '3.7' -pytest>=7.3.1 -pytest-cov>=4.0.0 -pytest-lazy-fixture>=0.6.3 -pytest-mock==3.10.0 -tf-explain>=0.3.1 -zennit>=0.4.5; python_version >= '3.7' -tensorflow>=2.5.0; python_version == '3.7' -tensorflow>=2.12.0; sys_platform != 'darwin' and python_version > '3.7' -tensorflow_macos>=2.9.0; sys_platform == 'darwin' and python_version > '3.7' -torch<=1.9.0; python_version == '3.7' -torch>=1.13.1; sys_platform != 'linux' and python_version > '3.7' -torch>=1.13.1, <2.0.0; sys_platform == 'linux' and python_version > '3.7' and python_version <= '3.10' -torch>=2.0.0; sys_platform == 'linux' and python_version >= '3.11' -torchvision<=0.12.0; python_version == '3.7' -torchvision>=0.15.1; sys_platform != 'linux' and python_version > '3.7' -torchvision>=0.14.0, <0.15.1; sys_platform == 'linux' and python_version > '3.7' and python_version <= '3.10' -torchvision>=0.15.1; sys_platform == 'linux' and python_version >= '3.11' diff --git a/setup.py b/setup.py deleted file mode 100644 index 1eb6454d5..000000000 --- a/setup.py +++ /dev/null @@ -1,74 +0,0 @@ -# This file is part of Quantus. -# Quantus is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. -# Quantus is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -# You should have received a copy of the GNU Lesser General Public License along with Quantus. If not, see . - -import os -from setuptools import setup, find_packages -from importlib import util - -with open(os.path.join(os.path.dirname(__file__), "requirements.txt")) as f: - - required = f.read().splitlines() - -with open(os.path.join(os.path.dirname(__file__), "requirements_test.txt")) as f: - required_tests = f.read().splitlines() - -# Define extras. -EXTRAS = {} -EXTRAS["torch"] = ( - [ - "torch<=1.11.0; python_version == '3.7'", - "torch>=1.13.1; sys_platform != 'linux' and python_version > '3.7'", - "torch>=1.13.1, <2.0.0; sys_platform == 'linux' and python_version > '3.7' and python_version <= '3.10'", - "torch>=2.0.0; sys_platform == 'linux' and python_version >= '3.11'", - "torchvision<=0.12.0; python_version == '3.7'", - "torchvision>=0.15.1; sys_platform != 'linux' and python_version > '3.7'", - "torchvision>=0.14.0, <0.15.1; sys_platform == 'linux' and python_version > '3.7' and python_version <= '3.10'", - "torchvision>=0.15.1; sys_platform == 'linux' and python_version >= '3.11'", - ] - if not (util.find_spec("torch")) - else [] -) -EXTRAS["tensorflow"] = ( - [ - "tensorflow>=2.5.0; python_version == '3.7'", - "tensorflow>=2.12.0; sys_platform != 'darwin' and python_version > '3.7'", - "tensorflow_macos>=2.12.0; sys_platform == 'darwin' and python_version > '3.7'", - ] - if not (util.find_spec("tensorflow")) - else [] -) -EXTRAS["captum"] = ( - (EXTRAS["torch"] + ["captum>=0.6.0"]) if not util.find_spec("captum") else [] -) -EXTRAS["tf-explain"] = ( - (EXTRAS["tensorflow"] + ["tf-explain>=0.3.1"]) - if not util.find_spec("tf-explain") - else [] -) -EXTRAS["zennit"] = ( - (EXTRAS["torch"] + ["zennit>=0.5.1"]) if not util.find_spec("zennit") else [] -) -EXTRAS["tests"] = required + required_tests[1:] -EXTRAS["full"] = EXTRAS["captum"] + EXTRAS["tf-explain"] + EXTRAS["zennit"] - -# Define setup. -setup( - name="quantus", - version="0.4.0", - description="A toolkit to evaluate neural network explanations.", - long_description=open("README.md", "r").read(), - long_description_content_type="text/markdown", - install_requires=required, - extras_require=EXTRAS, - url="http://github.com/understandable-machine-intelligence-lab/Quantus", - author="Anna Hedstrom", - author_email="hedstroem.anna@gmail.com", - keywords=["explainable ai", "xai", "machine learning", "deep learning"], - license="GNU LESSER GENERAL PUBLIC LICENSE VERSION 3", - packages=find_packages(), - zip_safe=False, - python_requires=">=3.7", - include_package_data=True, -) diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..2680a4fe9 --- /dev/null +++ b/tox.ini @@ -0,0 +1,49 @@ +[tox] +requires = + tox>=4.2 + virtualenv>20.2 +env_list = + lint + type + coverage + py{311, 310, 39, 38, 37} +skip_missing_interpreters = true + +[testenv] +description = run the tests with {basepython} +deps = + .[tests] +pass_env = + TF_XLA_FLAGS +commands = + pytest {posargs} + +[testenv:lint] +description = check the code style +deps = + flake8 +commands = + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics {posargs} + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics {posargs} + +[testenv:type] +description = run type checking +deps = + mypy==0.982 +commands = + mypy quantus {posargs} + +[testenv:coverage] +description = run the tests with coverage +deps = + {[testenv]deps} + coverage[toml] + pytest_cov +commands = + pytest --cov-report term --cov-report html:htmlcov --cov-report xml --cov=quantus {posargs} + +[gh] +python = + 3.8 = py38 + 3.9 = py39 + 3.10 = py310 From 03d51527b45b2a8f662f0dedba6dcabbeae75ceb Mon Sep 17 00:00:00 2001 From: aaarrti Date: Wed, 7 Jun 2023 18:48:51 +0200 Subject: [PATCH 02/34] * --- README.md | 57 ++++++++++++++++++++----------------------------------- 1 file changed, 21 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 8d5c0644f..bc8c1de0b 100644 --- a/README.md +++ b/README.md @@ -218,44 +218,29 @@ Let's first load the data and model. In this example, a pre-trained LeNet availa for the purpose of this tutorial is loaded, but generally, you might use any Pytorch (or TensorFlow) model instead. To follow this example, one needs to have quantus and torch installed, by e.g., `pip install 'quantus[torch]'`. ```python -from collections import OrderedDict -import urllib.request +import quantus +from quantus.helpers.model.models import LeNet import torch -# Download model weights and batch of sample data. -urllib.request.urlretrieve( - "https://raw.github.com/understandable-machine-intelligence-lab/Quantus/main/tests/assets/mnist", - "/tmp/lenet_mnist_weights.pickle" -) -urllib.request.urlretrieve( - "https://raw.github.com/understandable-machine-intelligence-lab/Quantus/main/tests/assets/mnist_x", - "/tmp/mnist_x_batch.pt" -) -urllib.request.urlretrieve( - "https://raw.github.com/understandable-machine-intelligence-lab/Quantus/main/tests/assets/mnist_y", - "/tmp/mnist_y_batch.pt" -) -# Enable GPU if available. +import torchvision +from torchvision import transforms + +# Enable GPU. device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") -# Define LeNet model. -model = torch.nn.Sequential(OrderedDict([ - ("conv_1", torch.nn.Conv2d(1, 6, 5)), - ("relu_1", torch.nn.ReLU(),), - ("max_pool_1", torch.nn.MaxPool2d(2, 2),), - ("conv_2", torch.nn.Conv2d(6, 16, 5)), - ("relu_2", torch.nn.ReLU(),), - ("max_pool_2", torch.nn.MaxPool2d(2, 2),), - ("flatten", torch.nn.Flatten(),), - ("fc_1", torch.nn.Linear(256, 120)), - ("relu_3", torch.nn.ReLU(),), - ("fc_2", torch.nn.Linear(120, 84),), - ("relu_4", torch.nn.ReLU(),), - ("fc_3", torch.nn.Linear(84, 10),), -])) -# Load weights. -model.load_state_dict(torch.load("/tmp/lenet_mnist_weights.pickle", map_location=device)) -# Load a batch of inputs and labels to use for XAI evaluation. -x_batch = torch.load('/tmp/mnist_x_batch.pt').to(device) -y_batch = torch.load('/tmp/mnist_y_batch.pt').to(device) + +# Load a pre-trained LeNet classification model (architecture at quantus/helpers/models). +model = LeNet() +if device.type == "cpu": + model.load_state_dict(torch.load("tests/assets/mnist", map_location=torch.device('cpu'))) +else: + model.load_state_dict(torch.load("tests/assets/mnist")) + +# Load datasets and make loaders. +test_set = torchvision.datasets.MNIST(root='./sample_data', download=True, transforms=transforms.Compose([transforms.ToTensor()])) +test_loader = torch.utils.data.DataLoader(test_set, batch_size=24) + +# Load a batch of inputs and outputs to use for XAI evaluation. +x_batch, y_batch = iter(test_loader).next() +x_batch, y_batch = x_batch.cpu().numpy(), y_batch.cpu().numpy() ``` From 16f768643a8910031244f36b37a69c76802e285a Mon Sep 17 00:00:00 2001 From: aaarrti Date: Wed, 14 Jun 2023 16:23:39 +0200 Subject: [PATCH 03/34] add github release script --- .coveragerc | 15 ++++++ .github/workflows/codecov.yml | 10 ---- .github/workflows/lint.yml | 17 +++---- .github/workflows/pre-release.yml | 27 ++++++++++ .github/workflows/python-package.yml | 27 +++------- CONTRIBUTING.md | 54 ++++++++++---------- mypy.ini | 18 +++++++ pyproject.toml | 73 +--------------------------- pytest.ini | 24 +++++++++ scripts/github_release.sh | 24 +++++++++ tests/README.md | 13 ----- 11 files changed, 151 insertions(+), 151 deletions(-) create mode 100644 .coveragerc create mode 100644 .github/workflows/pre-release.yml create mode 100644 mypy.ini create mode 100644 pytest.ini create mode 100755 scripts/github_release.sh delete mode 100644 tests/README.md diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..2001db9b2 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,15 @@ +[run] +source = quantus +omit = + /tutorials/* + /tests/* + quantus/metrics/base.py + quantus/helpers/asserts.py + quantus/helpers/warn_func.py + quantus/helpers/model_interface.py + quantus/helpers/models.py + quantus/helpers/constants.py + quantus/helpers/plotting.py + +[report] +ignore_errors = True diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 80237627c..af504aab4 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -1,19 +1,9 @@ name: codecov on: push: - # Run coverage only when pushed to main branches: - main - # and changes are in source or testing code. - paths: - - quantus - - tests - # Run coverage on any PR modifying source or testing code. pull_request: - paths: - - quantus - - tests - # Allow manual trigger. workflow_dispatch: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b8d983fb1..d0be221e5 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,19 +2,9 @@ name: Lint on: push: - # Run only when pushed to main branches: - main - # and changes are in source or testing code. - paths: - - quantus - - tests - # Run on any PR modifying source or testing code. pull_request: - paths: - - quantus - - tests - # Allow manual trigger. workflow_dispatch: @@ -22,13 +12,18 @@ jobs: lint: runs-on: ubuntu-latest name: Lint + strategy: + fail-fast: false + matrix: + python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] steps: - name: Check out source repository uses: actions/checkout@v3 - name: Set up Python environment uses: actions/setup-python@v4 with: - python-version: "3.10" + cache: 'pip' + python-version: ${{ matrix.python-version }} - name: Install tox-gh run: pip install tox-gh - name: Run flake8 diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml new file mode 100644 index 000000000..902a1f8d8 --- /dev/null +++ b/.github/workflows/pre-release.yml @@ -0,0 +1,27 @@ +name: Python package + +on: + schedule: + - cron: '* 8 * * *' + workflow_dispatch: + + +jobs: + build: + runs-on: ubuntu-latest + name: Pre Release + steps: + - uses: actions/checkout@v3 + - name: Setup python + uses: actions/setup-python@v4 + with: + cache: 'pip' + python-version: "3.10" + - name: Install tox-gh + run: pip install tox-gh + - name: Setup test environment + run: tox run --notest + - name: Test with PyTest + run: tox run + - name: Test building package + run: tox run -e build \ No newline at end of file diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index d6296983f..377e4adab 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -1,49 +1,34 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - name: Python package on: push: - # Run only when pushed to main branches: - main - # and changes are in source or testing code. - paths: - - quantus - - tests - # Run on any PR modifying source or testing code. pull_request: - paths: - - quantus - - tests - # Allow manual trigger. workflow_dispatch: + jobs: build: - runs-on: ubuntu-latest + name: Python Package strategy: fail-fast: false matrix: - python-version: [ "3.8", "3.9", "3.10" ] - + python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] steps: - uses: actions/checkout@v3 - - name: Setup python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: cache: 'pip' python-version: ${{ matrix.python-version }} - - name: Install tox-gh run: pip install tox-gh - - name: Setup test environment run: tox run --notest - - name: Test with PyTest - run: tox run --skip-pkg-install \ No newline at end of file + run: tox run + - name: Test building package + run: tox run -e build \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d10b47808..8efb4f5a5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,8 +48,10 @@ Make sure to install the latest version of Quantus from the main branch. ```bash git clone https://github.com/understandable-machine-intelligence-lab/Quantus.git cd quantus -pip install -r requirements_test.txt -pip install -e . +# Tox will provision dev environment with editable installation for you. +python3 -m pip install tox +python3 -m tox run -e dev +source .tox/dev/bin/activate ``` ### Branching @@ -64,28 +66,25 @@ We use [flake8](https://pypi.org/project/flake8/) for quick style checks and [bl ### Unit Tests Tests are written using [pytest](https://github.com/pytest-dev/pytest) and executed together with [codecov](https://github.com/codecov/codecov-action) for coverage reports. -To perform the tests, execute the following (make sure pytest is installed): +We use [tox](https://tox.wiki/en/latest/) for test automation. ```bash -pytest +# To list all test environments, run: +python3 -m tox list +# To execute all test environments run: +python3 -m tox run +# Or, e.g., you can run only tests for pytho3.8 with: +python3 -m tox run -e py38 +# Or, e.g., run type checking for python 3.10 with: +python3 -m tox run -e py310_type +# Or coverage using python 3.10 +python3 -m tox run -e py310_coverage +# If you want to pass CLI arguments to pytest, add them after --, e.g., you might want to split execution between 4 cpu cores, +# show verbose logging, and run only robustness tests. +python3 -m tox run py310 -- -n 4 -s -v -m "robustness" +# You still can run tests using your active local interpreter by: +python -m pytest ``` - -... optionally, you could split test execution between multiple CPU cores using [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) -```bash -pytest tests -n auto -``` - -... alternatively, to get additionaly coverage details, run: -```bash -pytest --cov=. --cov-report term-missing -``` - -It is possible to limit the scope of testing to specific sections of the codebase, for example, only test the Faithfulness metrics: -```bash -pytest -m faithfulness -s -``` -For a complete overview of the possible testing scopes, please refer to `pytest.ini`. - ### Documentation Make sure to add docstrings to every class, method and function that you add to the codebase. The docstring should include a description of all parameters and returns. Use the existing documentation as an example. @@ -104,11 +103,16 @@ black quantus/INSERT_YOUR_FILE_NAME.py ```bash flake8 quantus/INSERT_YOUR_FILE_NAME.py ``` -- Create `pytests` for new functionality (if needed) and add under `tests/` folder -- If the `pytests` include a new category of `@pytest.mark` then add that category with description to `pytest.ini` -- Make sure all unit tests are passed and the coverage level is maintained (we aim at ~100% code coverage for Quantus): +- Create tests for new functionality and add under `tests/` folder. +- If tests include a new category of `@pytest.mark` then add that category with description to `pytest.ini` +- Make sure all unit tests pass for all supported python version by running: +```shell +python -m tox run +``` +- Generally, every change should be covered with respective test-case, we aim at ~100% code coverage for Quantus. +You can verify it by running: ```bash -pytest tests -v --cov-report term --cov-report html:htmlcov --cov-report xml --cov=quantus +python -m tox run coverage ``` ### Pull Requests diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 000000000..09c5feae7 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,18 @@ +# Global options: +# https://mypy.readthedocs.io/en/stable/config_file.html#config-file-import-discovery-global + +# Instructions to run mypy. Go to library root, then run: +# > mypy quantus + +[mypy] +warn_return_any = False +warn_unused_configs = True +mypy_path = "quantus/" +ignore_missing_imports = True +no_site_packages = True +show_none_errors = False +ignore_errors = False + +[mypy-quantus.*] +disallow_untyped_defs = False +disable_error_code = misc, index, arg-type diff --git a/pyproject.toml b/pyproject.toml index d558db2b2..473a883b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ name = "quantus" # Required # # For a discussion on single-sourcing the version, see # https://packaging.python.org/guides/single-sourcing-package-version/ -version = "0.4.0" # Required +version = "0.4.1" # Required # This is a one-line description or tagline of what your project does. This # corresponds to the "Summary" metadata field: @@ -196,73 +196,4 @@ full = [ # These are the assumed default build requirements from pip: # https://pip.pypa.io/en/stable/reference/pip/#pep-517-and-518-support requires = ["flit-core >= 3.4"] -build-backend = "flit_core.buildapi" - - -# Global options: -# https://mypy.readthedocs.io/en/stable/config_file.html#config-file-import-discovery-global - -# Instructions to run mypy. Go to library root, then run: -# > mypy quantus - -[tool.mypy] -warn_return_any = false -warn_unused_configs = true -mypy_path = "quantus/" -ignore_missing_imports = true -no_site_packages = true -show_none_errors = false -plugins = ["numpy.typing.mypy_plugin"] - - -[[tool.mypy.overrides]] -module = "quantus.*" -disallow_untyped_defs = false -disable_error_code = ["misc", "index", "arg-type"] - -# PyTest confoguration, for all options refer to https://docs.pytest.org/en/7.2.x/reference/customize.html -[tool.pytest.ini_options] -# addopts = ["-s -v -n auto"] -markers = [ - "loss_func: loss_func tests.", - "similar_func: similar_func tests.", - "perturb_func: perturb_func tests.", - "normalise_func: normalise_func tests.", - "norm_func: norm_func tests.", - "explain_func: explain_func tests.", - "evaluate_func: evaluate tests.", - "utils: utils tests.", - "fixes: fixing tests.", - "pytorch_model: pytorch model interface tests.", - "tf_model: tensorflow model interface tests.", - "randomisation: randomisation metrics tests.", - "faithfulness: faithfulness metrics tests.", - "robustness: robustness metrics tests.", - "localisation: localisation metrics tests.", - "complexity: complexity metrics tests.", - "axiomatic: axiomatic metrics tests." -] -filterwarnings = [ - "error", - "ignore::UserWarning", - "ignore::DeprecationWarning" -] - -# Coverage configuration, for all options refer to https://coverage.readthedocs.io/en/latest/config.html -# Note, we need to run python 3.11 or have coverage[toml] installed. -[tool.coverage.run] -source = ["quantus.*"] -omit = [ - "/tutorials/*", - "/tests/*", - "quantus/metrics/base.py", - "quantus/helpers/asserts.py", - "quantus/helpers/warn_func.py", - "quantus/helpers/model_interface.py", - "quantus/helpers/models.py", - "quantus/helpers/constants.py", - "quantus/helpers/plotting.py" -] - -[tool.coverage.report] -ignore_errors = true +build-backend = "flit_core.buildapi" \ No newline at end of file diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..00a92ec87 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,24 @@ +[pytest] +markers = + loss_func: loss_func tests. + similar_func: similar_func tests. + perturb_func: perturb_func tests. + normalise_func: normalise_func tests. + norm_func: norm_func tests. + explain_func: explain_func tests. + evaluate_func: evaluate tests. + utils: utils tests. + fixes: fixing tests. + pytorch_model: pytorch model interface tests. + tf_model: tensorflow model interface tests. + randomisation: randomisation metrics tests. + faithfulness: faithfulness metrics tests. + robustness: robustness metrics tests. + localisation: localisation metrics tests. + complexity: complexity metrics tests. + axiomatic: axiomatic metrics tests. + +filterwarnings = + error + ignore::UserWarning + ignore::DeprecationWarning diff --git a/scripts/github_release.sh b/scripts/github_release.sh new file mode 100755 index 000000000..e06fbffe5 --- /dev/null +++ b/scripts/github_release.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env zsh +set -e +# Check no un-committed changes +echo -n "Checking if there are uncommited changes... " +trap 'echo -e "\033[0;31mCHANGED\033[0m"' ERR +git diff-index --quiet HEAD -- +trap - ERR +echo -e "\033[0;32mUNCHANGED\033[0m" +# Check provided 1 positional argument +if [ $# -eq 0 ]; then + echo -e "Must provide tag as positional argument" +fi +TAG=$1 +echo "TAG=${TAG}" +# Update main ref's and switch to main's HEAD. +git fetch --atomic --verbose && git checkout main +# Build wheel. +python3 -m build . +python3 -m twine check ./dist/* +# Tag release. +git tag "$TAG" +git push --follow-tags +# Create GitHub release draft. +gh release create "$TAG" ./dist/* --draft --latest diff --git a/tests/README.md b/tests/README.md deleted file mode 100644 index 44458e0be..000000000 --- a/tests/README.md +++ /dev/null @@ -1,13 +0,0 @@ -### How to run tests - -Run all tests at once: - -```pytest``` - -Run a subset of tests with e.g., localisation metrics (see available markers in the pytest.ini files): - -```pytest -m localisation -s``` - -Run pytest with coverage: - -```pytest tests -v --cov-report term --cov-report html:htmlcov --cov-report xml --cov=quantus``` \ No newline at end of file From 07ae0d53d3a72be9ca0a7708d6fbccde4310cf2b Mon Sep 17 00:00:00 2001 From: aaarrti Date: Wed, 14 Jun 2023 16:40:49 +0200 Subject: [PATCH 04/34] add github release script --- scripts/github_release.sh | 3 +-- tox.ini | 52 ++++++++++++++++++++++++++------------- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/scripts/github_release.sh b/scripts/github_release.sh index e06fbffe5..6da22e122 100755 --- a/scripts/github_release.sh +++ b/scripts/github_release.sh @@ -15,8 +15,7 @@ echo "TAG=${TAG}" # Update main ref's and switch to main's HEAD. git fetch --atomic --verbose && git checkout main # Build wheel. -python3 -m build . -python3 -m twine check ./dist/* +tox run -e build # Tag release. git tag "$TAG" git push --follow-tags diff --git a/tox.ini b/tox.ini index 2680a4fe9..018d42973 100644 --- a/tox.ini +++ b/tox.ini @@ -3,47 +3,65 @@ requires = tox>=4.2 virtualenv>20.2 env_list = - lint - type coverage + build + py{311, 310, 39, 38, 37}_{lint, type} py{311, 310, 39, 38, 37} skip_missing_interpreters = true [testenv] -description = run the tests with {basepython} +description = Run the tests with {basepython} deps = .[tests] pass_env = TF_XLA_FLAGS commands = - pytest {posargs} + pytest -s -v {posargs} -[testenv:lint] -description = check the code style +[testenv:dev] +desciption = Development environment +basepython = python3.10 +usedevelop = True + +[testenv:py{311, 310, 39, 38, 37}_lint] +description = Check the code style deps = flake8 commands = - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics {posargs} - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics {posargs} + python3 -m flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics {posargs} + python3 -m flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics {posargs} -[testenv:type] -description = run type checking +[testenv:py{311, 310, 39, 38, 37}_type] +description = Run type checking deps = + {[testenv]deps} mypy==0.982 commands = - mypy quantus {posargs} + python3 -m mypy quantus {posargs} [testenv:coverage] -description = run the tests with coverage +description = Run the tests with coverage deps = {[testenv]deps} - coverage[toml] + coverage pytest_cov commands = - pytest --cov-report term --cov-report html:htmlcov --cov-report xml --cov=quantus {posargs} + python3 -m pytest --cov-report term --cov-report html:htmlcov --cov-report xml --cov=quantus {posargs} + +[testenv:build] +description = Build environment +deps = + . + build + twine +commands = + python3 -m build . + python3 -m twine check ./dist/* --strict [gh] python = - 3.8 = py38 - 3.9 = py39 - 3.10 = py310 + 3.7 = py37, py37_{lint, type} + 3.8 = py38, py38_{lint, type} + 3.9 = py39, py39_{lint, type} + 3.10 = py310, py310_{lint, type}, coverage + 3.11 = py311, py311_{lint, type} \ No newline at end of file From d3177e7311587653c02bc26b50907f2b13bd3ea4 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Wed, 14 Jun 2023 17:05:53 +0200 Subject: [PATCH 05/34] * --- .github/workflows/lint.yml | 7 +------ tox.ini | 6 +++--- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d0be221e5..b44afe6e4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -29,9 +29,4 @@ jobs: - name: Run flake8 run: tox run -e lint - name: Run mypy - run: tox run -e mypy - - name: Verify black code-style - uses: psf/black@stable - with: - options: "--check --diff --color" - jupyter: true \ No newline at end of file + run: tox run -e mypy \ No newline at end of file diff --git a/tox.ini b/tox.ini index 018d42973..4d18fa9ad 100644 --- a/tox.ini +++ b/tox.ini @@ -28,8 +28,8 @@ description = Check the code style deps = flake8 commands = - python3 -m flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics {posargs} - python3 -m flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics {posargs} + python3 -m flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + python3 -m flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics [testenv:py{311, 310, 39, 38, 37}_type] description = Run type checking @@ -37,7 +37,7 @@ deps = {[testenv]deps} mypy==0.982 commands = - python3 -m mypy quantus {posargs} + python3 -m mypy quantus [testenv:coverage] description = Run the tests with coverage From 65239ead4d10ec337dbd263f6aed2ced286154b5 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Wed, 14 Jun 2023 17:35:01 +0200 Subject: [PATCH 06/34] * --- .github/workflows/lint.yml | 2 +- tox.ini | 58 +++++++++++++++++++------------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b44afe6e4..399f113fd 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -29,4 +29,4 @@ jobs: - name: Run flake8 run: tox run -e lint - name: Run mypy - run: tox run -e mypy \ No newline at end of file + run: tox run -e type \ No newline at end of file diff --git a/tox.ini b/tox.ini index 4d18fa9ad..51882be30 100644 --- a/tox.ini +++ b/tox.ini @@ -3,9 +3,6 @@ requires = tox>=4.2 virtualenv>20.2 env_list = - coverage - build - py{311, 310, 39, 38, 37}_{lint, type} py{311, 310, 39, 38, 37} skip_missing_interpreters = true @@ -18,29 +15,9 @@ pass_env = commands = pytest -s -v {posargs} -[testenv:dev] -desciption = Development environment -basepython = python3.10 -usedevelop = True - -[testenv:py{311, 310, 39, 38, 37}_lint] -description = Check the code style -deps = - flake8 -commands = - python3 -m flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - python3 -m flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - -[testenv:py{311, 310, 39, 38, 37}_type] -description = Run type checking -deps = - {[testenv]deps} - mypy==0.982 -commands = - python3 -m mypy quantus - [testenv:coverage] description = Run the tests with coverage +base_python = py310 deps = {[testenv]deps} coverage @@ -50,6 +27,7 @@ commands = [testenv:build] description = Build environment +base_python = py310 deps = . build @@ -58,10 +36,32 @@ commands = python3 -m build . python3 -m twine check ./dist/* --strict +[testenv:lint] +description = Check the code style +deps = + flake8 +commands = + python3 -m flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + python3 -m flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + +[testenv:type] +description = Run type checking +base_python = py310 +deps = + {[testenv]deps} + mypy==0.982 +commands = + python3 -m mypy quantus + +[testenv:dev] +base_python = py310 +package = editable +desciption = Development environment + [gh] python = - 3.7 = py37, py37_{lint, type} - 3.8 = py38, py38_{lint, type} - 3.9 = py39, py39_{lint, type} - 3.10 = py310, py310_{lint, type}, coverage - 3.11 = py311, py311_{lint, type} \ No newline at end of file + 3.7 = py37 + 3.8 = py38 + 3.9 = py39 + 3.10 = py310 + 3.11 = py311 From 87bd27baf72dacd60cad25f45529c0bb4035d6f9 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 15 Jun 2023 13:04:44 +0200 Subject: [PATCH 07/34] * --- scripts/github_release.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/github_release.sh b/scripts/github_release.sh index 6da22e122..1b901df00 100755 --- a/scripts/github_release.sh +++ b/scripts/github_release.sh @@ -14,8 +14,11 @@ TAG=$1 echo "TAG=${TAG}" # Update main ref's and switch to main's HEAD. git fetch --atomic --verbose && git checkout main +# Clean old artifacts. +rm -f -R build # Build wheel. -tox run -e build +python3 -m pip install tox +python3 -m tox tox run -e build # Tag release. git tag "$TAG" git push --follow-tags From 2d1c0dbda29369d28fb97abb39b21472d548e2ed Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 15 Jun 2023 13:05:29 +0200 Subject: [PATCH 08/34] * --- scripts/github_release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/github_release.sh b/scripts/github_release.sh index 1b901df00..cd89b235c 100755 --- a/scripts/github_release.sh +++ b/scripts/github_release.sh @@ -18,7 +18,7 @@ git fetch --atomic --verbose && git checkout main rm -f -R build # Build wheel. python3 -m pip install tox -python3 -m tox tox run -e build +python3 -m tox run -e build # Tag release. git tag "$TAG" git push --follow-tags From d9946ad8e65d8e2380be671e1e6a910cb76a3bc4 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 15 Jun 2023 13:06:01 +0200 Subject: [PATCH 09/34] * --- scripts/github_release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/github_release.sh b/scripts/github_release.sh index cd89b235c..b53efc269 100755 --- a/scripts/github_release.sh +++ b/scripts/github_release.sh @@ -13,7 +13,7 @@ fi TAG=$1 echo "TAG=${TAG}" # Update main ref's and switch to main's HEAD. -git fetch --atomic --verbose && git checkout main +# git fetch --atomic --verbose && git checkout main # Clean old artifacts. rm -f -R build # Build wheel. From 971a21f0c8d49fa3e7c3e4039558c7ac54a26fa6 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 15 Jun 2023 13:09:38 +0200 Subject: [PATCH 10/34] * --- scripts/github_release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/github_release.sh b/scripts/github_release.sh index b53efc269..cd89b235c 100755 --- a/scripts/github_release.sh +++ b/scripts/github_release.sh @@ -13,7 +13,7 @@ fi TAG=$1 echo "TAG=${TAG}" # Update main ref's and switch to main's HEAD. -# git fetch --atomic --verbose && git checkout main +git fetch --atomic --verbose && git checkout main # Clean old artifacts. rm -f -R build # Build wheel. From ce088a1598347c675a87b0a594b098500973cb6a Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 15 Jun 2023 13:13:19 +0200 Subject: [PATCH 11/34] * --- scripts/github_release.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/github_release.sh b/scripts/github_release.sh index cd89b235c..26ad9f8a4 100755 --- a/scripts/github_release.sh +++ b/scripts/github_release.sh @@ -10,6 +10,11 @@ echo -e "\033[0;32mUNCHANGED\033[0m" if [ $# -eq 0 ]; then echo -e "Must provide tag as positional argument" fi +if ! command -v gh &> /dev/null +then + echo "GitHub CLI not installed." + exit +fi TAG=$1 echo "TAG=${TAG}" # Update main ref's and switch to main's HEAD. From 08e8f25a094d1c6f3933a1d00215bdf6854a0f94 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 15 Jun 2023 13:14:05 +0200 Subject: [PATCH 12/34] * --- scripts/github_release.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/github_release.sh b/scripts/github_release.sh index 26ad9f8a4..f51ce7546 100755 --- a/scripts/github_release.sh +++ b/scripts/github_release.sh @@ -10,10 +10,9 @@ echo -e "\033[0;32mUNCHANGED\033[0m" if [ $# -eq 0 ]; then echo -e "Must provide tag as positional argument" fi -if ! command -v gh &> /dev/null +if ! command -v ghs &> /dev/null then - echo "GitHub CLI not installed." - exit + echo -e "GitHub CLI not installed." fi TAG=$1 echo "TAG=${TAG}" From 0bb7dd8193420b73602f31b0edce85918dc2bad5 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 15 Jun 2023 13:15:04 +0200 Subject: [PATCH 13/34] * --- scripts/github_release.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/github_release.sh b/scripts/github_release.sh index f51ce7546..5420c07ac 100755 --- a/scripts/github_release.sh +++ b/scripts/github_release.sh @@ -12,7 +12,8 @@ if [ $# -eq 0 ]; then fi if ! command -v ghs &> /dev/null then - echo -e "GitHub CLI not installed." + echo -e "\033[0;31m GitHub CLI not installed." + exit fi TAG=$1 echo "TAG=${TAG}" From 17796f726e2937083208bfdc2f67d12fe740210b Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 15 Jun 2023 13:17:14 +0200 Subject: [PATCH 14/34] * --- scripts/github_release.sh | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/scripts/github_release.sh b/scripts/github_release.sh index 5420c07ac..108b49db5 100755 --- a/scripts/github_release.sh +++ b/scripts/github_release.sh @@ -6,17 +6,21 @@ trap 'echo -e "\033[0;31mCHANGED\033[0m"' ERR git diff-index --quiet HEAD -- trap - ERR echo -e "\033[0;32mUNCHANGED\033[0m" + +echo -n "Looking for GitHub CLI... " +if ! command -v ghs &>/dev/null; then + echo -e "\033[0;31m GitHub CLI not installed.\033[0m" + exit +else + echo -e "\033[0;32OK\033[0m" +fi # Check provided 1 positional argument if [ $# -eq 0 ]; then - echo -e "Must provide tag as positional argument" -fi -if ! command -v ghs &> /dev/null -then - echo -e "\033[0;31m GitHub CLI not installed." - exit + echo -e "\033[0;31m Must provide tag as positional argument\033[0m" + exit fi TAG=$1 -echo "TAG=${TAG}" +echo -n "TAG=${TAG}" # Update main ref's and switch to main's HEAD. git fetch --atomic --verbose && git checkout main # Clean old artifacts. From 38f9528be70a9a292e99c795998aabf70ff1b90e Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 15 Jun 2023 13:17:27 +0200 Subject: [PATCH 15/34] * --- scripts/github_release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/github_release.sh b/scripts/github_release.sh index 108b49db5..0578de77b 100755 --- a/scripts/github_release.sh +++ b/scripts/github_release.sh @@ -8,7 +8,7 @@ trap - ERR echo -e "\033[0;32mUNCHANGED\033[0m" echo -n "Looking for GitHub CLI... " -if ! command -v ghs &>/dev/null; then +if ! command -v gh &>/dev/null; then echo -e "\033[0;31m GitHub CLI not installed.\033[0m" exit else From 302c2273042d994f231286ee48cd2b12a6ba015f Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 15 Jun 2023 13:17:44 +0200 Subject: [PATCH 16/34] * --- scripts/github_release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/github_release.sh b/scripts/github_release.sh index 0578de77b..28155949a 100755 --- a/scripts/github_release.sh +++ b/scripts/github_release.sh @@ -12,7 +12,7 @@ if ! command -v gh &>/dev/null; then echo -e "\033[0;31m GitHub CLI not installed.\033[0m" exit else - echo -e "\033[0;32OK\033[0m" + echo -e "\033[0;32mOK\033[0m" fi # Check provided 1 positional argument if [ $# -eq 0 ]; then From e9c686bb61524aab8fbebba960bfe8e613e82d76 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 15 Jun 2023 13:26:51 +0200 Subject: [PATCH 17/34] * --- .github/workflows/lint.yml | 6 +----- .github/workflows/pre-release.yml | 4 ++-- .github/workflows/python-package.yml | 2 ++ 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 399f113fd..cd43c2c47 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,10 +12,6 @@ jobs: lint: runs-on: ubuntu-latest name: Lint - strategy: - fail-fast: false - matrix: - python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] steps: - name: Check out source repository uses: actions/checkout@v3 @@ -23,7 +19,7 @@ jobs: uses: actions/setup-python@v4 with: cache: 'pip' - python-version: ${{ matrix.python-version }} + python-version: "3.10" - name: Install tox-gh run: pip install tox-gh - name: Run flake8 diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml index 902a1f8d8..b2f7b5f33 100644 --- a/.github/workflows/pre-release.yml +++ b/.github/workflows/pre-release.yml @@ -1,8 +1,8 @@ name: Python package on: - schedule: - - cron: '* 8 * * *' + #schedule: + # - cron: '* 8 * * *' workflow_dispatch: diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 377e4adab..285220503 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -1,6 +1,8 @@ name: Python package on: + schedule: + - cron: '* 8 * * *' push: branches: - main From 90a1282ebb8a85ec47769b75d74b548f9d05e549 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 15 Jun 2023 13:49:08 +0200 Subject: [PATCH 18/34] * --- pyproject.toml | 3 ++- quantus/__init__.py | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 473a883b0..3143478aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,8 @@ name = "quantus" # Required # # For a discussion on single-sourcing the version, see # https://packaging.python.org/guides/single-sourcing-package-version/ -version = "0.4.1" # Required +# version = "0.4.1" # Required +dynamic = ["version"] # This is a one-line description or tagline of what your project does. This # corresponds to the "Summary" metadata field: diff --git a/quantus/__init__.py b/quantus/__init__.py index c2e598fef..6c84d1beb 100644 --- a/quantus/__init__.py +++ b/quantus/__init__.py @@ -3,6 +3,12 @@ # Quantus is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. # You should have received a copy of the GNU Lesser General Public License along with Quantus. If not, see . # Quantus project URL: . +import subprocess + +commit_sha = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip().decode('utf-8') +version = subprocess.check_output(['git', 'describe', '--tags', '--always', '--dirty=-pre', commit_sha]).strip().decode('utf-8') + +__version__ = version # Expose quantus.evaluate to the user. from quantus.evaluation import evaluate From 0c0cdf67d0317798ed10431330d476914611b534 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 15 Jun 2023 14:00:39 +0200 Subject: [PATCH 19/34] * --- .github/workflows/pre-release.yml | 27 ---------------- .github/workflows/python-package.yml | 3 +- .github/workflows/snapshot-release.yml | 45 ++++++++++++++++++++++++++ quantus/__init__.py | 17 +++++++--- tox.ini | 2 ++ 5 files changed, 62 insertions(+), 32 deletions(-) delete mode 100644 .github/workflows/pre-release.yml create mode 100644 .github/workflows/snapshot-release.yml diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml deleted file mode 100644 index b2f7b5f33..000000000 --- a/.github/workflows/pre-release.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Python package - -on: - #schedule: - # - cron: '* 8 * * *' - workflow_dispatch: - - -jobs: - build: - runs-on: ubuntu-latest - name: Pre Release - steps: - - uses: actions/checkout@v3 - - name: Setup python - uses: actions/setup-python@v4 - with: - cache: 'pip' - python-version: "3.10" - - name: Install tox-gh - run: pip install tox-gh - - name: Setup test environment - run: tox run --notest - - name: Test with PyTest - run: tox run - - name: Test building package - run: tox run -e build \ No newline at end of file diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 285220503..17ab103e2 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -1,8 +1,9 @@ name: Python package on: + # Run tests every night, to catch potentially breaking changes in dependencies. schedule: - - cron: '* 8 * * *' + - cron: '0 0 * * *' push: branches: - main diff --git a/.github/workflows/snapshot-release.yml b/.github/workflows/snapshot-release.yml new file mode 100644 index 000000000..5adb38a59 --- /dev/null +++ b/.github/workflows/snapshot-release.yml @@ -0,0 +1,45 @@ +name: Python package + +on: + # Publish snapshot release every night. + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + inputs: + index: + type: choice + description: Index to publish snapshot release to + options: + - https://test.pypi.org/simple/ + - https://pypi.org/simple/ + default: https://test.pypi.org/simple/ + + + +jobs: + build: + runs-on: ubuntu-latest + name: Pre Release + steps: + - uses: actions/checkout@v3 + - name: Setup python + uses: actions/setup-python@v4 + with: + cache: 'pip' + python-version: "3.10" + - name: Install tox-gh + run: pip install tox-gh + - name: Setup test environment + run: tox run --notest + - name: Test with PyTest + run: tox run + - name: Test building package + run: tox run -e build + - name: Publish package + uses: pypa/gh-action-pypi-publish/v1 + env: + SNAPSHOT_RELEASE: 1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} + repository-url: ${{ github.event.inputs.index }} \ No newline at end of file diff --git a/quantus/__init__.py b/quantus/__init__.py index 6c84d1beb..a8b84f765 100644 --- a/quantus/__init__.py +++ b/quantus/__init__.py @@ -4,11 +4,20 @@ # You should have received a copy of the GNU Lesser General Public License along with Quantus. If not, see . # Quantus project URL: . import subprocess +import os -commit_sha = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip().decode('utf-8') -version = subprocess.check_output(['git', 'describe', '--tags', '--always', '--dirty=-pre', commit_sha]).strip().decode('utf-8') - -__version__ = version +if "SNAPSHOT_RELEASE" in os.environ: + commit_sha = ( + subprocess.check_output(["git", "rev-parse", "HEAD"]).strip().decode("utf-8") + ) + version = ( + subprocess.check_output(["git", "describe", "--tags", "--always", commit_sha]) + .strip() + .decode("utf-8") + ) + __version__ = version +else: + __version__ = "0.4.1" # Expose quantus.evaluate to the user. from quantus.evaluation import evaluate diff --git a/tox.ini b/tox.ini index 51882be30..462251b7f 100644 --- a/tox.ini +++ b/tox.ini @@ -32,6 +32,8 @@ deps = . build twine +pass_env = + SNAPSHOT_RELEASE commands = python3 -m build . python3 -m twine check ./dist/* --strict From 97750c3ffab1d0ca79e45e199b8b0d653be04f57 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 15 Jun 2023 14:02:37 +0200 Subject: [PATCH 20/34] * --- .github/workflows/snapshot-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/snapshot-release.yml b/.github/workflows/snapshot-release.yml index 5adb38a59..abff4211c 100644 --- a/.github/workflows/snapshot-release.yml +++ b/.github/workflows/snapshot-release.yml @@ -36,7 +36,7 @@ jobs: - name: Test building package run: tox run -e build - name: Publish package - uses: pypa/gh-action-pypi-publish/v1 + uses: pypa/gh-action-pypi-publish@release/v1 env: SNAPSHOT_RELEASE: 1 with: From 527ec9d8d6c637fce9ee411431a3c7c46b62eda3 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 15 Jun 2023 14:27:06 +0200 Subject: [PATCH 21/34] * --- .../{snapshot-release.yml => dev-release.yml} | 23 +++++++++++-------- quantus/__init__.py | 13 +---------- tox.ini | 2 -- 3 files changed, 14 insertions(+), 24 deletions(-) rename .github/workflows/{snapshot-release.yml => dev-release.yml} (61%) diff --git a/.github/workflows/snapshot-release.yml b/.github/workflows/dev-release.yml similarity index 61% rename from .github/workflows/snapshot-release.yml rename to .github/workflows/dev-release.yml index abff4211c..04c1a1fa8 100644 --- a/.github/workflows/snapshot-release.yml +++ b/.github/workflows/dev-release.yml @@ -1,18 +1,15 @@ name: Python package on: - # Publish snapshot release every night. - schedule: - - cron: '0 0 * * *' workflow_dispatch: inputs: index: type: choice description: Index to publish snapshot release to options: - - https://test.pypi.org/simple/ - - https://pypi.org/simple/ - default: https://test.pypi.org/simple/ + - Test PyPi + - PyPi + default: Test PyPi @@ -35,11 +32,17 @@ jobs: run: tox run - name: Test building package run: tox run -e build - - name: Publish package + - name: Publish to Test PyPi + if: ${{github.event.inputs.index == 'Test PyPi'}} + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + repository-url: https://test.pypi.org/simple/ + - name: Publish to PyPi + if: ${{github.event.inputs.index == 'PyPi'}} uses: pypa/gh-action-pypi-publish@release/v1 - env: - SNAPSHOT_RELEASE: 1 with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} - repository-url: ${{ github.event.inputs.index }} \ No newline at end of file + repository-url: https://pypi.org/simple/ \ No newline at end of file diff --git a/quantus/__init__.py b/quantus/__init__.py index a8b84f765..f7fcdb772 100644 --- a/quantus/__init__.py +++ b/quantus/__init__.py @@ -6,18 +6,7 @@ import subprocess import os -if "SNAPSHOT_RELEASE" in os.environ: - commit_sha = ( - subprocess.check_output(["git", "rev-parse", "HEAD"]).strip().decode("utf-8") - ) - version = ( - subprocess.check_output(["git", "describe", "--tags", "--always", commit_sha]) - .strip() - .decode("utf-8") - ) - __version__ = version -else: - __version__ = "0.4.1" +__version__ = "0.4.1.dev1" # Expose quantus.evaluate to the user. from quantus.evaluation import evaluate diff --git a/tox.ini b/tox.ini index 462251b7f..51882be30 100644 --- a/tox.ini +++ b/tox.ini @@ -32,8 +32,6 @@ deps = . build twine -pass_env = - SNAPSHOT_RELEASE commands = python3 -m build . python3 -m twine check ./dist/* --strict From 5deeb7c47a3ceffe87d60ca0bf53872439034186 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 15 Jun 2023 14:30:31 +0200 Subject: [PATCH 22/34] * --- .github/workflows/lint.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index cd43c2c47..655e24b12 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,9 +1,6 @@ name: Lint on: - push: - branches: - - main pull_request: workflow_dispatch: From f22de7117cb162a8904420ad14ce168ae8f981e8 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 15 Jun 2023 14:32:20 +0200 Subject: [PATCH 23/34] update CONTRIBUTING.md --- CONTRIBUTING.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8efb4f5a5..9f95eb701 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,10 +74,8 @@ python3 -m tox list python3 -m tox run # Or, e.g., you can run only tests for pytho3.8 with: python3 -m tox run -e py38 -# Or, e.g., run type checking for python 3.10 with: -python3 -m tox run -e py310_type -# Or coverage using python 3.10 -python3 -m tox run -e py310_coverage +# Or, e.g., run type checking with: +python3 -m tox run -e type # If you want to pass CLI arguments to pytest, add them after --, e.g., you might want to split execution between 4 cpu cores, # show verbose logging, and run only robustness tests. python3 -m tox run py310 -- -n 4 -s -v -m "robustness" From 060d5966c519bd991878f2f43d6a8d295b3d2c93 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 15 Jun 2023 14:35:03 +0200 Subject: [PATCH 24/34] update CONTRIBUTING.md --- pyproject.toml | 3 +-- quantus/__init__.py | 4 ---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3143478aa..948e5a4c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,8 +17,7 @@ name = "quantus" # Required # # For a discussion on single-sourcing the version, see # https://packaging.python.org/guides/single-sourcing-package-version/ -# version = "0.4.1" # Required -dynamic = ["version"] +version = "0.4.1.dev1" # Required # This is a one-line description or tagline of what your project does. This # corresponds to the "Summary" metadata field: diff --git a/quantus/__init__.py b/quantus/__init__.py index f7fcdb772..c2e598fef 100644 --- a/quantus/__init__.py +++ b/quantus/__init__.py @@ -3,10 +3,6 @@ # Quantus is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. # You should have received a copy of the GNU Lesser General Public License along with Quantus. If not, see . # Quantus project URL: . -import subprocess -import os - -__version__ = "0.4.1.dev1" # Expose quantus.evaluate to the user. from quantus.evaluation import evaluate From 3bf8686c9938c810e78cae05871ec152cd130695 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 15 Jun 2023 14:54:40 +0200 Subject: [PATCH 25/34] update CONTRIBUTING.md --- pyproject.toml | 2 +- scripts/github_release.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 948e5a4c4..59ae99c6a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,7 +82,7 @@ classifiers = [# Optional "Intended Audience :: Developers", # Pick your license as you wish "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", - "Topic :: Scientific :: Explainable Artificial Intelligence", + # "Topic :: Scientific :: Explainable Artificial Intelligence", "Programming Language :: Python :: 3 :: Only", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Scientific/Engineering :: Artificial Intelligence", diff --git a/scripts/github_release.sh b/scripts/github_release.sh index 28155949a..3e9896726 100755 --- a/scripts/github_release.sh +++ b/scripts/github_release.sh @@ -24,7 +24,7 @@ echo -n "TAG=${TAG}" # Update main ref's and switch to main's HEAD. git fetch --atomic --verbose && git checkout main # Clean old artifacts. -rm -f -R build +rm -f -R dist # Build wheel. python3 -m pip install tox python3 -m tox run -e build From 39f0a4095b0cfd40a68a47297efd09a7ce64bee0 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Fri, 16 Jun 2023 18:01:20 +0200 Subject: [PATCH 26/34] update CONTRIBUTING.md --- README.md | 14 ++++---------- docs/source/getting_started/installation.md | 13 +++---------- pyproject.toml | 10 +++++----- 3 files changed, 12 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index bc8c1de0b..82cd7b0ee 100644 --- a/README.md +++ b/README.md @@ -184,15 +184,6 @@ For TensorFlow, please run: pip install "quantus[tensorflow]" ``` -Alternatively, you can simply install Quantus with [requirements.txt](https://github.com/understandable-machine-intelligence-lab/Quantus/blob/main/requirements.txt). -Note that this installation requires that either [PyTorch](https://pytorch.org/) or [TensorFlow](https://www.TensorFlow.org) are already installed on your machine. - -```setup -pip install -r requirements.txt -``` - -For a more in-depth guide on how to install Quantus, please read more [here](https://quantus.readthedocs.io/en/latest/getting_started/installation.html). This includes instructions for how to install a desired deep learning framework such as PyTorch or TensorFlow together with Quantus. - ### Package requirements The package requirements are as follows: @@ -201,7 +192,10 @@ python>=3.7.0 torch>=1.11.0 tensorflow>=2.5.0 ``` -Please note that the exact [PyTorch](https://pytorch.org/) and/ or [TensorFlow](https://www.TensorFlow.org) versions to be installed depends on your Python version (3.7-3.11) and platform (`darwin`, `linux`, …). See `requirements_test.txt` to retrieve the exact versions of [PyTorch](https://pytorch.org/) and/ or [TensorFlow](https://www.TensorFlow.org). + +Please note that the exact [PyTorch](https://pytorch.org/) and/ or [TensorFlow](https://www.TensorFlow.org) versions +to be installed depends on your Python version (3.7-3.11) and platform (`darwin`, `linux`, …). +See `[project.optional-dependencies]` section in the `pyproject.toml` file. ## Getting started diff --git a/docs/source/getting_started/installation.md b/docs/source/getting_started/installation.md index 63ffb22db..7af01eef1 100644 --- a/docs/source/getting_started/installation.md +++ b/docs/source/getting_started/installation.md @@ -22,15 +22,6 @@ For TensorFlow, please run: pip install "quantus[tensorflow]" ``` -### Installing via requirements.txt - -Alternatively, you can simply install Quantus from the [requirements.txt](https://github.com/understandable-machine-intelligence-lab/Quantus/blob/main/requirements.txt). -Note that this installation requires that either [PyTorch](https://pytorch.org/) or [TensorFlow](https://www.TensorFlow.org) are already installed on your machine. - -```setup -pip install -r requirements.txt -``` - ### Installing additional XAI Library support (PyPI only) Most evaluation metrics in Quantus allow for a choice of either providing pre-computed explanations directly as an input, or instead making use of several wrappers implemented in `quantus.explain` around common explainability libraries. The @@ -88,4 +79,6 @@ python>=3.7.0 torch>=1.11.0 tensorflow>=2.5.0 ``` -Please note that the exact [PyTorch](https://pytorch.org/) and/ or [TensorFlow](https://www.TensorFlow.org) versions to be installed depends on your Python version (3.7-3.11) and platform (`darwin`, `linux`, …). See `requirements_test.txt` to retrieve the exact versions of [PyTorch](https://pytorch.org/) and/ or [TensorFlow](https://www.TensorFlow.org). \ No newline at end of file +Please note that the exact [PyTorch](https://pytorch.org/) and/ or [TensorFlow](https://www.TensorFlow.org) versions +to be installed depends on your Python version (3.7-3.11) and platform (`darwin`, `linux`, …). +See `[project.optional-dependencies]` section in the `pyproject.toml` file. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 59ae99c6a..96928fb2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,9 +3,9 @@ # package, this name will be registered for you. It will determine how # users can install this project, e.g.: # -# $ pip install sampleproject +# $ pip install quantus # -# And where it will live on PyPI: https://pypi.org/project/sampleproject/ +# And where it will live on PyPI: https://pypi.org/project/quantus/ # # There are some restrictions on what makes a valid project name # specification here: @@ -108,7 +108,7 @@ dependencies = [ # dependencies). Users will be able to install these using the "extras" # syntax, for example: # -# $ pip install sampleproject[dev] +# $ pip install quantus[dev] # # Similar to `dependencies` above, these must be valid existing # projects. @@ -182,10 +182,10 @@ full = [ # what's used to render the link text on PyPI. [project.urls] # Optional "Homepage" = "https://github.com/understandable-machine-intelligence-lab/Quantus" -#"Bug Reports" = "https://github.com/pypa/sampleproject/issues" +#"Bug Reports" = "https://github.com/pypa/quantus/issues" #"Funding" = "https://donate.pypi.org" #"Say Thanks!" = "http://saythanks.io/to/example" -#"Source" = "https://github.com/pypa/sampleproject/" +"Source" = "https://github.com/understandable-machine-intelligence-lab/Quantus" # The following would provide a command line executable called `sample` # which executes the function `main` from this package when invoked. From f9bc608c3a50dbe852683dd25e094a2152b593dd Mon Sep 17 00:00:00 2001 From: aaarrti Date: Fri, 16 Jun 2023 18:19:13 +0200 Subject: [PATCH 27/34] code review comments --- pyproject.toml | 120 +++++++------------------------------------------ 1 file changed, 15 insertions(+), 105 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 96928fb2c..fe94f6092 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,99 +1,33 @@ +# A complete reference of pyproject.toml format can be found on: https://flit.pypa.io/en/stable/pyproject_toml.html [project] -# This is the name of your project. The first time you publish this -# package, this name will be registered for you. It will determine how -# users can install this project, e.g.: -# -# $ pip install quantus -# -# And where it will live on PyPI: https://pypi.org/project/quantus/ -# -# There are some restrictions on what makes a valid project name -# specification here: -# https://packaging.python.org/specifications/core-metadata/#name -name = "quantus" # Required - +name = "quantus" # Versions should comply with PEP 440: # https://www.python.org/dev/peps/pep-0440/ -# -# For a discussion on single-sourcing the version, see -# https://packaging.python.org/guides/single-sourcing-package-version/ -version = "0.4.1.dev1" # Required - -# This is a one-line description or tagline of what your project does. This -# corresponds to the "Summary" metadata field: -# https://packaging.python.org/specifications/core-metadata/#summary -description = "A metrics toolkit to evaluate neural network explanations." # Optional - -# This is an optional longer description of your project that represents -# the body of text which users will see when they visit PyPI. -# -# Often, this is the same as your README, so you can just read it in from -# that file directly (as we have already done above) -# -# This field corresponds to the "Description" metadata field: -# https://packaging.python.org/specifications/core-metadata/#description-optional -readme = "README.md" # Optional - -# Specify which Python versions you support. In contrast to the -# 'Programming Language' classifiers above, 'pip install' will check this -# and refuse to install the project if the version does not match. See -# https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires +version = "0.4.1.dev1" +description = "A metrics toolkit to evaluate neural network explanations." +readme = "README.md" requires-python = ">=3.7" - -# This is either text indicating the license for the distribution, or a file -# that contains the license -# https://packaging.python.org/en/latest/specifications/core-metadata/#license license = { file = "LICENSE" } - -# This field adds keywords for your project which will appear on the -# project page. What does your project relate to? -# -# Note that this is a list of additional keywords, separated -# by commas, to be used to assist searching for the distribution in a -# larger catalog. -keywords = ["explainable ai", "xai", "machine learning", "deep learning"] # Optional - -# This should be your name or the name of the organization who originally -# authored the project, and a valid email address corresponding to the name -# listed. +keywords = ["explainable ai", "xai", "machine learning", "deep learning"] authors = [ { name = "Anna Hedstrom", email = "hedstroem.anna@gmail.com" } # Optional ] - -# This should be your name or the names of the organization who currently -# maintains the project, and a valid email address corresponding to the name -# listed. maintainers = [ { name = "Anna Hedstrom", email = "hedstroem.anna@gmail.com" } # Optional ] - -# Classifiers help users find your project by categorizing it. -# # For a list of valid classifiers, see https://pypi.org/classifiers/ -classifiers = [# Optional - # How mature is this project? Common values are - # 3 - Alpha - # 4 - Beta - # 5 - Production/Stable - "Development Status :: 5 - Production/Stable", - # Indicate who your project is intended for +classifiers = [ + "Development Status :: 4 - Beta", "Intended Audience :: Education", "Intended Audience :: Science/Research", "Intended Audience :: Developers", - # Pick your license as you wish "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + # Explainable Artificial Intelligence is not in the list of valid classifiers. # "Topic :: Scientific :: Explainable Artificial Intelligence", "Programming Language :: Python :: 3 :: Only", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Scientific/Engineering :: Artificial Intelligence", ] - -# This field lists other packages that your project depends on to run. -# Any package you put here will be installed by pip when your project is -# installed, so they must be valid existing projects. -# -# For an analysis of this field vs pip's requirements files see: -# https://packaging.python.org/discussions/install-requires-vs-requirements/ dependencies = [ "numpy>=1.19.5", "opencv-python>=4.5.5.62", @@ -104,15 +38,12 @@ dependencies = [ "matplotlib>=3.3.4" ] -# List additional groups of dependencies here (e.g. development -# dependencies). Users will be able to install these using the "extras" -# syntax, for example: +# List additional groups of dependencies here (e.g. development dependencies). +# Users will be able to install these using the "extras" syntax, for example: # -# $ pip install quantus[dev] +# $ pip install quantus[tensorflow] # -# Similar to `dependencies` above, these must be valid existing -# projects. -[project.optional-dependencies] # Optional +[project.optional-dependencies] tests = [ "captum>=0.6.0", "coverage>=7.2.3", @@ -169,31 +100,10 @@ zennit = [ full = [ "quantus[captum,tf_explain,zennit]" ] - -# List URLs that are relevant to your project -# -# This field corresponds to the "Project-URL" and "Home-Page" metadata fields: -# https://packaging.python.org/specifications/core-metadata/#project-url-multiple-use -# https://packaging.python.org/specifications/core-metadata/#home-page-optional -# -# Examples listed include a pattern for specifying where the package tracks -# issues, where the source is hosted, where to say thanks to the package -# maintainers, and where to support the project financially. The key is -# what's used to render the link text on PyPI. -[project.urls] # Optional -"Homepage" = "https://github.com/understandable-machine-intelligence-lab/Quantus" -#"Bug Reports" = "https://github.com/pypa/quantus/issues" -#"Funding" = "https://donate.pypi.org" -#"Say Thanks!" = "http://saythanks.io/to/example" +[project.urls] +"Documentation" = "https://quantus.readthedocs.io/en/latest/" "Source" = "https://github.com/understandable-machine-intelligence-lab/Quantus" -# The following would provide a command line executable called `sample` -# which executes the function `main` from this package when invoked. -#[project.scripts] # Optional -#sample = "sample:main" - [build-system] -# These are the assumed default build requirements from pip: -# https://pip.pypa.io/en/stable/reference/pip/#pep-517-and-518-support requires = ["flit-core >= 3.4"] build-backend = "flit_core.buildapi" \ No newline at end of file From fa6d8a94663cd1b0e38b46b4d746fca09d8116d7 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Fri, 16 Jun 2023 18:35:17 +0200 Subject: [PATCH 28/34] code review comments --- scripts/github_release.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/github_release.sh b/scripts/github_release.sh index 3e9896726..892bf5f8b 100755 --- a/scripts/github_release.sh +++ b/scripts/github_release.sh @@ -23,8 +23,6 @@ TAG=$1 echo -n "TAG=${TAG}" # Update main ref's and switch to main's HEAD. git fetch --atomic --verbose && git checkout main -# Clean old artifacts. -rm -f -R dist # Build wheel. python3 -m pip install tox python3 -m tox run -e build @@ -32,4 +30,4 @@ python3 -m tox run -e build git tag "$TAG" git push --follow-tags # Create GitHub release draft. -gh release create "$TAG" ./dist/* --draft --latest +gh release create "$TAG" "./dist/*${TAG}*" --draft --latest From 246cac786df44e20cd4c0858873f85e95f5139f5 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Wed, 21 Jun 2023 17:10:41 +0200 Subject: [PATCH 29/34] remove dev version --- .github/workflows/dev-release.yml | 48 ------------------------------- pyproject.toml | 2 +- 2 files changed, 1 insertion(+), 49 deletions(-) delete mode 100644 .github/workflows/dev-release.yml diff --git a/.github/workflows/dev-release.yml b/.github/workflows/dev-release.yml deleted file mode 100644 index 04c1a1fa8..000000000 --- a/.github/workflows/dev-release.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Python package - -on: - workflow_dispatch: - inputs: - index: - type: choice - description: Index to publish snapshot release to - options: - - Test PyPi - - PyPi - default: Test PyPi - - - -jobs: - build: - runs-on: ubuntu-latest - name: Pre Release - steps: - - uses: actions/checkout@v3 - - name: Setup python - uses: actions/setup-python@v4 - with: - cache: 'pip' - python-version: "3.10" - - name: Install tox-gh - run: pip install tox-gh - - name: Setup test environment - run: tox run --notest - - name: Test with PyTest - run: tox run - - name: Test building package - run: tox run -e build - - name: Publish to Test PyPi - if: ${{github.event.inputs.index == 'Test PyPi'}} - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.TEST_PYPI_API_TOKEN }} - repository-url: https://test.pypi.org/simple/ - - name: Publish to PyPi - if: ${{github.event.inputs.index == 'PyPi'}} - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} - repository-url: https://pypi.org/simple/ \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index fe94f6092..f05728f09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "quantus" # Versions should comply with PEP 440: # https://www.python.org/dev/peps/pep-0440/ -version = "0.4.1.dev1" +version = "0.4.1" description = "A metrics toolkit to evaluate neural network explanations." readme = "README.md" requires-python = ">=3.7" From 534e9073bf096752f0afc55fae5d1fb6e44ea587 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 22 Jun 2023 13:32:35 +0200 Subject: [PATCH 30/34] readme improvements --- CONTRIBUTING.md | 40 ++++++++++++++++++++-------------------- pyproject.toml | 2 -- tests/README.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ tox.ini | 3 ++- 4 files changed, 66 insertions(+), 23 deletions(-) create mode 100644 tests/README.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9f95eb701..9b46a3c56 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,8 +50,8 @@ git clone https://github.com/understandable-machine-intelligence-lab/Quantus.git cd quantus # Tox will provision dev environment with editable installation for you. python3 -m pip install tox -python3 -m tox run -e dev -source .tox/dev/bin/activate +python3 -m tox devenv +source venv/bin/activate ``` ### Branching @@ -66,23 +66,23 @@ We use [flake8](https://pypi.org/project/flake8/) for quick style checks and [bl ### Unit Tests Tests are written using [pytest](https://github.com/pytest-dev/pytest) and executed together with [codecov](https://github.com/codecov/codecov-action) for coverage reports. -We use [tox](https://tox.wiki/en/latest/) for test automation. -```bash -# To list all test environments, run: -python3 -m tox list -# To execute all test environments run: +We use [tox](https://tox.wiki/en/latest/) for test automation. For complete list of CLI commands, please refer to [tox - CLI interface](https://tox.wiki/en/latest/cli_interface.html). +To perform the tests for all supported python versions, linting and type checking execute the following CLI command: +```shell python3 -m tox run -# Or, e.g., you can run only tests for pytho3.8 with: -python3 -m tox run -e py38 -# Or, e.g., run type checking with: -python3 -m tox run -e type -# If you want to pass CLI arguments to pytest, add them after --, e.g., you might want to split execution between 4 cpu cores, -# show verbose logging, and run only robustness tests. -python3 -m tox run py310 -- -n 4 -s -v -m "robustness" -# You still can run tests using your active local interpreter by: -python -m pytest ``` +... alternatively, to get additionaly coverage details, run: +```bash +python3 -m tox run -e coverage +``` + +It is possible to limit the scope of testing to specific sections of the codebase, for example, only test the Faithfulness metrics using python3.10: +```bash +python3 -m tox run -e py310 -- -m faithfulness -s +``` +For a complete overview of the possible testing scopes, please refer to `pytest.ini`. + ### Documentation Make sure to add docstrings to every class, method and function that you add to the codebase. The docstring should include a description of all parameters and returns. Use the existing documentation as an example. @@ -101,16 +101,16 @@ black quantus/INSERT_YOUR_FILE_NAME.py ```bash flake8 quantus/INSERT_YOUR_FILE_NAME.py ``` -- Create tests for new functionality and add under `tests/` folder. -- If tests include a new category of `@pytest.mark` then add that category with description to `pytest.ini` +- Create a unit test for new functionality and add under `tests/` folder, add `@pytest.mark` with fitting category. +- If newly added test cases include a new category of `@pytest.mark` then add that category with description to `pytest.ini` - Make sure all unit tests pass for all supported python version by running: ```shell -python -m tox run +python3 -m tox run -e 'py3{7,8,9,10,11} ``` - Generally, every change should be covered with respective test-case, we aim at ~100% code coverage for Quantus. You can verify it by running: ```bash -python -m tox run coverage +python3 -m tox run -e coverage ``` ### Pull Requests diff --git a/pyproject.toml b/pyproject.toml index f05728f09..496cea34f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,8 +22,6 @@ classifiers = [ "Intended Audience :: Science/Research", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", - # Explainable Artificial Intelligence is not in the list of valid classifiers. - # "Topic :: Scientific :: Explainable Artificial Intelligence", "Programming Language :: Python :: 3 :: Only", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Scientific/Engineering :: Artificial Intelligence", diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..12d16a2c7 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,44 @@ +### How to run tests + +Run all tests for all supported python versions, coverage, linting and type-checking at once + +```shell +python3 -m tox run +``` + +To list all configured test environments, run + +```shell +python3 -m tox list +``` + +To run, e.g., only test for python3.8, run: + +```shell +python3 -m tox run -e py38 +``` + +If you need to provide additional CLI argument, they must follow after `--`, e.g., in this case, +we will split test execution between cpu cores using [pytest-xdist](https://github.com/pytest-dev/pytest-xdist): + +```shell +python3 -m tox run -e py310 -- -n auto +``` + +Run a subset of tests with e.g., localisation metrics (see available markers in the pytest.ini files): + +```shell +python3 -m tox run -- -m localisation -s +``` + +Run pytest with coverage: + +```shell +python3 -m tox run -e coverage +``` + +Run type checking using [mypy](https://github.com/python/mypy) + +```shell +python3 -m tox run -e type +``` \ No newline at end of file diff --git a/tox.ini b/tox.ini index 51882be30..745a8d6f9 100644 --- a/tox.ini +++ b/tox.ini @@ -54,9 +54,10 @@ commands = python3 -m mypy quantus [testenv:dev] +description = Development environment base_python = py310 package = editable -desciption = Development environment +commands = [gh] python = From e4a1d73f312334b5a1b8b27c1903759481965c48 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 22 Jun 2023 13:37:43 +0200 Subject: [PATCH 31/34] readme improvements --- CONTRIBUTING.md | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9b46a3c56..5f1672d31 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -65,22 +65,35 @@ We use [flake8](https://pypi.org/project/flake8/) for quick style checks and [bl ### Unit Tests -Tests are written using [pytest](https://github.com/pytest-dev/pytest) and executed together with [codecov](https://github.com/codecov/codecov-action) for coverage reports. -We use [tox](https://tox.wiki/en/latest/) for test automation. For complete list of CLI commands, please refer to [tox - CLI interface](https://tox.wiki/en/latest/cli_interface.html). -To perform the tests for all supported python versions, linting and type checking execute the following CLI command: +Tests are written using [pytest](https://github.com/pytest-dev/pytest) and executed together +with [codecov](https://github.com/codecov/codecov-action) for coverage reports. +We use [tox](https://tox.wiki/en/latest/) for test automation. For complete list of CLI commands, please refer +to [tox - CLI interface](https://tox.wiki/en/latest/cli_interface.html). +To perform the tests for all supported python versions execute the following CLI command: + ```shell python3 -m tox run ``` -... alternatively, to get additionaly coverage details, run: +... or to run all testing environments in parallel, execute: + +```shell +python3 -m tox run-parallel +``` + +... alternatively, to get additionally coverage details, run: + ```bash python3 -m tox run -e coverage ``` -It is possible to limit the scope of testing to specific sections of the codebase, for example, only test the Faithfulness metrics using python3.10: +It is possible to limit the scope of testing to specific sections of the codebase, for example, only test the +Faithfulness metrics using python3.10: + ```bash python3 -m tox run -e py310 -- -m faithfulness -s ``` + For a complete overview of the possible testing scopes, please refer to `pytest.ini`. ### Documentation @@ -94,21 +107,29 @@ Before creating a PR, double-check that the following tasks are completed: - Make sure that the latest version of the code from the `main` branch is merged into your working branch. - Run `black` to format source code: + ```bash black quantus/INSERT_YOUR_FILE_NAME.py ``` + - Run `flake8` for quick style checks, e.g.: + ```bash flake8 quantus/INSERT_YOUR_FILE_NAME.py ``` + - Create a unit test for new functionality and add under `tests/` folder, add `@pytest.mark` with fitting category. -- If newly added test cases include a new category of `@pytest.mark` then add that category with description to `pytest.ini` +- If newly added test cases include a new category of `@pytest.mark` then add that category with description + to `pytest.ini` - Make sure all unit tests pass for all supported python version by running: + ```shell -python3 -m tox run -e 'py3{7,8,9,10,11} +python3 -m tox run ``` -- Generally, every change should be covered with respective test-case, we aim at ~100% code coverage for Quantus. -You can verify it by running: + +- Generally, every change should be covered with respective test-case, we aim at ~100% code coverage in Quantus, you can + verify it by running: + ```bash python3 -m tox run -e coverage ``` From 21a544fda03cd828740172fed27532b3847419b0 Mon Sep 17 00:00:00 2001 From: aaarrti Date: Thu, 22 Jun 2023 13:38:49 +0200 Subject: [PATCH 32/34] readme improvements --- CONTRIBUTING.md | 6 ------ tests/README.md | 8 +++++++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5f1672d31..e4d37945f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -75,12 +75,6 @@ To perform the tests for all supported python versions execute the following CLI python3 -m tox run ``` -... or to run all testing environments in parallel, execute: - -```shell -python3 -m tox run-parallel -``` - ... alternatively, to get additionally coverage details, run: ```bash diff --git a/tests/README.md b/tests/README.md index 12d16a2c7..ed60d71f7 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,11 +1,17 @@ ### How to run tests -Run all tests for all supported python versions, coverage, linting and type-checking at once +Run all tests for all supported python versions, execute: ```shell python3 -m tox run ``` +... or to run all testing environments in parallel, execute: + +```shell +python3 -m tox run-parallel +``` + To list all configured test environments, run ```shell From 6055bf649011560c9ee951cfc5065c5432f6cad2 Mon Sep 17 00:00:00 2001 From: annahedstroem Date: Fri, 23 Jun 2023 11:23:18 +0200 Subject: [PATCH 33/34] tiny fixes and testing tox branch --- CONTRIBUTING.md | 18 ++++++---- docs/source/docs_dev/CONTRIBUTING.md | 54 ++++++++++++++++++++-------- pyproject.toml | 12 ++++--- 3 files changed, 57 insertions(+), 27 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e4d37945f..2ba5b3244 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,7 +48,10 @@ Make sure to install the latest version of Quantus from the main branch. ```bash git clone https://github.com/understandable-machine-intelligence-lab/Quantus.git cd quantus -# Tox will provision dev environment with editable installation for you. +``` + +Tox will provision dev environment with editable installation for you. +```bash python3 -m pip install tox python3 -m tox devenv source venv/bin/activate @@ -69,9 +72,10 @@ Tests are written using [pytest](https://github.com/pytest-dev/pytest) and execu with [codecov](https://github.com/codecov/codecov-action) for coverage reports. We use [tox](https://tox.wiki/en/latest/) for test automation. For complete list of CLI commands, please refer to [tox - CLI interface](https://tox.wiki/en/latest/cli_interface.html). -To perform the tests for all supported python versions execute the following CLI command: +To perform the tests for all supported python versions execute the following CLI command (a re-intall of tox is necessary): ```shell +python3 -m pip install tox python3 -m tox run ``` @@ -82,10 +86,10 @@ python3 -m tox run -e coverage ``` It is possible to limit the scope of testing to specific sections of the codebase, for example, only test the -Faithfulness metrics using python3.10: +Faithfulness metrics using python3.9 (make sure the python versions match in your environment): ```bash -python3 -m tox run -e py310 -- -m faithfulness -s +python3 -m tox run -e py39 -- -m faithfulness -s ``` For a complete overview of the possible testing scopes, please refer to `pytest.ini`. @@ -93,7 +97,6 @@ For a complete overview of the possible testing scopes, please refer to `pytest. ### Documentation Make sure to add docstrings to every class, method and function that you add to the codebase. The docstring should include a description of all parameters and returns. Use the existing documentation as an example. -TODO: Automatic docstring generation. ### Before You Create a Pull Request @@ -103,13 +106,13 @@ Before creating a PR, double-check that the following tasks are completed: - Run `black` to format source code: ```bash -black quantus/INSERT_YOUR_FILE_NAME.py +black quantus/*/INSERT_YOUR_FILE_NAME.py ``` - Run `flake8` for quick style checks, e.g.: ```bash -flake8 quantus/INSERT_YOUR_FILE_NAME.py +flake8 quantus/*/INSERT_YOUR_FILE_NAME.py ``` - Create a unit test for new functionality and add under `tests/` folder, add `@pytest.mark` with fitting category. @@ -154,6 +157,7 @@ See a more detailed description of those in [README](https://github.com/understa Identify which category your metric belongs to and create a Python file for your metric class in the respective folder in `quantus/metrics`. Add the metric to the `__init__.py` file in the respective folder. + ### Metric Class Every metric class inherits from the base `Metric` class: `quantus/metrics/base.py`. Importantly, Faithfulness and Robustness inherit not from the `Metric` class directly, but rather from its child `PerturbationMetric`. diff --git a/docs/source/docs_dev/CONTRIBUTING.md b/docs/source/docs_dev/CONTRIBUTING.md index 38096589f..e4d37945f 100644 --- a/docs/source/docs_dev/CONTRIBUTING.md +++ b/docs/source/docs_dev/CONTRIBUTING.md @@ -48,8 +48,10 @@ Make sure to install the latest version of Quantus from the main branch. ```bash git clone https://github.com/understandable-machine-intelligence-lab/Quantus.git cd quantus -pip install -r requirements_test.txt -pip install -e . +# Tox will provision dev environment with editable installation for you. +python3 -m pip install tox +python3 -m tox devenv +source venv/bin/activate ``` ### Branching @@ -63,20 +65,29 @@ We use [flake8](https://pypi.org/project/flake8/) for quick style checks and [bl ### Unit Tests -Tests are written using [pytest](https://github.com/pytest-dev/pytest) and executed together with [codecov](https://github.com/codecov/codecov-action) for coverage reports. -To perform the tests, execute the following (make sure pytest is installed): -```bash -pytest +Tests are written using [pytest](https://github.com/pytest-dev/pytest) and executed together +with [codecov](https://github.com/codecov/codecov-action) for coverage reports. +We use [tox](https://tox.wiki/en/latest/) for test automation. For complete list of CLI commands, please refer +to [tox - CLI interface](https://tox.wiki/en/latest/cli_interface.html). +To perform the tests for all supported python versions execute the following CLI command: + +```shell +python3 -m tox run ``` -... alternatively, to get additionaly coverage details, run: + +... alternatively, to get additionally coverage details, run: + ```bash -pytest --cov=. --cov-report term-missing +python3 -m tox run -e coverage ``` -It is possible to limit the scope of testing to specific sections of the codebase, for example, only test the Faithfulness metrics: +It is possible to limit the scope of testing to specific sections of the codebase, for example, only test the +Faithfulness metrics using python3.10: + ```bash -pytest -m faithfulness -s +python3 -m tox run -e py310 -- -m faithfulness -s ``` + For a complete overview of the possible testing scopes, please refer to `pytest.ini`. ### Documentation @@ -90,18 +101,31 @@ Before creating a PR, double-check that the following tasks are completed: - Make sure that the latest version of the code from the `main` branch is merged into your working branch. - Run `black` to format source code: + ```bash black quantus/INSERT_YOUR_FILE_NAME.py ``` + - Run `flake8` for quick style checks, e.g.: + ```bash flake8 quantus/INSERT_YOUR_FILE_NAME.py ``` -- Create `pytests` for new functionality (if needed) and add under `tests/` folder -- If the `pytests` include a new category of `@pytest.mark` then add that category with description to `pytest.ini` -- Make sure all unit tests are passed and the coverage level is maintained (we aim at ~100% code coverage for Quantus): + +- Create a unit test for new functionality and add under `tests/` folder, add `@pytest.mark` with fitting category. +- If newly added test cases include a new category of `@pytest.mark` then add that category with description + to `pytest.ini` +- Make sure all unit tests pass for all supported python version by running: + +```shell +python3 -m tox run +``` + +- Generally, every change should be covered with respective test-case, we aim at ~100% code coverage in Quantus, you can + verify it by running: + ```bash -pytest tests -v --cov-report term --cov-report html:htmlcov --cov-report xml --cov=quantus +python3 -m tox run -e coverage ``` ### Pull Requests @@ -174,4 +198,4 @@ Please note that by contributing to the project you agree that it will be licens ## Questions If you have any developer-related questions, please [open an issue](https://github.com/understandable-machine-intelligence-lab/Quantus/issues/new/choose) -or write us at [hedstroem.anna@gmail.com](mailto:hedstroem.anna@gmail.com). \ No newline at end of file +or write us at [hedstroem.anna@gmail.com](mailto:hedstroem.anna@gmail.com). diff --git a/pyproject.toml b/pyproject.toml index 496cea34f..7ffc4c8ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ # A complete reference of pyproject.toml format can be found on: https://flit.pypa.io/en/stable/pyproject_toml.html [project] name = "quantus" + # Versions should comply with PEP 440: # https://www.python.org/dev/peps/pep-0440/ version = "0.4.1" @@ -10,10 +11,10 @@ requires-python = ">=3.7" license = { file = "LICENSE" } keywords = ["explainable ai", "xai", "machine learning", "deep learning"] authors = [ - { name = "Anna Hedstrom", email = "hedstroem.anna@gmail.com" } # Optional + { name = "Anna Hedstrom", email = "hedstroem.anna@gmail.com" } ] maintainers = [ - { name = "Anna Hedstrom", email = "hedstroem.anna@gmail.com" } # Optional + { name = "Anna Hedstrom", email = "hedstroem.anna@gmail.com" } ] # For a list of valid classifiers, see https://pypi.org/classifiers/ classifiers = [ @@ -36,6 +37,10 @@ dependencies = [ "matplotlib>=3.3.4" ] +[project.urls] +"Documentation" = "https://quantus.readthedocs.io/en/latest/" +"Source" = "https://github.com/understandable-machine-intelligence-lab/Quantus" + # List additional groups of dependencies here (e.g. development dependencies). # Users will be able to install these using the "extras" syntax, for example: # @@ -98,9 +103,6 @@ zennit = [ full = [ "quantus[captum,tf_explain,zennit]" ] -[project.urls] -"Documentation" = "https://quantus.readthedocs.io/en/latest/" -"Source" = "https://github.com/understandable-machine-intelligence-lab/Quantus" [build-system] requires = ["flit-core >= 3.4"] From a33f42e8dc9c013a95460a9682b6b510c5174bfd Mon Sep 17 00:00:00 2001 From: annahedstroem Date: Fri, 23 Jun 2023 11:47:22 +0200 Subject: [PATCH 34/34] minor fixes, delete of github.release --- quantus/metrics/faithfulness/road.py | 1 - scripts/github_release.sh | 33 ---------------------------- 2 files changed, 34 deletions(-) delete mode 100755 scripts/github_release.sh diff --git a/quantus/metrics/faithfulness/road.py b/quantus/metrics/faithfulness/road.py index 3414e5f35..8333d3279 100644 --- a/quantus/metrics/faithfulness/road.py +++ b/quantus/metrics/faithfulness/road.py @@ -364,7 +364,6 @@ def custom_postprocess( """ # Calculate accuracy for every number of most important pixels removed. - self.last_results = { percentage: np.mean(np.array(self.last_results)[:, p_ix]) for p_ix, percentage in enumerate(self.percentages) diff --git a/scripts/github_release.sh b/scripts/github_release.sh deleted file mode 100755 index 892bf5f8b..000000000 --- a/scripts/github_release.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env zsh -set -e -# Check no un-committed changes -echo -n "Checking if there are uncommited changes... " -trap 'echo -e "\033[0;31mCHANGED\033[0m"' ERR -git diff-index --quiet HEAD -- -trap - ERR -echo -e "\033[0;32mUNCHANGED\033[0m" - -echo -n "Looking for GitHub CLI... " -if ! command -v gh &>/dev/null; then - echo -e "\033[0;31m GitHub CLI not installed.\033[0m" - exit -else - echo -e "\033[0;32mOK\033[0m" -fi -# Check provided 1 positional argument -if [ $# -eq 0 ]; then - echo -e "\033[0;31m Must provide tag as positional argument\033[0m" - exit -fi -TAG=$1 -echo -n "TAG=${TAG}" -# Update main ref's and switch to main's HEAD. -git fetch --atomic --verbose && git checkout main -# Build wheel. -python3 -m pip install tox -python3 -m tox run -e build -# Tag release. -git tag "$TAG" -git push --follow-tags -# Create GitHub release draft. -gh release create "$TAG" "./dist/*${TAG}*" --draft --latest