Merge branch 'release-v0.14.1' into 'v0.14-stable'

Resolve "Release v0.14.1"

Release issue #925

See merge request secml/secml!31

.gitlab-ci.yml

@@ -111,7 +111,13 @@ package:docs:
 
 
 code_quality:
-  interruptible: true
+  interruptible: true  
+  rules:
+    - if: '$CODE_QUALITY_DISABLED'
+      when: never
+    - if: '$CI_COMMIT_BRANCH == "stable" || $CI_COMMIT_BRANCH =~ /^release-.*$/ || $CI_COMMIT_BRANCH =~ /^.*-stable$/'
+      when: never
+    - if: '$CI_COMMIT_BRANCH'
   variables:
     REPORT_FORMAT: html
   artifacts:
@@ -562,6 +568,22 @@ release:gitlab-pages:
     - git commit -m "Release $CI_COMMIT_TAG"
     - git push
 
+release:zoo:
+  stage: release
+  variables:
+    RELEASE: $CI_COMMIT_TAG
+    PIP_CACHE_DIR: $PIP_CACHE_DIR
+    TORCH_HOME: $TORCH_HOME
+    SECML_HOME_DIR: $SECML_HOME_DIR
+  trigger:
+    project: secml/secml-zoo
+    strategy: depend
+  rules:
+    - if: '$CI_SERVER_HOST != "gitlab.com"'
+      when: never
+    - if: $CI_COMMIT_TAG
+      when: manual
+
 release:pypi:
   extends: .release
   image: ${CI_REGISTRY}/pralab/docker-helper-images/python36-setuptools:latest

CHANGELOG.md

@@ -1,3 +1,19 @@
+## v0.14.1 (22/04/2021)
+- This version brings fixes for a few issues with the optimizers and related classes, along with improvements to documentation for all attacks, optimizers, and related classes.
+
+### Fixed (3 changes)
+- #923 Fixed `COptimizerPGDLS` and `COptimizerPGDLS` not working properly if the classifier's gradient has multiple components with the same (max) value.
+- #919 Fixed `CConstraintL1` crashing when projecting sparse data using default center value (scalar 0).
+- #920 Fixed inconsistent results between dense and sparse data for `CConstraintL1` projection caused by type casting.
+
+### Removed & Deprecated (1 change)
+- #922 Removed unnecessary parameter `discrete` from `COptimizerPGDLS` and `COptimizerPGDExp`.
+
+### Documentation (2 changes)
+- #100017 Improved documentation of `CAttackEvasion`, `COptimizer`, `CLineSearch`, and corresponding subclasses.
+- #918 Installing the latest stable version of RobustBench instead of the master version.
+
+
 ## v0.14 (23/03/2021)
 - #795 Added new package `adv.attacks.evasion.foolbox` with a wrapper for [Foolbox](https://foolbox.readthedocs.io/en/stable/).
 - #623 `secml` is now tested for compatibility with Python 3.8.

src/secml/VERSION

@@ -1 +1 @@
-0.14
+0.14.1

src/secml/adv/attacks/evasion/c_attack_evasion.py

@@ -29,7 +29,7 @@ class CAttackEvasion(CAttack, metaclass=ABCMeta):
         belonging to the `y_target` class.
     attack_classes : 'all' or CArray, optional
         Array with the classes that can be manipulated by the attacker or
-         'all' (default) if all classes can be manipulated.
+        'all' (default) if all classes can be manipulated.
 
     """
     __super__ = 'CAttackEvasion'
@@ -200,6 +200,8 @@ def run(self, x, y, ds_init=None):
 
         y_pred = CArray(y_pred)
 
+        self.logger.info("y_pred after attack:\n{:}".format(y_pred))
+
         # Return the mean objective function value on the evasion points
         f_obj = fs_opt.mean()
 

src/secml/adv/attacks/evasion/c_attack_evasion_pgd.py

@@ -7,7 +7,6 @@
 .. moduleauthor:: Marco Melis <[email protected]>
 
 """
-from secml import _NoValue
 from secml.adv.attacks.evasion import CAttackEvasionPGDLS
 
 
@@ -41,8 +40,8 @@ class CAttackEvasionPGD(CAttackEvasionPGDLS):
     double_init_ds : CDataset or None, optional
         Dataset used to initialize an alternative init point (double init).
     double_init : bool, optional
-            If True (default), use double initialization point.
-            Needs double_init_ds not to be None.
+        If True (default), use double initialization point.
+        Needs double_init_ds not to be None.
     distance : {'l1' or 'l2'}, optional
         Norm to use for computing the distance of the adversarial example
         from the original sample. Default 'l2'.
@@ -58,10 +57,11 @@ class CAttackEvasionPGD(CAttackEvasionPGDLS):
         belonging to the `y_target` class.
     attack_classes : 'all' or CArray, optional
         Array with the classes that can be manipulated by the attacker or
-         'all' (default) if all classes can be manipulated.
+        'all' (default) if all classes can be manipulated.
     solver_params : dict or None, optional
-        Parameters for the solver. Default None, meaning that default
-        parameters will be used.
+        Parameters for the solver.
+        Default None, meaning that default parameters will be used.
+        See :class:`COptimizerPGD` for more information.
 
     Attributes
     ----------
@@ -77,7 +77,6 @@ def __init__(self, classifier,
                  dmax=0,
                  lb=0,
                  ub=1,
-                 discrete=_NoValue,
                  y_target=None,
                  attack_classes='all',
                  solver_params=None):
@@ -91,10 +90,6 @@ def __init__(self, classifier,
         # class (indiscriminate evasion). See _get_point_with_min_f_obj()
         self._xk = None
 
-        # pgd solver does not accepts parameter `discrete`
-        if discrete is not _NoValue:
-            raise ValueError("`pgd` solver does not work in discrete space.")
-
         super(CAttackEvasionPGD, self).__init__(
             classifier=classifier,
             double_init_ds=double_init_ds,

src/secml/adv/attacks/evasion/c_attack_evasion_pgd_exp.py

@@ -38,8 +38,8 @@ class onto the feasible domain and try again.
     double_init_ds : CDataset or None, optional
         Dataset used to initialize an alternative init point (double init).
     double_init : bool, optional
-            If True (default), use double initialization point.
-            Needs double_init_ds not to be None.
+        If True (default), use double initialization point.
+        Needs double_init_ds not to be None.
     distance : {'l1' or 'l2'}, optional
         Norm to use for computing the distance of the adversarial example
         from the original sample. Default 'l2'.
@@ -55,10 +55,11 @@ class onto the feasible domain and try again.
         belonging to the `y_target` class.
     attack_classes : 'all' or CArray, optional
         Array with the classes that can be manipulated by the attacker or
-         'all' (default) if all classes can be manipulated.
+        'all' (default) if all classes can be manipulated.
     solver_params : dict or None, optional
-        Parameters for the solver. Default None, meaning that default
-        parameters will be used.
+        Parameters for the solver.
+        Default None, meaning that default parameters will be used.
+        See :class:`COptimizerPGDExp` for more information.
 
     Attributes
     ----------

src/secml/adv/attacks/evasion/c_attack_evasion_pgd_ls.py

@@ -49,8 +49,8 @@ class CAttackEvasionPGDLS(CAttackEvasion, CAttackMixin):
     double_init_ds : CDataset or None, optional
         Dataset used to initialize an alternative init point (double init).
     double_init : bool, optional
-            If True (default), use double initialization point.
-            Needs double_init_ds not to be None.
+        If True (default), use double initialization point.
+        Needs double_init_ds not to be None.
     distance : {'l1' or 'l2'}, optional
         Norm to use for computing the distance of the adversarial example
         from the original sample. Default 'l2'.
@@ -66,10 +66,11 @@ class CAttackEvasionPGDLS(CAttackEvasion, CAttackMixin):
         belonging to the `y_target` class.
     attack_classes : 'all' or CArray, optional
         Array with the classes that can be manipulated by the attacker or
-         'all' (default) if all classes can be manipulated.
+        'all' (default) if all classes can be manipulated.
     solver_params : dict or None, optional
-        Parameters for the solver. Default None, meaning that default
-        parameters will be used.
+        Parameters for the solver.
+        Default None, meaning that default parameters will be used.
+        See :class:`COptimizerPGDLS` for more information.
 
     Attributes
     ----------

src/secml/adv/attacks/evasion/tests/c_attack_evasion_testcases.py

@@ -1,7 +1,9 @@
 from secml.testing import CUnitTest
 
-from numpy import *
+import os
+import numpy as np
 
+from secml.array import CArray
 from secml.data.loader import CDLRandomBlobs
 from secml.optim.constraints import \
     CConstraintBox, CConstraintL1, CConstraintL2
@@ -19,7 +21,7 @@
 class CAttackEvasionTestCases(CUnitTest):
     """Unittests interface for CAttackEvasion."""
     images_folder = IMAGES_FOLDER
-    make_figures = False  # Set as True to produce figures
+    make_figures = os.getenv('MAKE_FIGURES', False)  # True to produce figures
 
     def _load_blobs(self, n_feats, n_clusters, sparse=False, seed=None):
         """Load Random Blobs dataset.
@@ -71,6 +73,11 @@ def _discretize_data(ds, eta):
         else:  # eta is a single value
             ds.X = (ds.X / eta).round() * eta
 
+        # It is likely that after the discretization there are duplicates
+        new_array = [tuple(row) for row in ds.X.tondarray()]
+        uniques, uniques_idx = np.unique(new_array, axis=0, return_index=True)
+        ds = ds[uniques_idx.tolist(), :]
+
         return ds
 
     def _prepare_linear_svm(self, sparse, seed):
@@ -102,6 +109,35 @@ def _prepare_linear_svm(self, sparse, seed):
 
         return ds, clf
 
+    def _prepare_linear_svm_10d(self, sparse, seed):
+        """Preparare the data required for attacking a LINEAR SVM.
+
+        - load a blob 10D dataset
+        - create a SVM (C=1) and a minmax preprocessor
+
+        Parameters
+        ----------
+        sparse : bool
+        seed : int or None
+
+        Returns
+        -------
+        ds : CDataset
+        clf : CClassifierSVM
+
+        """
+        ds = self._load_blobs(
+            n_feats=10,  # Number of dataset features
+            n_clusters=2,  # Number of dataset clusters
+            sparse=sparse,
+            seed=seed
+        )
+
+        normalizer = CNormalizerMinMax(feature_range=(-1, 1))
+        clf = CClassifierSVM(C=1.0, preprocess=normalizer)
+
+        return ds, clf
+
     def _prepare_nonlinear_svm(self, sparse, seed):
         """Preparare the data required for attacking a NONLINEAR SVM.
 

src/secml/adv/attacks/evasion/tests/test_c_attack_evasion_pgd_exp.py

@@ -5,8 +5,8 @@
 from secml.array import CArray
 
 
-class TestCAttackEvasionPGDLS(CAttackEvasionTestCases):
-    """Unittests for CAttackEvasionPGDLS."""
+class TestCAttackEvasionPGDExp(CAttackEvasionTestCases):
+    """Unittests for CAttackEvasionPGDExp."""
 
     def _set_evasion(self, ds, params):
         """Prepare the evasion attack.
@@ -79,6 +79,93 @@ def test_linear_l1(self):
 
         self._plot_2d_evasion(evas, ds, x0, 'pgd_exp_linear_L1.pdf')
 
+    def test_linear_l1_discrete(self):
+        """Test evasion of a linear classifier using L1 distance (discrete)."""
+
+        eta = 0.5
+        sparse = True
+        seed = 10
+
+        ds, clf = self._prepare_linear_svm(sparse, seed)
+
+        ds = self._discretize_data(ds, eta)
+
+        evasion_params = {
+            "classifier": clf,
+            "double_init_ds": ds,
+            "distance": 'l1',
+            "dmax": 2,
+            "lb": -1,
+            "ub": 1,
+            "attack_classes": CArray([1]),
+            "y_target": 0,
+            "solver_params": {
+                "eta": eta,
+                "eta_min": None,
+                "eta_max": None
+            }
+        }
+
+        evas, x0, y0 = self._set_evasion(ds, evasion_params)
+
+        # Expected final optimal point
+        expected_x = CArray([0.5, -1])
+        expected_y = 0
+
+        self._run_evasion(evas, x0, y0, expected_x, expected_y)
+
+        self._plot_2d_evasion(evas, ds, x0, 'pgd_exp_linear_L1_discrete.pdf')
+
+    def test_linear_l1_discrete_10d(self):
+        """Test evasion of a linear classifier (10 features)
+        using L1 distance (discrete).
+        In this test we set few features to the same value to cover a
+        special case of the l1 projection, where there are multiple
+        features with the same max value. The optimizer should change
+        one of them at each iteration.
+        """
+
+        eta = 0.5
+        sparse = True
+        seed = 10
+
+        ds, clf = self._prepare_linear_svm_10d(sparse, seed)
+
+        ds = self._discretize_data(ds, eta)
+
+        evasion_params = {
+            "classifier": clf,
+            "double_init_ds": ds,
+            "distance": 'l1',
+            "dmax": 5,
+            "lb": -2,
+            "ub": 2,
+            "attack_classes": CArray([1]),
+            "y_target": 0,
+            "solver_params": {
+                "eta": eta,
+                "eta_min": None,
+                "eta_max": None
+            }
+        }
+
+        evas, x0, y0 = self._set_evasion(ds, evasion_params)
+
+        # Set few features to the same max value
+        w_new = clf.w.deepcopy()
+        w_new[CArray.randint(
+            clf.w.size, shape=3, random_state=seed)] = clf.w.max()
+        clf._w = w_new
+
+        # Expected final optimal point
+        # CAttackEvasionPGDExp uses CLineSearchBisectProj
+        # which brings the point outside of the grid
+        expected_x = \
+            CArray([-1.8333, -1.8333, 1.8333, 0, -0.5, 0, 0.5, -0.5, 1, 0.5])
+        expected_y = 0
+
+        self._run_evasion(evas, x0, y0, expected_x, expected_y)
+
     def test_linear_l2(self):
         """Test evasion of a linear classifier using L2 distance."""