From 4be7c1371a457d1c41787013ee8a4848cb623c85 Mon Sep 17 00:00:00 2001 From: Marco Melis Date: Thu, 22 Apr 2021 15:25:53 +0000 Subject: [PATCH 1/2] Release v0.14.1-rc1 --- .gitlab-ci.yml | 24 +- src/secml/VERSION | 2 +- .../adv/attacks/evasion/c_attack_evasion.py | 4 +- .../attacks/evasion/c_attack_evasion_pgd.py | 17 +- .../evasion/c_attack_evasion_pgd_exp.py | 11 +- .../evasion/c_attack_evasion_pgd_ls.py | 11 +- .../tests/c_attack_evasion_testcases.py | 40 +++- .../tests/test_c_attack_evasion_pgd_exp.py | 91 +++++++- .../tests/test_c_attack_evasion_pgd_ls.py | 82 +++++++ .../seceval/tests/test_c_sec_eval_evasion.py | 210 +++++++++--------- .../tests/test_dataloader_torchvision.py | 24 +- .../optim/constraints/c_constraint_l1.py | 5 +- .../constraints/tests/test_c_constraint.py | 11 +- .../constraints/tests/test_c_constraint_l1.py | 131 +++++++---- src/secml/optim/optimizers/c_optimizer.py | 9 +- src/secml/optim/optimizers/c_optimizer_pgd.py | 16 ++ .../optim/optimizers/c_optimizer_pgd_exp.py | 202 +++++------------ .../optim/optimizers/c_optimizer_pgd_ls.py | 101 +++++---- .../optimizers/line_search/c_line_search.py | 13 ++ .../line_search/c_line_search_bisect.py | 81 ++++--- .../line_search/c_line_search_bisect_proj.py | 43 +++- .../optimizers/tests/c_optimizer_testcases.py | 10 +- .../tests/test_c_optimizer_pgd_ls_discrete.py | 192 +++++++++------- tox.ini | 2 +- tutorials/14-RobustBench.ipynb | 6 +- 25 files changed, 843 insertions(+), 495 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1e68fd43..75632782 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -111,7 +111,13 @@ package:docs: code_quality: - interruptible: true + interruptible: true + rules: + - if: '$CODE_QUALITY_DISABLED' + when: never + - if: '$CI_COMMIT_BRANCH == "stable" || $CI_COMMIT_BRANCH =~ /^release-.*$/ || $CI_COMMIT_BRANCH =~ /^.*-stable$/' + when: never + - if: '$CI_COMMIT_BRANCH' variables: REPORT_FORMAT: html artifacts: @@ -562,6 +568,22 @@ release:gitlab-pages: - git commit -m "Release $CI_COMMIT_TAG" - git push +release:zoo: + stage: release + variables: + RELEASE: $CI_COMMIT_TAG + PIP_CACHE_DIR: $PIP_CACHE_DIR + TORCH_HOME: $TORCH_HOME + SECML_HOME_DIR: $SECML_HOME_DIR + trigger: + project: secml/secml-zoo + strategy: depend + rules: + - if: '$CI_SERVER_HOST != "gitlab.com"' + when: never + - if: $CI_COMMIT_TAG + when: manual + release:pypi: extends: .release image: ${CI_REGISTRY}/pralab/docker-helper-images/python36-setuptools:latest diff --git a/src/secml/VERSION b/src/secml/VERSION index c64601b2..ad4f01d7 100644 --- a/src/secml/VERSION +++ b/src/secml/VERSION @@ -1 +1 @@ -0.14 \ No newline at end of file +0.14.1-rc1 diff --git a/src/secml/adv/attacks/evasion/c_attack_evasion.py b/src/secml/adv/attacks/evasion/c_attack_evasion.py index e0748887..590b3a35 100644 --- a/src/secml/adv/attacks/evasion/c_attack_evasion.py +++ b/src/secml/adv/attacks/evasion/c_attack_evasion.py @@ -29,7 +29,7 @@ class CAttackEvasion(CAttack, metaclass=ABCMeta): belonging to the `y_target` class. attack_classes : 'all' or CArray, optional Array with the classes that can be manipulated by the attacker or - 'all' (default) if all classes can be manipulated. + 'all' (default) if all classes can be manipulated. """ __super__ = 'CAttackEvasion' @@ -200,6 +200,8 @@ def run(self, x, y, ds_init=None): y_pred = CArray(y_pred) + self.logger.info("y_pred after attack:\n{:}".format(y_pred)) + # Return the mean objective function value on the evasion points f_obj = fs_opt.mean() diff --git a/src/secml/adv/attacks/evasion/c_attack_evasion_pgd.py b/src/secml/adv/attacks/evasion/c_attack_evasion_pgd.py index aa0ca9f0..45535bf9 100644 --- a/src/secml/adv/attacks/evasion/c_attack_evasion_pgd.py +++ b/src/secml/adv/attacks/evasion/c_attack_evasion_pgd.py @@ -7,7 +7,6 @@ .. moduleauthor:: Marco Melis """ -from secml import _NoValue from secml.adv.attacks.evasion import CAttackEvasionPGDLS @@ -41,8 +40,8 @@ class CAttackEvasionPGD(CAttackEvasionPGDLS): double_init_ds : CDataset or None, optional Dataset used to initialize an alternative init point (double init). double_init : bool, optional - If True (default), use double initialization point. - Needs double_init_ds not to be None. + If True (default), use double initialization point. + Needs double_init_ds not to be None. distance : {'l1' or 'l2'}, optional Norm to use for computing the distance of the adversarial example from the original sample. Default 'l2'. @@ -58,10 +57,11 @@ class CAttackEvasionPGD(CAttackEvasionPGDLS): belonging to the `y_target` class. attack_classes : 'all' or CArray, optional Array with the classes that can be manipulated by the attacker or - 'all' (default) if all classes can be manipulated. + 'all' (default) if all classes can be manipulated. solver_params : dict or None, optional - Parameters for the solver. Default None, meaning that default - parameters will be used. + Parameters for the solver. + Default None, meaning that default parameters will be used. + See :class:`COptimizerPGD` for more information. Attributes ---------- @@ -77,7 +77,6 @@ def __init__(self, classifier, dmax=0, lb=0, ub=1, - discrete=_NoValue, y_target=None, attack_classes='all', solver_params=None): @@ -91,10 +90,6 @@ def __init__(self, classifier, # class (indiscriminate evasion). See _get_point_with_min_f_obj() self._xk = None - # pgd solver does not accepts parameter `discrete` - if discrete is not _NoValue: - raise ValueError("`pgd` solver does not work in discrete space.") - super(CAttackEvasionPGD, self).__init__( classifier=classifier, double_init_ds=double_init_ds, diff --git a/src/secml/adv/attacks/evasion/c_attack_evasion_pgd_exp.py b/src/secml/adv/attacks/evasion/c_attack_evasion_pgd_exp.py index b0340f54..0aad6389 100644 --- a/src/secml/adv/attacks/evasion/c_attack_evasion_pgd_exp.py +++ b/src/secml/adv/attacks/evasion/c_attack_evasion_pgd_exp.py @@ -38,8 +38,8 @@ class onto the feasible domain and try again. double_init_ds : CDataset or None, optional Dataset used to initialize an alternative init point (double init). double_init : bool, optional - If True (default), use double initialization point. - Needs double_init_ds not to be None. + If True (default), use double initialization point. + Needs double_init_ds not to be None. distance : {'l1' or 'l2'}, optional Norm to use for computing the distance of the adversarial example from the original sample. Default 'l2'. @@ -55,10 +55,11 @@ class onto the feasible domain and try again. belonging to the `y_target` class. attack_classes : 'all' or CArray, optional Array with the classes that can be manipulated by the attacker or - 'all' (default) if all classes can be manipulated. + 'all' (default) if all classes can be manipulated. solver_params : dict or None, optional - Parameters for the solver. Default None, meaning that default - parameters will be used. + Parameters for the solver. + Default None, meaning that default parameters will be used. + See :class:`COptimizerPGDExp` for more information. Attributes ---------- diff --git a/src/secml/adv/attacks/evasion/c_attack_evasion_pgd_ls.py b/src/secml/adv/attacks/evasion/c_attack_evasion_pgd_ls.py index 16bf519b..d2f0beb3 100644 --- a/src/secml/adv/attacks/evasion/c_attack_evasion_pgd_ls.py +++ b/src/secml/adv/attacks/evasion/c_attack_evasion_pgd_ls.py @@ -49,8 +49,8 @@ class CAttackEvasionPGDLS(CAttackEvasion, CAttackMixin): double_init_ds : CDataset or None, optional Dataset used to initialize an alternative init point (double init). double_init : bool, optional - If True (default), use double initialization point. - Needs double_init_ds not to be None. + If True (default), use double initialization point. + Needs double_init_ds not to be None. distance : {'l1' or 'l2'}, optional Norm to use for computing the distance of the adversarial example from the original sample. Default 'l2'. @@ -66,10 +66,11 @@ class CAttackEvasionPGDLS(CAttackEvasion, CAttackMixin): belonging to the `y_target` class. attack_classes : 'all' or CArray, optional Array with the classes that can be manipulated by the attacker or - 'all' (default) if all classes can be manipulated. + 'all' (default) if all classes can be manipulated. solver_params : dict or None, optional - Parameters for the solver. Default None, meaning that default - parameters will be used. + Parameters for the solver. + Default None, meaning that default parameters will be used. + See :class:`COptimizerPGDLS` for more information. Attributes ---------- diff --git a/src/secml/adv/attacks/evasion/tests/c_attack_evasion_testcases.py b/src/secml/adv/attacks/evasion/tests/c_attack_evasion_testcases.py index aa806b11..04a3c081 100644 --- a/src/secml/adv/attacks/evasion/tests/c_attack_evasion_testcases.py +++ b/src/secml/adv/attacks/evasion/tests/c_attack_evasion_testcases.py @@ -1,7 +1,9 @@ from secml.testing import CUnitTest -from numpy import * +import os +import numpy as np +from secml.array import CArray from secml.data.loader import CDLRandomBlobs from secml.optim.constraints import \ CConstraintBox, CConstraintL1, CConstraintL2 @@ -19,7 +21,7 @@ class CAttackEvasionTestCases(CUnitTest): """Unittests interface for CAttackEvasion.""" images_folder = IMAGES_FOLDER - make_figures = False # Set as True to produce figures + make_figures = os.getenv('MAKE_FIGURES', False) # True to produce figures def _load_blobs(self, n_feats, n_clusters, sparse=False, seed=None): """Load Random Blobs dataset. @@ -71,6 +73,11 @@ def _discretize_data(ds, eta): else: # eta is a single value ds.X = (ds.X / eta).round() * eta + # It is likely that after the discretization there are duplicates + new_array = [tuple(row) for row in ds.X.tondarray()] + uniques, uniques_idx = np.unique(new_array, axis=0, return_index=True) + ds = ds[uniques_idx.tolist(), :] + return ds def _prepare_linear_svm(self, sparse, seed): @@ -102,6 +109,35 @@ def _prepare_linear_svm(self, sparse, seed): return ds, clf + def _prepare_linear_svm_10d(self, sparse, seed): + """Preparare the data required for attacking a LINEAR SVM. + + - load a blob 10D dataset + - create a SVM (C=1) and a minmax preprocessor + + Parameters + ---------- + sparse : bool + seed : int or None + + Returns + ------- + ds : CDataset + clf : CClassifierSVM + + """ + ds = self._load_blobs( + n_feats=10, # Number of dataset features + n_clusters=2, # Number of dataset clusters + sparse=sparse, + seed=seed + ) + + normalizer = CNormalizerMinMax(feature_range=(-1, 1)) + clf = CClassifierSVM(C=1.0, preprocess=normalizer) + + return ds, clf + def _prepare_nonlinear_svm(self, sparse, seed): """Preparare the data required for attacking a NONLINEAR SVM. diff --git a/src/secml/adv/attacks/evasion/tests/test_c_attack_evasion_pgd_exp.py b/src/secml/adv/attacks/evasion/tests/test_c_attack_evasion_pgd_exp.py index 43780007..ddbdfece 100644 --- a/src/secml/adv/attacks/evasion/tests/test_c_attack_evasion_pgd_exp.py +++ b/src/secml/adv/attacks/evasion/tests/test_c_attack_evasion_pgd_exp.py @@ -5,8 +5,8 @@ from secml.array import CArray -class TestCAttackEvasionPGDLS(CAttackEvasionTestCases): - """Unittests for CAttackEvasionPGDLS.""" +class TestCAttackEvasionPGDExp(CAttackEvasionTestCases): + """Unittests for CAttackEvasionPGDExp.""" def _set_evasion(self, ds, params): """Prepare the evasion attack. @@ -79,6 +79,93 @@ def test_linear_l1(self): self._plot_2d_evasion(evas, ds, x0, 'pgd_exp_linear_L1.pdf') + def test_linear_l1_discrete(self): + """Test evasion of a linear classifier using L1 distance (discrete).""" + + eta = 0.5 + sparse = True + seed = 10 + + ds, clf = self._prepare_linear_svm(sparse, seed) + + ds = self._discretize_data(ds, eta) + + evasion_params = { + "classifier": clf, + "double_init_ds": ds, + "distance": 'l1', + "dmax": 2, + "lb": -1, + "ub": 1, + "attack_classes": CArray([1]), + "y_target": 0, + "solver_params": { + "eta": eta, + "eta_min": None, + "eta_max": None + } + } + + evas, x0, y0 = self._set_evasion(ds, evasion_params) + + # Expected final optimal point + expected_x = CArray([0.5, -1]) + expected_y = 0 + + self._run_evasion(evas, x0, y0, expected_x, expected_y) + + self._plot_2d_evasion(evas, ds, x0, 'pgd_exp_linear_L1_discrete.pdf') + + def test_linear_l1_discrete_10d(self): + """Test evasion of a linear classifier (10 features) + using L1 distance (discrete). + In this test we set few features to the same value to cover a + special case of the l1 projection, where there are multiple + features with the same max value. The optimizer should change + one of them at each iteration. + """ + + eta = 0.5 + sparse = True + seed = 10 + + ds, clf = self._prepare_linear_svm_10d(sparse, seed) + + ds = self._discretize_data(ds, eta) + + evasion_params = { + "classifier": clf, + "double_init_ds": ds, + "distance": 'l1', + "dmax": 5, + "lb": -2, + "ub": 2, + "attack_classes": CArray([1]), + "y_target": 0, + "solver_params": { + "eta": eta, + "eta_min": None, + "eta_max": None + } + } + + evas, x0, y0 = self._set_evasion(ds, evasion_params) + + # Set few features to the same max value + w_new = clf.w.deepcopy() + w_new[CArray.randint( + clf.w.size, shape=3, random_state=seed)] = clf.w.max() + clf._w = w_new + + # Expected final optimal point + # CAttackEvasionPGDExp uses CLineSearchBisectProj + # which brings the point outside of the grid + expected_x = \ + CArray([-1.8333, -1.8333, 1.8333, 0, -0.5, 0, 0.5, -0.5, 1, 0.5]) + expected_y = 0 + + self._run_evasion(evas, x0, y0, expected_x, expected_y) + def test_linear_l2(self): """Test evasion of a linear classifier using L2 distance.""" diff --git a/src/secml/adv/attacks/evasion/tests/test_c_attack_evasion_pgd_ls.py b/src/secml/adv/attacks/evasion/tests/test_c_attack_evasion_pgd_ls.py index 11e863b9..ba401f97 100644 --- a/src/secml/adv/attacks/evasion/tests/test_c_attack_evasion_pgd_ls.py +++ b/src/secml/adv/attacks/evasion/tests/test_c_attack_evasion_pgd_ls.py @@ -79,6 +79,88 @@ def test_linear_l1(self): self._plot_2d_evasion(evas, ds, x0, 'pgd_ls_linear_L1.pdf') + def test_linear_l1_discrete(self): + """Test evasion of a linear classifier using L1 distance (discrete).""" + + eta = 0.5 + sparse = True + seed = 10 + + ds, clf = self._prepare_linear_svm(sparse, seed) + + ds = self._discretize_data(ds, eta) + + evasion_params = { + "classifier": clf, + "double_init_ds": ds, + "distance": 'l1', + "dmax": 2, + "lb": -1, + "ub": 1, + "attack_classes": CArray([1]), + "y_target": 0, + "solver_params": { + "eta": eta + } + } + + evas, x0, y0 = self._set_evasion(ds, evasion_params) + + # Expected final optimal point + expected_x = CArray([0.5, -1]) + expected_y = 0 + + self._run_evasion(evas, x0, y0, expected_x, expected_y) + + self._plot_2d_evasion(evas, ds, x0, 'pgd_ls_linear_L1_discrete.pdf') + + def test_linear_l1_discrete_10d(self): + """Test evasion of a linear classifier (10 features) + using L1 distance (discrete). + In this test we set few features to the same value to cover a + special case of the l1 projection, where there are multiple + features with the same max value. The optimizer should change + one of them at each iteration. + """ + + eta = 0.5 + sparse = True + seed = 10 + + ds, clf = self._prepare_linear_svm_10d(sparse, seed) + + ds = self._discretize_data(ds, eta) + + evasion_params = { + "classifier": clf, + "double_init_ds": ds, + "distance": 'l1', + "dmax": 5, + "lb": -2, + "ub": 2, + "attack_classes": CArray([1]), + "y_target": 0, + "solver_params": { + "eta": eta, + "eta_min": None, + "eta_max": None + } + } + + evas, x0, y0 = self._set_evasion(ds, evasion_params) + + # Set few features to the same max value + w_new = clf.w.deepcopy() + w_new[CArray.randint( + clf.w.size, shape=3, random_state=seed)] = clf.w.max() + clf._w = w_new + + # Expected final optimal point + expected_x = CArray([-2., -1.5, 2, 0, -0.5, 0, 0.5, -0.5, 1, 0.5]) + expected_y = 0 + + self._run_evasion(evas, x0, y0, expected_x, expected_y) + def test_linear_l2(self): """Test evasion of a linear classifier using L2 distance.""" diff --git a/src/secml/adv/seceval/tests/test_c_sec_eval_evasion.py b/src/secml/adv/seceval/tests/test_c_sec_eval_evasion.py index 5e521ce9..a4d59e7b 100644 --- a/src/secml/adv/seceval/tests/test_c_sec_eval_evasion.py +++ b/src/secml/adv/seceval/tests/test_c_sec_eval_evasion.py @@ -1,139 +1,149 @@ +from secml.adv.attacks.evasion.tests import CAttackEvasionTestCases + from secml.adv.attacks.evasion import CAttackEvasionPGDLS from secml.adv.seceval import CSecEval from secml.array import CArray -from secml.data.loader import CDLRandomBlobs from secml.figure import CFigure from secml.ml.classifiers import CClassifierSVM -from secml.testing import CUnitTest -class TestCSecEval(CUnitTest): +class TestCSecEval(CAttackEvasionTestCases): """Unittests for CSecEval (evasion attack).""" def setUp(self): - self.classifier = CClassifierSVM(kernel='linear', C=1.0) - - self.lb = -2 - self.ub = +2 - - n_tr = 20 - n_ts = 10 - n_features = 2 - - n_reps = 1 - - self.sec_eval = [] - self.attack_ds = [] - for rep_i in range(n_reps): - self.logger.info( - "Loading `random_blobs` with seed: {:}".format(rep_i)) - loader = CDLRandomBlobs( - n_samples=n_tr + n_ts, - n_features=n_features, - centers=[(-0.5, -0.5), (+0.5, +0.5)], - center_box=(-0.5, 0.5), - cluster_std=0.5, - random_state=rep_i * 100 + 10) - ds = loader.load() - - self.tr = ds[:n_tr, :] - self.ts = ds[n_tr:, :] - - self.classifier.fit(self.tr.X, self.tr.Y) - - # only manipulate positive samples, targeting negative ones - self.y_target = None - self.attack_classes = CArray([1]) - - for create_fn in (self._attack_pgd_ls, self._attack_cleverhans): - # TODO: REFACTOR THESE UNITTESTS REMOVING THE FOR LOOP - - try: - import cleverhans - except ImportError: - continue - - self.attack_ds.append(self.ts) - attack, param_name, param_values = create_fn() - # set sec eval object - self.sec_eval.append( - CSecEval( - attack=attack, - param_name=param_name, - param_values=param_values, - ) - ) - - def _attack_pgd_ls(self): + self.clf = CClassifierSVM(C=1.0) + + self.n_tr = 40 + self.n_features = 10 + self.seed = 0 + + self.logger.info( + "Loading `random_blobs` with seed: {:}".format(self.seed)) + self.ds = self._load_blobs( + self.n_features, 2, sparse=False, seed=self.seed) + + self.tr = self.ds[:self.n_tr, :] + self.ts = self.ds[self.n_tr:, :] + + self.clf.fit(self.tr.X, self.tr.Y) + + def test_attack_pgd_ls(self): + """Test SecEval using CAttackEvasionPGDLS.""" params = { - "classifier": self.classifier, + "classifier": self.clf, "double_init_ds": self.tr, + "distance": 'l2', + "lb": -2, + "ub": 2, + "y_target": None, + "solver_params": {'eta': 0.1, 'eps': 1e-2} + } + attack = CAttackEvasionPGDLS(**params) + attack.verbose = 1 + + param_name = 'dmax' + + self._set_and_run(attack, param_name) + + def test_attack_pgd_ls_discrete(self): + """Test SecEval using CAttackEvasionPGDLS on a problematic + discrete case with L1 constraint. + We alter the classifier so that many weights have the same value. + The optimizer should be able to evade the classifier anyway, + by changing one feature each iteration. Otherwise, by changing + all the feature with the same value at once, the evasion will always + fail because the L1 constraint will be violated. + """ + self.ds = self._discretize_data(self.ds, eta=1) + self.ds.X[self.ds.X > 1] = 1 + self.ds.X[self.ds.X < -1] = -1 + + self.tr = self.ds[:self.n_tr, :] + self.ts = self.ds[self.n_tr:, :] + + self.clf.fit(self.tr.X, self.tr.Y) + + # Set few features to the same max value + w_new = self.clf.w.deepcopy() + w_new[CArray.randint( + self.clf.w.size, shape=5, random_state=0)] = self.clf.w.max() + self.clf._w = w_new + + params = { + "classifier": self.clf, + "double_init": False, "distance": 'l1', - "lb": self.lb, - "ub": self.ub, - "y_target": self.y_target, - "attack_classes": self.attack_classes, - "solver_params": {'eta': 0.5, 'eps': 1e-2} + "lb": -1, + "ub": 1, + "y_target": None, + "solver_params": {'eta': 1, 'eps': 1e-2} } attack = CAttackEvasionPGDLS(**params) attack.verbose = 1 - # sec eval params param_name = 'dmax' - dmax = 2 - dmax_step = 0.5 - param_values = CArray.arange( - start=0, step=dmax_step, - stop=dmax + dmax_step) - return attack, param_name, param_values + self._set_and_run(attack, param_name, dmax_step=1) - def _attack_cleverhans(self): + def test_attack_cleverhans(self): + """Test SecEval using CAttackEvasionCleverhans+FastGradientMethod.""" + try: + import cleverhans + except ImportError as e: + import unittest + raise unittest.SkipTest(e) from cleverhans.attacks import FastGradientMethod from secml.adv.attacks import CAttackEvasionCleverhans + params = { + "classifier": self.clf, + "surrogate_data": self.tr, + "y_target": None, + "clvh_attack_class": FastGradientMethod, + 'eps': 0.1, + 'clip_max': 2, + 'clip_min': -2, + 'ord': 2 + } + attack = CAttackEvasionCleverhans(**params) - attack_params = {'eps': 0.1, - 'clip_max': self.ub, - 'clip_min': self.lb, - 'ord': 1} + param_name = 'attack_params.eps' - attack = CAttackEvasionCleverhans( - classifier=self.classifier, - surrogate_data=self.tr, - y_target=self.y_target, - clvh_attack_class=FastGradientMethod, - ** attack_params) + self._set_and_run(attack, param_name) - param_name = 'attack_params.eps' - dmax = 2 - dmax_step = 0.5 + def _set_and_run(self, attack, param_name, dmax=2, dmax_step=0.5): + """Create the SecEval and run it on test set.""" param_values = CArray.arange( start=0, step=dmax_step, stop=dmax + dmax_step) - return attack, param_name, param_values + sec_eval = CSecEval( + attack=attack, + param_name=param_name, + param_values=param_values, + ) + + sec_eval.run_sec_eval(self.ts) + + self._plot_sec_eval(sec_eval) + + # At the end of the seceval we expect 0% accuracy + self.assertFalse( + CArray(sec_eval.sec_eval_data.Y_pred[-1] == self.ts.Y).any()) + + @staticmethod + def _plot_sec_eval(sec_eval): - def _plot_sec_eval(self): - # figure creation figure = CFigure(height=5, width=5) - for sec_eval in self.sec_eval: - sec_eval_data = sec_eval.sec_eval_data - # plot security evaluation - figure.sp.plot_sec_eval(sec_eval_data, label='SVM', marker='o', - show_average=True, mean=True) + figure.sp.plot_sec_eval(sec_eval.sec_eval_data, + label='SVM', marker='o', + show_average=True, mean=True) + figure.sp.title(sec_eval.attack.__class__.__name__) figure.subplots_adjust() figure.show() - def test_sec_eval(self): - # evaluate classifier security - for sec_eval_i, sec_eval in enumerate(self.sec_eval): - sec_eval.run_sec_eval(self.attack_ds[sec_eval_i]) - - self._plot_sec_eval() - if __name__ == '__main__': - CUnitTest.main() + CAttackEvasionTestCases.main() diff --git a/src/secml/data/loader/tests/test_dataloader_torchvision.py b/src/secml/data/loader/tests/test_dataloader_torchvision.py index f12de3da..118a9d78 100644 --- a/src/secml/data/loader/tests/test_dataloader_torchvision.py +++ b/src/secml/data/loader/tests/test_dataloader_torchvision.py @@ -29,17 +29,19 @@ def setUp(self): def _create_ds(self): torchvision_dataset = datasets.MNIST - # Workaround for the original MNIST site not being available - torchvision_dataset.resources = [ - ('https://ossci-datasets.s3.amazonaws.com/mnist/train-images-idx3-ubyte.gz', - 'f68b3c2dcbeaaa9fbdd348bbdeb94873'), - ('https://ossci-datasets.s3.amazonaws.com/mnist/train-labels-idx1-ubyte.gz', - 'd53e105ee54ea40749a09fcbcd1e9432'), - ('https://ossci-datasets.s3.amazonaws.com/mnist/t10k-images-idx3-ubyte.gz', - '9fb629c4189551a2d022fa330f9573f3'), - ('https://ossci-datasets.s3.amazonaws.com/mnist/t10k-labels-idx1-ubyte.gz', - 'ec29112dd5afa0611ce80d1b7f02629c') - ] + # Workaround for the original MNIST site not being available + # Only needed for torchvision < 0.9.1 + if torchvision.__version__ < '0.9.1': # TODO: REMOVE AFTER BUMPING DEPS + torchvision_dataset.resources = [ + ('https://ossci-datasets.s3.amazonaws.com/mnist/train-images-idx3-ubyte.gz', + 'f68b3c2dcbeaaa9fbdd348bbdeb94873'), + ('https://ossci-datasets.s3.amazonaws.com/mnist/train-labels-idx1-ubyte.gz', + 'd53e105ee54ea40749a09fcbcd1e9432'), + ('https://ossci-datasets.s3.amazonaws.com/mnist/t10k-images-idx3-ubyte.gz', + '9fb629c4189551a2d022fa330f9573f3'), + ('https://ossci-datasets.s3.amazonaws.com/mnist/t10k-labels-idx1-ubyte.gz', + 'ec29112dd5afa0611ce80d1b7f02629c') + ] ds = CDataLoaderTorchDataset( torchvision_dataset, train=True, download=True).load() splitter = CTrainTestSplit(train_size=self.n_samples_tr, diff --git a/src/secml/optim/constraints/c_constraint_l1.py b/src/secml/optim/constraints/c_constraint_l1.py index 1a40af8e..e4c782d6 100644 --- a/src/secml/optim/constraints/c_constraint_l1.py +++ b/src/secml/optim/constraints/c_constraint_l1.py @@ -43,7 +43,7 @@ def center(self): @center.setter def center(self, value): """Center of the constraint.""" - self._center = CArray(value) + self._center = value @property def radius(self): @@ -178,10 +178,11 @@ def _euclidean_proj_simplex(self, v, s=1): theta = (cssv[rho] - s) / (rho + 1.0) # compute the projection by thresholding v using theta - w = v + w = v.deepcopy() if w.issparse: p = CArray(w.nnz_data) p -= theta + w = w.astype(p) # p dtype may change after subtraction w[w.nnz_indices] = p else: w -= theta diff --git a/src/secml/optim/constraints/tests/test_c_constraint.py b/src/secml/optim/constraints/tests/test_c_constraint.py index 86ed2b7e..a90010e6 100644 --- a/src/secml/optim/constraints/tests/test_c_constraint.py +++ b/src/secml/optim/constraints/tests/test_c_constraint.py @@ -180,7 +180,7 @@ def check_projection(cons, point, expected): check_projection(c, p_out, p_out_expected) check_projection(c, p_out.astype(int), p_out_expected) - def _test_plot(self, c, *points): + def _test_plot(self, c, *points, label=''): """Visualize the constraint. Parameters @@ -189,6 +189,8 @@ def _test_plot(self, c, *points): *points : CArray A series of point to plot. Each point will be plotted before and after cosntraint projection. + label : str, optional + Additional suffix for image filename. """ self.logger.info("Plotting constrain {:}".format(c.class_type)) @@ -213,7 +215,12 @@ def _test_plot(self, c, *points): "Plotting point (color {:}): {:}".format(colors[p_i], p_proj)) fig.sp.scatter(*p_proj, c=colors[p_i], zorder=10) - filename = "test_constraint_{:}.pdf".format(c.class_type) + if label: + filename = \ + "test_constraint_{:}_{:}.pdf".format(c.class_type, label) + else: + filename = \ + "test_constraint_{:}.pdf".format(c.class_type) fig.savefig(fm.join(fm.abspath(__file__), filename)) diff --git a/src/secml/optim/constraints/tests/test_c_constraint_l1.py b/src/secml/optim/constraints/tests/test_c_constraint_l1.py index 75099033..f66c9aa8 100644 --- a/src/secml/optim/constraints/tests/test_c_constraint_l1.py +++ b/src/secml/optim/constraints/tests/test_c_constraint_l1.py @@ -11,79 +11,128 @@ class TestCConstraintL1(CConstraintTestCases): def setUp(self): + self.c0 = CConstraintL1() # center=0, radius=1 self.c = CConstraintL1(center=1, radius=1) self.c_array = CConstraintL1(center=CArray([1, 1]), radius=1) - # create a point that lies inside the constraint - self.p1_inside = CArray([1., 1.]) - # create a point that lies outside the constraint - self.p2_outside = CArray([2., 2.]) - # create a point that lies on the constraint - self.p3_on = CArray([0., 1.]) + # create a point that lies inside the constraints + self.c0_p1_inside = CArray([0., 0.]) + self.c_p1_inside = CArray([1., 1.]) + # create a point that lies outside the constraints + self.c0_p2_outside = CArray([1., 1.]) + self.c_p2_outside = CArray([2., 2.]) + # create a point that lies on the constraints + self.c0_p3_on = CArray([0., 1.]) + self.c_p3_on = CArray([0., 1.]) def test_is_active(self): """Test for CConstraint.is_active().""" self._test_is_active( - self.c, self.p1_inside, self.p2_outside, self.p3_on) + self.c0, self.c0_p1_inside, self.c0_p2_outside, self.c0_p3_on) self._test_is_active( - self.c_array, self.p1_inside, self.p2_outside, self.p3_on) + self.c, self.c_p1_inside, self.c_p2_outside, self.c_p3_on) + self._test_is_active( + self.c_array, self.c_p1_inside, self.c_p2_outside, self.c_p3_on) - # Test for sparse arrays, works only for a center defined as CArray + # Test for sparse arrays + self._test_is_violated( + self.c0, + self.c0_p1_inside.tosparse(), + self.c0_p2_outside.tosparse(), + self.c0_p3_on.tosparse()) self._test_is_violated( - self.c_array, self.p1_inside.tosparse(), - self.p2_outside.tosparse(), self.p3_on.tosparse()) + self.c_array, + self.c_p1_inside.tosparse(), + self.c_p2_outside.tosparse(), + self.c_p3_on.tosparse()) def test_is_violated(self): """Test for CConstraint.is_violated().""" self._test_is_violated( - self.c, self.p1_inside, self.p2_outside, self.p3_on) + self.c0, self.c0_p1_inside, self.c0_p2_outside, self.c0_p3_on) self._test_is_violated( - self.c_array, self.p1_inside, self.p2_outside, self.p3_on) + self.c, self.c_p1_inside, self.c_p2_outside, self.c_p3_on) + self._test_is_violated( + self.c_array, self.c_p1_inside, self.c_p2_outside, self.c_p3_on) - # Test for sparse arrays, works only for a center defined as CArray + # Test for sparse arrays + self._test_is_active( + self.c0, + self.c0_p1_inside.tosparse(), + self.c0_p2_outside.tosparse(), + self.c0_p3_on.tosparse()) self._test_is_active( - self.c_array, self.p1_inside.tosparse(), - self.p2_outside.tosparse(), self.p3_on.tosparse()) + self.c_array, + self.c_p1_inside.tosparse(), + self.c_p2_outside.tosparse(), + self.c_p3_on.tosparse()) def test_constraint(self): """Test for CConstraint.constraint().""" self._test_constraint( - self.c, self.p1_inside, self.p2_outside, self.p3_on) + self.c0, self.c0_p1_inside, self.c0_p2_outside, self.c0_p3_on) self._test_constraint( - self.c_array, self.p1_inside, self.p2_outside, self.p3_on) + self.c, self.c_p1_inside, self.c_p2_outside, self.c_p3_on) + self._test_constraint( + self.c_array, self.c_p1_inside, self.c_p2_outside, self.c_p3_on) - # Test for sparse arrays, works only for a center defined as CArray + # Test for sparse arrays + self._test_constraint( + self.c0, + self.c0_p1_inside.tosparse(), + self.c0_p2_outside.tosparse(), + self.c0_p3_on.tosparse()) self._test_constraint( - self.c_array, self.p1_inside.tosparse(), - self.p2_outside.tosparse(), self.p3_on.tosparse()) + self.c_array, + self.c_p1_inside.tosparse(), + self.c_p2_outside.tosparse(), + self.c_p3_on.tosparse()) def test_projection(self): """Test for CConstraint.projection().""" - self._test_projection(self.c, self.p1_inside, self.p2_outside, - self.p3_on, CArray([1.5, 1.5])) - self._test_projection(self.c_array, self.p1_inside, self.p2_outside, - self.p3_on, CArray([1.5, 1.5])) - - # Test for sparse arrays, works only for a center defined as CArray + self._test_projection(self.c0, self.c0_p1_inside, self.c0_p2_outside, + self.c0_p3_on, CArray([0.5, 0.5])) + self._test_projection(self.c, self.c_p1_inside, self.c_p2_outside, + self.c_p3_on, CArray([1.5, 1.5])) + self._test_projection(self.c_array, self.c_p1_inside, self.c_p2_outside, + self.c_p3_on, CArray([1.5, 1.5])) + + # Test for sparse arrays + self._test_projection( + self.c0, self.c0_p1_inside.tosparse(), + self.c0_p2_outside.tosparse(), self.c0_p3_on.tosparse(), + CArray([0.5, 0.5], tosparse=True)) self._test_projection( - self.c_array, self.p1_inside.tosparse(), - self.p2_outside.tosparse(), self.p3_on.tosparse(), + self.c_array, self.c_p1_inside.tosparse(), + self.c_p2_outside.tosparse(), self.c_p3_on.tosparse(), CArray([1.5, 1.5], tosparse=True)) def test_gradient(self): """Test for CConstraint.gradient().""" + # [0. 0.] is the center of the constraint, expected grad [0, c0] + # however, numerical gradient is struggling so we avoid its comparison + self.assert_array_almost_equal( + self.c0.gradient(self.c0_p1_inside), CArray([0, 0])) + # [1. 1.] is the center of the constraint, expected grad [0, c0] - # however, numerical gradient is struggling so we avoid its omparison + # however, numerical gradient is struggling so we avoid its comparison self.assert_array_almost_equal( - self.c.gradient(self.p1_inside), CArray([0, 0])) + self.c.gradient(self.c_p1_inside), CArray([0, 0])) + + self._test_gradient(self.c0, CArray([0.1, 0.2])) + self._test_gradient(self.c0, self.c0_p2_outside) self._test_gradient(self.c, CArray([1.1, 1.2])) - self._test_gradient(self.c, self.p2_outside) + self._test_gradient(self.c, self.c_p2_outside) + # [0. 1.] is the verge of the constraint, expected grad [0, 1] + # however, numerical gradient is struggling so we avoid its comparison + self.assert_array_almost_equal( + self.c0.gradient(self.c0_p3_on), CArray([0., 1.])) # [0. 1.] is the verge of the constraint, expected grad [-1, 1] # however, numerical gradient is struggling so we avoid its comparison self.assert_array_almost_equal( - self.c.gradient(self.p3_on), CArray([-1., 0.])) + self.c.gradient(self.c_p3_on), CArray([-1., 0.])) def test_subgradient(self): """Check if the subgradient is computed correctly @@ -91,14 +140,12 @@ def test_subgradient(self): Subgradient should lie in the cone made up by the subgradients. """ - c = CConstraintL1(center=0, radius=1) - x0 = CArray([0, 1]) p_min = CArray([1, 1]) p_max = CArray([-1, 1]) - gradient = c.gradient(x0) + gradient = self.c0.gradient(x0) # normalize the points norm_center = x0 / x0.norm(2) @@ -117,8 +164,16 @@ def test_subgradient(self): def test_plot(self): """Visualize the constraint.""" # Plotting constraint and "critical" points - self._test_plot( - self.c, self.p1_inside, self.p2_outside, self.p3_on) + self._test_plot(self.c0, + self.c0_p1_inside, + self.c0_p2_outside, + self.c0_p3_on, + label='c0') + self._test_plot(self.c, + self.c_p1_inside, + self.c_p2_outside, + self.c_p3_on, + label='c') if __name__ == '__main__': diff --git a/src/secml/optim/optimizers/c_optimizer.py b/src/secml/optim/optimizers/c_optimizer.py index 4adf6444..fe648c3e 100644 --- a/src/secml/optim/optimizers/c_optimizer.py +++ b/src/secml/optim/optimizers/c_optimizer.py @@ -25,9 +25,12 @@ class COptimizer(CCreator, metaclass=ABCMeta): Parameters ---------- fun : CFunction - The objective function to be optimized, - along with 1st-order (Jacobian) and 2nd-order (Hessian) derivatives - (if available). + The objective function to be optimized, along with 1st-order (Jacobian) + and 2nd-order (Hessian) derivatives (if available). + constr : CConstraintL1 or CConstraintL2 or None, optional + A distance constraint. Default None. + bounds : CConstraintBox or None, optional + A box constraint. Default None. """ __super__ = 'COptimizer' diff --git a/src/secml/optim/optimizers/c_optimizer_pgd.py b/src/secml/optim/optimizers/c_optimizer_pgd.py index cddeb2fc..3ae8aaa7 100644 --- a/src/secml/optim/optimizers/c_optimizer_pgd.py +++ b/src/secml/optim/optimizers/c_optimizer_pgd.py @@ -23,6 +23,22 @@ class COptimizerPGD(COptimizer): The solution algorithm is based on the classic gradient descent algorithm. + Parameters + ---------- + fun : CFunction + The objective function to be optimized, along with 1st-order (Jacobian) + and 2nd-order (Hessian) derivatives (if available). + constr : CConstraintL1 or CConstraintL2 or None, optional + A distance constraint. Default None. + bounds : CConstraintBox or None, optional + A box constraint. Default None. + eta : scalar, optional + Step of the Projected Gradient Descent. Default 1e-3. + eps : scalar, optional + Tolerance of the stop criterion. Default 1e-4. + max_iter : int, optional + Maximum number of iterations. Default 200. + Attributes ---------- class_type : 'pgd' diff --git a/src/secml/optim/optimizers/c_optimizer_pgd_exp.py b/src/secml/optim/optimizers/c_optimizer_pgd_exp.py index 72c86978..0762db0f 100644 --- a/src/secml/optim/optimizers/c_optimizer_pgd_exp.py +++ b/src/secml/optim/optimizers/c_optimizer_pgd_exp.py @@ -5,13 +5,14 @@ .. moduleauthor:: Battista Biggio """ +import numpy as np from secml.array import CArray -from secml.optim.optimizers import COptimizer +from secml.optim.optimizers import COptimizerPGDLS from secml.optim.optimizers.line_search import CLineSearchBisectProj -class COptimizerPGDExp(COptimizer): +class COptimizerPGDExp(COptimizerPGDLS): """Solves the following problem: min f(x) @@ -24,8 +25,30 @@ class COptimizerPGDExp(COptimizer): The solution algorithm is based on a line-search exploring one feature (i.e., dimension) at a time (for l1-constrained problems), or all features - (for l2-constrained problems). + (for l2-constrained problems). This solver also works for discrete + problems where x and the grid discretization (eta) are integer valued. + Parameters + ---------- + fun : CFunction + The objective function to be optimized, along with 1st-order (Jacobian) + and 2nd-order (Hessian) derivatives (if available). + constr : CConstraintL1 or CConstraintL2 or None, optional + A distance constraint. Default None. + bounds : CConstraintBox or None, optional + A box constraint. Default None. + eta : scalar, optional + Minimum resolution of the line-search grid. Default 1e-3. + eta_min : scalar or None, optional + Initial step of the line search. Gets multiplied or divided by 2 + at each step until convergence. If None, will be set equal to eta. + Default None. + eta_max : scalar or None, optional + Maximum step of the line search. Default None. + max_iter : int, optional + Maximum number of iterations. Default 1000. + eps : scalar, optional + Tolerance of the stop criterion. Default 1e-4. Attributes ---------- @@ -36,98 +59,23 @@ class COptimizerPGDExp(COptimizer): def __init__(self, fun, constr=None, bounds=None, - discrete=False, eta=1e-3, eta_min=None, eta_max=None, max_iter=1000, eps=1e-4): - COptimizer.__init__(self, fun=fun, - constr=constr, bounds=bounds) - - # Read/write attributes - self.eta = eta - self.eta_min = eta_min - self.eta_max = eta_max - self.max_iter = max_iter - self.eps = eps - self.discrete = discrete - - # Internal attributes - self._line_search = None - - ########################################################################### - # READ-WRITE ATTRIBUTES - ########################################################################### - - @property - def eta(self): - return self._eta - - @eta.setter - def eta(self, value): - self._eta = value - - @property - def eta_min(self): - return self._eta_min - - @eta_min.setter - def eta_min(self, value): - self._eta_min = value - - @property - def eta_max(self): - return self._eta_max - - @eta_max.setter - def eta_max(self, value): - self._eta_max = value - - @property - def max_iter(self): - """Returns the maximum number of descent iterations""" - return self._max_iter - - @max_iter.setter - def max_iter(self, value): - """Set the maximum number of descent iterations""" - self._max_iter = int(value) - - @property - def eps(self): - """Return tolerance value for stop criterion""" - return self._eps - - @eps.setter - def eps(self, value): - """Set tolerance value for stop criterion""" - self._eps = float(value) - - @property - def discrete(self): - """True if feature space is discrete, False if continuous.""" - return self._discrete - - @discrete.setter - def discrete(self, value): - """True if feature space is discrete, False if continuous.""" - self._discrete = bool(value) + COptimizerPGDLS.__init__( + self, fun=fun, constr=constr, bounds=bounds, + eta=eta, eta_min=eta_min, eta_max=eta_max, + max_iter=max_iter, eps=eps) ########################################## # METHODS ########################################## - def _init_line_search( - self, eta, eta_min, eta_max, discrete): + def _init_line_search(self, eta, eta_min, eta_max): """Initialize line-search optimizer""" - - if discrete is True and self.constr is not None and \ - self.constr.class_type == 'l2': - raise NotImplementedError( - "L2 constraint is not supported for discrete optimization") - self._line_search = CLineSearchBisectProj( fun=self._fun, constr=self._constr, @@ -135,50 +83,6 @@ def _init_line_search( max_iter=20, eta=eta, eta_min=eta_min, eta_max=eta_max) - @staticmethod - def _l1_projected_gradient(grad): - """ - Find v that maximizes v'grad onto the unary-norm l1 ball. - This is the maximization of an inner product over the l1 ball, - and the optimal (sparse) direction v is found by setting - v = sign(grad) when abs(grad) is maximum and 0 elsewhere. - """ - abs_grad = abs(grad) - grad_max = abs_grad.max() - argmax_pos = abs_grad == grad_max - # TODO: not sure if proj_grad should be always sparse - # (grad is not) - proj_grad = CArray.zeros(shape=grad.shape, sparse=grad.issparse) - proj_grad[argmax_pos] = grad[argmax_pos].sign() - return proj_grad - - def _box_projected_gradient(self, x, grad): - """ - Exclude from descent direction those features which, - if modified according to the given descent direction, - would violate the box constraint. - - """ - if self.bounds is None: - return grad # all features are feasible - - # x_lb and x_ub are feature manipulations that violate box - # (the first vector is potentially sparse, so it has to be - # the first argument of logical_and to avoid conversion to dense) - # FIXME: the following condition is error-prone. - # Use (ad wrap in CArray) np.isclose with atol=1e-6, rtol=0 - # FIXME: converting grad to dense as the sparse vs sparse logical_and - # is too slow - x_lb = (x.round(6) == CArray(self.bounds.lb).round(6)).logical_and( - grad.todense() > 0).astype(bool) - - x_ub = (x.round(6) == CArray(self.bounds.ub).round(6)).logical_and( - grad.todense() < 0).astype(bool) - - # reset gradient for unfeasible features - grad[x_lb + x_ub] = 0 - return grad - def _xk(self, x, fx, *args): """Returns a new point after gradient descent.""" @@ -195,12 +99,12 @@ def _xk(self, x, fx, *args): # filter modifications that would violate bounds (to sparsify gradient) grad = self._box_projected_gradient(x, grad) - if self.discrete or ( - self.constr is not None and self.constr.class_type == 'l1'): + if self.constr is not None and self.constr.class_type == 'l1': # project z onto l1 constraint (via dual norm) grad = self._l1_projected_gradient(grad) - next_point = x - grad * self._line_search.eta + next_point = CArray(x - grad * self._line_search.eta, + dtype=self._dtype, tosparse=x.issparse) if self.constr is not None and self.constr.is_violated(next_point): self.logger.debug("Line-search on distance constraint.") @@ -287,8 +191,7 @@ def minimize(self, x_init, args=(), **kwargs): # initialize line search (and re-assign fun to it) self._init_line_search(eta=self.eta, eta_min=self.eta_min, - eta_max=self.eta_max, - discrete=self.discrete) + eta_max=self.eta_max) # constr.radius = 0, exit if self.constr is not None and self.constr.radius == 0: @@ -299,36 +202,43 @@ def minimize(self, x_init, args=(), **kwargs): warnings.warn( "x0 " + str(x0) + " is outside of the given bounds.", category=RuntimeWarning) - self._x_seq = CArray.zeros((1, x0.size), - sparse=x0.issparse, dtype=x0.dtype) + self._x_seq = CArray.zeros( + (1, x0.size), sparse=x0.issparse, dtype=x0.dtype) self._f_seq = CArray.zeros(1) self._x_seq[0, :] = x0 self._f_seq[0] = self._fun.fun(x0, *args) self._x_opt = x0 return x0 + x = x_init.deepcopy() # TODO: IS DEEPCOPY REALLY NEEDED? + # if x is outside of the feasible domain, project it - if self.bounds is not None and self.bounds.is_violated(x_init): - x_init = self.bounds.projection(x_init) + if self.bounds is not None and self.bounds.is_violated(x): + x = self.bounds.projection(x) - if self.constr is not None and self.constr.is_violated(x_init): - x_init = self.constr.projection(x_init) + if self.constr is not None and self.constr.is_violated(x): + x = self.constr.projection(x) - if (self.bounds is not None and self.bounds.is_violated(x_init)) or \ - (self.constr is not None and self.constr.is_violated(x_init)): + if (self.bounds is not None and self.bounds.is_violated(x)) or \ + (self.constr is not None and self.constr.is_violated(x)): raise ValueError( - "x_init " + str(x_init) + " is outside of feasible domain.") + "x_init " + str(x) + " is outside of feasible domain.") + + # dtype depends on x and eta (the grid discretization) + if np.issubdtype(x_init.dtype, np.floating): + # x is float, res dtype should be float + self._dtype = x_init.dtype + else: # x is int, res dtype depends on the grid discretization + self._dtype = self._line_search.eta.dtype # initialize x_seq and f_seq - self._x_seq = CArray.zeros( - (self.max_iter, x_init.size), sparse=x_init.issparse) - if self.discrete is True: - self._x_seq.astype(x_init.dtype) # this may set x_seq to int + self._x_seq = CArray.zeros((self.max_iter, x_init.size), + sparse=x_init.issparse, + dtype=self._dtype) self._f_seq = CArray.zeros(self.max_iter) # The first point is obviously the starting point, # and the constraint is not violated (false...) - x = x_init.deepcopy() fx = self._fun.fun(x, *args) # eval fun at x, for iteration 0 self._x_seq[0, :] = x self._f_seq[0] = fx diff --git a/src/secml/optim/optimizers/c_optimizer_pgd_ls.py b/src/secml/optim/optimizers/c_optimizer_pgd_ls.py index fffc2a05..629ca423 100644 --- a/src/secml/optim/optimizers/c_optimizer_pgd_ls.py +++ b/src/secml/optim/optimizers/c_optimizer_pgd_ls.py @@ -5,6 +5,7 @@ .. moduleauthor:: Battista Biggio """ +import numpy as np from secml.array import CArray from secml.optim.optimizers import COptimizer @@ -25,13 +26,34 @@ class COptimizerPGDLS(COptimizer): The solution algorithm is based on a line-search exploring one feature (i.e., dimension) at a time (for l1-constrained problems), or all features (for l2-constrained problems). This solver also works for discrete - problems, where x is integer valued. In this case, exploration works - by manipulating one feature at a time. + problems where x and the grid discretization (eta) are integer valued. Differently from standard line searches, it explores a subset of `n_dimensions` at a time. In this sense, it is an extension of the classical line-search approach. + Parameters + ---------- + fun : CFunction + The objective function to be optimized, along with 1st-order (Jacobian) + and 2nd-order (Hessian) derivatives (if available). + constr : CConstraintL1 or CConstraintL2 or None, optional + A distance constraint. Default None. + bounds : CConstraintBox or None, optional + A box constraint. Default None. + eta : scalar, optional + Minimum resolution of the line-search grid. Default 1e-3. + eta_min : scalar or None, optional + Initial step of the line search. Gets multiplied or divided by 2 + at each step until convergence. If None, will be set equal to eta. + Default None. + eta_max : scalar or None, optional + Maximum step of the line search. Default None. + max_iter : int, optional + Maximum number of iterations. Default 1000. + eps : scalar, optional + Tolerance of the stop criterion. Default 1e-4. + Attributes ---------- class_type : 'pgd-ls' @@ -41,7 +63,6 @@ class COptimizerPGDLS(COptimizer): def __init__(self, fun, constr=None, bounds=None, - discrete=False, eta=1e-3, eta_min=None, eta_max=None, @@ -57,10 +78,10 @@ def __init__(self, fun, self.eta_max = eta_max self.max_iter = max_iter self.eps = eps - self.discrete = discrete # Internal attributes self._line_search = None + self._dtype = None ########################################################################### # READ-WRITE ATTRIBUTES @@ -110,29 +131,12 @@ def eps(self, value): """Set tolerance value for stop criterion""" self._eps = float(value) - @property - def discrete(self): - """True if feature space is discrete, False if continuous.""" - return self._discrete - - @discrete.setter - def discrete(self, value): - """True if feature space is discrete, False if continuous.""" - self._discrete = bool(value) - ########################################## # METHODS ########################################## - def _init_line_search( - self, eta, eta_min, eta_max, discrete): + def _init_line_search(self, eta, eta_min, eta_max): """Initialize line-search optimizer""" - - if discrete is True and self.constr is not None and \ - self.constr.class_type == 'l2': - raise NotImplementedError( - "L2 constraint is not supported for discrete optimization") - self._line_search = CLineSearchBisect( fun=self._fun, constr=self._constr, @@ -150,10 +154,11 @@ def _l1_projected_gradient(grad): """ abs_grad = abs(grad) grad_max = abs_grad.max() - argmax_pos = abs_grad == grad_max - # TODO: not sure if proj_grad should be always sparse - # (grad is not) - proj_grad = CArray.zeros(shape=grad.shape, sparse=grad.issparse) + # Get one of the features having grad_max value + argmax_pos = CArray(abs_grad == grad_max).nnz_indices[1][:1] + # TODO: not sure if proj_grad should be always sparse (grad is not) + proj_grad = CArray.zeros( + shape=grad.shape, sparse=grad.issparse, dtype=grad.dtype) proj_grad[argmax_pos] = grad[argmax_pos].sign() return proj_grad @@ -200,12 +205,12 @@ def _xk(self, x, fx, *args): # filter modifications that would violate bounds (to sparsify gradient) grad = self._box_projected_gradient(x, grad) - if self.discrete or ( - self.constr is not None and self.constr.class_type == 'l1'): + if self.constr is not None and self.constr.class_type == 'l1': # project z onto l1 constraint (via dual norm) grad = self._l1_projected_gradient(grad) - next_point = x - grad * self._line_search.eta + next_point = CArray(x - grad * self._line_search.eta, + dtype=self._dtype, tosparse=x.issparse) if self.constr is not None and self.constr.is_violated(next_point): self.logger.debug("Line-search on distance constraint.") @@ -257,8 +262,7 @@ def minimize(self, x_init, args=(), **kwargs): # initialize line search (and re-assign fun to it) self._init_line_search(eta=self.eta, eta_min=self.eta_min, - eta_max=self.eta_max, - discrete=self.discrete) + eta_max=self.eta_max) # constr.radius = 0, exit if self.constr is not None and self.constr.radius == 0: @@ -269,36 +273,43 @@ def minimize(self, x_init, args=(), **kwargs): warnings.warn( "x0 " + str(x0) + " is outside of the given bounds.", category=RuntimeWarning) - self._x_seq = CArray.zeros((1, x0.size), - sparse=x0.issparse, dtype=x0.dtype) + self._x_seq = CArray.zeros( + (1, x0.size), sparse=x0.issparse, dtype=x0.dtype) self._f_seq = CArray.zeros(1) self._x_seq[0, :] = x0 self._f_seq[0] = self._fun.fun(x0, *args) self._x_opt = x0 return x0 + x = x_init.deepcopy() # TODO: IS DEEPCOPY REALLY NEEDED? + # if x is outside of the feasible domain, project it - if self.bounds is not None and self.bounds.is_violated(x_init): - x_init = self.bounds.projection(x_init) + if self.bounds is not None and self.bounds.is_violated(x): + x = self.bounds.projection(x) - if self.constr is not None and self.constr.is_violated(x_init): - x_init = self.constr.projection(x_init) + if self.constr is not None and self.constr.is_violated(x): + x = self.constr.projection(x) - if (self.bounds is not None and self.bounds.is_violated(x_init)) or \ - (self.constr is not None and self.constr.is_violated(x_init)): + if (self.bounds is not None and self.bounds.is_violated(x)) or \ + (self.constr is not None and self.constr.is_violated(x)): raise ValueError( - "x_init " + str(x_init) + " is outside of feasible domain.") + "x_init " + str(x) + " is outside of feasible domain.") + + # dtype depends on x and eta (the grid discretization) + if np.issubdtype(x_init.dtype, np.floating): + # x is float, res dtype should be float + self._dtype = x_init.dtype + else: # x is int, res dtype depends on the grid discretization + self._dtype = self._line_search.eta.dtype # initialize x_seq and f_seq - self._x_seq = CArray.zeros( - (self.max_iter, x_init.size), sparse=x_init.issparse) - if self.discrete is True: - self._x_seq.astype(x_init.dtype) # this may set x_seq to int + self._x_seq = CArray.zeros((self.max_iter, x_init.size), + sparse=x_init.issparse, + dtype=self._dtype) self._f_seq = CArray.zeros(self.max_iter) # The first point is obviously the starting point, # and the constraint is not violated (false...) - x = x_init.deepcopy() fx = self._fun.fun(x, *args) # eval fun at x, for iteration 0 self._x_seq[0, :] = x self._f_seq[0] = fx diff --git a/src/secml/optim/optimizers/line_search/c_line_search.py b/src/secml/optim/optimizers/line_search/c_line_search.py index cb91edaf..da180e27 100644 --- a/src/secml/optim/optimizers/line_search/c_line_search.py +++ b/src/secml/optim/optimizers/line_search/c_line_search.py @@ -18,6 +18,19 @@ class CLineSearch(CCreator, metaclass=ABCMeta): direction in the feasible domain, potentially subject to constraints. The search is normally stopped when the objective improves at a satisfying level, to keep the search fast. + + Parameters + ---------- + fun : CFunction + The function to use for the optimization. + constr : CConstraintL1 or CConstraintL2 or None, optional + A distance constraint. Default None. + bounds : CConstraintBox or None, optional + A box constraint. Default None. + eta : scalar, optional + Minimum resolution of the line-search grid. Default 1e-4. + max_iter : int, optional + Maximum number of iterations of the line search. Default 20. """ __super__ = 'CLineSearch' diff --git a/src/secml/optim/optimizers/line_search/c_line_search_bisect.py b/src/secml/optim/optimizers/line_search/c_line_search_bisect.py index b4ff2900..12260456 100644 --- a/src/secml/optim/optimizers/line_search/c_line_search_bisect.py +++ b/src/secml/optim/optimizers/line_search/c_line_search_bisect.py @@ -14,6 +14,25 @@ class CLineSearchBisect(CLineSearch): """Binary line search. + Parameters + ---------- + fun : CFunction + The function to use for the optimization. + constr : CConstraintL1 or CConstraintL2 or None, optional + A distance constraint. Default None. + bounds : CConstraintBox or None, optional + A box constraint. Default None. + eta : scalar, optional + Minimum resolution of the line-search grid. Default 1e-4. + eta_min : scalar or None, optional + Initial step of the line search. Gets multiplied or divided by 2 + at each step until convergence. If None, will be set equal to eta. + Default 0.1. + eta_max : scalar or None, optional + Maximum step of the line search. Default None. + max_iter : int, optional + Maximum number of iterations of the line search. Default 20. + Attributes ---------- class_type : 'bisect' @@ -43,6 +62,7 @@ def __init__(self, fun, constr=None, bounds=None, self._fz = None # cached value of fun at current z during line search self._fun_idx_max = None self._fun_idx_min = None + self._dtype = None @property def eta_max(self): @@ -91,7 +111,7 @@ def n_iter(self): def _update_z(self, x, eta, d): """Update z and its cached score fz.""" - z = x + eta * d + z = CArray(x + eta * d, dtype=self._dtype, tosparse=x.issparse) self._fz = self.fun.fun(z) return z @@ -110,27 +130,15 @@ def _is_feasible(self, x): def _select_best_point(self, x, d, idx_min, idx_max, **kwargs): """Returns best point among x and the two points found by the search. In practice, if f(x + eta*d) increases on d, we return x.""" - - # dtype of x1 and x2 depends on x and eta (the grid discretization) - if np.issubdtype(x.dtype, np.floating): - # if x is float res dtype should be float - dtype = x.dtype - else: # x is int, so the res dtype depends on the grid discretization - dtype = self.eta.dtype - x1 = CArray(x + d * self.eta * idx_min, - dtype=dtype, tosparse=x.issparse) + dtype=self._dtype, tosparse=x.issparse) x2 = CArray(x + d * self.eta * idx_max, - dtype=dtype, tosparse=x.issparse) + dtype=self._dtype, tosparse=x.issparse) - self.logger.info("Select best point between: " + - str(x + self.eta * d * idx_min) + ", " + - str(x + self.eta * d * idx_max) + ", " + - str(x)) - self.logger.debug("f[a], f[b]: [" + - str(self._fun_idx_min) + "," + - str(self._fun_idx_max) + "]") - self.logger.debug("f[x] " + str(self._fx)) + self.logger.info("Select best point between...") + self.logger.info("x (f: {:}) -> \n{:}".format(self._fx, x)) + self.logger.info("x1 (f: {:}) -> \n{:}".format(self._fun_idx_min, x1)) + self.logger.info("x2 (f: {:}) -> \n{:}".format(self._fun_idx_max, x2)) f0 = self._fx @@ -168,11 +176,11 @@ def _select_best_point(self, x, d, idx_min, idx_max, **kwargs): # else return best point among x1, x2 and x self.logger.debug("f0: {:}, f1: {:}, f2: {:}".format(f0, f1, f2)) - if f2 <= f0 and f2 < f1: + if f2 <= f0 and f2 <= f1: self.logger.debug("Returning x2.") return x2, f2 - if f1 <= f0 and f1 < f2: + if f1 <= f0 and f1 <= f2: self.logger.debug("Returning x1.") return x1, f1 @@ -196,10 +204,10 @@ def _is_decreasing(self, x, d, **kwargs): delta = self.fun.fun(x + 0.1 * self.eta * d, **kwargs) - self._fz if delta <= 0: - # feasible point, decreasing / stationary score + # feasible point and decreasing / stationary score return True - # feasible point, increasing score + # feasible point but increasing score return False def _compute_eta_max(self, x, d, **kwargs): @@ -297,7 +305,14 @@ def minimize(self, x, d, fx=None, tol=1e-4, **kwargs): The value `f(x')`. """ - d = CArray(d, tosparse=d.issparse).ravel() + d = CArray(d).ravel() + + # dtype depends on x and eta (the grid discretization) + if np.issubdtype(x.dtype, np.floating): + # if x is float res dtype should be float + self._dtype = x.dtype + else: # x is int, so the res dtype depends on the grid discretization + self._dtype = self.eta.dtype self._n_iter = 0 @@ -317,25 +332,27 @@ def minimize(self, x, d, fx=None, tol=1e-4, **kwargs): # exponential search if self.eta_max is None: - self.logger.debug("Exponential search ") + self.logger.debug("Exponential search") eta_max = self._compute_eta_max(x, d, **kwargs) idx_max = (eta_max / self.eta).ceil().astype(int) idx_min = (idx_max / 2).astype(int) # this only searches within [eta, 2*eta] # the values fun_idx_min and fun_idx_max are already cached else: - self.logger.debug("Binary search ") + self.logger.debug("Binary search") idx_max = (self.eta_max / self.eta).ceil().astype(int) idx_min = 0 self._fun_idx_min = self._fx self._fun_idx_max = None # this has not been cached - self.logger.info("Running binary line search in: [" + - str(x + self.eta * d * idx_min) + "," + - str(x + self.eta * d * idx_max) + "]") - self.logger.debug("f[a], f[b]: [" + - str(self._fun_idx_min) + "," + - str(self._fun_idx_max) + "]") + x1 = CArray(x + d * self.eta * idx_min, + dtype=self._dtype, tosparse=x.issparse) + x2 = CArray(x + d * self.eta * idx_max, + dtype=self._dtype, tosparse=x.issparse) + + self.logger.info("Running binary line search in...") + self.logger.info("x1 (f: {:}) -> \n{:}".format(self._fun_idx_min, x1)) + self.logger.info("x2 (f: {:}) -> \n{:}".format(self._fun_idx_max, x2)) while self._n_iter < self.max_iter: diff --git a/src/secml/optim/optimizers/line_search/c_line_search_bisect_proj.py b/src/secml/optim/optimizers/line_search/c_line_search_bisect_proj.py index 7e9b4483..6ebe1976 100644 --- a/src/secml/optim/optimizers/line_search/c_line_search_bisect_proj.py +++ b/src/secml/optim/optimizers/line_search/c_line_search_bisect_proj.py @@ -5,6 +5,7 @@ .. moduleauthor:: Battista Biggio """ +import numpy as np from secml.array import CArray from secml.optim.optimizers.line_search import CLineSearchBisect @@ -13,6 +14,25 @@ class CLineSearchBisectProj(CLineSearchBisect): """Binary line search including projections. + Parameters + ---------- + fun : CFunction + The function to use for the optimization. + constr : CConstraintL1 or CConstraintL2 or None, optional + A distance constraint. Default None. + bounds : CConstraintBox or None, optional + A box constraint. Default None. + eta : scalar, optional + Minimum resolution of the line-search grid. Default 1e-4. + eta_min : scalar or None, optional + Initial step of the line search. Gets multiplied or divided by 2 + at each step until convergence. If None, will be set equal to eta. + Default 0.1. + eta_max : scalar or None, optional + Maximum step of the line search. Default None. + max_iter : int, optional + Maximum number of iterations of the line search. Default 20. + Attributes ---------- class_type : 'bisect-proj' @@ -30,10 +50,11 @@ def __init__(self, fun, constr=None, bounds=None, self._best_score = None self._best_eta = None + self._dtype = None def _update_z(self, x, eta, d, projection=False): """Update z and its cached score fz.""" - z = x + eta * d + z = CArray(x + eta * d, dtype=self._dtype, tosparse=x.issparse) if projection: z = self.bounds.projection(z) if self.bounds is not None else z z = self.constr.projection(z) if self.constr is not None else z @@ -59,7 +80,8 @@ def _select_best_point(self, x, d, idx_min, idx_max, **kwargs): """Returns best point among x and the two points found by the search. In practice, if f(x + eta*d) increases on d, we return x.""" - v = x + d * self._best_eta + v = CArray(x + d * self._best_eta, + dtype=self._dtype, tosparse=x.issparse) if self.bounds is not None: v = self.bounds.projection(v) if self.bounds is not None else v if self.constr is not None: @@ -67,13 +89,15 @@ def _select_best_point(self, x, d, idx_min, idx_max, **kwargs): if self._is_feasible(v): return v, self._best_score - x1 = CArray(x + d * self.eta * idx_min) + x1 = CArray(x + d * self.eta * idx_min, + dtype=self._dtype, tosparse=x.issparse) if self.bounds is not None: x1 = self.bounds.projection(x1) if self.bounds is not None else x1 if self.constr is not None: x1 = self.constr.projection(x1) if self.constr is not None else x1 - x2 = CArray(x + d * self.eta * idx_max) + x2 = CArray(x + d * self.eta * idx_max, + dtype=self._dtype, tosparse=x.issparse) if self.bounds is not None: x2 = self.bounds.projection(x2) if self.bounds is not None else x2 if self.constr is not None: @@ -186,6 +210,8 @@ def _compute_eta_max(self, x, d, **kwargs): # this helps getting closer to the violated constraint t = CArray(eta / self.eta).round() + # FIXME: MANY UNUSED VARIABLES IN THE FOLLOWING + # update z and fz z = self._update_z(x, eta, d, projection=True) @@ -263,7 +289,14 @@ def minimize(self, x, d, fx=None, tol=1e-4, **kwargs): The value `f(x')`. """ - d = CArray(d, tosparse=d.issparse).ravel() + d = CArray(d).ravel() + + # dtype depends on x and eta (the grid discretization) + if np.issubdtype(x.dtype, np.floating): + # if x is float res dtype should be float + self._dtype = x.dtype + else: # x is int, so the res dtype depends on the grid discretization + self._dtype = self.eta.dtype self._n_iter = 0 diff --git a/src/secml/optim/optimizers/tests/c_optimizer_testcases.py b/src/secml/optim/optimizers/tests/c_optimizer_testcases.py index d19602c9..78fab29d 100644 --- a/src/secml/optim/optimizers/tests/c_optimizer_testcases.py +++ b/src/secml/optim/optimizers/tests/c_optimizer_testcases.py @@ -1,5 +1,7 @@ from secml.testing import CUnitTest +import os + from secml.optim.function import CFunction from secml.optim.constraints import CConstraintBox, CConstraintL2 from secml.array import CArray @@ -9,6 +11,7 @@ class COptimizerTestCases(CUnitTest): """Unittests interface for COptimizer.""" + make_figures = os.getenv('MAKE_FIGURES', False) # True to produce figures def setUp(self): @@ -56,7 +59,7 @@ def setUp(self): poly = self._create_poly(d=n) self.test_funcs['poly-2'] = { 'fun': poly, - 'x0': CArray.ones((n,)) * 2, + 'x0': CArray.ones((n,), dtype=int) * 2, 'vmin': -10, 'vmax': 5, 'grid_limits': [(-1, 1), (-1, 1)] } @@ -161,7 +164,7 @@ def _test_minimize(self, opt_class, fun_id, self.logger.info("Found minimum: {:}".format(min_x)) self.logger.info("Fun value @ minimum: {:}".format(opt.f_opt)) - if fun.global_min_x().size == 2: + if self.make_figures and fun.global_min_x().size == 2: self._plot_optimization(opt, fun_dict['x0'], min_x, grid_limits=fun_dict['grid_limits'], method=minimize_params.get('method'), @@ -180,8 +183,9 @@ def _test_minimize(self, opt_class, fun_id, # Check if solution has expected int dtype or not self.assertIsSubDtype(min_x.dtype, int if out_int is True else float) + @staticmethod def _plot_optimization( - self, solver, x_0, g_min, grid_limits, + solver, x_0, g_min, grid_limits, method=None, vmin=None, vmax=None, label=None): """Plots the optimization problem. diff --git a/src/secml/optim/optimizers/tests/test_c_optimizer_pgd_ls_discrete.py b/src/secml/optim/optimizers/tests/test_c_optimizer_pgd_ls_discrete.py index 90ece07a..ba4f9807 100644 --- a/src/secml/optim/optimizers/tests/test_c_optimizer_pgd_ls_discrete.py +++ b/src/secml/optim/optimizers/tests/test_c_optimizer_pgd_ls_discrete.py @@ -1,48 +1,71 @@ from secml.optim.optimizers.tests import COptimizerTestCases +from secml.array import CArray from secml.optim.optimizers import COptimizerPGDLS -from secml.optim.constraints import CConstraintBox, CConstraintL2 +from secml.optim.constraints import CConstraintBox, CConstraintL1 class TestCOptimizerPGDLSDiscrete(COptimizerTestCases): - """Unittests for COptimizerPGDLSDiscrete.""" + """Unittests for COptimizerPGDLS in discrete space.""" def test_minimize_3h_camel(self): - """Test for COptimizer.minimize() method on 3h-camel fun. This - function tests the optimization in discrete space, with a floating - eta and an integer starting point. The solution expected by this - test is a float vector.""" + """Test for COptimizer.minimize() method on 3h-camel fun. + This function tests the optimization in discrete space, + with an integer eta and an integer starting point. + The solution expected by this test is a integer vector. + """ + opt_params = { + 'eta': 1, 'eta_min': 1, 'eps': 1e-12, + 'bounds': CConstraintBox(lb=-1, ub=1) + } - opt_params = {'eta': 0.5, 'eta_min': 0.5, 'eps': 1e-12, - 'discrete': True, - 'bounds': CConstraintBox(lb=-1, ub=1)} + self._test_minimize(COptimizerPGDLS, '3h-camel', + opt_params=opt_params, + label='discrete', + out_int=True) + + def test_minimize_3h_camel_l1(self): + """Test for COptimizer.minimize() method on 3h-camel fun. + This function tests the optimization in discrete space, + with a floating eta (l1 constraint) and an integer starting point. + The solution expected by this test is a float vector. + """ + opt_params = { + 'eta': 0.5, 'eta_min': 0.5, 'eps': 1e-12, + 'constr': CConstraintL1(radius=2), + 'bounds': CConstraintBox(lb=-1, ub=1) + } self._test_minimize(COptimizerPGDLS, '3h-camel', opt_params=opt_params, - label='discrete') + label='discrete-l1') def test_minimize_beale(self): - """Test for COptimizer.minimize() method on 3h-camel fun. This - function tests the optimization in discrete space, with a floating - eta and an integer starting point. The solution expected by this - test is a float vector.""" - opt_params = {'eta': 1e-6, 'eta_min': 1e-4, 'eps': 1e-12, - 'discrete': True, - 'bounds': CConstraintBox(lb=0, ub=4)} + """Test for COptimizer.minimize() method on 3h-camel fun. + This function tests the optimization in discrete space, + with a floating eta (l1 constraint) and an integer starting point. + The solution expected by this test is a float vector. + """ + opt_params = { + 'eta': 1e-6, 'eta_min': 1e-4, 'eps': 1e-12, + 'constr': CConstraintL1(center=CArray([2, 0]), radius=2), + 'bounds': CConstraintBox(lb=0, ub=4) + } self._test_minimize(COptimizerPGDLS, 'beale', opt_params=opt_params, - label='discrete') + label='discrete-l1') def test_minimize_quad2d_no_bound(self): """Test for COptimizer.minimize() method on a quadratic function in - a 2-dimensional space. This function tests the optimization in discrete - space, with an integer eta, an integer starting point and without any - bound. The solution expected by this test is an integer vector.""" - - # test a simple function without any bound - opt_params = {'eta': 1, 'eta_min': 1, 'eps': 1e-12, - 'discrete': True} + a 2-dimensional space. + This function tests the optimization in discrete space, + with an integer eta, an integer starting point and without any bound. + The solution expected by this test is an integer vector. + """ + opt_params = { + 'eta': 1, 'eta_min': 1, 'eps': 1e-12 + } # both the starting point and eta are integer, # therefore we expect an integer solution @@ -53,14 +76,15 @@ def test_minimize_quad2d_no_bound(self): def test_minimize_quad2d_bound(self): """Test for COptimizer.minimize() method on a quadratic function in - a 2-dimensional space. This function tests the optimization in discrete - space, with an integer eta, an integer starting point and with a box - constraint. The solution expected by this test is an integer vector.""" - - # Testing bounded optimization - opt_params = {'eta': 1, 'eta_min': 1, 'eps': 1e-12, - 'discrete': True, - 'bounds': CConstraintBox(lb=-2, ub=3)} + a 2-dimensional space. + This function tests the optimization in discrete space, with an + integer eta, an integer starting point and with a box constraint. + The solution expected by this test is an integer vector. + """ + opt_params = { + 'eta': 1, 'eta_min': 1, 'eps': 1e-12, + 'bounds': CConstraintBox(lb=-2, ub=3) + } self._test_minimize( COptimizerPGDLS, 'quad-2', @@ -70,14 +94,15 @@ def test_minimize_quad2d_bound(self): def test_minimize_quad100d_sparse(self): """Test for COptimizer.minimize() method on a quadratic function in - a 100-dimensional space. This function tests the optimization in - discrete space, with an integer eta, an integer and sparse starting - point with box constraint. The solution expected by this test is an - integer and sparse vector.""" - - opt_params = {'eta': 1, 'eta_min': 1, 'eps': 1e-12, - 'discrete': True, - 'bounds': CConstraintBox(lb=-2, ub=3)} + a 100-dimensional space. + This function tests the optimization in discrete space, with an + integer eta, an integer and sparse starting point with box constraint. + The solution expected by this test is an integer sparse vector. + """ + opt_params = { + 'eta': 1, 'eta_min': 1, 'eps': 1e-12, + 'bounds': CConstraintBox(lb=-2, ub=3) + } self._test_minimize( COptimizerPGDLS, 'quad-100-sparse', @@ -85,32 +110,55 @@ def test_minimize_quad100d_sparse(self): label='quad-100-sparse-discrete-bounded', out_int=True) - def test_minimize_expsum2d_bounded(self): - """Test for COptimizer.minimize() method on a polynomial function in - a 2-dimensional space. This function tests the optimization in - discrete space, with an integer eta, an floating starting point with - a box constraint. The solution expected by this test is an integer - vector.""" + def test_minimize_quad100d_l1_sparse(self): + """Test for COptimizer.minimize() method on a quadratic function in + a 100-dimensional space. + This function tests the optimization in discrete space, with an + integer eta (l1 constraint), an integer sparse starting point + with box constraint. + The solution expected by this test is an integer sparse vector. + """ + opt_params = { + 'eta': 1, 'eta_min': 1, 'eps': 1e-12, + 'constr': CConstraintL1(radius=100), + 'bounds': CConstraintBox(lb=-2, ub=3) + } - opt_params = {'eta': 1, 'eta_min': 1, 'eps': 1e-12, - 'discrete': True, - 'bounds': CConstraintBox(lb=-1, ub=1)} + self._test_minimize( + COptimizerPGDLS, 'quad-100-sparse', + opt_params=opt_params, + label='quad-100-sparse-discrete-bounded-l1', + out_int=True) + + def test_minimize_poly_2d_bounded(self): + """Test for COptimizer.minimize() method on a polynomial function in + a 2-dimensional space. + This function tests the optimization in discrete space, with an + integer eta, an integer starting point with a box constraint. + The solution expected by this test is an integer vector. + """ + opt_params = { + 'eta': 1, 'eta_min': 1, 'eps': 1e-12, + 'bounds': CConstraintBox(lb=-1, ub=1)} self._test_minimize( COptimizerPGDLS, 'poly-2', opt_params=opt_params, label='poly-discrete-bounded', + out_int=True ) - def test_minimize_expsum100d_bounded(self): + def test_minimize_poly_100d_bounded(self): """Test for COptimizer.minimize() method on a polynomial function in - a 2-dimensional space. This function tests the optimization in - discrete space, with an integer eta, an integer starting point with - a box constraint. The solution of this problem is integer and sparse - (it is a zero vector)""" - opt_params = {'eta': 1, 'eta_min': 1, 'eps': 1e-12, - 'discrete': True, - 'bounds': CConstraintBox(lb=-1, ub=1)} + a 2-dimensional space. + This function tests the optimization in discrete space, with an + integer eta, an integer starting point with a box constraint. + The solution of this problem is an integer vector (of zeros). + """ + opt_params = { + 'eta': 1, 'eta_min': 1, 'eps': 1e-12, + 'bounds': CConstraintBox(lb=-1, ub=1) + } self._test_minimize( COptimizerPGDLS, 'poly-100-int', @@ -120,13 +168,16 @@ def test_minimize_expsum100d_bounded(self): def test_minimize_poly_100d_bounded_sparse(self): """Test for COptimizer.minimize() method on a polynomial function in - a 100-dimensional space. This function tests the optimization in - discrete space, with an integer eta, an integer and sparse starting - point (it is a zero vector) with a box constraint. The solution - expected by this test is an integer and sparse vector.""" - opt_params = {'eta': 1, 'eta_min': 1, 'eps': 1e-12, - 'discrete': True, - 'bounds': CConstraintBox(lb=-1, ub=1)} + a 100-dimensional space. + This function tests the optimization in discrete space, with an + integer eta, an integer and sparse starting point (zeros vector) + with a box constraint. + The solution expected by this test is an integer sparse vector (of zeros). + """ + opt_params = { + 'eta': 1, 'eta_min': 1, 'eps': 1e-12, + 'bounds': CConstraintBox(lb=-1, ub=1) + } self._test_minimize( COptimizerPGDLS, 'poly-100-int-sparse', @@ -134,17 +185,6 @@ def test_minimize_poly_100d_bounded_sparse(self): label='poly-int-sparse-discrete-bounded', out_int=True) - def test_minimize_poly_100d_lc_constr(self): - """Discrete optimization + L2 constraint is not supported. This - function tests if the optimzier raises correctly an error when it - receives as parameter discrete = True and an L2 constraint.""" - - with self.assertRaises(NotImplementedError): - opt_params = {'eta': 1e-6, 'eta_min': 1e-4, 'eps': 1e-12, - 'discrete': True, 'constr': CConstraintL2()} - self._test_minimize( - COptimizerPGDLS, 'beale', opt_params=opt_params) - if __name__ == '__main__': COptimizerTestCases.main() diff --git a/tox.ini b/tox.ini index 0a26301b..843e2d3a 100644 --- a/tox.ini +++ b/tox.ini @@ -54,7 +54,7 @@ description = env with latest versions of dependencies for testing notebooks commands = py.test src/secml/test_simple.py --nbval tutorials {posargs} deps = - git+https://github.com/RobustBench/robustbench + git+https://github.com/RobustBench/robustbench.git@v0.1 # Documentation [testenv:docs] diff --git a/tutorials/14-RobustBench.ipynb b/tutorials/14-RobustBench.ipynb index 5768844d..f97d2afa 100644 --- a/tutorials/14-RobustBench.ipynb +++ b/tutorials/14-RobustBench.ipynb @@ -57,7 +57,7 @@ "To install the library, just open a terminal and execute the following command:\n", "\n", "```bash\n", - "pip install git+https://github.com/RobustBench/robustbench\n", + "pip install git+https://github.com/RobustBench/robustbench.git@v0.1", "```" ] }, @@ -77,7 +77,7 @@ "try:\n", " import robustbench\n", "except ImportError:\n", - " %pip install git+https://github.com/RobustBench/robustbench" + " %pip install git+https://github.com/RobustBench/robustbench.git@v0.1" ] }, { @@ -325,4 +325,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} From 96b42059245e9ac921d62536ef7063c85837d29f Mon Sep 17 00:00:00 2001 From: Marco Melis Date: Thu, 22 Apr 2021 16:41:52 +0000 Subject: [PATCH 2/2] Release v0.14.1 --- CHANGELOG.md | 16 ++++++++++++++++ src/secml/VERSION | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8a6b713..bf3f13da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +## v0.14.1 (22/04/2021) +- This version brings fixes for a few issues with the optimizers and related classes, along with improvements to documentation for all attacks, optimizers, and related classes. + +### Fixed (3 changes) +- #923 Fixed `COptimizerPGDLS` and `COptimizerPGDLS` not working properly if the classifier's gradient has multiple components with the same (max) value. +- #919 Fixed `CConstraintL1` crashing when projecting sparse data using default center value (scalar 0). +- #920 Fixed inconsistent results between dense and sparse data for `CConstraintL1` projection caused by type casting. + +### Removed & Deprecated (1 change) +- #922 Removed unnecessary parameter `discrete` from `COptimizerPGDLS` and `COptimizerPGDExp`. + +### Documentation (2 changes) +- #100017 Improved documentation of `CAttackEvasion`, `COptimizer`, `CLineSearch`, and corresponding subclasses. +- #918 Installing the latest stable version of RobustBench instead of the master version. + + ## v0.14 (23/03/2021) - #795 Added new package `adv.attacks.evasion.foolbox` with a wrapper for [Foolbox](https://foolbox.readthedocs.io/en/stable/). - #623 `secml` is now tested for compatibility with Python 3.8. diff --git a/src/secml/VERSION b/src/secml/VERSION index ad4f01d7..930e3000 100644 --- a/src/secml/VERSION +++ b/src/secml/VERSION @@ -1 +1 @@ -0.14.1-rc1 +0.14.1