Bug fix in models.py

README.md

@@ -36,6 +36,13 @@ Note, that the optimization problem being solved uses linear approximation of th
 
 Additionally, the only well-defined point for an adversarial change is the point on the decision boundary. In order to find it we perform a binary search for the smallest `c` in the optimization problem, which is enough to change the class. The code is in `ae_generation.py`.
 
+## Updated Cross-Lipschitz regularizer
+
+A better regularization would be (keeping the bounds closed):
+<!--<img src="http://latex.codecogs.com/gif.latex?\Omega(f)%20=%20\frac{1}{n}\sum_{i=1}^n%20\frac{||\nabla%20f_{y_{i}}(x_i)%20-%20\nabla%20\max_{m}f_m(x_i)||_2^2}{||f_{y_{i}}(x_i)-\max_{m}f_m(x_i)||_2^2}" />-->
+<img src="http://latex.codecogs.com/gif.latex?j%20=%20\underset{m}{\mathrm{argmax}} \nabla f_m(x_i)" />
+
+<img src="http://latex.codecogs.com/gif.latex?\Omega(f)%20=%20\frac{1}{n}\sum_{i=1}^n%20\frac{||\nabla%20f_{y_{i}}(x_i)%20-%20\nabla%20f_j(x_i)||_2^2}{||f_{y_{i}}(x_i)-f_j(x_i)||_2^2}" />
 
 ## Running options
 Supported neural network types:
@@ -46,6 +53,7 @@ Supported neural network types:
 
 Regularization types:
 
+- **`cross_lipschitz_updated`: the updated varient of the proposed Cross-Lipschitz regularization (with closed bounds)**
 - **`cross_lipschitz`: the proposed Cross-Lipschitz regularization**
 - `weight_decay`: a standard L2 weight decay
 - `dropout`: in case of ResNets it is implemented as suggested in [Wide Residual Networks](https://arxiv.org/pdf/1605.07146v1.pdf).

data.py

@@ -1,11 +1,19 @@
 import numpy
 import numpy as np
-import pickle
 import csv
 import scipy.io
 from PIL import Image
 from tensorflow.examples.tutorials.mnist import input_data
 
+import os
+import sys
+import wget
+import tarfile
+import shutil
+if sys.version_info[0] == 3:
+    import pickle
+else:
+    import cPickle as pickle
 
 def get_next_batch(X, Y, batch_size, augm_flag=False):
     n_batches = len(X) // batch_size
@@ -71,10 +79,32 @@ def normalize(x_train, x_test):
 def get_cifar10(as_image=True, onehot=True, validation_size=5000):
     """load all CIFAR-10 data and merge training batches"""
 
+    def download_cifar10_dataset(folder):
+        archieveFileName = 'cifar-10-python.tar.gz'
+        if os.path.exists(archieveFileName):
+            os.remove(archieveFileName)
+        print ("Downloading CIFAR-10 dataset")
+        url = 'http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
+        filename = wget.download(url)
+
+        # Extract the tar file
+        print ("Extracting archive")
+        tar = tarfile.open(filename)
+        tar.extractall()
+        tar.close()
+
+        # Remove the data to this location
+        shutil.move('cifar-10-batches-py', folder)
+        os.remove(filename)
+
     def load_cifar10_file(filename):
         """load data from single CIFAR-10 file"""
         with open(filename, 'rb') as f:
-            data_dict = pickle.load(f, encoding='latin1')
+            # data_dict = pickle.load(f, encoding='latin1')
+            if sys.version_info[0] == 3:
+                data_dict = pickle.load(f, encoding='latin1')
+            else:
+                data_dict = pickle.load(f)
             x = data_dict['data']
             y = data_dict['labels']
             x = x.astype(float)
@@ -87,6 +117,9 @@ def reshape_cifar(x):
         return x
 
     folder = 'data/cifar10/'
+    if not os.path.exists(folder):
+        download_cifar10_dataset(folder)
+
     xs, ys = [], []
     for i in range(1, 6):
         filename = folder + 'data_batch_' + str(i)

models.py

@@ -71,6 +71,7 @@ def build_graph(self, x):
         Build the core model within the graph.
           x: Batches of images. [batch_size, image_size, image_size, 3]
         """
+        self.x = x
         filters = self.filters
         res_func = self.res_func
 

regularizers.py

@@ -27,6 +27,42 @@ def cross_lipschitz(f, x, n_ex, hps):
     return reg / n_summations
 
 
+def cross_lipschitz_updated(f, x, y, n_ex, hps):
+    """
+    Calculates Cross-Lipschitz regularization in a straightforward way using tf.gradients to calculate the required
+    derivatives. Suitable for all differentiable classifiers. It is calculated for the given batch (details:
+    "Formal Guarantees on the Robustness of a Classifier against Adversarial Manipulation",
+    http://www.ml.uni-saarland.de/Publications/HeiAnd-FormGuarAdvManipLongVersion.pdf).
+
+    f: tensor, (batch_size, n_classes) - values of the output layer before softmax for a batch
+    x: tensor, (batch_size, n_input) or (batch_size, height, width, color) - input images
+    y: tensor, (batch_size) - input image labels
+    n_classes: int - number of classes
+    n_ex: int - number of examples in a batch
+    """
+    # n_summations = tf.cast(hps.n_classes ** 2 * n_ex, tf.float32)  # normalizing factor to unify scale of lambda across datasets
+    n_summations = tf.cast(n_ex, tf.float32)  # normalizing factor to unify scale of lambda across datasets (based on only the number of examples)
+    reg = 0
+    grad_matrix_list = [tf.gradients(f[:, k], x)[0] for k in range(hps.n_classes)]  # take each gradient wrt input only once
+    if hps.as_image:  # if x has shape (batch_size, height, width, color), then we need to flatten it first
+        grad_matrix_list = [tf.reshape(grad, [-1, hps.real_height * hps.real_width * hps.n_colors]) for grad in grad_matrix_list]
+
+    for inputIter in range(n_ex):
+        maxClsGradIdx = 0
+        l = y[inputIter] # GT class for the current example
+        for m in range(1, hps.n_classes):
+            if tf.norm(grad_matrix_list[m], ord=2, axis=1) > tf.norm(grad_matrix_list[maxClsGradIdx], ord=2, axis=1):
+                maxClsGradIdx = m
+
+        grad_diff_matrix = grad_matrix_list[l] - grad_matrix_list[maxClsGradIdx]  # difference of gradients for a class pair (l, m)
+        numerator = tf.norm(grad_diff_matrix, ord=2, axis=1)
+        denominator = tf.norm(f[:, l] - f[:, maxClsGradIdx], ord=2, axis=1)
+        norm_for_batch = numerator / denominator
+
+        reg += tf.reduce_sum(tf.square(norm_for_batch))
+    return reg / n_summations
+
+
 def cross_lipschitz_analytical_1hl(model, n_ex, hps):
     """
     Calculates Cross-Lipschitz regularization in analytic form for 1 hidden layer Neural Network. (details:

requirements.txt

@@ -1,5 +1,5 @@
-tensorflow_gpu==1.2.1
-matplotlib==2.0.2
-numpy==1.13.1
-scipy==0.19.1
-Pillow==4.3.0
+tensorflow_gpu
+matplotlib
+numpy
+scipy
+Pillow

run.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+python worker.py --experiment_name=advers_training --adv_train_flag --reg_type=cross_lipschitz_updated --dataset=cifar10 --nn_type=resnet --gpu_number=0 --gpu_memory=0.9 --lr=0.2 --lmbd=0.0001 --batch_size=128 --n_epochs=200

worker.py

@@ -73,8 +73,10 @@ def create_mixed_ae_batch(x_batch, x_tf, model, sess, hps):
                          'the paper.')
 parser.add_argument('--dataset', type=str, default='cifar10',
                     help='mnist, cifar10, gtrsrb (German traffic roadsign dataset).')
-parser.add_argument('--reg_type', type=str, default='cross_lipschitz',
-                    help='cross_lipschitz, no, weight_decay, dropout')
+# parser.add_argument('--reg_type', type=str, default='cross_lipschitz',
+#                     help='cross_lipschitz, no, weight_decay, dropout')
+parser.add_argument('--reg_type', type=str, default='cross_lipschitz_updated',
+                    help='cross_lipschitz, cross_lipschitz_updated, no, weight_decay, dropout')
 parser.add_argument('--opt_method', type=str, default='sgd',
                     help='Optimization method: sgd or momentum (default momentum: 0.9)')
 parser.add_argument('--n_epochs', type=int, default=200, help='Number of epochs.')
@@ -196,6 +198,12 @@ def create_mixed_ae_batch(x_batch, x_tf, model, sess, hps):
             reg = regularizers.cross_lipschitz_analytical_1hl(model, hps.n_ex, hps)
         else:
             reg = regularizers.cross_lipschitz(f, X_input, hps.n_ex, hps)
+    elif 'cross_lipschitz_updated' in hps.reg_type:
+        if hps.nn_type == 'mlp1layer':
+            raise NotImplementedError
+        else:
+            print ("Using updated cross-lipschitz regularizer")
+            reg = regularizers.cross_lipschitz_updated(f, X_input, Y, hps.n_ex, hps)
     else:
         reg = tf.constant(0.0)  # 'dropout' and 'no' cases go here