examples/VGG-Cifar10-100-TransferLearning-FineTuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Let's see how to use DBT to:\n",
    "# 1: train a VGG-like network on CIFAR-10\n",
    "# 2: continue a train from the last iteration\n",
    "# 3: do TRANSFER LEARNING from the trained model to another model that will be able to classify CIFAR-100\n",
    "# 4: do FINE TUNING of the model trained on CIFAR-10 to solve the CIFAR-100 classification problem\n",
    "# 5: compare the train/validation/test performance of the models\n",
    "\n",
    "import pandas as pd\n",
    "import pprint\n",
    "import tensorflow as tf\n",
    "from dytb.inputs.predefined import Cifar10, Cifar100\n",
    "from dytb.train import train\n",
    "from dytb.models.predefined.VGG import VGG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Instantiate the model\n",
    "vgg = VGG()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Instantiate the CIFAR-10 input source\n",
    "cifar10 = Cifar10.Cifar10()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original training set size 50000. Augmented training set size: 100000\n",
      "<tf.Variable 'VGG/64/conv1/W:0' shape=(3, 3, 3, 64) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/64/conv1/b:0' shape=(64,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/64/conv2/W:0' shape=(3, 3, 64, 64) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/64/conv2/b:0' shape=(64,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/128/conv3/W:0' shape=(3, 3, 64, 128) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/128/conv3/b:0' shape=(128,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/128/conv4/W:0' shape=(3, 3, 128, 128) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/128/conv4/b:0' shape=(128,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv5/W:0' shape=(3, 3, 128, 256) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv5/b:0' shape=(256,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv6/W:0' shape=(3, 3, 256, 256) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv6/b:0' shape=(256,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv7/W:0' shape=(3, 3, 256, 256) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv7/b:0' shape=(256,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv8/W:0' shape=(3, 3, 256, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv8/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv9/W:0' shape=(3, 3, 512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv9/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv10/W:0' shape=(3, 3, 512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv10/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv11/W:0' shape=(3, 3, 512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv11/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv12/W:0' shape=(3, 3, 512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv12/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv13/W:0' shape=(3, 3, 512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv13/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/fc/W:0' shape=(512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/fc/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/softmax_linear/W:0' shape=(512, 10) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/softmax_linear/b:0' shape=(10,) dtype=float32_ref>\n",
      "Model VGG: trainable parameters: 14982474. Size: 59929.896 KB\n",
      "[!] No checkpoint file found\n",
      "2017-05-22 15:54:25.113484: step 0, loss = 2.5562 (18.3 examples/sec; 2.726 sec/batch)\n",
      "2017-05-22 15:54:31.518555: step 200, loss = 2.0407 (1772.2 examples/sec; 0.028 sec/batch)\n",
      "2017-05-22 15:54:38.177017: step 400, loss = 1.7758 (1668.4 examples/sec; 0.030 sec/batch)\n",
      "2017-05-22 15:54:44.512619: step 600, loss = 1.6580 (1730.2 examples/sec; 0.029 sec/batch)\n",
      "2017-05-22 15:54:51.121688: step 800, loss = 1.6563 (1893.3 examples/sec; 0.026 sec/batch)\n",
      "2017-05-22 15:54:57.477840: step 1000, loss = 1.6784 (1622.5 examples/sec; 0.031 sec/batch)\n",
      "2017-05-22 15:55:03.983570: step 1200, loss = 1.4601 (1897.1 examples/sec; 0.026 sec/batch)\n",
      "2017-05-22 15:55:10.386923: step 1400, loss = 1.2355 (1740.0 examples/sec; 0.029 sec/batch)\n",
      "2017-05-22 15:55:17.030403: step 1600, loss = 1.2162 (1677.3 examples/sec; 0.030 sec/batch)\n",
      "2017-05-22 15:55:23.402104: step 1800, loss = 1.0021 (1757.7 examples/sec; 0.028 sec/batch)\n",
      "2017-05-22 15:55:29.881070: step 2000, loss = 1.1827 (1824.0 examples/sec; 0.027 sec/batch)\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-10_Adam_l2=1e-05_fliplr/model.ckpt-2000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-10_Adam_l2=1e-05_fliplr/model.ckpt-2000\n",
      "2017-05-22 15:55:47.020577 (1): train accuracy = 0.587 validation accuracy = 0.572\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-10_Adam_l2=1e-05_fliplr/best/model.ckpt-2000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-10_Adam_l2=1e-05_fliplr/best/model.ckpt-2000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-10_Adam_l2=1e-05_fliplr/best/model.ckpt-2000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-10_Adam_l2=1e-05_fliplr/best/model.ckpt-2000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-10_Adam_l2=1e-05_fliplr/best/model.ckpt-2000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-10_Adam_l2=1e-05_fliplr/best/model.ckpt-2000\n"
     ]
    }
   ],
   "source": [
    "# 1: Train VGG on Cifar10 for an Epoch\n",
    "\n",
    "# Place the train process on GPU:0\n",
    "device = '/gpu:0'\n",
    "with tf.device(device):\n",
    "    info = train(\n",
    "        model=vgg,\n",
    "        dataset=cifar10,\n",
    "        hyperparameters={\n",
    "            \"epochs\": 1,\n",
    "            \"batch_size\": 50,\n",
    "            \"regularizations\": {\n",
    "                \"l2\": 1e-5,\n",
    "                \"augmentation\": {\n",
    "                    \"name\": \"FlipLR\",\n",
    "                    \"fn\": tf.image.random_flip_left_right,\n",
    "                    # On average the training set size double appling this\n",
    "                    # transformation, thus factor=2\n",
    "                    \"factor\": 2,\n",
    "                }\n",
    "            },\n",
    "            \"gd\": {\n",
    "                \"optimizer\": tf.train.AdamOptimizer,\n",
    "                \"args\": {\n",
    "                    \"learning_rate\": 1e-3,\n",
    "                    \"beta1\": 0.9,\n",
    "                    \"beta2\": 0.99,\n",
    "                    \"epsilon\": 1e-8\n",
    "                }\n",
    "            }\n",
    "        },\n",
    "        force_restart=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test</th>\n",
       "      <th>train</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.5718</td>\n",
       "      <td>0.59188</td>\n",
       "      <td>0.5717</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            test    train  validation\n",
       "accuracy  0.5718  0.59188      0.5717"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Info contains every information related to the trained model.\n",
    "# We're interested in stats only, thus we extract only them from the info dict\n",
    "# Display the results in a table. Let's use a Pandas DataFrame for that\n",
    "\n",
    "# Extract the accuracyes measured in every set (train/validation/test)\n",
    "accuracies = {key: value[\"accuracy\"] for key, value in info[\"stats\"].items()}\n",
    "df = pd.DataFrame.from_records(accuracies, index=[\"accuracy\"])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2945.0</td>\n",
       "      <td>178.0</td>\n",
       "      <td>75.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>1137.0</td>\n",
       "      <td>423.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12.0</td>\n",
       "      <td>4019.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>99.0</td>\n",
       "      <td>838.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1138.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>1146.0</td>\n",
       "      <td>525.0</td>\n",
       "      <td>265.0</td>\n",
       "      <td>757.0</td>\n",
       "      <td>597.0</td>\n",
       "      <td>207.0</td>\n",
       "      <td>127.0</td>\n",
       "      <td>202.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>244.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>138.0</td>\n",
       "      <td>1234.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>2354.0</td>\n",
       "      <td>367.0</td>\n",
       "      <td>118.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>340.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>302.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>422.0</td>\n",
       "      <td>399.0</td>\n",
       "      <td>1887.0</td>\n",
       "      <td>516.0</td>\n",
       "      <td>602.0</td>\n",
       "      <td>611.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>233.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>140.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>109.0</td>\n",
       "      <td>516.0</td>\n",
       "      <td>75.0</td>\n",
       "      <td>3598.0</td>\n",
       "      <td>109.0</td>\n",
       "      <td>161.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>228.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>64.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>588.0</td>\n",
       "      <td>106.0</td>\n",
       "      <td>171.0</td>\n",
       "      <td>3610.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>245.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>115.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>47.0</td>\n",
       "      <td>118.0</td>\n",
       "      <td>167.0</td>\n",
       "      <td>999.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>3006.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>530.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>494.0</td>\n",
       "      <td>253.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>3904.0</td>\n",
       "      <td>229.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>63.0</td>\n",
       "      <td>434.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>88.0</td>\n",
       "      <td>4331.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        0       1       2       3       4       5       6       7       8  \\\n",
       "0  2945.0   178.0    75.0    60.0     6.0    62.0    31.0    37.0  1137.0   \n",
       "1    12.0  4019.0     1.0    16.0     0.0     8.0    17.0     6.0    99.0   \n",
       "2  1138.0    28.0  1146.0   525.0   265.0   757.0   597.0   207.0   127.0   \n",
       "3   244.0    23.0   138.0  1234.0    60.0  2354.0   367.0   118.0    65.0   \n",
       "4   302.0    33.0   422.0   399.0  1887.0   516.0   602.0   611.0    36.0   \n",
       "5   140.0    16.0   109.0   516.0    75.0  3598.0   109.0   161.0    32.0   \n",
       "6    64.0    62.0   115.0   588.0   106.0   171.0  3610.0    24.0    55.0   \n",
       "7   115.0    10.0    47.0   118.0   167.0   999.0    36.0  3006.0    23.0   \n",
       "8   494.0   253.0    14.0    43.0     1.0    29.0    14.0    10.0  3904.0   \n",
       "9    63.0   434.0     3.0    14.0     0.0    33.0     4.0    18.0    88.0   \n",
       "\n",
       "        9  \n",
       "0   423.0  \n",
       "1   838.0  \n",
       "2   202.0  \n",
       "3   340.0  \n",
       "4   233.0  \n",
       "5   228.0  \n",
       "6   245.0  \n",
       "7   530.0  \n",
       "8   229.0  \n",
       "9  4331.0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Extract the confusion matrices \n",
    "confusion_matrices = {key: value[\"confusion_matrix\"] for key, value in info[\"stats\"].items()}\n",
    "# Display the confusione matrices for the training set\n",
    "df = pd.DataFrame(confusion_matrices[\"train\"])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original training set size 50000. Augmented training set size: 100000\n",
      "<tf.Variable 'VGG/64/conv1/W:0' shape=(3, 3, 3, 64) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/64/conv1/b:0' shape=(64,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/64/conv2/W:0' shape=(3, 3, 64, 64) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/64/conv2/b:0' shape=(64,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/128/conv3/W:0' shape=(3, 3, 64, 128) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/128/conv3/b:0' shape=(128,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/128/conv4/W:0' shape=(3, 3, 128, 128) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/128/conv4/b:0' shape=(128,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv5/W:0' shape=(3, 3, 128, 256) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv5/b:0' shape=(256,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv6/W:0' shape=(3, 3, 256, 256) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv6/b:0' shape=(256,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv7/W:0' shape=(3, 3, 256, 256) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv7/b:0' shape=(256,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv8/W:0' shape=(3, 3, 256, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv8/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv9/W:0' shape=(3, 3, 512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv9/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv10/W:0' shape=(3, 3, 512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv10/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv11/W:0' shape=(3, 3, 512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv11/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv12/W:0' shape=(3, 3, 512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv12/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv13/W:0' shape=(3, 3, 512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv13/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/fc/W:0' shape=(512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/fc/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/softmax_linear/W:0' shape=(512, 10) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/softmax_linear/b:0' shape=(10,) dtype=float32_ref>\n",
      "Model VGG: trainable parameters: 14982474. Size: 59929.896 KB\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-10_Adam_l2=1e-05_fliplr/model.ckpt-2000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-10_Adam_l2=1e-05_fliplr/best/model.ckpt-2000\n",
      "2017-05-22 15:56:41.150636: step 2200, loss = 0.6995 (1489.9 examples/sec; 0.034 sec/batch)\n",
      "2017-05-22 15:56:47.728752: step 2400, loss = 1.2488 (1825.8 examples/sec; 0.027 sec/batch)\n",
      "2017-05-22 15:56:54.283833: step 2600, loss = 0.9503 (1493.5 examples/sec; 0.033 sec/batch)\n",
      "2017-05-22 15:57:00.739826: step 2800, loss = 0.8572 (1549.1 examples/sec; 0.032 sec/batch)\n",
      "2017-05-22 15:57:07.264069: step 3000, loss = 1.0171 (1491.0 examples/sec; 0.034 sec/batch)\n",
      "2017-05-22 15:57:13.645133: step 3200, loss = 0.7402 (1206.1 examples/sec; 0.041 sec/batch)\n",
      "2017-05-22 15:57:20.242200: step 3400, loss = 0.9686 (1741.3 examples/sec; 0.029 sec/batch)\n",
      "2017-05-22 15:57:26.501463: step 3600, loss = 0.9150 (1587.2 examples/sec; 0.032 sec/batch)\n",
      "2017-05-22 15:57:33.058041: step 3800, loss = 0.6963 (1597.9 examples/sec; 0.031 sec/batch)\n",
      "2017-05-22 15:57:39.353625: step 4000, loss = 1.1301 (1725.1 examples/sec; 0.029 sec/batch)\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-10_Adam_l2=1e-05_fliplr/model.ckpt-4000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-10_Adam_l2=1e-05_fliplr/model.ckpt-4000\n",
      "2017-05-22 15:57:57.462822 (2): train accuracy = 0.749 validation accuracy = 0.724\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-10_Adam_l2=1e-05_fliplr/best/model.ckpt-4000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-10_Adam_l2=1e-05_fliplr/best/model.ckpt-4000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-10_Adam_l2=1e-05_fliplr/best/model.ckpt-4000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-10_Adam_l2=1e-05_fliplr/best/model.ckpt-4000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-10_Adam_l2=1e-05_fliplr/best/model.ckpt-4000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-10_Adam_l2=1e-05_fliplr/best/model.ckpt-4000\n"
     ]
    }
   ],
   "source": [
    "# 2: train it again for another epoch\n",
    "# Note the `force_restart` parameter removed.\n",
    "# `epochs` is the TOTAL number of epoch for the trained model\n",
    "# Thus since we trained it before for a single epoch,\n",
    "# we set \"epochs\": 2 in order to train it for another epoch\n",
    "\n",
    "with tf.device(device):\n",
    "    info = train(\n",
    "        model=vgg,\n",
    "        dataset=cifar10,\n",
    "        hyperparameters={\n",
    "            \"epochs\": 2,\n",
    "            \"batch_size\": 50,\n",
    "            \"regularizations\": {\n",
    "                \"l2\": 1e-5,\n",
    "                \"augmentation\": {\n",
    "                    \"name\": \"FlipLR\",\n",
    "                    \"fn\": tf.image.random_flip_left_right,\n",
    "                    \"factor\": 2,\n",
    "                }\n",
    "            },\n",
    "            \"gd\": {\n",
    "                \"optimizer\": tf.train.AdamOptimizer,\n",
    "                \"args\": {\n",
    "                    \"learning_rate\": 1e-3,\n",
    "                    \"beta1\": 0.9,\n",
    "                    \"beta2\": 0.99,\n",
    "                    \"epsilon\": 1e-8\n",
    "                }\n",
    "            }\n",
    "        })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test</th>\n",
       "      <th>train</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.7241</td>\n",
       "      <td>0.74684</td>\n",
       "      <td>0.724</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            test    train  validation\n",
       "accuracy  0.7241  0.74684       0.724"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Display the results in a table. Let's use a Pandas DataFrame for that\n",
    "accuracies = {key: value[\"accuracy\"] for key, value in info[\"stats\"].items()}\n",
    "df = pd.DataFrame.from_records(accuracies, index=[\"accuracy\"])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Save last trained model info\n",
    "vggInfo = info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original training set size 50000. Augmented training set size: 100000\n",
      "<tf.Variable 'VGG/64/conv1/W:0' shape=(3, 3, 3, 64) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/64/conv1/b:0' shape=(64,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/64/conv2/W:0' shape=(3, 3, 64, 64) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/64/conv2/b:0' shape=(64,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/128/conv3/W:0' shape=(3, 3, 64, 128) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/128/conv3/b:0' shape=(128,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/128/conv4/W:0' shape=(3, 3, 128, 128) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/128/conv4/b:0' shape=(128,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv5/W:0' shape=(3, 3, 128, 256) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv5/b:0' shape=(256,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv6/W:0' shape=(3, 3, 256, 256) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv6/b:0' shape=(256,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv7/W:0' shape=(3, 3, 256, 256) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv7/b:0' shape=(256,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv8/W:0' shape=(3, 3, 256, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv8/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv9/W:0' shape=(3, 3, 512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv9/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv10/W:0' shape=(3, 3, 512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv10/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv11/W:0' shape=(3, 3, 512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv11/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv12/W:0' shape=(3, 3, 512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv12/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv13/W:0' shape=(3, 3, 512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv13/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/fc/W:0' shape=(512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/fc/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/softmax_linear/W:0' shape=(512, 100) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/softmax_linear/b:0' shape=(100,) dtype=float32_ref>\n",
      "Model VGG: trainable parameters: 15028644. Size: 60114.576 KB\n",
      "[!] No checkpoint file found\n",
      "2017-05-22 15:58:46.456678: step 0, loss = 4.8550 (18.5 examples/sec; 2.706 sec/batch)\n",
      "2017-05-22 15:58:53.423647: step 200, loss = 4.6459 (1422.8 examples/sec; 0.035 sec/batch)\n",
      "2017-05-22 15:58:59.787396: step 400, loss = 4.6378 (1633.2 examples/sec; 0.031 sec/batch)\n",
      "2017-05-22 15:59:06.429601: step 600, loss = 4.6263 (1706.8 examples/sec; 0.029 sec/batch)\n",
      "2017-05-22 15:59:12.840329: step 800, loss = 4.6233 (1677.9 examples/sec; 0.030 sec/batch)\n",
      "2017-05-22 15:59:19.697066: step 1000, loss = 4.6179 (1469.4 examples/sec; 0.034 sec/batch)\n",
      "2017-05-22 15:59:26.260296: step 1200, loss = 4.6106 (1363.8 examples/sec; 0.037 sec/batch)\n",
      "2017-05-22 15:59:32.943655: step 1400, loss = 4.6101 (1664.8 examples/sec; 0.030 sec/batch)\n",
      "2017-05-22 15:59:39.250269: step 1600, loss = 4.6050 (1547.8 examples/sec; 0.032 sec/batch)\n",
      "2017-05-22 15:59:46.037072: step 1800, loss = 4.6037 (1602.8 examples/sec; 0.031 sec/batch)\n",
      "2017-05-22 15:59:52.438255: step 2000, loss = 4.6048 (1608.5 examples/sec; 0.031 sec/batch)\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-100_Adam_l2=1e-05_fliplr/model.ckpt-2000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-100_Adam_l2=1e-05_fliplr/model.ckpt-2000\n",
      "2017-05-22 16:00:11.160246 (1): train accuracy = 0.010 validation accuracy = 0.010\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-100_Adam_l2=1e-05_fliplr/best/model.ckpt-2000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-100_Adam_l2=1e-05_fliplr/best/model.ckpt-2000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-100_Adam_l2=1e-05_fliplr/best/model.ckpt-2000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-100_Adam_l2=1e-05_fliplr/best/model.ckpt-2000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-100_Adam_l2=1e-05_fliplr/best/model.ckpt-2000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-100_Adam_l2=1e-05_fliplr/best/model.ckpt-2000\n"
     ]
    }
   ],
   "source": [
    "# 3: TRANSFER LEARNING\n",
    "# Use the best model trained on Cifar10, to classify Cifar 100 images.\n",
    "# Thus we train ONLY the softmax linear scope (that has 100 neurons, now),\n",
    "# keeping constant any other previosly trained layer\n",
    "# We load the weights from the previous trained model, or better\n",
    "# DyTB saves the \"best\" model (w.r.t. a metric) in a separate folder\n",
    "# So we extract the info[\"paths\"][\"best\"] path, that's the path of the best\n",
    "# model trained so far.\n",
    "cifar100 = Cifar100.Cifar100()\n",
    "with tf.device(device):\n",
    "    transferInfo = train(\n",
    "        model=vgg,\n",
    "        dataset=cifar100,\n",
    "        hyperparameters={\n",
    "            \"epochs\": 1,\n",
    "            \"batch_size\": 50,\n",
    "            \"regularizations\": {\n",
    "                \"l2\": 1e-5,\n",
    "                \"augmentation\": {\n",
    "                    \"name\": \"FlipLR\",\n",
    "                    \"fn\": tf.image.random_flip_left_right,\n",
    "                    \"factor\": 2,\n",
    "                }\n",
    "            },\n",
    "            \"gd\": {\n",
    "                \"optimizer\": tf.train.AdamOptimizer,\n",
    "                \"args\": {\n",
    "                    \"learning_rate\": 1e-3,\n",
    "                    \"beta1\": 0.9,\n",
    "                    \"beta2\": 0.99,\n",
    "                    \"epsilon\": 1e-8\n",
    "                    }\n",
    "                }\n",
    "        },\n",
    "        force_restart=True,\n",
    "        surgery={\n",
    "            \"checkpoint_path\": vggInfo[\"paths\"][\"best\"],\n",
    "            \"exclude_scopes\": \"VGG/softmax_linear\",\n",
    "            \"trainable_scopes\": \"VGG/softmax_linear\"\n",
    "        })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original training set size 50000. Augmented training set size: 100000\n",
      "<tf.Variable 'VGG/64/conv1/W:0' shape=(3, 3, 3, 64) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/64/conv1/b:0' shape=(64,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/64/conv2/W:0' shape=(3, 3, 64, 64) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/64/conv2/b:0' shape=(64,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/128/conv3/W:0' shape=(3, 3, 64, 128) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/128/conv3/b:0' shape=(128,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/128/conv4/W:0' shape=(3, 3, 128, 128) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/128/conv4/b:0' shape=(128,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv5/W:0' shape=(3, 3, 128, 256) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv5/b:0' shape=(256,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv6/W:0' shape=(3, 3, 256, 256) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv6/b:0' shape=(256,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv7/W:0' shape=(3, 3, 256, 256) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/256/conv7/b:0' shape=(256,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv8/W:0' shape=(3, 3, 256, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv8/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv9/W:0' shape=(3, 3, 512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv9/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv10/W:0' shape=(3, 3, 512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512/conv10/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv11/W:0' shape=(3, 3, 512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv11/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv12/W:0' shape=(3, 3, 512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv12/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv13/W:0' shape=(3, 3, 512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/512b2/conv13/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/fc/W:0' shape=(512, 512) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/fc/b:0' shape=(512,) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/softmax_linear/W:0' shape=(512, 100) dtype=float32_ref>\n",
      "<tf.Variable 'VGG/softmax_linear/b:0' shape=(100,) dtype=float32_ref>\n",
      "Model VGG: trainable parameters: 15028644. Size: 60114.576 KB\n",
      "[!] No checkpoint file found\n",
      "2017-05-22 16:00:58.062810: step 0, loss = 4.7794 (23.6 examples/sec; 2.122 sec/batch)\n",
      "2017-05-22 16:01:04.976976: step 200, loss = 4.6412 (1874.0 examples/sec; 0.027 sec/batch)\n",
      "2017-05-22 16:01:11.427974: step 400, loss = 4.6381 (1594.2 examples/sec; 0.031 sec/batch)\n",
      "2017-05-22 16:01:18.114518: step 600, loss = 4.6311 (1877.8 examples/sec; 0.027 sec/batch)\n",
      "2017-05-22 16:01:24.520331: step 800, loss = 4.6238 (1550.1 examples/sec; 0.032 sec/batch)\n",
      "2017-05-22 16:01:31.066939: step 1000, loss = 4.6181 (1501.1 examples/sec; 0.033 sec/batch)\n",
      "2017-05-22 16:01:37.486925: step 1200, loss = 4.6126 (1623.5 examples/sec; 0.031 sec/batch)\n",
      "2017-05-22 16:01:43.917332: step 1400, loss = 4.6089 (1853.3 examples/sec; 0.027 sec/batch)\n",
      "2017-05-22 16:01:50.444527: step 1600, loss = 4.6070 (1363.7 examples/sec; 0.037 sec/batch)\n",
      "2017-05-22 16:01:56.757677: step 1800, loss = 4.6046 (1828.3 examples/sec; 0.027 sec/batch)\n",
      "2017-05-22 16:02:03.368083: step 2000, loss = 4.6055 (1511.6 examples/sec; 0.033 sec/batch)\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-100_Adam_l2=1e-05_fliplr/model.ckpt-2000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-100_Adam_l2=1e-05_fliplr/model.ckpt-2000\n",
      "2017-05-22 16:02:21.476925 (1): train accuracy = 0.010 validation accuracy = 0.010\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-100_Adam_l2=1e-05_fliplr/best/model.ckpt-2000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-100_Adam_l2=1e-05_fliplr/best/model.ckpt-2000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-100_Adam_l2=1e-05_fliplr/best/model.ckpt-2000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-100_Adam_l2=1e-05_fliplr/best/model.ckpt-2000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-100_Adam_l2=1e-05_fliplr/best/model.ckpt-2000\n",
      "INFO:tensorflow:Restoring parameters from /data/pgaleone/dtb_work/examples/log/VGG/CIFAR-100_Adam_l2=1e-05_fliplr/best/model.ckpt-2000\n"
     ]
    }
   ],
   "source": [
    "# 4: FINE TUNING:\n",
    "# Use the model pointed by vggInfo to fine tune the whole network\n",
    "# and tune it on Cifar100.\n",
    "# Let's retrain the whole network end-to-end, starting from the learned weights\n",
    "# Just remove the \"traiable_scopes\" section from the surgery parameter\n",
    "with tf.device(device):\n",
    "    fineTuningInfo = train(\n",
    "        model=vgg,\n",
    "        dataset=cifar100,\n",
    "        hyperparameters={\n",
    "            \"epochs\": 1,\n",
    "            \"batch_size\": 50,\n",
    "            \"regularizations\": {\n",
    "                \"l2\": 1e-5,\n",
    "                \"augmentation\": {\n",
    "                    \"name\": \"FlipLR\",\n",
    "                    \"fn\": tf.image.random_flip_left_right,\n",
    "                    \"factor\": 2,\n",
    "                }\n",
    "            },\n",
    "            \"gd\": {\n",
    "                \"optimizer\": tf.train.AdamOptimizer,\n",
    "                \"args\": {\n",
    "                    \"learning_rate\": 1e-3,\n",
    "                    \"beta1\": 0.9,\n",
    "                    \"beta2\": 0.99,\n",
    "                    \"epsilon\": 1e-8\n",
    "                }\n",
    "            }\n",
    "        },\n",
    "        force_restart=True,\n",
    "        surgery={\n",
    "            \"checkpoint_path\": vggInfo[\"paths\"][\"best\"],\n",
    "            \"exclude_scopes\": \"VGG/softmax_linear\"\n",
    "        })\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test</th>\n",
       "      <th>train</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.01</td>\n",
       "      <td>0.01032</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          test    train  validation\n",
       "accuracy  0.01  0.01032        0.01"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compare the performance of Transfer learning and Fine Tuning\n",
    "accuracies = {key: value[\"accuracy\"] for key, value in transferInfo[\"stats\"].items()}\n",
    "df = pd.DataFrame.from_records(accuracies, index=[\"accuracy\"])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test</th>\n",
       "      <th>train</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.01</td>\n",
       "      <td>0.0101</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          test   train  validation\n",
       "accuracy  0.01  0.0101        0.01"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracies = {key: value[\"accuracy\"] for key, value in fineTuningInfo[\"stats\"].items()}\n",
    "df = pd.DataFrame.from_records(accuracies, index=[\"accuracy\"])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{   'args': {   'batch_size': 50,\n",
      "                'checkpoint_path': '',\n",
      "                'comment': '',\n",
      "                'dataset': <dytb.inputs.predefined.Cifar10.Cifar10 object at 0x7f42e04a86a0>,\n",
      "                'epochs': 2,\n",
      "                'exclude_scopes': None,\n",
      "                'force_restart': False,\n",
      "                'gd': {   'args': {   'beta1': 0.9,\n",
      "                                      'beta2': 0.99,\n",
      "                                      'epsilon': 1e-08,\n",
      "                                      'learning_rate': 0.001},\n",
      "                          'optimizer': <class 'tensorflow.python.training.adam.AdamOptimizer'>},\n",
      "                'lr_decay': {'enabled': False, 'epochs': 25, 'factor': 0.1},\n",
      "                'model': <dytb.models.predefined.VGG.VGG object at 0x7f4289c98ef0>,\n",
      "                'regularizations': {   'augmentation': {   'factor': 2,\n",
      "                                                           'fn': <function random_flip_left_right at 0x7f4289d5c7b8>,\n",
      "                                                           'name': 'FlipLR'},\n",
      "                                       'l2': 1e-05},\n",
      "                'seed': None,\n",
      "                'trainable_scopes': None},\n",
      "    'paths': {   'best': '/data/pgaleone/dtb_work/examples/log/VGG/CIFAR-10_Adam_l2=1e-05_fliplr/best',\n",
      "                 'current': '/data/pgaleone/dtb_work/examples',\n",
      "                 'log': '/data/pgaleone/dtb_work/examples/log/VGG/CIFAR-10_Adam_l2=1e-05_fliplr'},\n",
      "    'stats': {   'test': {   'accuracy': 0.72409998625516891,\n",
      "                             'confusion_matrix': array([[ 628.,    7.,  141.,   10.,   42.,    2.,   10.,   58.,   73.,\n",
      "          29.],\n",
      "       [   9.,  861.,    3.,    0.,    5.,    2.,   25.,    5.,   20.,\n",
      "          70.],\n",
      "       [  26.,    0.,  655.,   29.,  108.,   54.,  107.,   16.,    3.,\n",
      "           2.],\n",
      "       [  10.,    5.,  107.,  364.,   94.,  194.,  179.,   26.,   10.,\n",
      "          12.],\n",
      "       [   8.,    0.,   67.,   14.,  766.,   10.,   75.,   57.,    3.,\n",
      "           0.],\n",
      "       [   3.,    0.,   79.,  119.,   64.,  657.,   39.,   35.,    1.,\n",
      "           2.],\n",
      "       [   2.,    0.,   44.,   23.,   26.,    3.,  895.,    3.,    1.,\n",
      "           3.],\n",
      "       [   2.,    0.,   39.,   27.,   63.,   95.,   14.,  757.,    1.,\n",
      "           2.],\n",
      "       [  61.,   21.,   32.,    6.,    6.,    4.,   21.,    4.,  828.,\n",
      "          17.],\n",
      "       [  17.,   72.,    5.,    8.,    4.,    0.,   19.,   33.,   12.,\n",
      "         830.]])},\n",
      "                 'train': {   'accuracy': 0.74683998459577561,\n",
      "                              'confusion_matrix': array([[  3.20800000e+03,   1.50000000e+01,   7.17000000e+02,\n",
      "          3.60000000e+01,   2.13000000e+02,   1.40000000e+01,\n",
      "          4.40000000e+01,   3.37000000e+02,   2.77000000e+02,\n",
      "          1.06000000e+02],\n",
      "       [  2.70000000e+01,   4.43800000e+03,   1.30000000e+01,\n",
      "          9.00000000e+00,   1.90000000e+01,   2.00000000e+00,\n",
      "          1.38000000e+02,   1.00000000e+01,   7.70000000e+01,\n",
      "          2.66000000e+02],\n",
      "       [  1.49000000e+02,   4.00000000e+00,   3.37000000e+03,\n",
      "          1.65000000e+02,   5.16000000e+02,   1.84000000e+02,\n",
      "          5.26000000e+02,   8.60000000e+01,   3.30000000e+01,\n",
      "          5.00000000e+00],\n",
      "       [  4.30000000e+01,   2.00000000e+00,   4.88000000e+02,\n",
      "          1.95500000e+03,   4.33000000e+02,   1.08700000e+03,\n",
      "          8.02000000e+02,   7.80000000e+01,   4.20000000e+01,\n",
      "          3.00000000e+01],\n",
      "       [  2.80000000e+01,   0.00000000e+00,   2.98000000e+02,\n",
      "          7.20000000e+01,   3.93200000e+03,   6.40000000e+01,\n",
      "          3.22000000e+02,   2.93000000e+02,   5.00000000e+00,\n",
      "          4.00000000e+00],\n",
      "       [  5.00000000e+00,   0.00000000e+00,   3.10000000e+02,\n",
      "          6.19000000e+02,   3.21000000e+02,   3.36200000e+03,\n",
      "          2.34000000e+02,   1.62000000e+02,   5.00000000e+00,\n",
      "          1.20000000e+01],\n",
      "       [  8.00000000e+00,   6.00000000e+00,   1.94000000e+02,\n",
      "          1.13000000e+02,   1.28000000e+02,   2.10000000e+01,\n",
      "          4.54100000e+03,   3.00000000e+00,   1.70000000e+01,\n",
      "          4.00000000e+00],\n",
      "       [  1.40000000e+01,   0.00000000e+00,   1.75000000e+02,\n",
      "          1.22000000e+02,   3.33000000e+02,   3.60000000e+02,\n",
      "          4.20000000e+01,   3.89200000e+03,   1.00000000e+01,\n",
      "          1.60000000e+01],\n",
      "       [  2.30000000e+02,   7.80000000e+01,   1.45000000e+02,\n",
      "          3.40000000e+01,   8.00000000e+00,   5.00000000e+00,\n",
      "          8.50000000e+01,   2.20000000e+01,   4.32600000e+03,\n",
      "          4.60000000e+01],\n",
      "       [  9.30000000e+01,   2.57000000e+02,   2.40000000e+01,\n",
      "          4.70000000e+01,   2.10000000e+01,   1.40000000e+01,\n",
      "          6.10000000e+01,   1.01000000e+02,   4.20000000e+01,\n",
      "          4.35000000e+03]])},\n",
      "                 'validation': {   'accuracy': 0.72399998486042028,\n",
      "                                   'confusion_matrix': array([[ 627.,    7.,  141.,   10.,   42.,    2.,   10.,   59.,   73.,\n",
      "          29.],\n",
      "       [   9.,  861.,    3.,    0.,    5.,    2.,   25.,    5.,   20.,\n",
      "          70.],\n",
      "       [  26.,    0.,  655.,   29.,  108.,   54.,  107.,   16.,    3.,\n",
      "           2.],\n",
      "       [  10.,    5.,  107.,  363.,   94.,  194.,  179.,   26.,   10.,\n",
      "          12.],\n",
      "       [   8.,    0.,   67.,   14.,  766.,   10.,   75.,   57.,    3.,\n",
      "           0.],\n",
      "       [   3.,    0.,   79.,  119.,   64.,  658.,   39.,   35.,    1.,\n",
      "           2.],\n",
      "       [   2.,    0.,   44.,   23.,   26.,    3.,  895.,    3.,    1.,\n",
      "           3.],\n",
      "       [   2.,    0.,   39.,   27.,   63.,   95.,   14.,  757.,    1.,\n",
      "           2.],\n",
      "       [  61.,   21.,   32.,    6.,    6.,    4.,   21.,    4.,  828.,\n",
      "          17.],\n",
      "       [  17.,   72.,    5.,    8.,    4.,    0.,   19.,   33.,   12.,\n",
      "         830.]])}},\n",
      "    'steps': {'decay': 50000, 'epoch': 2000, 'log': 200, 'max': 4000}}\n"
     ]
    }
   ],
   "source": [
    "# For completeness, lets see what a info object contains\n",
    "pprint.pprint(info, indent=4)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}