1_PyTorch for Deep Neural Networks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AI61002\n",
    "# Tutorial 1 : PyTorch for Deep Neural Networks\n",
    "### 18 January 2019\n",
    "\n",
    "## Follow instructions given in the PyTorch [website](http://pytorch.org/) for installation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensor initialization and slicing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing the library\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 2,  2]) tensor([[ 0.8977,  0.8976],\n",
      "        [ 0.1018,  0.4081]])\n"
     ]
    }
   ],
   "source": [
    "# Defining tensors\n",
    "x = torch.tensor((2,2)) # Initialized with specific values\n",
    "y = torch.rand(2,2) # Random initialization\n",
    "print(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2])\n"
     ]
    }
   ],
   "source": [
    "# Obtaining size of tensors\n",
    "print(x.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0.8977,  0.8976],\n",
      "        [ 0.1018,  0.4081]]) tensor([[ 0.8056,  0.7025],\n",
      "        [ 0.6916,  0.6005]]) tensor([[ 1.7033,  1.6002],\n",
      "        [ 0.7933,  1.0085]])\n",
      "tensor([[ 0.8056,  0.7025],\n",
      "        [ 0.6916,  0.6005]]) tensor([[ 1.6111,  1.4051],\n",
      "        [ 1.3831,  1.2009]])\n"
     ]
    }
   ],
   "source": [
    "# Arithmetic operations\n",
    "z = torch.rand(2,2)\n",
    "print(y,z,y+z)\n",
    "print(z,2*z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0.8977,  0.8976],\n",
      "        [ 0.1018,  0.4081]])\n",
      "tensor(0.4081)\n",
      "tensor([ 0.8977,  0.1018])\n"
     ]
    }
   ],
   "source": [
    "# Indexing is similar to numpy indexing\n",
    "print(y)\n",
    "print(y[1,1])\n",
    "print(y[:,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'torch.Tensor'>\n",
      "<class 'numpy.ndarray'>\n"
     ]
    }
   ],
   "source": [
    "# Converting to numpy\n",
    "y_np = y.numpy()\n",
    "print(type(y))\n",
    "print(type(y_np))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1. 1. 1.]\n",
      " [1. 1. 1.]\n",
      " [1. 1. 1.]] tensor([[ 1.,  1.,  1.],\n",
      "        [ 1.,  1.,  1.],\n",
      "        [ 1.,  1.,  1.]], dtype=torch.float64)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "# Converting from numpy to tensor\n",
    "x_np = np.ones((3,3))\n",
    "x_py = torch.from_numpy(x_np)\n",
    "print(x_np,x_py)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU computation completed in 0.000138s, GPU computation completed in 0.000184s\n"
     ]
    }
   ],
   "source": [
    "# Improving computational time with GPU acceleration\n",
    "import time\n",
    "use_gpu = torch.cuda.is_available()\n",
    "x = torch.randn(10000,10000)\n",
    "if use_gpu:\n",
    "    x = torch.randn(100,100)\n",
    "    cpuStart = time.time()\n",
    "    y = x*x\n",
    "    cpuEnd = time.time()-cpuStart\n",
    "    x = x.cuda()\n",
    "    gpuStart = time.time()\n",
    "    y = x*x\n",
    "    gpuEnd = time.time()-gpuStart\n",
    "    print('CPU computation completed in {:.6f}s, GPU computation completed in {:.6f}s'\\\n",
    "          .format(cpuEnd,gpuEnd))    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Autograd\n",
    " This package provides automatic differentiation for all operations on Tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 1.,  1.],\n",
      "        [ 1.,  1.]])\n",
      "tensor([[ 1.,  1.],\n",
      "        [ 1.,  1.]])\n"
     ]
    }
   ],
   "source": [
    "from torch.autograd import Variable\n",
    "a = torch.ones(2,2)\n",
    "a_var = Variable(a,requires_grad=True)\n",
    "print(a)\n",
    "print(a_var)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 3.,  3.],\n",
      "        [ 3.,  3.]])\n"
     ]
    }
   ],
   "source": [
    "b = a_var+2\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<AddBackward0 object at 0x7f42bfddfe10>\n"
     ]
    }
   ],
   "source": [
    "print(b.grad_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 27.,  27.],\n",
      "        [ 27.,  27.]])\n",
      "tensor(27.)\n"
     ]
    }
   ],
   "source": [
    "c = b*b*3\n",
    "d = c.mean()\n",
    "print(c)\n",
    "print(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "d.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 4.5000,  4.5000],\n",
      "        [ 4.5000,  4.5000]])\n"
     ]
    }
   ],
   "source": [
    "# Gradients\n",
    "print(a_var.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Torchvision datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import torch\n",
    "from torchvision import datasets,transforms\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "apply_transform = transforms.Compose([transforms.ToTensor()])\n",
    "trainDset = datasets.MNIST('./MNIST',train=True, download=True, transform= apply_transform)\n",
    "testDset = datasets.MNIST('./MNIST',train=False, download=True, transform= apply_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60000 10000\n"
     ]
    }
   ],
   "source": [
    "# Number of samples\n",
    "print(len(trainDset),len(testDset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label: tensor(5)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADgdJREFUeJzt3X9sXfV5x/HPs9D8QRoIXjUTpWFpIhQUIuZOJkwoGkXM5YeCggGhWkLKRBT3j1ii0hQNZX8MNAVFg2RqBKrsqqHJ1KWZBCghqpp0CZBOTBEmhF9mKQylqi2TFAWTH/zIHD/74x53Lvh+r3Pvufdc+3m/JMv3nuecex4d5ZPz8/pr7i4A8fxJ0Q0AKAbhB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8Q1GWNXJmZ8TghUGfublOZr6Y9v5ndYWbHzex9M3ukls8C0FhW7bP9ZjZL0m8kdUgalPSqpC53H0gsw54fqLNG7PlXSHrf3T9w9wuSfi5pdQ2fB6CBagn/Akm/m/B+MJv2R8ys28z6zay/hnUByFndL/i5e5+kPonDfqCZ1LLnH5K0cML7b2bTAEwDtYT/VUnXmtm3zGy2pO9J2ptPWwDqrerDfncfNbMeSfslzZK03d3fya0zAHVV9a2+qlbGOT9Qdw15yAfA9EX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUFUP0S1JZnZC0llJFyWNunt7Hk0hP7NmzUrWr7zyyrquv6enp2zt8ssvTy67dOnSZH39+vXJ+pNPPlm21tXVlVz2888/T9Y3b96crD/22GPJejOoKfyZW939oxw+B0ADcdgPBFVr+F3SATN7zcy682gIQGPUeti/0t2HzOzPJP3KzP7b3Q9PnCH7T4H/GIAmU9Oe392Hst+nJD0vacUk8/S5ezsXA4HmUnX4zWyOmc0dfy3pu5LezqsxAPVVy2F/q6TnzWz8c/7N3X+ZS1cA6q7q8Lv7B5L+IsdeZqxrrrkmWZ89e3ayfvPNNyfrK1euLFubN29ectn77rsvWS/S4OBgsr5t27ZkvbOzs2zt7NmzyWXfeOONZP3ll19O1qcDbvUBQRF+ICjCDwRF+IGgCD8QFOEHgjJ3b9zKzBq3sgZqa2tL1g8dOpSs1/trtc1qbGwsWX/ooYeS9XPnzlW97uHh4WT9448/TtaPHz9e9brrzd1tKvOx5weCIvxAUIQfCIrwA0ERfiAowg8ERfiBoLjPn4OWlpZk/ciRI8n64sWL82wnV5V6HxkZSdZvvfXWsrULFy4kl436/EOtuM8PIInwA0ERfiAowg8ERfiBoAg/EBThB4LKY5Te8E6fPp2sb9iwIVlftWpVsv76668n65X+hHXKsWPHkvWOjo5k/fz588n69ddfX7b28MMPJ5dFfbHnB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgKn6f38y2S1ol6ZS7L8+mtUjaLWmRpBOSHnD39B8618z9Pn+trrjiimS90nDSvb29ZWtr165NLvvggw8m67t27UrW0Xzy/D7/TyXd8aVpj0g66O7XSjqYvQcwjVQMv7sflvTlR9hWS9qRvd4h6Z6c+wJQZ9We87e6+/h4Rx9Kas2pHwANUvOz/e7uqXN5M+uW1F3regDkq9o9/0kzmy9J2e9T5WZ09z53b3f39irXBaAOqg3/XklrstdrJO3Jpx0AjVIx/Ga2S9J/SVpqZoNmtlbSZkkdZvaepL/J3gOYRiqe87t7V5nSbTn3EtaZM2dqWv6TTz6petl169Yl67t3707Wx8bGql43isUTfkBQhB8IivADQRF+ICjCDwRF+IGgGKJ7BpgzZ07Z2gsvvJBc9pZbbknW77zzzmT9wIEDyToajyG6ASQRfiAowg8ERfiBoAg/EBThB4Ii/EBQ3Oef4ZYsWZKsHz16NFkfGRlJ1l988cVkvb+/v2zt6aefTi7byH+bMwn3+QEkEX4gKMIPBEX4gaAIPxAU4QeCIvxAUNznD66zszNZf+aZZ5L1uXPnVr3ujRs3Jus7d+5M1oeHh5P1qLjPDyCJ8ANBEX4gKMIPBEX4gaAIPxAU4QeCqnif38y2S1ol6ZS7L8+mPSppnaTfZ7NtdPdfVFwZ9/mnneXLlyfrW7duTdZvu636kdx7e3uT9U2bNiXrQ0NDVa97OsvzPv9PJd0xyfR/cfe27Kdi8AE0l4rhd/fDkk43oBcADVTLOX+Pmb1pZtvN7KrcOgLQENWG/0eSlkhqkzQsaUu5Gc2s28z6zaz8H3MD0HBVhd/dT7r7RXcfk/RjSSsS8/a5e7u7t1fbJID8VRV+M5s/4W2npLfzaQdAo1xWaQYz2yXpO5K+YWaDkv5R0nfMrE2SSzoh6ft17BFAHfB9ftRk3rx5yfrdd99dtlbpbwWYpW9XHzp0KFnv6OhI1mcqvs8PIInwA0ERfiAowg8ERfiBoAg/EBS3+lCYL774Ilm/7LL0Yyijo6PJ+u2331629tJLLyWXnc641QcgifADQRF+ICjCDwRF+IGgCD8QFOEHgqr4fX7EdsMNNyTr999/f7J+4403lq1Vuo9fycDAQLJ++PDhmj5/pmPPDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBcZ9/hlu6dGmy3tPTk6zfe++9yfrVV199yT1N1cWLF5P14eHhZH1sbCzPdmYc9vxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EFTF+/xmtlDSTkmtklxSn7v/0MxaJO2WtEjSCUkPuPvH9Ws1rkr30ru6usrWKt3HX7RoUTUt5aK/vz9Z37RpU7K+d+/ePNsJZyp7/lFJf+fuyyT9laT1ZrZM0iOSDrr7tZIOZu8BTBMVw+/uw+5+NHt9VtK7khZIWi1pRzbbDkn31KtJAPm7pHN+M1sk6duSjkhqdffx5ys/VOm0AMA0MeVn+83s65KelfQDdz9j9v/Dgbm7lxuHz8y6JXXX2iiAfE1pz29mX1Mp+D9z9+eyySfNbH5Wny/p1GTLunufu7e7e3seDQPIR8XwW2kX/xNJ77r71gmlvZLWZK/XSNqTf3sA6qXiEN1mtlLSryW9JWn8O5IbVTrv/3dJ10j6rUq3+k5X+KyQQ3S3tqYvhyxbtixZf+qpp5L166677pJ7ysuRI0eS9SeeeKJsbc+e9P6Cr+RWZ6pDdFc853f3/5RU7sNuu5SmADQPnvADgiL8QFCEHwiK8ANBEX4gKMIPBMWf7p6ilpaWsrXe3t7ksm1tbcn64sWLq+opD6+88kqyvmXLlmR9//79yfpnn312yT2hMdjzA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQYe7z33TTTcn6hg0bkvUVK1aUrS1YsKCqnvLy6aeflq1t27Ytuezjjz+erJ8/f76qntD82PMDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFBh7vN3dnbWVK/FwMBAsr5v375kfXR0NFlPfed+ZGQkuSziYs8PBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0GZu6dnMFsoaaekVkkuqc/df2hmj0paJ+n32awb3f0XFT4rvTIANXN3m8p8Uwn/fEnz3f2omc2V9JqkeyQ9IOmcuz851aYIP1B/Uw1/xSf83H1Y0nD2+qyZvSup2D9dA6Bml3TOb2aLJH1b0pFsUo+ZvWlm283sqjLLdJtZv5n119QpgFxVPOz/w4xmX5f0sqRN7v6cmbVK+kil6wD/pNKpwUMVPoPDfqDOcjvnlyQz+5qkfZL2u/vWSeqLJO1z9+UVPofwA3U21fBXPOw3M5P0E0nvTgx+diFwXKekty+1SQDFmcrV/pWSfi3pLUlj2eSNkroktal02H9C0vezi4Opz2LPD9RZrof9eSH8QP3ldtgPYGYi/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBNXoIbo/kvTbCe+/kU1rRs3aW7P2JdFbtfLs7c+nOmNDv8//lZWb9bt7e2ENJDRrb83al0Rv1SqqNw77gaAIPxBU0eHvK3j9Kc3aW7P2JdFbtQrprdBzfgDFKXrPD6AghYTfzO4ws+Nm9r6ZPVJED+WY2Qkze8vMjhU9xFg2DNopM3t7wrQWM/uVmb2X/Z50mLSCenvUzIaybXfMzO4qqLeFZvaimQ2Y2Ttm9nA2vdBtl+irkO3W8MN+M5sl6TeSOiQNSnpVUpe7DzS0kTLM7ISkdncv/J6wmf21pHOSdo6PhmRm/yzptLtvzv7jvMrd/75JentUlzhyc516Kzey9N+qwG2X54jXeShiz79C0vvu/oG7X5D0c0mrC+ij6bn7YUmnvzR5taQd2esdKv3jabgyvTUFdx9296PZ67OSxkeWLnTbJfoqRBHhXyDpdxPeD6q5hvx2SQfM7DUz6y66mUm0ThgZ6UNJrUU2M4mKIzc30pdGlm6abVfNiNd544LfV61097+UdKek9dnhbVPy0jlbM92u+ZGkJSoN4zYsaUuRzWQjSz8r6QfufmZirchtN0lfhWy3IsI/JGnhhPffzKY1BXcfyn6fkvS8SqcpzeTk+CCp2e9TBffzB+5+0t0vuvuYpB+rwG2XjSz9rKSfuftz2eTCt91kfRW13YoI/6uSrjWzb5nZbEnfk7S3gD6+wszmZBdiZGZzJH1XzTf68F5Ja7LXayTtKbCXP9IsIzeXG1laBW+7phvx2t0b/iPpLpWu+P+PpH8ooocyfS2W9Eb2807RvUnapdJh4P+qdG1kraQ/lXRQ0nuS/kNSSxP19q8qjeb8pkpBm19QbytVOqR/U9Kx7Oeuorddoq9CthtP+AFBccEPCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQ/weCC5r/92q6mAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Displaying sample image from the dataset\n",
    "img = trainDset[0][0].numpy().transpose(1,2,0).squeeze(2)\n",
    "plt.imshow(img,'gray')\n",
    "print('Label: '+str(trainDset[0][1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating dataloader for loading data in batches\n",
    "trainLoader = torch.utils.data.DataLoader(trainDset, batch_size=10, shuffle=True, num_workers=1, pin_memory=False)\n",
    "testLoader = torch.utils.data.DataLoader(testDset, batch_size=10, shuffle=True, num_workers=1, pin_memory=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining a multi-layer perceptron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn.functional as F\n",
    "import torch.nn as nn\n",
    "class MLP(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(MLP, self).__init__()\n",
    "        self.fc1 = nn.Linear(28*28,100)\n",
    "        self.fc2 = nn.Linear(100,10)\n",
    "    def forward(self, x):\n",
    "        x = self.fc1(x)\n",
    "        x = torch.sigmoid(x)\n",
    "        x = self.fc2(x)\n",
    "        x = F.log_softmax(x,dim=1)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MLP(\n",
      "  (fc1): Linear(in_features=784, out_features=100, bias=True)\n",
      "  (fc2): Linear(in_features=100, out_features=10, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "net = MLP()\n",
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dimensions of first layer weights: torch.Size([100, 784])\n",
      "Dimensions of first layer bias: torch.Size([100])\n",
      "torch.Size([100, 784])\n",
      "torch.Size([100])\n",
      "torch.Size([10, 100])\n",
      "torch.Size([10])\n",
      "Total number of parameters: 79510\n"
     ]
    }
   ],
   "source": [
    "params = list(net.parameters())\n",
    "# print('No. of parameters :'+str(len(params)))\n",
    "print('Dimensions of first layer weights: '+str(params[0].size())) # Weights of fc1\n",
    "print('Dimensions of first layer bias: '+str(params[1].size())) # Biases of fc1\n",
    "\n",
    "totalParams = 0\n",
    "for param in params:    \n",
    "    print(param.size())\n",
    "    totalParams += np.sum(np.prod(param.size()))\n",
    "print('Total number of parameters: '+str(totalParams))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "inp = trainLoader.dataset[0][0]\n",
    "label = trainLoader.dataset[0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 28, 28])\n",
      "torch.Size([1, 10])\n"
     ]
    }
   ],
   "source": [
    "from torch.autograd import Variable\n",
    "# Feed-forward data through network\n",
    "out = net(Variable(inp.view(-1,28*28)))\n",
    "print(inp.size())\n",
    "print(out.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Backpropagating gradients\n",
    "net.zero_grad()\n",
    "out.backward(torch.randn(1, 10)) # Using random gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(2.7730)\n"
     ]
    }
   ],
   "source": [
    "out = net(Variable(inp.view(-1,28*28)))\n",
    "# Defining loss function\n",
    "criterion = nn.NLLLoss() # Negative log-likelihood loss\n",
    "label = label*torch.ones(1).long() # Converting to tensor\n",
    "loss = criterion(out,Variable(label)) # NLLLoss() expects the labels to be of dtype 'long'\n",
    "print(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bias gradient of fc1 before backward\n",
      "tensor([ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.])\n",
      "Bias gradient of fc1 after backward\n",
      "tensor(1.00000e-02 *\n",
      "       [-0.1304,  0.2909, -0.6959, -1.9853,  0.1184,  0.8501, -0.7394,\n",
      "         2.2983, -1.7532, -0.2429])\n"
     ]
    }
   ],
   "source": [
    "# Backprogattion\n",
    "net.zero_grad()     # zeroes the gradient buffers of all parameters\n",
    "\n",
    "print('Bias gradient of fc1 before backward')\n",
    "print(net.fc1.bias.grad[:10])\n",
    "\n",
    "loss.backward()\n",
    "\n",
    "print('Bias gradient of fc1 after backward')\n",
    "print(net.fc1.bias.grad[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(1.00000e-02 *\n",
      "       [ 0.0116,  0.6293, -6.6868,  5.7245,  1.2912])\n",
      "tensor(1.00000e-02 *\n",
      "       [-2.7393, -2.2785, -9.6455,  2.9049, -2.0219])\n"
     ]
    }
   ],
   "source": [
    "import copy\n",
    "# Updataing weights of the network\n",
    "learning_rate = 1\n",
    "init_params = copy.deepcopy(net.fc2.weight.data) # Copying initial parameters\n",
    "\n",
    "for f in net.parameters():    \n",
    "    f.data.sub_(f.grad.data * learning_rate)\n",
    "updated_params = net.fc2.weight.data\n",
    "print(init_params[0,:5])\n",
    "print(updated_params[0,:5])\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}