Q3.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.7.9 64-bit"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import datasets,layers,models\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_test, y_test) = keras.datasets.cifar10.load_data()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Noof classes: 10\nNoof training images: 50000\nNoof test images: 10000\nNoof features: 3072\n(50000, 1)\n"
     ]
    }
   ],
   "source": [
    "K = len(np.unique(y_train)) # Classes\n",
    "Ntr = x_train.shape[0]\n",
    "Nte = x_test.shape[0]\n",
    "Din = 3072 # CIFAR10\n",
    "print('Noof classes:',K)\n",
    "print('Noof training images:',Ntr)\n",
    "print('Noof test images:',Nte)\n",
    "print('Noof features:',Din)\n",
    "\n",
    "x_train=x_train[range(Ntr),:]\n",
    "y_train=y_train[range(Ntr),:]\n",
    "print(y_train.shape)\n",
    "x_test=x_test[range(Nte),:]\n",
    "x_test=x_test[range(Nte),:]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "original labels: (50000, 1)\n",
      "labels in class matrix: (50000, 10)\n",
      "x_train: (50000, 3072)\n",
      "x_test: (10000, 3072)\n"
     ]
    }
   ],
   "source": [
    "print('original labels:',y_train.shape)\n",
    "y_train = tf.keras.utils.to_categorical(y_train, num_classes=K)\n",
    "print('labels in class matrix:',y_train.shape)\n",
    "y_test = tf.keras.utils.to_categorical(y_test, num_classes=K)\n",
    "\n",
    "x_train = np.reshape(x_train,(Ntr,Din))\n",
    "x_test = np.reshape(x_test,(Nte,Din))\n",
    "print('x_train:', x_train.shape)\n",
    "print('x_test:', x_test.shape)\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "\n",
    "# Normalize pixel values\n",
    "#x_train, x_test = x_train / 255.0, x_test / 255.0\n",
    "mean_image = np.mean(x_train, axis=0)\n",
    "x_train = x_train - mean_image\n",
    "x_test = x_test - mean_image\n",
    "\n"
   ]
  },
  {
   "source": [
    "Two Layer Fully Connected Network"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "w1: (3072, 200)\nb1: (200,)\nw2: (200, 10)\nb2: (10,)\n"
     ]
    }
   ],
   "source": [
    "H=200\n",
    "std=1e-6\n",
    "w1 = std*np.random.randn(Din, H)\n",
    "w2 = std*np.random.randn(H, K)\n",
    "b1 = np.zeros(H)\n",
    "b2=np.zeros(K)\n",
    "print(\"w1:\", w1.shape)\n",
    "print(\"b1:\", b1.shape)\n",
    "print(\"w2:\", w2.shape)\n",
    "print(\"b2:\", b2.shape)"
   ]
  },
  {
   "source": [
    "Stochastic Gradient Descent (Going through batches in order)"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "epoch 0/300: train loss= 0.747635-- ,test loss= 0.771751--,train accuracy= 0.795200--, test accuracy= 0.780480\n",
      "epoch 10/300: train loss= 0.782559-- ,test loss= 0.779988--,train accuracy= 0.770400--, test accuracy= 0.774110\n",
      "epoch 20/300: train loss= 0.773794-- ,test loss= 0.775199--,train accuracy= 0.770600--, test accuracy= 0.782600\n",
      "epoch 30/300: train loss= 0.779001-- ,test loss= 0.775631--,train accuracy= 0.792600--, test accuracy= 0.784080\n",
      "epoch 40/300: train loss= 0.783227-- ,test loss= 0.775139--,train accuracy= 0.779200--, test accuracy= 0.788540\n",
      "epoch 50/300: train loss= 0.751493-- ,test loss= 0.767492--,train accuracy= 0.801200--, test accuracy= 0.782460\n",
      "epoch 60/300: train loss= 0.763654-- ,test loss= 0.771161--,train accuracy= 0.762200--, test accuracy= 0.780100\n",
      "epoch 70/300: train loss= 0.765465-- ,test loss= 0.772608--,train accuracy= 0.788000--, test accuracy= 0.781350\n",
      "epoch 80/300: train loss= 0.764949-- ,test loss= 0.772618--,train accuracy= 0.775400--, test accuracy= 0.777780\n",
      "epoch 90/300: train loss= 0.765781-- ,test loss= 0.775972--,train accuracy= 0.766800--, test accuracy= 0.773680\n",
      "epoch 100/300: train loss= 0.739031-- ,test loss= 0.768255--,train accuracy= 0.799400--, test accuracy= 0.782850\n",
      "epoch 110/300: train loss= 0.776545-- ,test loss= 0.774678--,train accuracy= 0.783200--, test accuracy= 0.775660\n",
      "epoch 120/300: train loss= 0.755447-- ,test loss= 0.765866--,train accuracy= 0.792400--, test accuracy= 0.785080\n",
      "epoch 130/300: train loss= 0.766554-- ,test loss= 0.766412--,train accuracy= 0.795600--, test accuracy= 0.788830\n",
      "epoch 140/300: train loss= 0.768722-- ,test loss= 0.767870--,train accuracy= 0.775000--, test accuracy= 0.789700\n",
      "epoch 150/300: train loss= 0.743189-- ,test loss= 0.761773--,train accuracy= 0.804000--, test accuracy= 0.786110\n",
      "epoch 160/300: train loss= 0.750277-- ,test loss= 0.763867--,train accuracy= 0.784200--, test accuracy= 0.783710\n",
      "epoch 170/300: train loss= 0.755134-- ,test loss= 0.763793--,train accuracy= 0.789200--, test accuracy= 0.788440\n",
      "epoch 180/300: train loss= 0.754116-- ,test loss= 0.762556--,train accuracy= 0.780800--, test accuracy= 0.785220\n",
      "epoch 190/300: train loss= 0.754684-- ,test loss= 0.767964--,train accuracy= 0.772400--, test accuracy= 0.780720\n",
      "epoch 200/300: train loss= 0.727202-- ,test loss= 0.761151--,train accuracy= 0.803400--, test accuracy= 0.786480\n",
      "epoch 210/300: train loss= 0.764079-- ,test loss= 0.764380--,train accuracy= 0.781800--, test accuracy= 0.785950\n",
      "epoch 220/300: train loss= 0.745755-- ,test loss= 0.762998--,train accuracy= 0.795800--, test accuracy= 0.787890\n",
      "epoch 230/300: train loss= 0.759412-- ,test loss= 0.763613--,train accuracy= 0.795800--, test accuracy= 0.787900\n",
      "epoch 240/300: train loss= 0.752214-- ,test loss= 0.758083--,train accuracy= 0.795000--, test accuracy= 0.793880\n",
      "epoch 250/300: train loss= 0.730079-- ,test loss= 0.757043--,train accuracy= 0.823200--, test accuracy= 0.791830\n",
      "epoch 260/300: train loss= 0.741482-- ,test loss= 0.758161--,train accuracy= 0.784000--, test accuracy= 0.788280\n",
      "epoch 270/300: train loss= 0.744892-- ,test loss= 0.760921--,train accuracy= 0.794400--, test accuracy= 0.788540\n",
      "epoch 280/300: train loss= 0.742871-- ,test loss= 0.759450--,train accuracy= 0.774800--, test accuracy= 0.787610\n",
      "epoch 290/300: train loss= 0.742591-- ,test loss= 0.764060--,train accuracy= 0.780600--, test accuracy= 0.784450\n"
     ]
    }
   ],
   "source": [
    "batch_size = 500\n",
    "step=Ntr/batch_size\n",
    "iterations = 300\n",
    "lr = 1.4e-2\n",
    "lr_decay=0.999\n",
    "reg =5e-6\n",
    "train_loss_history = []\n",
    "train_acc_history = []\n",
    "test_loss_history = []\n",
    "test_acc_history = []\n",
    "seed = 0\n",
    "\n",
    "for t in range(iterations):\n",
    "    start=int(batch_size*((t)%step))\n",
    "    x=x_train[start:start+batch_size,:]\n",
    "    y=y_train[start:start+batch_size,:]\n",
    "\n",
    "    # Forward pass\n",
    "    z1=1.0/(1.0 + np.exp(-(x.dot(w1)+b1)))\n",
    "    y_pred=z1.dot(w2)+b2\n",
    "    z=1.0/(1.0 + np.exp(-(x_test.dot(w1)+b1)))\n",
    "    y_pred_test=z.dot(w2)+b2\n",
    "\n",
    "    #loss\n",
    "    loss=(1.0/batch_size)*np.square(y_pred-y).sum() + reg * (np.sum(w1*w1) + np.sum(w2*w2))\n",
    "    test_loss=(1.0/Nte)*np.square(y_pred_test-y_test).sum() + reg * (np.sum(w1*w1) + np.sum(w2*w2))\n",
    "    train_loss_history.append(loss)\n",
    "    test_loss_history.append(test_loss)\n",
    "\n",
    "    #accuracy\n",
    "    train_acc = 1.0 - (1/(batch_size*K))*(np.abs(np.argmax(y,axis=1) - np.argmax(y_pred,axis=1))).sum()\n",
    "    train_acc_history.append(train_acc)\n",
    "    test_acc = 1.0 - (1/(Nte*K))*(np.abs(np.argmax(y_test,axis=1) - np.argmax(y_pred_test,axis=1))).sum()\n",
    "    test_acc_history.append(test_acc)\n",
    "\n",
    "    if t%10==0:\n",
    "        print('epoch %d/%d: train loss= %f-- ,test loss= %f--,train accuracy= %f--, test accuracy= %f' % (t,iterations,loss,test_loss,train_acc,test_acc))\n",
    "\n",
    "        \n",
    "    # Backward pass\n",
    "\n",
    "    dy_pred=(1./batch_size)*2.0*(y_pred-y)\n",
    "    dw2=z1.T.dot(dy_pred) + reg * w2\n",
    "    db2=dy_pred.sum(axis=0)\n",
    "    dz1=dy_pred.dot(w2.T)\n",
    "    dw1=x.T.dot(dz1*z1*(1-z1))+reg*w1\n",
    "    db1=(dz1*z1*(1-z1)).sum(axis=0)\n",
    "\n",
    "\n",
    "    \n",
    "    w1 -= lr*dw1\n",
    "    w2 -= lr*dw2\n",
    "    b1 -= lr*db1\n",
    "    b2 -= lr*db2\n",
    "    lr *= lr_decay\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "source": [
    "Stochastic Gradient Descent (Going through batches in a random order)"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "epoch 0/300: \n",
      "train loss= 0.716159-- ,test loss= 0.746209--,train accuracy= 0.809000--, test accuracy= 0.793060\n",
      "epoch 0/300: \n",
      "train loss= 0.726833-- ,test loss= 0.766283--,train accuracy= 0.797600--, test accuracy= 0.783790\n",
      "epoch 0/300: \n",
      "train loss= 0.754795-- ,test loss= 0.777773--,train accuracy= 0.781800--, test accuracy= 0.768990\n",
      "epoch 0/300: \n",
      "train loss= 0.722950-- ,test loss= 0.755010--,train accuracy= 0.795600--, test accuracy= 0.786830\n",
      "epoch 0/300: \n",
      "train loss= 0.720323-- ,test loss= 0.759143--,train accuracy= 0.813600--, test accuracy= 0.789960\n",
      "epoch 10/300: \n",
      "train loss= 0.644920-- ,test loss= 0.748028--,train accuracy= 0.847000--, test accuracy= 0.797090\n",
      "epoch 10/300: \n",
      "train loss= 0.653220-- ,test loss= 0.747254--,train accuracy= 0.861800--, test accuracy= 0.795560\n",
      "epoch 10/300: \n",
      "train loss= 0.634627-- ,test loss= 0.748866--,train accuracy= 0.851000--, test accuracy= 0.797200\n",
      "epoch 10/300: \n",
      "train loss= 0.645471-- ,test loss= 0.747715--,train accuracy= 0.857800--, test accuracy= 0.798000\n",
      "epoch 10/300: \n",
      "train loss= 0.627113-- ,test loss= 0.751945--,train accuracy= 0.852800--, test accuracy= 0.796250\n",
      "epoch 20/300: \n",
      "train loss= 0.589036-- ,test loss= 0.754420--,train accuracy= 0.893200--, test accuracy= 0.798660\n",
      "epoch 20/300: \n",
      "train loss= 0.590546-- ,test loss= 0.754066--,train accuracy= 0.895000--, test accuracy= 0.798140\n",
      "epoch 20/300: \n",
      "train loss= 0.586744-- ,test loss= 0.754493--,train accuracy= 0.882400--, test accuracy= 0.797820\n",
      "epoch 20/300: \n",
      "train loss= 0.572324-- ,test loss= 0.754504--,train accuracy= 0.888000--, test accuracy= 0.800570\n",
      "epoch 20/300: \n",
      "train loss= 0.598604-- ,test loss= 0.755102--,train accuracy= 0.892000--, test accuracy= 0.797700\n",
      "epoch 30/300: \n",
      "train loss= 0.564391-- ,test loss= 0.758781--,train accuracy= 0.900800--, test accuracy= 0.796800\n",
      "epoch 30/300: \n",
      "train loss= 0.576553-- ,test loss= 0.758757--,train accuracy= 0.893200--, test accuracy= 0.797530\n",
      "epoch 30/300: \n",
      "train loss= 0.567372-- ,test loss= 0.758938--,train accuracy= 0.897000--, test accuracy= 0.798290\n",
      "epoch 30/300: \n",
      "train loss= 0.570414-- ,test loss= 0.758799--,train accuracy= 0.884000--, test accuracy= 0.798070\n",
      "epoch 30/300: \n",
      "train loss= 0.568090-- ,test loss= 0.759362--,train accuracy= 0.893200--, test accuracy= 0.796560\n",
      "epoch 40/300: \n",
      "train loss= 0.574479-- ,test loss= 0.760952--,train accuracy= 0.890200--, test accuracy= 0.796900\n",
      "epoch 40/300: \n",
      "train loss= 0.579116-- ,test loss= 0.760927--,train accuracy= 0.886600--, test accuracy= 0.797980\n",
      "epoch 40/300: \n",
      "train loss= 0.570462-- ,test loss= 0.761150--,train accuracy= 0.890000--, test accuracy= 0.797550\n",
      "epoch 40/300: \n",
      "train loss= 0.558898-- ,test loss= 0.760980--,train accuracy= 0.896600--, test accuracy= 0.797260\n",
      "epoch 40/300: \n",
      "train loss= 0.581155-- ,test loss= 0.761023--,train accuracy= 0.897600--, test accuracy= 0.797360\n",
      "epoch 50/300: \n",
      "train loss= 0.561018-- ,test loss= 0.761672--,train accuracy= 0.898000--, test accuracy= 0.797250\n",
      "epoch 50/300: \n",
      "train loss= 0.575392-- ,test loss= 0.761753--,train accuracy= 0.887600--, test accuracy= 0.797540\n",
      "epoch 50/300: \n",
      "train loss= 0.575381-- ,test loss= 0.761780--,train accuracy= 0.895200--, test accuracy= 0.797200\n",
      "epoch 50/300: \n",
      "train loss= 0.561373-- ,test loss= 0.761721--,train accuracy= 0.894800--, test accuracy= 0.797080\n",
      "epoch 50/300: \n",
      "train loss= 0.570775-- ,test loss= 0.761878--,train accuracy= 0.886600--, test accuracy= 0.797260\n",
      "epoch 60/300: \n",
      "train loss= 0.554415-- ,test loss= 0.762044--,train accuracy= 0.901600--, test accuracy= 0.797400\n",
      "epoch 60/300: \n",
      "train loss= 0.572986-- ,test loss= 0.762050--,train accuracy= 0.882200--, test accuracy= 0.797180\n",
      "epoch 60/300: \n",
      "train loss= 0.569285-- ,test loss= 0.762044--,train accuracy= 0.873200--, test accuracy= 0.797210\n",
      "epoch 60/300: \n",
      "train loss= 0.569541-- ,test loss= 0.762079--,train accuracy= 0.889200--, test accuracy= 0.797530\n",
      "epoch 60/300: \n",
      "train loss= 0.558269-- ,test loss= 0.762061--,train accuracy= 0.897400--, test accuracy= 0.797320\n",
      "epoch 70/300: \n",
      "train loss= 0.545547-- ,test loss= 0.762157--,train accuracy= 0.906000--, test accuracy= 0.797340\n",
      "epoch 70/300: \n",
      "train loss= 0.549908-- ,test loss= 0.762174--,train accuracy= 0.899600--, test accuracy= 0.797420\n",
      "epoch 70/300: \n",
      "train loss= 0.563515-- ,test loss= 0.762163--,train accuracy= 0.890400--, test accuracy= 0.797410\n",
      "epoch 70/300: \n",
      "train loss= 0.574546-- ,test loss= 0.762165--,train accuracy= 0.891600--, test accuracy= 0.797480\n",
      "epoch 70/300: \n",
      "train loss= 0.574202-- ,test loss= 0.762168--,train accuracy= 0.884400--, test accuracy= 0.797540\n",
      "epoch 80/300: \n",
      "train loss= 0.572548-- ,test loss= 0.762200--,train accuracy= 0.883400--, test accuracy= 0.797360\n",
      "epoch 80/300: \n",
      "train loss= 0.571745-- ,test loss= 0.762203--,train accuracy= 0.889800--, test accuracy= 0.797330\n",
      "epoch 80/300: \n",
      "train loss= 0.555867-- ,test loss= 0.762201--,train accuracy= 0.912400--, test accuracy= 0.797360\n",
      "epoch 80/300: \n",
      "train loss= 0.573974-- ,test loss= 0.762203--,train accuracy= 0.885200--, test accuracy= 0.797270\n",
      "epoch 80/300: \n",
      "train loss= 0.547369-- ,test loss= 0.762205--,train accuracy= 0.899600--, test accuracy= 0.797330\n",
      "epoch 90/300: \n",
      "train loss= 0.539313-- ,test loss= 0.762216--,train accuracy= 0.896600--, test accuracy= 0.797390\n",
      "epoch 90/300: \n",
      "train loss= 0.566732-- ,test loss= 0.762217--,train accuracy= 0.893000--, test accuracy= 0.797310\n",
      "epoch 90/300: \n",
      "train loss= 0.557519-- ,test loss= 0.762217--,train accuracy= 0.897400--, test accuracy= 0.797310\n",
      "epoch 90/300: \n",
      "train loss= 0.554201-- ,test loss= 0.762215--,train accuracy= 0.904600--, test accuracy= 0.797400\n",
      "epoch 90/300: \n",
      "train loss= 0.532158-- ,test loss= 0.762215--,train accuracy= 0.902400--, test accuracy= 0.797390\n",
      "epoch 100/300: \n",
      "train loss= 0.574409-- ,test loss= 0.762222--,train accuracy= 0.884200--, test accuracy= 0.797390\n",
      "epoch 100/300: \n",
      "train loss= 0.569471-- ,test loss= 0.762222--,train accuracy= 0.896200--, test accuracy= 0.797410\n",
      "epoch 100/300: \n",
      "train loss= 0.580449-- ,test loss= 0.762221--,train accuracy= 0.895400--, test accuracy= 0.797410\n",
      "epoch 100/300: \n",
      "train loss= 0.536779-- ,test loss= 0.762222--,train accuracy= 0.902200--, test accuracy= 0.797390\n",
      "epoch 100/300: \n",
      "train loss= 0.561164-- ,test loss= 0.762221--,train accuracy= 0.908000--, test accuracy= 0.797390\n",
      "epoch 110/300: \n",
      "train loss= 0.568922-- ,test loss= 0.762224--,train accuracy= 0.890200--, test accuracy= 0.797390\n",
      "epoch 110/300: \n",
      "train loss= 0.553974-- ,test loss= 0.762224--,train accuracy= 0.901600--, test accuracy= 0.797390\n",
      "epoch 110/300: \n",
      "train loss= 0.560314-- ,test loss= 0.762224--,train accuracy= 0.897000--, test accuracy= 0.797390\n",
      "epoch 110/300: \n",
      "train loss= 0.536769-- ,test loss= 0.762224--,train accuracy= 0.902200--, test accuracy= 0.797410\n",
      "epoch 110/300: \n",
      "train loss= 0.566400-- ,test loss= 0.762224--,train accuracy= 0.893200--, test accuracy= 0.797390\n",
      "epoch 120/300: \n",
      "train loss= 0.559900-- ,test loss= 0.762224--,train accuracy= 0.904600--, test accuracy= 0.797390\n",
      "epoch 120/300: \n",
      "train loss= 0.590467-- ,test loss= 0.762224--,train accuracy= 0.897400--, test accuracy= 0.797390\n",
      "epoch 120/300: \n",
      "train loss= 0.542067-- ,test loss= 0.762224--,train accuracy= 0.905200--, test accuracy= 0.797390\n",
      "epoch 120/300: \n",
      "train loss= 0.555133-- ,test loss= 0.762224--,train accuracy= 0.894400--, test accuracy= 0.797390\n",
      "epoch 120/300: \n",
      "train loss= 0.555240-- ,test loss= 0.762224--,train accuracy= 0.895000--, test accuracy= 0.797390\n",
      "epoch 130/300: \n",
      "train loss= 0.572940-- ,test loss= 0.762225--,train accuracy= 0.899600--, test accuracy= 0.797390\n",
      "epoch 130/300: \n",
      "train loss= 0.563047-- ,test loss= 0.762225--,train accuracy= 0.897600--, test accuracy= 0.797390\n",
      "epoch 130/300: \n",
      "train loss= 0.554305-- ,test loss= 0.762225--,train accuracy= 0.899200--, test accuracy= 0.797390\n",
      "epoch 130/300: \n",
      "train loss= 0.543474-- ,test loss= 0.762225--,train accuracy= 0.903200--, test accuracy= 0.797390\n",
      "epoch 130/300: \n",
      "train loss= 0.554164-- ,test loss= 0.762225--,train accuracy= 0.904600--, test accuracy= 0.797390\n",
      "epoch 140/300: \n",
      "train loss= 0.537417-- ,test loss= 0.762225--,train accuracy= 0.906000--, test accuracy= 0.797390\n",
      "epoch 140/300: \n",
      "train loss= 0.555203-- ,test loss= 0.762225--,train accuracy= 0.904600--, test accuracy= 0.797390\n",
      "epoch 140/300: \n",
      "train loss= 0.568987-- ,test loss= 0.762225--,train accuracy= 0.897400--, test accuracy= 0.797390\n",
      "epoch 140/300: \n",
      "train loss= 0.562749-- ,test loss= 0.762225--,train accuracy= 0.894600--, test accuracy= 0.797390\n",
      "epoch 140/300: \n",
      "train loss= 0.563846-- ,test loss= 0.762225--,train accuracy= 0.890600--, test accuracy= 0.797390\n",
      "epoch 150/300: \n",
      "train loss= 0.573886-- ,test loss= 0.762225--,train accuracy= 0.884800--, test accuracy= 0.797390\n",
      "epoch 150/300: \n",
      "train loss= 0.572486-- ,test loss= 0.762225--,train accuracy= 0.883400--, test accuracy= 0.797390\n",
      "epoch 150/300: \n",
      "train loss= 0.559899-- ,test loss= 0.762225--,train accuracy= 0.904600--, test accuracy= 0.797390\n",
      "epoch 150/300: \n",
      "train loss= 0.569455-- ,test loss= 0.762225--,train accuracy= 0.896200--, test accuracy= 0.797390\n",
      "epoch 150/300: \n",
      "train loss= 0.539287-- ,test loss= 0.762225--,train accuracy= 0.896600--, test accuracy= 0.797390\n",
      "epoch 160/300: \n",
      "train loss= 0.580435-- ,test loss= 0.762225--,train accuracy= 0.895400--, test accuracy= 0.797390\n",
      "epoch 160/300: \n",
      "train loss= 0.553420-- ,test loss= 0.762225--,train accuracy= 0.903600--, test accuracy= 0.797390\n",
      "epoch 160/300: \n",
      "train loss= 0.550500-- ,test loss= 0.762225--,train accuracy= 0.895000--, test accuracy= 0.797390\n",
      "epoch 160/300: \n",
      "train loss= 0.565618-- ,test loss= 0.762225--,train accuracy= 0.892800--, test accuracy= 0.797390\n",
      "epoch 160/300: \n",
      "train loss= 0.569163-- ,test loss= 0.762225--,train accuracy= 0.888800--, test accuracy= 0.797390\n",
      "epoch 170/300: \n",
      "train loss= 0.551573-- ,test loss= 0.762225--,train accuracy= 0.909600--, test accuracy= 0.797390\n",
      "epoch 170/300: \n",
      "train loss= 0.553481-- ,test loss= 0.762225--,train accuracy= 0.901800--, test accuracy= 0.797390\n",
      "epoch 170/300: \n",
      "train loss= 0.557010-- ,test loss= 0.762225--,train accuracy= 0.887600--, test accuracy= 0.797390\n",
      "epoch 170/300: \n",
      "train loss= 0.559899-- ,test loss= 0.762225--,train accuracy= 0.904600--, test accuracy= 0.797390\n",
      "epoch 170/300: \n",
      "train loss= 0.574401-- ,test loss= 0.762225--,train accuracy= 0.884200--, test accuracy= 0.797390\n",
      "epoch 180/300: \n",
      "train loss= 0.564466-- ,test loss= 0.762225--,train accuracy= 0.878800--, test accuracy= 0.797390\n",
      "epoch 180/300: \n",
      "train loss= 0.574340-- ,test loss= 0.762225--,train accuracy= 0.891600--, test accuracy= 0.797390\n",
      "epoch 180/300: \n",
      "train loss= 0.574401-- ,test loss= 0.762225--,train accuracy= 0.884200--, test accuracy= 0.797390\n",
      "epoch 180/300: \n",
      "train loss= 0.566116-- ,test loss= 0.762225--,train accuracy= 0.889800--, test accuracy= 0.797390\n",
      "epoch 180/300: \n",
      "train loss= 0.549312-- ,test loss= 0.762225--,train accuracy= 0.907000--, test accuracy= 0.797390\n",
      "epoch 190/300: \n",
      "train loss= 0.554872-- ,test loss= 0.762225--,train accuracy= 0.901000--, test accuracy= 0.797390\n",
      "epoch 190/300: \n",
      "train loss= 0.578892-- ,test loss= 0.762225--,train accuracy= 0.896400--, test accuracy= 0.797390\n",
      "epoch 190/300: \n",
      "train loss= 0.549312-- ,test loss= 0.762225--,train accuracy= 0.907000--, test accuracy= 0.797390\n",
      "epoch 190/300: \n",
      "train loss= 0.571180-- ,test loss= 0.762225--,train accuracy= 0.905200--, test accuracy= 0.797390\n",
      "epoch 190/300: \n",
      "train loss= 0.565317-- ,test loss= 0.762225--,train accuracy= 0.891600--, test accuracy= 0.797390\n",
      "epoch 200/300: \n",
      "train loss= 0.553419-- ,test loss= 0.762225--,train accuracy= 0.903600--, test accuracy= 0.797390\n",
      "epoch 200/300: \n",
      "train loss= 0.555769-- ,test loss= 0.762225--,train accuracy= 0.913400--, test accuracy= 0.797390\n",
      "epoch 200/300: \n",
      "train loss= 0.577127-- ,test loss= 0.762225--,train accuracy= 0.873800--, test accuracy= 0.797390\n",
      "epoch 200/300: \n",
      "train loss= 0.572486-- ,test loss= 0.762225--,train accuracy= 0.883400--, test accuracy= 0.797390\n",
      "epoch 200/300: \n",
      "train loss= 0.567779-- ,test loss= 0.762225--,train accuracy= 0.903400--, test accuracy= 0.797390\n",
      "epoch 210/300: \n",
      "train loss= 0.545572-- ,test loss= 0.762225--,train accuracy= 0.893800--, test accuracy= 0.797390\n",
      "epoch 210/300: \n",
      "train loss= 0.573271-- ,test loss= 0.762225--,train accuracy= 0.894600--, test accuracy= 0.797390\n",
      "epoch 210/300: \n",
      "train loss= 0.553214-- ,test loss= 0.762225--,train accuracy= 0.893800--, test accuracy= 0.797390\n",
      "epoch 210/300: \n",
      "train loss= 0.562659-- ,test loss= 0.762225--,train accuracy= 0.894000--, test accuracy= 0.797390\n",
      "epoch 210/300: \n",
      "train loss= 0.549730-- ,test loss= 0.762225--,train accuracy= 0.899600--, test accuracy= 0.797390\n",
      "epoch 220/300: \n",
      "train loss= 0.553980-- ,test loss= 0.762225--,train accuracy= 0.892800--, test accuracy= 0.797390\n",
      "epoch 220/300: \n",
      "train loss= 0.559069-- ,test loss= 0.762225--,train accuracy= 0.878600--, test accuracy= 0.797390\n",
      "epoch 220/300: \n",
      "train loss= 0.574070-- ,test loss= 0.762225--,train accuracy= 0.885800--, test accuracy= 0.797390\n",
      "epoch 220/300: \n",
      "train loss= 0.549829-- ,test loss= 0.762225--,train accuracy= 0.907000--, test accuracy= 0.797390\n",
      "epoch 220/300: \n",
      "train loss= 0.569827-- ,test loss= 0.762225--,train accuracy= 0.881200--, test accuracy= 0.797390\n",
      "epoch 230/300: \n",
      "train loss= 0.557432-- ,test loss= 0.762225--,train accuracy= 0.896000--, test accuracy= 0.797390\n",
      "epoch 230/300: \n",
      "train loss= 0.560469-- ,test loss= 0.762225--,train accuracy= 0.901600--, test accuracy= 0.797390\n",
      "epoch 230/300: \n",
      "train loss= 0.569163-- ,test loss= 0.762225--,train accuracy= 0.888800--, test accuracy= 0.797390\n",
      "epoch 230/300: \n",
      "train loss= 0.555769-- ,test loss= 0.762225--,train accuracy= 0.913400--, test accuracy= 0.797390\n",
      "epoch 230/300: \n",
      "train loss= 0.553481-- ,test loss= 0.762225--,train accuracy= 0.901800--, test accuracy= 0.797390\n",
      "epoch 240/300: \n",
      "train loss= 0.538592-- ,test loss= 0.762225--,train accuracy= 0.898800--, test accuracy= 0.797390\n",
      "epoch 240/300: \n",
      "train loss= 0.559069-- ,test loss= 0.762225--,train accuracy= 0.878600--, test accuracy= 0.797390\n",
      "epoch 240/300: \n",
      "train loss= 0.553980-- ,test loss= 0.762225--,train accuracy= 0.892800--, test accuracy= 0.797390\n",
      "epoch 240/300: \n",
      "train loss= 0.547197-- ,test loss= 0.762225--,train accuracy= 0.908200--, test accuracy= 0.797390\n",
      "epoch 240/300: \n",
      "train loss= 0.545377-- ,test loss= 0.762225--,train accuracy= 0.906000--, test accuracy= 0.797390\n",
      "epoch 250/300: \n",
      "train loss= 0.557492-- ,test loss= 0.762225--,train accuracy= 0.897400--, test accuracy= 0.797390\n",
      "epoch 250/300: \n",
      "train loss= 0.553481-- ,test loss= 0.762225--,train accuracy= 0.901800--, test accuracy= 0.797390\n",
      "epoch 250/300: \n",
      "train loss= 0.550500-- ,test loss= 0.762225--,train accuracy= 0.895000--, test accuracy= 0.797390\n",
      "epoch 250/300: \n",
      "train loss= 0.547380-- ,test loss= 0.762225--,train accuracy= 0.902400--, test accuracy= 0.797390\n",
      "epoch 250/300: \n",
      "train loss= 0.554163-- ,test loss= 0.762225--,train accuracy= 0.904600--, test accuracy= 0.797390\n",
      "epoch 260/300: \n",
      "train loss= 0.550500-- ,test loss= 0.762225--,train accuracy= 0.895000--, test accuracy= 0.797390\n",
      "epoch 260/300: \n",
      "train loss= 0.545572-- ,test loss= 0.762225--,train accuracy= 0.893800--, test accuracy= 0.797390\n",
      "epoch 260/300: \n",
      "train loss= 0.551573-- ,test loss= 0.762225--,train accuracy= 0.909600--, test accuracy= 0.797390\n",
      "epoch 260/300: \n",
      "train loss= 0.575598-- ,test loss= 0.762225--,train accuracy= 0.897600--, test accuracy= 0.797390\n",
      "epoch 260/300: \n",
      "train loss= 0.566703-- ,test loss= 0.762225--,train accuracy= 0.893200--, test accuracy= 0.797390\n",
      "epoch 270/300: \n",
      "train loss= 0.569455-- ,test loss= 0.762225--,train accuracy= 0.896200--, test accuracy= 0.797390\n",
      "epoch 270/300: \n",
      "train loss= 0.542066-- ,test loss= 0.762225--,train accuracy= 0.905200--, test accuracy= 0.797390\n",
      "epoch 270/300: \n",
      "train loss= 0.580435-- ,test loss= 0.762225--,train accuracy= 0.895400--, test accuracy= 0.797390\n",
      "epoch 270/300: \n",
      "train loss= 0.558005-- ,test loss= 0.762225--,train accuracy= 0.898800--, test accuracy= 0.797390\n",
      "epoch 270/300: \n",
      "train loss= 0.577240-- ,test loss= 0.762225--,train accuracy= 0.887200--, test accuracy= 0.797390\n",
      "epoch 280/300: \n",
      "train loss= 0.549312-- ,test loss= 0.762225--,train accuracy= 0.907000--, test accuracy= 0.797390\n",
      "epoch 280/300: \n",
      "train loss= 0.569163-- ,test loss= 0.762225--,train accuracy= 0.888800--, test accuracy= 0.797390\n",
      "epoch 280/300: \n",
      "train loss= 0.563047-- ,test loss= 0.762225--,train accuracy= 0.897600--, test accuracy= 0.797390\n",
      "epoch 280/300: \n",
      "train loss= 0.557010-- ,test loss= 0.762225--,train accuracy= 0.887600--, test accuracy= 0.797390\n",
      "epoch 280/300: \n",
      "train loss= 0.550500-- ,test loss= 0.762225--,train accuracy= 0.895000--, test accuracy= 0.797390\n",
      "epoch 290/300: \n",
      "train loss= 0.547249-- ,test loss= 0.762225--,train accuracy= 0.899600--, test accuracy= 0.797390\n",
      "epoch 290/300: \n",
      "train loss= 0.559899-- ,test loss= 0.762225--,train accuracy= 0.904600--, test accuracy= 0.797390\n",
      "epoch 290/300: \n",
      "train loss= 0.560469-- ,test loss= 0.762225--,train accuracy= 0.901600--, test accuracy= 0.797390\n",
      "epoch 290/300: \n",
      "train loss= 0.545377-- ,test loss= 0.762225--,train accuracy= 0.906000--, test accuracy= 0.797390\n",
      "epoch 290/300: \n",
      "train loss= 0.567252-- ,test loss= 0.762225--,train accuracy= 0.905000--, test accuracy= 0.797390\n"
     ]
    }
   ],
   "source": [
    "batch_size = 500\n",
    "step=Ntr/batch_size\n",
    "iterations = 300\n",
    "lr = 1.4e-2\n",
    "lr_decay=0.999\n",
    "reg =5e-6\n",
    "train_loss_history = []\n",
    "train_acc_history = []\n",
    "test_loss_history = []\n",
    "test_acc_history = []\n",
    "seed = 0\n",
    "rng = np.random.default_rng(seed=seed)\n",
    "indices=np.arange(0,step)\n",
    "for t in range(iterations):\n",
    "    rng.shuffle(indices)\n",
    "    for order in range(0,len(indices)):        \n",
    "        start=int(batch_size*indices[order])\n",
    "        x=x_train[start:start+batch_size,:]\n",
    "        y=y_train[start:start+batch_size,:]\n",
    "        \n",
    "\n",
    "        # Forward pass\n",
    "        z1=1.0/(1.0 + np.exp(-(x.dot(w1)+b1)))\n",
    "        y_pred=z1.dot(w2)+b2\n",
    "        z=1.0/(1.0 + np.exp(-(x_test.dot(w1)+b1)))\n",
    "        y_pred_test=z.dot(w2)+b2\n",
    "\n",
    "        #loss\n",
    "        loss=(1.0/batch_size)*np.square(y_pred-y).sum() + reg * (np.sum(w1*w1) + np.sum(w2*w2))\n",
    "        test_loss=(1.0/Nte)*np.square(y_pred_test-y_test).sum() + reg * (np.sum(w1*w1) + np.sum(w2*w2))\n",
    "        train_loss_history.append(loss)\n",
    "        test_loss_history.append(test_loss)\n",
    "\n",
    "        #accuracy\n",
    "        train_acc = 1.0 - (1/(batch_size*K))*(np.abs(np.argmax(y,axis=1) - np.argmax(y_pred,axis=1))).sum()\n",
    "        train_acc_history.append(train_acc)\n",
    "        test_acc = 1.0 - (1/(Nte*K))*(np.abs(np.argmax(y_test,axis=1) - np.argmax(y_pred_test,axis=1))).sum()\n",
    "        test_acc_history.append(test_acc)\n",
    "\n",
    "        if t%10==0 and order%20==0:\n",
    "            print('epoch %d/%d: ' % (t,iterations))\n",
    "            print('train loss= %f-- ,test loss= %f--,train accuracy= %f--, test accuracy= %f' % (loss,test_loss,train_acc,test_acc))\n",
    "\n",
    "                    \n",
    "        # Backward pass\n",
    "\n",
    "        dy_pred=(1./batch_size)*2.0*(y_pred-y)\n",
    "        dw2=z1.T.dot(dy_pred) + reg * w2\n",
    "        db2=dy_pred.sum(axis=0)\n",
    "        dz1=dy_pred.dot(w2.T)\n",
    "        dw1=x.T.dot(dz1*z1*(1-z1))+reg*w1\n",
    "        db1=(dz1*z1*(1-z1)).sum(axis=0)\n",
    "    \n",
    "        w1 -= lr*dw1\n",
    "        w2 -= lr*dw2\n",
    "        b1 -= lr*db1\n",
    "        b2 -= lr*db2\n",
    "        lr *= lr_decay\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}