MLP from scratch for XOR and Iris dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fbeb5dfc",
   "metadata": {},
   "source": [
    "# XOR Problem \n",
    "\n",
    "f(0, 0) => 0 \n",
    "\n",
    "f(1, 0) => 1\n",
    "\n",
    "f(0, 1) => 1\n",
    "\n",
    "f(1, 1) => 0 \n",
    "\n",
    "In fact, linear models *cannot* solve this problem!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "816b002c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch=0 and mse=0.2914809379598231\n",
      "epoch=100 and mse=0.25132493190245053\n",
      "epoch=200 and mse=0.2350917411445105\n",
      "epoch=300 and mse=0.1958778316906303\n",
      "epoch=400 and mse=0.14556776894980394\n",
      "epoch=500 and mse=0.09662363196609818\n",
      "epoch=600 and mse=0.05999120706179112\n",
      "epoch=700 and mse=0.03842152278682629\n",
      "epoch=800 and mse=0.02590619393376082\n",
      "epoch=900 and mse=0.018780635137008426\n",
      "[[0.1782055 ]\n",
      " [0.90234507]\n",
      " [0.90253518]\n",
      " [0.08097861]]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np \n",
    "\n",
    "np.random.seed(42)\n",
    "learning_rate = 0.1\n",
    "epochs = 1_000 \n",
    "\n",
    "def relu(z):\n",
    "    return np.maximum(z, 0)\n",
    "\n",
    "def relu_derivative(z):\n",
    "    return np.where(z > 0, 1, 0)\n",
    "\n",
    "def sigmoid(z):\n",
    "    return 1 / (1 + np.exp(-z))\n",
    "\n",
    "def sigmoid_derivative(z):\n",
    "    return z * (1 - z)\n",
    "\n",
    "\n",
    "X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])\n",
    "y = np.array([[0], [1], [1], [0]])\n",
    "\n",
    "input_layer_neurons = 2\n",
    "hidden_layer_neurons = 3 # NOTE: 3 is also valid, 2 seems invalid\n",
    "output_layer_neurons = 1\n",
    "\n",
    "hidden_weights = np.random.uniform(size=(input_layer_neurons, hidden_layer_neurons))\n",
    "hidden_bias = np.zeros((1, hidden_layer_neurons))\n",
    "output_weights = np.random.uniform(size=(hidden_layer_neurons, output_layer_neurons))\n",
    "output_bias = np.zeros((1, output_layer_neurons))\n",
    "\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    # feedforward process \n",
    "    hidden_layer_input = np.dot(X, hidden_weights) + hidden_bias\n",
    "    hidden_layer_activation = relu(hidden_layer_input)\n",
    "    output_layer_input = np.dot(hidden_layer_activation, output_weights) + output_bias\n",
    "    predicted_outputs = sigmoid(output_layer_input)\n",
    "    \n",
    "    # loss functions \n",
    "    error = y - predicted_outputs \n",
    "    mse = np.mean(np.square(error))\n",
    "    \n",
    "    # backpropagation \n",
    "    d_predicted_outputs = error * sigmoid_derivative(predicted_outputs)\n",
    "    \n",
    "    error_hidden_layer = d_predicted_outputs.dot(output_weights.T)\n",
    "    d_hidden_layer = error_hidden_layer * relu_derivative(hidden_layer_activation)\n",
    "    \n",
    "    output_weights = output_weights + learning_rate * hidden_layer_activation.T.dot(d_predicted_outputs)\n",
    "    output_bias = output_bias + learning_rate * np.sum(d_predicted_outputs, axis=0, keepdims=True)\n",
    "    \n",
    "    hidden_weights = hidden_weights + learning_rate * X.T.dot(d_hidden_layer)\n",
    "    hidden_bias = hidden_bias + learning_rate * np.sum(d_hidden_layer, axis=0, keepdims=True)\n",
    "    \n",
    "    if epoch % 100 == 0:\n",
    "        print(f\"{epoch=} and {mse=}\") # NOTE: epoch=100\n",
    "        \n",
    "    \n",
    "print(predicted_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f363ebba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0],\n",
       "       [1],\n",
       "       [1],\n",
       "       [0]])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddaa4d62",
   "metadata": {},
   "source": [
    "# Iris dataset \n",
    "\n",
    "Classification problem\n",
    "\n",
    "Number of labels (classes) = 3 \n",
    "Data is tabular (not image) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0e7d3301",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_iris \n",
    "from sklearn.model_selection import train_test_split \n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "361471b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "iris = load_iris()\n",
    "X = iris.data\n",
    "y = iris.target \n",
    "\n",
    "encoder = OneHotEncoder(sparse_output=False)\n",
    "y = encoder.fit_transform(y.reshape(-1, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fee21004",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e0d3df0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "learning_rate = 0.01\n",
    "epochs = 10_000\n",
    "\n",
    "def relu(z):\n",
    "    return np.maximum(z, 0)\n",
    "\n",
    "def relu_derivative(z):\n",
    "    return np.where(z > 0, 1, 0)\n",
    "\n",
    "def softmax(z):\n",
    "    exps = np.exp(z - np.max(z, axis=1, keepdims=True))\n",
    "    \n",
    "    return exps / np.sum(exps, axis=1, keepdims=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "09c8f086",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_layer_neurons = X_train.shape[1]\n",
    "hidden_layer_neurons = 5 # XXX: you can change it \n",
    "output_layer_neurons = y_train.shape[1]\n",
    "\n",
    "hidden_weights = np.random.uniform(size=(input_layer_neurons, hidden_layer_neurons))\n",
    "hidden_bias = np.random.uniform(size=(1, hidden_layer_neurons)) # NOTE: we can change it np.random.uniform()\n",
    "output_weights = np.random.uniform(size=(hidden_layer_neurons, output_layer_neurons))\n",
    "output_bias = np.zeros((1, output_layer_neurons)) # NOTE: we can change it np.random.uniform()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "65dd5777",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch=0 and loss=6.426498398422818\n",
      "epoch=1000 and loss=0.17483620451744405\n",
      "epoch=2000 and loss=0.10257819894920667\n",
      "epoch=3000 and loss=0.08403806062035032\n",
      "epoch=4000 and loss=0.07621786753883336\n",
      "epoch=5000 and loss=0.07200393936700504\n",
      "epoch=6000 and loss=0.06938077754030263\n",
      "epoch=7000 and loss=0.06758210297826528\n",
      "epoch=8000 and loss=0.0662553785847467\n",
      "epoch=9000 and loss=0.06522204988842592\n",
      "[[9.99635859e-01 3.64141430e-04 2.10633591e-19]\n",
      " [9.99919654e-01 8.03457343e-05 7.94089003e-23]\n",
      " [4.59913364e-04 9.99452150e-01 8.79363429e-05]\n",
      " [9.98814810e-01 1.18518998e-03 8.72076816e-18]\n",
      " [9.99079598e-01 9.20401546e-04 1.55798958e-17]\n",
      " [1.13944609e-08 1.27504619e-02 9.87249527e-01]\n",
      " [1.95041512e-04 9.99080219e-01 7.24739726e-04]\n",
      " [9.99595736e-01 4.04264309e-04 1.45404870e-19]\n",
      " [9.99704770e-01 2.95230459e-04 7.10890376e-20]\n",
      " [9.99858812e-01 1.41187963e-04 1.42272041e-21]\n",
      " [1.63282181e-09 2.82167685e-03 9.97178322e-01]\n",
      " [1.41317988e-04 9.97498743e-01 2.35993897e-03]\n",
      " [9.52550817e-05 9.98297581e-01 1.60716383e-03]\n",
      " [9.99836969e-01 1.63031047e-04 3.13416575e-21]\n",
      " [9.99798037e-01 2.01963147e-04 8.93962398e-21]\n",
      " [1.64333263e-03 9.98001284e-01 3.55383279e-04]\n",
      " [3.04137397e-06 5.03965701e-01 4.96031258e-01]\n",
      " [5.05571108e-08 5.53505823e-02 9.44649367e-01]\n",
      " [3.21566977e-04 9.99425226e-01 2.53206873e-04]\n",
      " [1.09439669e-10 2.35381096e-03 9.97646189e-01]\n",
      " [3.07539419e-04 9.97406118e-01 2.28634257e-03]\n",
      " [1.95252527e-12 1.85747744e-04 9.99814252e-01]\n",
      " [4.65112841e-05 9.31101300e-01 6.88521889e-02]\n",
      " [9.99692858e-01 3.07142226e-04 8.37786725e-20]\n",
      " [6.02122814e-13 8.52784831e-05 9.99914722e-01]\n",
      " [9.91552378e-04 9.98757372e-01 2.51075401e-04]\n",
      " [9.99650265e-01 3.49735229e-04 1.17671459e-19]\n",
      " [9.99659540e-01 3.40459568e-04 1.53325737e-19]\n",
      " [9.97757392e-01 2.24260819e-03 1.46756865e-17]\n",
      " [7.70667039e-04 9.90258368e-01 8.97096513e-03]\n",
      " [5.52651800e-07 1.71355188e-01 8.28644259e-01]\n",
      " [9.96135800e-01 3.86419985e-03 2.36525152e-16]\n",
      " [9.97408400e-01 2.59160043e-03 1.06637127e-16]\n",
      " [9.94776145e-01 5.22385537e-03 4.48044276e-16]\n",
      " [1.65819079e-04 9.87479880e-01 1.23543012e-02]\n",
      " [9.99421927e-01 5.78073108e-04 5.98708064e-19]\n",
      " [5.83329741e-04 9.98867468e-01 5.49202523e-04]\n",
      " [4.12191035e-10 1.04388416e-02 9.89561158e-01]\n",
      " [9.98899343e-01 1.10065652e-03 1.63564420e-17]\n",
      " [3.56009722e-04 9.99070581e-01 5.73408988e-04]\n",
      " [1.51351879e-10 4.77361438e-04 9.99522638e-01]\n",
      " [9.99897132e-01 1.02868448e-04 2.55151186e-22]\n",
      " [3.28800200e-06 4.48641311e-01 5.51355401e-01]\n",
      " [1.63282181e-09 2.82167685e-03 9.97178322e-01]\n",
      " [5.23136906e-04 9.98164246e-01 1.31261695e-03]\n",
      " [3.43863306e-05 8.54183042e-01 1.45782572e-01]\n",
      " [2.23620320e-09 6.75691969e-03 9.93243078e-01]\n",
      " [1.15299953e-04 9.43508865e-01 5.63758347e-02]\n",
      " [9.99425354e-01 5.74646452e-04 1.37824300e-19]\n",
      " [3.61370655e-03 9.96063515e-01 3.22778617e-04]\n",
      " [4.21751338e-09 8.40241477e-03 9.91597581e-01]\n",
      " [9.99442936e-01 5.57064443e-04 8.19922695e-19]\n",
      " [9.99557751e-01 4.42249030e-04 5.27187349e-19]\n",
      " [7.50330305e-04 9.97666317e-01 1.58335251e-03]\n",
      " [9.30366202e-06 8.90613282e-01 1.09377415e-01]\n",
      " [9.98483333e-01 1.51666659e-03 2.46866356e-17]\n",
      " [1.10764141e-11 1.04154326e-04 9.99895846e-01]\n",
      " [9.99356101e-01 6.43898503e-04 1.01313351e-18]\n",
      " [9.96485131e-01 3.51486870e-03 1.38392060e-16]\n",
      " [7.13625415e-06 6.58023623e-01 3.41969241e-01]\n",
      " [1.45481291e-02 9.85436400e-01 1.54709662e-05]\n",
      " [5.53378628e-10 4.23364458e-03 9.95766355e-01]\n",
      " [1.49801625e-07 6.49852706e-02 9.35014580e-01]\n",
      " [4.73259212e-09 7.83005601e-03 9.92169939e-01]\n",
      " [7.70418922e-11 1.98833503e-03 9.98011665e-01]\n",
      " [1.98103711e-04 9.83445280e-01 1.63566163e-02]\n",
      " [9.97114702e-01 2.88529758e-03 6.73855973e-16]\n",
      " [9.99121306e-01 8.78693918e-04 2.15028307e-17]\n",
      " [1.80896263e-08 1.25040000e-02 9.87495982e-01]\n",
      " [3.26165957e-08 8.44772237e-02 9.15522744e-01]\n",
      " [9.97904787e-01 2.09521294e-03 1.19577845e-16]\n",
      " [9.99552412e-01 4.47587907e-04 2.65502069e-19]\n",
      " [9.98483727e-01 1.51627336e-03 9.22079673e-17]\n",
      " [2.38601661e-06 3.79869854e-01 6.20127760e-01]\n",
      " [1.29694789e-10 1.14315023e-03 9.98856850e-01]\n",
      " [9.99014877e-01 9.85123390e-04 8.96327403e-18]\n",
      " [5.19948562e-09 1.52377384e-02 9.84762256e-01]\n",
      " [2.01065738e-13 1.62859733e-05 9.99983714e-01]\n",
      " [9.99333963e-01 6.66036660e-04 3.21539681e-18]\n",
      " [4.32913571e-05 9.68643567e-01 3.13131417e-02]\n",
      " [5.32573452e-05 9.87614463e-01 1.23322794e-02]\n",
      " [1.89394382e-06 3.43294652e-01 6.56703454e-01]\n",
      " [2.49711819e-04 9.99640198e-01 1.10090148e-04]\n",
      " [3.24554145e-09 8.77764611e-03 9.91222351e-01]\n",
      " [9.99684167e-01 3.15833010e-04 5.13530276e-20]\n",
      " [5.99743562e-08 8.28044156e-02 9.17195524e-01]\n",
      " [2.25108435e-04 9.97246632e-01 2.52825950e-03]\n",
      " [8.89495140e-08 7.47149631e-02 9.25284948e-01]\n",
      " [7.36417737e-03 9.92625439e-01 1.03835627e-05]\n",
      " [2.37440000e-04 9.82450057e-01 1.73125034e-02]\n",
      " [8.44440744e-05 9.91569138e-01 8.34641774e-03]\n",
      " [9.88567665e-01 1.14323352e-02 8.84636371e-14]\n",
      " [1.67655873e-04 9.98937001e-01 8.95343570e-04]\n",
      " [7.36729535e-05 9.36926574e-01 6.29997530e-02]\n",
      " [9.99779797e-01 2.20202803e-04 1.45116147e-20]\n",
      " [5.26686343e-04 9.98567835e-01 9.05478706e-04]\n",
      " [1.53459123e-10 2.46863789e-03 9.97531362e-01]\n",
      " [4.88924701e-09 1.66125062e-02 9.83387489e-01]\n",
      " [9.99405595e-01 5.94405040e-04 4.37416064e-19]\n",
      " [4.16301231e-03 9.95578400e-01 2.58587927e-04]\n",
      " [4.48168738e-11 7.21150917e-04 9.99278849e-01]\n",
      " [3.79302879e-07 3.37825172e-01 6.62174448e-01]\n",
      " [9.99696533e-01 3.03467418e-04 7.93606717e-20]\n",
      " [7.38340742e-11 8.94485903e-04 9.99105514e-01]\n",
      " [9.98687712e-01 1.31228842e-03 1.99810951e-17]\n",
      " [5.35957989e-05 9.97090995e-01 2.85540951e-03]\n",
      " [2.97060779e-10 3.68863380e-03 9.96311366e-01]\n",
      " [1.76648691e-09 5.27050504e-03 9.94729493e-01]\n",
      " [3.55544340e-04 9.97505495e-01 2.13896056e-03]\n",
      " [3.55490871e-08 4.48574703e-02 9.55142494e-01]\n",
      " [5.76632966e-05 9.65667615e-01 3.42747214e-02]\n",
      " [4.41867711e-04 9.99309628e-01 2.48504669e-04]\n",
      " [3.86093852e-09 4.31864406e-03 9.95681352e-01]\n",
      " [1.21739508e-07 6.04922886e-02 9.39507590e-01]\n",
      " [9.99079121e-01 9.20879282e-04 1.11673878e-18]\n",
      " [8.13712569e-04 9.98987817e-01 1.98470664e-04]\n",
      " [1.87971482e-08 6.08019101e-03 9.93919790e-01]\n",
      " [9.99899191e-01 1.00809074e-04 1.90180862e-22]\n",
      " [5.26486345e-04 9.98337652e-01 1.13586137e-03]\n",
      " [3.11904612e-10 3.20855540e-03 9.96791444e-01]]\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(epochs):\n",
    "    \n",
    "    # forward propagation \n",
    "    hidden_layer_input = np.dot(X_train, hidden_weights) + hidden_bias\n",
    "    hidden_layer_activation = relu(hidden_layer_input)\n",
    "    \n",
    "    output_layer_input = np.dot(hidden_layer_activation, output_weights) + output_bias\n",
    "    predicted_outputs = softmax(output_layer_input)\n",
    "    \n",
    "    # log loss \n",
    "    \n",
    "    loss = - np.mean(np.sum(y_train * np.log(predicted_outputs + 1e-8), axis=1))\n",
    "    error = y_train - predicted_outputs\n",
    "    \n",
    "    \n",
    "    # backpropagation \n",
    "    d_output_weights = hidden_layer_activation.T.dot(error) / X_train.shape[0]\n",
    "    d_output_bias = np.sum(error, axis=0, keepdims=True) / X_train.shape[0]\n",
    "    \n",
    "    error_hidden_layer = error.dot(output_weights.T) * relu_derivative(hidden_layer_activation)\n",
    "    d_hidden_weights = X_train.T.dot(error_hidden_layer) / X_train.shape[0]\n",
    "    d_hidden_bias = np.sum(error_hidden_layer, axis=0, keepdims=True) / X_train.shape[0]\n",
    "    \n",
    "    output_weights += learning_rate * d_output_weights\n",
    "    output_bias += learning_rate * d_output_bias\n",
    "    \n",
    "    hidden_weights += learning_rate * d_hidden_weights\n",
    "    hidden_bias += learning_rate * d_hidden_bias\n",
    "    \n",
    "    if epoch % 1000 == 0:\n",
    "        print(f\"{epoch=} and {loss=}\")\n",
    "\n",
    "print(predicted_outputs)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "6f3268a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden_layer_input = np.dot(X_test, hidden_weights) + hidden_bias\n",
    "hidden_layer_activation = relu(hidden_layer_input)\n",
    "\n",
    "output_layer_input = np.dot(hidden_layer_activation, output_weights) + output_bias\n",
    "predicted_test_outputs = softmax(output_layer_input)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "a7b39d66",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_hat = np.argmax(predicted_test_outputs, axis=1)\n",
    "y_test_argmax = np.argmax(y_test, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "0887278f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0, 2, 1, 1, 0, 1, 2, 2, 1, 2, 0, 0, 0, 0, 1, 2, 1, 1, 2, 0, 2,\n",
       "       0, 2, 2, 2, 2, 2, 0, 0])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_hat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "2656f2b3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0, 2, 1, 1, 0, 1, 2, 1, 1, 2, 0, 0, 0, 0, 1, 2, 1, 1, 2, 0, 2,\n",
       "       0, 2, 2, 2, 2, 2, 0, 0])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_argmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "f816bc1a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9666666666666667"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_test_argmax, y_hat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba76e45f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}