lecture_9_svm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Basic setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Create anaconda environment\n",
    "<br>\n",
    "```bash\n",
    "conda create -n ml python=3.7.4 jupyter\n",
    "```\n",
    "Install fastai library\n",
    "<br>\n",
    "```bash\n",
    "conda install -c pytorch -c fastai fastai\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Supported Vector Machine (SVM)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Large Margin Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "From logistic regression we know that \n",
    "$$f(x, W) = \\frac{1}{1 + e^{-W^TX}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "def sigm(x):\n",
    "    return 1 / (1 + np.exp(-x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "x = np.linspace(-6, 6, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11ddc9810>]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAcj0lEQVR4nO3deXxV1b3+8c9XlEEGBYNAAUUrihNWjKiX+7PW6SIq9FoHnECFS1WwtuIAoqJQFAd+igJVqhRxQsUprVhLHShXBYkDKCCKCBIRGaSMSsB87x8rSAyBnCQnWWd43q/XfuUMm5znaPKwWGfvtc3dERGR9LdL7AAiIpIcKnQRkQyhQhcRyRAqdBGRDKFCFxHJELvGeuGcnBxv06ZNrJcXEUlL77333kp3b1rWc9EKvU2bNuTn58d6eRGRtGRmi3f0nKZcREQyhApdRCRDqNBFRDKECl1EJEOo0EVEMkS5hW5m48xsuZl9vIPnzczuN7MFZjbbzDokP6aIiJQnkRH6eKDzTp4/DWhbvPUB/lT1WCIiUlHlHofu7v8yszY72aUbMMHDOrzTzWxPM2vh7l8nKaOISHTu8P33sHEjfPdduL316/ffw6ZN274WFoavmzeH21u3rffPPBOOPjr5GZNxYlFLYEmJ+wXFj21X6GbWhzCKZ5999knCS4uI7Jw7rF8Pq1bBt9+GbfXqsK1ZA//+N6xdu21bv/6n24YNYdu4MXmZWrZM3UK3Mh4r86oZ7j4WGAuQm5urK2uISKVt3gxLl0JBQdiWLoVly+Drr2H5cvjmm/B15cowKt6RXXaBRo22bQ0bhq8/+xnUr79t2333bVu9emGrWzds9epBnTo/3Xbb7ae3a9cOX2vVAiurNZMgGYVeALQucb8VsDQJ31dEstzKlTB/PixYAJ9/DgsXwqJFYVu6NIy+S6pdG5o3D1urVnDkkdC0KeTkwF57ha1JE2jcGPbcM2z161dfwda0ZBR6HtDPzCYCxwBrNH8uIhWxejXMmgUffRS2OXPgk0/C9MhWu+wCrVvDfvvBKafAvvuG+61ahSmMn/0sFHWmlHNllFvoZvYUcAKQY2YFwGBgNwB3fxCYDHQBFgAbgUurK6yIpL+NG+G99+Cdd2DmzHD7iy+2Pd+kCRx2GJxzDrRrBwceCAccAG3ahBG47FgiR7mcX87zDvRNWiIRyShr1sC0aTB1atjefx9++CE8t//+kJsLffqE6ZHDD4cWLbJ7lF0V0ZbPFZHMVFQURt2vvAL/+AdMnx4KvHZtOOYYuOEGOO64cLtpmat6S2Wp0EWkyjZvhtdfhxdegL/+NXxgaQZHHQUDBsDJJ4cCr1cvdtLMpkIXkUpxh//9X3jySZg0KRyRUr8+/Nd/Qbdu0KVLOLpEao4KXUQqZOlSePRRGDcuHE5Yrx507Qrdu0PnzuG4bIlDhS4i5do6Gn/gAXj++TAn/stfws03w1lnQYMGsRMKqNBFZCd++CFMp9x5J3zwQTjO+5prwlEpBxwQO52UpkIXke1s3gwTJsDw4WFapV07GDsWLrwwnPouqUmFLiI/KiqCp5+GW24JRd6hAzz3HPz61+FMTUltKnQRAeCNN8J0yocfhhN88vLgjDN0kk860d+5Illu8eJwmv2JJ4Y1VR5/PJT6mWeqzNONCl0kS23ZEj7sbNcOXn4ZhgyBefPCPLmmV9KTplxEstCHH0KvXmFdlV//GkaOBF1zJv3p72GRLLJlSxiJ5+bCV1+FQxJfeEFlnik0QhfJEosWwUUXwVtvwQUXhJOEmjSJnUqSSYUukgUmTQpTLBA+9Lzwwrh5pHpoykUkg23ZAtddF45iOfjgMHeuMs9cGqGLZKjly+G88+DNN+HKK+Hee3XFn0ynQhfJQHPnwumnw7JlYWXEHj1iJ5KaoEIXyTCvvx5WQKxbF/71Lzj66NiJpKZoDl0kgzz2WLjARKtWMGOGyjzbqNBFMsSoUWFq5fjjw6GJ++4bO5HUNBW6SJpzh2HD4KqrwlmfL78Me+wRO5XEoEIXSWPucOONcNNNcPHF8OyzugRcNlOhi6SxW24JF6H47W9h/HjYVYc5ZDUVukiaGjoU/vhH6N0bxozRComiQhdJS3ffHUbnPXvCQw+pzCXQj4FImhk/Hq6/PpwF+sgjKnPZRj8KImlk8uQwxXLyyeEizrVqxU4kqUSFLpImZswIi2wdcQQ8/7zWZZHtqdBF0sCiReEan82bh1F6w4axE0kqSqjQzayzmc03swVmNqCM5/cxszfM7AMzm21mXZIfVSQ7rVsHXbtCYWEo82bNYieSVFVuoZtZLWA0cBpwCHC+mR1SarebgGfc/UigOzAm2UFFslFRUbjK0Ny54aShgw6KnUhSWSIj9I7AAndf6O6FwESgW6l9HGhUfHsPYGnyIopkr0GDIC8P7rsPTjkldhpJdYkUektgSYn7BcWPlXQrcJGZFQCTgavK+kZm1sfM8s0sf8WKFZWIK5I9nntu21mgffvGTiPpIJFCtzIe81L3zwfGu3sroAvwmJlt973dfay757p7btOmTSueViRLzJ8Pl14KxxwD998PVtZvoUgpiRR6AdC6xP1WbD+l0gt4BsDd3wHqAjnJCCiSbTZsgN/8BurUCfPmOjxREpVIoc8E2prZfmZWm/ChZ16pfb4ETgIws4MJha45FZEKcg9TLHPnwlNPQevW5f8Zka3KLXR33wL0A14F5hGOZpljZkPMrGvxbv2B/zGzWcBTwCXuXnpaRkTKMX48PPEEDBkSzgYVqQiL1bu5ubmen58f5bVFUtGnn0KHDtCxI0yZotP6pWxm9p6755b1nM4UFUkBmzbB+eeHefPHHlOZS+VoOXyRFDBoELz/Prz4IrQsfVCwSII0QheJ7I03YMQIuPxy6Fb6lD2RClChi0S0dm043rxt21DqIlWhKReRiPr3hyVLYNo02H332Gkk3WmELhLJK6/Aww/DtdfCf/xH7DSSCVToIhGsXh2uPHTooXDbbbHTSKbQlItIBNdfD8uWwUsvQd26sdNIptAIXaSGvflmmGrp3x9yyzw9RKRyVOgiNei776BPH9h/f7j11thpJNNoykWkBg0dCp99Bv/8p45qkeTTCF2khsyeDXfdBZddBiedFDuNZCIVukgNKCqCK66Axo3h7rtjp5FMpSkXkRrw6KPw9tvwl79Akyax00im0ghdpJqtWgXXXQedOkGPHrHTSCZToYtUsxtvhH//G8aMgV30GyfVSD9eItXo3Xfhz3+Gq6+G9u1jp5FMp0IXqSZFRXDVVdCsGQweHDuNZAN9KCpSTR5/PIzQx4+HRo1ip5FsoBG6SDVYtw5uuCFcH/Tii2OnkWyhEbpINRg2LCy+9eKL+iBUao5+1ESS7PPP4d57oWdPOOaY2Gkkm6jQRZLs+utht93gjjtiJ5Fso0IXSaJp0+D552HAAGjRInYayTYqdJEkKSoKa5y3bAnXXBM7jWQjfSgqkiRPPQUzZ4Z1W7Q0rsSgEbpIEnz3HQwcCB06wEUXxU4j2UojdJEkGDkSliyBCRN0mKLEox89kSpatQqGD4czzoATToidRrKZCl2kim6/PZwZqsMUJTYVukgVLFoEo0aFk4gOOyx2Gsl2CRW6mXU2s/lmtsDMBuxgn3PNbK6ZzTGzJ5MbUyQ13XJLmDO/7bbYSUQS+FDUzGoBo4FTgAJgppnlufvcEvu0BQYCndx9tZntXV2BRVLFrFlhRcXrroPWrWOnEUlshN4RWODuC929EJgIdCu1z/8Ao919NYC7L09uTJHUM3Ag7LlnOCtUJBUkUugtgSUl7hcUP1bSgcCBZvaWmU03s85lfSMz62Nm+WaWv2LFisolFkkB06bBK6+EMm/cOHYakSCRQrcyHvNS93cF2gInAOcDD5vZntv9Ifex7p7r7rlNmzataFaRlOAeRuctWkC/frHTiGyTyIlFBUDJGcJWwNIy9pnu7puBL8xsPqHgZyYlpUgKmTwZ3noL/vQnneIvqSWREfpMoK2Z7WdmtYHuQF6pfV4EfgVgZjmEKZiFyQwqkgqKiuDGG+HnP4devWKnEfmpckfo7r7FzPoBrwK1gHHuPsfMhgD57p5X/NypZjYX+AG4zt1XVWdwkRgmToTZs+HJJ8Oa5yKpxNxLT4fXjNzcXM/Pz4/y2iKVsXkzHHJImGb54AOt2SJxmNl77p5b1nNanEskQY8+CgsWQF6eylxSk34sRRKwaRMMGQIdO4ZFuERSkUboIgkYOzYsjztuHFhZB/KKpACN0EXKsWEDDBsWlsY96aTYaUR2TCN0kXKMHg3ffAPPPafRuaQ2jdBFdmLtWrjzTujcGTp1ip1GZOdU6CI7MXIkfPstDB0aO4lI+VToIjuwejWMGAHdukFumUf9iqQWFbrIDowYAWvWhMMVRdKBCl2kDCtWhOmWc8+F9u1jpxFJjApdpAx33QUbN8Ktt8ZOIpI4FbpIKcuWhUMVL7gADj44dhqRxKnQRUoZPhwKC2Hw4NhJRCpGhS5SQkEBPPgg9OwJBxwQO41IxajQRUq4/fZwEYubb46dRKTiVOgixRYvhocfDlciatMmdhqRilOhixQbOjSs1XLjjbGTiFSOCl2EcOGK8ePht7+F1q3L3V0kJanQRQhng+62GwwcGDuJSOWp0CXrzZsHTzwB/fpBixax04hUngpdst6tt0K9enD99bGTiFSNCl2y2uzZ8MwzcPXV0LRp7DQiVaNCl6w2eDA0agT9+8dOIlJ1KnTJWvn58OKLcM010KRJ7DQiVadCl6x1002w117whz/ETiKSHLpItGSladPg1VfDMrmNGsVOI5IcGqFL1nGHQYOgeXPo2zd2GpHk0Qhdss6UKWGEPmoU7L577DQiyaMRumQV9zB3vs8+0Lt37DQiyaURumSVF1+EmTPhkUegTp3YaUSSSyN0yRo//BDmztu1gx49YqcRSb6ECt3MOpvZfDNbYGYDdrLf2WbmZpabvIgiyfHYY2Hdlj/+EXbVv00lA5Vb6GZWCxgNnAYcApxvZoeUsV9D4HfAjGSHFKmqTZvCWaG5uXDWWbHTiFSPREboHYEF7r7Q3QuBiUC3MvYbCtwFfJ/EfCJJ8dBD8OWX4RJzZrHTiFSPRAq9JbCkxP2C4sd+ZGZHAq3d/W87+0Zm1sfM8s0sf8WKFRUOK1IZ69bBsGFw4olw8smx04hUn0QKvazxjP/4pNkuwL1AucsbuftYd89199ymWtpOasiIEbB8Odxxh0bnktkSKfQCoORFuVoBS0vcbwgcBrxpZouAY4E8fTAqqeCbb+Cee+Ccc6Bjx9hpRKpXIoU+E2hrZvuZWW2gO5C39Ul3X+PuOe7ext3bANOBru6eXy2JRSpgyJDwgeiwYbGTiFS/cgvd3bcA/YBXgXnAM+4+x8yGmFnX6g4oUlmffQZjx0KfPtC2bew0ItUvoaNx3X0yMLnUY7fsYN8Tqh5LpOoGDQpng95S5k+qSObRmaKSkWbMgGefDVciatYsdhqRmqFCl4zjHq5C1Lw5XHdd7DQiNUcnQEvGmTQJ3n4b/vxnaNAgdhqRmqMRumSUTZvghhvg8MPh0ktjpxGpWRqhS0YZNQq++CJcXq5WrdhpRGqWRuiSMVatCispdu4Mp54aO41IzVOhS8a4+eawbss998ROIhKHCl0ywqxZYUXFK6+EQw+NnUYkDhW6pD13uPpqaNwYbrstdhqRePShqKS9SZNg6lQYMyaUuki20ghd0tp338G110L79mHNFpFsphG6pLU77ghXInr0UR2mKKIRuqStTz+FO++ECy+EE06InUYkPhW6pCV36NcP6tbVYYoiW2nKRdLSpEkwZQrcf39YhEtENEKXNLRuHfz+93DkkXDFFbHTiKQOjdAl7dx0E3z9NTz3HOyqn2CRH2mELmll+nR44AHo2xeOPTZ2GpHUokKXtFFYCL17Q8uWcPvtsdOIpB79g1XSxvDhMGcO/PWv0LBh7DQiqUcjdEkLc+fCsGHQvTuccUbsNCKpSYUuKW/LFrjkknA5ufvui51GJHVpykVS3p13wsyZ8PTT0KxZ7DQiqUsjdElps2aFJXHPOw/OPTd2GpHUpkKXlFVYCD16QJMmMHp07DQiqU9TLpKyBg+G2bPhpZdgr71ipxFJfRqhS0p6/fUwd967N3TtGjuNSHpQoUvKWbkSLr4YDjpIR7WIVISmXCSluEOvXqHUX34Z6tePnUgkfajQJaWMGQN5eXDvvfCLX8ROI5JeNOUiKePdd+EPf4DTT4err46dRiT9JFToZtbZzOab2QIzG1DG89eY2Vwzm21mr5nZvsmPKpls5Uo4++yw8NaECWAWO5FI+im30M2sFjAaOA04BDjfzA4ptdsHQK67twcmAXclO6hkrh9+gAsugOXLwxrnTZrETiSSnhIZoXcEFrj7QncvBCYC3Uru4O5vuPvG4rvTgVbJjSmZbPDgcDm50aOhQ4fYaUTSVyKF3hJYUuJ+QfFjO9ILeKWsJ8ysj5nlm1n+ihUrEk8pGeupp8Iqir17h6NbRKTyEin0smYzvcwdzS4CcoG7y3re3ce6e6675zZt2jTxlJKR3n0XLr0Ujj9ep/aLJEMihy0WAK1L3G8FLC29k5mdDAwCfunum5ITTzJVQQF06wYtWoR589q1YycSSX+JjNBnAm3NbD8zqw10B/JK7mBmRwIPAV3dfXnyY0omWbsWzjwT1q8PVx/KyYmdSCQzlDtCd/ctZtYPeBWoBYxz9zlmNgTId/c8whRLA+BZC8ebfenuWoFDtrNpE/z3f8PHH4cyP+yw2IlEMkdCZ4q6+2RgcqnHbilx++Qk55IMVFQEPXuGhbcmTIDOnWMnEsksOlNUaoR7OAv06afhrrvC4lsiklwqdKl27jBgANx/fyj1a6+NnUgkM6nQpdoNHhxG5VdcASNG6LR+keqiQpdqNXRo2Hr1glGjVOYi1UnL50q1cIeBA8NVh3r0gLFjYRcNH0SqlQpdkq6oCPr2hQcfhMsvD2eBqsxFqp9+zSSpNm0KR7A8+CDccEO4YIXKXKRmaIQuSfPtt3DWWTB1KtxxRziyRURqjgpdkuLzz6FLF1i0CJ54IqxvLiI1S4UuVfbaa3DeeeGD0Ndeg//8z9iJRLKTZjel0txh+HA49VRo1gymT1eZi8SkEbpUyurV4djyF16Ac8+FRx6BBg1ipxLJbhqhS4VNnQpHHBFWSxwxAiZOVJmLpAIVuiSssBAGDYJf/Qrq1IG334ZrrtHZnyKpQlMukpAZM8IUy5w5cNllMHKkRuUiqUYjdNmpdevCKPy442DNGvjb3zRfLpKqVOhSJnd4/HE46CC4995wCv+cOXD66bGTiciOqNBlO2+/DZ06hVP4W7UKhyOOGQONGsVOJiI7o0KXH82eHS7e3KkTLFwI48aFMj/mmNjJRCQRKnTh/ffhnHPgF7+AadNg2LBwKv+ll2phLZF0oqNcspR7uFjzPffA3/8eplMGDoT+/aFJk9jpRKQyVOhZZv368GHnAw/A3LnQtCncfjtceSXssUfsdCJSFSr0LOAePugcNw6efho2bICjjoLx48OiWnXrxk4oIsmgQs9gH30UTsufODF8yNmgAXTvHk4QOvZYneEpkmlU6BmkqCgclfLSS2GbPz98qHnSSXDzzXD22TohSCSTqdDT3FdfwZQp8Oqr4euqVbDrrnDCCfC734US33vv2ClFpCao0NNIURF89hm88044vHDq1HB4IYT1yLt0gdNOC9uee8bNKiI1T4WeooqKYPFi+OADeO+9sL37bliHHKBxYzj+eOjbF048Edq315y4SLZToUe2ZUso7vnz4ZNPYN48+PjjsK1fH/apVQsOPRR+85uwSNaxx0K7djrpR0R+SoVezQoLYelSWLIkbIsXhwspL1oUpksWLw6lvlVOTijvSy6Bww8PZ2+2b69DC0WkfAkVupl1BkYCtYCH3X14qefrABOAo4BVwHnuvii5UVPD5s1h2uPbb8O2cmX4IHLFCli+PGzLloXt66/D86Xl5MC++0JubjgO/Oc/DyPugw6Cvfaq+fckIpmh3EI3s1rAaOAUoACYaWZ57j63xG69gNXufoCZdQfuBM6rjsDlcQ+lu2lT2L7/Pnz97rvtt40bw7ZhQ9jWr9+2rV0b1gJfuzasA75127Bhx69dr174cHLvvWH//cMiV82bQ+vWYdXCVq1CkdevX3P/PUQkeyQyQu8ILHD3hQBmNhHoBpQs9G7ArcW3JwGjzMzc3ZOYFQhnO959d5jK2Lz5p1+3bpVVqxY0bBgKd489wu1GjaBly3DUyB57hK9NmoStceMw2s7JCSPrBg30waSIxJNIobcElpS4XwCUXlD1x33cfYuZrQH2An4y4WBmfYA+APvss0+lAufkhDnl3XbbttWpA7Vr//R2nTph3rlu3XC7Xr1t93ffPdyvVy+U99atdm0Vsoikr0QKvayKKz3yTmQf3H0sMBYgNze3UqP3rl3DJiIiP5XIgW8FQOsS91sBS3e0j5ntCuwBfJuMgCIikphECn0m0NbM9jOz2kB3IK/UPnlAz+LbZwOvV8f8uYiI7Fi5Uy7Fc+L9gFcJhy2Oc/c5ZjYEyHf3POAR4DEzW0AYmXevztAiIrK9hI5Dd/fJwORSj91S4vb3wDnJjSYiIhWhk8dFRDKECl1EJEOo0EVEMoQKXUQkQ1isowvNbAWwuJJ/PIdSZ6GmMb2X1JMp7wP0XlJVVd7Lvu7etKwnohV6VZhZvrvnxs6RDHovqSdT3gfovaSq6novmnIREckQKnQRkQyRroU+NnaAJNJ7ST2Z8j5A7yVVVct7Scs5dBER2V66jtBFRKQUFbqISIZI60I3s6vMbL6ZzTGzu2LnqSozu9bM3MxyYmepDDO728w+MbPZZvaCme0ZO1NFmVnn4p+pBWY2IHaeyjKz1mb2hpnNK/79uDp2pqows1pm9oGZ/S12lqowsz3NbFLx78k8Mzsumd8/bQvdzH5FuJZpe3c/FLgncqQqMbPWhAtxfxk7SxVMAQ5z9/bAp8DAyHkqpMQF0U8DDgHON7ND4qaqtC1Af3c/GDgW6JvG7wXgamBe7BBJMBL4u7u3A44gye8pbQsduAIY7u6bANx9eeQ8VXUvcD1lXLovXbj7P9x9S/Hd6YSrW6WTHy+I7u6FwNYLoqcdd//a3d8vvr2OUBwt46aqHDNrBZwOPBw7S1WYWSPgeML1I3D3Qnf/dzJfI50L/UDg/5nZDDObamZHxw5UWWbWFfjK3WfFzpJElwGvxA5RQWVdED0tS7AkM2sDHAnMiJuk0u4jDHaKYgepov2BFcBfiqePHjaz+sl8gYQucBGLmf0TaF7GU4MI2RsT/jl5NPCMme2fqpe+K+e93AicWrOJKmdn78PdXyreZxDhn/xP1GS2JEjoYufpxMwaAM8Bv3f3tbHzVJSZnQEsd/f3zOyE2HmqaFegA3CVu88ws5HAAODmZL5AynL3k3f0nJldATxfXODvmlkRYcGbFTWVryJ29F7M7HBgP2CWmUGYpnjfzDq6+7IajJiQnf0/ATCznsAZwEmp+pfrTiRyQfS0YWa7Ecr8CXd/PnaeSuoEdDWzLkBdoJGZPe7uF0XOVRkFQIG7b/2X0iRCoSdNOk+5vAicCGBmBwK1ScOV2Nz9I3ff293buHsbwv/0DqlY5uUxs87ADUBXd98YO08lJHJB9LRgYXTwCDDP3f9/7DyV5e4D3b1V8e9Gd8IF6NOxzCn+nV5iZgcVP3QSMDeZr5HSI/RyjAPGmdnHQCHQMw1HhJlmFFAHmFL8r43p7n553EiJ29EF0SPHqqxOwMXAR2b2YfFjNxZfH1jiuQp4onjAsBC4NJnfXKf+i4hkiHSechERkRJU6CIiGUKFLiKSIVToIiIZQoUuIpIhVOgiIhlChS4ikiH+DztBK7yAdqbrAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(x, sigm(x),'b', label='linspace(-10,10,100)')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "if $y = 1$ then our goal is $f(x, W) \\approx 1$ and by the nature the logistic function $W^Tx >> 0$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "On the other heand if we have $y = 0$ then $f(x, W) \\approx 0$ and again by the nature of logistic function $W^Tx << 0$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Cost function for logistic regression for the single training example:\n",
    "$$-(y\\log f(x, W) + (1 - y)\\log(1 - f(x, W)))$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Or if we put $f(x, W)$:\n",
    "$$-y\\log(\\frac{1}{1 + e^{-W^TX}}) - (1 - y)\\log(1 - \\frac{1}{1 + e^{-W^TX}})$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Now if $y = 1$ then only matters:\n",
    "$$-y\\log(\\frac{1}{1 + e^{-W^TX}})$$ part of equasion ($(1 - y) = 0$ and second part will be zero)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Denote $Z = W^Tx$ then we have \n",
    "$$\\frac{1}{1 + e^{-Z}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "Z = np.linspace(-6, 6, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "def left_sig(x):\n",
    "    return -np.log((1 / (1 + np.exp(-x))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11df29bd0>]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAazklEQVR4nO3deXiU9b3+8fcnCft+JKKyCF6IGyBqRD0cqeCCVkSOdQFxqVo4Ilrl4G5t/elRRH6iaC1KAbXKcQO3WksFj3sFDYhVRD1qq6IoERXBhfVz/vgmDWogk2Rmvs/M3K/req5MkmFyP7XcPHnmu5i7IyIiyVUUO4CIiGydilpEJOFU1CIiCaeiFhFJOBW1iEjClWTiRdu3b+9du3bNxEuLiOSlhQsXfubupTV9LyNF3bVrV8rLyzPx0iIiecnM3t/S93TrQ0Qk4VTUIiIJp6IWEUk4FbWISMKpqEVEEi6lojaztmY2y8zeNLOlZnZApoOJiEiQ6vC8ycAcdz/WzBoDzTOYSURENlPrFbWZtQb6A9MB3H2du3+Z7iCbNsE118DChel+ZRGR3JbKrY+dgArgdjN7xcymmVmLHz7JzEaZWbmZlVdUVNQ5yKpVcOutcOyx8Pnndf7jIiJ5K5WiLgH2Bqa4+17A18DFP3ySu0919zJ3LystrXEW5Fa1awezZsFHH8HJJ4crbBERSa2olwHL3H1B5eezCMWddn37wo03wuOPw/jxmfgJIiK5p9aidvdPgA/NbJfKLx0MvJGpQKNHw4knwuWXw7x5mfopIiK5I9Vx1OcAM83sb0Af4JpMBTKDqVNh991h+HBYtixTP0lEJDekVNTuvrjy/nNvdx/q7l9kMlSLFjB7Nnz3HRx3HKxbl8mfJiKSbImdmbjLLnD77TB/PlxwQew0IiLxJLaoIQzVGzsWbroJ7r03dhoRkTgSXdQAEyZAv37wi1/AGxl7C1NEJLkSX9SNGsF994X71j/7GaxeHTuRiEh2Jb6oATp2DLc+3n4bRo4E99iJRESyJyeKGmDAALj66nB1fdNNsdOIiGRPzhQ1wIUXwpAhcP758Ne/xk4jIpIdOVXURUVw553QpUsYX71iRexEIiKZl1NFDdC2bZgM8/nnMGwYbNgQO5GISGblXFED9OkDU6bAU0/Br38dO42ISGblZFED/PznYQTI+PHwxz/GTiMikjk5W9QQRn/svXdYv/q992KnERHJjJwu6qZNw2YDRUVhMsy338ZOJCKSfjld1ADdusFdd8HixTBmTOw0IiLpl/NFDXDkkfCrX4XV9qZPj51GRCS98qKoAa64Ag49NFxVL1oUO42ISPrkTVEXF8N//zdsu224X62dzEUkX+RNUQO0bw8PPKCdzEUkv+RVUQPst1/1TubXZGxnRxGR7Mm7ooawk/mIEWHW4ty5sdOIiDRMXha1Gdx2G+yxR9jJ/IMPYicSEam/vCxqqN7JfN26sNLe2rWxE4mI1E/eFjVAjx5wxx3w0kswblzsNCIi9ZPXRQ1wzDFho4FbboGZM2OnERGpu5JUnmRm/wBWAxuBDe5elslQ6TZ+fLiqHjkSeveGXr1iJxIRSV1drqgHuHufXCtpgJKSsNdimzZhMsyqVbETiYikLu9vfVTZbju4//6wHOrpp2sncxHJHakWtQNPmNlCMxtV0xPMbJSZlZtZeUVFRfoSptGBB8LEifDgg3D99bHTiIikxjyFS0sz28HdPzazbYG5wDnu/uyWnl9WVubl5eVpjJk+7nD88fDQQ/Dkk/CTn8ROJCICZrZwS7eWU7qidvePKz+uAB4C+qYvXnaZhaVQu3eHE06Ajz+OnUhEZOtqLWoza2FmraoeA4cBr2c6WCa1bh0mw6xZE66u16+PnUhEZMtSuaLuADxvZq8CLwF/cvc5mY2VeXvsAdOmwQsvwEUXxU4jIrJltY6jdvf3gD2zkCXrhg2DF1+EG26A/fcPV9ciIklTMMPztmTiRDjggDBkb+nS2GlERH6s4Iu6ceOw2UCLFmG6+erVsROJiHxfwRc1QMeOcO+98Pbb8ItfaDKMiCSLirrSgAFhR5j774fJk2OnERGppqLezIUXwtChcMEF8PzzsdOIiAQq6s2YhfWru3ULI0A++SR2IhERFfWPtGkTJsN8+WWYuajJMCISm4q6Br16we9/D88+C5dcEjuNiBQ6FfUWjBgBY8aEVfYeeCB2GhEpZCrqrZg0KcxY1GQYEYlJRb0VVZNhmjULO8OsWRM7kYgUIhV1LTp1CpNh3noLzjhDk2FEJPtU1CkYOLB6MsyNN8ZOIyKFRkWdogsvhH//9zAZ5rnnYqcRkUKiok6RGdx+O+y0U5gMs3x57EQiUihU1HXQpk3YGPerr7QzjIhkj4q6jnr2DDvDPP98uA0iIpJpKup6GD4cfvnLsMrevffGTiMi+U5FXU8TJ0K/fmHI3pIlsdOISD5TUddT48ZhuF6rVmFnmK++ip1IRPKViroBdtghlPW778Kpp2oyjIhkhoq6gfr3D7dBHn4YrrsudhoRyUcq6jQ477ywdvWll8KTT8ZOIyL5RkWdBmZhyN6uu8KwYfDhh7ETiUg+SbmozazYzF4xs8cyGShXtWwZJsOsXRtW2lu7NnYiEckXdbmiPhfQqsxbscsu8Ic/wMsvh3HWIiLpkFJRm1kn4EhgWmbj5L6hQ8P2XVOnwowZsdOISD5I9Yr6RuBCYNOWnmBmo8ys3MzKKyoq0hIuV111FRxyCJx1FpSXx04jIrmu1qI2s8HACndfuLXnuftUdy9z97LS0tK0BcxFxcVwzz3QoUO4X/3ZZ7ETiUguS+WKuh8wxMz+AdwLDDSzuzOaKg+0bw+zZ8Onn4a1QTZujJ1IRHJVrUXt7pe4eyd37woMA/7H3U/KeLI8UFYGt9wC8+bB5ZfHTiMiuUrjqDPsjDNg5EgYPz7MXhQRqas6FbW7P+3ugzMVJl/dfDPsuy+cckrYJFdEpC50RZ0FTZqE+9VNm4Z9F1evjp1IRHKJijpLOncOmwy89RacfrpW2hOR1Kmos2jgQJgwAWbNCivuiYikQkWdZePGwXHHhdmLWmlPRFKhos4yszC1fNddw9Ko778fO5GIJJ2KOoKWLeGhh2D9+jBz8bvvYicSkSRTUUfSowfcdRcsXBjWBNGbiyKyJSrqiIYMCTMWb78dbr01dhoRSSoVdWS/+Q0ccQScey789a+x04hIEqmoIysuhpkzoUsXOPZYWL48diIRSRoVdQK0axfWAVm1KgzdW7cudiIRSRIVdUL07BnuVb/wAowdGzuNiCRJSewAUu3448OOMBMnhiVSTzstdiIRSQJdUSfMNdeEbbxGjw6b5IqIqKgTpqQkLN603XZwzDGwYkXsRCISm4o6gbbZJsxc/Oyz8Obi+vWxE4lITCrqhNprL5g2DZ59Fs4/P3YaEYlJbyYm2IgRYYr5DTfAPvuEHWJEpPDoijrhrrsOBgyAUaPCiBARKTwq6oQrKYH77oMOHcI2Xp9+GjuRiGSbijoHlJbqzUWRQqaizhF77x3eXHzuOc1cFCk0ejMxh4wYAYsWwaRJobhPPz12IhHJBl1R55gJE6pnLs6fHzuNiGRDrUVtZk3N7CUze9XMlpjZ/8tGMKlZ1czFjh3DzEUtiyqS/1K5ol4LDHT3PYE+wOFmtn9mY8nWbLNN9bKoP/sZrF0bO5GIZFKtRe3BmspPG1Ue2uEvst694c474cUXYcwY7bkoks9SukdtZsVmthhYAcx19wU1PGeUmZWbWXlFRUW6c0oNjj0WLrsMpk+H3/0udhoRyZSUitrdN7p7H6AT0NfMetbwnKnuXubuZaWlpenOKVtw5ZVw1FFhz8Wnn46dRkQyoU6jPtz9S+Bp4PCMpJE6KyqCu++GHj3CFfbf/x47kYikWyqjPkrNrG3l42bAIcCbmQ4mqWvdGh55BDZuhKFDYc2a2v+MiOSOVK6otweeMrO/AS8T7lE/ltlYUlc77xyG7b3+Ovz857BpU+xEIpIuqYz6+Ju77+Xuvd29p7tfmY1gUneDBoX9FmfPhquuip1GRNJFU8jzzNix8OqrcMUV0KtXmBQjIrlNU8jzjBncdhvstx+cfHIobRHJbSrqPNS0aVgWtV07OPpobZArkutU1Hlq++3DNPNPPw3D9tati51IROpLRZ3HysrgjjvCGtZnnaVp5iK5Sm8m5rkTToDXXoOrr4aePeG882InEpG60hV1AbjyyrDf4rhxMGdO7DQiUlcq6gJQVAR/+EMYrnfCCbB0aexEIlIXKuoC0bIlPPooNGsGgwfDypWxE4lIqlTUBaRLlzBs76OPwoYDGgkikhtU1AXmgAPC+tXPPKORICK5QqM+CtCIEeE+9dVXw267hTcZRSS5VNQF6sor4c034YILwlrWRx0VO5GIbIlufRSoqpEg++wDw4fD4sWxE4nIlqioC1jz5mEkSLt24Yp6+fLYiUSkJirqArf99vDHP8IXX8CQIfD117ETicgPqaiFPn3gnntg0aKwNOrGjbETicjmVNQChFsfkyaFcdYXXRQ7jYhsTqM+5J9++Ut45x24/nro3h3OPDN2IhEBFbVsxgxuuAHeew/OPht23BGOOCJ2KhHRrQ/5npKSsJt5795w/PEatieSBCpq+ZFWreCxx6BtWzjySFi2LHYikcKmopYa7bADPP44rFkDP/0prFoVO5FI4VJRyxb16gWzZ4d1QbTvokg8tRa1mXU2s6fMbKmZLTGzc7MRTJLhkENg2jSYNw9GjtRqeyIxpDLqYwMwzt0XmVkrYKGZzXX3NzKcTRLi1FPhgw/g17+Gzp3hv/4rdiKRwlJrUbv7cmB55ePVZrYU6AioqAvIr34Vyvrqq0NZ/8d/xE4kUjjqNI7azLoCewELavjeKGAUQJcuXdIQTZLEDKZMCQs3nXUWdOgAQ4fGTiVSGFJ+M9HMWgKzgfPc/asfft/dp7p7mbuXlZaWpjOjJERJCdx3H5SVhaVRX3ghdiKRwpBSUZtZI0JJz3T3BzMbSZKsRYswxrpz57A+yJIlsROJ5L9URn0YMB1Y6u6TMh9Jkq60FP7yF2jSBAYNCveuRSRzUrmi7gecDAw0s8WVx08znEsSrls3mDMHVq8OZf3ZZ7ETieSvVEZ9PA9YFrJIjtlzz7DpwGGHweDBYax1y5axU4nkH81MlAbp3z8s4vTyy3DMMbB2bexEIvlHRS0NNnRomL04dy6cdJJ2iBFJNxW1pMVpp4UdYmbNChsOaKq5SPpo4wBJm7FjYeXKMHuxTRuYODFMlBGRhlFRS1pddVVYEvX668O61r/5TexEIrlPRS1pZQaTJ4d1rK+4IpT1f/5n7FQiuU1FLWlXVAS//30o63HjoHlzbZQr0hAqasmIkhKYORO++w5Gjw6zGE87LXYqkdykUR+SMY0bwwMPhAkxZ5wRiltE6k5FLRnVtCk8/DAcdBCcckpYfU9E6kZFLRnXrFmYat6vH4wYobIWqSsVtWRFixZhV/N//VeVtUhdqagla1q2VFmL1IeKWrKqqqz79YMTT4S77oqdSCT5VNSSdVVlfdBBYYfzGTNiJxJJNhW1RFG1pdegQWHo3u9+FzuRSHKpqCWaZs3C0L0hQ2DMGLj22tiJRJJJRS1RNWkSlkY98US45BK4+GItkSryQ5pCLtE1ahTeVGzVCiZMCKvv/fa3UFwcO5lIMqioJRGKimDKFGjbNpR1RQXcfXeY2ShS6FTUkhhm4T51hw5hadSVK+GRR6B169jJROLSPWpJnLFjw62Q558Pm+d+/HHsRCJxqaglkU46KQzfe/dd2H9/eP312IlE4lFRS2INGgTPPQcbNoSZjE8+GTuRSBy1FrWZzTCzFWamaxrJuj59YP586NwZDj887BwjUmhSuaK+Azg8wzlEtqhLF3jhBTj4YBg1KrzRuHFj7FQi2VNrUbv7s8DnWcgiskVt2oR71uecAzfcEGYzrloVO5VIdqTtHrWZjTKzcjMrr6ioSNfLivxTSQncdFNYF+SJJ6BvX3jzzdipRDIvbUXt7lPdvczdy0pLS9P1siI/Mnp0eGPxiy9CWT/6aOxEIpmlUR+Sk/r3h4ULoUcPOPpouPTSMDpEJB+pqCVnde4cJsWMHAnjx8Ohh8Inn8ROJZJ+qQzPuwd4EdjFzJaZ2RmZjyWSmqZNYepUuOMOWLAA9toL5s2LnUokvVIZ9THc3bd390bu3sndp2cjmEhdnHpqKOp27eCww8KSqevXx04lkh669SF5o1cvePnlsGPMtdfCgQfC//5v7FQiDaeilrzSokWYvXjfffDWW2Fm45Qp2oxAcpuKWvLS8ceHhZz+7d/grLPgiCPgww9jpxKpHxW15K2OHWHOHLjllrC40x57wK23wqZNsZOJ1I2KWvKaWbiifu21MDlm9GgYOFAzGiW3qKilIOy0E8ydC9Onw6uvQu/ecNll8M03sZOJ1E5FLQXDDE4/PbzJeOKJcM01sPvuMHu23myUZFNRS8HZdtswQeaZZ8LO58ceCwMGwCuvxE4mUjMVtRSs/v1DOU+ZAkuWwD77hIkz778fO5nI96mopaCVlMCZZ4aJMeefH8Zf9+gB550Hn34aO51IoKIWAdq2heuug3fegZNPhptvhm7dwm4yWuhJYlNRi2ymUyeYNg2WLoXjjoPJk0Nhn3MO/P3vsdNJoVJRi9SgRw+4884wQmT4cLjtNujeHYYNg5deip1OCo2KWmQruneHGTPC1fT558Of/wz77ReOu+6CtWtjJ5RCoKIWSUHHjjBhQlgv5Oabw8a6p5wSvj52bFhXRCRTVNQiddC6NZx9driH/cQTYTr6LbeEJVb79g2b72q0iKSbilqkHszC1l/33w8ffQSTJoWNCs49N1xlDxoUllutqIidVPKBeQbmzpaVlXl5eXnaX1ck6ZYsgZkzQ4G/+y4UFYWJNUcdBYMHhzcpRWpiZgvdvazG76moRdLPPSz+NGsWPPJI9T3s7t3DVmGHHhqmrbdpEzenJIeKWiSyf/wDHnssjBp5+umwal9REey9N/zkJ+E44ABo3z52UolFRS2SIOvWwYsvht3Sn3kmbMq7bl343s47w/77w777QlkZ7LknNG8eN69kh4paJMG+/TZsyvvii+GYP7965EhREeyyS1g/u3dv6NkTdtstrK9dXBw3t6TX1oq6JNthROT7mjULbzj27x8+d4ePP4bycli4MNzrXrAgLBhVpUmTcL97553DG5Tdu4ep7jvtBJ07Q6NGcc5FMkNFLZIwZmGIX8eOcPTR1V9ftSpsIfbGG+F4++3w+Z/+FIYGbv7nd9gBunQJa5dUvVbHjrDddrD99tChQ1iIyiz75yd1l1JRm9nhwGSgGJjm7tdmNJWI/EibNtXT1ze3cSMsWxamub/3XlhP+4MPwsfFi0OR17TlWEkJlJaGY5ttqo927eBf/iV8bNOm+mjdOhytWkHLluG2jGRHrUVtZsXALcChwDLgZTN71N3fyHQ4EaldcTHsuGM4Djrox993D1fjy5eHJVurjooKWLEifFy5MgwhXLkSvvgCNmyo/ec2bw4tWnz/aN48HM2aVR9Nm4ajSZPqj40bh49Vjxs3Drdrqj5WHSUl1R9/eBQXV3+s6SgqCr8x5MNvDalcUfcF3nH39wDM7F7gaEBFLZIDzMJtjrZtwxuRtXGHr78Ohb1qVTi+/BJWrw7HV1/BmjXhWL06PPebb6o/rlgRPn73XXij9Ntvw+O1a+PsTVlUFI6q8t78MPvx4x9+rS5HaSk8+2z6zyGVou4IfLjZ58uA/X74JDMbBYwC6NKlS1rCiUj2mYVbGy1bhjcm08U93Etfu7b6WL8+DE2selz1+fr14aq+6msbNlQfGzfW/HjTpnBs3BiOqsebf829+vOqx5s2/fixe/Vz6nK0bZu+/702l0pR1/SLw4/+XXT3qcBUCMPzGphLRPKMWfVtjlatYqfJLam8HbAM2Pzf1U7Ax5mJIyIiP5RKUb8M7Gxm3cysMTAMeDSzsUREpEqttz7cfYOZnQ38hTA8b4a7L8l4MhERAVIcR+3ujwOPZziLiIjUQEPWRUQSTkUtIpJwKmoRkYRTUYuIJFxG1qM2swrg/Xr+8fbAZ2mME1O+nEu+nAfoXJIoX84DGnYuO7p7aU3fyEhRN4SZlW9p8excky/nki/nATqXJMqX84DMnYtufYiIJJyKWkQk4ZJY1FNjB0ijfDmXfDkP0LkkUb6cB2ToXBJ3j1pERL4viVfUIiKyGRW1iEjCJbaozewcM3vLzJaY2XWx8zSEmZ1vZm5m7WNnqS8zm2hmb5rZ38zsITPL0F4WmWFmh1f+/+kdM7s4dp76MrPOZvaUmS2t/LtxbuxMDWVmxWb2ipk9FjtLQ5hZWzObVfn3ZKmZHZCu105kUZvZAMK+jL3dfQ/g/0eOVG9m1pmwMfAHsbM00Fygp7v3Bt4GLomcJ2WbbdB8BLA7MNzMdo+bqt42AOPcfTdgf2BMDp9LlXOBpbFDpMFkYI677wrsSRrPKZFFDYwGrnX3tQDuviJynoa4AbiQGrYvyyXu/oS7V+1NPZ+w00+u+OcGze6+DqjaoDnnuPtyd19U+Xg1oQw6xk1Vf2bWCTgSmBY7S0OYWWugPzAdwN3XufuX6Xr9pBZ1D+BAM1tgZs+Y2b6xA9WHmQ0BPnL3V2NnSbPTgT/HDlEHNW3QnLPlVsXMugJ7AQviJmmQGwkXMptiB2mgnYAK4PbK2zjTzKxFul48pY0DMsHM5gHb1fCtywi52hF+tdsXuN/MdvIEjiWs5TwuBQ7LbqL629q5uPsjlc+5jPDr98xsZmuglDZoziVm1hKYDZzn7l/FzlMfZjYYWOHuC83soNh5GqgE2Bs4x90XmNlk4GLg8nS9eBTufsiWvmdmo4EHK4v5JTPbRFjspCJb+VK1pfMws15AN+BVM4Nwq2CRmfV190+yGDFlW/tvAmBmpwKDgYOT+I/mVuTVBs1m1ohQ0jPd/cHYeRqgHzDEzH4KNAVam9nd7n5S5Fz1sQxY5u5Vv93MIhR1WiT11sfDwEAAM+sBNCbHVtdy99fcfVt37+ruXQn/IfdOaknXxswOBy4Chrj7N7Hz1FHebNBs4V/96cBSd58UO09DuPsl7t6p8u/HMOB/crSkqfx7/aGZ7VL5pYOBN9L1+tGuqGsxA5hhZq8D64BTc+wKLh/9FmgCzK38DWG+u58ZN1Jq8myD5n7AycBrZra48muXVu5rKnGdA8ysvBh4DzgtXS+sKeQiIgmX1FsfIiJSSUUtIpJwKmoRkYRTUYuIJJyKWkQk4VTUIiIJp6IWEUm4/wMpIgBjPFYodQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(Z, left_sig(Z),'b', label='left')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "When $Z$ is large then value is near zero"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "def right_sig(x):\n",
    "    return -np.log(1 - (1 / (1 + np.exp(-x))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11e00bc10>]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAZ6UlEQVR4nO3deZSU1Z3G8e/PpgERcG0VFSRoxBAdo7YY48REHXHHfSGKYlSEEUPiLi4R9XjAuKHiBAQcIrgFVMSIwqjBHW0WjYgoEUYWOTRRXAAFun/zx+0eELG7urveum9VPZ9z3lNFd1v9FMjD2/d9773m7oiISHptFjuAiIjUTUUtIpJyKmoRkZRTUYuIpJyKWkQk5Zol8aLbbbedd+zYMYmXFhEpSNOnT1/u7mWb+lwiRd2xY0cqKiqSeGkRkYJkZv/7Q5/T0IeISMqpqEVEUk5FLSKScipqEZGUU1GLiKRcRkVtZluZ2Tgz+8DM5pjZQUkHExGRINPb84YAz7n7qWbWHGiVYCYREdlAvWfUZtYWOAQYCeDua9x9RdLBRETyySuvwN13QxIrR2cy9NEJqAQeNLOZZjbCzLbY+IvMrLeZVZhZRWVlZdaDioik1dKlcPrpcP/9sGpV9l8/k6JuBuwH/Je77wusBK7e+Ivcfbi7l7t7eVnZJmdBiogUnHXr4Iwz4IsvYPx42OJ7p7FNl0lRLwIWufu0ml+PIxS3iEjRu/ZaePllGD4c9t47me9Rb1G7+1JgoZl1rvnQ4cD7ycQREckfTz0Ft90GffrA2Wcn930yvevjEmBszR0fHwPnJRdJRCT95s2Dc8+F8vJwETFJGRW1u88CypONIiKSH1atglNOgWbNYNw4aNEi2e+XyDKnIiKFyh3+8z/hH/+AZ5+FXXdN/ntqCrmISAOMGAGjR8MNN8BRR+Xme6qoRUQyVFEB/fpBt25w/fW5+74qahGRDHz2GZx6KuywA4wdCyUlufveGqMWEalHdTX07AlLlsCrr8J22+X2+6uoRUTqceut4cLh0KHQtWvuv7+GPkRE6jBlSrhw+JvfQN++cTKoqEVEfsAnn0CPHtClS5gibhYnh4paRGQTvv0WTjsN1qxJbrGlTGmMWkRkEy67DN56K8w87Ny5/q9Pks6oRUQ28vDD4cLhpZeGqeKxqahFRDYwezZceCH88pcwaFDsNIGKWkSkxpdfwsknQ5s28NhjUFoaO1GgMWoREcJiS+efD//8J7zwArRrFzvReipqERHCmtLjxsHgwfCrX8VO810a+hCRovfKK3DFFXDSSeExbVTUIlLUli4Nm9N26gQPPhhvUktdNPQhIkVr3To480xYsQKefx623DJ2ok1TUYtI0RowAKZOhb/8JbkdxLNBQx8iUpSeeAL+9Kew0FLPnrHT1E1FLSJF58MPoVcvOOAAuOuu2Gnqp6IWkaKyalXYqaV589zsIJ4NGqMWkaLhDhddBO+9B5MmQYcOsRNlRkUtIkXjz3+GMWNg4EA48sjYaTKXUVGb2QLgK6AKWOfu5UmGEhHJtrfegv794eij4brrYqdpmIacUR/q7ssTSyIikpDly8O49E47hTPqzfLs6pyGPkSkoFVVhf0Oly2D116DbbaJnajhMv13xYHJZjbdzHpv6gvMrLeZVZhZRWVlZfYSiog0wcCBYYPa++6D/fePnaZxMi3qg919P+Bo4GIzO2TjL3D34e5e7u7lZWVlWQ0pItIYf/sb3HwznHdeWMI0X2VU1O6+pOZxGfAk0DXJUCIiTTV/Ppx9NvzsZ2FbrTQutpSpeovazLYwsza1z4FuwHtJBxMRaazVq9fvdTh+PGy+edw8TZXJxcQdgCct/HPUDHjY3Z9LNJWISBNccgnMnAkTJ4blS/NdvUXt7h8D++Qgi4hIk40cGY7rroPjjoudJjvy7G5CEZEfNmMGXHwxHHEE3Hhj7DTZo6IWkYLw2WdhXHr77WHsWCgpiZ0oezThRUTyXnV1WFN68WJ49VUotDuEVdQikvduuQWefRbuvx+6FuDNwxr6EJG89vzzYTy6Z0/o0yd2mmSoqEUkby1YENbx2GuvsIRpPk9qqYuKWkTy0jffhBXxqqrC/oetWsVOlByNUYtIXvrd72D6dJgwAXbfPXaaZOmMWkTyzoMPwgMPwDXXQPfusdMkT0UtInllxgzo2xcOPzysjFcMVNQikjdqJ7WUlcEjjxTWpJa6aIxaRPLChpNaXnml8Ca11EVFLSJ5YcNJLQceGDtNbmnoQ0RSb9Kkwp/UUhcVtYik2vz5cNZZsPfehT2ppS4qahFJrdWrw6SW6urCn9RSF41Ri0gquUO/fuF2vIkTYbfdYieKR2fUIpJKI0bAqFGFtVNLY6moRSR13n47nE1361ZYO7U0lopaRFJl+fIwqaVdO3j44eKZ1FIXjVGLSGpUVYVlS5ctCzu1bLtt7ETpoKIWkdS44QaYMiUsuFReHjtNemjoQ0RSYcIEuPVWuOCCcMh6KmoRie6jj+Ccc8JZ9L33xk6TPhkXtZmVmNlMM3smyUAiUlxWroSTT4bSUhg3Dlq2jJ0ofRoyRt0fmAO0TSiLiBQZ9zDM8f778NxzsOuusROlU0Zn1Ga2C3AsMCLZOCJSTIYMgUcfDSvjHXFE7DTplenQx93AlUD1D32BmfU2swozq6isrMxKOBEpXC+/DJdfDieeCFdfHTtNutVb1GZ2HLDM3afX9XXuPtzdy929vKyYVvQWkQZbsgROPz2s3zF6dHGuiNcQmYxRHwx0N7NjgJZAWzMb4+5nJxtNRArRmjVw2mnw9dfwwgvQVle96lXvGbW7X+Puu7h7R+BM4EWVtIg01mWXweuvh53Ef/rT2Gnyg+6jFpGcGTMG7rsvlPVpp8VOkz8aNIXc3f8O/D2RJCJS0GbNgt694de/hkGDYqfJLzqjFpHEffZZmNSyzTbw2GPQTKsMNYh+u0QkUVVVYc/DxYvDLXnbbx87Uf5RUYtIogYODLMOhw2DAw+MnSY/aehDRBLz9NNw883w29/ChRfGTpO/VNQikoi5c6Fnz7Ai3tChmtTSFCpqEcm6r74KFw+bN4fx47UiXlNpjFpEsso9DHV88EHYraVDh9iJ8p+KWkSy6rbbwrrSt90Ghx0WO01h0NCHiGTNlCkwYEBYcOnyy2OnKRwqahHJigUL4MwzoUsXGDlSFw+zSUUtIk22alW4eFhVBU8+Ca1bx05UWDRGLSJN4g4XXRTW8pg4EXbfPXaiwqOiFpEmuffesCreTTfBscfGTlOYNPQhIo02dSpceimccAJce23sNIVLRS0ijbJwYVhTevfd4S9/gc3UJonRb62INNjq1eHi4TffwFNPaTutpGmMWkQaxB369oWKCpgwAfbcM3aiwqczahFpkKFDw87hf/wjdO8eO01xUFGLSMZefhn+8Ac4/ni44YbYaYqHilpEMrJwIZx6Kuy2Gzz0kC4e5pJ+q0WkXqtXw0knrb94uOWWsRMVF11MFJE6uUOfPjB9ui4exqIzahGp0z33hPukb7xRFw9jUVGLyA968UW47LIw8/D662OnKV71FrWZtTSzt8zsHTObbWYDcxFMROKaPz+sK925sy4expbJGPW3wGHu/rWZlQKvmtkkd38z4WwiEsnKleHiYVVVuHjYpk3sRMWt3qJ2dwe+rvllac3hSYYSkXhq9zx891149ln48Y9jJ5KMfpgxsxIzmwUsA6a4+7RNfE1vM6sws4rKysps5xSRHBk0CB5/PDwedVTsNAIZFrW7V7n7z4BdgK5mttcmvma4u5e7e3lZWVm2c4pIDjzzTFiutEcPuOKK2GmkVoMuD7j7CuDvgP6dFSkwH3wAv/kN7LsvjBihPQ/TJJO7PsrMbKua55sD/wF8kHQwEcmdzz8P90i3bBn2PGzVKnYi2VAmd320A0abWQmh2B9392eSjSUiuVJVFYY6FiwI90136BA7kWwsk7s+3gX2zUEWEYngqqvg+efhgQfg3/89dhrZFN3CLlLEHnoI7rgD+vWDCy6InUZ+iIpapEi9+WYo58MOgzvvjJ1G6qKiFilCixbBiSdC+/bhnunS0tiJpC5a5lSkyKxaFRZZWrUqXDzcdtvYiaQ+KmqRIlI7PXzmTJg4Ebp0iZ1IMqGiFikit9wCjz0GgwfDscfGTiOZ0hi1SJEYPz5sSHvOOZoenm9U1CJFYOZM6NkTDjoIhg3T9PB8o6IWKXCffhqmh2+3XZge3rJl7ETSUBqjFilgq1eHOzw+/xxeew122CF2ImkMFbVIgaquhl69oKIi7NKyzz6xE0ljqahFCtRNN4XJLIMHa/fwfKcxapEC9PDDMHBgOKPWHR75T0UtUmBefz1MajnkEN3hUShU1CIFZP789Wt4PPEENG8eO5Fkg4papEB88QUcfzysXRv2PtQaHoVDFxNFCsDatXDaaTB3btgEoHPn2Ikkm1TUInnOHS6+GKZMgVGjwvrSUlg09CGS526/PWyjNWAAnHde7DSSBBW1SB7761/hyivh9NPh5ptjp5GkqKhF8tTrr4eFln7xCxg9GjbT3+aCpT9akTw0b15Yw6N9e5gwQQstFToVtUieWb4cjjkmXEScNCmsiieFTXd9iOSR1avDuh0LF8ILL8Duu8dOJLlQ7xm1mbU3s5fMbI6ZzTaz/rkIJiLfVVUFZ50Fb74JY8eGsWkpDpmcUa8DLnP3GWbWBphuZlPc/f2Es4lIDXe49NKw8P/dd8PJJ8dOJLlU7xm1u3/q7jNqnn8FzAF2TjqYiKx3++1wzz3whz9Af/1MW3QadDHRzDoC+wLTNvG53mZWYWYVlZWV2UknIowZE+6VPuOMUNhSfDIuajNrDYwHfu/uX278eXcf7u7l7l5eVlaWzYwiRWvy5DDb8NBDda90Mcvoj93MSgklPdbdn0g2kogATJ8Op5wCXbqEsekWLWInklgyuevDgJHAHHe/M/lIIvLRR3D00WGp0kmTYMstYyeSmDI5oz4Y6AkcZmazao5jEs4lUrSWLIFu3cKdHpMnw047xU4ksdV7e567vwpoMx+RHFixIpxJV1bCSy/BHnvETiRpoJmJIimxahUcdxzMmQN/+xsccEDsRJIWKmqRFFizBk49NayI99hjcMQRsRNJmqioRSKrqoJzzw0XDYcPD1tqiWxId2WKRFS7jdajj8KgQXDhhbETSRqpqEUicYcrroBhw+Dqq+Gqq2InkrRSUYtEctNNcMcd0K8f3Hpr7DSSZipqkQhuvx1uvBF69YIhQ8B0A6zUQUUtkmP33BOGPM44A0aM0PodUj/9LyKSQ8OGhWVKTzoJHnoISkpiJ5J8oKIWyZFRo6BPnzCp5dFHobQ0diLJFypqkRwYNQouuACOPBL++ldo3jx2IsknKmqRhNWWdLdu8NRT0LJl7ESSb1TUIglSSUs2qKhFEvLnP8P556ukpelU1CIJGDIE+vaF44+HCRNU0tI0KmqRLBs8GH7/+7CN1rhx2kJLmk5FLZIl7jBgQFi3o0cPeOQR3d0h2aFlTkWyoLoafvc7GDoUeveG++/XZBbJHp1RizTR2rVhPemhQ+Hyy8NFRJW0ZJPOqEWaYOXKsND/pElwyy1h6EMLLEm2qahFGulf/wrTwd96K+zMokX/JSkqapFGmD8fjjkmPI4bFxZZEkmKilqkgSoqwpn0mjUweTIcckjsRFLodDFRpAGeeQZ+9SvYfHN47TWVtORGvUVtZqPMbJmZvZeLQCJp5A533QUnnAA/+Qm88UZ4FMmFTM6o/xs4KuEcIqm1di1cdBFcemko6qlTYccdY6eSYlJvUbv7y8BnOcgikjrLl4c1pB94AK65Jlw43GKL2Kmk2GTtYqKZ9QZ6A3To0CFbLysSzaxZcOKJsHQpjB4N55wTO5EUq6xdTHT34e5e7u7lZWVl2XpZkSgefRR+8QtYtw5eflklLXHprg+RDaxZE1a+69ED9t8/3IrXtWvsVFLsVNQiNRYuDLfeDRkSyvqFF3TRUNIhk9vzHgHeADqb2SIzOz/5WCK59cwzsO++MHt22Hz2rru0RKmkR70XE929Ry6CiMTwzTdw1VVwzz2wzz7w+OOwxx6xU4l8l4Y+pGjNng0//3ko6f794c03VdKSTipqKTpVVXDHHeFi4ZIl8PTTcPfd2tdQ0kuLMklR+ec/w87gU6dC9+5hIsv228dOJVI3nVFLUaiqgjvvhL33hhkzYNQoeOoplbTkB51RS8F7992wj+G0aXDssWGrrF12iZ1KJHM6o5aC9dVXYSGl/fYLQx4PPwwTJ6qkJf+oqKXguIdS3nPPcJHwggtg7tww21D7GUo+UlFLQZk2LazRcdZZYVbh66+HoY5ttomdTKTxVNRSEObNgzPPDPdFL1gADz4Ib78dfi2S71TUktcWL4a+fcNuKxMnwnXXwYcfQq9esJn+75YCobs+JC8tXAiDB4f7oKurww4s112nRZSkMKmoJa/MnRtmFY4eHQq6Vy8YMAB+9KPYyUSSo6KW1HMPO37fcQdMmBBWtTvvPLj6aujYMXY6keSpqCW1Vq8Ot9ndd1/YFmvrreHaa6FfP9hhh9jpRHJHRS2pM3MmjBwJY8fCihWw114wbFi45U4by0oxUlFLKixZAo88AmPGhLPnFi3glFPC1O9DDtFEFSluKmqJZulSePJJGDcOXnopjEUfcADce284e95669gJRdJBRS054w5z5oRtryZODBcI3aFzZ7j++lDOWrhf5PtU1JKozz+HF1+EKVNg8mSYPz98fJ994I9/hFNPhS5dNLQhUhcVtWTVsmVhfY2pU8Mxa1Y4a27dGg49FK68Miw12r597KQi+UNFLY329dehiCsqwvHGG/Dxx+FzLVvCQQeFs+bDD4cDD4TS0rh5RfKVilrqtW5dWM/5/ffDhrDvvgvvvAMffRTOlgF22iksgNS3byjo8vJw54aINJ2KWgBYsyasn/Hxx+GYNy8U8YcfhpJes2b913bqFMaYe/QIhbz//tCuXbzsIoVORV3gqqvDpJGlS8Px6afhnuXFi8OxaBF88kn4eO3ZMYSz4d12C3dkHH98uODXpUtYjL9Nm3jvR6QYZVTUZnYUMAQoAUa4+6BEU8l3VFXBypXw5Zdhe6kvvwzl+8UX4fj8c/jss/D4r3+tPyorw1FV9f3XbN0adt45HEceCbvuCh06hLPlTp3CUEZJSe7fq4h8X71FbWYlwFDgCGAR8LaZPe3u7ycdLmnu4aiuDmW28bFu3fcfNzzWrl3/uHZtGB6ofax9/u233z+++SY8rl4dnq9eDatWrT9Wrlz/+PXX4fP1KS0NE0S23TYcnTqFC3jbbw9lZWH5zx13DEMU7dpB27bJ//6KSHZkckbdFZjn7h8DmNmjwAlA1ot6//1DQdUWaKZHdXU4ap9v6mMbHlVV65/nmlm4I6JFC9h88+8erVqFLaPatw9rWrRqFYYZWrcOR9u24WjTBrbaCrbcMhxbbx2+VvciixSmTIp6Z2DhBr9eBBy48ReZWW+gN0CHDh0aFaZLl3Cmadawo6Rk/fPNNgvHxs9LSr77vPbXGz5u/LxZs/WPGz4vLQ3PS0vDUfux5s3XP254tGgRjtrPq1BFpCEyKepN1Yp/7wPuw4HhAOXl5d/7fCYeeqgx/5WISGHLZFe5RcCG88h2AZYkE0dERDaWSVG/DfzYzH5kZs2BM4Gnk40lIiK16h36cPd1ZtYPeJ5we94od5+deDIREQEyvI/a3Z8Fnk04i4iIbEImQx8iIhKRilpEJOVU1CIiKaeiFhFJOXNv1NyUul/UrBL430b+59sBy7MYJ6ZCeS+F8j5A7yWNCuV9QNPey67uXrapTyRS1E1hZhXuXh47RzYUynsplPcBei9pVCjvA5J7Lxr6EBFJORW1iEjKpbGoh8cOkEWF8l4K5X2A3ksaFcr7gITeS+rGqEVE5LvSeEYtIiIbUFGLiKRcaovazC4xs7lmNtvMboudpynM7HIzczPbLnaWxjKzP5nZB2b2rpk9aWZbxc7UEGZ2VM3/T/PM7OrYeRrLzNqb2UtmNqfm70b/2JmaysxKzGymmT0TO0tTmNlWZjau5u/JHDM7KFuvncqiNrNDCfsy/pu7/xS4PXKkRjOz9oSNgT+JnaWJpgB7ufu/AR8C10TOk7ENNmg+GugC9DCzLnFTNdo64DJ3/wnwc+DiPH4vtfoDc2KHyIIhwHPuviewD1l8T6ksaqAvMMjdvwVw92WR8zTFXcCVbGL7snzi7pPdfV3NL98k7PSTL/5/g2Z3XwPUbtCcd9z9U3efUfP8K0IZ7Bw3VeOZ2S7AscCI2FmawszaAocAIwHcfY27r8jW66e1qPcAfmlm08xsqpkdEDtQY5hZd2Cxu78TO0uW/RaYFDtEA2xqg+a8LbdaZtYR2BeYFjdJk9xNOJGpjh2kiToBlcCDNcM4I8xsi2y9eEYbByTBzP4H2HETn7qWkGtrwo92BwCPm1knT+G9hPW8jwFAt9wmary63ou7T6j5mmsJP36PzWW2Jspog+Z8YmatgfHA7939y9h5GsPMjgOWuft0M/t17DxN1AzYD7jE3aeZ2RDgauD6bL14FO7+Hz/0OTPrCzxRU8xvmVk1YbGTylzly9QPvQ8z2xv4EfCOmUEYKphhZl3dfWkOI2asrj8TADM7FzgOODyN/2jWoaA2aDazUkJJj3X3J2LnaYKDge5mdgzQEmhrZmPc/ezIuRpjEbDI3Wt/uhlHKOqsSOvQx1PAYQBmtgfQnDxbXcvd/+Hu27t7R3fvSPiD3C+tJV0fMzsKuAro7u6rYudpoILZoNnCv/ojgTnufmfsPE3h7te4+y41fz/OBF7M05Km5u/1QjPrXPOhw4H3s/X60c6o6zEKGGVm7wFrgHPz7AyuEN0HtACm1PyE8Ka794kbKTMFtkHzwUBP4B9mNqvmYwNq9jWVuC4BxtacDHwMnJetF9YUchGRlEvr0IeIiNRQUYuIpJyKWkQk5VTUIiIpp6IWEUk5FbWISMqpqEVEUu7/ADyxmQLo1DkeAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(x, right_sig(Z),'b', label='right')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "cost_1 = left_sig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11e07dad0>]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAazklEQVR4nO3deXiU9b3+8fcnCft+JKKyCF6IGyBqRD0cqeCCVkSOdQFxqVo4Ilrl4G5t/elRRH6iaC1KAbXKcQO3WksFj3sFDYhVRD1qq6IoERXBhfVz/vgmDWogk2Rmvs/M3K/req5MkmFyP7XcPHnmu5i7IyIiyVUUO4CIiGydilpEJOFU1CIiCaeiFhFJOBW1iEjClWTiRdu3b+9du3bNxEuLiOSlhQsXfubupTV9LyNF3bVrV8rLyzPx0iIiecnM3t/S93TrQ0Qk4VTUIiIJp6IWEUk4FbWISMKpqEVEEi6lojaztmY2y8zeNLOlZnZApoOJiEiQ6vC8ycAcdz/WzBoDzTOYSURENlPrFbWZtQb6A9MB3H2du3+Z7iCbNsE118DChel+ZRGR3JbKrY+dgArgdjN7xcymmVmLHz7JzEaZWbmZlVdUVNQ5yKpVcOutcOyx8Pnndf7jIiJ5K5WiLgH2Bqa4+17A18DFP3ySu0919zJ3LystrXEW5Fa1awezZsFHH8HJJ4crbBERSa2olwHL3H1B5eezCMWddn37wo03wuOPw/jxmfgJIiK5p9aidvdPgA/NbJfKLx0MvJGpQKNHw4knwuWXw7x5mfopIiK5I9Vx1OcAM83sb0Af4JpMBTKDqVNh991h+HBYtixTP0lEJDekVNTuvrjy/nNvdx/q7l9kMlSLFjB7Nnz3HRx3HKxbl8mfJiKSbImdmbjLLnD77TB/PlxwQew0IiLxJLaoIQzVGzsWbroJ7r03dhoRkTgSXdQAEyZAv37wi1/AGxl7C1NEJLkSX9SNGsF994X71j/7GaxeHTuRiEh2Jb6oATp2DLc+3n4bRo4E99iJRESyJyeKGmDAALj66nB1fdNNsdOIiGRPzhQ1wIUXwpAhcP758Ne/xk4jIpIdOVXURUVw553QpUsYX71iRexEIiKZl1NFDdC2bZgM8/nnMGwYbNgQO5GISGblXFED9OkDU6bAU0/Br38dO42ISGblZFED/PznYQTI+PHwxz/GTiMikjk5W9QQRn/svXdYv/q992KnERHJjJwu6qZNw2YDRUVhMsy338ZOJCKSfjld1ADdusFdd8HixTBmTOw0IiLpl/NFDXDkkfCrX4XV9qZPj51GRCS98qKoAa64Ag49NFxVL1oUO42ISPrkTVEXF8N//zdsu224X62dzEUkX+RNUQO0bw8PPKCdzEUkv+RVUQPst1/1TubXZGxnRxGR7Mm7ooawk/mIEWHW4ty5sdOIiDRMXha1Gdx2G+yxR9jJ/IMPYicSEam/vCxqqN7JfN26sNLe2rWxE4mI1E/eFjVAjx5wxx3w0kswblzsNCIi9ZPXRQ1wzDFho4FbboGZM2OnERGpu5JUnmRm/wBWAxuBDe5elslQ6TZ+fLiqHjkSeveGXr1iJxIRSV1drqgHuHufXCtpgJKSsNdimzZhMsyqVbETiYikLu9vfVTZbju4//6wHOrpp2sncxHJHakWtQNPmNlCMxtV0xPMbJSZlZtZeUVFRfoSptGBB8LEifDgg3D99bHTiIikxjyFS0sz28HdPzazbYG5wDnu/uyWnl9WVubl5eVpjJk+7nD88fDQQ/Dkk/CTn8ROJCICZrZwS7eWU7qidvePKz+uAB4C+qYvXnaZhaVQu3eHE06Ajz+OnUhEZOtqLWoza2FmraoeA4cBr2c6WCa1bh0mw6xZE66u16+PnUhEZMtSuaLuADxvZq8CLwF/cvc5mY2VeXvsAdOmwQsvwEUXxU4jIrJltY6jdvf3gD2zkCXrhg2DF1+EG26A/fcPV9ciIklTMMPztmTiRDjggDBkb+nS2GlERH6s4Iu6ceOw2UCLFmG6+erVsROJiHxfwRc1QMeOcO+98Pbb8ItfaDKMiCSLirrSgAFhR5j774fJk2OnERGppqLezIUXwtChcMEF8PzzsdOIiAQq6s2YhfWru3ULI0A++SR2IhERFfWPtGkTJsN8+WWYuajJMCISm4q6Br16we9/D88+C5dcEjuNiBQ6FfUWjBgBY8aEVfYeeCB2GhEpZCrqrZg0KcxY1GQYEYlJRb0VVZNhmjULO8OsWRM7kYgUIhV1LTp1CpNh3noLzjhDk2FEJPtU1CkYOLB6MsyNN8ZOIyKFRkWdogsvhH//9zAZ5rnnYqcRkUKiok6RGdx+O+y0U5gMs3x57EQiUihU1HXQpk3YGPerr7QzjIhkj4q6jnr2DDvDPP98uA0iIpJpKup6GD4cfvnLsMrevffGTiMi+U5FXU8TJ0K/fmHI3pIlsdOISD5TUddT48ZhuF6rVmFnmK++ip1IRPKViroBdtghlPW778Kpp2oyjIhkhoq6gfr3D7dBHn4YrrsudhoRyUcq6jQ477ywdvWll8KTT8ZOIyL5RkWdBmZhyN6uu8KwYfDhh7ETiUg+SbmozazYzF4xs8cyGShXtWwZJsOsXRtW2lu7NnYiEckXdbmiPhfQqsxbscsu8Ic/wMsvh3HWIiLpkFJRm1kn4EhgWmbj5L6hQ8P2XVOnwowZsdOISD5I9Yr6RuBCYNOWnmBmo8ys3MzKKyoq0hIuV111FRxyCJx1FpSXx04jIrmu1qI2s8HACndfuLXnuftUdy9z97LS0tK0BcxFxcVwzz3QoUO4X/3ZZ7ETiUguS+WKuh8wxMz+AdwLDDSzuzOaKg+0bw+zZ8Onn4a1QTZujJ1IRHJVrUXt7pe4eyd37woMA/7H3U/KeLI8UFYGt9wC8+bB5ZfHTiMiuUrjqDPsjDNg5EgYPz7MXhQRqas6FbW7P+3ugzMVJl/dfDPsuy+cckrYJFdEpC50RZ0FTZqE+9VNm4Z9F1evjp1IRHKJijpLOncOmwy89RacfrpW2hOR1Kmos2jgQJgwAWbNCivuiYikQkWdZePGwXHHhdmLWmlPRFKhos4yszC1fNddw9Ko778fO5GIJJ2KOoKWLeGhh2D9+jBz8bvvYicSkSRTUUfSowfcdRcsXBjWBNGbiyKyJSrqiIYMCTMWb78dbr01dhoRSSoVdWS/+Q0ccQScey789a+x04hIEqmoIysuhpkzoUsXOPZYWL48diIRSRoVdQK0axfWAVm1KgzdW7cudiIRSRIVdUL07BnuVb/wAowdGzuNiCRJSewAUu3448OOMBMnhiVSTzstdiIRSQJdUSfMNdeEbbxGjw6b5IqIqKgTpqQkLN603XZwzDGwYkXsRCISm4o6gbbZJsxc/Oyz8Obi+vWxE4lITCrqhNprL5g2DZ59Fs4/P3YaEYlJbyYm2IgRYYr5DTfAPvuEHWJEpPDoijrhrrsOBgyAUaPCiBARKTwq6oQrKYH77oMOHcI2Xp9+GjuRiGSbijoHlJbqzUWRQqaizhF77x3eXHzuOc1cFCk0ejMxh4wYAYsWwaRJobhPPz12IhHJBl1R55gJE6pnLs6fHzuNiGRDrUVtZk3N7CUze9XMlpjZ/8tGMKlZ1czFjh3DzEUtiyqS/1K5ol4LDHT3PYE+wOFmtn9mY8nWbLNN9bKoP/sZrF0bO5GIZFKtRe3BmspPG1Ue2uEvst694c474cUXYcwY7bkoks9SukdtZsVmthhYAcx19wU1PGeUmZWbWXlFRUW6c0oNjj0WLrsMpk+H3/0udhoRyZSUitrdN7p7H6AT0NfMetbwnKnuXubuZaWlpenOKVtw5ZVw1FFhz8Wnn46dRkQyoU6jPtz9S+Bp4PCMpJE6KyqCu++GHj3CFfbf/x47kYikWyqjPkrNrG3l42bAIcCbmQ4mqWvdGh55BDZuhKFDYc2a2v+MiOSOVK6otweeMrO/AS8T7lE/ltlYUlc77xyG7b3+Ovz857BpU+xEIpIuqYz6+Ju77+Xuvd29p7tfmY1gUneDBoX9FmfPhquuip1GRNJFU8jzzNix8OqrcMUV0KtXmBQjIrlNU8jzjBncdhvstx+cfHIobRHJbSrqPNS0aVgWtV07OPpobZArkutU1Hlq++3DNPNPPw3D9tati51IROpLRZ3HysrgjjvCGtZnnaVp5iK5Sm8m5rkTToDXXoOrr4aePeG882InEpG60hV1AbjyyrDf4rhxMGdO7DQiUlcq6gJQVAR/+EMYrnfCCbB0aexEIlIXKuoC0bIlPPooNGsGgwfDypWxE4lIqlTUBaRLlzBs76OPwoYDGgkikhtU1AXmgAPC+tXPPKORICK5QqM+CtCIEeE+9dVXw267hTcZRSS5VNQF6sor4c034YILwlrWRx0VO5GIbIlufRSoqpEg++wDw4fD4sWxE4nIlqioC1jz5mEkSLt24Yp6+fLYiUSkJirqArf99vDHP8IXX8CQIfD117ETicgPqaiFPn3gnntg0aKwNOrGjbETicjmVNQChFsfkyaFcdYXXRQ7jYhsTqM+5J9++Ut45x24/nro3h3OPDN2IhEBFbVsxgxuuAHeew/OPht23BGOOCJ2KhHRrQ/5npKSsJt5795w/PEatieSBCpq+ZFWreCxx6BtWzjySFi2LHYikcKmopYa7bADPP44rFkDP/0prFoVO5FI4VJRyxb16gWzZ4d1QbTvokg8tRa1mXU2s6fMbKmZLTGzc7MRTJLhkENg2jSYNw9GjtRqeyIxpDLqYwMwzt0XmVkrYKGZzXX3NzKcTRLi1FPhgw/g17+Gzp3hv/4rdiKRwlJrUbv7cmB55ePVZrYU6AioqAvIr34Vyvrqq0NZ/8d/xE4kUjjqNI7azLoCewELavjeKGAUQJcuXdIQTZLEDKZMCQs3nXUWdOgAQ4fGTiVSGFJ+M9HMWgKzgfPc/asfft/dp7p7mbuXlZaWpjOjJERJCdx3H5SVhaVRX3ghdiKRwpBSUZtZI0JJz3T3BzMbSZKsRYswxrpz57A+yJIlsROJ5L9URn0YMB1Y6u6TMh9Jkq60FP7yF2jSBAYNCveuRSRzUrmi7gecDAw0s8WVx08znEsSrls3mDMHVq8OZf3ZZ7ETieSvVEZ9PA9YFrJIjtlzz7DpwGGHweDBYax1y5axU4nkH81MlAbp3z8s4vTyy3DMMbB2bexEIvlHRS0NNnRomL04dy6cdJJ2iBFJNxW1pMVpp4UdYmbNChsOaKq5SPpo4wBJm7FjYeXKMHuxTRuYODFMlBGRhlFRS1pddVVYEvX668O61r/5TexEIrlPRS1pZQaTJ4d1rK+4IpT1f/5n7FQiuU1FLWlXVAS//30o63HjoHlzbZQr0hAqasmIkhKYORO++w5Gjw6zGE87LXYqkdykUR+SMY0bwwMPhAkxZ5wRiltE6k5FLRnVtCk8/DAcdBCcckpYfU9E6kZFLRnXrFmYat6vH4wYobIWqSsVtWRFixZhV/N//VeVtUhdqagla1q2VFmL1IeKWrKqqqz79YMTT4S77oqdSCT5VNSSdVVlfdBBYYfzGTNiJxJJNhW1RFG1pdegQWHo3u9+FzuRSHKpqCWaZs3C0L0hQ2DMGLj22tiJRJJJRS1RNWkSlkY98US45BK4+GItkSryQ5pCLtE1ahTeVGzVCiZMCKvv/fa3UFwcO5lIMqioJRGKimDKFGjbNpR1RQXcfXeY2ShS6FTUkhhm4T51hw5hadSVK+GRR6B169jJROLSPWpJnLFjw62Q558Pm+d+/HHsRCJxqaglkU46KQzfe/dd2H9/eP312IlE4lFRS2INGgTPPQcbNoSZjE8+GTuRSBy1FrWZzTCzFWamaxrJuj59YP586NwZDj887BwjUmhSuaK+Azg8wzlEtqhLF3jhBTj4YBg1KrzRuHFj7FQi2VNrUbv7s8DnWcgiskVt2oR71uecAzfcEGYzrloVO5VIdqTtHrWZjTKzcjMrr6ioSNfLivxTSQncdFNYF+SJJ6BvX3jzzdipRDIvbUXt7lPdvczdy0pLS9P1siI/Mnp0eGPxiy9CWT/6aOxEIpmlUR+Sk/r3h4ULoUcPOPpouPTSMDpEJB+pqCVnde4cJsWMHAnjx8Ohh8Inn8ROJZJ+qQzPuwd4EdjFzJaZ2RmZjyWSmqZNYepUuOMOWLAA9toL5s2LnUokvVIZ9THc3bd390bu3sndp2cjmEhdnHpqKOp27eCww8KSqevXx04lkh669SF5o1cvePnlsGPMtdfCgQfC//5v7FQiDaeilrzSokWYvXjfffDWW2Fm45Qp2oxAcpuKWvLS8ceHhZz+7d/grLPgiCPgww9jpxKpHxW15K2OHWHOHLjllrC40x57wK23wqZNsZOJ1I2KWvKaWbiifu21MDlm9GgYOFAzGiW3qKilIOy0E8ydC9Onw6uvQu/ecNll8M03sZOJ1E5FLQXDDE4/PbzJeOKJcM01sPvuMHu23myUZFNRS8HZdtswQeaZZ8LO58ceCwMGwCuvxE4mUjMVtRSs/v1DOU+ZAkuWwD77hIkz778fO5nI96mopaCVlMCZZ4aJMeefH8Zf9+gB550Hn34aO51IoKIWAdq2heuug3fegZNPhptvhm7dwm4yWuhJYlNRi2ymUyeYNg2WLoXjjoPJk0Nhn3MO/P3vsdNJoVJRi9SgRw+4884wQmT4cLjtNujeHYYNg5deip1OCo2KWmQruneHGTPC1fT558Of/wz77ReOu+6CtWtjJ5RCoKIWSUHHjjBhQlgv5Oabw8a6p5wSvj52bFhXRCRTVNQiddC6NZx9driH/cQTYTr6LbeEJVb79g2b72q0iKSbilqkHszC1l/33w8ffQSTJoWNCs49N1xlDxoUllutqIidVPKBeQbmzpaVlXl5eXnaX1ck6ZYsgZkzQ4G/+y4UFYWJNUcdBYMHhzcpRWpiZgvdvazG76moRdLPPSz+NGsWPPJI9T3s7t3DVmGHHhqmrbdpEzenJIeKWiSyf/wDHnssjBp5+umwal9REey9N/zkJ+E44ABo3z52UolFRS2SIOvWwYsvht3Sn3kmbMq7bl343s47w/77w777QlkZ7LknNG8eN69kh4paJMG+/TZsyvvii+GYP7965EhREeyyS1g/u3dv6NkTdtstrK9dXBw3t6TX1oq6JNthROT7mjULbzj27x8+d4ePP4bycli4MNzrXrAgLBhVpUmTcL97553DG5Tdu4ep7jvtBJ07Q6NGcc5FMkNFLZIwZmGIX8eOcPTR1V9ftSpsIfbGG+F4++3w+Z/+FIYGbv7nd9gBunQJa5dUvVbHjrDddrD99tChQ1iIyiz75yd1l1JRm9nhwGSgGJjm7tdmNJWI/EibNtXT1ze3cSMsWxamub/3XlhP+4MPwsfFi0OR17TlWEkJlJaGY5ttqo927eBf/iV8bNOm+mjdOhytWkHLluG2jGRHrUVtZsXALcChwDLgZTN71N3fyHQ4EaldcTHsuGM4Djrox993D1fjy5eHJVurjooKWLEifFy5MgwhXLkSvvgCNmyo/ec2bw4tWnz/aN48HM2aVR9Nm4ajSZPqj40bh49Vjxs3Drdrqj5WHSUl1R9/eBQXV3+s6SgqCr8x5MNvDalcUfcF3nH39wDM7F7gaEBFLZIDzMJtjrZtwxuRtXGHr78Ohb1qVTi+/BJWrw7HV1/BmjXhWL06PPebb6o/rlgRPn73XXij9Ntvw+O1a+PsTVlUFI6q8t78MPvx4x9+rS5HaSk8+2z6zyGVou4IfLjZ58uA/X74JDMbBYwC6NKlS1rCiUj2mYVbGy1bhjcm08U93Etfu7b6WL8+DE2selz1+fr14aq+6msbNlQfGzfW/HjTpnBs3BiOqsebf829+vOqx5s2/fixe/Vz6nK0bZu+/702l0pR1/SLw4/+XXT3qcBUCMPzGphLRPKMWfVtjlatYqfJLam8HbAM2Pzf1U7Ax5mJIyIiP5RKUb8M7Gxm3cysMTAMeDSzsUREpEqttz7cfYOZnQ38hTA8b4a7L8l4MhERAVIcR+3ujwOPZziLiIjUQEPWRUQSTkUtIpJwKmoRkYRTUYuIJFxG1qM2swrg/Xr+8fbAZ2mME1O+nEu+nAfoXJIoX84DGnYuO7p7aU3fyEhRN4SZlW9p8excky/nki/nATqXJMqX84DMnYtufYiIJJyKWkQk4ZJY1FNjB0ijfDmXfDkP0LkkUb6cB2ToXBJ3j1pERL4viVfUIiKyGRW1iEjCJbaozewcM3vLzJaY2XWx8zSEmZ1vZm5m7WNnqS8zm2hmb5rZ38zsITPL0F4WmWFmh1f+/+kdM7s4dp76MrPOZvaUmS2t/LtxbuxMDWVmxWb2ipk9FjtLQ5hZWzObVfn3ZKmZHZCu105kUZvZAMK+jL3dfQ/g/0eOVG9m1pmwMfAHsbM00Fygp7v3Bt4GLomcJ2WbbdB8BLA7MNzMdo+bqt42AOPcfTdgf2BMDp9LlXOBpbFDpMFkYI677wrsSRrPKZFFDYwGrnX3tQDuviJynoa4AbiQGrYvyyXu/oS7V+1NPZ+w00+u+OcGze6+DqjaoDnnuPtyd19U+Xg1oQw6xk1Vf2bWCTgSmBY7S0OYWWugPzAdwN3XufuX6Xr9pBZ1D+BAM1tgZs+Y2b6xA9WHmQ0BPnL3V2NnSbPTgT/HDlEHNW3QnLPlVsXMugJ7AQviJmmQGwkXMptiB2mgnYAK4PbK2zjTzKxFul48pY0DMsHM5gHb1fCtywi52hF+tdsXuN/MdvIEjiWs5TwuBQ7LbqL629q5uPsjlc+5jPDr98xsZmuglDZoziVm1hKYDZzn7l/FzlMfZjYYWOHuC83soNh5GqgE2Bs4x90XmNlk4GLg8nS9eBTufsiWvmdmo4EHK4v5JTPbRFjspCJb+VK1pfMws15AN+BVM4Nwq2CRmfV190+yGDFlW/tvAmBmpwKDgYOT+I/mVuTVBs1m1ohQ0jPd/cHYeRqgHzDEzH4KNAVam9nd7n5S5Fz1sQxY5u5Vv93MIhR1WiT11sfDwEAAM+sBNCbHVtdy99fcfVt37+ruXQn/IfdOaknXxswOBy4Chrj7N7Hz1FHebNBs4V/96cBSd58UO09DuPsl7t6p8u/HMOB/crSkqfx7/aGZ7VL5pYOBN9L1+tGuqGsxA5hhZq8D64BTc+wKLh/9FmgCzK38DWG+u58ZN1Jq8myD5n7AycBrZra48muXVu5rKnGdA8ysvBh4DzgtXS+sKeQiIgmX1FsfIiJSSUUtIpJwKmoRkYRTUYuIJJyKWkQk4VTUIiIJp6IWEUm4/wMpIgBjPFYodQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(x, cost_1(Z),'b', label='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    " def cost_1(x):\n",
    "    return [max(0, -x_i + 1) for x_i in x]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11e156cd0>]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAZsklEQVR4nO3deZTU1ZnG8e9LsymIGy2riMYlwRVtt2HUqFH2RYIC4oJiGhQR44ILbhhBJWGbxDgSMImGYBARFBHjRI2jozgQFUWM4xZBBVqFcQeBd/64zcSQhq7urqr7q189n3M4It2WTx0Pjz9u3fdec3dERCS56sUOICIi26eiFhFJOBW1iEjCqahFRBJORS0iknD1c/GizZs39/bt2+fipUVEUmnJkiUfuXtpVV/LSVG3b9+exYsX5+KlRURSycz+tq2vaelDRCThVNQiIgmnohYRSTgVtYhIwqmoRUQSrtqiNrMDzOylb/341MwuzUc4ERHJYHueu/8VOAzAzEqA94EHc5xLREQq1XTp42TgLXff5n6/2nKHsWPhxRez/coiIoWtpkU9AJhZ1RfMrNzMFpvZ4oqKihoHWbsW7roLunaFd96p8T8uIpJaGRe1mTUEegH3V/V1d5/q7mXuXlZaWuUU5HbtthssXAgbNkDnzlCLrhcRSaWaPFF3Bf7i7qtzFaZDB3j4YVixArp3hy++yNW/SUSkcNSkqAeyjWWPbOrUCe67D5YsgTPOgG++yfW/UUQk2TIqajPbETgFmJPbOEHv3nDnnbBgAZSXhw8aRUSKVUan57n7l8DuOc7yD8rL4cMP4aaboHXrsCNERKQY5eSY02y54Qb44AMYNw5atYKLL46dSEQk/xJd1GZwxx2wejVccgm0bAn9+sVOJSKSX4k/66N+fZg5E449FgYNgqeeip1IRCS/El/UADvsELbtfec74YPGpUtjJxIRyZ+CKGr4+0DMTjuF6cW/ZX2IXUQkmQqmqAHatQtl/eWX0KULfPxx7EQiIrlXUEUNcNBBMG9eOA+kZ89Q2iIiaVZwRQ1w/PHw+9/D88/DgAGwcWPsRCIiuVOQRQ3Qty/84hfhQ8YLL9T0ooikV6L3UVfnoovCQMzYsWF6ccyY2IlERLKvoIsa4Cc/CaPmN98cpheHDYudSEQkuwq+qM3ChQOrV8Pw4WF6sU+f2KlERLKnYNeov61+ffjDH+DII2HgQHjmmdiJRESyJxVFDdCkCcyfH/Za9+wJy5bFTiQikh2pKWqA5s3hscegceMwELNiRexEIiJ1l6qiBmjfPkwvfvppGDVfuzZ2IhGRukldUQMceijMnQv/8z/hEKevvoqdSESk9lJZ1AAnngj33hs+WBw0CDZtip1IRKR2UlvUEC7HnTwZHnww3A6j6UURKUQZ7aM2s12AacBBgAPnu/tzuQyWLZdcEqYXb78d2rSB666LnUhEpGYyHXiZAix0935m1hDYMYeZsu7WW8P04vXXh4GYCy6InUhEJHPVFrWZNQOOBwYDuPsGYENuY2WXGUybFqYXhw6FFi3CXmsRkUKQyRr1PkAF8Gsze9HMpplZk62/yczKzWyxmS2uqKjIetC6atAAZs+Gww+H/v3huYJYuBERyayo6wOHA3e6e0fgC+Dqrb/J3ae6e5m7l5WWlmY5ZnY0bQqPPBLWqnv0gNdfj51IRKR6mRT1SmCluy+q/PvZhOIuSHvsEaYX69eHzp3DB40iIklWbVG7+ypghZkdUPlLJwOv5TRVju2zDzz6KHzySRg1X7cudiIRkW3LdB/1CGCGmS0FDgPG5S5Sfhx+OMyZE5Y/TjsN1q+PnUhEpGoZFbW7v1S5/nyIu/dx91ScoHHKKfCb38BTT8HZZ2t6UUSSqeAvDqirM8+EVavg8svDHuspU8J2PhGRpCj6oga47DJ4/32YODHsCLnqqtiJRET+TkVd6ac/DdOLV18dnqzPPTd2IhGRQEVdqV69sF5dUQFDhoRtfF27xk4lIpLy0/NqqmFDeOABOOQQ6NcPXnghdiIRERX1P2nWDBYsCOeBdO8Ob7wRO5GIFDsVdRVatgzTixAGYlatiptHRIqbinob9tsvnAuyejV06xbuYBQRiUFFvR1HHRVO3Fu6FPr2hQ0FdbiriKSFiroaXbvC9Onwpz/B4MGweXPsRCJSbLQ9LwPnnhv2WF9zDbRqBRMmxE4kIsVERZ2hq64KR6JOnAitW4eRcxGRfFBRZ8gMJk0KO0CuuCLsDBk0KHYqESkGKuoaKCmBe+4J04uDB0NpKZx6auxUIpJ2+jCxhho3hrlzoUMH+OEPYcmS2IlEJO1U1LWw887hhpjddgt7rN96K3YiEUkzFXUttW4dphc3bgx3L65ZEzuRiKSViroOvvvdML34wQfhyfrzz2MnEpE0UlHX0THHwKxZ8NJL4cS9b76JnUhE0kZFnQU9esBdd4WlkCFDwD12IhFJk4y255nZu8BnwCZgo7uX5TJUIRoyJEwvXn99mF68/fbYiUQkLWqyj/pEd/8oZ0lSYPTosF49fnz4sHHkyNiJRCQNNPCSRWbw85+Ho1F//OMwvdi/f+xUIlLoMl2jduCPZrbEzMqr+gYzKzezxWa2uKKiInsJC0xJCcyYAf/6r3DOOfDEE7ETiUihy7SoO7n74UBXYLiZHb/1N7j7VHcvc/ey0tLSrIYsNI0bw7x54fKBPn3CjhARkdrKqKjd/YPKv64BHgSOymWoNNh1V1i4MEwxdu0K774bO5GIFKpqi9rMmpjZTlt+DpwKvJrrYGnQtm3Ysrd+fZhe/EgfxYpILWTyRN0CeMbMXgZeAB5x94W5jZUeHTrAww/De++F/dZffBE7kYgUmmp3fbj728CheciSWp06wcyZ4bS9/v3D6Xv1td9GRDKkycQ86dMHfvnLcDbI0KGaXhSRzOm5Lo+GDg0DMTffHKYXb7kldiIRKQQq6jy76aZQ1mPHhunFiy6KnUhEkk5FnWdmcOedYXrx4ovD9GLfvrFTiUiSaY06gvr14b77whGpZ54JTz8dO5GIJJmKOpIddwzb9vbeG3r1gldeiZ1IRJJKRR3R7ruH6cUmTcL04nvvxU4kIkmkoo5sr71CWX/+OXTpAp98EjuRiCSNijoBDj44HOL01lvQsyd89VXsRCKSJCrqhDjhhHA86nPPwYAB4XZzERFQUSdKv34wZQo89BAMH67pRREJtI86YUaMCHcv3nprGIi58cbYiUQkNhV1Ao0dG6YXb7opjJqXV3mnjogUCxV1ApnBr34Fa9bAhRdCixbQu3fsVCISi9aoE6pBA7j/fjjiiPDh4rPPxk4kIrGoqBOsSZNwLOqee4Zte6+9FjuRiMSgok640tJwnVejRmEgZuXK2IlEJN9U1AVg771hwQJYty6Mmq9bFzuRiOSTirpAdOwIDz4If/1r+GDx669jJxKRfMm4qM2sxMxeNLP5uQwk23byyXDPPeFY1LPOgk2bYicSkXyoyRP1SGB5roJIZgYMgEmT4IEHYORITS+KFIOMitrM2gLdgWm5jSOZuPRSuPJKuOOOMMEoIumW6cDLZGAUsNO2vsHMyoFygHbt2tU9mWzXbbeFUfPRo8P04nnnxU4kIrlS7RO1mfUA1rj7ku19n7tPdfcydy8rLS3NWkCpWr16MH06nHoq/OhHYb+1iKRTJksfnYBeZvYucB9wkpn9LqepJCMNG8Ls2XDYYXD66bBoUexEIpIL1Ra1u1/j7m3dvT0wAHjC3c/KeTLJyE47hafpVq2ge/ewfU9E0kX7qFOgRYswvVivHnTuHE7eE5H0qFFRu/tT7t4jV2Gk9vbdN0wvfvQRdOsG//u/sROJSLboiTpFysrC/uply+C002D9+tiJRCQbVNQp07kz3H03PPkknHMObN4cO5GI1JUuDkihs8+GVatg1KjwIeOkSeEyAhEpTCrqlLriivCh4uTJ0KZNmGQUkcKkok4pM5gwIUwvjhoFLVuGJ20RKTwq6hSrVw9++1uoqIDzz4c99ghr2CJSWPRhYso1ahTOsT7wQPjhD2Hx4tiJRKSmVNRFoFkzePTRcK1Xt27w5puxE4lITaioi0SrVmF6cfPmsPyxenXsRCKSKRV1Edl//3AuyKpV4cn6s89iJxKRTKioi8zRR8OsWfDyy2HNesOG2IlEpDoq6iLUvTv86lfw+ONhN4imF0WSTdvzitR55/39hpiWLeFnP4udSES2RUVdxK65JkwvTpgArVvDZZfFTiQiVVFRFzEzmDIlfLh4+eVhZ8jAgbFTicjWVNRFrqQEfve7ML147rlhevHkk2OnEpFv04eJQuPGMG8eHHBAOMf6xRdjJxKRb1NRCwC77AILF4a/du0K77wTO5GIbKGilv/Xpk2YXtywIUwvVlTETiQikEFRm1ljM3vBzF42s2VmNiYfwSSO730P5s+HlSvDfuvPP4+dSEQyeaJeD5zk7ocChwFdzOyY3MaSmP7lX+APf4AlS+CMM+Cbb2InEilu1Ra1B1ueqxpU/vCcppLoevaEu+4Kp+796Efg+i8uEk1G2/PMrARYAuwL3OHui6r4nnKgHKBdu3bZzCiRXHBBmF684YYwEDNuXOxEIsUpow8T3X2Tux8GtAWOMrODqvieqe5e5u5lpaWl2c4pkVx3HQwdCrfeCj//eew0IsWpRgMv7r7OzJ4CugCv5iSRJIoZ3HFHOL965MhwLsjpp8dOJVJcMtn1UWpmu1T+fAfgB8DruQ4myVFSAr//ffiQ8ayz4KmnYicSKS6ZLH20Ap40s6XAfwOPu/v83MaSpNlhB3joIdh3X+jdG5YujZ1IpHhUu/Th7kuBjnnIIgm3225hevHYY6FLF3juOdhrr9ipRNJPk4lSI3vuGaYXv/oqTC9+/HHsRCLpp6KWGjvwwLAM8u67Yb/1l1/GTiSSbipqqZXjjoOZM2HRIhgwADZujJ1IJL1U1FJrp50Wtu49/DBceKGmF0VyRRcHSJ0MGxau8/rJT8L04hgd2SWSdSpqqbMxY0JZ33xzuM5r2LDYiUTSRUUtdWYG//7vYXpx+HBo0SIsi4hIdmiNWrKifv1wNOpRR4ULcp95JnYikfRQUUvW7LhjuHSgffuwbW/ZstiJRNJBRS1ZtfvuYXpxhx3C9OKKFbETiRQ+FbVkXfv24cKBTz8NZb12bexEIoVNRS05ceihMHcuvPkm9OoVRs5FpHZU1JIzJ54I994Lzz4LZ54JmzbFTiRSmFTUklNnnAFTpoSn6+HDNb0oUhvaRy05N2JEGIi57bYwvXjDDbETiRQWFbXkxbhxoaxvvDGU9QUXxE4kUjhU1JIXZjBtGqxZEy7L3WOP8CGjiFRPa9SSNw0awP33wxFHQP/+8F//FTuRSGFQUUteNW0KjzwSborp2ROWL4+dSCT5MrmFfE8ze9LMlpvZMjMbmY9gkl6lpWF6sUGDMBDz/vuxE4kkWyZP1BuBy939e8AxwHAz65DbWJJ2++wTphfXroWuXWHdutiJRJKr2qJ29w/d/S+VP/8MWA60yXUwSb+OHWHOHHj9dejTB77+OnYikWSq0Rq1mbUHOgKLqvhauZktNrPFFRUV2UknqfeDH8Bvfwt//jOcfbamF0WqknFRm1lT4AHgUnf/dOuvu/tUdy9z97LS0tJsZpSUGzgQJkyA2bNh5EhNL4psLaN91GbWgFDSM9x9Tm4jSTG67LIwEDNhQhiIufba2IlEkqPaojYzA6YDy919Yu4jSbEaPx4+/BBGjw5lPXhw7EQiyZDJ0kcn4GzgJDN7qfJHtxznkiJUrx78+tdh3fqCC2DBgtiJRJKh2idqd38GsDxkEaFhw7AT5Pvfh9NPhyeegKOPjp1KJC5NJkri7LRTeJpu2RK6d4c33oidSCQuFbUkUosWYXqxXj3o3BlWrYqdSCQeFbUk1n77hXNBKirC9OKn/7QpVKQ4qKgl0Y48MuyvfvVVOO00WL8+diKR/FNRS+J16QLTp4cPFgcPhs2bYycSyS9dHCAF4Zxzwh7rq68OHzJOnBguIxApBipqKRijRoXpxcmTw0DMlVfGTiSSHypqKRhmMGlS2AEyahS0agVnnRU7lUjuqailoNSrB/fcE+5ePO+8cPfiqafGTiWSW/owUQpOo0Ywdy506AB9+8LixbETieSWiloK0s47hxtimjeHbt3gzTdjJxLJHRW1FKzWreGxx8J2vS5dYPXq2IlEckNFLQXtgANg/vywG6R7d/jss9iJRLJPRS0F75hjYNYseOkl6NcPNmyInUgku1TUkgo9esDUqfDHP8KQIZpelHTR9jxJjfPPD9OL110X1q9vvz12IpHsUFFLqlx7bVivHj8+DMRcemnsRCJ1p6KWVDGDf/u3ML344x+Hc0EGDIidSqRutEYtqVNSAjNmwHHHhcOcnngidiKRulFRSyo1bgzz5sH++0OfPmFHiEihqraozexuM1tjZq/mI5BItuy6a7jOa+edww0x77wTO5FI7WTyRP0boEuOc4jkRNu2YXpx/fpw92JFRexEIjVXbVG7+9PAJ3nIIpITHTrAww/DihVhv/UXX8ROJFIzWVujNrNyM1tsZosr9NgiCdOpE8ycGU7aO/10+Oab2IlEMpe1onb3qe5e5u5lpaWl2XpZkazp0wd++ctw6l55ObjHTiSSGe2jlqIydGiYXhwzBtq0gVtuiZ1IpHoqaik6N94YphfHjg3Ti8OHx04ksn2ZbM+bCTwHHGBmK81sSO5jieSOWVgC6dULRoyABx6InUhk+6p9onb3gfkIIpJP9euHDxdPOQXOPDOcunfCCbFTiVRNk4lStHbcMWzb+853oHdveOWV2IlEqqailqK2225herFJk3Cd13vvxU4k8s9U1FL02rULZf3FF2F68eOPYycS+UcqahHg4IPDIU5vvw09e8KXX8ZOJPJ3KmqRSiecEI5Hff55GDgQNm6MnUgkUFGLfEu/fuHigYcegosu0vSiJIMGXkS2cvHFYXpx3Lhw9+JNN8VOJMVORS1ShVtuCdOLY8aE6cWhQ2MnkmKmohapghlMnQpr1oQlkBYtwqFOIjFojVpkGxo0gFmzoKwsfLj47LOxE0mxUlGLbEeTJvDII2Gvdc+e8NprsRNJMVJRi1SjefMwENOoUZheXLkydiIpNipqkQzsvXe4cGDdulDWa9fGTiTFREUtkqHDDoO5c+GNN8IhTl9/HTuRFAsVtUgNnHQS3Hsv/Od/wqBBsGlT7ERSDFTUIjXUvz9Mngxz5sAll2h6UXJP+6hFamHkyDAQM358mF4cPTp2IkkzFbVILd16axg1v+66ML14/vmxE0laqahFaqlePZg+PUwvlpdDaWnYay2SbRmtUZtZFzP7q5m9aWZX5zqUSKFo0ABmz4aOHaFvX7jrrtiJJI0yuYW8BLgD6Ap0AAaaWYdcBxMpFE2bwp/+FC7KHTYsrF/rLGvJpkyWPo4C3nT3twHM7D6gN6BhWpFKzZqFi3KvvBImTQr7rZs2jZ1K8m333eHpp7P/upkUdRtgxbf+fiVw9NbfZGblQDlAu3btshJOpJCUlMDEiXDEEaGopfjssktuXjeTorYqfu2fdo66+1RgKkBZWZl2lkrRGjQo/BDJlkw+TFwJ7Pmtv28LfJCbOCIisrVMivq/gf3MbG8zawgMAB7KbSwREdmi2qUPd99oZhcDjwElwN3uviznyUREBMhw4MXdFwALcpxFRESqoEOZREQSTkUtIpJwKmoRkYRTUYuIJJx5Dk49N7MK4G+1/MebAx9lMU5MaXkvaXkfoPeSRGl5H1C397KXu5dW9YWcFHVdmNlidy+LnSMb0vJe0vI+QO8lidLyPiB370VLHyIiCaeiFhFJuCQW9dTYAbIoLe8lLe8D9F6SKC3vA3L0XhK3Ri0iIv8oiU/UIiLyLSpqEZGES2xRm9mIygt1l5nZ+Nh56sLMrjAzN7PmsbPUlpn91MxeN7OlZvagmeXoLovcSMsFzWa2p5k9aWbLK39vjIydqa7MrMTMXjSz+bGz1IWZ7WJmsyt/nyw3s2Oz9dqJLGozO5FwL+Mh7n4g8LPIkWrNzPYETgHei52ljh4HDnL3Q4A3gGsi58lYyi5o3ghc7u7fA44Bhhfwe9liJLA8dogsmAIsdPfvAoeSxfeUyKIGLgRuc/f1AO6+JnKeupgEjKKK68sKibv/0d233K39POGmn0Lx/xc0u/sGYMsFzQXH3T90979U/vwzQhm0iZuq9sysLdAdmBY7S12YWTPgeGA6gLtvcPd12Xr9pBb1/sBxZrbIzP5sZkfGDlQbZtYLeN/dX46dJcvOBx6NHaIGqrqguWDLbQszaw90BBbFTVInkwkPMptjB6mjfYAK4NeVyzjTzKxJtl48o4sDcsHM/gNoWcWXRhNy7Ur4o92RwCwz28cTuJewmvdxLXBqfhPV3vbei7vPq/ye0YQ/fs/IZ7Y6yuiC5kJiZk2BB4BL3f3T2Hlqw8x6AGvcfYmZfT92njqqDxwOjHD3RWY2BbgauD5bLx6Fu/9gW18zswuBOZXF/IKZbSYcdlKRr3yZ2tb7MLODgb2Bl80MwlLBX8zsKHdflceIGdvefxMAMzsX6AGcnMT/aW5Hqi5oNrMGhJKe4e5zYuepg05ALzPrBjQGmpnZ79z9rMi5amMlsNLdt/zpZjahqLMiqUsfc4GTAMxsf6AhBXa6lru/4u57uHt7d29P+A95eFJLujpm1gW4Cujl7l/GzlNDqbmg2cL/9acDy919Yuw8deHu17h728rfHwOAJwq0pKn8fb3CzA6o/KWTgdey9frRnqircTdwt5m9CmwAzi2wJ7g0+gXQCHi88k8Iz7v7sLiRMpOyC5o7AWcDr5jZS5W/dm3lvaYS1whgRuXDwNvAedl6YY2Qi4gkXFKXPkREpJKKWkQk4VTUIiIJp6IWEUk4FbWISMKpqEVEEk5FLSKScP8HJjPbKWy3yk4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#<div class=\"pirk\">\n",
    "plt.plot(x, cost_1(Z),'b', label='left')\n",
    "#</div><i class=\"fa fa-lightbulb-o \">\n",
    "#</i>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "cost_0 = right_sig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11e2349d0>]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAZ6UlEQVR4nO3deZSU1Z3G8e/PpgERcG0VFSRoxBAdo7YY48REHXHHfSGKYlSEEUPiLi4R9XjAuKHiBAQcIrgFVMSIwqjBHW0WjYgoEUYWOTRRXAAFun/zx+0eELG7urveum9VPZ9z3lNFd1v9FMjD2/d9773m7oiISHptFjuAiIjUTUUtIpJyKmoRkZRTUYuIpJyKWkQk5Zol8aLbbbedd+zYMYmXFhEpSNOnT1/u7mWb+lwiRd2xY0cqKiqSeGkRkYJkZv/7Q5/T0IeISMqpqEVEUk5FLSKScipqEZGUU1GLiKRcRkVtZluZ2Tgz+8DM5pjZQUkHExGRINPb84YAz7n7qWbWHGiVYCYREdlAvWfUZtYWOAQYCeDua9x9RdLBRETyySuvwN13QxIrR2cy9NEJqAQeNLOZZjbCzLbY+IvMrLeZVZhZRWVlZdaDioik1dKlcPrpcP/9sGpV9l8/k6JuBuwH/Je77wusBK7e+Ivcfbi7l7t7eVnZJmdBiogUnHXr4Iwz4IsvYPx42OJ7p7FNl0lRLwIWufu0ml+PIxS3iEjRu/ZaePllGD4c9t47me9Rb1G7+1JgoZl1rvnQ4cD7ycQREckfTz0Ft90GffrA2Wcn930yvevjEmBszR0fHwPnJRdJRCT95s2Dc8+F8vJwETFJGRW1u88CypONIiKSH1atglNOgWbNYNw4aNEi2e+XyDKnIiKFyh3+8z/hH/+AZ5+FXXdN/ntqCrmISAOMGAGjR8MNN8BRR+Xme6qoRUQyVFEB/fpBt25w/fW5+74qahGRDHz2GZx6KuywA4wdCyUlufveGqMWEalHdTX07AlLlsCrr8J22+X2+6uoRUTqceut4cLh0KHQtWvuv7+GPkRE6jBlSrhw+JvfQN++cTKoqEVEfsAnn0CPHtClS5gibhYnh4paRGQTvv0WTjsN1qxJbrGlTGmMWkRkEy67DN56K8w87Ny5/q9Pks6oRUQ28vDD4cLhpZeGqeKxqahFRDYwezZceCH88pcwaFDsNIGKWkSkxpdfwsknQ5s28NhjUFoaO1GgMWoREcJiS+efD//8J7zwArRrFzvReipqERHCmtLjxsHgwfCrX8VO810a+hCRovfKK3DFFXDSSeExbVTUIlLUli4Nm9N26gQPPhhvUktdNPQhIkVr3To480xYsQKefx623DJ2ok1TUYtI0RowAKZOhb/8JbkdxLNBQx8iUpSeeAL+9Kew0FLPnrHT1E1FLSJF58MPoVcvOOAAuOuu2Gnqp6IWkaKyalXYqaV589zsIJ4NGqMWkaLhDhddBO+9B5MmQYcOsRNlRkUtIkXjz3+GMWNg4EA48sjYaTKXUVGb2QLgK6AKWOfu5UmGEhHJtrfegv794eij4brrYqdpmIacUR/q7ssTSyIikpDly8O49E47hTPqzfLs6pyGPkSkoFVVhf0Oly2D116DbbaJnajhMv13xYHJZjbdzHpv6gvMrLeZVZhZRWVlZfYSiog0wcCBYYPa++6D/fePnaZxMi3qg919P+Bo4GIzO2TjL3D34e5e7u7lZWVlWQ0pItIYf/sb3HwznHdeWMI0X2VU1O6+pOZxGfAk0DXJUCIiTTV/Ppx9NvzsZ2FbrTQutpSpeovazLYwsza1z4FuwHtJBxMRaazVq9fvdTh+PGy+edw8TZXJxcQdgCct/HPUDHjY3Z9LNJWISBNccgnMnAkTJ4blS/NdvUXt7h8D++Qgi4hIk40cGY7rroPjjoudJjvy7G5CEZEfNmMGXHwxHHEE3Hhj7DTZo6IWkYLw2WdhXHr77WHsWCgpiZ0oezThRUTyXnV1WFN68WJ49VUotDuEVdQikvduuQWefRbuvx+6FuDNwxr6EJG89vzzYTy6Z0/o0yd2mmSoqEUkby1YENbx2GuvsIRpPk9qqYuKWkTy0jffhBXxqqrC/oetWsVOlByNUYtIXvrd72D6dJgwAXbfPXaaZOmMWkTyzoMPwgMPwDXXQPfusdMkT0UtInllxgzo2xcOPzysjFcMVNQikjdqJ7WUlcEjjxTWpJa6aIxaRPLChpNaXnml8Ca11EVFLSJ5YcNJLQceGDtNbmnoQ0RSb9Kkwp/UUhcVtYik2vz5cNZZsPfehT2ppS4qahFJrdWrw6SW6urCn9RSF41Ri0gquUO/fuF2vIkTYbfdYieKR2fUIpJKI0bAqFGFtVNLY6moRSR13n47nE1361ZYO7U0lopaRFJl+fIwqaVdO3j44eKZ1FIXjVGLSGpUVYVlS5ctCzu1bLtt7ETpoKIWkdS44QaYMiUsuFReHjtNemjoQ0RSYcIEuPVWuOCCcMh6KmoRie6jj+Ccc8JZ9L33xk6TPhkXtZmVmNlMM3smyUAiUlxWroSTT4bSUhg3Dlq2jJ0ofRoyRt0fmAO0TSiLiBQZ9zDM8f778NxzsOuusROlU0Zn1Ga2C3AsMCLZOCJSTIYMgUcfDSvjHXFE7DTplenQx93AlUD1D32BmfU2swozq6isrMxKOBEpXC+/DJdfDieeCFdfHTtNutVb1GZ2HLDM3afX9XXuPtzdy929vKyYVvQWkQZbsgROPz2s3zF6dHGuiNcQmYxRHwx0N7NjgJZAWzMb4+5nJxtNRArRmjVw2mnw9dfwwgvQVle96lXvGbW7X+Puu7h7R+BM4EWVtIg01mWXweuvh53Ef/rT2Gnyg+6jFpGcGTMG7rsvlPVpp8VOkz8aNIXc3f8O/D2RJCJS0GbNgt694de/hkGDYqfJLzqjFpHEffZZmNSyzTbw2GPQTKsMNYh+u0QkUVVVYc/DxYvDLXnbbx87Uf5RUYtIogYODLMOhw2DAw+MnSY/aehDRBLz9NNw883w29/ChRfGTpO/VNQikoi5c6Fnz7Ai3tChmtTSFCpqEcm6r74KFw+bN4fx47UiXlNpjFpEsso9DHV88EHYraVDh9iJ8p+KWkSy6rbbwrrSt90Ghx0WO01h0NCHiGTNlCkwYEBYcOnyy2OnKRwqahHJigUL4MwzoUsXGDlSFw+zSUUtIk22alW4eFhVBU8+Ca1bx05UWDRGLSJN4g4XXRTW8pg4EXbfPXaiwqOiFpEmuffesCreTTfBscfGTlOYNPQhIo02dSpceimccAJce23sNIVLRS0ijbJwYVhTevfd4S9/gc3UJonRb62INNjq1eHi4TffwFNPaTutpGmMWkQaxB369oWKCpgwAfbcM3aiwqczahFpkKFDw87hf/wjdO8eO01xUFGLSMZefhn+8Ac4/ni44YbYaYqHilpEMrJwIZx6Kuy2Gzz0kC4e5pJ+q0WkXqtXw0knrb94uOWWsRMVF11MFJE6uUOfPjB9ui4exqIzahGp0z33hPukb7xRFw9jUVGLyA968UW47LIw8/D662OnKV71FrWZtTSzt8zsHTObbWYDcxFMROKaPz+sK925sy4expbJGPW3wGHu/rWZlQKvmtkkd38z4WwiEsnKleHiYVVVuHjYpk3sRMWt3qJ2dwe+rvllac3hSYYSkXhq9zx891149ln48Y9jJ5KMfpgxsxIzmwUsA6a4+7RNfE1vM6sws4rKysps5xSRHBk0CB5/PDwedVTsNAIZFrW7V7n7z4BdgK5mttcmvma4u5e7e3lZWVm2c4pIDjzzTFiutEcPuOKK2GmkVoMuD7j7CuDvgP6dFSkwH3wAv/kN7LsvjBihPQ/TJJO7PsrMbKua55sD/wF8kHQwEcmdzz8P90i3bBn2PGzVKnYi2VAmd320A0abWQmh2B9392eSjSUiuVJVFYY6FiwI90136BA7kWwsk7s+3gX2zUEWEYngqqvg+efhgQfg3/89dhrZFN3CLlLEHnoI7rgD+vWDCy6InUZ+iIpapEi9+WYo58MOgzvvjJ1G6qKiFilCixbBiSdC+/bhnunS0tiJpC5a5lSkyKxaFRZZWrUqXDzcdtvYiaQ+KmqRIlI7PXzmTJg4Ebp0iZ1IMqGiFikit9wCjz0GgwfDscfGTiOZ0hi1SJEYPz5sSHvOOZoenm9U1CJFYOZM6NkTDjoIhg3T9PB8o6IWKXCffhqmh2+3XZge3rJl7ETSUBqjFilgq1eHOzw+/xxeew122CF2ImkMFbVIgaquhl69oKIi7NKyzz6xE0ljqahFCtRNN4XJLIMHa/fwfKcxapEC9PDDMHBgOKPWHR75T0UtUmBefz1MajnkEN3hUShU1CIFZP789Wt4PPEENG8eO5Fkg4papEB88QUcfzysXRv2PtQaHoVDFxNFCsDatXDaaTB3btgEoHPn2Ikkm1TUInnOHS6+GKZMgVGjwvrSUlg09CGS526/PWyjNWAAnHde7DSSBBW1SB7761/hyivh9NPh5ptjp5GkqKhF8tTrr4eFln7xCxg9GjbT3+aCpT9akTw0b15Yw6N9e5gwQQstFToVtUieWb4cjjkmXEScNCmsiieFTXd9iOSR1avDuh0LF8ILL8Duu8dOJLlQ7xm1mbU3s5fMbI6ZzTaz/rkIJiLfVVUFZ50Fb74JY8eGsWkpDpmcUa8DLnP3GWbWBphuZlPc/f2Es4lIDXe49NKw8P/dd8PJJ8dOJLlU7xm1u3/q7jNqnn8FzAF2TjqYiKx3++1wzz3whz9Af/1MW3QadDHRzDoC+wLTNvG53mZWYWYVlZWV2UknIowZE+6VPuOMUNhSfDIuajNrDYwHfu/uX278eXcf7u7l7l5eVlaWzYwiRWvy5DDb8NBDda90Mcvoj93MSgklPdbdn0g2kogATJ8Op5wCXbqEsekWLWInklgyuevDgJHAHHe/M/lIIvLRR3D00WGp0kmTYMstYyeSmDI5oz4Y6AkcZmazao5jEs4lUrSWLIFu3cKdHpMnw047xU4ksdV7e567vwpoMx+RHFixIpxJV1bCSy/BHnvETiRpoJmJIimxahUcdxzMmQN/+xsccEDsRJIWKmqRFFizBk49NayI99hjcMQRsRNJmqioRSKrqoJzzw0XDYcPD1tqiWxId2WKRFS7jdajj8KgQXDhhbETSRqpqEUicYcrroBhw+Dqq+Gqq2InkrRSUYtEctNNcMcd0K8f3Hpr7DSSZipqkQhuvx1uvBF69YIhQ8B0A6zUQUUtkmP33BOGPM44A0aM0PodUj/9LyKSQ8OGhWVKTzoJHnoISkpiJ5J8oKIWyZFRo6BPnzCp5dFHobQ0diLJFypqkRwYNQouuACOPBL++ldo3jx2IsknKmqRhNWWdLdu8NRT0LJl7ESSb1TUIglSSUs2qKhFEvLnP8P556ukpelU1CIJGDIE+vaF44+HCRNU0tI0KmqRLBs8GH7/+7CN1rhx2kJLmk5FLZIl7jBgQFi3o0cPeOQR3d0h2aFlTkWyoLoafvc7GDoUeveG++/XZBbJHp1RizTR2rVhPemhQ+Hyy8NFRJW0ZJPOqEWaYOXKsND/pElwyy1h6EMLLEm2qahFGulf/wrTwd96K+zMokX/JSkqapFGmD8fjjkmPI4bFxZZEkmKilqkgSoqwpn0mjUweTIcckjsRFLodDFRpAGeeQZ+9SvYfHN47TWVtORGvUVtZqPMbJmZvZeLQCJp5A533QUnnAA/+Qm88UZ4FMmFTM6o/xs4KuEcIqm1di1cdBFcemko6qlTYccdY6eSYlJvUbv7y8BnOcgikjrLl4c1pB94AK65Jlw43GKL2Kmk2GTtYqKZ9QZ6A3To0CFbLysSzaxZcOKJsHQpjB4N55wTO5EUq6xdTHT34e5e7u7lZWVl2XpZkSgefRR+8QtYtw5eflklLXHprg+RDaxZE1a+69ED9t8/3IrXtWvsVFLsVNQiNRYuDLfeDRkSyvqFF3TRUNIhk9vzHgHeADqb2SIzOz/5WCK59cwzsO++MHt22Hz2rru0RKmkR70XE929Ry6CiMTwzTdw1VVwzz2wzz7w+OOwxx6xU4l8l4Y+pGjNng0//3ko6f794c03VdKSTipqKTpVVXDHHeFi4ZIl8PTTcPfd2tdQ0kuLMklR+ec/w87gU6dC9+5hIsv228dOJVI3nVFLUaiqgjvvhL33hhkzYNQoeOoplbTkB51RS8F7992wj+G0aXDssWGrrF12iZ1KJHM6o5aC9dVXYSGl/fYLQx4PPwwTJ6qkJf+oqKXguIdS3nPPcJHwggtg7tww21D7GUo+UlFLQZk2LazRcdZZYVbh66+HoY5ttomdTKTxVNRSEObNgzPPDPdFL1gADz4Ib78dfi2S71TUktcWL4a+fcNuKxMnwnXXwYcfQq9esJn+75YCobs+JC8tXAiDB4f7oKurww4s112nRZSkMKmoJa/MnRtmFY4eHQq6Vy8YMAB+9KPYyUSSo6KW1HMPO37fcQdMmBBWtTvvPLj6aujYMXY6keSpqCW1Vq8Ot9ndd1/YFmvrreHaa6FfP9hhh9jpRHJHRS2pM3MmjBwJY8fCihWw114wbFi45U4by0oxUlFLKixZAo88AmPGhLPnFi3glFPC1O9DDtFEFSluKmqJZulSePJJGDcOXnopjEUfcADce284e95669gJRdJBRS054w5z5oRtryZODBcI3aFzZ7j++lDOWrhf5PtU1JKozz+HF1+EKVNg8mSYPz98fJ994I9/hFNPhS5dNLQhUhcVtWTVsmVhfY2pU8Mxa1Y4a27dGg49FK68Miw12r597KQi+UNFLY329dehiCsqwvHGG/Dxx+FzLVvCQQeFs+bDD4cDD4TS0rh5RfKVilrqtW5dWM/5/ffDhrDvvgvvvAMffRTOlgF22iksgNS3byjo8vJw54aINJ2KWgBYsyasn/Hxx+GYNy8U8YcfhpJes2b913bqFMaYe/QIhbz//tCuXbzsIoVORV3gqqvDpJGlS8Px6afhnuXFi8OxaBF88kn4eO3ZMYSz4d12C3dkHH98uODXpUtYjL9Nm3jvR6QYZVTUZnYUMAQoAUa4+6BEU8l3VFXBypXw5Zdhe6kvvwzl+8UX4fj8c/jss/D4r3+tPyorw1FV9f3XbN0adt45HEceCbvuCh06hLPlTp3CUEZJSe7fq4h8X71FbWYlwFDgCGAR8LaZPe3u7ycdLmnu4aiuDmW28bFu3fcfNzzWrl3/uHZtGB6ofax9/u233z+++SY8rl4dnq9eDatWrT9Wrlz/+PXX4fP1KS0NE0S23TYcnTqFC3jbbw9lZWH5zx13DEMU7dpB27bJ//6KSHZkckbdFZjn7h8DmNmjwAlA1ot6//1DQdUWaKZHdXU4ap9v6mMbHlVV65/nmlm4I6JFC9h88+8erVqFLaPatw9rWrRqFYYZWrcOR9u24WjTBrbaCrbcMhxbbx2+VvciixSmTIp6Z2DhBr9eBBy48ReZWW+gN0CHDh0aFaZLl3Cmadawo6Rk/fPNNgvHxs9LSr77vPbXGz5u/LxZs/WPGz4vLQ3PS0vDUfux5s3XP254tGgRjtrPq1BFpCEyKepN1Yp/7wPuw4HhAOXl5d/7fCYeeqgx/5WISGHLZFe5RcCG88h2AZYkE0dERDaWSVG/DfzYzH5kZs2BM4Gnk40lIiK16h36cPd1ZtYPeJ5we94od5+deDIREQEyvI/a3Z8Fnk04i4iIbEImQx8iIhKRilpEJOVU1CIiKaeiFhFJOXNv1NyUul/UrBL430b+59sBy7MYJ6ZCeS+F8j5A7yWNCuV9QNPey67uXrapTyRS1E1hZhXuXh47RzYUynsplPcBei9pVCjvA5J7Lxr6EBFJORW1iEjKpbGoh8cOkEWF8l4K5X2A3ksaFcr7gITeS+rGqEVE5LvSeEYtIiIbUFGLiKRcaovazC4xs7lmNtvMboudpynM7HIzczPbLnaWxjKzP5nZB2b2rpk9aWZbxc7UEGZ2VM3/T/PM7OrYeRrLzNqb2UtmNqfm70b/2JmaysxKzGymmT0TO0tTmNlWZjau5u/JHDM7KFuvncqiNrNDCfsy/pu7/xS4PXKkRjOz9oSNgT+JnaWJpgB7ufu/AR8C10TOk7ENNmg+GugC9DCzLnFTNdo64DJ3/wnwc+DiPH4vtfoDc2KHyIIhwHPuviewD1l8T6ksaqAvMMjdvwVw92WR8zTFXcCVbGL7snzi7pPdfV3NL98k7PSTL/5/g2Z3XwPUbtCcd9z9U3efUfP8K0IZ7Bw3VeOZ2S7AscCI2FmawszaAocAIwHcfY27r8jW66e1qPcAfmlm08xsqpkdEDtQY5hZd2Cxu78TO0uW/RaYFDtEA2xqg+a8LbdaZtYR2BeYFjdJk9xNOJGpjh2kiToBlcCDNcM4I8xsi2y9eEYbByTBzP4H2HETn7qWkGtrwo92BwCPm1knT+G9hPW8jwFAt9wmary63ou7T6j5mmsJP36PzWW2Jspog+Z8YmatgfHA7939y9h5GsPMjgOWuft0M/t17DxN1AzYD7jE3aeZ2RDgauD6bL14FO7+Hz/0OTPrCzxRU8xvmVk1YbGTylzly9QPvQ8z2xv4EfCOmUEYKphhZl3dfWkOI2asrj8TADM7FzgOODyN/2jWoaA2aDazUkJJj3X3J2LnaYKDge5mdgzQEmhrZmPc/ezIuRpjEbDI3Wt/uhlHKOqsSOvQx1PAYQBmtgfQnDxbXcvd/+Hu27t7R3fvSPiD3C+tJV0fMzsKuAro7u6rYudpoILZoNnCv/ojgTnufmfsPE3h7te4+y41fz/OBF7M05Km5u/1QjPrXPOhw4H3s/X60c6o6zEKGGVm7wFrgHPz7AyuEN0HtACm1PyE8Ka794kbKTMFtkHzwUBP4B9mNqvmYwNq9jWVuC4BxtacDHwMnJetF9YUchGRlEvr0IeIiNRQUYuIpJyKWkQk5VTUIiIpp6IWEUk5FbWISMqpqEVEUu7/ADyxmQLo1DkeAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(x, cost_0(Z),'b', label='right')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "def cost_0(x):\n",
    "    return [max(0, x_i - 1) for x_i in x]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11e3777d0>]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAWmElEQVR4nO3deZBU1dnH8e8jsoioaMClQBxNFI1bIANxCSa4By2i0ZRSpaYUGAUEFARZRNxQVAIKKIugkhKTaMQluCCo4FKROCiCikbcIi8uEyMuyD7P+8cZ3tdlmOmZ6dvndvfvUzXFQDe3f7eoeXj69FnM3RERkfTaLnYAERGpmQq1iEjKqVCLiKScCrWISMqpUIuIpNz2SVy0VatWXlJSksSlRUQK0pIlS/7j7q2reyyRQl1SUkJ5eXkSlxYRKUhm9sG2HtPQh4hIyqlQi4iknAq1iEjKqVCLiKScCrWISMplNOvDzN4HvgK2AJvdvTTJUCIi8v/qMj2vq7v/J7EkIiJSLQ19iIhkwdNPw8SJsGVL9q+daaF24EkzW2JmZdU9wczKzKzczMorKiqyl1BEJOW++QZ69YLJk2HTpuxfP9Ohj6PdfbWZ7Q7MN7M33f3Zbz/B3acD0wFKS0t1GoGIFI3Ro+G992DhQmjWLPvXz6ijdvfVVb9+CjwIdM5+FBGR/LNkCYwfD2Vl8KtfJfMatRZqM9vRzHba+j1wIvBaMnFERPLHpk1hyGOPPeDGG5N7nUyGPvYAHjSzrc+/192fSC6SiEh+GD8eli6FOXOgZcvkXqfWQu3u7wKHJxdBRCT/vP02XHUV/O53cPrpyb6WpueJiNRRZSX07g1Nm4aZHklLZD9qEZFCduedsGgR3HEH7LVX8q+njlpEpA5Wr4bLLoNf/xp69szNa6pQi4jUQf/+sH49TJ8OYY5F8jT0ISKSoTlzwtf118P+++fuddVRi4hkYM0auPhi+NnPwtBHLqmjFhHJwNCh8Mkn8Pe/Q+PGuX1tddQiIrVYuDDM8Bg0CH7+89y/vgq1iEgN1q0L+3jstx9cfXWcDBr6EBGpwbXXhlWI8+dD8+ZxMqijFhHZhldfhZtugvPPh+OPj5dDhVpEpBqbN4cFLa1awbhxcbNo6ENEpBq33hr2mr7vPthtt7hZ1FGLiHzPu+/CqFHQvTuceWbsNCrUIiLf4Q4XXgjbbw+33Za7ZeI10dCHiMi3zJoFCxbAlCnQtm3sNIE6ahGRKp98Eha1/PKXYe50WqhQi4hUGTAA1q4NqxC3S1F1TFEUEZF4HnkkzPC44go48MDYab5LhVpEit6XX0LfvnDIIXD55bHT/JA+TBSRojd8eDi55YEHoEmT2Gl+SB21iBS1F16A228P49O/+EXsNNVToRaRorVhA/TqBfvsA9ddFzvNtmnoQ0SK1pgx8Oab8Pjj0KJF7DTbpo5aRIrS8uVwww1wzjlw8smx09RMhVpEis6WLdC7N7RsCRMmxE5TOw19iEjRue02WLwYZs8O25imnTpqESkqH3wAI0bAb34DPXrETpMZFWoRKRpbd8aDsOlSGnbGy4SGPkSkaNx7L8ybFw4F2Gef2Gkyp45aRIpCRQUMHAhHHAH9+sVOUzcZF2oza2Rmr5jZ3CQDiYgkYdCgsKfHjBnQqFHsNHVTl456ILAiqSAiIkl54gm4556wp8fBB8dOU3cZFWozawucAsxINo6ISHZ9/XX4APGgg8Jsj3yU6YeJtwBDgZ229QQzKwPKANq1a9fwZCIiWXDFFfDhh/D889C0aew09VNrR21mpwKfuvuSmp7n7tPdvdTdS1u3bp21gCIi9bV4MUycGPaaPuqo2GnqL5Ohj6OB7mb2PvAX4FgzuyfRVCIiDbRxY9gZr00buP762GkaptZC7e7D3b2tu5cAZwNPu/s5iScTEWmAG2+E114LC1t23jl2mobRPGoRKTgrVoT9pc86C049NXaahqvTykR3XwgsTCSJiEgWVFaGnfFatAjj04VAS8hFpKBMnRqO17r7bth999hpskNDHyJSMFatgmHD4IQT4LzzYqfJHhVqESkI7tCnTzgUYNq0/NkZLxMa+hCRgnDffTB3LowbB/vuGztNdqmjFpG899ln0L8/lJaGHfIKjTpqEcl7l10Gn38O8+fD9gVY1dRRi0heW7AgzPAYOhQOPzx2mmSoUItI3vrmGygrgwMOgFGjYqdJTgG+SRCRYjF6NLz3HixaBM2axU6THHXUIpKXliyB8ePDXtPHHBM7TbJUqEUk72zaBD17wh57hM2XCp2GPkQk7/zxj/Dqq/Dgg7DLLrHTJE8dtYjklbffhquugjPOgNNOi50mN1SoRSRvbN0Zb4cdYNKk2GlyR0MfIpI3Zs4MMzxmzIC99oqdJnfUUYtIXli9GoYMga5d4YILYqfJLRVqEckL/fvDhg2FtzNeJjT0ISKpN2dO+Bo7FvbfP3aa3FNHLSKptmYN9OsHHTrA4MGx08ShjlpEUm3oUKiogEcfLcyd8TKhjlpEUmvhQrjjDhg0CDp2jJ0mHhVqEUmldevCnOn99gsLXIpZkb6REJG0u+YaWLky7DfdvHnsNHGpoxaR1Fm6FG6+OcyXPu642GniU6EWkVTZvBl69YJWrUKxFg19iEjK3HJL2Gv6vvtgt91ip0kHddQikhrvvANXXgndu8OZZ8ZOkx4q1CKSCu7htJbGjeH224tvmXhNNPQhIqkwaxY89RRMmQJt2sROky7qqEUkuk8+CYtaunQJp4rLd9VaqM2smZn908xeNbPXzezqXAQTkeIxYACsXRtWIW6n9vEHMhn62AAc6+5fm1lj4Hkze9zdX0w4m4gUgUceCTM8rr0W2rePnSadai3U7u7A11W/bVz15UmGEpHi8OWX0LcvHHpo2HxJqpfRmwwza2RmS4FPgfnuvria55SZWbmZlVdUVGQ7p4gUoOHD4aOPwtFaTZrETpNeGRVqd9/i7j8D2gKdzeyQap4z3d1L3b20devW2c4pIgXmhRfCNLyBA6Fz59hp0q1Ow/buvgZYCJycSBoRKQrr14dl4iUlYWxaapbJrI/WZtay6vsdgOOBN5MOJiKF6/rr4c03YepU2HHH2GnSL5NZH3sBs8ysEaGw3+fuc5ONJSKFavlyuOEGOPdcOOmk2GnyQyazPpYBHXKQRUQK3JYt4TCAli1h/PjYafKHlpCLSM7cdhssXgyzZ4dtTCUzWgMkIjnxwQcwYgR06wY9esROk19UqEUkcVt3xoOw6ZJ2xqsbDX2ISOLuvRfmzYOJE6Fdu9hp8o86ahFJVEVFWNRyxBFhubjUnQq1iCTq0kvDnh4zZkCjRrHT5CcVahFJzOOPhxkew4fDwQfHTpO/VKhFJBFffw0XXQQHHRRme0j96cNEEUnEyJHw4Yfw/PPQtGnsNPlNHbWIZN2LL8KkSdCvHxx1VOw0+U+FWkSyauPGsDNemzZh8yVpOA19iEhW3XgjvP46zJ0LO+0UO01hUEctIlmzYgVcd11YIn7KKbHTFA4VahHJisrKsDNeixZwyy2x0xQWDX2ISFZMnRqO15o1C3bfPXaawqKOWkQabNUqGDYMTjghHAgg2aVCLSIN4g59+oRDAaZN0854SdDQh4g0yP33hxke48fDvvvGTlOY1FGLSL199hn07w+dOsGAAbHTFC511CJSb5ddBv/9Lzz5pHbGS5I6ahGplwUL4O67YehQOPzw2GkKmwq1iNTZ2rVQVgYHHACjRsVOU/g09CEidTZ6NLz3HixaBM2axU5T+NRRi0idlJfDhAnhsNpjjomdpjioUItIxjZtgp49YY89wuZLkhsa+hCRjI0bB8uWwZw5sMsusdMUD3XUIpKRf/0Lrr4azjgDTj89dpriokItIrWqrAyzPHbYASZPjp2m+GjoQ0RqNXNmmOExYwbsuWfsNMVHHbWI1Gj1ahgyBLp2hQsuiJ2mOKlQi0iNLr4YNmzQzngx1VqozWxvM3vGzFaY2etmNjAXwUQkvjlz4MEH4aqrYP/9Y6cpXpmMUW8GBrv7y2a2E7DEzOa7+xsJZxORiNasgX79oEMHGDw4dpriVmuhdvePgI+qvv/KzFYAbQAVapECNnQoVFTAo4/C9pp2EFWdxqjNrAToACyu5rEyMys3s/KKiorspBORKBYuhDvugEGDoGPH2GnE3D2zJ5q1ABYBY9x9Tk3PLS0t9fLy8izEE5FcW7cODjsszJ1evhyaN4+dqDiY2RJ3L63usYze0JhZY+ABYHZtRVpE8ts118DKlfDUUyrSaZHJrA8DZgIr3H188pFEJJZXXoGbbw7zpY89NnYa2SqTMeqjgXOBY81sadVXt4RziUiObd4MvXpBq1Zh8yVJj0xmfTwPaJq7SIGbMAFefjmcKr7rrrHTyLdpZaKIsHIlXHkldO8edseTdFGhFily7uG0liZN4PbbtUw8jTSNXaTI3XUXPP00TJkCbdrETiPVUUctUsQ+/jgsD+/SJew3LemkQi1SxAYMgG++CasQt1M1SC3904gUqYcfDjM8Ro2C9u1jp5GaqFCLFKEvvoC+feHQQ8PmS5Ju+jBRpAgNGwYffRT2mm7SJHYaqY06apEi89xzMHUqDBwInTvHTiOZUKEWKSLr10Pv3rDPPnDttbHTSKY09CFSRMaMgbfegieegBYtYqeRTKmjFikSy5bB2LFw7rlw0kmx00hdqFCLFIEtW8LOeC1bwnhtVpx3NPQhUgQmTYKXXoJ77w3bmEp+UUctUuDefx9GjoRu3eDss2OnkfpQoRYpYO5w0UVhefiUKdoZL19p6EOkgN1zD8ybF4Y+2rWLnUbqSx21SIH69FO45BI48kjo0yd2GmkIFWqRAnXppfDVV2FnvEaNYqeRhlChFilAjz0WZngMHw4HHxw7jTSUCrVIgfnqq/AB4kEHwYgRsdNINujDRJECM3IkrFoFL7wATZvGTiPZoI5apID84x8weTL06xc+RJTCoEItUiA2bgzLxNu2heuvj51GsklDHyIF4oYb4I03YO5c2Gmn2Gkkm9RRixSAN94IW5j26AGnnBI7jWSbCrVInqusDEMeO+8Mt94aO40kQUMfInluypTwIeKf/gStW8dOI0lQRy2Sx/7973BQ7YknwjnnxE4jSVGhFslT7mEPj8pKmDZNO+MVsloLtZndaWafmtlruQgkIpn561/DUvExY6CkJHYaSVImHfXdwMkJ5xCROvjsMxgwADp1gv79Y6eRpNX6YaK7P2tmJclHEZFMDR4Mn38OCxZoZ7xikLUxajMrM7NyMyuvqKjI1mVF5HuefBJmzYLLL4fDDoudRnLB3L32J4WOeq67H5LJRUtLS728vLxhyUTkB9auhUMOCZstLV0KzZrFTiTZYmZL3L20usc0j1okj1x5ZTis9tlnVaSLiabnieSJl16CW24Je0136RI7jeRSJtPz/gz8A2hvZqvMrGfysUTk2zZtCsvE99wTxo6NnUZyLZNZHz1yEUREtu3mm2HZMnjoIdhll9hpJNc09CGScm+9BddcA7//Pfz2t7HTSAwq1CIpVlkJZWWwww4wcWLsNBKLZn2IpNiMGWGGx8yZYXxaipM6apGUWr0ahgyBY4+F88+PnUZiUqEWSSH3cEDtxo3aGU809CGSSnPmhBkeN90EP/lJ7DQSmzpqkZT5/HO4+GLo0AEuvTR2GkkDddQiKTNkCFRUwKOPwvb6CRXUUYukyjPPhBkegwdDx46x00haqFCLpMS6ddC7N/z4xzB6dOw0kiZ6YyWSEldfDe+8A08/Dc2bx04jaaKOWiQFXn4Zxo2Dnj2ha9fYaSRtVKhFItu8OeyM17p12HxJ5Ps09CES2YQJ8MorcP/9sOuusdNIGqmjFolo5cpwastpp8EZZ8ROI2mlQi0SiTtceCE0aQKTJ2uZuGybhj5EIrnrrjDDY9o0aNMmdhpJM3XUIhF8/HFY1HLMMeGDRJGaqFCLRDBgQFjgMn06bKefQqmFhj5Ecuzhh8MMjzFjoH372GkkH+j/cpEc+uIL6NsXDjssbL4kkgl11CI5NGxYGJ9+6CFo3Dh2GskX6qhFcuS552DqVLjkEujUKXYayScq1CI5sH592BmvpASuuSZ2Gsk3GvoQyYExY+Ctt2DePNhxx9hpJN+ooxZJ2LJlMHYsnHcenHhi7DSSj1SoRRK0ZUtY0LLrrjB+fOw0kq809CGSoEmT4KWX4M9/hh/9KHYayVfqqEUS8v77MHIkdOsGZ50VO43kMxVqkQS4w0UXheXhU6ZoZzxpGA19iCRg9uwww2PSJGjXLnYayXcZddRmdrKZvWVmK81sWNKhRPJZRUVY1HLkkWG5uEhD1VqozawRcBvwG+CnQA8z+2nSwUTyzccfw003hQL95ZcwY4Z2xpPsyGToozOw0t3fBTCzvwC/Bd7IdpjS0rD1o0i+cYe33w4H1XbpAhMnwk/VzkiWZFKo2wAffuv3q4BffP9JZlYGlAG0q+eg3IEHwoYN9fqrItF17w7nn6+tSyX7MinU1X1e7T/4A/fpwHSA0tLSHzyeiXvuqc/fEhEpbJmMoK0C9v7W79sCq5OJIyIi35dJoX4J2N/M9jWzJsDZwCPJxhIRka1qHfpw981mdjEwD2gE3OnuryeeTEREgAwXvLj7Y8BjCWcREZFqaJaniEjKqVCLiKScCrWISMqpUIuIpJy512ttSs0XNasAPqjnX28F/CeLcWIqlHsplPsA3UsaFcp9QMPuZR93b13dA4kU6oYws3J3L42dIxsK5V4K5T5A95JGhXIfkNy9aOhDRCTlVKhFRFIujYV6euwAWVQo91Io9wG6lzQqlPuAhO4ldWPUIiLyXWnsqEVE5FtUqEVEUi61hdrM+lcdqPu6md0UO09DmNllZuZm1ip2lvoys5vN7E0zW2ZmD5pZy9iZ6qJQDmg2s73N7BkzW1H1szEwdqaGMrNGZvaKmc2NnaUhzKylmf2t6udkhZkdma1rp7JQm1lXwrmMh7n7wcC4yJHqzcz2Bk4A/h07SwPNBw5x98OAfwHDI+fJWIEd0LwZGOzuBwFHAP3y+F62GgisiB0iC24FnnD3A4HDyeI9pbJQA32Ase6+AcDdP42cpyEmAEOp5viyfOLuT7r75qrfvkg46Sdf/N8Bze6+Edh6QHPecfeP3P3lqu+/IhSDNnFT1Z+ZtQVOAWbEztIQZrYzcAwwE8DdN7r7mmxdP62F+gCgi5ktNrNFZtYpdqD6MLPuwP+4+6uxs2TZBcDjsUPUQXUHNOdtcdvKzEqADsDiuEka5BZCI1MZO0gD7QdUAHdVDePMMLMds3XxjA4OSIKZLQD2rOahkYRcuxLe2nUC7jOz/TyFcwlruY8RwIm5TVR/Nd2Luz9c9ZyRhLffs3OZrYEyOqA5n5hZC+AB4BJ3/zJ2nvows1OBT919iZn9OnaeBtoe6Aj0d/fFZnYrMAwYla2LR+Hux2/rMTPrA8ypKsz/NLNKwmYnFbnKl6lt3YeZHQrsC7xqZhCGCl42s87u/nEOI2aspn8TADP7A3AqcFwa/9OsQUEd0GxmjQlFera7z4mdpwGOBrqbWTegGbCzmd3j7udEzlUfq4BV7r713c3fCIU6K9I69PEQcCyAmR0ANCHPdtdy9+Xuvru7l7h7CeEfsmNai3RtzOxk4HKgu7t/EztPHRXMAc0W/tefCaxw9/Gx8zSEuw9397ZVPx9nA0/naZGm6uf6QzNrX/VHxwFvZOv60TrqWtwJ3GlmrwEbgT/kWQdXiCYDTYH5Ve8QXnT3i+JGykyBHdB8NHAusNzMllb92Yiqc00lrv7A7Kpm4F3g/GxdWEvIRURSLq1DHyIiUkWFWkQk5VSoRURSToVaRCTlVKhFRFJOhVpEJOVUqEVEUu5/AfD/w6XMXAURAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(x, cost_0(Z),'b', label='left')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Cost for logistic regression:\n",
    "$$\\min_W\\frac{1}{m} [\\sum_{i=1}^{m}y^{(i)}(-\\log f(W^{(i)}, x^{(i)}) - (1 - y^{(i)})(-\\log(1 - f(W^{(i)}, x^{(i)}))))]$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Or with regularization:\n",
    "Cost for logistic regression:\n",
    "$$\\min_W\\frac{1}{m} [\\sum_{i=1}^{m}y^{(i)}(-\\log f(W^{(i)}, x^{(i)}) - (1 - y^{(i)})(-\\log(1 - f(W^{(i)}, x^{(i)}))))] + \\frac{\\lambda}{2m}\\sum_{i=1}^{n}w_i^2$$\n",
    "Different numbers $m$ and $n$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Now let's remove $\\frac{1}{m}$ and lets $C = \\frac{1}{\\lambda}$ then:\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "$$\\min_WC[\\sum_{i=1}^{m}y^{(i)}cost_1(W^{(i)^T}x^{(i)}) - (1 - y^{(i)})cost_0(W^{(i)T}x^{(i)}))] + \\frac{1}{2}\\sum_{i=1}^{n}w_i^2$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Supported Vector Machine does not output the probability:\n",
    "$$f(W, x) = \\begin{cases}1 & \\text{if }W^Tx \\ge 0 \\\\ 0 & \\text{otherwise }\\end{cases}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "$$\\min_W[\\sum_{i=1}^{m}y^{(i)}cost_1(W^{(i)^T}x^{(i)}) - (1 - y^{(i)})cost_0(W^{(i)^T}x^{(i)}))] + \\frac{1}{2}\\sum_{i=1}^{n}w_i^2$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "if $y = 1$ we want $W^Tx \\ge 1$ instead of $W^Tx \\ge 0$\n",
    "<br>\n",
    "if $y = 0$ we want $W^Tx \\le -1$ instead of $W^Tx \\lt 0$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Large Margin Classifier\n",
    "<img src=\"images/svm/svm1.png\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Maximum among the minimums"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Let us choose higher value of $C$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Choise of $C$ parameter:\n",
    "<img src=\"images/svm/svm2.png\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "By the definition $C = \\frac{1}{\\lambda}$ and therefore large $C$ means small $\\lambda$ and small $C$ means the larger $\\lambda$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Geometry Behind the SVM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Let us observer optimization objectives of SVM:\n",
    "$$\\frac{1}{2}\\sum_{i=1}^{n}w_i^2$$\n",
    "$$W^Tx \\ge 1$$\n",
    "$$W^Tx \\le -1$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "From the definition of Euclidean norm $||v|| = \\sqrt{\\sum_{i=1}^{n}v_i^2}$\n",
    "$$\\sum_{i=1}^{m}w_i^2 = ||W||^2$$\n",
    "Where $W = (w_1, w_2, \\dots, w_n)$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Inner product of vectors:\n",
    "$$W^Tx = x_w||W||$$ \n",
    "Where $x_w$ is a length of the projection of $x$ vector on the $W$ vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Then we have a optimization objectives:\n",
    "$$\\frac{1}{2}||W||^2$$\n",
    "$$x_w||W|| \\ge 1$$\n",
    "$$x_w||W|| \\le -1$$\n",
    "<br>\n",
    "<img src=\"images/svm/svm3.png\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Kernels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Suppose we have a data:\n",
    "<img src=\"images/svm/svm4.png\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "We could have a polinomial function:\n",
    "$$f(x, W) = w_0 + x_1w_1 + x_2w_1w_2 + \\dots x_nw_n^r$$\n",
    "<br>\n",
    "and then find:\n",
    "$$f(x, W) \\ge 0$$\n",
    "$$f(x, W) \\le -0$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Let us put some lendmarks near the data points $l^{(1)}, l^{(2)}, \\dots l^{(m)}$ and define features $f_1, f_2, \\dots f_m$:\n",
    "<br>\n",
    "$$f_i = \\exp(-\\frac{||x - l^{(i)}||^2}{2\\sigma^2})$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/svm/svm5.png\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Choosing the Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Lets choose the features by the training examples:\n",
    "$$x = (x^{(1)}, x^{(2)}, \\dots, x^{(m)})$$\n",
    "<br>\n",
    "Choose the features:\n",
    "$$f = (f^{(1)}, f^{(2)}, \\dots, f^{(m)})$$\n",
    "<br>\n",
    "by the Gaussian distance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "This means that we will need the $W \\in \\mathbb{R}^{m+1}$ instead of $W \\in \\mathbb{R}^{n+1}$ (with bias)\n",
    "<br>\n",
    "and our cost will be:\n",
    "$$\\min_WC[\\sum_{i=1}^{m}y^{(i)}cost_1(w^{(i)^T}f^{(i)}) - (1 - y^{(i)})cost_0(w^{(i)^T}f^{(i)}))] + \\frac{1}{2}\\sum_{i=1}^{m}w_i^2$$\n",
    "<br>\n",
    "Here parameters $m$ and $n$ are the same for training examples and for regularization respectively"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "For regularization instead of $\\sum_{i=1}^{m}w_i^2 = W^TW$ offen is used \n",
    "$$W^TMW$$ \n",
    "for some matrix $M$ for large $m$ ($m = 10000$ or even $m \\ge 1000000$) for optimization purposes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "We can aolso apply the kernels to the logistic regression but computation will be very slow, this computation reduction does not goes well with logistic regression and that's why SVM is preffered in many cases"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "So we kn ow about optimization objectives and the cost function of SVM, for practical use better use softwares which has numerical optimized implementations and we don't have to worry about them"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Choosing the C Parameter (Bias Variance Tradeoff)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Recall that:\n",
    "$$C = \\frac{1}{\\lambda}$$\n",
    "and therefore:\n",
    "- Large value of $C$ means small $\\lambda$ which implies lower bias / high variance (overfitting)\n",
    "- Small value of $C$ means large $\\lambda$ which implies higher bias / low variance (underfitting)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "If we use Gaussian kernel then:\n",
    "- If $\\sigma^2$ is large, features $f_i$ vary smoothly and this might cause higher bias / low variance (underfitting)\n",
    "- If $\\sigma^2$ is small, features $f_i$ vary less smoothly and this might cause lower bias / high variance (overfitting)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Recall that:\n",
    "$$f_i = \\exp(-\\frac{||x - l^{(i)}||^2}{2\\sigma^2})$$\n",
    "<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/svm/svm6.png\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "The Gaussian kernel is an instance of more general Radial Basis Function:\n",
    "$$\\varphi(\\mathbf{x}) = \\varphi(\\left\\|\\mathbf{x}-\\mathbf{c}\\right\\|)$$\n",
    "or even\n",
    "$$\\varphi(\\mathbf{x}) = \\varphi(d(\\mathbf{x},\\mathbf{c}))$$\n",
    "Which is defined as function of some metrics (Euclidean distance generated from norm or even some arbitrary metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Implementation of SVM with scikit-learn library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already up-to-date: scikit-learn in /Users/levan/anaconda3/envs/edu/lib/python3.7/site-packages (0.22)\n",
      "Requirement already satisfied, skipping upgrade: joblib>=0.11 in /Users/levan/anaconda3/envs/edu/lib/python3.7/site-packages (from scikit-learn) (0.14.0)\n",
      "Requirement already satisfied, skipping upgrade: numpy>=1.11.0 in /Users/levan/anaconda3/envs/edu/lib/python3.7/site-packages (from scikit-learn) (1.17.2)\n",
      "Requirement already satisfied, skipping upgrade: scipy>=0.17.0 in /Users/levan/anaconda3/envs/edu/lib/python3.7/site-packages (from scikit-learn) (1.3.1)\n"
     ]
    }
   ],
   "source": [
    "! pip install -U scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn import svm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "lin_clf = svm.LinearSVC()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "krn_clf = svm.SVC(C=10.0, kernel='rbf', degree=3, gamma='scale')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "??krn_clf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = StandardScaler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = [[0, 0], [1, 1]]\n",
    "y = [0, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
       "          intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
       "          multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
       "          verbose=0)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lin_clf.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=10.0, break_ties=False, cache_size=200, class_weight=None, coef0=0.0,\n",
       "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='rbf',\n",
       "    max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
       "    tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "krn_clf.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}