07-Automatic_differentiation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Automatic differentiation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "**Note**: the code below requires autograd for computing the gradient of the logistic regression, which you can install from the command line by typing ```conda install --channel conda-forge autograd```. If this fails you can also load the full gradient expresion of the gradient from the first class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import autograd.numpy as np # Thinly−wrapped numpy\n",
    "from autograd import elementwise_grad\n",
    "\n",
    "def f(x): # Define a function\n",
    "    y = np.exp( - x )\n",
    "    return (1.0 - y ) / ( 1.0 + y )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "D_f = elementwise_grad(f) # Obtain gradient function\n",
    "D2_f = elementwise_grad(D_f) # 2nd derivative\n",
    "D3_f = elementwise_grad(D2_f) # 3rd derivative\n",
    "D4_f = elementwise_grad(D3_f) # etc.\n",
    "D5_f = elementwise_grad(D4_f)\n",
    "D6_f = elementwise_grad(D5_f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xl8VNXd+PHPuXe2zGTfQ8gKYd8JoIjgBgJu1bqgVqtW\nqVVrtdVf7Wb7PK1tba19bKuiVWvd9x3cRUBUdgk7hJCE7Ps6+73n98cEDBCSSQgg5rx5zWtm7j33\nzpmQ3O89u5BSoiiKoij7aMc7A4qiKMo3iwoMiqIoygFUYFAURVEOoAKDoiiKcgAVGBRFUZQDqMCg\nKIqiHEAFBkVRFOUAKjAoiqIoB1CBQVEURTmA5XhnoC8SExNldnb28c6GoijKCWXdunV1UsqkntKd\nkIEhOzubtWvXHu9sKIqinFCEECXhpFNVSYqiKMoBVGBQFEVRDqACg6IoinIAFRgURVGUA/RLYBBC\nPCGEqBFCbD7MfiGE+IcQolAIUSCEmNRp31whxI6OfXf1R34URVGUvuuvEsOTwNxu9s8D8joeC4GH\nAYQQOvBgx/5RwOVCiFH9lCdFURSlD/olMEgplwMN3SS5AHhKhnwJxAoh0oCpQKGUskhK6Qde6Eir\nKIqiHCfHahxDOrC30/uyjm1dbZ92jPKkKIpyRAxT4g+aoYfR8Qh2ehgG/qAkYJgYUmIYEkNKTFMS\nNCWmlBj7XpuhfYZ50OOg4y6cNJicRNdR/V4nzAA3IcRCQtVQZGZmHufcKIpyopFS4g2YNHn8NLkD\nNLkDNHe8bvMF8fgN2v0Gbn+Qdl/Hs9/A7Qvu3+72G/gCBgFD4jdMDFMe8+8xKSvuWxMYyoGMTu8H\nd2yzHmb7IaSUjwKPAuTn5x/7/w1FUb6RDFNS1eKlusVLTYuPmtavn6tbfNS0+qhv89HkCeAPmt2e\ny2bRcNl0nDYLLvvXz7FOW8d7HZuuYbN0PHT969cWDXunfdZ9r3UNm0WgCYFF09A00DWBRQtt07VO\nDyHQ9u3reH/wvmPhWAWGt4BbhBAvEKoqapZSVgohaoE8IUQOoYCwALjiGOVJUZQThGFKiuvb2VXd\nxt4GNyUN7ZQ2eCitb6es0UPwoDt3XRMkRdpJjraTHutgXHoMsS4rsRE2Yp1WYiOsxDhD72OcViLt\nFpw2HauuevBDPwUGIcTzwGlAohCiDPgtodIAUspFwBJgPlAIuIFrO/YFhRC3AO8DOvCElHJLf+RJ\nUZQTU2O7n80VzeyoamVbZSs7qlvYVd2Gr9PdfkyElawEJ6PTY5g/No2MeCep0Q6SouykRDuId9nQ\nj9Hd9beRkPLEq5XJz8+XahI9RTnxSSnZVdPG2uJG1peGHkW17fv3J0XZGZEaxYjUKIanRjMsJZKs\neBcxTutxzPWJSwixTkqZ31O6E6bxWVGUb4faVh+fFdayYmcdKwrrqG31ARDvsjEpM5bvThrMhIxY\nRqRGkRBpP865HZhUYFAU5agrrXezeFMl726upKCsGQgFghlDE5mRl8jU7HiyEpwIoap/vglUYFAU\n5aiobPbw+oZyFhdUsqWiBYDxGbHcefZwZg1LYlRa9DHrZaP0jgoMiqL0G8OULN9Zy7OrSvlkezWm\nhAkZsfxq/kjmjU1lcJzzeGdRCYMKDIqiHLFmd4BnVpXw3KpSyps8JEba+OGsISyYkkFWwtEdjKX0\nPxUYFEXps5oWL49/todnV5XS5gtycm4Cv5g/gjmjUrFZ1JiAE5UKDIqi9FqT28/Dn+7myc+LCRgm\n54wbxI2zchk9KOZ4Z03pByowKIoSNl/Q4D8ri3loaSGtviAXTkjn1jPzyD7Kc/cox5YKDIqihGX5\nzlp++9YW9tS1c8aIZO48ezgj06KPd7aUo0AFBkVRulXb6uO3b21myaYqchJd/Pe6qcwalnS8s6Uc\nRSowKIpyWIsLKvn1G5to9xvcMWcYN8zMxW7Rj3e2lKNMBQZFUQ7R7Anw6zc28/bGCsYNjuH+S8cz\nNDnqeGdLOUZUYFCUg5jeIIGKNhACW3b0gJumYUtFMzc9u56yRg8/nT2Mm04bgkVNRz2gqMCgKJ0Y\nbX6qH9iA2eoHIO6iPFxTU49zro4NKSUvrtnL3W9tId5p48WFJ5GfHX+8s6UcByowKEonTW/uxnQH\niL9yJO2rKml8qxBreiS29MjjnbWjKmiY/O87W3nqixJOzUvk/y6boGY2HcBU+VBROni21uPZVEf0\nWZk4xyYSv2A4WoSF5vf2HO+sHVWt3gDXP7WWp74oYeHMXJ68dqoKCgNcvwQGIcRcIcQOIUShEOKu\nLvbfKYT4quOxWQhhCCHiO/YVCyE2dexTq+8ox03bFxXosXaiZg4GQI+04cpPxVfYhNFRtfRtU9Xs\n5ZJFX7BiVx1/vHAsv5w/Uq18phx5YBBC6MCDwDxgFHC5EGJU5zRSyr9KKSdIKScAvwCWSSkbOiU5\nvWN/jysLKcrREGz24StswjkpGdGpodU5IQkkuAtqj2Pujo69DW4ueeRzyho9PHntFK6Ylnm8s6R8\nQ/RHiWEqUCilLJJS+oEXgAu6SX858Hw/fK6i9Bv3hhqQ4JqUcsB2a4oLa5oL91ffrsBQWNPGJYu+\noMUT5Nnrp3FqnhqwpnytPwJDOrC30/uyjm2HEEI4gbnAq502S+AjIcQ6IcTCfsiPovSKlBL3+mps\nWdFYEiMO2e+ckExgbyvBBu9xyF3/21HVymWPfEHQNHlh4UmMz4g93llSvmGOdePzecDKg6qRZnRU\nMc0DbhZCzOzqQCHEQiHEWiHE2trab9fdm3J8GfVegjUenOO7vmt2DI8DwLen+Vhm66gormvne4+v\nwqILXvrhyWquI6VL/REYyoGMTu8Hd2zrygIOqkaSUpZ3PNcArxOqmjqElPJRKWW+lDI/KUkVe5X+\n4y1sBMA+LK7L/ZZkJ8Kh4y9pOZbZ6ncVTR6ufGwVQcPkmR9MIzfp290FV+m7/ggMa4A8IUSOEMJG\n6OL/1sGJhBAxwCzgzU7bXEKIqH2vgTnA5n7Ik6KEzbuzCT3WjiXB0eV+oQlsmdH4TuDAUNfm43uP\nraLFE+Cp66aRl6Kmt1AO74gDg5QyCNwCvA9sA16SUm4RQtwohLixU9ILgQ+klO2dtqUAnwkhNgKr\ngcVSyveONE+KEi5pSHy7m3AMi+t26gt7VjTBajemO3AMc9c/vAGDG55aS0WzhyeuncLYwWoxHaV7\n/TLyWUq5BFhy0LZFB71/EnjyoG1FwPj+yIOi9IW/rBXpM7AP7b4B1pYdqov3lbYSMeLEmSbCNCU/\ne3kjX+1t4uErJzFFTXGhhEGNfFYGNF9hEwiwD+khMGREgcYJ187wtw93sLigkrvmjmDumLTjnR3l\nBKECgzKg+UpasKY40V3WbtNpNh1rWiT+va3HKGdH7vUNZTy4dDcLpmSwcGbu8c6OcgJRgUEZsKQp\n8Ze2YMsMr8umNdVFoLq954TfAFsrWrjr1U1My4nn998ZM+CmDleOjAoMyoAVrHUjvQa2rHADgxOz\nNYDR/s1ugG72BPjRs+uIdVr51xWTsKq1FJReUr8xyoDlLwlVC9kyw+u6aU1xARCo+uaWGkxT8rOX\nNlLe6OGhKyeRFKVmSVV6TwUGZcDylbSgOS1dToPRFWuqE4BgtftoZuuI/HtFER9tq+ZX54xkcpbq\ngaT0jQoMyoC1r30h3Pp3LcqG5rR8Y9sZNpU1c98HO5g3JpVrpmcf7+woJzAVGJQByfQECdZ6wq5G\nAhBCYElxEqj65pUYPH6Dn7y4gQSXnT9dNFY1NitHRAUGZUDyl7cBYBvcu6khrCmhnklSyqORrT77\nw+Kt7Klr5/7LxhPrtB3v7CgnOBUYlAEp0BEYrL1cy9ma6kJ6DYzmb86Kbh9trebZVaUsPDWX6UMS\nj3d2lG8BFRiUAclf3ooea+9xYNvBrMkdDdA134zqpGZ3gF+8vomRadH8dM6w450d5VtCBQZlQAqU\nt2HrZWkBwJIYmoE1WO/p7yz1ye8Xb6Wh3c9fLx6H3aIf7+wo3xIqMCgDjukNEqz3Yh3c+8CgRdkQ\nVo1g/fFfzW3ZzlpeWVfGjbNyGZOuZkxV+o8KDMqAs7/hOb33axIIIbAkRBz3EkObL8gvX9vEkCQX\nPz4j77jmRfn2UYFBGXD2NzwPcvXpeEuCg2Dd8Q0M972/g4pmD3+5eDwOq6pCUvqXCgzKgOOvaEOP\nsaFH9q1bp54YQbDBizSPT5fVzeXNPPVFMVedlMXkrK6XI1WUI9EvgUEIMVcIsUMIUSiEuKuL/acJ\nIZqFEF91PO4O91hF6W+BynasaX1f79iS4ABDYjT7+jFX4TFNyW/e3Ey8y8bP5gw/5p+vDAxHvIKb\nEEIHHgRmA2XAGiHEW1LKrQclXSGlPLePxypKv5ABk2Ctm4hRCX0+hyUhNLdSsM6DJa7rdaKPlpfX\n7WVDaRN/u2Q8MRG962qrKOHqjxLDVKBQSlkkpfQDLwAXHINjFaXXAjVuMMGa1rf2BWD/pHv7eyaZ\nJrTXYVRtw793I2ZdIXj7f6W3xnY/f353O1Oy47hoUnq/n19R9umPNZ/Tgb2d3pcB07pIN10IUQCU\nA3dIKbf04lhF6ReByo6G5yMIDLrDAF0SXPchbHwcs3Yna6ri+aIuC0NqRFm8zEnbRXayDsmjIPNk\nGD4PBk2EI5jD6L4PdtDiDaqFd5Sj7lg1Pq8HMqWU44B/Am/09gRCiIVCiLVCiLW1tbX9nkFlYAhU\ntiOs2v7qoF4pXw9v3Iz4ay4Ws5RgRS3SFsVbzafzWW0OOcOyOfXMfKwxyby6dywFlpnga4EV98G/\nT4d/ToJP7oGm0l5/9PaqFp5fXcpVJ2UxIjW8hYUUpa/6o8RQDmR0ej+4Y9t+UsqWTq+XCCEeEkIk\nhnNsp+MeBR4FyM/P/2bNYKacMAKV7VhTXQgtzDtuKWHXh7Dsz1C+DqwuGH8Zlr2DMXzDWROfxe7S\nJ5n1vevIP+8iACb5/bx53x/4ZP0mUn7/H1JS42Hb27DplVCQ+Ox+GL8AZvwUEoaEkQXJPYu3EeWw\ncttZasyCcvT1R4lhDZAnhMgRQtiABcBbnRMIIVJFR9lXCDG143PrwzlWUfqLlBJ/ZXv41Ujl6+G/\n58Fzl4C7HubeCz/bBuc9gJ4+mECjj89eeIphJ81g8rkX7j/MYrMx75afEREVzeIH/oJhjYJJV8P3\n34KfFED+D0JB4l/58Nat4G7oNhuf7qxlxa46bj0zT82cqhwTRxwYpJRB4BbgfWAb8JKUcosQ4kYh\nxI0dyS4GNgshNgL/ABbIkC6PPdI8KUpXjGY/0hPsOTB4m+Htn4Sqf2q2wry/ws1r4KQbwRGaesIS\n5wC/xOWMYfYNtxxS5++MjuGs62+isbKcLZ9+9PWO2AyY/5dQgJj6Q9jwTKiKad2ToUbsgwQMk3sW\nbyMn0cVVJ2Ud6Y9AUcLSH1VJSCmXAEsO2rao0+t/Af8K91hFORrCanje9WEoKLRWwsm3wKyfg+PQ\nOv0WTz0AU06/EEdk12MicidNJS1vOF+89gKjZp6Bxdbpbj8qBeb9OVSSWHJH6DM3vggXLoK4rwPA\nC6tLKaxp49GrJmOzqPGoyrGhftOUASNQFVqS05raRWAIeODt2+DZi8EeBT/4EM6+p8ugALBp9YcA\n5I056bCfJ4TglMuuoq2+jk1LP+g6UcoouGYxXPAQVG+Gh0+BjS+AlLT5gvzfR7uYlhPP7FEpvfuy\ninIEVGBQBoxAZTt6vAPNcVBBuW4XPHYWrPsPTL8VfrgcBucf9jyNVRUUbl0VetPefT+IrLETSMnN\nY+MHSw6/6psQMPFKuPEzSB0Dr/8QXruBp5dtob7dz13zRqjuqcoxpQKDMmDs65F0gC2vwyOzQlVH\nV74Cc34PFnu359n08fv48YFFYDT2PC3G+NnzqC8rpWLHtu4TxmWFSg+n/xq5+VXOWnklV+X5mZip\n5kNSji0VGJQBwfQbBOs8X7cvSAnL/wovXwOpY0N363mzezxPMBBg89IPGTJ5GpY4B0Zjz+syjJg+\nE1uEk40fvdtzRjUdZt3Jc3l/J54mflf9Y9i+uOfjFKUfqcCgDAjBajdIsKW5IOiHN26CT/4A4y4L\ndSONHhTWeYrWrcLT2sL4s+aixzkINvVcYrA6HIyaeTo7v/wMn7u9x/QVTR7+Z2sKDw97Aj0xD164\nAj77eyiYKcoxoAKDMiD49/VIigvC0xfCxufgtF/ChY/0WHXU2fbPl+OMiSVz3AQscXaMpvBWchs5\n43SMQIDCNV/2mPaBj3aBhGvmz4Br34UxF8NHvwsFs+Cxn9FVGXhUYFAGhEBlO8Im0F85B8pWw0WP\nwWk/79XcRX6Pmz3r1zLspBlomo4e58BsD2L6jR6PTcsbTnRSMju+WNFtutJ6N6+sL+OKaZkMjnOC\n1QHffSwUxDY+B099B9rrw86zovSFCgzKgBDYU4nV2I7wNcL334Zxl/T6HLvXriIY8DNi+kwALLGh\nkkY47QxCCIaffColBRvwtB5+5tUHlxaia4IfnTak88GhIHbxE1CxHh47A2p39Dr/ihIuFRiUbz35\n1QsEqtqxOmrh+o8g8/BjD7qz/fPlRCYkMmjYCAD0jrUYwmlnABh+8qmYhsGu1Z93uX9vg5tX15dx\n+ZQMUqK7WOdhzHdDvZb8bnh8NuxZ3qfvoSg9UYFB+faSEj65B+O1u5G4sJ5+KcTn9ulUfq+Hkk1f\nMWzqdIQW+rPRY0Ijmc1mf1jnSM4ZQkxK6mHbGR76tBBNCG48rZuJ9Qbnh4JbVFqorWTDs737IooS\nBhUYlG+ngBdevR6W/4VAzjUAWDOT+ny6koINGIEAQ/K/Lm3oUTYQEAxziU8hBEMmT6N080b8Xs8B\n+8oa3by8tozLpmSQFtPDlOBxWXDd+5A9A97s6F2leiwp/UgFBuXbp70OnjofNr8CZ/2OwOArQRxm\nKoww7V67GrvLRfqIUfu3CV1Di7T1au3nIZOnYQQClBRsOGD7Q5/uRggObFvoTkRsaEDexKtC4zFe\nuyEUDBWlH6jAoHy71O6Af58BlRvhkv/CjNvxV7VjSYhAs+l9OqVpGhStX03uxCnolgOn09BjbBgt\n4VUlAaSPGIXd5WL32tX7t5U3eXh57V4uzc9gUGwvFhDSrXD+P+HM38Kml+Fp1WNJ6R8qMCjfHkWf\nwmOzQxPiXbMERn8H6JgK4wiW8qzcuQNPawtD8g9ddVaPtveqxKBbLORMyKdo/WpMM9TN9eFPCwG4\n6fShvc+cEHDqT+Hi/4TWj3j8LKjf3fvzKEonKjAo3w7rn4JnvhsawXzDxzB4MgCmN4jR4D2iwLDn\nq3UITSNr3MRD9ukxNowwG5/3yZ08FU9rC9W7C6ls9vDSmjIunpxBem9KCwcbcxFc805oLYnHzoSS\nrns+KUo4VGBQTmymAR/eDW/9GHJmwg/eh9jM/bv3T7V9BIGhpGA9aXkjcLgOXXdBj7EjvUFMX8+D\n3PbJGjsBhKB443r+vXwPppTcFG7bQncypoZ6LDkT4akLoODlIz+nMiD1S2AQQswVQuwQQhQKIe7q\nYv+VQogCIcQmIcTnQojxnfYVd2z/Sgixtj/yowwQ3pbQPEIrH4D86+CKl/evsLZPoDw0FYYtPapP\nH+FpbaGqqJDsLkoLEAoMAEZL+NVJzugYUnKGUrhhLc+vLuX8CYPIiHf2KX+HiM+FH3wAg6fCa9fD\nsr+oHktKrx1xYBBC6MCDwDxgFHC5EGLUQcn2ALOklGOB3wOPHrT/dCnlBCnl4SfBV5TOGvbA43NC\nK67Nvw/O/Tvohy5I6K9oR4uyokf3ba3kkk1fgZRdViMB+8/b2+qk7PGTqNm9C8Pr5sZZ/VBa6MwZ\nD1e9DuMvh6X3dMyx1Lv8KQNbfyztORUolFIWAQghXgAuALbuSyCl7Fzh+SUwuB8+Vxmo9iyHl64O\n3Qlf9RrknnbYpIHyNmyDul56MxwlBRuwu1ykDsnrcr9lX4mhFw3QAGmjx8PrL3JOQivDUkKlmZqS\nFrZ+VkFFYTPeNj+6RSMxI4qs0fEMm5aK7eAFhrpjscF3Hg6VIJbeAw1FcOl/ISq1V/lUBqb+CAzp\nwN5O78uAQ7tvfO0HQOeJ6SXwkRDCAB6RUh5cmgBACLEQWAiQmZnZVRLl205KWPUIfPAriB8Clz8P\nCYe/25YBk0BNO45R8X38OElxwQayxkxA07vu6rpv9HNvqpIAljU68QkbU6y1eFr9LH1mO3s21mG1\n6wzKi2VQXixBn0F1cQvFBXV88fpuJs/PZvwZGejhrv0sBMz6f5AwFN68BR6ZGerCm3Vyr/KqDDz9\nERjCJoQ4nVBgmNFp8wwpZbkQIhn4UAixXUp5yCQwHQHjUYD8/HxVaTrQeFvgrVtg65swbB5c9Mgh\n7QkHC1S1g0mfSwwN5WW01deR9d0Fh00jrDqa09KrqqSAYfL456WclpBNy87NvPCH1XjbA0w7P4dx\np2dgizjwz7JqTzPrlhTzxWu72bmqmrNvGE1cbwbrjbkIkkfCC1fCf8+FOffAtB/2amZZZWDpj8bn\nciCj0/vBHdsOIIQYBzwGXCCl3D8KR0pZ3vFcA7xOqGpKUb5WtQkenQXb3oHZ/xsqKfQQFAD8FR1r\nMKT3LTAUb1wPQNbYrtsX9untWIa3N1ZQ3uRh5OhJtDfVIY0GLv5/+eTPzzkkKACk5sRwzs3jmX/T\nONqbfbz0p7Xs2Vjbuy+TPBIWLoW8s+G9n4emC/G1Ig1JoMZNsN6DDITfs0r5duuPEsMaIE8IkUMo\nICwAruicQAiRCbwGXCWl3NlpuwvQpJStHa/nAP/bD3lSvg2khA1Pw5I7wREb6qefNT3swwPlbWhO\nC3ps+AvxdFZSsJ64tHRiklO6Tdeb0c+mKVm0bDfj4124d4QKviOm+UnK7LnXVM64RC771VTeXVTA\nu4s2cdr3RjDqlPBWngNCwfSyZ+Cz+wl+/AgtWx7FE5yKDHTstwjsubFEnpSGY0Q8QlMlioHqiAOD\nlDIohLgFeB/QgSeklFuEEDd27F8E3A0kAA+JUPE12NEDKQV4vWObBXhOSvnekeZJ+RZor4O3fwLb\n3wmNT/ju4xCZ3KtT+MvbsA6KRPShyiQYCLB362bGnjGnx7R6jB1/R7fYnizdUUNRVRt3WmIRWjQx\nyWlUFm4CLg7r+Mg4OxfcPpH3Ht3M0qe3420LMHFOZvjfUdPwxF9DgzEN6fPj0t/DNm4oDJtLoNKN\nZ0s99U9txZoeSdyFQ7EN7ls3X+XE1i9tDFLKJcCSg7Yt6vT6euD6Lo4rAsYfvF0Z4LYvgbdvDY3i\nnf17OPlm0A4/z1Gbv43dzbspaSmhuLmYem89Xp+HH1XMZ23WLgpWvkysI5ZUZyrD44czOmE0DksX\n6x10UrFjK0G/77DdVDvTo22YbQFk0ET00DC8aNluzjEjCNb7OOemcRSty2fTJx8Q9Pux2MLrUmtz\nWDjnpnF8/ORWvnh9NwGfwbTzw5tO3L2pjobnt2NLjyL+okFYVj4Lm/8F3ulw0SPEzM/H/VUtze8V\nU/PgV0SfmUnUGZmq9DDAHNPGZ0XplrcF3v8FbHgGUsfC1W9BysFDYqDV38oXFV+wtnotG2o2sLNx\nJ6Y0AbAIC/GOeIb5s7FIne22IlaWr6bB10DQDAJg02zkp+YzJ2sOc7LnEGU79K64uGADmm4hY/TY\nHrP99SA3P5b4wwecDaWNVO1q5rR2O+PPyCB7XCJmcBIb3nub8u1byRo3IawfE4Bu0Zh93WgsNp21\nS4qx2nUmnZ3V7TGB6nYaXtyBbXAkideNQXNYQiWxvDmw+A546GTEmXfjmnI9EaMTaHpzNy0fleIr\nbSX+suHoLmvY+VNObCowKMeflLDlNXjvl9BeA6feAbN+HuqL36HOU8f7xe+ztHQp66rXEZRBIiwR\njE8az43jbmRkwkiyo7NJj0rHqllpX1NF4+Zd/Pz8u/l1YgRSSmrcNWxr2MaaqjV8uvdTfvfF77h3\nzb2cP+R8rhtzHYMiv66vL964nkHDR2Bz9Dx/UefRz90FhieWFTHPayMq0cG074Tu8DNGjUW3WCgu\nWN+rwAAgNMFp3xtBwG/wxeu7sdp1xp7W9RAhGTRpeHEHml0j4apRoaAAoZ5J4xdA5snwzu3w7v+D\ngpfQzv8HcZeOwpYdTdNbu6n55wYSrhqFrY8N+cqJRQUG5fiq2wVL7gjNjJo2AS5/DtJDE+B5g14+\n3fspb+1+i88rPseQBkNjh/L90d9n5uCZjEsah0Xr+lfYX9GGsOv7L9RCCFJcKaS4Ujgt4zTuyL+D\nLfVbeHHHi7y26zVe3fUqV464kpsm3IRs91FbXMSMBVeH9RX2j2XopmdSeZOH+vV1jDCsnHHVSKwd\nU4BbHQ7SR4yiZON6+N514f7U9tM0wVnXjiLoN1n+wk6sDp0RJ6Udkq71s3ICFe0kXDUqtMDQweKy\n4Huvhqbvfu8ueGQm4pSfEHnqHdjSx1P/9DZqF20kfsFwIkYn9jqfyolFBQbl+PC1wmf/F5rnyOoM\nTWuRfx2mEKyvWsvbRW/zQfEHtAXaSHGmcO2Yazkv9zxyY8OrSw+Ut2Ed5Dps3bgQgjGJYxiTOIab\nJ9zMwxsf5r9b/8sHJR/wQ2touu7s8ZPC+qz9JYZuxjI89fFupnosDBobz+DhcQfsyxo3iRXPPUlb\nYwORcb0fjKfrGmffMJrFDxbwyVPbsTks5E74erU60xOkdVkZjhHxRIxOOPyJhIBxl8LQs+D9X8GK\nv8FXz2M78zck33QRdc9sp/6ZbcTMzSFyZnqfGvWVE4OaXVU5toI++PJheGACrLgvtMD9j9dSPHw2\n/9z4EPNfm8+171/Le3ve48zMM3lszmN8cPEH/GTST8IOCtKUBCrbwx7YlupK5X+m/w9PzXsKl9XF\nR8tewnToRKaHN32EsOsIm37YEkObL0jFiip0TXDmguGH7N8XgA5e1a03LFadeTeOJSkzivcf20zZ\n9ob9+1pXlCE9QaLndN8GsZ8zHi58GK59NzSFxhs/Qn/+LJJnNxMxNpHmd/fQ+OouZNDsc36VbzYV\nGJRjwzRZ1nHJAAAgAElEQVRCC9f/c3KoqiJ5JE3ff4sXRp3Olctv57w3zuOxTY+RHZ3Nn079E0sv\nXcofZvyBaWnT0ETvfk2DtW5kwOz1wLaJyRN58dwXGdqUQElcC1e9fzUlLSU9HieE6HYsw4ufFDHM\nK0jPTyI64dA2i6TMbJwxsfsH1PWVzWHhvFvGE5vsZMnDm6gubsH0BWlbWUHE2MTejwDPmg7Xfxxq\noPY0I547n3jjN0Tl67jXVlP3xGZMd6Dn8ygnHBUYlKMr4IG1/4EHp8KbN+FzxvPRvN9xW9ZQTv/s\nNu5ZdQ+eoIefTf4ZH178IYtmL+Lc3HNxWvs+DbV/bytAn/rgN5eVQ7uPs2ZdRq27lgXvLGB52SEz\ntBxCj+l69LNhSrZ9XIYUgrO/O6zLY/ctAlRSsAFpHtlduCPSyvm3TsARaeWdf26k9tMypM8g8tT0\nvp1Q02DsxXDLGjjrfxAV64jZPI+4tHfwFTdR89BGgnWeI8qz8s2jAoNydLgbQmsB/H0Mxju3sdph\n47dTLuT0qCC3b3+Cr2q+4ooRV/DKea/w2vmvcc2Ya0h29m4A2+H4S1sRDguWxN6viFbcUZ1z5qyL\nefHcF8mMzuTWT27ljcI3uj1Oj7Z1GRgWf15KVpskblw8rpjDj8DOHj8JT2sLNcVFvc7zwVyxdi64\nbQKaDg1L96IlR2DLOMKBalYHzLgNbtsEc+7BFXiDJMtdmA311PxjNb5ddUecb+WbQzU+K/1HSij9\nEjY8jdz8Glu1IO8OHs271kHU+JtwNm3jrKyzOCfnHKamTT1sj6Ij5S9twZYZ1adBWSUFG0gYnElU\nfCJRwH/O/g+3Lb2N36z8DY3eRq4dc22Xx+kxdoxWP9KUB3zumsV7SBSCCxaM6PZzs8aGuqoWb1xP\nSm4f1n4+SEySk3MuycP/6i621fuIafF3G5jCZo+E6bfAlOuxb3yO5GV/oa7uB9Q+bhI3uhDXeWcf\nsIKecmJSgUE5cq3VsPF5AhueZn37Xj6JiuGTzMFUSR8W0cKMlBncmXsOswbPIsJyBOsah8H0BQlU\nu4ke0/sulQGfl7Jtm5kwZ/7+bU6rk3+d+S9++dkvuX/d/bT4W7h14q2H9MjRY2xggtnmR48OXYC/\n+KqK5EYD68gYouO6H2ntio0jKTuX4oL1TLvw0l7nvSvWva34LRp7WgJU/30DF9w+sX+CA4RKEPnX\nYZn0fZK3fEz9m5U0bhlBYPs/iBlRhhj/XRg+LxRIlBOOCgxK3zSXw7a3ad72OqtqC/jUGcGyqEha\nolOw63amD5rOLZlnMjN9JpEtAfxFu/F89hrNJSUE62oxGhoxGhsxvV6kz4cMBBBWK1pEBJrTiSUl\nBWvGYGyDM3CMGU3EmDGIMKaM8O9tBQm2zOhef6XybVswAgGyxx3YTdWm27j31HuJskXx2KbH0IXO\nLRNvOSDNvmBgNH8dGJa+XogTuOyKkWF9fva4iaxb/CZ+jxtbxJEt9SkNiWdzHRGjE5g3OYV3Hizg\n9b+t5zu3TyQyzkGwvh7vtu34tm/Dt6uQYH09wYZ6jKYmCBr7lwPVXC60qCj0qCgsSUlY0lKxpg3C\nmpaKddAgbJmZaGPnkDjKpPm1AtrWXYBvRynxu36H1fZjGD431PNs6OxQMFFOCCowKOExDaj8Cu+u\nD9iwewlftpXwZYSDbTY7MiWRON3FeY6pzDByGNJoR36wF1/Rc9QU/YGqtq8nmNOcTizJyegJCVgz\nM9AinAi7DWG1Iv1+pMeD0d5OoLyc9lWrkG43AMLhIGL8eKLOOJ3oc8/FktB1f3x/aUfDcx/q1Pd8\ntQ7daiV95OhD9umazm9O+g2GafBIwSPYdBsLxy38en/nldwyothd2kxktY9AtovkpPDWTsgeP4k1\nb73K3q2bGDK5u7WueuYrasJ0B3GOSyRhWBzn/Xg87/xjA6/c/Sn55c+ib/16eXVLSkooECcl48gb\nBlYLQgiklEi3G6O1DaOlGV9REcGaGujcQK7r2DIysOXmYh+SiyNvBL6SLKqNh4lNLsBV9HfEltdD\nY1VyZkLe7NAUHKq66RtNBQala6YJ9buoL/qYTUUfUNCwlY26ZKtmI7FRkFkfzWx3Aje1RpNY40Uv\nq0YGPgSgCbAkJWEbMoSY88/ff9Gw5Q7BkpwU9sAoKSXB2lo8GzfiWbuW9i9XUf2nP1P91/uIPPVU\n4q68Etcp0w84n7+0FUtyBFoX6xr0pGjDGjJHj8Nq7/rOVhMavz35twTNIP/c8E/sup3vj/4+cOjo\n5zdf3o4VOOcwPZG6Mmj4KCx2O8Ub1x9xYPAU1CFsOrbsSJpeex3vf//L+AoPX427hS8TL2Pmj+aQ\nftJwHMOHocfGhn1eGQwSrK0lUFlFoGwvvqIi/EV78O8pom3FCggEEPYYHJO+T5MxnsbAfVgs63AY\nO7EXbMG++X10m4TE4ZBzaqhLbOZ0iD50tLZy/KjAoISqDdqqaSpbxa6S5eys3sCuunJqWwTWFp30\nOkl6vYUfNGjENhuIfevnaXXYMiKw5eZhP/NsbLlDQgEgJwc9uvdVOQcTQmBNTsY6ezbRs2cD4Css\npPmNN2h6803arr+eiPzJJP/kJzinTEGaEl9xM85xST2c+VANFeU0VVUyef53uk2nazr/e8r/4jf9\n3Lf2PuId8Zw35Dw0pxV0gdHqp6nFB4VttCXYGJEX/khmi9VKxqixRzTQDULVSO7NdWgRLew+ezZG\nXR32vDyG3fkDcsdN5t2ni1laGMlZM7MZ0ougACAsFqxpaVjT0mDSgTPPymCQQFkZvt278RYW4du9\nDsRwgsymsdCFf0cF0m1giXNhjzWw2V/HHvUc9ugg9qxB6Hknh6ZFGTQhNImirRer1Cn9SgWGAUa6\nG6iv2sjeitWUFW2gqnwPrbUtmE0Q3SQY1CAZ0QCT/BoSQdDiwO+KRWQOReTl4k5Oh/hk9MRE9PgE\nAppOO2CxaljtOlZNJ6IeIk0fEdE2tH6ertk+dCjJd9xB4q230vTKK9Q/vIiSq64m8owzSPjRz5Fe\nA3tOz6u7Haxo/WoAcidN6TGtRbPwxxl/pMnXxN0r7ybOEceM9BnoUTaMZj+vvrwduxRMOien1/nI\nHj+JpRvW0lxTRUxyeCOvD9bywSqkJ0jbsmewDxlC4l/uxXnyyftLVhf/PJElDxfw3iObmXZ+DpPm\nZvfL/5OwWLBlZ2PLzibqzDMBML1Bmj8spv2LGdiyT0GPasJs3oC/cBNNRRLp2dcY7sfi/BRb1AfY\nooLYIg2saSnYsodgzRuDnjkGEoeF1q+2Ht0ODAoIKY98+WQhxFzgAUIL9TwmpfzzQftFx/75gBu4\nRkq5Ppxju5Kfny/Xrl3bU7KBR0qC7nrq6rZTu7eA+tKdNFeV4K6uxd8QRLbasLpdRPqcOP0upOYi\nYHUStISefc5ogs5YTFs0QeHAHzyyYS5CE8QkRZCQHkliRiSpOdGkDYlFt/bf8BnT66Xh6aep+9eD\nWIeehT3vAlLvmoIltncNnS///pe0NzVxzd8eCvuYNn8b175/LSUtJTxx9hMkvxQEi8aTBXUEbIJf\n3X96b78O9eV7efKnP+Ks629m/Ox5vTrW9Hio/uOfcG/2Yxs2l+hT/USfM7vLqrug3+CTp7eza001\ng/JiOevaUUR1MzPskQo2+2hdupf2NVVgShzD43FOTUGP9OAv2o1/9258uwrx7dxGYG8ZRmv7Acfr\nNhNrZBCry8Aa58KSmBBqGxmUgTVzKJac0WgpQyAyFXR1v3s4Qoh1HYukdZ/uSAODEEIHdgKzgTJC\nS31eLqXc2inNfODHhALDNOABKeW0cI7tyoAJDFLiczfQWLmDpvLdNFWW01rbgKexHV9rkIAbgh4N\nM2BDBm0I046QdhA2TM1GwGInqFsIahbABIIgg0gMkMGObQKL1YLVYcfmsGFzRhARGYUzOhpnbAyR\ncTE4ox3YXVYcHQ+rXQcRqurZd80JBkwCPoOA18Dd6qe90Utbo4/GKjd1Za201HkBsNg00ofFkT0u\nkbwpKdj70BbQFX9xMVX3LwfThSaWMejPf0JzhVcV4XO389D1VzD53AuZecU1vfrcOk8d31vyPdwB\nN89678db7OeDKi9Jc9O59DuHzovUEykl/77lOpKzh/CdO38d9nG+3bspv+12fLt2Ef3d+7EOSiX5\n5u4XGZJSsv2LKpa/uBNdF8y8fBh5+SlHdXK8YJOP9tWVtK+pwmwNoEVasQxxEUwDf5SPgOEn6Pfj\nb27GV1tDsKERo7Eeo64as6Eeo7kFw+NDC5hopkSTEr3j2aIb2KwGVhtYI6zYIh3Yo11YY2OxJSRi\nSUpFT0hGT0hBi0tCi0tBi0tBRMTAAJkQMNzA0B9/lVOBwo7V2BBCvABcAHS+uF8APCVDUehLIUSs\nECINyA7j2OMmGPAT8LTj87Thb3cT8How3R58Hh+mz0fQ78Pv8RJwt+Fv9xLw+gn6fAR9AQy/gRkw\nMYMmZlAiDcAUSFOErsdSgASkDggEGgIdhEAgOv44Q88CEEIisCBIwkoSNgFCSrBJTKuBKQ2kNDEJ\nYEovhjQImD78ppdAwEfA9OE12jFk8NDv6QmtkXM4rrh4YlNSiU1JIzEji5TcoSTnDMXuDL9Lpc8T\npHJXE6VbGyjdUs+y53aw8uVdDJmczJiZ6aTm9r76pzNrZhZ6Qh6atZ6mZz6m+PIrGPzQg9gGd70+\nQWclBRswDYPciT3+vRwiMSKRRWct4up3r+bjhmXM9Eyh1QI3nNO3QWpCCHInTWXLso/CXtWtZckS\nKn71a7SICNL/9SgtH4FjZDezqHb6rJHT00gbEsOHT2zhw8e3smV5BTMuzSPpSEdKH6S1oY7KXTuo\nKy2hvnwvTU1lRLZHk9KWTVpLLtaNNjD9NPsqqPOWUectp8lfg890H3iimKjQI1wtQIsXSvaiyVJ0\nc18gCQUWXUp0aaIjsQiJLgQWDXRdYLVo6LqG1aKjWyxYLBYsdlvoYbOjOyLQbTZ0mx3N4cAS4UC3\nOdDtEWh2O7o9IvTa4UB3uNAdLrQIF7rTiW6LQLM7wGpDaN+8CSj6IzCkA3s7vS8jVCroKU16mMf2\nm49v/ieDHFmAtv/iu/+f6Lg4H7AtlK7zJG4CcKADzo5HNwRg7XgcBRLZ8TG9uNtxaGjRVvRYO5ZY\nO1qCHRFvRcTpmBaJ3+PB29qKp60Fb2sr7pZmWmpraK6porhgA1uWfbz/VEnZueSMn0T2+EkMGj4K\n3XL4Xyd7hIXscYlkj0sMLZpT0sq2lRXsWlPNji+ryBgVz7TzcknJ6VujdaDaHZpB9NwpRM16lPKf\n/oziiy8h/YEHcE2b2u2xRevX4HBFMmhYeOMNDpYdk82DZz7IJ4+9g00I4sa6sFkPvxRpT4ZOnsrG\nDxZTumUjuRMP3+YhpaThif9Q89e/EjF5Mun334+/TAI7cQwPv9E7NsXJd3+ez7aVFXz5ZhEv/XEN\nI05KZcLsTBJ6O/FeB7/HzZ6v1lG0bjVl27fSUlsd2iEEscmpxKcPJmp4IsS4aI0OEOV3Ym1wkFbj\nIrU+e/95hNOCnmhHi7aiRdvQoi1oUVakQ2BqBoZmYIggRjBAMOAnGAgQ9PswAgGCfn/o5q2tjUBj\nPYHmOgKtLQTc7QS93q/TBQ2ChoHblBhSYkgwEBimwJACDAE+iYaBED4EAYRwo9HccY0IXTc0tP3v\ntX3Xk45th7zvuBnUukivCX3/defAc4aeI4dpnHzHHX36fwnXCVMZJ4RYCCwEyMzsWx9ow+qj2d+I\n7PQPKTn4Hwc973u9rzuOFKHXktCdPJoATaBZBJquoVl1dKsF3W7FYrNicTqwOSKwRUZij44mIioa\nW5QL3WZFt1oQuhaaRqHjEXodqqNHdGzTQ6+/Tsf+Ir80JRgSaZhIo+N1wMD0GpjeINITxPQEMdr8\nGE0+jCYfwSYf3uJ2pN/o+AGDNcWJLTuGqJx0HBNGh3raHMTd3ER1USFVu3dRunkja995ndVvvoIz\nJpYRp8xi9KwzSc7ufnpsIQQp2dGkZEdzysV5bF5WzvoPSnjl3rXkjE9k5oJhRPYwUvhgvl2NANiH\nxmKJOYWcl15k7003s/f660l/4AGizui6vl+aJkUb1pI9YTKa3veL+diksWxsrQBgT9yrBIxJWPW+\n3REMHj0OqyOC3WtXHTYwSNOk5t57afjvU0TNncuge/+MZrfTumwHmsuKNa13PXo0TTD61HSGTk5m\nzeJitiwvZ/sXVWSOjmfsaYPJGBmP3sN61qZpULRuDZs//ZDijesxAgEioqIZPGoMk+efz6BhI0nI\nzMJq6370tekJ4i9vJVDlJlDVTrDWg1Hmwd/SFCptd0FYNay6wKZrCN0OlojQ34oukKYMPaTEcJqY\ndgNTyv3bkbKjFC8RUiBk6J4udKkO3UZ+k+w0th/1z+iPNoaTgd9JKc/ueP8LACnlnzqleQT4VEr5\nfMf7HcBphKqSuj22KwOmjeEok1JiNPoIVLUTqGzHV9yMv6QF6TdBE9hzY4gYnUDEuKTDrvfrc7sp\n3fwV2z77lKJ1qzGCQdLyhjPtwkvJnTQ17PpqvzdIwSdlrHuvGKEJpl80lNEzBoU931Htvwsw2gKk\n3j55/zajqYnSGxbi3baN9L/+heh5hzbmVu7awXO//hnzb/kZI0/tfWPxPoVFTay9fz0zIq3clfl/\nZIwdxh9n/LHP9fVv3/8nKnZuY+FDTx5S1SANg4pf/IKWt94m7uqrSLnrrv1pKv+8GltGFAlX9q30\ns4+nzc+W5eUUfFqOp8WP3WkhZ3wiOeOTSBsaQ0Tk11Vc3vY2Cj56j40fLqGltobIuHjyTjqFYVNP\nYdCIkWha3wNuZ0bAxFPbjrfKjb/Jj7/FR6AtgL/Ng6/dS8AbwAgYyIAJhkAzBJqpIaRg31XucM8m\nJkFhYAgDU5iYmsQUJlKTSCGRukRqhLrIdDwLi0BYQOg6mh66KUQP3cRpFg1NFwhd67hZ1NAteuhh\n7Xi2WEKvrTpW3YJm0dCtOkLXseg6ulXDYrWiWzQ0iwWLVceih6q0+upYtjGsAfKEEDlAObAAuOKg\nNG8Bt3S0IUwDmqWUlUKI2jCOVY4SIQSWeAeWeAcRo0J10tKQ+Mtb8W6tx7OlnqY3d9O0uAjn2CRc\n01KxZUUfcLGzO53kTZ1O3tTpeNpa2bbiU9YtfoM3/vJ7EjOzmX7JFQydcnKPF0ibw0L+/GyGTU1h\n6TPbWfbcDgrXVjP7B6N7nN/H9AXxFbcQecqBU0vrsbFk/ucJ9v7wRsp/dgemz0fsdw4cp7B73WqE\n0MieMJkj8c4rO4g2Q5eZBYMu4RdF95DsTOb2ybf36XxD8qexc9VKqop2kTb060ZsaZpU/urXtLz1\nNkm3/YSEH/5w/8822ODFaPJhn9lzu0pPIiJt5M/PYeLsLEq3NbB7fQ1FX9Wx/YsqIFT9lJQZQXvD\nWvZufp+A101a3mhmLLiOYSeddNhqRSklRsAk6DfxugP43EF87YHQ6/YgPncAb3vHtvYAnrYAnnY/\nnjY/AbcBh7l7N0QQr6Udn8WN19KOYfcjIg30CLC6dBxOK06ngyiXi0iXkyinE5fLSUxkJNGRkcQ6\nY3BanVi1o1Tve4I54sAgpQwKIW4B3icUT5+QUm4RQtzYsX8RsIRQj6RCQt1Vr+3u2CPNk9J3QhfY\nM6OxZ0YTMzcHf2U77asrca+vwb2hBltmFNFnZWHPiz3kYh8RGcWkeecxfvY8dny+nFWvv8Rbf/sj\nWeMmcsa1NxI/qOc1AaITIzj/JxPY9nklK17cyUv3rGHuwjGkDT38QCxfYTMYEsdBS2YC6JGRZP77\nUfbefDOVv/glmsNB9Ny5QOgitfPLzxg8agwRUX0fkNfY5IE9bdQk2CAIM2Omc2nKpTyx+QmSnclc\nOfLKXp8zd9JUNN3Czi9X7g8M0jSpvPtumt94g8Qf30LijTcecIxvTzMA9iNsyO9Mt2rkjEskZ1wi\nRsCkuriFyt1N7PxyJVs+XoxptKBZsrBFzaCxLoVPnvHz6fMrQnfQHVWgmiYwTYnhNwkGel5vQlgl\npi2A19pOm9ZMi2jAE9WON74dr6UdS4QgKjqC6GgX8bExJMcnkBKbTJorh1RXKokRiUdt5t6Bol9+\nelLKJYQu/p23Ler0WgI3h3us8s1hS3Nhu2AoMfNycK+voXXpXuqe2IwtK5qY+TnYsw69oOoWC6Nm\nnsGIU2bx1fvvsPKlZ/nvHTcz9TsXc9JFC7ptpIZQSWbUKYNIyY7m3UWbeOP+DZxySR7jTu/6Tti7\nswFh07vMC4TmZ8p46CFKf3A95Xf+PzRXJJGnzqCutJjGynImn3NB738wnbz68g6sUjDxvFzEuyUY\nLX5+OeuX1HnquHf1vSRFJDEne06vzumIjCRr7Hh2fvkZM68MTfVdfc8faX7lVRJv+hFJNx/65+Qr\nakZzWrAkH9kEfIejWzVikiRr336Oim2fk5w9hFMW3El00lBa6rx42wN420KlANOUSENimhLTMNE0\ngW7TsVg1LDYN3arRKpoo85ew27uL3Z6d7PJsp1U0YWoGFs3CkJgh5MbmMiQ6i6zoEWRHZ5MZnUm0\n7chH1SvdU2FVCYtm04k8KQ1Xfgrta6to+WQvtQ9vxJmfQsy8nC7bIDRdZ9L8Cxg+fSbLnn6cL199\ngZKCDcz/8Z3EpvQ8qjchPZJLfpHPR09uY8WLO2lt8DL9oiEHlFSkKfFub8A+NBbRTeOoFhFBxqKH\nKbn6+5TdeiuZjz/Ozl2bEUIjb+r0vv1QgIDfoOmrBtwuwcypg6heWYnZ4kfXdO6deS8LP1zIXSvu\nItYey9S07ntHHWzYyafy/sP/R/XuXVg/XU7js88Sf+21JP74x12m9+1pxpYT06d1KMKxc9VKPnz0\nXwR8Xk694hryz71wf4N9Yg+r5QXMAJvrNrOhZgMbajawsWYjjb5Qh4EoWxSj4kdxQfw5DI8fzvC4\n4eTG5Pa58V45cv0y8vlYU43Px5/pM2j5uJS2z8rRHDqx5w3BObH7Fdi2f76cj/79IFKazL7hFkac\nMiusz5KmZMVLu9j0aRkjTkrl9KtGhBr6AF9xM7WLCoi/bHiPnw8QrKuj5MrvEWhsZOWUUUSnpnHJ\nb/4YVj668uprO6j6oJyk+YO59Pxh1D62CekzSL45tPBOs6+Za967hvK2ch6Z/QgTk7sfdNaZt62N\nhxd+j9HDR5Px/GtEz5/PoPv+2mW/d6PZR+WfVhNzbi5RM/q4jOdhmIbB8ueeZN07r5M6dBhzf3Q7\nCYMzejxub8teVlasZGXFSlZXrsYdDI1JyIrOYkLSBCYmT2RC8gRyYnJ6va630jfHsvFZGYA0u07s\n/Bxck5JpfG0XDS/uwLuzkdgLhqA5uv61GjF9JoPyRrD4n/ex+B9/pba0mBmXXdXjAB+hCU69LI+I\nKCur396Dtz3A2QvHYLHqeArqwCJwjAyv374lMZGMxx9n49VX0lRXy6Qz5/b6u+8jpaRwRQWGFRbO\nGwKElvj07W7anybGHsO/5/yba9+7lh999CMenf0o45LGhXV+R2Qkg7Ny2LVxHcMmTybtz3867M/K\nt2+68cz+HZjW3tTIOw/cS9nWzUw4+xxOu/p6dEvXd/IBM8CayjV8svcTPq/4nL2toSFK6ZHpnJN7\nDtMHTWdSyiTiHeGPsVCODxUYlCNiTXWR9MPxtC4tpeWTUnwlLSRcPuKw6yFEJyVz6d338PETi1j9\nxss0VpQz7+afYnV0P25BCMGUc3KIiLSy7PmdvPfIZubeMAb3pjocw+IPG4y6YhucTuP8OWirVhLx\nn2cwzprXp9lgV6wo+//t3Xd41eX5+PH3fWb2HoSQsARkKSAiQkURRHCPFm21tdiqWLVqa6vVrxU7\nLFpba3+1WqzWXRUXFkUEBBUHsvcIIgnZJISEzLOe3x/ngAlkkZPkJOF+XRfX+ZzPvMmVnPs8m6ga\ng3VcIvbAgLbGlvhMCk/i39P+zazFs5i9ZDbzps1jRNKIFu9ft3s3SV+sISc1Ft8tN2FpZhS0a18F\n2ARHGwekNaa8uJD5f/g/qsrKmHHLLxg26dxjY/TW8UX+FyzJXsKKfSuocFUQbgvn9F6nc83Qa5jY\neyJ9Y/p26DQbqv1pYlBBE6sEeirFc+C/Oyh+aiPxl59E5NjG2xGsNjvn3XAriemZfPziM7z24D1c\n8ZsHiYhpuTfNiLP7IBZhxcs7+eyfGxl4yEXEKce3jKfH5SJr2yYGDB2BeXsRubfcSsa/n8bibP2y\nl8YYvlq4B6/FcOPMb9dztsYElviscmON/vaDPDUylWemPcOsxbP4yeKf8Pdz/84ZaU0P8ncXF5Nz\n44308kJYRCTbVn/BgAlnNXm+K/sQjt5RzbazHI/S3H288Yf78LhczHzgoQZdZt1eNyvzVrJo7yI+\nyf2EKncV0fZozsk4h6l9pzKh9wTCbLpaW3emFXuq3Tj7xpBy22ic/WMpeyOLg+9+jfE23j1RRDjt\nwku57Nf3U5q7j1cfuJuKkuJWPWf4WelMunowjn2H8Ao4Bh/bTbU5X6/9itqqSk797tX0/tOfqF69\nmvy778H4Wu5KediG1YWEV3iRYbHER32bUKwxDRfsqS8tKo3npz9P76je3Lz0Zj7c+2Gj9/ZWVrFv\n9my8B8vp99RTnHzWZHav/oLaqspGzzceH668yjYtZ9qYom++5rU5d+Pz+Zg5Zy5pJw3BGMOWki08\ntOohzp1/Lj9f/nO+zP+S6f2m8+TUJ/n4qo956KyHODfzXE0KPYAmBtWurJF2kmaNIOo76VR+nk/J\nf7biq3Y3ef6AMadz5b0PUlV2gFd/ezcH8vNa9Zxhp6fSJ8xCdq2XFa9l+ac2aKVNS98nOjGZzBGn\nEHvRhaT8+tcc+uADiv40l9Z0xjDG8Mnbu6kQHzPrlRag3trPFa5Gr02NTOW56c8xPHE4d318Fy9v\nf4m3W28AACAASURBVLnBM43bTd6dd1K3cxd9/vYY4cOHM+KcqXjdbrauWNboPd0FVeDxtUv7Qv6u\n7cz/3b3YHE6umvMwngQnT296mkveuYTvv/d93tz1JmekncETU55g2cxlzJkwh++kf0d7EPUwmhhU\nuxOrEHfRAOK/N5i6b8opfnIjntKaJs/vM3QEMx/4Ex63i1cf+DXFe/e0+IzqtUWID6LP7M3OVYV8\nOj+rVR/q+7O/IWfLJkadf+GRqRoSZv2YhOt+RNmLL3Lg2WdbvMeujfuxlbmpHBhJv5SGdfpHSgxN\nJAbwN0jPmzaPczLOYe5Xc7n707updldjjKHwd7+j6tNP6fXAb4maNAmA1AEn0XvIMNYv/h8+n/eY\n+7ly/FPjBltiyN68gTf+cD9h0dHEXz+Fuzbfz7Q3p/H39X8nISyBOWfOYflVy3n07EeZ1GeSjhLu\nwTQxqA4TeVoqyT8ZgbfSTfE/N1KX3fTc3qn9B3L1g49gtduZ/7t7Kdy9q8lzjddH5ZcFOPrFMOp7\ngzh1Sgabl+fy1cJvWoxp3aJ3sTmcjJxy/pF9IkLK3XcTPWM6xX9+lPL//a/pZxvDR29kUS4+rmhk\nPWdLlAMEvBXHViXVF24L52+T/8bPR/+cxXsXc83715D11z9ycP4bJM6+ifiZMxucP2bGJZQXFbJn\n7epj7uXKrcQS7cAW1/o2kqNlrfmSN+c+QE0UvHDqFn639WGKqoq4ZdQtLLpiEc/PeJ4rB1+pg8tO\nEJoYVIdyDogj5WenYgmzsv/pTVRvbLodIaF3OlfPmYszMpL5f7iP3B2Nz45StaYI74Faos/JQESY\n+N2TOHlCGmve28vGZfsavQb8XS+3r1zB8LPPJTyqYbWLWCz0fvhhIsaNI//e+6j6/PNG77FnYwm+\nkjoK0h2c1v/YNQ/EKlgCS3y2xCIWbjjlBp6a+hTDPs3F+/TLFE46mdhbbz7m3EHjziQ6MZk1C98+\npmTkyj2Eo0/beiPlVOTwt5fv5Z1Hf09xZDXvjcvn3KHTeXHGiyy8fCGzT51Nn+jg515S3YsmBtXh\n7MkRJP9sFI4+0Rz4704qluU0We0Tm9KLq+Y8TGRcAm8+9FuyN29ocNy4/QPrHH1jjsyNJCJMvmYI\nA0Yns3J+Fts/z2/03l++9RrG5+O0iy5v9LjF4aDPE//A2b8/ubfeRu22hutFed0+lv53B6UWHzMu\nanohHmuMo8USQ30jtlVx7fvV7BuRwp3js/ju/77LyryVDX5GFquV0y+5grwdW8netP7Ifl+dB09J\nDY4WRh7XV1BZwAtbX+Da96/ljr9fhefdjdSmOpl45y18eO1HzJkwh1Epo7SL6QlME4PqFNZIO8k/\nHUnE6BQqlmRTNn8XxtN4L6DoxCSumjOXuJRevP3wg+xZ9231ScWKXHwVLmKmNewbb7FamHb9cDKG\nxrP8xR18vb5hyaS8uIhNSz9gxOTziO/Vu+k4o6PJeHoelthYcm66CVdu7pFjGz/ah6fczeYUC+ef\nktb0PWKczbYx1Fe16ivyfvFLwkaOYOoLi/jn+fPwGi83L72Zaxddywd7P8Dt8zfej5wynZjkFFa+\n+sKRpOHOqwID9vSmSwzGGL4++DXPbXmOH7z3A6a9OY0/r/kz8VuqmLg5kfSRI7nnkVe4eOhl2qNI\nAZoYVCcSm4X4mYOJmZpJ9bpi9j+zBW9V4z2WIuPimfnAn0jsk8mCR//IrlWfUZdTwaHlOUSMTiFs\n4LGzrVrtFmbMPoXU/jF8+MxW9m0/cOTYp688h8ViYfyVV7cYpz01lcyn52Fcbvb99AY8ZWWU769m\n1cJvyLJ5ueC8/libmY/IX2JoOTFUffEF+266CXtmBhlPPYUlIoIJ6RNYcOkC7h9/P6U1pfzq418x\n5fUpPPjFg3yc/wkjL7mYoj272bFyBQCuvMCI53qJodZTy+b9m3l95+vct/I+ps6fymULLuMva/+C\n13j5+eif82fbLfRb62bwGROZec/vsTs1Iahv6VxJKiSq1xdz4I1dWCLtJFw1pNEPevAvAvPW3DmU\nf5PPhYNmY3eGkXrHmGZHOtdWuXnnr+spL6nh0ttHUVm6nQWP/oEJ37uGM7/7/dbHuHYtOdf/BHu/\nfmwcfw+5udW8kuBm6f9NIcrZ9PMrludQsTib9N9PQJpY4rNy5Wfk3nILjsxMMp/7D7bEY9srvD4v\nn+V/xsI9C1mxbwU1nhrEwMVfpBNdY6fwqkwu2j2R9LJEnpr0PiU1JRTXFLO/ej9e4++9lBCWwLhe\n4ziz95mMTxtPr/BUPnpuHhs/fI/hZ09l2k23BbVynepeWjtXkiYGFTKu/EoOvLIDT0mNf5bW8/s1\nGC18WHVeGXn/WIXNa6NyrIsR35vRYv13VXkdbz+6jsqDZXhrXiE6IZ5rHnqsxSm/j1b52Wd89tBb\n7M64kK+khKGXnMI9M05u9pqqtUWUzd9Fr7vGYksKP+b4oY8+Iu+OO3EMGEDmf57FFt/yAD23182m\nkk3sOLCD7K+3YX95I2XpwpVRP6UgvIRnTl5IcngyyRHJpEWmMTRxKMMShtErsteRn1VddTULH3+Y\nvRvWMvbiK5j0gx93yYXoVcfRSfRUl+foHUXKz0f7Z2n9NJfqDcVEnJpC+NAErHFOfLUeaneUUflF\nPhHOGLY6vmLzm0spKNvN5Fk3Nbt2cGSskwtvGcaL9/wKd00l4797x3EnBYAD8SfzdeaFJJZu4uYd\nL5Lx/X+0eE39sQz1E4MxhgPP/ofiRx8lbMQIMub9q1VJAcButXNa6mmclnoaDIU15i0+e/klUvsl\nMOis0Vw05SfNXn+wsIAFf/kjpbk5nHfjrZwSxOSBqucLKjGISALwGv61m/cCM40xZUedkwG8AKTi\nX2J1njHm8cCxOcANwP7A6fcGFu5RJwiLw0rcjP5End6LQ5/kUr1xP9Vri749QSBiVAqxM/qTFnUG\nEa8nsOrt18nbuZ3zZ99O78GNf3uvrihnyb/m4qktJGXgTFYtqMARlsuIs9Nb3dumaG8Fi5/eSkxq\nBE85knk4L5aKm2/AefvtJP7keqSJKphvE8O3PZM8ZWUU/N/9VC5bRvT559N77p+whB9bmmit0y66\nnNo95VAAWV9/xejJ6Y2urezzedm87EM+fulZLBYLV/7md/Q9ZVSbn6tODEFVJYnII8ABY8xcEbkH\niDfG3H3UOWlAmjFmnYhEA2uBy4wx2wKJodIY8+jxPFerknou4/Xhyq3EV+VGbBYcmdHHtCfs3bSe\nxU/+jcoDpQwaN4FTppxPxohTsNrs1FVXs+Ozj/ny7deorajgvJtuY9AZk1j89BayN5cy+IxUzr56\nCI7w5r8T5WeV8d4/NxMWaaNgTCxPrt7LspvG4nh8LocWfUDYsGGk3vsbIsYeWyr31XrIn/MFsTP6\nE3VmCgfffof9jz2Gt6qK1Lt+SfyPftQuXUErVuZSsfAb3s15gsTB/TjjiqvIGDYSq81GTeUhdq/+\ngnXvLaBkXzYZw09h+s/uICap5TUrVM/VKW0MIrITOMcYUxBIACuMMUNauGYB8A9jzBJNDKqtXDXV\nrFn4DuveX0BddRWIEBYReWSiudQBJ3HejbeR2t+/ToLPZ1i7aC+rF35DeLSD8ZcNYPC4XliPmo3U\nVethw5Ic1ry/l9iUCM6dPZxpT33OWYOT+Oc1p2GMOTKvkqe4mPDRo4m99BIiJ0zA3qcPYrHgq6uj\n4A9fYXHsp3LZ/8NTWEj42NPo9dvfEjb42NHSbXXgjV3Ubi+ldOIhVr76AtXlBxGLBUd4OHVVVQDE\n9+7DxJnXMnj8RB2XoDotMRw0xsQFtgUoO/y+ifP7AZ8AI4wxFYHEMAsoB9YAvzy6KqoxmhjUYR63\nm70b1lK892uqy8uJTkwi/eRhpJ88vNEPwqK9FXzy350UZx8iPMZB3xGJJPaOBKA0r5JvNpVQV+Vh\n0OmpnHPNEJ5dlc3cRTt499aJnNLn219tX3U1B998i7KXXsKVnQ2A2O2Iw4GvpoaIc36Lr7IIi2U9\nibN+TOSkSe3+wVz0j/VYwmwk/3QkHreb3au/oCQnm9qqSmKSkskYPpJeAwdrQlBHtFtiEJGlQGMT\n698HPF8/EYhImTGm0dY0EYkCPgb+aIx5K7AvFSjB3/bwe/xVTtc3cf2NwI0AmZmZp2UH/hiVOl7G\nZ9i3/QBbV+aTv+sgtYGxFGFRdvoMiWfU1ExS+8dQ6/Zy1iPLGZwaxcs/Hd/4vYyhLiuLmo0bcefk\nYFxuLFFReKqHY3FGkXrH6R3zf/Aa8h74nKjxacRdNKBDnqF6nnbrlWSMmdrMQ4pEJK1eVVKjE+GI\niB14E3j5cFII3Luo3jlPAwubiWMeMA/8JYaW4laqKWIRMocnkjk80f/BXu0BwBlha/Dt+u31eew/\nVMdjM5turBURwgYPPqaK6MAbu6jdeaCJq4LnKa0Bjw97WmSHPUOduILtxPwucF1g+zpgwdEnBKqY\nngG2G2P+etSx+vMKXA5sCTIepY6LiBAWaScs0t4gKXh9hnmf7GFEegwTTzp28FlLrLFOfJXuJqf9\nCJY739+WYm/HpTyVOizYxDAXOE9EsoCpgfeISG8ROdztdCLwQ+BcEdkQ+HdB4NgjIrJZRDYBk4E7\ng4xHqXbxwZZCvimpYvbZA9tUR2+Lc4Jpfl2GYLgLqsAq2JPb3uVVqaYENY7BGFMKTGlkfz5wQWB7\nJdDoX5Yx5ofBPF+pjuDzGf7fR1kMSIpkxoimJ8trjjX28EpuddgS2n8eIldBFfaUiHZb41mp+vS3\nSqmjfLitkB2Fh7j13JOanSyvOdbYwCC3g62ffvt4uAuqtH1BdRhNDErV4/MZHl+2m36JEVxyatPT\nc7fkSImhFQv2HC9vpQvfIRf2NG1fUB1DE4NS9SzZXsT2ggpuPXcQNmvb/zwsYTbEacVb3v4lBneB\nf/CalhhUR9HEoFSAMYa/L8uib2IEl41qe2nhMGusE48mBtUNaWJQKmDZ9mK25ldwy+STgiotHGaN\nc3ZYicEa48AaaW/3eysFmhiUAvylhceXZZGREM7lo9Pb5Z62WGeHND678it1/ILqUJoYlAKW7yxm\nc145t04+CXs7lBbAX2LwVbox7vYb5GY8Pjz7a7QaSXUoTQzqhGeM4fGlWfSJD+eKMX3a7b7WOH/P\npPZsZ3AXVYPPaGJQHUoTgzrhLdlWxMbc9i0tANji/QPbvGW17XZPbXhWnUETgzqheX2GPy/eyYCk\nSL57WvuVFgCs8YGxDGXtWGIoqETsFmyJOhWG6jiaGNQJ7e31eWQVV3LX+UPapSdSfdYYJ1jAc7B9\nSwz2XpFIG0dkK9UamhjUCavO4+WxJbsYmR7LjBGNLTkSHLEK1hhnu5UYjDH+OZK0Gkl1ME0M6oT1\nyqoc8g7W8OvpQzpslTNrvBNPO7UxeMtdmBqPJgbV4TQxqBPSoVo3//hoNxMGJvKdk5I67Dm2+LB2\nKzG4CwJrMGhiUB1ME4M6IT254mtKq1zcPf3kDl0T2RrnxFtRh/EGP5bBna89klTn0MSgTji5ZdX8\ne+U3XD46nVMz4lq+IAi2+DD/gj3tMMuqu7AKa2IYFmdQy6go1aKgEoOIJIjIEhHJCrzGN3He3sBK\nbRtEZM3xXq9Ue3rkg50I8Kvzh3T4sw53WW2PdgZXfiUOLS2oThBsieEeYJkxZhCwLPC+KZONMaOM\nMWPbeL1SQVuXU8a7G/O5cdIAesd1/FgAW1xgkFuQcyb5aj14S2uxp+scSarjBZsYLgWeD2w/D1zW\nydcr1WrGGH6/cBvJ0U5mnz2wU55pjXOCgOdAcCWGI+0LOnme6gTBJoZUY0xBYLsQSG3iPAMsFZG1\nInJjG65XKmhvrctjfc5BfjVtCJGdVE8vNou/Abq0Jqj7uPL9PZIcmhhUJ2jxr0NElgKNjf65r/4b\nY4wREdPEbb5jjMkTkRRgiYjsMMZ8chzXE0goNwJkZma2FLZSDZTXuPnTou2Myohr96kvWmJLDMdd\nGmyJoRJLtB1rtKOdolKqaS0mBmPM1KaOiUiRiKQZYwpEJA0obuIeeYHXYhF5GxgHfAK06vrAtfOA\neQBjx45tMoEo1Zi/friTA1Uunps1DksnTydhSwyjZnNJUPdw51dqaUF1mmCrkt4FrgtsXwcsOPoE\nEYkUkejD28A0YEtrr1cqWFvyynnxy2yuHd+XEemxnf58W1I4vmoPvmp3m643bh/u4mptX1CdJtjE\nMBc4T0SygKmB94hIbxF5P3BOKrBSRDYCXwHvGWM+aO56pdqLz2e4f8EWEiId/HJax3dPbczhmVA9\nbaxOchdWgU8bnlXnCaoFzhhTCkxpZH8+cEFgew9w6vFcr1R7eWlVNutzDvKX751KbHho1ki2Jfq7\nrHpKa3BkRB/39d82POsYBtU5dOSz6rFyy6p5eNEOzhqUxBVj2mcd57awJYT7u6yWtK1nkju/Egmz\nYk0Ia+fIlGqcJgbVIxlj+M1bmzHAQ5eP7ND5kFoidgvWGGfbq5Lyq3D0jgrp/0GdWDQxqB7pjbW5\nfJpVwj0zTiYjISLU4WBLDMPThrEMxhtYg0HbF1Qn0sSgepyiilp+v3Ab4/olcO0ZfUMdDuDvmdSW\nxOApqQaPD7u2L6hOpIlB9Sg+n+Gu+RtxeX3MvXJkp49ZaIotORxflQdv1fF1WXUFpsLQMQyqM2li\nUD3Kfz7fy6dZJdx/0TAGJHedD1N7qv8bv7uw6riuc+dVgs2CLTn01WHqxKGJQfUY2wsqeHjRDqYO\nTeUH47rWtCn2Xv4Pdk9R9XFd58qrxJ4WiVi7RslHnRg0Mageodbt5fZX1xMbYefhK0PbC6kxlmgH\nlggb7qLWlxiM1+DOPYSzDWMflAqGLgWleoQHFmxlV1Elz806ncQoZ6jDOYaIYEuNwF3Y+hKDu7AK\n4/bhyNTEoDqXlhhUt/fa6hxeW7OPWyYP5JwhKaEOp0n21Ej/h71p3RyQrpwKAByZMR0ZllLH0MSg\nurXNueXcv2Ar3zkpiV+cF5q5kFrL3isSU+dt9frPruwKLNGOI8uDKtVZNDGobqusysXNL68lKdLB\n41ePwtpFuqY25XADdGt7JtXlHMKZGd3l2ktUz6eJQXVLLo+P2S+tpbiijieuGdMl2xWOdjxdVr2H\nXHgP1OLoq9VIqvNp47Pqdg7Pg7TqmwP87apRjM6MD3VIrWIJt2FNDMO171CL59btDbQvaGJQIaAl\nBtXtPLF8N2+uy+WOqYO4bHToZk1tC2ffGFzZFS02QNftLkOcVhx9us4gPXXi0MSgupV31ufx6Ie7\nuHx0OrdPGRTqcI6bo28Mvko33hZmWq3NOohzQCxi1T9R1fmC+q0TkQQRWSIiWYHXY8r0IjJERDbU\n+1chIncEjs0Rkbx6xy4IJh7Vs324tZBfzt/ImQMSmdsFB7G1hrOfv2rocFVRYzylNXgP1BI2qHtU\nkameJ9ivI/cAy4wxg4BlgfcNGGN2GmNGGWNGAacB1cDb9U557PBxY8z7R1+vFMDKrBJufWU9I9Nj\nefq6sTht1lCH1Ca25AgkzIYru+nEUJt1EADnoLjOCkupBoJNDJcCzwe2nwcua+H8KcDXxpjsIJ+r\nTiCr9x7ghhfWMCA5kudmnU6Us/v2mRCL4OwbTV12eZPn1GWVYY11YksK78TIlPpWsIkh1RhTENgu\nBFJbOP9q4L9H7btNRDaJyLONVUUdJiI3isgaEVmzf//+IEJW3cnKrBJ+9MxXpMWF8eJPziAuwhHq\nkILm6B+Lp7gGb3ndMcd8tR5qd5URdnJ8t6wqUz1Di4lBRJaKyJZG/l1a/zzj72bRZFcLEXEAlwDz\n6+1+EhgAjAIKgL80db0xZp4xZqwxZmxycnJLYaseYMm2Iq5/bjV9EyN47cYzSY7u+mMVWiN8RBIA\n1RuP/YJTs7kE4/YRMaal71hKdZwWy+TGmKlNHRORIhFJM8YUiEgaUNzMrWYA64wxRfXufWRbRJ4G\nFrYubNXTvb0+l7vmb2JE7xiev35cjygpHGZPCsfeJ4rqDcVET+rT4FjVumJsSeE6cZ4KqWCrkt4F\nrgtsXwcsaObc73NUNVIgmRx2ObAlyHhUN2eM4bElu7jztY2c3i+el37aM6qPjhYxKgV3fhXu4m9n\nW/UcqMX1TTkRY1K0GkmFVLCJYS5wnohkAVMD7xGR3iJypIeRiEQC5wFvHXX9IyKyWUQ2AZOBO4OM\nR3VjtW4vd7y2gceXZXHlmD68cP0ZRIfZQx1Wh4g4NRkEKj/PB/wJsfy9PWAVrUZSIRdU9w5jTCn+\nnkZH788HLqj3vgpIbOS8HwbzfNVz5JZVc8sr69m47yC/On8IPztnYI/+1myNdhA5Po2qLwpwDojF\nuLzUbC0lZno/bHE9oy1FdV/dt9+f6jGW7yjmztc34PUanrp2DNNHpLV8UQ8Qd+EAXPsOceCVHQDY\n+0QRfVafFq5SquNpYlAhU+fx8tclu/jXx3sYmhbDk9eMoV9SZKjD6jRis5B03XCq1xdjjXYQNjRB\n13ZWXYImBhUSW/LK+cXrG9hVVMn3x2XywMXDCLN3z9HMwbBGO47pmaRUqGliUJ2q1u3lieW7eXLF\n1yREOvjPrNOZ3IWX41TqRKSJQXWapduKmPO/reSW1XD56HQeuHhYj+yKqlR3p4lBdbjtBRU8/MEO\nVuzcz6CUKP57w3jOHHhMJzWlVBehiUF1mJzSah5buot3NuQR7bRx7wUnM2tif+y6xoBSXZomBtXu\ndhYe4skVu/nfpgJsFuGmSQO5+eyBxEb0zMFqSvU0mhhUu/D6DMt3FPPyqmyW79xPhMPKrAn9uGHS\nAFJjwkIdnlLqOGhiUEEpLK/l1dU5vLZ6HwXltSRHO7lz6mCum9BXG5aV6qY0MajjdqDKxeKthby/\nuYDPvy7F6zOcNSiJBy4expShqdqGoFQ3p4lBtUpuWTUf79rPB1sKjySDvokRzD57ADPHZtA38cQZ\nsaxUT6eJQTWqotbNV3sO8GnWfj7NKmFPSRUA/RIjuGnSAC4Ymcbw3jE9eqI7pU5UmhgUXp9hz/5K\n1uccZF1OGetyysgqrsQYCLdbOXNgIteO78tZg5I4KSVKk4FSPZwmhhOIMYaiijq+KaliV9EhthdU\nsL2ggp1Fh6h1+wCIDbczOjOOi0/pzdh+CYzpG4fTduLNYaTUiSyoxCAi3wPmAEOBccaYNU2cNx14\nHLAC/zbGHF7QJwF4DegH7AVmGmPKgonpRGaM4WC1m4LyWooqaimsqCW7tJq9JVXsLfX/O5wAAOIj\n7AxNi+EH4/oyNC2a0ZnxDEiKxGLREoFSJ7JgSwxbgCuAfzV1gohYgSfwr+CWC6wWkXeNMduAe4Bl\nxpi5InJP4P3dQcbUYxhjqHX7KKt2UVbt4mC1279d5aIssF1S6aKo3J8ECitqcXl8De5htwoZCRH0\nT4xk4klJ9EuKpH9iJCelRJEa49RqIaXUMYJdwW070NKHyzhgtzFmT+DcV4FLgW2B13MC5z0PrKCL\nJQZjDF6fweMz+Ezg1eff5/UZ3D5DnduLy+vD5fFR5/G/+re9R94f2e/1Uef2Ue32UFXnoarOS2Wd\nh2qXh8o6b2CfJ7DPi9dnmowt2mkjIcpBakwYozLi6BUbRq+YMHrFhpF6+DXaiU27jyqljkNntDGk\nA/vqvc8FzghspxpjCgLbhUCHLnb792VZvLMhz//Bbgxeb+DVV+/Dv14S8PoMzXwuB8VhsxDltBHp\ntBLpsBHltBEbbic9LoxIh43Iw8ecNuIjHMRH2P2vkQ7iIuzEhTtw2PQDXynV/lpMDCKyFOjVyKH7\njDEL2isQY4wRkSY/hkXkRuBGgMzMzDY9IyXaydC0GKwi2CyCxXLUqwjWRvbVf289vM/qf7VbBafN\nisNmwWmz4LBZcFgtOO3WwGvgvc1y5DyHzYJV6/GVUl1Ui4nBGDM1yGfkARn13vcJ7AMoEpE0Y0yB\niKQBxc3EMQ+YBzB27Ng2fY+/elwmV49rW1JRSqkTRWfURawGBolIfxFxAFcD7waOvQtcF9i+Dmi3\nEohSSqm2CSoxiMjlIpILnAm8JyKLA/t7i8j7AMYYD3ArsBjYDrxujNkauMVc4DwRyQKmBt4rpZQK\nITGmg1pXO9DYsWPNmjWNDplQSinVBBFZa4wZ29J52q1FKaVUA5oYlFJKNaCJQSmlVAOaGJRSSjWg\niUEppVQD3bJXkojsB7JDHcdRkoCSUAfRSt0pVuhe8XanWKF7xdudYoWuGW9fY0xySyd1y8TQFYnI\nmtZ0A+sKulOs0L3i7U6xQveKtzvFCt0v3vq0KkkppVQDmhiUUko1oImh/cwLdQDHoTvFCt0r3u4U\nK3SveLtTrND94j1C2xiUUko1oCUGpZRSDWhiaGcicpuI7BCRrSLySKjjaYmI/FJEjIgkhTqW5ojI\nnwM/100i8raIxIU6pqOJyHQR2SkiuwNrmHdJIpIhIstFZFvg9/T2UMfUGiJiFZH1IrIw1LE0R0Ti\nROSNwO/rdhE5M9QxHS9NDO1IRCbjX8f6VGPMcODREIfULBHJAKYBOaGOpRWWACOMMacAu4DfhDie\nBkTECjwBzACGAd8XkWGhjapJHuCXxphhwHjgli4ca32345+6v6t7HPjAGHMycCrdI+YGNDG0r5uB\nucaYOgBjTJMr0nURjwG/Brp8Q5Mx5sPA2h4AX+JfCbArGQfsNsbsMca4gFfxf0nocowxBcaYdYHt\nQ/g/uNJDG1XzRKQPcCHw71DH0hwRiQUmAc8AGGNcxpiDoY3q+GliaF+DgbNEZJWIfCwip4c6oKaI\nyKVAnjFmY6hjaYPrgUWhDuIo6cC+eu9z6eIftgAi0g8YDawKbSQt+hv+LzG+UAfSgv7AfuA/gWqv\nf4tIZKiDOl4trvmsGhKRpUCvRg7dh//nmYC/eH468LqIDDAh6vrVQqz34q9G6jKai9cYsyBwTFsQ\nIgAAAb1JREFUzn34q0Je7szYeiIRiQLeBO4wxlSEOp6miMhFQLExZq2InBPqeFpgA8YAtxljVonI\n48A9wP2hDev4aGI4TsaYqU0dE5GbgbcCieArEfHhny9lf2fFV19TsYrISPzfbDaKCPirZdaJyDhj\nTGEnhthAcz9bABH5MXARMCVUybYZeUBGvfd9Avu6JBGx408KLxtj3gp1PC2YCFwiIhcAYUCMiLxk\njLk2xHE1JhfINcYcLoG9gT8xdCtaldS+3gEmA4jIYMBB15tEC2PMZmNMijGmnzGmH/5f5jGhTAot\nEZHp+KsSLjHGVIc6nkasBgaJSH8RcQBXA++GOKZGif/bwDPAdmPMX0MdT0uMMb8xxvQJ/K5eDXzU\nRZMCgb+hfSIyJLBrCrAthCG1iZYY2tezwLMisgVwAdd1wW+23dU/ACewJFDK+dIYMzu0IX3LGOMR\nkVuBxYAVeNYYszXEYTVlIvBDYLOIbAjsu9cY834IY+pJbgNeDnxB2APMCnE8x01HPiullGpAq5KU\nUko1oIlBKaVUA5oYlFJKNaCJQSmlVAOaGJRSSjWgiUEppVQDmhiUUko1oIlBKaVUA/8fqO8P8IYR\nWJ0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1132c6d30>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "x = np.linspace ( -7 , 7 , 200)\n",
    "plt.plot(x , f(x))\n",
    "plt.plot(x, D_f(x))\n",
    "plt.plot(x, D2_f(x))\n",
    "plt.plot(x, D3_f(x))\n",
    "plt.plot(x, D4_f(x))\n",
    "plt.plot(x, D5_f(x))\n",
    "plt.plot(x, D6_f(x))\n",
    "plt.show ()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading training data...\n",
      "    Epoch      |    Train err  |   Test error  \n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-9d16c01e73b4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    192\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    193\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_epochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 194\u001b[0;31m         \u001b[0mprint_perf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    195\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0midxs\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mbatch_idxs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    196\u001b[0m             \u001b[0mgrad_W\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_images\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midxs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_labels\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midxs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-7-9d16c01e73b4>\u001b[0m in \u001b[0;36mprint_perf\u001b[0;34m(epoch, W)\u001b[0m\n\u001b[1;32m    184\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mprint_perf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    185\u001b[0m         \u001b[0mtest_perf\u001b[0m  \u001b[0;34m=\u001b[0m \u001b[0mfrac_err\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_images\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_labels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 186\u001b[0;31m         \u001b[0mtrain_perf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfrac_err\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_images\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_labels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    187\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"{0:15}|{1:15}|{2:15}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_perf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_perf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    188\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-7-9d16c01e73b4>\u001b[0m in \u001b[0;36mfrac_err\u001b[0;34m(W_vect, X, T)\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfrac_err\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mW_vect\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mT\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpred_fun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mW_vect\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredictions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfrac_err\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-7-9d16c01e73b4>\u001b[0m in \u001b[0;36mpredictions\u001b[0;34m(W_vect, inputs)\u001b[0m\n\u001b[1;32m     53\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mlayer\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlayer_specs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m             \u001b[0mcur_weights\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mW_vect\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 55\u001b[0;31m             \u001b[0mcur_units\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlayer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward_pass\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcur_units\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcur_weights\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     56\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcur_units\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-7-9d16c01e73b4>\u001b[0m in \u001b[0;36mforward_pass\u001b[0;34m(self, inputs, param_vector)\u001b[0m\n\u001b[1;32m     77\u001b[0m         \u001b[0mparams\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam_vector\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'params'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     78\u001b[0m         \u001b[0mbiases\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam_vector\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'biases'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 79\u001b[0;31m         \u001b[0mconv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconvolve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxes\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdot_axes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'valid'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     80\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mconv\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mbiases\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     81\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/fabian/anaconda3/lib/python3.6/site-packages/autograd/core.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     67\u001b[0m                 \u001b[0mprogenitors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprogenitors\u001b[0m \u001b[0;34m&\u001b[0m \u001b[0mactive_progenitors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 69\u001b[0;31m         \u001b[0mresult_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margvals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     70\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mprogenitors\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mnew_node\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparents\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprogenitors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/fabian/anaconda3/lib/python3.6/site-packages/autograd/scipy/signal.py\u001b[0m in \u001b[0;36mconvolve\u001b[0;34m(A, B, axes, dot_axes, mode)\u001b[0m\n\u001b[1;32m     28\u001b[0m             \u001b[0mignore_A\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m             \u001b[0mconv\u001b[0m     \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mconvolve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mB\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mA\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxes\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdot_axes\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdot_axes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mignore_A\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mignore_B\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mconv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mmode\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'full'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/fabian/anaconda3/lib/python3.6/site-packages/autograd/core.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     67\u001b[0m                 \u001b[0mprogenitors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprogenitors\u001b[0m \u001b[0;34m&\u001b[0m \u001b[0mactive_progenitors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 69\u001b[0;31m         \u001b[0mresult_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margvals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     70\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mprogenitors\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mnew_node\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparents\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprogenitors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/fabian/anaconda3/lib/python3.6/site-packages/autograd/scipy/signal.py\u001b[0m in \u001b[0;36mconvolve\u001b[0;34m(A, B, axes, dot_axes, mode)\u001b[0m\n\u001b[1;32m     44\u001b[0m     \u001b[0mA_view\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mA\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mflipped_idxs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m     \u001b[0mall_axes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdot_axes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 46\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0meinsum_tensordot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mA_view\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mB_view\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mall_axes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0meinsum_tensordot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mA\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mB\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreverse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/fabian/anaconda3/lib/python3.6/site-packages/autograd/scipy/signal.py\u001b[0m in \u001b[0;36meinsum_tensordot\u001b[0;34m(A, B, axes, reverse)\u001b[0m\n\u001b[1;32m     54\u001b[0m         \u001b[0mA_axnums\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_A\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msum_axnum\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mi_sum\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m         \u001b[0mB_axnums\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_B\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msum_axnum\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mi_sum\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mnpo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meinsum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mA\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mA_axnums\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mB\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mB_axnums\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     57\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mpad_to_full\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mA\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mB\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "\"\"\"Convolutional neural net on MNIST, modeled on 'LeNet-5',\n",
    "http://yann.lecun.com/exdb/publis/pdf/lecun-98.pdf\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import print_function\n",
    "\n",
    "import autograd.numpy as np\n",
    "import autograd.numpy.random as npr\n",
    "import autograd.scipy.signal\n",
    "from autograd import grad\n",
    "from builtins import range\n",
    "import data_mnist\n",
    "\n",
    "convolve = autograd.scipy.signal.convolve\n",
    "\n",
    "class WeightsParser(object):\n",
    "    \"\"\"A helper class to index into a parameter vector.\"\"\"\n",
    "    def __init__(self):\n",
    "        self.idxs_and_shapes = {}\n",
    "        self.N = 0\n",
    "\n",
    "    def add_weights(self, name, shape):\n",
    "        start = self.N\n",
    "        self.N += np.prod(shape)\n",
    "        self.idxs_and_shapes[name] = (slice(start, self.N), shape)\n",
    "\n",
    "    def get(self, vect, name):\n",
    "        idxs, shape = self.idxs_and_shapes[name]\n",
    "        return np.reshape(vect[idxs], shape)\n",
    "\n",
    "def make_batches(N_total, N_batch):\n",
    "    start = 0\n",
    "    batches = []\n",
    "    while start < N_total:\n",
    "        batches.append(slice(start, start + N_batch))\n",
    "        start += N_batch\n",
    "    return batches\n",
    "\n",
    "def logsumexp(X, axis, keepdims=False):\n",
    "    max_X = np.max(X)\n",
    "    return max_X + np.log(np.sum(np.exp(X - max_X), axis=axis, keepdims=keepdims))\n",
    "\n",
    "def make_nn_funs(input_shape, layer_specs, L2_reg):\n",
    "    parser = WeightsParser()\n",
    "    cur_shape = input_shape\n",
    "    for layer in layer_specs:\n",
    "        N_weights, cur_shape = layer.build_weights_dict(cur_shape)\n",
    "        parser.add_weights(layer, (N_weights,))\n",
    "\n",
    "    def predictions(W_vect, inputs):\n",
    "        \"\"\"Outputs normalized log-probabilities.\n",
    "        shape of inputs : [data, color, y, x]\"\"\"\n",
    "        cur_units = inputs\n",
    "        for layer in layer_specs:\n",
    "            cur_weights = parser.get(W_vect, layer)\n",
    "            cur_units = layer.forward_pass(cur_units, cur_weights)\n",
    "        return cur_units\n",
    "\n",
    "    def loss(W_vect, X, T):\n",
    "        log_prior = -L2_reg * np.dot(W_vect, W_vect)\n",
    "        log_lik = np.sum(predictions(W_vect, X) * T)\n",
    "        return - log_prior - log_lik\n",
    "\n",
    "    def frac_err(W_vect, X, T):\n",
    "        return np.mean(np.argmax(T, axis=1) != np.argmax(pred_fun(W_vect, X), axis=1))\n",
    "\n",
    "    return parser.N, predictions, loss, frac_err\n",
    "\n",
    "class conv_layer(object):\n",
    "    def __init__(self, kernel_shape, num_filters):\n",
    "        self.kernel_shape = kernel_shape\n",
    "        self.num_filters = num_filters\n",
    "\n",
    "    def forward_pass(self, inputs, param_vector):\n",
    "        # Input dimensions:  [data, color_in, y, x]\n",
    "        # Params dimensions: [color_in, color_out, y, x]\n",
    "        # Output dimensions: [data, color_out, y, x]\n",
    "        params = self.parser.get(param_vector, 'params')\n",
    "        biases = self.parser.get(param_vector, 'biases')\n",
    "        conv = convolve(inputs, params, axes=([2, 3], [2, 3]), dot_axes = ([1], [0]), mode='valid')\n",
    "        return conv + biases\n",
    "\n",
    "    def build_weights_dict(self, input_shape):\n",
    "        # Input shape : [color, y, x] (don't need to know number of data yet)\n",
    "        self.parser = WeightsParser()\n",
    "        self.parser.add_weights('params', (input_shape[0], self.num_filters)\n",
    "                                          + self.kernel_shape)\n",
    "        self.parser.add_weights('biases', (1, self.num_filters, 1, 1))\n",
    "        output_shape = (self.num_filters,) + \\\n",
    "                       self.conv_output_shape(input_shape[1:], self.kernel_shape)\n",
    "        return self.parser.N, output_shape\n",
    "\n",
    "    def conv_output_shape(self, A, B):\n",
    "        return (A[0] - B[0] + 1, A[1] - B[1] + 1)\n",
    "\n",
    "class maxpool_layer(object):\n",
    "    def __init__(self, pool_shape):\n",
    "        self.pool_shape = pool_shape\n",
    "\n",
    "    def build_weights_dict(self, input_shape):\n",
    "        # input_shape dimensions: [color, y, x]\n",
    "        output_shape = list(input_shape)\n",
    "        for i in [0, 1]:\n",
    "            assert input_shape[i + 1] % self.pool_shape[i] == 0, \\\n",
    "                \"maxpool shape should tile input exactly\"\n",
    "            output_shape[i + 1] = input_shape[i + 1] / self.pool_shape[i]\n",
    "        return 0, output_shape\n",
    "\n",
    "    def forward_pass(self, inputs, param_vector):\n",
    "        new_shape = inputs.shape[:2]\n",
    "        for i in [0, 1]:\n",
    "            pool_width = self.pool_shape[i]\n",
    "            img_width = inputs.shape[i + 2]\n",
    "            new_shape += (pool_width, img_width // pool_width)\n",
    "        result = inputs.reshape(new_shape)\n",
    "        return np.max(np.max(result, axis=2), axis=3)\n",
    "\n",
    "class full_layer(object):\n",
    "    def __init__(self, size):\n",
    "        self.size = size\n",
    "\n",
    "    def build_weights_dict(self, input_shape):\n",
    "        # Input shape is anything (all flattened)\n",
    "        input_size = np.prod(input_shape, dtype=int)\n",
    "        self.parser = WeightsParser()\n",
    "        self.parser.add_weights('params', (input_size, self.size))\n",
    "        self.parser.add_weights('biases', (self.size,))\n",
    "        return self.parser.N, (self.size,)\n",
    "\n",
    "    def forward_pass(self, inputs, param_vector):\n",
    "        params = self.parser.get(param_vector, 'params')\n",
    "        biases = self.parser.get(param_vector, 'biases')\n",
    "        if inputs.ndim > 2:\n",
    "            inputs = inputs.reshape((inputs.shape[0], np.prod(inputs.shape[1:])))\n",
    "        return self.nonlinearity(np.dot(inputs[:, :], params) + biases)\n",
    "\n",
    "class tanh_layer(full_layer):\n",
    "    def nonlinearity(self, x):\n",
    "        return np.tanh(x)\n",
    "\n",
    "class softmax_layer(full_layer):\n",
    "    def nonlinearity(self, x):\n",
    "        return x - logsumexp(x, axis=1, keepdims=True)\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    # Network parameters\n",
    "    L2_reg = 1.0\n",
    "    input_shape = (1, 28, 28)\n",
    "    layer_specs = [conv_layer((5, 5), 6),\n",
    "                   maxpool_layer((2, 2)),\n",
    "                   conv_layer((5, 5), 16),\n",
    "                   maxpool_layer((2, 2)),\n",
    "                   tanh_layer(120),\n",
    "                   tanh_layer(84),\n",
    "                   softmax_layer(10)]\n",
    "\n",
    "    # Training parameters\n",
    "    param_scale = 0.1\n",
    "    learning_rate = 1e-3\n",
    "    momentum = 0.9\n",
    "    batch_size = 256\n",
    "    num_epochs = 50\n",
    "\n",
    "    # Load and process MNIST data\n",
    "    print(\"Loading training data...\")\n",
    "    add_color_channel = lambda x : x.reshape((x.shape[0], 1, x.shape[1], x.shape[2]))\n",
    "    one_hot = lambda x, K : np.array(x[:,None] == np.arange(K)[None, :], dtype=int)\n",
    "    train_images, train_labels, test_images, test_labels = data_mnist.mnist()\n",
    "    train_images = add_color_channel(train_images) / 255.0\n",
    "    test_images  = add_color_channel(test_images)  / 255.0\n",
    "    train_labels = one_hot(train_labels, 10)\n",
    "    test_labels = one_hot(test_labels, 10)\n",
    "    N_data = train_images.shape[0]\n",
    "\n",
    "    # Make neural net functions\n",
    "    N_weights, pred_fun, loss_fun, frac_err = make_nn_funs(input_shape, layer_specs, L2_reg)\n",
    "    loss_grad = grad(loss_fun)\n",
    "\n",
    "    # Initialize weights\n",
    "    rs = npr.RandomState()\n",
    "    W = rs.randn(N_weights) * param_scale\n",
    "\n",
    "    # Check the gradients numerically, just to be safe\n",
    "    # quick_grad_check(loss_fun, W, (train_images[:50], train_labels[:50]))\n",
    "\n",
    "    print(\"    Epoch      |    Train err  |   Test error  \")\n",
    "    def print_perf(epoch, W):\n",
    "        test_perf  = frac_err(W, test_images, test_labels)\n",
    "        train_perf = frac_err(W, train_images, train_labels)\n",
    "        print(\"{0:15}|{1:15}|{2:15}\".format(epoch, train_perf, test_perf))\n",
    "\n",
    "    # Train with sgd\n",
    "    batch_idxs = make_batches(N_data, batch_size)\n",
    "    cur_dir = np.zeros(N_weights)\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        print_perf(epoch, W)\n",
    "        for idxs in batch_idxs:\n",
    "            grad_W = loss_grad(W, train_images[idxs], train_labels[idxs])\n",
    "            cur_dir = momentum * cur_dir + (1.0 - momentum) * grad_W\n",
    "            W -= learning_rate * cur_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}