notebooks/08_naive_bayes_roc_auc.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**CLASS**: Naive Bayes SMS spam classifier using sklearn\n",
    "\n",
    "Data source: https://archive.ics.uci.edu/ml/datasets/SMS+Spam+Collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## READING IN THE DATA\n",
    "\n",
    "# read tab-separated file using pandas\n",
    "import pandas as pd\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = pd.read_table('../data/sms.tsv',\n",
    "                   sep='\\t', header=None, names=['label', 'msg'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>msg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>  ham</td>\n",
       "      <td> Go until jurong point, crazy.. Available only ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>  ham</td>\n",
       "      <td>                     Ok lar... Joking wif u oni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td> spam</td>\n",
       "      <td> Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>  ham</td>\n",
       "      <td> U dun say so early hor... U c already then say...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>  ham</td>\n",
       "      <td> Nah I don't think he goes to usf, he lives aro...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label                                                msg\n",
       "0   ham  Go until jurong point, crazy.. Available only ...\n",
       "1   ham                      Ok lar... Joking wif u oni...\n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...\n",
       "3   ham  U dun say so early hor... U c already then say...\n",
       "4   ham  Nah I don't think he goes to usf, he lives aro..."
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# examine the data\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x105e06610>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEQCAYAAABV+ASvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFFFJREFUeJzt3X+sZHd53/H3B68J5Ae9OAH/WG+ypNgqbp1eo8q4UaRe\nQmM7LrIttQRHar1LraKW0kaoamOi1AaFViFSAqIISFOXXTkJthUCmCaF3QJHbZVi4oABx3G9jljq\nNbVJbZY2pEnt+Okfc+56WM/dXc2cO+fOOe+XNPL5fu+Zmef6nn3mez/nzNxUFZKkYXpe3wVIkraP\nTV6SBswmL0kDZpOXpAGzyUvSgNnkJWnATtvkkxxN8qUkX0jyuXbunCSHkzyU5FCStan935PkSJIv\nJrlsan5fu/9DSW7cnm9HkjTtTFbyBWxU1WVVdXk7dzNwuKouBj7VjklyDfDyqroIeCPw/nb+HOAW\n4PL2duv0C4MkaXucaVyTk8bXAgfb7YPA9e32dZvzVXUPsJbkPOAq4FBVHa+q48Bh4OpFCpcknd6Z\nruQPJbk3yT9o586tqscBquox4KXt/AXAI1P3PQbsbuePzZiXJG2jXWewzw9X1WNJXgIcTvLgKfYN\nz13114y5zXlJ0jY6bZNvV+pU1R8l+QiTTP3xJOe1zf984Ovt7seAPVN3vxB4tJ3fmJrfA3z65OdK\nYuOXpDlU1azF9KnjmiTfmeR72u3vAq4EvgzcDexrd9sHfLTdvhu4sd3/CuB4G+scAq5MspbkxcCP\nAZ/colBvHd1uvfXW3mvw5m3WzWOz29upnG4lfy7wkSSb+/5aVR1Kci9wV5KbgK8Cr2sb9G8nuSbJ\nw8C3gDe0808m+Tngd9vHfXtNTsBqGx09erTvEqSZPDaX55RNvqq+AqzPmH8S+Jtb3OfNW8x/EPjg\nHDVKkubkO14HbP/+/X2XIM3ksbk8OV2es0xJaifVI0mrIAk1z4lXrbamafouQZrJY3N5bPKSNGDG\nNXNorzZSh1bh5y7tVKeKa87kHa+ayabUHV80pe1iXDNoTd8FSDOZyS+PTV6SBsxMfg6TTH7n17k6\nYiYvLcBLKCVppGzyg9b0XYA0k5n88tjkJWnAzOTnYCbfNTN5aRFm8pI0Ujb5QWv6LkCayUx+eWzy\nkjRgZvJzMJPvmpm8tAgzeUkaKZv8oDV9FyDNZCa/PDZ5SRowM/k5mMl3zUxeWoSZvCSNlE1+0Jq+\nC5BmMpNfHpu8JA2YmfwczOS7ZiYvLcJMXpJGyiY/aE3fBUgzmckvj01ekgbMTH4OZvJdM5OXFmEm\nL0kjZZMftKbvAqSZzOSXxyYvSQNmJj8HM/mumclLizCTl6SRsskPWtN3AdJMZvLLY5OXpAE7oyaf\n5KwkX0jy8Xb8siT3JHkoyR1Jzm7nvyPJnUmOJPlskh+Yeoy3tvMPJrlye74dfbuNvguQZtrY2Oi7\nhNE405X8TwEP8OzZxncCv1hVFwPfAG5q528Cnqiqi4B3tfuR5BLg9cAlwNXA+5L4W4QkbbPTNtok\nFwLXAP8O2Dx7+2rgN9rtg8D17fa17Rjgw8Br2u3rgA9V1VNVdRR4GLh80eJ1Ok3fBUgzmckvz5ms\npt8F/HPgGYAk3wscr6pn2q8/Cuxut3cDjwBU1dPAN9v9LwCOTT3msan7SJK2ya5TfTHJa4GvV9UX\nkmy007NeGDZjnFnXadYW88/MmGP//v3s3bsXgLW1NdbX10/kd5uv/n2Pn7U53tih4825nVLPVuN2\ntEN+vo63f7yxsbGj6lm1cdM0HDhwAOBEv9zKKd8MleRfA38PeBp4AfAi4CPAVcC5VfVMkr8O3FJV\nP57kE8DbquqzSXYB/7OqXpLkZoCq+vn2cT8B3FpV95z0fL4ZapR8M5S0iLnfDFVVP1NVe6rqZcAN\nwKer6u8CnwFe1+62D/hYu313Owb4O8CnpuZvSPL8JC8DLgI+N+83pDPV9F2ANJOZ/PKcMq6ZYXO5\n9dPAHUneAXweuK2dvw24PckR4AkmLwxU1QNJ7mJyhc7TwJtWYskuSSvOz66Zg3FN14xrpEX42TWS\nNFI2+UFr+i5AmslMfnls8pI0YGbyczCT75qZvLQIM3lJGimb/KA1fRcgzWQmvzw2eUkaMDP5OZjJ\nd81MXlqEmbwkjZRNftCavguQZjKTXx6bvCQNmJn8HMzku2YmLy3CTF6SRsomP2hN3wVIM5nJL49N\nXpIGzEx+DmbyXTOTlxZhJi9JI2WTH7Sm7wKkmczkl8cmL0kDZiY/BzP5rpnJS4swk5ekkbLJD1rT\ndwHSTGbyy2OTl6QBM5Ofg5l818zkpUWYyUvSSNnkB63puwBpJjP55bHJS9KAmcnPwUy+a2by0iLM\n5CVppGzyg9b0XYA0k5n88tjkJWnAzOTnYCbfNTN5aRFm8pI0Ujb5QWv6LkCayUx+eWzykjRgp2zy\nSV6Q5J4k9yW5P8nb2vmXtfMPJbkjydnt/HckuTPJkSSfTfIDU4/11nb+wSRXbut3pdZG3wVIM21s\nbPRdwmicsslX1Z8Cr66qdWAduDrJq4B3Ar9YVRcD3wBuau9yE/BEVV0EvKvdjySXAK8HLgGuBt6X\nxN8iJGmbnbbRVtWftJvPB85mclnJq4HfaOcPAte329e2Y4APA69pt68DPlRVT1XVUeBh4PJFi9fp\nNH0XIM1kJr88p23ySZ6X5D7gceAQ8IfA8ap6pt3lUWB3u70beASgqp4Gvpnke4ELgGNTD3ts6j6S\npG1yJiv5Z9q45kLgVcArZu3W/nfWdZq1xfwzM+bUqY2+C5BmMpNfnl1numNVfTNJA1wBrCV5Xrua\nv5DJah4mK/TvB76WZBfwF6rqySTHgD1TD3ch8LVZz7N//3727t0LwNraGuvr6ycOiM1f8foeP2tz\nvOF4oXE72iE/X8eOd/q4aRoOHDgAcKJfbuWU73hN8n3A01V1PMkLgU8yOZm6D/hwVd2Z5APAfVX1\ngSRvAi6tqn+U5Abg+qq6oT3x+utMcvjdwH8CXn7y21t9x2vXGlZjNe87XsemaRpX8x061TteT7eS\nPx84mOQsJtHOnVX1W0keAO5I8g7g88Bt7f63AbcnOQI8AdwAUFUPJLkLeAB4GnjTSnRzSVpxfnbN\nHFZnJb8qXMlLi/CzayRppGzyg9b0XYA0k9fJL49NXpIGzEx+DmbyXTOTlxZhJi9JI2WTH7Sm7wKk\nmczkl8cmL0kDZiY/BzP5rpnJS4swk5ekkbLJD1rTdwHSTGbyy2OTl6QBM5Ofg5l818zkpUWYyUvS\nSNnkB63puwBpJjP55bHJS9KAmcnPwUy+a2by0iLM5CVppGzyg9b0XYA0k5n88tjkJWnAzOTnYCbf\nNTN5aRFm8pI0Ujb5QWv6LkCayUx+eWzykjRgZvJzMJPvmpm8tAgzeUkaKZv8oDV9FyDNZCa/PDZ5\nSRowM/k5mMl3zUxeWoSZvCSNlE1+0Jq+C5BmMpNfHpu8JA2YmfwczOS7ZiYvLcJMXpJGyiY/aE3f\nBUgzmckvj01ekgbMTH4OZvJdM5OXFjF3Jp9kT5LPJHkgyf1J/mk7f06Sw0keSnIoydrUfd6T5EiS\nLya5bGp+X7v/Q0lu7OqbkyRt7XRxzVPAW6rqEuAK4B8neQVwM3C4qi4GPtWOSXIN8PKqugh4I/D+\ndv4c4Bbg8vZ26/QLg7ZL03cB0kxm8stzyiZfVY9V1X3t9h8DfwDsBq4FDra7HQSub7ev25yvqnuA\ntSTnAVcBh6rqeFUdBw4DV3f8vUiSTnLGJ16T7AUuA+4Bzq2qx2HyQgC8tN3tAuCRqbsdY/KicEG7\nffK8ttVG3wVIM21sbPRdwmicUZNP8t3Ah4Gfqqr/c6pd29u0mjG3OS9J2ka7TrdDkrOZNPjbq+qj\n7fTjSc6rqseSnA98vZ0/BuyZuvuFwKPt/MbU/B7g07Oeb//+/ezduxeAtbU11tfXT7zqb+Z4fY+f\ntTne2KHjdwPrO6iercbtaIf8fB1v/3j639JOqGfVxk3TcODAAYAT/XIrp7yEMpNrBQ8CT1TVW6bm\nf6Gde2eSm4G1qrq5PfH65qq6JskVwLur6or2xOu9wCuZrOp/D3hlm89PP5+XUHaqYTUiGy+hHJum\naYxsOnSqSyhP1+R/BPjPwJd4tqu9FfgccBfw/cBXgddtNuwk72VyUvVbwBuq6vPt/BuAn2kf4x1V\ntXnidvr5bPKjZJOXFjF3k182m/xY2eSlRfgBZaPV9F2ANJPXyS+PTV6SBsy4Zg7GNV0zrpEWYVwj\nSSNlkx+0pu8CpJnM5JfHJi9JA2YmPwcz+a6ZyUuLMJOXpJGyyQ9a03cB0kxm8stjk5ekATOTn4OZ\nfNfM5KVFmMlL0kjZ5Aet6bsAaSYz+eWxyUvSgJnJz8FMvmtm8tIizOQlaaRs8oPW9F2ANJOZ/PLY\n5CVpwMzk52Am3zUzeWkRZvKSNFI2+UFr+i5AmslMfnls8pI0YGbyczCT75qZvLQIM3lJGimb/KA1\nfRcgzWQmvzw2eUkaMDP5OZjJd81MXlqEmbwkjZRNftCavguQZjKTXx6bvCQNmJn8HMzku2YmLy3C\nTF6SRsomP2hN3wVIM5nJL49NXpIGzEx+DmbyXTOTlxZhJi9JI3XaJp/k3yd5PMmXp+bOSXI4yUNJ\nDiVZm/rae5IcSfLFJJdNze9r938oyY3dfyt6rqbvAqSZzOSX50xW8h8Erj5p7mbgcFVdDHyqHZPk\nGuDlVXUR8Ebg/e38OcAtwOXt7dbpFwZJ0vY4bZOvqv8CfOOk6WuBg+32QeD6dvu6zfmqugdYS3Ie\ncBVwqKqOV9Vx4DDPfeFQ5zb6LkCaaWNjo+8SRmPeTP7cqnocoKoeA17azl8APDK13zFgdzt/bMa8\nJGkb7er48dLeptWMuc3559i/fz979+4FYG1tjfX19ROv+ps5Xt/jZ22ON3bo+N3A+g6qZ6txO9oh\nP1/H2z+e/re0E+pZtXHTNBw4cADgRL/cyhldQplkL/Dxqrq0HT8IbFTVY0nOBz5TVX8pyQeApqru\nmNrvbwCvbvf/h+38LwOfrqo7T3oeL6HsVMNqRDZeQjk2TdMY2XRoOy6hvBvY127vAz46NX9j+6RX\nAMfbWOcQcGWStSQvBn4M+OScz60zttF3AdJMNvjlOW1ck+RDTFbj35fkESZXyfw8cFeSm4CvAq8D\nqKrfTnJNkoeBbwFvaOefTPJzwO+2D/v29gSsJGkb+Y7XORjXdM24ZmyMa7rlO14laaRcyc9hdVby\nq8KVvLQIV/KSNFI2+UFr+i5AmsnPrlkem7wkDZiZ/BzM5LtmJi8twkxekkaq68+u0Y7SsBrXyasr\nk98y1aVV/y3TJi8Nzio0pYbVWICs/oummfwczOS7ZibfFY/Nrq3GsWkmL0kjZZMftKbvAqQtNH0X\nMBo2eUkaMDP5OZh7dm01cs9V4LHZtdU4Ns3kJWmkbPKD1vRdgLSFpu8CRsMmL0kDZiY/B3PPrq1G\n7rkKPDa7thrHppm8JI2UTX7Qmr4LkLbQ9F3AaNjkJWnAzOTnYO7ZtdXIPVeBx2bXVuPYNJOXpJGy\nyQ9a03cB0haavgsYDZu8JA2YmfwczD27thq55yrw2OzaahybZvKSNFI2+UFr+i5A2kLTdwGjYZOX\npAEzk5+DuWfXViP3XAUem11bjWPTTF6SRsomP2hN3wVIW2j6LmA0bPKSNGBm8nMw9+zaauSeq8Bj\ns2urcWyayUvSSC21ySe5OsmDSY4k+ellPvc4NX0XIG2h6buA0Vhak09yFvBe4GrgEuAnk7xiWc8/\nTvf1XYC0BY/NZVnmSv5y4OGqOlpVTwF3ANct8flH6HjfBUhb8NhclmU2+d3AI1PjY+2cJGmb9H3i\ndeeftl5pR/suQNrC0b4LGI1dS3yuY8CeqfEe4NGTd5pcArYKVqXOg30XcEZW5+e+Clbl/6XH5jIs\n7Tr5JLuA/w68Bvga8DngJ6vqD5ZSgCSN0NJW8lX1dJI3A58EzgJus8FL0vbaUe94lSR1q+8Tr5Kk\nbbTME69agiR/FdjLsz/bqqrf7K8i6cQ5ub/Fc4/NX+qtqJGwyQ9Ikg8ClwK/Dzwz9SWbvPr2ceD/\nAl/m249NbTOb/LC8CvjLK/FRnhqb3VX1Q30XMUZm8sPyWSafCyTtNJ9IclXfRYyRK/lhOQD8TpLH\ngT9r58oVlHaA/wb8ZvtBhU+1c1VVL+qxplHwEsoBSfKHwFuA+5nKPavqaF81SQBJjgLXAvdXlZn8\nErmSH5avV9XdfRchzfA/gN+3wS+fTX5YvpDk15lcyfD/2jkvodRO8BXgM0n+I99+bHoJ5TazyQ/L\ndzLJ4q88ad4mr759pb09v735x2iXxExekgbMlfyAJHkhcBOTyyhfSLtSqqq/32ddUpKXAv+CZ49N\nmMQ1P9pfVePgdfLDcjtwLpO/o9sw+cz+P+6zIKn1a8CDwA8Cb2PyV0Pu7bGe0TCuGZAk91XVepIv\nVdUPJTkb+K9V9aq+a9O4Jfl8Vb1y89hs5+6tqr/Wd21DZ1wzLJtXLXwzyaXAY8BLeqxH2rR5bD6W\n5LVM/nDQi3usZzRs8sPyK0nOAX4W+Bjw3cAt/ZYkAfCvkqwB/wz4N8CLmLxxT9vMuGZAkrwA+Nt8\n+8e5UlVv76smSf3yxOuwfIzJW8efYnLCdfMm9SrJX0zy8ST/K8kfJflYkh/su64xcCU/IEnur6q/\n0ncd0smS3AO8F7ijnXo98E+8KGD7uZIflt9J4idOaid6YVXdXlVPtbdfBV7Qd1Fj4Ep+AJJ8ud08\nC7iIydvH/ahh7RhJ3gkcBz7UTr2eydU1vwBQVU/2VNrg2eQHIMneU33djxpW39qPGt6q2VRVmc9v\nE5u8pG2X5CeAT1TV/05yC3AZ8I6q+r2eSxs8M3lJy/Av2wb/I8CPArcB7+u5plGwyUtahj9v//ta\n4Feq6j8w+chhbTObvKRleDTJvwV+Avit9o179p8lMJOXtO2SfBeTT0f9UlUdSXI+cGlVHeq5tMGz\nyUvSgPnrkiQNmE1ekgbMJi9JA2aTl6QBs8lL0oD9f3BXsHgoGFCEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10546cb10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.label.value_counts().plot(kind='bar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ham     0.865937\n",
       "spam    0.134063\n",
       "dtype: float64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get the null accuracy rate\n",
    "df.label.value_counts() / df.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count                       5572\n",
       "unique                      5169\n",
       "top       Sorry, I'll call later\n",
       "freq                          30\n",
       "Name: msg, dtype: object"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.msg.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>msg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td> 0</td>\n",
       "      <td> Go until jurong point, crazy.. Available only ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td> 0</td>\n",
       "      <td>                     Ok lar... Joking wif u oni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td> 1</td>\n",
       "      <td> Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td> 0</td>\n",
       "      <td> U dun say so early hor... U c already then say...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td> 0</td>\n",
       "      <td> Nah I don't think he goes to usf, he lives aro...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label                                                msg\n",
       "0      0  Go until jurong point, crazy.. Available only ...\n",
       "1      0                      Ok lar... Joking wif u oni...\n",
       "2      1  Free entry in 2 a wkly comp to win FA Cup fina...\n",
       "3      0  U dun say so early hor... U c already then say...\n",
       "4      0  Nah I don't think he goes to usf, he lives aro..."
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# convert label to a quantitative binary variable\n",
    "df['label'] = df.label.map({'ham':0, 'spam':1})\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# split into training and testing sets\n",
    "from sklearn.cross_validation import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(df.msg, df.label, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'44', u'cab', u'call', u'me', u'please', u'tonight', u'you']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "# start with a simple example\n",
    "train_simple = ['call you tonight',\n",
    "                'Call me a cab',\n",
    "                'please call me... PLEASE 44!']\n",
    "\n",
    "# learn the 'vocabulary' of the training data\n",
    "vect = CountVectorizer()\n",
    "train_simple_dtm = vect.fit_transform(train_simple)\n",
    "vect.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<3x7 sparse matrix of type '<type 'numpy.int64'>'\n",
       "\twith 10 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# transform training data into a 'document-term matrix'\n",
    "train_simple_dtm = vect.transform(train_simple)\n",
    "train_simple_dtm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>44</th>\n",
       "      <th>cab</th>\n",
       "      <th>call</th>\n",
       "      <th>me</th>\n",
       "      <th>please</th>\n",
       "      <th>tonight</th>\n",
       "      <th>you</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 1</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td> 0</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td> 1</td>\n",
       "      <td> 0</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 2</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   44  cab  call  me  please  tonight  you\n",
       "0   0    0     1   0       0        1    1\n",
       "1   0    1     1   1       0        0    0\n",
       "2   1    0     1   1       2        0    0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# examine the vocabulary and document-term matrix together\n",
    "pd.DataFrame(train_simple_dtm.toarray(), columns=vect.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>44</th>\n",
       "      <th>cab</th>\n",
       "      <th>call</th>\n",
       "      <th>me</th>\n",
       "      <th>please</th>\n",
       "      <th>tonight</th>\n",
       "      <th>you</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 1</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   44  cab  call  me  please  tonight  you\n",
       "0   0    0     1   1       1        0    0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# transform testing data into a document-term matrix (using existing vocabulary, notice don't is missing)\n",
    "test_simple = [\"please don't call me\"]\n",
    "test_simple_dtm = vect.transform(test_simple)\n",
    "test_simple_dtm.toarray()\n",
    "pd.DataFrame(test_simple_dtm.toarray(), columns=vect.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<4179x7456 sparse matrix of type '<type 'numpy.int64'>'\n",
       "\twith 55209 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## REPEAT PATTERN WITH SMS DATA\n",
    "\n",
    "# instantiate the vectorizer\n",
    "vect = CountVectorizer()\n",
    "\n",
    "# learn vocabulary and create document-term matrix in a single step\n",
    "train_dtm = vect.fit_transform(X_train)\n",
    "train_dtm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1393x7456 sparse matrix of type '<type 'numpy.int64'>'\n",
       "\twith 17604 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# transform testing data into a document-term matrix\n",
    "test_dtm = vect.transform(X_test)\n",
    "test_dtm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([u'00',\n",
       "  u'000',\n",
       "  u'008704050406',\n",
       "  u'0121',\n",
       "  u'01223585236',\n",
       "  u'01223585334',\n",
       "  u'0125698789',\n",
       "  u'02',\n",
       "  u'0207',\n",
       "  u'02072069400'],\n",
       " [u'zed',\n",
       "  u'zeros',\n",
       "  u'zhong',\n",
       "  u'zindgi',\n",
       "  u'zoe',\n",
       "  u'zoom',\n",
       "  u'zouk',\n",
       "  u'zyada',\n",
       "  u'\\xe8n',\n",
       "  u'\\u3028ud'])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# store feature names and examine them\n",
    "train_features = vect.get_feature_names()\n",
    "train_features\n",
    "train_features[:10], train_features[-10:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       ..., \n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# convert train_dtm to a regular array\n",
    "train_arr = train_dtm.toarray()\n",
    "# remember that each ROW is an sms\n",
    "# and each COLUMN is a token\n",
    "train_arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 2, 3, 4],\n",
       "       [5, 6, 7, 8]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## SIMPLE SUMMARIES OF THE TRAINING DATA\n",
    "# refresher on numpy\n",
    "import numpy as np\n",
    "arr = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])\n",
    "arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(arr[0, :]) #sum of the first row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(arr[:,0]) # sum of the first column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "################\n",
    "### EXERCISE ###\n",
    "################\n",
    "# count how many times the 0th token appears across ALL messages in train_arr\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "################\n",
    "### EXERCISE ###\n",
    "################\n",
    "#calculate the number of tokens in the 0th message in train_arr\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(arr) # adds up all numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 6,  8, 10, 12])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(arr, axis=0) # adds up by column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([10, 26])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(arr, axis=1)  # adds up by row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "################\n",
    "### EXERCISE ###\n",
    "################\n",
    "# count how many times EACH token appears across ALL messages in train_arr\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>token</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6656</th>\n",
       "      <td> 1670</td>\n",
       "      <td>        to</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7420</th>\n",
       "      <td> 1660</td>\n",
       "      <td>       you</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6542</th>\n",
       "      <td> 1004</td>\n",
       "      <td>       the</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>929 </th>\n",
       "      <td>  717</td>\n",
       "      <td>       and</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3502</th>\n",
       "      <td>  683</td>\n",
       "      <td>        in</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3612</th>\n",
       "      <td>  679</td>\n",
       "      <td>        is</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4238</th>\n",
       "      <td>  601</td>\n",
       "      <td>        me</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3623</th>\n",
       "      <td>  568</td>\n",
       "      <td>        it</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4489</th>\n",
       "      <td>  550</td>\n",
       "      <td>        my</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2821</th>\n",
       "      <td>  518</td>\n",
       "      <td>       for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7424</th>\n",
       "      <td>  508</td>\n",
       "      <td>      your</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4704</th>\n",
       "      <td>  460</td>\n",
       "      <td>        of</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6539</th>\n",
       "      <td>  453</td>\n",
       "      <td>      that</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1552</th>\n",
       "      <td>  443</td>\n",
       "      <td>      call</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3235</th>\n",
       "      <td>  416</td>\n",
       "      <td>      have</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4743</th>\n",
       "      <td>  390</td>\n",
       "      <td>        on</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4662</th>\n",
       "      <td>  361</td>\n",
       "      <td>       now</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1016</th>\n",
       "      <td>  358</td>\n",
       "      <td>       are</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1574</th>\n",
       "      <td>  349</td>\n",
       "      <td>       can</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6017</th>\n",
       "      <td>  344</td>\n",
       "      <td>        so</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4647</th>\n",
       "      <td>  338</td>\n",
       "      <td>       not</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1522</th>\n",
       "      <td>  332</td>\n",
       "      <td>       but</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4778</th>\n",
       "      <td>  306</td>\n",
       "      <td>        or</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3465</th>\n",
       "      <td>  302</td>\n",
       "      <td>        if</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2995</th>\n",
       "      <td>  299</td>\n",
       "      <td>       get</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1081</th>\n",
       "      <td>  298</td>\n",
       "      <td>        at</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6904</th>\n",
       "      <td>  297</td>\n",
       "      <td>        ur</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7257</th>\n",
       "      <td>  293</td>\n",
       "      <td>      with</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7120</th>\n",
       "      <td>  293</td>\n",
       "      <td>        we</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2290</th>\n",
       "      <td>  292</td>\n",
       "      <td>        do</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3157</th>\n",
       "      <td>    1</td>\n",
       "      <td>       gut</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3178</th>\n",
       "      <td>    1</td>\n",
       "      <td>   haircut</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3156</th>\n",
       "      <td>    1</td>\n",
       "      <td>      gurl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3155</th>\n",
       "      <td>    1</td>\n",
       "      <td>   guoyang</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3154</th>\n",
       "      <td>    1</td>\n",
       "      <td>     gumby</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3153</th>\n",
       "      <td>    1</td>\n",
       "      <td>    guitar</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3151</th>\n",
       "      <td>    1</td>\n",
       "      <td>    guides</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3149</th>\n",
       "      <td>    1</td>\n",
       "      <td>  guidance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3147</th>\n",
       "      <td>    1</td>\n",
       "      <td>   guessin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3176</th>\n",
       "      <td>    1</td>\n",
       "      <td>      hail</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3179</th>\n",
       "      <td>    1</td>\n",
       "      <td>    haiyoh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3216</th>\n",
       "      <td>    1</td>\n",
       "      <td>   hardest</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3199</th>\n",
       "      <td>    1</td>\n",
       "      <td>    hangin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3215</th>\n",
       "      <td>    1</td>\n",
       "      <td>    harder</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3210</th>\n",
       "      <td>    1</td>\n",
       "      <td>   happily</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3209</th>\n",
       "      <td>    1</td>\n",
       "      <td>  happiest</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3208</th>\n",
       "      <td>    1</td>\n",
       "      <td>   happier</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3202</th>\n",
       "      <td>    1</td>\n",
       "      <td> hannaford</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3201</th>\n",
       "      <td>    1</td>\n",
       "      <td>     hanks</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3200</th>\n",
       "      <td>    1</td>\n",
       "      <td>   hanging</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3198</th>\n",
       "      <td>    1</td>\n",
       "      <td>    hanger</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3182</th>\n",
       "      <td>    1</td>\n",
       "      <td>   half8th</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3197</th>\n",
       "      <td>    1</td>\n",
       "      <td>      hang</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3196</th>\n",
       "      <td>    1</td>\n",
       "      <td> handsomes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3195</th>\n",
       "      <td>    1</td>\n",
       "      <td>  handsome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3188</th>\n",
       "      <td>    1</td>\n",
       "      <td>    hamper</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3187</th>\n",
       "      <td>    1</td>\n",
       "      <td>       ham</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3184</th>\n",
       "      <td>    1</td>\n",
       "      <td>     halla</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3183</th>\n",
       "      <td>    1</td>\n",
       "      <td>      hall</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7455</th>\n",
       "      <td>    1</td>\n",
       "      <td>       〨ud</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7456 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      count      token\n",
       "6656   1670         to\n",
       "7420   1660        you\n",
       "6542   1004        the\n",
       "929     717        and\n",
       "3502    683         in\n",
       "3612    679         is\n",
       "4238    601         me\n",
       "3623    568         it\n",
       "4489    550         my\n",
       "2821    518        for\n",
       "7424    508       your\n",
       "4704    460         of\n",
       "6539    453       that\n",
       "1552    443       call\n",
       "3235    416       have\n",
       "4743    390         on\n",
       "4662    361        now\n",
       "1016    358        are\n",
       "1574    349        can\n",
       "6017    344         so\n",
       "4647    338        not\n",
       "1522    332        but\n",
       "4778    306         or\n",
       "3465    302         if\n",
       "2995    299        get\n",
       "1081    298         at\n",
       "6904    297         ur\n",
       "7257    293       with\n",
       "7120    293         we\n",
       "2290    292         do\n",
       "...     ...        ...\n",
       "3157      1        gut\n",
       "3178      1    haircut\n",
       "3156      1       gurl\n",
       "3155      1    guoyang\n",
       "3154      1      gumby\n",
       "3153      1     guitar\n",
       "3151      1     guides\n",
       "3149      1   guidance\n",
       "3147      1    guessin\n",
       "3176      1       hail\n",
       "3179      1     haiyoh\n",
       "3216      1    hardest\n",
       "3199      1     hangin\n",
       "3215      1     harder\n",
       "3210      1    happily\n",
       "3209      1   happiest\n",
       "3208      1    happier\n",
       "3202      1  hannaford\n",
       "3201      1      hanks\n",
       "3200      1    hanging\n",
       "3198      1     hanger\n",
       "3182      1    half8th\n",
       "3197      1       hang\n",
       "3196      1  handsomes\n",
       "3195      1   handsome\n",
       "3188      1     hamper\n",
       "3187      1        ham\n",
       "3184      1      halla\n",
       "3183      1       hall\n",
       "7455      1        〨ud\n",
       "\n",
       "[7456 rows x 2 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a dataframe with two columns, \"count\" and \"token\" that holds the token count for each token\n",
    "# use the numpy sum methods\n",
    "train_token_counts = pd.DataFrame({'token':train_features, 'count':np.sum(train_arr, axis=0)})\n",
    "train_token_counts.sort_index(by='count', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## MODEL BUILDING WITH NAIVE BAYES\n",
    "## http://scikit-learn.org/stable/modules/naive_bayes.html\n",
    "\n",
    "# train a Naive Bayes model using train_dtm\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "nb = MultinomialNB()\n",
    "nb.fit(train_dtm, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, ..., 0, 1, 0])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make predictions on test data using test_dtm\n",
    "preds = nb.predict(test_dtm)\n",
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.988513998564\n",
      "[[1203    5]\n",
      " [  11  174]]\n"
     ]
    }
   ],
   "source": [
    "# compare predictions to true labels\n",
    "from sklearn import metrics\n",
    "print metrics.accuracy_score(y_test, preds)\n",
    "print metrics.confusion_matrix(y_test, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.986643100054\n"
     ]
    }
   ],
   "source": [
    "# predict (poorly calibrated) probabilities and calculate AUC\n",
    "probs = nb.predict_proba(test_dtm)[:, 1]\n",
    "probs\n",
    "print metrics.roc_auc_score(y_test, probs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "################\n",
    "### EXERCISE ###\n",
    "################\n",
    "# show the message text for the false positives\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "################\n",
    "### EXERCISE ###\n",
    "################\n",
    "# show the message text for the false negatives\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## COMPARE NAIVE BAYES AND LOGISTIC REGRESSION\n",
    "## USING ALL DATA AND CROSS-VALIDATION\n",
    "vect = CountVectorizer(ngram_range=(1,3), stop_words='english')\n",
    "# create a document-term matrix using all data\n",
    "all_dtm = vect.fit_transform(df.msg)\n",
    "\n",
    "# instantiate logistic regression\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "logreg = LogisticRegression()\n",
    "\n",
    "# compare AUC using cross-validation\n",
    "# note: this is slightly improper cross-validation... can you figure out why?\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.87251 Seconds for logistic regression 0.976848539122\n"
     ]
    }
   ],
   "source": [
    "now = datetime.now()\n",
    "mean = cross_val_score(logreg, all_dtm, df.label, cv=10, scoring='accuracy').mean()\n",
    "print (datetime.now()-now).total_seconds(), \"Seconds for logistic regression\", mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.150404 Seconds for naive bayes 0.967516659054\n"
     ]
    }
   ],
   "source": [
    "now = datetime.now()\n",
    "mean = cross_val_score(nb, all_dtm, df.label, cv=10, scoring='accuracy').mean()\n",
    "print (datetime.now()-now).total_seconds(), \"Seconds for naive bayes\", mean\n",
    "# about 4 times faster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>default</th>\n",
       "      <th>student</th>\n",
       "      <th>balance</th>\n",
       "      <th>income</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0   </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  729.526495</td>\n",
       "      <td> 44361.625070</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1   </th>\n",
       "      <td> 0</td>\n",
       "      <td> Yes</td>\n",
       "      <td>  817.180407</td>\n",
       "      <td> 12106.134700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2   </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td> 1073.549164</td>\n",
       "      <td> 31767.138950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3   </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  529.250605</td>\n",
       "      <td> 35704.493940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4   </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  785.655883</td>\n",
       "      <td> 38463.495880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5   </th>\n",
       "      <td> 0</td>\n",
       "      <td> Yes</td>\n",
       "      <td>  919.588530</td>\n",
       "      <td>  7491.558572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6   </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  825.513330</td>\n",
       "      <td> 24905.226580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7   </th>\n",
       "      <td> 0</td>\n",
       "      <td> Yes</td>\n",
       "      <td>  808.667504</td>\n",
       "      <td> 17600.451340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8   </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td> 1161.057854</td>\n",
       "      <td> 37468.529290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9   </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>    0.000000</td>\n",
       "      <td> 29275.268290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10  </th>\n",
       "      <td> 0</td>\n",
       "      <td> Yes</td>\n",
       "      <td>    0.000000</td>\n",
       "      <td> 21871.073090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11  </th>\n",
       "      <td> 0</td>\n",
       "      <td> Yes</td>\n",
       "      <td> 1220.583753</td>\n",
       "      <td> 13268.562220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12  </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  237.045114</td>\n",
       "      <td> 28251.695340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13  </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  606.742343</td>\n",
       "      <td> 44994.555850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14  </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td> 1112.968401</td>\n",
       "      <td> 23810.174050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15  </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  286.232560</td>\n",
       "      <td> 45042.413040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16  </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>    0.000000</td>\n",
       "      <td> 50265.312350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17  </th>\n",
       "      <td> 0</td>\n",
       "      <td> Yes</td>\n",
       "      <td>  527.540184</td>\n",
       "      <td> 17636.539620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18  </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  485.936864</td>\n",
       "      <td> 61566.106120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19  </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td> 1095.072735</td>\n",
       "      <td> 26464.631390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20  </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  228.952550</td>\n",
       "      <td> 50500.182200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21  </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  954.261793</td>\n",
       "      <td> 32457.509080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22  </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td> 1055.956605</td>\n",
       "      <td> 51317.883080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23  </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  641.984389</td>\n",
       "      <td> 30466.103260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24  </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  773.211725</td>\n",
       "      <td> 34353.314310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25  </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  855.008522</td>\n",
       "      <td> 25211.332160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26  </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  642.999738</td>\n",
       "      <td> 41473.511800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27  </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td> 1454.863272</td>\n",
       "      <td> 32189.094950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28  </th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  615.704277</td>\n",
       "      <td> 39376.394620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29  </th>\n",
       "      <td> 0</td>\n",
       "      <td> Yes</td>\n",
       "      <td> 1119.569353</td>\n",
       "      <td> 16556.070210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9970</th>\n",
       "      <td> 0</td>\n",
       "      <td> Yes</td>\n",
       "      <td> 1294.500408</td>\n",
       "      <td> 25687.326050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9971</th>\n",
       "      <td> 0</td>\n",
       "      <td> Yes</td>\n",
       "      <td>  180.620128</td>\n",
       "      <td> 20975.560500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9972</th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  755.432801</td>\n",
       "      <td> 14455.865360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9973</th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  876.119027</td>\n",
       "      <td> 37668.366790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9974</th>\n",
       "      <td> 0</td>\n",
       "      <td> Yes</td>\n",
       "      <td>  933.332025</td>\n",
       "      <td> 26051.398320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9975</th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  908.315934</td>\n",
       "      <td> 21287.942490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9976</th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  218.417559</td>\n",
       "      <td> 25401.133120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9977</th>\n",
       "      <td> 0</td>\n",
       "      <td> Yes</td>\n",
       "      <td>  915.439827</td>\n",
       "      <td> 16624.339110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9978</th>\n",
       "      <td> 1</td>\n",
       "      <td>  No</td>\n",
       "      <td> 2202.462395</td>\n",
       "      <td> 47287.257110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9979</th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  173.249172</td>\n",
       "      <td> 30697.245060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9980</th>\n",
       "      <td> 0</td>\n",
       "      <td> Yes</td>\n",
       "      <td>  770.015741</td>\n",
       "      <td> 13684.789950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9981</th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  739.418018</td>\n",
       "      <td> 40656.951450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9982</th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  623.526119</td>\n",
       "      <td> 59441.309980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9983</th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  506.625453</td>\n",
       "      <td> 49861.003410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9984</th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  875.241640</td>\n",
       "      <td> 52861.744200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9985</th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  842.949429</td>\n",
       "      <td> 39957.127860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9986</th>\n",
       "      <td> 0</td>\n",
       "      <td> Yes</td>\n",
       "      <td>  401.332673</td>\n",
       "      <td> 15332.017830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9987</th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td> 1092.906583</td>\n",
       "      <td> 45479.466990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9988</th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>    0.000000</td>\n",
       "      <td> 41740.686600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9989</th>\n",
       "      <td> 0</td>\n",
       "      <td> Yes</td>\n",
       "      <td>  999.281112</td>\n",
       "      <td> 20013.350640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9990</th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  372.379238</td>\n",
       "      <td> 25374.899090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9991</th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  658.799558</td>\n",
       "      <td> 54802.078220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9992</th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td> 1111.647317</td>\n",
       "      <td> 45490.682460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9993</th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  938.836241</td>\n",
       "      <td> 56633.448740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9994</th>\n",
       "      <td> 0</td>\n",
       "      <td> Yes</td>\n",
       "      <td>  172.412988</td>\n",
       "      <td> 14955.941690</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  711.555020</td>\n",
       "      <td> 52992.378910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  757.962918</td>\n",
       "      <td> 19660.721770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td>  845.411989</td>\n",
       "      <td> 58636.156980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td> 0</td>\n",
       "      <td>  No</td>\n",
       "      <td> 1569.009053</td>\n",
       "      <td> 36669.112360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td> 0</td>\n",
       "      <td> Yes</td>\n",
       "      <td>  200.922183</td>\n",
       "      <td> 16862.952320</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      default student      balance        income\n",
       "0           0      No   729.526495  44361.625070\n",
       "1           0     Yes   817.180407  12106.134700\n",
       "2           0      No  1073.549164  31767.138950\n",
       "3           0      No   529.250605  35704.493940\n",
       "4           0      No   785.655883  38463.495880\n",
       "5           0     Yes   919.588530   7491.558572\n",
       "6           0      No   825.513330  24905.226580\n",
       "7           0     Yes   808.667504  17600.451340\n",
       "8           0      No  1161.057854  37468.529290\n",
       "9           0      No     0.000000  29275.268290\n",
       "10          0     Yes     0.000000  21871.073090\n",
       "11          0     Yes  1220.583753  13268.562220\n",
       "12          0      No   237.045114  28251.695340\n",
       "13          0      No   606.742343  44994.555850\n",
       "14          0      No  1112.968401  23810.174050\n",
       "15          0      No   286.232560  45042.413040\n",
       "16          0      No     0.000000  50265.312350\n",
       "17          0     Yes   527.540184  17636.539620\n",
       "18          0      No   485.936864  61566.106120\n",
       "19          0      No  1095.072735  26464.631390\n",
       "20          0      No   228.952550  50500.182200\n",
       "21          0      No   954.261793  32457.509080\n",
       "22          0      No  1055.956605  51317.883080\n",
       "23          0      No   641.984389  30466.103260\n",
       "24          0      No   773.211725  34353.314310\n",
       "25          0      No   855.008522  25211.332160\n",
       "26          0      No   642.999738  41473.511800\n",
       "27          0      No  1454.863272  32189.094950\n",
       "28          0      No   615.704277  39376.394620\n",
       "29          0     Yes  1119.569353  16556.070210\n",
       "...       ...     ...          ...           ...\n",
       "9970        0     Yes  1294.500408  25687.326050\n",
       "9971        0     Yes   180.620128  20975.560500\n",
       "9972        0      No   755.432801  14455.865360\n",
       "9973        0      No   876.119027  37668.366790\n",
       "9974        0     Yes   933.332025  26051.398320\n",
       "9975        0      No   908.315934  21287.942490\n",
       "9976        0      No   218.417559  25401.133120\n",
       "9977        0     Yes   915.439827  16624.339110\n",
       "9978        1      No  2202.462395  47287.257110\n",
       "9979        0      No   173.249172  30697.245060\n",
       "9980        0     Yes   770.015741  13684.789950\n",
       "9981        0      No   739.418018  40656.951450\n",
       "9982        0      No   623.526119  59441.309980\n",
       "9983        0      No   506.625453  49861.003410\n",
       "9984        0      No   875.241640  52861.744200\n",
       "9985        0      No   842.949429  39957.127860\n",
       "9986        0     Yes   401.332673  15332.017830\n",
       "9987        0      No  1092.906583  45479.466990\n",
       "9988        0      No     0.000000  41740.686600\n",
       "9989        0     Yes   999.281112  20013.350640\n",
       "9990        0      No   372.379238  25374.899090\n",
       "9991        0      No   658.799558  54802.078220\n",
       "9992        0      No  1111.647317  45490.682460\n",
       "9993        0      No   938.836241  56633.448740\n",
       "9994        0     Yes   172.412988  14955.941690\n",
       "9995        0      No   711.555020  52992.378910\n",
       "9996        0      No   757.962918  19660.721770\n",
       "9997        0      No   845.411989  58636.156980\n",
       "9998        0      No  1569.009053  36669.112360\n",
       "9999        0     Yes   200.922183  16862.952320\n",
       "\n",
       "[10000 rows x 4 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "Model evaluation metrics (confusion matrix, ROC/AUC)\n",
    "'''\n",
    "\n",
    "## READ DATA AND SPLIT INTO TRAIN/TEST\n",
    "\n",
    "# read in the data\n",
    "import pandas as pd\n",
    "data = pd.read_csv('../data/Default.csv')\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create X and y\n",
    "X = data[['balance']]\n",
    "y = data.default"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# split into train and test\n",
    "from sklearn.cross_validation import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, penalty='l2', random_state=None, tol=0.0001)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create logistic regression model\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "logreg = LogisticRegression()\n",
    "logreg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9748"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# predict and calculate accuracy in one step\n",
    "logreg.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9748\n"
     ]
    }
   ],
   "source": [
    "# predict in one step, calculate accuracy in a separate step\n",
    "preds = logreg.predict(X_test)\n",
    "from sklearn import metrics\n",
    "print metrics.accuracy_score(y_test, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.96840000000000004"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# compare to null accuracy rate\n",
    "y_test.mean()\n",
    "1 - y_test.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[2416    5]\n",
      " [  58   21]]\n"
     ]
    }
   ],
   "source": [
    "## CONFUSION MATRIX\n",
    "\n",
    "# print confusion matrix\n",
    "print metrics.confusion_matrix(y_test, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  |    0    1 |\n",
      "--+-----------+\n",
      "0 |<2416>   5 |\n",
      "1 |   58  <21>|\n",
      "--+-----------+\n",
      "(row = reference; col = test)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# nicer confusion matrix\n",
    "from nltk import ConfusionMatrix\n",
    "print ConfusionMatrix(list(y_test), list(preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.26582278481\n",
      "0.997934737712\n"
     ]
    }
   ],
   "source": [
    "# sensitivity: percent of correct predictions when reference value is 'default'\n",
    "21 / float(58 + 21)\n",
    "print metrics.recall_score(y_test, preds)\n",
    "\n",
    "# specificity: percent of correct predictions when reference value is 'not default'\n",
    "print 2416 / float(2416 + 5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([  2.23000000e+03,   1.23000000e+02,   6.30000000e+01,\n",
       "          2.90000000e+01,   2.30000000e+01,   1.10000000e+01,\n",
       "          1.00000000e+01,   6.00000000e+00,   1.00000000e+00,\n",
       "          4.00000000e+00]),\n",
       " array([  1.87839293e-04,   8.91943925e-02,   1.78200946e-01,\n",
       "          2.67207499e-01,   3.56214052e-01,   4.45220605e-01,\n",
       "          5.34227159e-01,   6.23233712e-01,   7.12240265e-01,\n",
       "          8.01246818e-01,   8.90253371e-01]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEACAYAAAC6d6FnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEdZJREFUeJzt3X+sZGV9x/H3R8AfrZSVUvm5Si1goZGyNOCmJnKNFJAo\nYMAfGApVghq71diYCKZld2NqtI2WGAONARGrglhTAhECKzJUo4DIsoD82m3A7kUW6w+M0LQB+faP\nOavj5e69c+/dOzP6vF/JSZ7zzDlzvnNmdz7znGfObqoKSVJ7njPuAiRJ42EASFKjDABJapQBIEmN\nMgAkqVEGgCQ1as4ASLIyyU1J7k1yT5L3dv3rkkwn2dgtrxvY57wkm5Pcn+S4gf4Tur7NST64fC9J\nkjSMzHUfQJJ9gH2q6s4kLwS+C5wCvBn4eVV9Ysb2hwFfBI4C9ge+BhxMP2geAI4FHgG+A5xeVfft\n9FckSRrKrnM9WFXbgG1d+4kk99H/YAfILLucDFxeVU8BDyfZAryy23ZLVT0MkOSKblsDQJLGZOg5\ngCQHAquAW7quNUk2JbkkyYqubz9gemC3afqBsR+wdZZ+SdKYDBUA3eWffwPeV1VPABcBLwOOAB4F\nPr5901l2rzn6JUljMuclIIAkuwFfAT5fVVcBVNUPBx6/GLimW50GVg7sfgD9a/6Z0b+y6595LENB\nkhahqmb7oj2n+X4FFOAS4N6qumCgf9+Bzd4I3N21rwbemuS5Sf6Q/gTwbcDtwMFJDkzyXOAt3baz\nvYiJWtauXTv2Gn4TaprUuqzJmlqoa7HmGwG8CjgDuCvJxq7vQ8DpSY6gfxnnIeBd3Yf3vUmuBO4F\nngbeU/3qnk6yBrge2AW4pPwFkCSN1Xy/Avoms48Srptjn48AH5ml/7q59pMkjZZ3As9jampq3CU8\nyyTWBJNZlzUNx5qGN6l1LcacN4KNWpKapHok6TdBEmpnTwJLkn57GQCS1CgDQJIaZQBIUqMMAElq\nlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEbN\n95/Cj9whhxw11uOvXr2Kz33u02OtQZJGYeICYPPmC8d49Dt4wQu+OMbjS9LoTFwAwDhHAP8zxmNL\n0mg5ByBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCk\nRhkAktQoA0CSGjVnACRZmeSmJPcmuSfJe7v+PZNsSPJgkhuSrBjY55NJNifZlGTVQP9Z3fYPJjlz\n+V6SJGkY840AngLeX1WHAauBv05yKHAusKGqDgFu7NZJciJwUFUdDLwTuKjr3xM4Hzi6W9YOhoYk\nafTmDICq2lZVd3btJ4D7gP2Bk4DLus0uA07p2idv76+qW4EVSfYBjgduqKrHq+pxYANwwk5+LZKk\nBRh6DiDJgcAq4FZg76p6DPohAby422w/YOvAbtP0A2O/rj2zX5I0JkP9l5BJXgh8BXhfVf08yQ43\n7ZZBNUvf9v5ZrBtoT3WLJGm7Xq9Hr9db8vPMGwBJdqP/4f+vVXVV1/1Ykn2qaluSfYEfdv3TwMqB\n3Q8AHun6pwb6VwJfn/2I64avXpIaNDU1xdTU1C/X169fv6jnme9XQAEuAe6tqgsGHroaOKtrnwVc\nNdB/ZrfvauDx7lLRDcBxSVYkeRHwF8D1i6pYkrRTzDcCeBVwBnBXko1d33nAR4Erk5wNfB94E0BV\nXZvkxCRbgCeBt3f9P0nyYeA73XOs7yaDJUljMmcAVNU32fEo4dgd7LNmB/2XApcuqDpJ0rLxTmBJ\napQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRG\nGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQB\nIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlR8wZAks8keSzJ3QN9\n65JMJ9nYLa8beOy8JJuT3J/kuIH+E7q+zUk+uPNfiiRpIYYZAVwKnDCjr4BPVNWqbrkOIMlhwFuA\nw7p9LkzfLsCnur7DgNOTHLqzXoQkaeHmDYCq+gbw01keyix9JwOXV9VTVfUwsAV4JXA0sKWqHq6q\np4Arum0lSWOylDmANUk2JbkkyYqubz9gemCbaWD/rn/rLP2SpDFZbABcBLwMOAJ4FPh41z/bqKDm\n6Jckjcmui9mpqn64vZ3kYuCabnUaWDmw6QHAI/QDYLB/Zdc/i3UD7alukSRt1+v16PV6S36eVM3/\nRTzJgcA1VfWKbn3fqnq0a78fOKqq3tZNAn+R/jX//YGvAQcBuwAPAK8FfgDcBpxeVffNOE6Nd2Bw\nM4cffj6bNt08xhokaWGSUFWzXWmZ07wjgCSXA8cAeyXZCqwFppIcQf/T+iHgXQBVdW+SK4F7gaeB\n91Q/YZ5Osga4nn4YXDLzw1+SNFpDjQBGxRGAJC3cYkcA3gksSY0yACSpUQaAJDXKAJCkRhkAktQo\nA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIA\nJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CS\nGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEbNGwBJPpPksSR3D/TtmWRDkgeT3JBkxcBjn0yy\nOcmmJKsG+s/qtn8wyZk7/6VIkhZimBHApcAJM/rOBTZU1SHAjd06SU4EDqqqg4F3Ahd1/XsC5wNH\nd8vawdCQJI3evAFQVd8Afjqj+yTgsq59GXBK1z55e39V3QqsSLIPcDxwQ1U9XlWPAxt4dqhIkkZo\nsXMAe1fVYwBVtQ14cde/H7B1YLtpYP+uf3qWfknSmOzsSeB0y6CapW97vyRpTHZd5H6PJdmnqrYl\n2Rf4Ydc/Dawc2O4A4JGuf2qgfyXw9dmfet1Ae2rGbpKkXq9Hr9db8vOkav4v4kkOBK6pqld06/8I\n/LiqPpbkXGBFVZ3bTQKvqaoTk6wGLqiq1d0k8O3AkfRHA98FjuzmAwaPU+MdGNzM4Yefz6ZNN4+x\nBklamCRU1WxXWuY07wggyeXAMcBeSbbS/zXPR4Erk5wNfB94E0BVXZvkxCRbgCeBt3f9P0nyYeA7\n3dOun/nhL0karaFGAKPiCECSFm6xIwDvBJakRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEG\ngCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBI\nUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1\nygCQpEYZAJLUKANAkhplAEhSo5YUAEkeTnJXko1Jbuv69kyyIcmDSW5IsmJg+08m2ZxkU5JVSy1e\nkrR4Sx0BFDBVVauq6uiu71xgQ1UdAtzYrZPkROCgqjoYeCdw0RKPLUlagp1xCSgz1k8CLuvalwGn\ndO2Tt/dX1a3AiiR774TjS5IWYWeMAG5IcnuSc7q+vavqMYCq2ga8uOvfD9g6sO80cMASjy9JWqRd\nl7j/n1fVtiR/AGxIcv8c24ZnjxaeefZm6wbaU90iSdqu1+vR6/WW/DypqqVXAyRZCzwBnEN/XmBb\nkn2Bm6rqj5P8C9Crqiu67e8Hjtk+Wuj6qj+oGJebOfzw89m06eYx1iBJC5OEqpr5BXtei74ElOR3\nkuzetX8XOA64G7gaOKvb7Czgqq59NXBmt/1q4PHBD39J0mgt5RLQ3sC/J9n+PF+oqhuS3A5cmeRs\n4PvAmwCq6tokJybZAjwJvH1ppUuSlmLRAVBVDwFHzNL/E+DYHeyzZrHHkyTtXN4JLEmNMgAkqVEG\ngCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBI\nUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1\nygCQpEYZAJLUqFTVuGv4pSQF46znZmBqjMf/lUl6XyRNtiRUVRa6367LUcxvvnF/+C74fZSkBfMS\nkCQ1ygCQpEYZAJLUKOcAJlQy/nkAJ6Kl324jHQEkOSHJ/Uk2J/ngKI/9m6fGvEj6bTeyAEiyC/Ap\n4ATgMOD0JIeO6viL1xt3AbPojbuAWfV6vXGX8CzWNBxrGt6k1rUYoxwBHA1sqaqHq+op4Arg5BEe\nf5F64y5gFr2RHCXJgpbXvOY1C95nmGUpJvEvqzUNZxJrgsmtazFGGQD7A1sH1qe7Pk2shV42WruI\nfea/FLWU8Fi/fv3YQ0iaVCO7EzjJacDxVXVOt34GcHRVvXdgm/q933v9SOqZzS9+8WOefPLb/Po1\n8HXdMkqZUcNM61j+muarYTbr2Pl1LaaOQetYek0GwCQZ948TJuULweB5WOydwKMMgNXAuqo6oVs/\nD3imqj42sI2zj5K0CJMeALsCDwCvBX4A3AacXlX3jaQASdKvGdl9AFX1dJI1wPXALsAlfvhL0vhM\n1L8GKkkanbH8UxDz3RCW5HlJvtQ9fkuSl05ATa9OckeSp5Kcutz1DFnT3yb5XpJNSb6W5CUTUNO7\nk9yVZGOSb4zqXo9hbzJMclqSZ5IcOe6akvxVkv/uztXGJO8Yd03dNm/u/lzdk+QL464pyScGztED\nSX46ATW9JMlN3WfCpiSvW+6ahqzrpUlu7Gq6Kcncv7SsqpEu9C//bAEOBHYD7gQOnbHNe4ALu/Zb\ngCsmoKaXAq8ALgNOnZDzNAU8v2u/e0LO0+4D7TcA103CudpeG/AfwLeAI8ddE3AW8MnlPj8LrOlg\n4A5gj259r3HXNGP7NcDF464J+DTwrq59KPDQhLx/Xwb+smu/BvjcXM85jhHAMDeEnUT/gxbgK/Qn\njsdaU1V9v6ruBp5Z5loWUlOvqv63W70VOGACavr5wOoLGc35GvYmww8DHwP+j+X/becwNWUEdSy0\npnOAT1XVzwCq6kcTUNOgtwGXT0BNzwB7dO0VwCPLXNOwdR0K3Ni1e7M8/mvGEQDD3BD2y22q6mng\nZ0n2HHNNo7bQms4Grl3WioasKcl7kmyh/2H73pmPj6OuJKuA/avqq13Xck9+DXOuCji1G65/Ocly\nB/gwNR0MvDzJN5N8O8nxE1AT0L+8Qf/b79cnoKZ1wBlJtgJfBf5mmWsatq5NwGld+43A7kletKMn\nnJR/DnrmX8bZvhWNerZ6EmfHZ62pu6nuSOCfRlsOMEtNVXVhVR0EfBD4+9GX1C9jeyPJc4B/Bj4w\n8Pg47uaZea6uAV5aVX8KfI1fjXpHaWZNuwIHAccApwMXJ9njWXuNtqbt3gp8ubrrGyM285hvAy6t\nqpXAicDnR18S8Oy6PgAck+QO4NX0RyZP72jncQTANLByYH0lzx4+TQMvgV/eP7BHVS3nxM8wNQ0a\nxR/AoWpKcizwIeCkblg49poGfAk4ZVkr6puvrt2BPwF6SR4CVgNXL/NE8Lznqqp+MvCeXQz82TLW\nM1RN3TZXV9Uvquph+vfuHDTmmrZ7C8t/+QeGq+kdwJUAVXUL8Pwke427rqp6tKpOraojgb/r+gYv\ny/665Z64mGUiY1fgP+kP5Z7LjieBL+rab2X5JzfnrWlg288ymkngYc7TKvqTQn80Qe/dQQPtNwC3\nTUJdM7a/ieWfBB7mXO0z0H4j8K0JqOl44LNdey/gv4AXjfu9A17OCCZaF3CergXO6tqHAo9MSF2/\nDzyna/8D/X99YcfPOYoTOssLeR39bxZbgPO6vvXAG7r28+in62bgFuDACajpKPrX354AfgTcPcaa\nXt+1NwCPAhu75aoJOE8XAPd09dw41wfxKOuase2yB8CQ5+oj3bm6sztXh4y7pm7948D3gLuAN09I\nTWuBj4ziz9KQ792hwDe7924jcOyE1HUa8GC3zaeB3eZ6Pm8Ek6RGTcoksCRpxAwASWqUASBJjTIA\nJKlRBoAkNcoAkKRGGQCS1CgDQJIa9f+DsW4Z9lkSPwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x102f81e50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# predict probabilities\n",
    "import matplotlib.pyplot as plt\n",
    "probs = logreg.predict_proba(X_test)[:, 1]\n",
    "plt.hist(probs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  |    0    1 |\n",
      "--+-----------+\n",
      "0 |<2416>   5 |\n",
      "1 |   58  <21>|\n",
      "--+-----------+\n",
      "(row = reference; col = test)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# use 0.5 cutoff for predicting 'default'\n",
    "import numpy as np\n",
    "preds = np.where(probs > 0.5, 1, 0)\n",
    "print ConfusionMatrix(list(y_test), list(preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  |    0    1 |\n",
      "--+-----------+\n",
      "0 |<2340>  81 |\n",
      "1 |   34  <45>|\n",
      "--+-----------+\n",
      "(row = reference; col = test)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# change cutoff for predicting default to 0.2\n",
    "preds = np.where(probs > 0.2, 1, 0)\n",
    "print ConfusionMatrix(list(y_test), list(preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.954\n",
      "0.569620253165\n",
      "0.966542750929\n"
     ]
    }
   ],
   "source": [
    "# check accuracy, sensitivity, specificity\n",
    "print metrics.accuracy_score(y_test, preds)\n",
    "print 45 / float(34 + 45)\n",
    "print 2340 / float(2340 + 81)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x118f9ef10>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEPCAYAAACtCNj2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XuYHFWd//H3JwkkIVcgEbkEgxAQg7oJCgiiw48AUQFB\nBSS6AsvjFUQUd7O4guO6LqK7uovIggJxUW6KqOEi12UUQW4GQgLhJkQIF0OAXEwI5PL9/VE1nU6n\np7t6ZqqrZ+bzep5+uqr6dNV3amb626dOnXMUEZiZmQEMKjoAMzNrHU4KZmZW4qRgZmYlTgpmZlbi\npGBmZiVOCmZmVpJrUpB0saS/SppXo8w5kh6XNFfSlDzjMTOz2vKuKcwCpnf1oqQPALtExCTg08D/\n5ByPmZnVkGtSiIjbgVdqFDkc+N+07N3AWEnb5BmTmZl1reg2he2BZ8rWFwE7FBSLmdmAV3RSUPoo\nt76IQMzMDIYUfPxFwISy9R2A5yoLSfIATWZm3RARlV+8ayo6KcwGTgaukLQPsDQi/lqtoAfuS7S3\nt9Pe3l50GLlZswaWL89W9uyz25k5sz3XePqKrs7FwoVw1FHw5JNND6kw/f1/pBFSQ/kAyDkpSLoc\neB8wTtIzwNeBzQAi4oKIuF7SByQ9AawETsgzHmt9p5wCl1wCw4bVL7tqFVx0Uf4x9QW1zsXUqc2N\nxfq2XJNCRByboczJecZgfcuqVXDeeXDccfXLtrcnD/O5sN5T9OUja1BbW1vRIWSycGHyaNQLL2Qv\n21fORTP4XGzgc9Ez6gvX6iVFX4jTNjjkkOQDfsstG3/vd74De+3V+zGZDTSS+lxDs/VT69bBf/4n\nTJtWdCRm1ggnBevSTTfBo492773PPFO/jJm1Hl8+si5NmQK77Qbjxzf+3kGD4PTT4Y1v7P24zCwb\nXz6yXjdzZpIczGxgcFIYwFatSj70X3+9+utPP93ceMyseL58NID9+c+w997wrW9Vf32zzeDjH4eh\nQ5sbl5n1Dl8+soaNGQOf+UzRUZhZq3BS6Oe+/nW4+ebqr61eDUP8F2BmZfyR0M/dfjt87GPwzndW\nf/0Nb2huPGbW2pwUBoA99oB99y06CjPrC5wUCnbqqfmO9PnqqzByZH77N7P+xUmhYM8+Cz/8IRx5\nZD77HzQIRozIZ99m1v84KeTkueeSCWPqWbkShg+HUaPyj8nMrB4nhRw89hhMngzbbVe/rATbb59/\nTGZmWTgp5OC11+Atb4F584qOxMysMU4KvSgCbr0VHnmk6EjMzLrHSaEXPfccHHpoModAXg3HZmZ5\nclLoRevXJ8NMX3tt0ZGYmXXPoKID6E+WLYOlS4uOwsys+5wUetGLL8LWWxcdhZlZ9zkp9CIJJk4s\nOgozs+5zUuglixfDWWclicHMrK9yQ3MvefLJZNKan/+86EjMzLrPNYVeNG4cTJ1adBRmZt3nmkIP\nLF0KBx2U9GBeuTLbsBZmZq3MSaEHli5NRjm94YZkfZttio3HzKynnBR6aPPN4e1vLzoKM7Pe4TaF\njF55JRniWtrw2Gkn2GqroiMzM+s9rilk9OqrMHZsMr6RmVl/5aSQQURyu2lnDcHMrL/y5aMM5s6F\ngw+GffctOhIzs3zVrClImgB8DNgf2A54FZgPXAv8NiLW5x5hC1izBvbYA666quhIzMzy1WVSkDQL\n2AG4Bvg28CIwDNgVeD/wNUkzI+L3zQi0KGvXwm23FR2FmVlzKCKqvyDtERHzu3yjNBSYEBFP5BVc\n2bGiqzjzNn8+7LcfnHkmnHZaISGYmXWLJCKioZbQLtsUOhOCpEMlbVIuIl6rlxAkTZf0iKTHJc2s\n8vqOkm6TNEfSXEnvbyT4ZoiAHXd0QjCzgSHL3UfHAudIugqYFRELsuxY0mDgXGAa8Cxwr6TZFe//\nGnBFRFwgaXfgemCnhn6CHFxwATz9dLK8eHGxsZiZNVPdu48i4uPAFOBJYJakP0r6tKRRdd66F/BE\nRCyMiDXAFcCHKsqsB8aky2NJkkfhzjwzmVpziy2S+RHa24uOyMysOTL1U4iIZZJ+CQwHTgWOBP5J\n0jkRcU4Xb9seeKZsfRGwd0WZduAmSV8ARgAHNhB7rk491WMZmdnAUzcpSDocOAHYBfgp8K6IWCxp\nC2AB0FVSqKaytXgGySWp70vaB/gZMLnaG9vLvq63tbXR1tbWwGGz+drX4MEHk4HuBrkHh5n1MR0d\nHXR0dPRoH13efVQqIF0CXFjt1lNJ0yLili7etw/QHhHT0/XTgfURcXZZmfnAIRHxbLr+Z2DviFhS\nsa+m3H20227wxS/CpEkwbZp7L5tZ39ardx+VeaEyIUg6G6CrhJC6D5gkaaKkzYFjgNkVZZ4maYgm\nbWgeVpkQmu3AA5M5EpwQzGwgypIUDqqy7QP13hQRa4GTgRuBh4ErI2KBpG9IOiwtdhrwKUkPAJcB\nx2ULu/e98go89hgMHVpUBGZmxavVee1zwOeBnYE/l700CrgjvSupKZpx+WjhQthzT3jppVwPY2bW\nNN25fFSrofky4LckQ1zMBDp3vCIi+t1H55o1yXwJZmYDWa2awuiIWC5paza9a4iIeDnv4Mpiyb2m\nsPPOsGoVPP98rocxM2ua3q4pXA58EPgTmyaFAN7cWHitbeRIuPrqoqMwMytWl0khIj6YPk9sWjQF\nWrq06AjMzIpX9+4jSbMlzZA0ohkBFeXpp2HMmPrlzMz6syy3pH6PZJKdhyVdJemjkoblHFfTjRgB\n48YVHYWZWbHqDnMRER1Ah6QhwAHAp4CLgdH5htYcL7+ctCWsWVN0JGZmxcs0IJ6k4cDhwNHAVOB/\n8wyqmW65Bb71LTjppGRUVDOzgSzLgHhXAvsANwA/BH4XEevyDqyZ3vUu+N73io7CzKx4WWoKs4AZ\n/SURrFsHZ5wBK1cm6489BqPqzQxhZjZA1Oq8dmBE3CrpI2zcT0FARETT7urvzc5rL78MO+wAZ521\nYdvee8M++/TK7s3MWkZvd157L3ArcBhVejQDfbar17BhyRDZZma2sVqd176eLv5rRDxZ/pqkPtub\n+ayzkhFRzcxsU1n6KVxVZdsvejuQZrntNjj33KKjMDNrTV3WFNJJb94KjJX0YdK2BJL+CX2289rw\n4fD2txcdhZlZa6rVprAbSXvCmPS50wqSDmx90p13Fh2BmVnryjJH87sj4o9NiqerGHrt7qPx42Hu\nXNhuu17ZnZlZy+rVu48kzYyIs4EZkmZUvBwRcUp3gjQzs9ZV6/LRw+lz+XwKnRkn3xlvcvLaa7Bk\nSTJ3gpmZbaru5aONCkuDgZERsSy/kKoet1cuH61alYyEumpVLwRlZtbiunP5KMt8CpdJGp3OpzCP\nZAjtf+pukEVauhRefbXoKMzMWleWfgqTI2I5cATwW2Ai8Pd5BpWXyy6DrbcuOgozs9aVJSkMkbQZ\nSVK4JiLW0EfbFACOP77oCMzMWleWpHABsBAYCfxe0kSgqW0KveUPf4BeurPVzKxfaqihGUCSgMER\nsTafkKoes1camseOhfPOgxmVN9iamfVDvT1KaudOhwEfIWlL6CwfwL82GmDRttrKQ2SbmdWSZZKd\n3wBLSforrM43nPzcdBM89VTRUZiZtbYsSWH7iDgk90hydv310NYGb3pT0ZGYmbWuLA3Nd0rq8+OK\nSnD44TB4cNGRmJm1rixJYX/gT5IekzQvfTyYd2C94fzz4V3vSh6XXQZDstSLzMwGsCwfk+/PPYqc\nzJkDBx0ERx6ZrE+eXGw8Zmatrm5SiIiFkvYHdomIWZLGk/RZ6BMmTkxqCmZmVl+WsY/agX8CTk83\nbQ78LMeYeuykk5IZ1i66CEaPLjoaM7O+I8vloyOBKSS3pBIRz0oalWtUPfTCC0lCOPLIJDmYmVk2\nWRqaX4uI9Z0r6WipLW35chg61AnBzKxRWZLCLyRdAIyV9GngVuDCLDuXNF3SI5IelzSzizJHS3pI\n0nxJl2YPvWu33JJMu2lmZo3JNPaRpIOBg9PVGyPi5gzvGQw8CkwDngXuBY6NiAVlZSYBVwIHRMQy\nSeMiYkmVfTU09tHIkcklJM+wZmYDWS5jHwFExE2S5gDvBV7OuO+9gCciYmEa3BXAh4AFZWU+BZzb\nOZNbtYTQqAhYubKnezEzG5i6vHwk6TpJe6TL2wLzgROAn0r6UoZ9bw88U7a+KN1WbhKwm6Q/SPqj\npB4Pp/Hii8mz2xPMzBpXq6YwMSLmp8snADdFxCfTO4/uBL7fjeNVXgMaAuwCvA+YANwuaY+ezAEd\nAW94g4ezMDPrjlpJYU3Z8jTgxwARsULS+upv2cgikg/6ThNI2hYqy9wVEeuAhZIeJUkSf6rcWXt7\ne2m5ra2Ntra2qgf9zW9g8eIM0ZmZ9TMdHR10dHT0aB9dNjRLuha4keSD/CLgzRHxiqQtgHsjouag\nEZKGkDQ0Hwg8B9zDpg3Nh6Tbjpc0DpgDvCMiXqnYV+aG5re9DfbfP5lMx8xsIOtOQ3OtW1JPBPYA\njgOOKfug3huYVW/H6cxsJ5MkloeBKyNigaRvSDosLXMj8JKkh4D/A75SmRAaNWIEfPKTPdmDmdnA\n1fB0nEVopKaw5ZZw3XWw7745B2Vm1uJ6taYg6ceS3tbFayMlnSjpE40Gmbdly5JB8MzMrHG1GprP\nA85ME8N84EVgGElD8BjgYlpwYLyttoLNNy86CjOzvqnu5aP0FtR3AtsCq4AFEfFoE2Irj6Hu5aP3\nvAdeegkefxyWLIGxY5sUnJlZi+rO5aN+06YgwUMPwRZb+PKRmRk4KbB+ffJsZma9f0tqn/HCC0VH\nYGbWP2ROCq08j8KqVbDTTq4lmJn1VJbpOPeV9DDwSLr+d5Jaqr/wihXJxDpmZtYzWWoK/wVMB5YA\nRMQDJAPYtYxZs5JOa2Zm1jOZLh9FxNMVm9bmEEu3DRkCn/lM0VGYmfV9WZLC05L2A5A0VNJX2Hii\nnEK9+ircXHceODMzyyJLUvgccBLJBDmLgCnpeku47z548MGk85qZmfVMluk4d42IGeUb0prDHfmE\n1Lj3vAf22afoKMzM+r4sNYVzM25runXr4JJLio7CzKz/6LKmIOndwL7AeElfBjp7AYyiRTq9LVsG\nP/sZ/OpXRUdiZtY/1Lp8tDlJAhicPndaDnw0z6AaMXw4TJ9edBRmZv1DllFSJ0bEwuaE02UMVcc+\nevll2GWX5NnMzDbWnbGPsjQ0r5L0H8BbgeHptoiI/9dogGZm1tqytA1cSjLExZuBdmAhcF9+IZmZ\nWVGyJIWtI+JC4PWI+F1EnAC0RC3hnnvglVeKjsLMrP/Icvno9fT5BUmHAs8BLTHS0NKlcOihRUdh\nZtZ/ZEkK35I0FjgN+AEwGvhSrlFlsHo1rFwJI1p2QG8zs76nblKIiGvSxaVAG5R6NBdqr73gqafg\ns58tOhIzs/6jVue1IcDRwHbADRExX9JhwOnAFsDfNSfE6v72N5g7F9785iKjMDPrX2rVFC4CdgDu\nAc6R9DywJ/DPEfHrZgTXlTVr4JlniozAzKx/qpUU3gm8LSLWSxoGvADsHBEvNSe0rnV0wNq1sPXW\nRUdiZta/1Lol9fWIWA8QEauBp1ohIUAyEN4hh8CYMUVHYmbWv9SqKbxF0ryy9Z3L1iMi3p5jXDUt\nXgyvv16/nJmZNaZWUti9aVE0aNkyGDq06CjMzPqfLpNC0YPg1SLBzjsXHYWZWf/TEvMiNOq662D9\n+qKjMDPrf/pkUnjwQTjggKKjMDPrfzIlBUlbSNot72CyGjMGJk8uOgozs/6nblKQdDhwP3Bjuj5F\n0uy8A6tlwYKkXcHMzHpXlppCO7A38ApARNwP7JRjTJnsVHgEZmb9T5aksDYillZsqz2HZ84239w1\nBTOzPGRJCvMlfRwYImmSpB8Ad2bZuaTpkh6R9LikmTXKfVTSeklTa+1vxgzYcksYNCh5mJlZ78ry\n0foFYDLwGnA5sBw4td6bJA0GzgWmk8zvfKykTTrESRoFnALcVW+ff/kLXHYZLFkCm22WIXIzM2tI\nlqSwa0R8NSLemT7+JR0LqZ69gCciYmFErAGuAD5Updw3gbNJkk7Ni0LLl8Po0Z5Yx8wsL1mSwvcl\nPSrpm5IauRF0e6B8gOtF6bYSSVOA7SPiunRTl20VjzwC8+fD+PENRGBmZg2pmxQioo1kxrUlwI8k\nzZN0RjePV/rQlzQI+D7wlbLXu6wprF4N73gH7LprN49sZmZ1ZZmjmYh4HvhvSf8HzATOJLnsU8si\nYELZ+gTg2bL1USRtFR1KbiV6IzBb0mERMadyZ+ef384LL0B7O7S1tdHW1pYldDOzAaOjo4OOjo4e\n7UMRte8ulfRWkmk5jwJeImkbuCoiFtd53xDgUeBA4DmSGdyOjYgFXZS/DTitWkKQFPffHxx/PDzw\nQN2fyczMAElEREM38GepKcwiuevo4Ih4tl7hThGxVtLJJD2hBwMXRcQCSd8A7ouIaxoJ1MzM8le3\nptAKXFMwM2tcr9YUJP0iIo6qmH2tU6Ezr5mZWT5qXT76Yvp8KJveFdT61QszM2tYl7ekRsRz6eLn\n0w5opQfw+aZEV+b552HFimYf1cxsYMnSee3gKts+0NuB1LNqFWyzTbOPamY2sNRqU/gcSY1g54p2\nhVHAHXkHVs222xZxVDOzgaPLu48kjQG2BL5N0mGts11hRUS81JzwSrHE5MnBxIlw7bXNPLKZWd/V\nnbuPaiWF0RGxXNLWVGlYjoiXuxdm4yTF2LHBLbfAnns266hmZn1bb3deuxz4IPAnqt9t1NS5z0aP\nhnHjmnlEM7OBp890XoNg8WKPkmpmllV3agp17z6StJ+kkeny30v6nqQ3dTfI7ho0yAnBzCxvWW5J\nPR9YJekdwJeBJ4FLco2qivXrm31EM7OBJ0tSWBsR64EjgB9GxLkkt6WamVk/k2WU1BWSvgp8Atg/\nHRK76TMkf+ELzT6imdnAk6WmcAzJ/Mn/EBEvANsB3801KjMzK0SW6TifBy4Fxko6FFgdEU1vUzAz\ns/xlufvoaOBukpnXjgbukXRU3oGZmVnzZWlT+Brwrs7pNyWNB24FfpFnYJX6QHcKM7M+L0ubgoAX\ny9ZfYtP5FXI3YUKzj2hmNvBkqSncANwo6TKSZHAM8Ntco6pi8uRmH9HMbOCpmxQi4h8lfRh4D0lS\nuCAifpV7ZGZm1nS15lPYleTW012AB4F/jIhFzQrMzMyar1abwsXAtcBHgDnAOU2JyMzMClPr8tHI\niPhxuvyIpPubEVBXNmt6H2ozs4GnVlIYJmlquixgeLouICJiTu7RlTnggGYezcxsYKo181oHG0+u\no/L1iGjax7SkWLMmGJLlXikzMwN6eTrOVuKkYGbWuFwm2TEzs4HDScHMzEqcFMzMrCTLKKmD0rmZ\nz0zXd5S0V/6hmZlZs2WpKZwHvBuYka7/Ld1mZmb9TJb7efaOiCmdndci4mVJTe9KNnhws49oZjbw\nZKkpvC6p9JGczqewPr+QqlPTB+s2Mxt4siSFHwC/At4g6d+BO4Czco3KzMwKkanzmqTdgQPT1Vsj\nYkGuUW16/OgLnezMzFpJLj2aJe3YuZg+B0BEPN1whN3kpGBm1ri8ksJ8Nox5NAzYCXg0IjLNhSZp\nOvBfwGDgwog4u+L1LwMnAmtJpv38h8qE46RgZta4pox9lI6UelJEnJih7GDgUWAa8CxwL3Bs+eUn\nSW3AXRGxWtJngbaI+FjFfpwUzMwa1JSxj9Ihs/fOWHwv4ImIWBgRa4ArgA9V7K8jIlanq3cDOzQa\nk5mZ9Y66/RQknVa2OgiYSvKtP4vtgWfK1hdRO6GcCFyfcd9mZtbLsnReG1m2vJZkis5f9uCYVa8D\nSfoEScL5UrXX29vbS8ttbW20tbX1IAQzs/6no6ODjo6OHu2jZptC2ibwnYg4rctCtXYu7QO0R8T0\ndP10YH2VxuZpJHNAvzcillTZj9sUzMwa1KttCpKGRMQ6YD+p2/2J7wMmSZooaXPgGGB2xXGmAOcD\nh1VLCGZm1jy1Lh/dQ3I55wHgN5J+AaxKX4uIuLreziNiraSTgRtJbkm9KCIWSPoGcG9EXAt8BxgB\nXJXmnr9ExBHd/onMzKzbas3RfH86EN5PqNIOEBEn5BxbeSy+fGRm1qDuXD6qVVMYn3Ysm9ezsMzM\nrK+olRQGA6OaFYiZmRWv7uWjJsdTlS8fmZk1rik9ms3MrP+qVVPYOiJeanI8VbmmYGbWuKYMiFcE\nJwUzs8b58pGZmfWIk4KZmZU4KZiZWYmTgpmZlTgpmJlZiZOCmZmVOCmYmVmJk4KZmZU4KZiZWYmT\ngpmZlTgpmJlZiZOCmZmVOCmYmVmJk4KZmZU4KZiZWYmTgpmZlTgpmJlZiZOCmZmVOCmYmVmJk4KZ\nmZU4KZiZWYmTgpmZlTgpmJlZiZOCmZmVOCmYmVmJk4KZmZU4KZiZWYmTgpmZlTgpmJlZSa5JQdJ0\nSY9IelzSzCqvD5V0Zfr6XZLelGc8ZmZWW25JQdJg4FxgOvBW4FhJu1cUOxF4KSImAd8Hzs4rnv6i\no6Oj6BBahs/FBj4XG/hc9EyeNYW9gCciYmFErAGuAD5UUeZw4H/T5V8CB+YYT7/gP/gNfC428LnY\nwOeiZ/JMCtsDz5StL0q3VS0TEWuBZZK2yjEmMzOrodkNzVGxrgxlzMysSRSRz2ewpH2A9oiYnq6f\nDqyPiLPLytyQlrlL0hDg+YgYX2VfThRmZt0QEdW+fHdpSF6BAPcBkyRNBJ4DjgGOrSgzGzgOuAv4\nKHBrtR01+kOZmVn35JYUImKtpJOBG4HBwEURsUDSN4D7IuIa4CLgp5IeB14CPpZXPGZmVl9ul4/M\nzKzvaakeze7stkGGc/FlSQ9JmivpFkk7FhFn3uqdh7JyH5W0XtLUZsbXTFnOhaSj07+L+ZIubXaM\nzZLh/2NHSbdJmpP+j7y/iDibQdLFkv4qaV6NMuek52qupCk1dxgRLfEgucT0BDAR2Ax4ANi9oszn\ngfPS5WOAK4qOu8Bz0QYMS5c/2x/PRZbzkJYbBfweuBOYWnTcBf5NTALmAGPS9XFFx13gufgR8Jl0\neXfgqaLjzvF87A9MAeZ18foHgOvT5b2Bu2rtr5VqCu7stkHdcxERHRGxOl29G9ihyTE2Q5a/CYBv\nkvSGf43qtzn3B1nOxaeAcyNiGUBELGlyjM2S5VysB8aky2OBZ5sYX1NFxO3AKzWKlD43I+JuYKyk\nbboq3EpJwZ3dNshyLsqdCFyfa0TFqHse0qrw9hFxXbqpvzaSZfmbmATsJukPkv4o6ZCmRddcWc5F\nO/AJSc8A1wFfaE5oLana+eryS2QrJYVq3Nltg6o/p6RPAFOB7zY3nMKUzoOkQSRjZn2l7PX+WlOo\npvJvYgiwC/A+ktu/L5Q0ZpN39U+V52IGMCsiJpBcPvlZ80NqGWLT/4v1XRVupaSwCJhQtj6BTat8\ni4AdAdLObmMiola1qa/Kci6QNA34KnB4Wo3ub+qdh1HAZKBD0lPAPsDsftrYnPX/Y3ZErIuIhcCj\nJEmiv8lyLv4B+DlARNwFDJM0rjnhtZzK87UDSd+xqlopKZQ6u0nanKQheXZFmc7OblCjs1s/UPdc\npJdNzgcO68fXjmueh4hYFhHjI2KniNiJpBPkYRExp6B485Tl/+PXwAEA6QfgrsCTTY2yObKci6eB\naQDp6MzD+vH/ST2zgU9CaaSJpRHx164K59mjuSHhzm4ldc7FvRFxLfAdYARwlSSAv0TEEYUFnYOM\nfxMDQpZzERE3SjpY0kPAOuAr/bEmnfHv4jTgx5K+RHJp6biu99i3Sbqc5JLhuLQN5eskd2URERdE\nxPWSPiDpCWAlcELN/aW3KZmZmbXU5SMzMyuYk4KZmZU4KZiZWYmTgpmZlTgpmJlZiZOCmZmVOCkM\nMJLWSbq/7NHlkNuS/tYLx/uJpCfTY/0p7TzT6D5+LOkt6fJXK167o6cxpvvpPC/zJM2uNzyEpHd0\nZzhmSdtKuiZd3jod3nmFpB90M+5/SYfJnpvGv1d39lNj/9dJGp0unyLpYUk/lXRYraHM0/J3pM9v\nklQ562K18h+U1N4rgVu3uZ/CACNpRUSM6u2yNfYxC7gmIq6WdBDwHxHxjh7sr8cx1duvpJ8Aj0XE\nv9cofzywZ0Q0NNCapO8Cv4+IayRtQTLk8R7AHt3Y17uB/wTeFxFr0sEhh0bE843sp4HjLQAOjIgu\nh0jo4n1twGkRcViGsnOA/SLi1e5FaT3lmsIAJ2mEkkl6/iTpQUmHVymzraTfl32Tfk+6/WBJd6bv\n/bmkEV0dJn2+nXQsHiWTBM1LH18si+U6SQ+k249Kt3dI2lPSt4HhaRw/TV/7W/p8Zfk397SGcqSk\nQZK+K+me9Nv0pzOclj+SjropaS9JdyiZrOUOSbumQyv8K3BMGstRaewXp8eZU+08pj4M3AAQEasi\n4g6SIb+7443Aks5xryLi5c6EIGmhpLPT3+ndknZOt4+XdFUa5z2S9k23j5Q0Ky0/V9KRZfvZWtL5\nwJuBGySdKun4ztqNpG0k/Sr9vT3QWRvUhprmt4H903N1avq3VPpikJ7XPdLVDuDQbp4P6w1FTxDh\nR3MfwFrg/vTxS5JhAkalr40DHi8ruyJ9Pg34aro8CBiZlv0dMDzdPhM4o8rxZgEfSZePIvnAnQo8\nCAwnGapjPvB3wEeAH5W9d3T6fBvp5DmdMVWJ8QjgJ+ny5iRj3wwFPg38S7p9KHAvMLFKnJ37GUwy\nkNrB6fooYHC6PA24Kl0+Djin7P3/Dnw8XR5LMhjdFhXH2IlkGIbKYx8H/KAbv8sR6e/xUeCHwHvL\nXnsKOD1d/nuS2hrAZSTfxCEZXPLhdPls4Htl7x9btp+tqiyXYgauBE4p+/vo/L11ntP3dR4/Xf8k\n8P10eVeSoVs6X5tRfl79aP6jZcY+sqZ5NSJK0/FJ2gw4S9L+JMPpbifpDRGxuOw99wAXp2V/HRFz\n00sCbwXuVDL20uYkM59VEvBdSV8DFpPM/XAQcHWklwgkXU0ye9QNadlvA9dGxB8a+LluAM5Jv8W/\nH/hdRLxBdjJmAAADl0lEQVQm6WDgbZI+mpYbTVJbWVjx/uGS7iepISwAbkm3jwUukbQLyRg6nf8z\nlcMRHwwcJqlzGO+hJCNTPlpWZlvgxQZ+ppoiYqWkPUnO3QHAlZL+OSI6J6K6PH2+gmSIcUgS2+7p\n7wxglKSRJBNWHVO276UNhHIA8In0feuB5RWvVw7bfBVwhqR/JBnNdFbZay8C2zVwbOtlTgr2cZJv\n/VMjYp2SIaiHlReIiNvTpHEo8BNJ3yOZ6enmiJhRZ/9BMjDb1Z0b0raF8g8KJYeJx5UMe/1B4N8k\n3RoR38zyQ0TEakkdwCHA0STfiDudHBE319nFqxExRdJwkoHWTgJ+QDKr260RcaSSOcE7auzjwxHx\neI3XV1FxbutR0nB8Qbp6RiSDIZakH8K/A36nZI7e49gwO+FGRTt3CewTERtdslJZluimzO+PiFWS\nbiap3R0F7Fn28jDA7QkFcpuCjQYWpwnhAOBNlQWU3KG0JCIuBC4kaRy9C9iv7Fr1FpImdXGMyg+M\n3wNHSBqetkMcAdwuaVtgdURcCvxHepxKa5TMpVHNFSTfPPcn+WAnff5853vSNoEtung/ae3lFOAr\n6XtGs2Hs+fLRJZeTXFrqdGP6PtLjVIv9cZJ5hSt1+YEaEfdExJT0sVFCSH+W8nM+hY1rQMeUPXfW\n4m6ibBaysmv7NwEnl20f21VMVWK+Ffhc+r7BkipvBFjBxucKkr+jc4B7KmoluwJdTkBv+XNSGHgq\nbze7FHinpAdJrj0vqFL2AOB+JXeGHA38dyRj0x8PXC5pLklbwW5ZjhkR9wM/IbksdRfw44iYC7wN\nuDu9jHMm8G9V9vUj4MHOhuaKfd9EkhBujmS6Vkg+fB4G5qTfpP+H6jXk0n4i4gFgLsmH6XdILq/N\nIWlv6Cx3G/DWzoZmkhrFZmlD7TzgG5scIGIl8OfORApJQy7JHUTHS3pa6a23GY0kqbk9lP4O3kIy\nDWWnLdPtXwC+lG47heT3PVfJENufSbf/W1p+nqQHgLYqx4uK5c71LwIHpH9D95FcViwvPxdYmzZC\nfxEgkjkvlrHxpSPS416HFca3pJo1kaQjSG5lPSPn4zyVHuflPI/TXZK2A26LiN3Ktm0DXBoR04qL\nzFxTMGuiiPg1mzZy53KoJhyjWyR9kqSG+NWKlyYAX25+RFbONQUzMytxTcHMzEqcFMzMrMRJwczM\nSpwUzMysxEnBzMxKnBTMzKzk/wMDd5yQ0uAOAQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x118ffced0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## ROC CURVES and AUC\n",
    "\n",
    "# plot ROC curve\n",
    "fpr, tpr, thresholds = metrics.roc_curve(y_test, probs)\n",
    "plt.plot(fpr, tpr)\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.0])\n",
    "plt.xlabel('False Positive Rate (1 - Specificity)')\n",
    "plt.ylabel('True Positive Rate (Sensitivity)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.941011926236\n"
     ]
    }
   ],
   "source": [
    "# calculate AUC\n",
    "print metrics.roc_auc_score(y_test, probs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.94819099843061938"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# use AUC as evaluation metric for cross-validation\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "X = data[['balance']]\n",
    "y = data.default\n",
    "logreg = LogisticRegression()\n",
    "cross_val_score(logreg, X, y, cv=10, scoring='roc_auc').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.59503752064417714"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# compare to a model with an additional feature\n",
    "X = data[['balance', 'income']]\n",
    "cross_val_score(logreg, X, y, cv=10, scoring='roc_auc').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# ROC/AUC are very meaningful when:\n",
    "# 1. class sizes are inbalances\n",
    "# 2. comparing across different binary classification models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## BONUS: CALCULATE THE 'SPAMMINESS' OF EACH TOKEN\n",
    "\n",
    "\n",
    "# create separate DataFrames for ham and spam\n",
    "df_ham = df[df.label==0]\n",
    "df_spam = df[df.label==1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# learn the vocabulary of ALL messages and save it\n",
    "vect = CountVectorizer(ngram_range=(1,1), stop_words='english')\n",
    "vect.fit(df.msg)\n",
    "all_features = vect.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# create document-term matrix of ham, then convert to a regular array\n",
    "ham_dtm = vect.transform(df_ham.msg)\n",
    "ham_arr = ham_dtm.toarray()\n",
    "ham_dtm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# create document-term matrix of spam, then convert to a regular array\n",
    "spam_dtm = vect.transform(df_spam.msg)\n",
    "spam_arr = spam_dtm.toarray()\n",
    "spam_dtm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# count how many times EACH token appears across ALL messages in ham_arr\n",
    "ham_counts = np.sum(ham_arr, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# count how many times EACH token appears across ALL messages in spam_arr\n",
    "spam_counts = np.sum(spam_arr, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# create a DataFrame of tokens with their separate ham and spam counts\n",
    "all_token_counts = pd.DataFrame({'token':all_features, 'ham':ham_counts, 'spam':spam_counts})\n",
    "all_token_counts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# add one to ham counts and spam counts so that ratio calculations (below) make more sense\n",
    "all_token_counts['ham'] = all_token_counts.ham + 1\n",
    "all_token_counts['spam'] = all_token_counts.spam + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# calculate ratio of spam-to-ham for each token\n",
    "all_token_counts['spam_ratio'] = all_token_counts.spam / all_token_counts.ham\n",
    "all_token_counts.sort_index(by='spam_ratio', ascending = False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}