full_data_clean_preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Subject:__ Topics of Tweets of US Congress through the Lens of Campaign Finance\n",
    "\n",
    "__Date:__ 09/10/2018\n",
    "\n",
    "__Author:__ Edmund D. Chitwood\n",
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Summary:__<br>\n",
    "<br>The following Notebook \n",
    "- reads Tweets of all Congress Members into a Pandas DataFrame,\n",
    "- performs EDA,\n",
    "- cleans them,\n",
    "- preprocceses them in preparation for topic modeling.\n",
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read in Data and EDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import nltk\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets = pd.read_pickle('congress_tweets.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1852507, 9)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Dataset includes 1.8 million Tweets\n",
    "tweets.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>retweets</th>\n",
       "      <th>favorites</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.852507e+06</td>\n",
       "      <td>1.852507e+06</td>\n",
       "      <td>1.852507e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>7.488233e+01</td>\n",
       "      <td>1.814578e+02</td>\n",
       "      <td>6.145035e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>9.643075e+02</td>\n",
       "      <td>2.401487e+03</td>\n",
       "      <td>2.937498e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>4.452310e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>3.901738e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>6.601509e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.700000e+01</td>\n",
       "      <td>8.681510e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>4.369710e+05</td>\n",
       "      <td>9.486740e+05</td>\n",
       "      <td>1.038613e+18</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           retweets     favorites            id\n",
       "count  1.852507e+06  1.852507e+06  1.852507e+06\n",
       "mean   7.488233e+01  1.814578e+02  6.145035e+17\n",
       "std    9.643075e+02  2.401487e+03  2.937498e+17\n",
       "min    0.000000e+00  0.000000e+00  4.452310e+08\n",
       "25%    1.000000e+00  0.000000e+00  3.901738e+17\n",
       "50%    3.000000e+00  3.000000e+00  6.601509e+17\n",
       "75%    1.000000e+01  1.700000e+01  8.681510e+17\n",
       "max    4.369710e+05  9.486740e+05  1.038613e+18"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1852507 entries, 0 to 8768\n",
      "Data columns (total 9 columns):\n",
      "username     object\n",
      "date         object\n",
      "retweets     int64\n",
      "favorites    int64\n",
      "text         object\n",
      "mentions     object\n",
      "hashtags     object\n",
      "id           int64\n",
      "permalink    object\n",
      "dtypes: int64(3), object(6)\n",
      "memory usage: 141.3+ MB\n"
     ]
    }
   ],
   "source": [
    "tweets.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thx @marklevinshow https:// twitter.com/foxnews/status /970503582595338240 … \n",
      "\n",
      "Looking forward to being with @marklevinshow on his new Fox News show! Much better than Oscars! https:// twitter.com/marklevinshow/ status/970471715930345473 … \n",
      "\n",
      "My daughter sees @TuckerCarlson report on TV saying California is becoming 3rd world and asks me why. I tell her CA has highest taxes in USA that make it hard for people to afford to live here. She responds “Don’t you get paid to fix it?” \n",
      "\n",
      "Congrats! Media hacks win award for Fakenews! https://www. washingtontimes.com/news/2018/mar/ 1/fbi-debunked-russia-trump-story-helped-new-york-ti/ … \n",
      "\n",
      "Must read by @NRO ...nothing more needs to be said unless you’re a Russian Bot... https://www. nationalreview.com/2018/02/schiff -memo-russia-investigation-harms-democrats-more-than-helps-them/ … \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for tweet in tweets['text'][100000:100005]:\n",
    "    print(tweet, '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DevinNunes \n",
      "\n",
      "DevinNunes \n",
      "\n",
      "DevinNunes \n",
      "\n",
      "DevinNunes \n",
      "\n",
      "DevinNunes \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for tweet in tweets['username'][100000:100005]:\n",
    "    print(tweet, '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert Tweet objects to strings\n",
    "tweets['text']  = tweets['text'].apply(lambda x: str(x)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get number of characters in each Tweet\n",
    "tweets['length'] = tweets['text'].apply( lambda x: len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>retweets</th>\n",
       "      <th>favorites</th>\n",
       "      <th>id</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.852507e+06</td>\n",
       "      <td>1.852507e+06</td>\n",
       "      <td>1.852507e+06</td>\n",
       "      <td>1.852507e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>7.488233e+01</td>\n",
       "      <td>1.814578e+02</td>\n",
       "      <td>6.145035e+17</td>\n",
       "      <td>1.483871e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>9.643075e+02</td>\n",
       "      <td>2.401487e+03</td>\n",
       "      <td>2.937498e+17</td>\n",
       "      <td>5.893152e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>4.452310e+08</td>\n",
       "      <td>1.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>3.901738e+17</td>\n",
       "      <td>1.190000e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>6.601509e+17</td>\n",
       "      <td>1.380000e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.700000e+01</td>\n",
       "      <td>8.681510e+17</td>\n",
       "      <td>1.640000e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>4.369710e+05</td>\n",
       "      <td>9.486740e+05</td>\n",
       "      <td>1.038613e+18</td>\n",
       "      <td>1.254000e+03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           retweets     favorites            id        length\n",
       "count  1.852507e+06  1.852507e+06  1.852507e+06  1.852507e+06\n",
       "mean   7.488233e+01  1.814578e+02  6.145035e+17  1.483871e+02\n",
       "std    9.643075e+02  2.401487e+03  2.937498e+17  5.893152e+01\n",
       "min    0.000000e+00  0.000000e+00  4.452310e+08  1.000000e+00\n",
       "25%    1.000000e+00  0.000000e+00  3.901738e+17  1.190000e+02\n",
       "50%    3.000000e+00  3.000000e+00  6.601509e+17  1.380000e+02\n",
       "75%    1.000000e+01  1.700000e+01  8.681510e+17  1.640000e+02\n",
       "max    4.369710e+05  9.486740e+05  1.038613e+18  1.254000e+03"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa4AAAEcCAYAAACf91DkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAH/xJREFUeJzt3XuUZFV96PFv94xhMDPxMTSS4TGIOD+NQQ2iAYNgVNQsRWKMKAoTyfUCmmhMiFdEQRTRCUqMChGUcEUwRIzIIy6N8QGKSoIPEsHrD1Tez6Z5DvIYpvv+sU8PNU1Xd9V0Pc6B72etXtV1dp2zf6fqVP1q77Nrn5GpqSkkSWqK0WEHIElSN0xckqRGMXFJkhrFxCVJahQTlySpUUxckqRGMXGpryLixIg4okfb2i4i1kbEour++RHx5l5su9reVyPiz3q1vS7q/WBE3BoRNw267iaJiCURMRUR23T4+DURcXK/4xp0XYLFww5AzRURVwFPAh4E1gM/Az4HfDozJwEy85AutvXmzPxGu8dk5jXA0oXE3FLfUcCOmbl/y/b/qBfb7jKObYFDgZWZecuMsjcCJ1V3FwGbAb+eLs/MXj0X/wJcmpkfbFO+BLgX2DYzr+tFnR3GdRFwfGaePqg651J9qfkrIIC7gB8BH8zMi4Ya2KOQLS4t1N6ZuQxYCawB3gX8U68riYhH6peslcDEzKQFkJmfz8ylVYL6I+CG6fu9SlrqTEQcTjm+jwLGKK/bycA+fajrkXqs98yIM2doU83WSoqI5wEXAc/MzEsj4rPAdZn53ojYAvgssDswCVwG7AmcCrwRuJ/ScvsAcCZwJfBm4H3AVcDqatljMvPBiDgf+AHwYsq34POBAzPztoh4IXB6Zm7oVpqOl9LTcC4wUtX5y8x8VrW90zPz5IgYBQ4H/jewOfA14G2ZeWdEbF/F8SbgaOCxwMcy85g2z9PjgE9Sks+vgc8AHwJeBJzHQy2pf83MN7XZxmz78xbgRZn52ur+tcC3M3N1df8WYI/M/HlE/C7wCeD3gJuAd2fm2RHxduC46vVYB3x1enst9czZ4oqIg4G/oXyg/wA4KDOvb1nvIOAw4InAZzPzr6v1FgN/T3nt7wA+Dvx9Zi6OiOOAd1QxPUhpeb5nru3NEtcaSoL5DeClwM+BN2XmZVX39dMy840tj/8M5UvEYTO2sxy4HnhtZp43R13bUhoDr6QcH6sz85Kq/EjgQGAL4GrgXZn5larsEGBf4P8BbwCOa9f6VWGLSz2Vmf8FXAe8YJbiQ6uyMUoX4+HAVGYeAFxDab0tzcxjW9bZE3g68LI2Va4G/hxYQfmA+0QHMX6Nkji+UNX3rFke9qbq7w+BHShdlMfPeMzulIT5YuDIiHh6myo/CTyu2s6eVcwHVgm/tSX1pvlin+GCantExA6U/d+juv87lOf25xHxW8B/UFrCW1T1nxIRO2bmJ4AvAUdXMbx2lnraiojXUxLM3pTX9CfAzK69P6IkzJ2BA6skDPCXVfy/CzwP+NPpFTLzUOBiyhejpdX9+bY3m9dQvhg9ETgHOKs6R/o54FURsbTaj82q+k+bZRsvAKaAf5ujHoBXA6cAjwe+CfxDS1kCz6ccB38H/Ev1RW7aHsAllNfnuHnqedQzcakfbqB8UMy0DvhtyvmcdZn53cycr8l/VGbek5n3tik/LTMvzcx7gCOAfacHbyzQGynf/n+VmWuBdwOvn9GN8/7MvDcz/xv4b+BhCbCK5XWUFs7dmXkV5YPpgIUGmJk/q+r4HcoH33nAXVWLcE9KYoPygXpp1fW4PjMvrh77moXGABxMOc9zeWauA94P7B4RT2p5zIcy867MvBL4DvDsavm+lOf4xsycAI6lM+22N5vvZ+a5VWxrKIlh58y8Gvgh5bmBknivzMzLZtnGcuDmDo7Vb2Xmf2TmekoC3BBXZn6h2s/JzDyN0oJ7Tsu6v8rMz1SvT7tjXRX7UtUPWwO3zbL8I5RzBF+PCCiDONbMs61ruyi/GngM5cNpoVZU22vd9mJKq2Ja6yjAXzP7wJEtKF1VM7e1dQ9ihPLB/UJgF+Ar1bI92ThxrQT2iIg7WtZbDNzeg/pXAidGxAktyx4EtgHurO63e55WsPHrN99rPa2T5/1h26y6l2+o6oXSEtufkmSmb2czATwpIkbmSV5t44qI/0UZ2LFdtWgpGx+nne67MHGpxyLiuZQP5QtnlmXm3ZTuwkMj4hnAtyPi4sz8JqUrZjbzfcvdtuX/7SituluBeyjnnqbjWkTpoux0uzdQPpRbt/0gcDPlQ7lTt1YxraSMupze1vVdbGMuF1AS1+9RBsYAvILSAps+T3It8PXM3LvNNhZyovta4J2Z+aWZBdU5rrncyMbP5bYzyntxAn7DNqtjYAXltQX4V+Dj1bH4UqDdCNgLKedDX8H83YUPExGrKN3FLwL+KzMnI+Ln1TanOdigCyYu9UR1HmUPygn20zPzp7M85pWUE+S/pAwnXl/9QUkIO2xC1ftHxOcogzc+QBngsD4iLgeWRMQrgK9Tzqdt1rLezcBeETE6PXR/hjOAd0XEV4FxHjon9mDVWuxIFcuZwDERsZrShfo3wEe73dE2LgCOAa7NzPGIuIBynmUdZfALwNnA0RHxOuAsygfmzsBtmXk5nT/3m81IRg8AJwLvjYhLMzMj4gmUASMPS2SzOBP464j4ehXv384o39RjotXzq+Pu36vtTwA/BsjMtRFxLuW1Pj8zZ/0dXWbeGhFHAydVA1G+SfkS83Jgt8w8fJ4YllIGv4wDoxFxELDjAvfrUc1zXFqo8yLibso37/dQRokd2OaxTwW+AayljD77x8w8vyr7MOUD8I6ImPkBNpfTKCMVbwKWAG8HyMw7gbdShixfT2mBtY6I+2J1OxERP55lu6dU2/4OZYTYfcDbuoir1duq+n9F+fb+z9X2e+GnlA/970D5kKW8FhvOH2bm7ZTBLQdSWjk3UFpjj6m28WngudVz/y9z1PULyqi+6b83ZOYZlEErZ0XEXZQBBnt1GPvxwPcpLdGLKa2Z+1vKPwasjojbI6LT818zfYkyeOd2yjm911TnoKadCuxE+25CAKoRo++hjCK9lTKY6CDKgI85ZeaPKQn+h5Tn/8nV/9pEDoeXVAsR8WpgTWZ23qRdeJ2rKElkq8z89XyPVz3YVShpKCJiGbAbpetta+C9wJcHWP8iSrft6SatZjFxSRqWUcoQ9VWUrtRzeWhASV9FxBMp3X2/ov1vBFVTdhVKkhrFwRmSpEaxq7B3NgOeSxk1tH6ex0qSikWUGXUuZuNRpW2ZuHrnucB3hx2EJDXUC5hl4oLZmLh650aA22+/h8nJ7s8bLl++lImJtT0PalCaHj80fx+aHj+4D3Uw6PhHR0d4whN+E6rP0E6YuHpnPcDk5NQmJa7pdZus6fFD8/eh6fGD+1AHQ4q/41MsDs6QJDWKiUuS1CgmLklSo5i4JEmNYuKSJDWKiUuS1CgmLklSo/g7rke5Zb+1OUs2681hMDa2rKvH33f/g9x91709qVvSo4eJ61FuyWaL2fvQeS/i2hfnHbcPdw+lZklNZlehJKlRTFySpEYxcUmSGsXEJUlqFBOXJKlRTFySpEYxcUmSGsXEJUlqFBOXJKlRTFySpEYxcUmSGsXEJUlqFBOXJKlRTFySpEYxcUmSGsXEJUlqFBOXJKlRTFySpEYxcUmSGmXxICqJiI8CrwG2B3bKzEur5auAU4HlwASwOjOvqFuZJKk+BtXiOhvYA7h6xvITgRMycxVwAnBSTcskSTUxkBZXZl4IEBEblkXElsDOwF7VojOA4yNiDBipS1lmjvfiOZAk9cYwz3FtC1yfmesBqtsbquV1KpMk1chAWlyPJsuXL93kdcfGlvUwkmao2z7XLZ5uNT1+cB/qoO7xDzNxXQtsHRGLMnN9RCwCVlTLR2pU1pWJibVMTk51/WSMjS1jfPzurtdbqGEfoMPY53aG9Rr0StPjB/ehDgYd/+joSNdf+IfWVZiZtwCXAPtVi/YDfpKZ43Uq6/2eS5IWYlDD4T8B/AmwFfCNiJjIzGcAhwCnRsSRwO3A6pbV6lQmSaqJkamp7ru1NKvtgSub2FW496HnDLxegPOO26dWXSp28Qyf+zB8Q+wqfDJwVUfr9DMgSZJ6zcQlSWoUE5ckqVFMXJKkRjFxSZIaxcQlSWoUE5ckqVFMXJKkRjFxSZIaxcQlSWoUE5ckqVFMXJKkRjFxSZIaxcQlSWoUE5ckqVFMXJKkRjFxSZIaxcQlSWoUE5ckqVFMXJKkRjFxSZIaxcQlSWoUE5ckqVFMXJKkRjFxSZIaxcQlSWoUE5ckqVFMXJKkRlk87AAAIuKVwNHACCWZHpWZZ0XEKuBUYDkwAazOzCuqdQZaJkmqh6G3uCJiBDgNOCAznw3sD5waEaPAicAJmbkKOAE4qWXVQZdJkmqgFi0uYBJ4XPX/44EbgS2AnYG9quVnAMdHxBilZTawsswc7+3uSpI21dATV2ZORcS+wDkRcQ+wDHgFsC1wfWaurx63PiJuqJaPDLis48S1fPnSTX4uxsaWbfK6TVW3fa5bPN1qevzgPtRB3eMfeuKKiMXAu4F9MvN7EfEHwBeAA4Yb2aaZmFjL5ORU1+uNjS1jfPzuPkQ0f73DNIx9bmdYr0GvND1+cB/qYNDxj46OdP2Ff+jnuIBnAysy83sA1e09wH3A1hGxCKC6XQFcW/0NskySVBN1SFzXAdtERABExNOBrYArgEuA/arH7Qf8JDPHM/OWQZb1Za8lSZtk6F2FmXlTRLwF+NeImKwWH5iZt0XEIZQRhkcCtwOrW1YddJkkqQaGnrgAMvPzwOdnWf5z4PfbrDPQMklSPdShq1CSpI6ZuCRJjWLikiQ1SseJKyLeHhFb9DMYSZLm002L6yXAVRHxbxHxuojYrF9BSZLUTseJKzNfBawEvgq8A7gpIk6OiD36FZwkSTN1NRw+Mycos6afEBHPpMzqfmBEXAt8Bvh4Zq7tfZiSJBVdD86IiBdHxP8FzgdupvxI9wDg9yitMUmS+qbjFldEfBR4PXAn8DngvZl5fUv5RZTZJiRJ6ptuugqXAK/OzItnK8zMdRGxS2/CkiRpdt0krg8Dv25dEBFPADbPzBtgw5RJkiT1TTfnuM4GtpmxbBvgy70LR5KkuXWTuCIzf9q6oLr/tN6GJElSe90krlsiYsfWBdX9id6GJElSe92c4zoF+FJEvAf4FfAU4Gjg5H4EJknSbLpJXGuAdcBHgW0pl7Q/Gfj7PsQlSdKsOk5cmTkJfKT6kyRpKLqa8ikiAngWsLR1eWae0sugJElqp5uZMw4HjgT+m41/zzVFOf8lSVLfddPiegfwvMz8n34FI0nSfLoZDn8v4MwYkqSh6qbFdQTwyYg4ijIr/AbVwA1Jkvqum8T12er2zS3LRijnuBb1KiBJkubSTeJ6ct+ikCSpQ938jutqgIgYBZ6UmTf2LSpJktroeHBGRDw+Iv4ZuA/4RbXsVRHxwX4FJ0nSTN2MKjyRcvXjlcAD1bIfAK/rdVCSJLXTzTmuFwMrqisdTwFk5nhEbLnQICJiCfAx4CWUFt0PMvOgiFgFnAosp8xCvzozr6jWGWiZJKkeumlx3Qls0bogIrYDenGu61hKwlqVmTtRht5DaeWdkJmrgBOAk1rWGXSZJKkGumlxncxDlzUZjYjdgA9RPuw3WUQsBVYD22TmdEvu5qoltzOwV/XQM4DjI2KMMgx/YGWZOb6QfZQk9U43La6/A86ktEQeQ5mf8Bzg4wuM4SmUbrn3RcQPI+L8iNidcumU6zNzPUB1e0O1fNBlkqSa6GY4/BTwD9Vfr2PYAfhJZr4zIn4fOA94bY/rGYjly5fO/6A2xsaW9TCSZqjbPtctnm41PX5wH+qg7vF3Mzv8i9qVZea3FhDD1cCDlK45MvM/I+JWytyIW0fEosxcHxGLgBWUC1iODLisYxMTa5mcnOr6SRgbW8b4+N1dr7dQwz5Ah7HP7QzrNeiVpscP7kMdDDr+0dGRrr/wd9NV+E8z/s4FvkY597XJMvNW4NtU55aqkX1bApcDlwD7VQ/dj9IqG8/MWwZZtpD9kyT1VjddhRtN+VS1SN4L9CI1HwKcEhHHAeuAAzLzjog4BDg1Io4EbqcM4mhdZ5BlffXAuvVDb/1IUhOMTE113601LSIWA9dl5la9C6mxtgeuXEhX4d6HntPzoOZz3nH7DKXe6brr1KViF8/wuQ/DN8SuwicDV3W0zgLr3AvwkiaSpIHpZnDGtZRLmEx7LLAEeGuvg5IkqZ1ufoC8/4z79wCXZ+ZdPYxHkqQ5dTM444J+BiJJUie66So8jY27CmeVmQMbiSdJevTpZnDGHcAfA4uA66p196mW/7LlT5KkvunmHNcq4BWZ+d3pBdWcgkdk5st6HpkkSbPopsW1K3DRjGX/CezWu3AkSZpbN4nrJ8CHImJzgOr2GMo0SZIkDUQ3ietNwB8Ad0bEzZQLS+4O/Fkf4pIkaVbdDIe/Cnh+RGxLmTX9xsy8pl+BSZI0m66mfIqI5cALgT0z85qIWBER2/QlMkmSZtFx4oqIPYEE3ggcUS1+KvCpPsQlSdKsumlx/QPwusx8OeXCj1BGFT6v51FJktRGN4lr+8z8ZvX/9AwaD9Ddb8EkSVqQbhLXzyJi5g+NXwL8tIfxSJI0p25aS4cC/xYRXwE2j4iTgL0p0z5JkjQQHbe4MvMi4JnAZcApwJXA8zLz4j7FJknSw3TU4oqIRcA3gZdl5rH9DUmSpPY6anFl5nrgyZ0+XpKkfunmHNf7gU9FxPsolzXZcG2uzJzsdWCSJM2mm8R1cnW7moeS1kj1/6JeBiVJUjvzJq6I2Cozb6J0FUqSNFSdtLguB34rM68GiIizMvNP+huWJEmz62SwxciM+y/sQxySJHWkk8Q1Nf9DJEkajE66ChdHxB/yUMtr5n0y81v9CE6SpJk6SVy3UGbKmDYx4/4UsEMvg5IkqZ15E1dmbj+AOACofiN2FLBTZl4aEbsCJwGbA1cB+2fmLdVjB1omSaqH2syEERE7A7sC11T3R4DTgb/IzFXAd4A1wyiTJNVHLRJXRGwGnAC8lYcGg+wC3JeZF1b3TwT2HVKZJKkm6nIRyA8Ap2fmlRExvWw74OrpO5l5a0SMRsQTB12Wmbd1uiPLly/tbs8f5cbGlg07hI3ULZ5uNT1+cB/qoO7xDz1xRcRuwHOBw4YdSy9MTKxlcrL7XxDU/UDpl/Hxu4cdwgZjY8tqFU+3mh4/uA91MOj4R0dHuv7CX4euwj2BpwFXRsRVwDbAvwM7AiunHxQRWwBTVevnmgGXSZJqYuiJKzPXZOaKzNy+GsF4HfAy4COUKy3vXj30EODM6v8fDbhMklQTQ09c7VSXSjmAcimVKygts8OGUSZJqo+hn+OaqfV3Y5n5fWCnNo8baJkkqR5q2+KSJGk2Ji5JUqOYuCRJjWLikiQ1iolLktQoJi5JUqOYuCRJjWLikiQ1iolLktQoJi5JUqOYuCRJjWLikiQ1iolLktQoJi5JUqOYuCRJjWLikiQ1iolLktQoJi5JUqOYuCRJjWLikiQ1iolLktQoJi5JUqOYuCRJjWLikiQ1iolLktQoJi5JUqOYuCRJjbJ42AFExHLgNOApwP3AL4CDM3M8InYFTgI2B64C9s/MW6r1BlomSaqHOrS4poBjMzMy85nAL4E1ETECnA78RWauAr4DrAEYdJkkqT6Gnrgy87bMPL9l0UXASmAX4L7MvLBafiKwb/X/oMskSTUx9MTVKiJGgbcA5wLbAVdPl2XmrcBoRDxxCGWSpJoY+jmuGT4JrAWOB1495Fg2yfLlS4cdQqOMjS0bdggbqVs83Wp6/OA+1EHd469N4oqIjwJPBfbOzMmIuIbSZThdvgUwlZm3Dbqsm/2YmFjL5ORUdztP/Q+Ufhkfv3vYIWwwNrasVvF0q+nxg/tQB4OOf3R0pOsv/LXoKoyIY4DnAH+cmfdXi38EbB4Ru1f3DwHOHFKZJKkmhp64IuIZwOHACuD7EXFJRHw5MyeBA4BPRcQVwJ7AYQCDLpMk1cfQuwoz8zJgpE3Z94Gd6lAmSaqHobe4JEnqholLktQoJi5JUqOYuCRJjWLikiQ1iolLktQoJi5JUqMM/XdcevR6YN36oUx1dd/9D3L3XfcOvF5JvWHi0tD8xmMWsfeh5wy83vOO24fmziQnya5CSVKjmLgkSY1i4pIkNYqJS5LUKCYuSVKjmLgkSY1i4pIkNYqJS5LUKCYuSVKjmLgkSY1i4pIkNYqJS5LUKCYuSVKjmLgkSY1i4pIkNYqJS5LUKCYuSVKjmLgkSY1i4pIkNcriYQdQNxGxCjgVWA5MAKsz84rhRiVJmmaL6+FOBE7IzFXACcBJQ45HktTCFleLiNgS2BnYq1p0BnB8RIxl5vg8qy8CGB0d2eT6t3zC5pu87kIMq95h1f3AuvWMjS2btazd8l65//4HWbv2vr5tfyHHX124D8M3yPhb6lrU6TojU1NT/YmmgSLiOcDnMvMZLct+BuyfmT+eZ/Xdge/2Mz5JegR7AXBhJw+0xdU7F1Oe+BuB9UOORZKaYhHw25TP0I6YuDZ2LbB1RCzKzPURsQhYUS2fz/10+G1BkrSRX3bzYAdntMjMW4BLgP2qRfsBP+ng/JYkaUA8xzVDRDyNMhz+CcDtlOHwOdyoJEnTTFySpEaxq1CS1CgmLklSo5i4JEmNYuKSJDWKv+MasiZM6hsRy4HTgKdQfq/2C+DgzByPiF0p8zluDlxFmWXklmq9tmXDEhHvA44CdsrMS5sUf0QsAT4GvAS4D/hBZh401zFUp+MrIl4JHA2MUL40H5WZZ9U5/oj4KPAaYHuqY2a+uOq0P7PFP9f7uVqn9u8JW1zD14RJfaeAYzMzMvOZlB8LromIEeB04C+q+L8DrAGYq2xYImJnYFfgmup+o+IHjqUkrFWZuRNwRLV8rmOoFsdX9XyeBhyQmc8G9gdOjYjReWIcdvxnA3sAV89YvqkxD3p/Zot/1vczNOc9YeIaopZJfc+oFp0B7BwRY8OL6uEy87bMPL9l0UXASmAX4L7MnJ4x5ERg3+r/ucoGLiI2o3xQvJXyxoVmxb8UWA0ckZlTAJl581zHUA2Pr0ngcdX/j6dMj7YFNY4/My/MzI1mztnU53wY+zNb/HO8n6Eh7wkT13BtC1yfmesBqtsbquW1VH1DfgtwLrAdLd/kMvNWYDQinjhP2TB8ADg9M69sWdak+J9C6Vp6X0T8MCLOj4jdmfsYqs3xVSXbfYFzIuJqSkvgz5oS/wybGnPt9mfG+xka8p4wcalbnwTWAscPO5BORcRuwHOBfxx2LAuwGNiBMgXZLsC7gLOApUONqkMRsRh4N7BPZq4E9ga+QEPifwRr3PsZTFzDtmFSX4AuJ/UduOpE71OB12XmJOVc0cqW8i2Aqcy8bZ6yQdsTeBpwZURcBWwD/Duw4xwx1il+KN90H6TqZsrM/wRuBe6l/TFUp+Pr2cCKzPweQHV7D+WcXRPibzVXXJtaNnCzvJ+hIe9pE9cQNWlS34g4BngO8MeZeX+1+EfA5lWXFcAhwJkdlA1UZq7JzBWZuX1mbg9cB7wM+MgcMdYmftjQLfNtqoucVqPTtgQup80xVLPj6zpgm4gIgIh4OrAVcEW7GGsW/wZzxbWpZYOLvmjzfoaGvKedq3DImjCpb0Q8A7iU8iF5b7X4ysx8dUQ8nzIyagkPDY+9uVqvbdkwVa2uV1ZDgxsTf0TsAJxCGUq9DnhPZn51rmOoTsdXRLwROIwySAPgfZl5dp3jj4hPAH9CSbK3AhOZ+YxNjXnQ+zNb/JRzjbO+n6t1av+eMHFJkhrFrkJJUqOYuCRJjWLikiQ1iolLktQoJi5JUqOYuCQNTES8PCJ+Mew41Gxe1kRaoIhY23L3sZRLRayv7h+cmZ8fQAw3AX/aMgHqzPKXA8dn5o79jqWlziWU3wltm5nXDapePfKZuKQFyswN8+1VP25+c2Z+Y2gBSY9wJi6pjyJiGTAObJmZd0XEB4F3Ao/PzHur+eIezMzDImJz4MOUC/89Bvgi8LfTU/JExKuB91Nm6f4ppTX3s4j4ImX6p69HxCRweGZ+oosY29Y73VKjzNjx18ADwP+ZbkVWl+o4FfgD4GfA+cAumfkSyvWaADIipijX4LqvWu/w2bYndcJzXFIfZebdwP8AL6gW7UGZt2/XlvsXVP9/jDIB8E5AAKsoUyRNX3n2H4EDKVM+nQacHRGLM/O1wC3ASzNzaTdJa756KyspVy1eAfwlcGJ1fTCAT1MS85OAgyiXKpm2R3UbVVxnd7A9aV4mLqn/LgD2rC5m+VTgU9X9ZcAzge9Vl/34c+CvMvOOzLyTcnXZ11fbOJhyjupHmbk+Mz8NbEaZKHWTdVAvwK+BD2fmusz8MuVCnDtW57BeRbm45b2Z+T9AJy2nWbe3kP3Qo4tdhVL/XQAcCfw+8EPgW8BxlCvP/rTqQtyO0k13WTWBOpRWyYPV/yuBfSPinS3b/Q1g6wXGtmKeegHGWy57ASXxLKVM3DpCaUFOu5ZyCZO5tNue1BETl9R/3wWeBbyCksQuoVwf7KU81E14IyVZPCUzJ2bZxrXAVzLzuDZ1bOps2fPVO5ebqnq3plyrCTa+mq8zeKsv7CqU+qzqfruMcon0C6rWxg+BN1MlrsxcRxkA8fGI2CIiRiJi24jYq9rMp4G3RcQuVdnSiHhVRDy2Kr+ZcoXkuYxExJLWvw7qnWu/7gPOA95fbe93gTe0lN8P3NlBXFJXTFzSYFxA6Vb7ccv93wRaf3f1DuAGSlK7E/ga1bmf6orBb6dcC+kOyrWU3sBDrZpjgGMi4o6I+Ms2MexA+V3Vhr+I2GauejtwMKW7cRw4mXKF5tYLEx4JfLGK61UdblOak9fjktQzEfFxYElmHjzsWPTI5TkuSZus6h6covyGazdgNQ9dml7qCxOXpIV4HOU3ZVtRBmt8MDO/NtyQ9EhnV6EkqVEcnCFJahQTlySpUUxckqRGMXFJkhrFxCVJahQTlySpUf4/ZWVze7+IWHoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Import seaborn for data visualization\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set()\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "# Create a histogram of all Tweet lengths\n",
    "ax = tweets.length.hist(bins=20)\n",
    "ax.set(xlabel='Tweet Length', ylabel='Frequency', \n",
    "       title='Distribution of Tweet Length by Char');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1357    X\n",
       "4116    E\n",
       "5811    z\n",
       "1504    #\n",
       "5551    Q\n",
       "Name: text, dtype: object"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Shortest Tweets are single chars\n",
    "tweets.sort_values(by='length',ascending=True).iloc[0:5]['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://twitter.com/PeteSessions/status/869267749779808256'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Tweets are accessible via permalinks\n",
    "tweets.sort_values(by='length',ascending=False).iloc[1]['permalink']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'President Trump, you made a big mistake. By trying to divide us up by race, religion, gender and nationality you actually brought us closer. pic.twitter.com/U7deCCTFx9'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Most Retweeted Tweet is by Senator Bernie Sanders\n",
    "tweets.sort_values(by='retweets',ascending=False).iloc[0]['text']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop Tweets that are too short and may not contain much meaning\n",
    "tweets.drop(tweets[tweets['length']<20].index, inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "from dateutil.parser import parse\n",
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parse date strings\n",
    "tweets.date = tweets.date.apply(lambda x : pd.to_datetime(parse(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets['year'] = tweets.date.apply(lambda x: x.year)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2013    118846\n",
       "2015    114652\n",
       "2017    112912\n",
       "2014    107984\n",
       "2016    100778\n",
       "2012     77780\n",
       "2011     70979\n",
       "2018     59708\n",
       "2010     30139\n",
       "2009     14775\n",
       "2008       439\n",
       "2007         2\n",
       "Name: year, dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Dataset includes Tweets from every year dating back to 2007\n",
    "tweets.year.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Natural Language Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/ubuntu/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add custom stop words\n",
    "more_stops =(['ive', 'isnt', 'theres', 'doesnt', 'wont', 'im', \\\n",
    "              'dont','youre', 'weve',  'theyre', 'around', 'thats',\\\n",
    "              'heres','hasnt','cant','sanderssenategovnewsroomnewsw'])             "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "import string\n",
    "import re\n",
    "\n",
    "def custom_tokenizer(text):\n",
    "    \n",
    "    # Remove URLs\n",
    "    text = re.sub(r'http[s]?\\:\\/\\/[\\S\\s]\\S+', '', text)\n",
    "\n",
    "    # Remove mentions (@username)\n",
    "    text = re.sub(r'\\@[a-zA-Z1-9]+', '', text)\n",
    "    \n",
    "    # Remove hashtags (#tag)\n",
    "    text = re.sub(r'\\#[a-zA-Z1-9]+', '', text)\n",
    "\n",
    "    # Remove punctuation\n",
    "    full_punc = '’‘“”.–…�★➠—' + string.punctuation\n",
    "    remove_punct = str.maketrans('', '', full_punc)\n",
    "    text = text.translate(remove_punct)\n",
    "\n",
    "\n",
    "    # Remove digits and convert to lower case\n",
    "    remove_digits = str.maketrans('', '', string.digits)\n",
    "    text = text.lower().translate(remove_digits)\n",
    "\n",
    "    # Tokenize\n",
    "    tokens = word_tokenize(text)\n",
    "\n",
    "    # Remove stop words\n",
    "    stop_words = stopwords.words('english')\n",
    "    stop_words = stop_words + more_stops\n",
    "    tokens_stop = [y for y in tokens if y not in stop_words]\n",
    "\n",
    "    # Stem\n",
    "    stemmer = nltk.stem.porter.PorterStemmer(mode='NLTK_EXTENSIONS')\n",
    "    tokens_stem = [stemmer.stem(y) for y in tokens_stop] \n",
    "\n",
    "    return tokens_stem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets['tokens'] = tweets.text.apply(lambda x :custom_tokenizer(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "On this day, we celebrate the ability of American women to participate freely in public life, but the fight for full equality hasn't ended. \n",
      "\n",
      "The Sunshine State needs a governor with real-world experience who can get the state’s economy back on track. Alex Sink is that leader. \n",
      "\n",
      "Kendrick Meek waged a fierce campaign on behalf of all Floridians. I congratulate him on this win and look forward to working with him. \n",
      "\n",
      "I served as a missionary in a country without voting rights—an experience that taught me to cherish the right to vote. #whyivote \n",
      "\n",
      "Voting is the right that safeguards all of our other rights. #whyivote \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Compare full Tweets to preprocessed Tweets\n",
    "for tweet in tweets['text'][len(tweets)-5:len(tweets)]:\n",
    "    print(tweet, '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['day', 'celebrate', 'ability', 'american', 'women', 'participate', 'freely', 'public', 'life', 'fight', 'full', 'equality', 'ended'] \n",
      "\n",
      "['sunshine', 'state', 'needs', 'governor', 'realworld', 'experience', 'get', 'states', 'economy', 'back', 'track', 'alex', 'sink', 'leader'] \n",
      "\n",
      "['kendrick', 'meek', 'waged', 'fierce', 'campaign', 'behalf', 'floridians', 'congratulate', 'win', 'look', 'forward', 'working'] \n",
      "\n",
      "['served', 'missionary', 'country', 'without', 'voting', 'rightsan', 'experience', 'taught', 'cherish', 'right', 'vote'] \n",
      "\n",
      "['voting', 'right', 'safeguards', 'rights'] \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for tweet in tweets['tokens'][len(tweets)-5:len(tweets)]:\n",
    "    print(tweet, '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>username</th>\n",
       "      <th>date</th>\n",
       "      <th>retweets</th>\n",
       "      <th>favorites</th>\n",
       "      <th>text</th>\n",
       "      <th>mentions</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>id</th>\n",
       "      <th>permalink</th>\n",
       "      <th>length</th>\n",
       "      <th>year</th>\n",
       "      <th>tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AnthonyBrownMD4</td>\n",
       "      <td>2018-09-07 17:31:08</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>Last night, @realDonaldTrump attacked Maryland...</td>\n",
       "      <td>@realDonaldTrump @BenJealous</td>\n",
       "      <td>#Dreamers</td>\n",
       "      <td>1038117477622992900</td>\n",
       "      <td>https://twitter.com/AnthonyBrownMD4/status/103...</td>\n",
       "      <td>299</td>\n",
       "      <td>2018</td>\n",
       "      <td>[last, night, attack, maryland, valu, go, immi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AnthonyBrownMD4</td>\n",
       "      <td>2018-08-19 18:29:15</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>The Maryland 4th Congressional District Office...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1031246733114662912</td>\n",
       "      <td>https://twitter.com/AnthonyBrownMD4/status/103...</td>\n",
       "      <td>301</td>\n",
       "      <td>2018</td>\n",
       "      <td>[maryland, th, congression, district, offic, s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AnthonyBrownMD4</td>\n",
       "      <td>2018-05-23 18:11:56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Congratulations @AmyMcGrathKY</td>\n",
       "      <td>@AmyMcGrathKY</td>\n",
       "      <td>NaN</td>\n",
       "      <td>999352242829438983</td>\n",
       "      <td>https://twitter.com/AnthonyBrownMD4/status/999...</td>\n",
       "      <td>29</td>\n",
       "      <td>2018</td>\n",
       "      <td>[congratul]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AnthonyBrownMD4</td>\n",
       "      <td>2018-04-14 12:29:37</td>\n",
       "      <td>3</td>\n",
       "      <td>11</td>\n",
       "      <td>Mission Accomplished? Not so easy Mr. Presiden...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>985132970746044416</td>\n",
       "      <td>https://twitter.com/AnthonyBrownMD4/status/985...</td>\n",
       "      <td>279</td>\n",
       "      <td>2018</td>\n",
       "      <td>[mission, accomplish, easi, mr, presid, missio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>AnthonyBrownMD4</td>\n",
       "      <td>2018-03-17 03:46:08</td>\n",
       "      <td>4</td>\n",
       "      <td>23</td>\n",
       "      <td>First 16th seed men’s basketball team to beat ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>#1</td>\n",
       "      <td>974854369169563648</td>\n",
       "      <td>https://twitter.com/AnthonyBrownMD4/status/974...</td>\n",
       "      <td>93</td>\n",
       "      <td>2018</td>\n",
       "      <td>[first, th, seed, men, basketbal, team, beat, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          username                date  retweets  favorites  \\\n",
       "0  AnthonyBrownMD4 2018-09-07 17:31:08         6         12   \n",
       "2  AnthonyBrownMD4 2018-08-19 18:29:15         1          3   \n",
       "3  AnthonyBrownMD4 2018-05-23 18:11:56         1          1   \n",
       "6  AnthonyBrownMD4 2018-04-14 12:29:37         3         11   \n",
       "8  AnthonyBrownMD4 2018-03-17 03:46:08         4         23   \n",
       "\n",
       "                                                text  \\\n",
       "0  Last night, @realDonaldTrump attacked Maryland...   \n",
       "2  The Maryland 4th Congressional District Office...   \n",
       "3                      Congratulations @AmyMcGrathKY   \n",
       "6  Mission Accomplished? Not so easy Mr. Presiden...   \n",
       "8  First 16th seed men’s basketball team to beat ...   \n",
       "\n",
       "                       mentions   hashtags                   id  \\\n",
       "0  @realDonaldTrump @BenJealous  #Dreamers  1038117477622992900   \n",
       "2                           NaN        NaN  1031246733114662912   \n",
       "3                 @AmyMcGrathKY        NaN   999352242829438983   \n",
       "6                           NaN        NaN   985132970746044416   \n",
       "8                           NaN         #1   974854369169563648   \n",
       "\n",
       "                                           permalink  length  year  \\\n",
       "0  https://twitter.com/AnthonyBrownMD4/status/103...     299  2018   \n",
       "2  https://twitter.com/AnthonyBrownMD4/status/103...     301  2018   \n",
       "3  https://twitter.com/AnthonyBrownMD4/status/999...      29  2018   \n",
       "6  https://twitter.com/AnthonyBrownMD4/status/985...     279  2018   \n",
       "8  https://twitter.com/AnthonyBrownMD4/status/974...      93  2018   \n",
       "\n",
       "                                              tokens  \n",
       "0  [last, night, attacked, maryland, values, go, immi...  \n",
       "2  [maryland, th, congressional, district, office, s...  \n",
       "3                                        [congratulations]  \n",
       "6  [mission, accomplished, easy, mr, president, missio...  \n",
       "8  [first, th, seed, mens, basketball, team, beat, ...  "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Join stems to create strings to be used with Count Vectorizer\n",
    "tweets['clean_text'] = tweets.tokens.apply(lambda x: ' '.join(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets.to_pickle('preprocessed_tweets.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}