Data_Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing on commentary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "# text = data['Comment'].str.split(',')     #splitting the comment based on commas\n",
    "# data['Comment'] = text.str.slice(start=2) #removing player names and the ball_event\n",
    "# players = text.str.get(0).str.split(' to ') #getting the first element after splitting and splitting that based on 'to'\n",
    "# data['Bowler'] = players.str.get(0)       #Bowler name is the first element after splitting players\n",
    "# data['Batsman'] = players.str.get(1)      #Batsman name is the second element after splitting players\n",
    "# data[['Ball_Event','Comment']]            # printing two columns together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Ali to Anwar Ali, OUT, tossed up, driven, plucked low at short cover, and England have sealed the series! Or have they? A quick check upstairs to make sure it carried, but the fielders are in little doubt. A pretty futile faff, the umpire's soft signal was out, and sure enough, it is!\""
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"/Users/murali/code/gyandata/table154.csv\")\n",
    "df.loc[0, \"Comment\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    44636\n",
       "1     1240\n",
       "Name: y, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading the scraped data for commentary of matches from the last 3 years\n",
    "df1 = pd.read_csv(\"/Users/murali/code/gyandata/table154.csv\")\n",
    "df2 = pd.read_csv(\"/Users/murali/code/gyandata/table155.csv\")\n",
    "df3 = pd.read_csv(\"/Users/murali/code/gyandata/table156.csv\")\n",
    "df4 = pd.read_csv(\"/Users/murali/code/gyandata/table157.csv\")\n",
    "df5 = pd.read_csv(\"/Users/murali/code/gyandata/table158.csv\")\n",
    "data = pd.concat([df1,df2,df3,df4,df5])\n",
    "team_names = ['England','Pakistan','New Zealand','India','Australia','Bangladesh','West Indies','Afghanistan','Sri Lanka','South Africa']\n",
    "data = data[(data.Team_1.isin(team_names))&(data.Team_2.isin(team_names))]\n",
    "data[\"Comment\"] = data[\"Comment\"].apply(lambda x: x.split(\",\"))\n",
    "data[\"Bowler\"] = data[\"Comment\"].apply(lambda x: x[0].split(' to ')[0]) #space is included in split parameter for names like stokes, stoinis\n",
    "data['Batsman'] = data[\"Comment\"].apply(lambda x: x[0].split(' to ')[1])\n",
    "data[\"Comment\"] = data[\"Comment\"].apply(lambda x: x[2:])\n",
    "data['Comment'] = data['Comment'].apply(lambda x: word_tokenize(''.join(x))) #the comment is now split into its words\n",
    "#data[\"Comment\"] = data[\"Comment\"].apply(lambda x: ''.join(x).split(' '))   # alternate method\n",
    "\n",
    "data.reset_index(inplace=True, drop=True)\n",
    "data.drop(\"Unnamed: 0\", inplace=True, axis=1)\n",
    "\n",
    "t_f = {True:1,False:0}\n",
    "#data['y'] = [1 if x =='W' else 0 for x in data['Ball_Event']]\n",
    "data['y'] = data['Ball_Event'].apply(lambda x: t_f[x=='W'])\n",
    "data['y'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    44636\n",
       "1     1240\n",
       "Name: y, dtype: int64"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.corpus import stopwords\n",
    "stop_words = set(stopwords.words('english'))\n",
    "stop_words.remove('on')\n",
    "stop_words.remove('off')\n",
    "stop_words.remove('above')\n",
    "stop_words.remove('below')\n",
    "stop_words.remove('between')\n",
    "stop_words.remove('down')\n",
    "stop_words.remove('through')\n",
    "stop_words.remove('with')\n",
    "stop_words.remove('against')\n",
    "\n",
    "dummy = data.copy(deep=True) #deep copying ensures that changes made in dummy arent reflected in data\n",
    "def process(sentence):\n",
    "    for word in sentence:\n",
    "        if word in stop_words:\n",
    "            sentence.remove(word)\n",
    "    return sentence        \n",
    "dummy['Comment'] = list(map(lambda x : process(x), dummy['Comment']))\n",
    "dummy[['Comment','y']]\n",
    "dummy['y'].value_counts()\n",
    "#filter(lambda x: stop_words not in x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_words = []\n",
    "for x in dummy['Comment']:\n",
    "    if len(x)!=0:\n",
    "        for word in x:\n",
    "            if word not in ['.','!',\"'s\",\"n't\",'the','a']:\n",
    "                all_words.append(word.lower())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('outside', 11679), ('leg', 10604), ('ball', 10469), ('back', 10136), ('length', 9345), ('short', 7469), ('full', 6872), ('off', 5501), ('cover', 4784), ('midwicket', 4525), ('point', 4409), ('away', 4402), ('square', 4245), ('good', 3819), ('middle', 3815), ('stump', 3792), ('side', 3659), ('wide', 3578), ('gets', 3515), ('deep', 3457), ('one', 3230), ('towards', 2869), ('man', 2843), ('edge', 2662), ('on', 2617), ('third', 2532), ('straight', 2498), ('across', 2479), ('bat', 2390), ('line', 2353), ('delivery', 2352), ('foot', 2336), ('single', 2236), ('goes', 2170), ('forward', 2165), ('front', 2164), ('pushed', 2064), ('bowler', 1970), ('fine', 1878), ('stumps', 1878), ('driven', 1870), ('pads', 1848), ('covers', 1838), ('drive', 1832), ('time', 1829), ('long-on', 1738), ('mid-off', 1738), ('another', 1698), ('defended', 1690), ('backward', 1615)]\n",
      "<class 'list'>\n"
     ]
    }
   ],
   "source": [
    "\n",
    "all_words = nltk.FreqDist(all_words)\n",
    "print(all_words.most_common(50))\n",
    "print(type(all_words.most_common(50)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "imp_words = all_words.most_common(2000)\n",
    "word_features = []\n",
    "for i in thousand:\n",
    "    word_features.append(i[0])\n",
    "def find_features(document):\n",
    "    features = {}\n",
    "    for w in word_features:\n",
    "        # the dictionary features contains top 1000 most frequent words as keys and a bool value \n",
    "        # as the value which indicates whether that word was present in the review \n",
    "        features[w] = (w in document)  # boolean value \n",
    "    return features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# documents is a list of tuples with the comment and 1/0 values for all deliveries of all matches \n",
    "documents = []\n",
    "for comment,y in zip(dummy.Comment, dummy.y):\n",
    "    documents.append((comment,y))\n",
    "\n",
    "featuresets = [(find_features(comment),y) for (comment,y) in documents]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "96.23287671232876\n"
     ]
    }
   ],
   "source": [
    "training_set = featuresets[:45000]\n",
    "testing_set = featuresets[45000:]\n",
    "clf = nltk.NaiveBayesClassifier.train(training_set)\n",
    "print(nltk.classify.accuracy(clf,testing_set)*100)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Informative Features\n",
      "                  finger = True                1 : 0      =    100.5 : 1.0\n",
      "                   catch = True                1 : 0      =     70.8 : 1.0\n",
      "                  simple = True                1 : 0      =     45.7 : 1.0\n",
      "                    gone = True                1 : 0      =     43.7 : 1.0\n",
      "                 strikes = True                1 : 0      =     36.7 : 1.0\n",
      "                    what = True                1 : 0      =     35.9 : 1.0\n",
      "                   bails = True                1 : 0      =     32.7 : 1.0\n",
      "                decision = True                1 : 0      =     31.2 : 1.0\n",
      "                 replays = True                1 : 0      =     30.9 : 1.0\n",
      "                   taken = True                1 : 0      =     29.1 : 1.0\n"
     ]
    }
   ],
   "source": [
    "clf.show_most_informative_features(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}