1_Descriptive_Statistics_Centrality.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Descriptive Statistics\n",
    "\n",
    "Descriptive Statistics or Summary Statistics is used to describe a data set using a small number of statistical quantities such as:\n",
    "\n",
    "1. measure of location of centrality\n",
    "2. measure of variability or spread\n",
    "3. measures of distribution\n",
    "4. measures of distribution shape\n",
    "\n",
    "In this notebook,  statistical quantities are used to describe a 1-D dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-09T04:24:32.258757Z",
     "start_time": "2018-11-09T04:24:31.367585Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Standard Imports\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "#Display options\n",
    "pd.options.display.max_rows = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-09T04:27:22.676698Z",
     "start_time": "2018-11-09T04:27:22.643888Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>total_bill</th>\n",
       "      <th>tip</th>\n",
       "      <th>sex</th>\n",
       "      <th>smoker</th>\n",
       "      <th>day</th>\n",
       "      <th>time</th>\n",
       "      <th>size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16.99</td>\n",
       "      <td>1.01</td>\n",
       "      <td>Female</td>\n",
       "      <td>No</td>\n",
       "      <td>Sun</td>\n",
       "      <td>Dinner</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10.34</td>\n",
       "      <td>1.66</td>\n",
       "      <td>Male</td>\n",
       "      <td>No</td>\n",
       "      <td>Sun</td>\n",
       "      <td>Dinner</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>21.01</td>\n",
       "      <td>3.50</td>\n",
       "      <td>Male</td>\n",
       "      <td>No</td>\n",
       "      <td>Sun</td>\n",
       "      <td>Dinner</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23.68</td>\n",
       "      <td>3.31</td>\n",
       "      <td>Male</td>\n",
       "      <td>No</td>\n",
       "      <td>Sun</td>\n",
       "      <td>Dinner</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>24.59</td>\n",
       "      <td>3.61</td>\n",
       "      <td>Female</td>\n",
       "      <td>No</td>\n",
       "      <td>Sun</td>\n",
       "      <td>Dinner</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   total_bill   tip     sex smoker  day    time  size\n",
       "0       16.99  1.01  Female     No  Sun  Dinner     2\n",
       "1       10.34  1.66    Male     No  Sun  Dinner     3\n",
       "2       21.01  3.50    Male     No  Sun  Dinner     3\n",
       "3       23.68  3.31    Male     No  Sun  Dinner     2\n",
       "4       24.59  3.61  Female     No  Sun  Dinner     4"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Dataset : demo dataset\n",
    "tips = sns.load_dataset(\"tips\") #tips is a Pandas DataFrame \n",
    "tips.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-09T04:32:02.423360Z",
     "start_time": "2018-11-09T04:32:02.415087Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([16.99, 10.34, 21.01, 23.68, 24.59, 25.29,  8.77, 26.88, 15.04,\n",
       "       14.78])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Convert total_bill column to Numpy Array\n",
    "data = tips.total_bill.values\n",
    "data[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Measures of Centrality\n",
    "\n",
    "#### MEAN\n",
    "\n",
    "1. Arithmetic Mean: Average (typical middle value)\n",
    "2. Geometric Mean: n-th root of a set of values\n",
    "3. Harmonic Mean: mean of ratio quantities\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-09T06:39:26.574341Z",
     "start_time": "2018-11-09T06:39:26.565284Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Arithmetic Mean = 19.7859\n",
      "Geometric Mean = 17.9989\n",
      "Harmonic Mean = 16.3090\n"
     ]
    }
   ],
   "source": [
    "# Computing the different Pythagorean Means\n",
    "print('Arithmetic Mean = {:6.4f}'.format(np.mean(data)))\n",
    "print('Geometric Mean = {:6.4f}'.format(sp.stats.gmean(data)))\n",
    "print('Harmonic Mean = {:6.4f}'.format(sp.stats.hmean(data)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-09T06:40:50.968359Z",
     "start_time": "2018-11-09T06:40:50.957491Z"
    }
   },
   "source": [
    "Summary:\n",
    "- The mean (or average) is a commonly used descriptive statistic. However, some data are distributed in ways that make the mean a poor measure for the typical value. \n",
    "- For example, if data are bi-modal (with lots of low values and lots of high values) the mean will typically lie between these two clumps of data. \n",
    "- Likewise, this can occur if a data set has a number of outliers at one end of a range (such as sales contracts that are much higher than the rest of the sales contracts in a list).\n",
    "In this case, other measures can provide more robust estimates. Some of the more common such measures include the median, the mode, and trimmed mean."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-09T13:14:48.151375Z",
     "start_time": "2018-11-09T13:14:48.129470Z"
    }
   },
   "source": [
    "#### MEDIAN\n",
    "\n",
    "The median is conceptually an easy measure of centrality; half the data lie above and below the median. Thus, to compute the median you first sort the data and select the middle element. While conceptually simple, in practice computing the median is complicated by several issues:\n",
    "We need to sort the data, which is challenging for big data (and also implies the data can be sorted).\n",
    "For a data set with an even number of values there is no middle value.\n",
    "In the last case, for example, given the list of points [0, 1, 2, 3, 4, 5], we can use three different techniques to compute a median: \n",
    "\n",
    "- a. average the two middle values (median=2.5), \n",
    "- b. take the lower of the two middle values (median=2), or \n",
    "- c. take the higher of the two middle values (median=3).\n",
    "\n",
    "Often, a Python module by default employs the first technique (average the two middle values), although, as demonstrated below, exceptions to this do occur, such as in the built-in Python statistics module, which can return any of these three values.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-09T13:32:07.723647Z",
     "start_time": "2018-11-09T13:32:07.718143Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Median Value = 17.7950\n"
     ]
    }
   ],
   "source": [
    "print(\"Median Value = {:6.4f}\".format(np.median(data)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### MODE\n",
    "\n",
    "The mode is simply the most common value in a data set. Often, a mode makes the most sense when the data have been binned, \n",
    "in which case the data have been aggregated and it becomes simple to find the bin with the most values (we will see\n",
    "this in a later lesson on distributions). A mode can also make sense when data are categorical, or explicitly limited to a discrete set of values. \n",
    "When calculating the mode (or modal value), we also have the number of times this value occurred, as demonstrated below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-09T13:31:52.622620Z",
     "start_time": "2018-11-09T13:31:52.609137Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Modal Value = 13.4200, occured 3 times.\n"
     ]
    }
   ],
   "source": [
    "mode = sp.stats.mode(data)\n",
    "print(\" Modal Value = {0:6.4f}, occured {1} times.\".format(mode[0][0], mode[1][0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-09T13:17:02.876428Z",
     "start_time": "2018-11-09T13:17:02.869723Z"
    }
   },
   "source": [
    "#### TRIMMED MEAN\n",
    "One simple technique to compute a more robust, average value is to trim outlier points before calculating the mean value, producing \n",
    "a _trimmed mean_. The trimming process generally eliminates either a set number of points from the calculation, \n",
    "or any values that are more extreme than a certain range (such as three standard deviations from the untrimmed mean). \n",
    "In NumPy, the latter approach is used, and a lower and upper limit can be applied to eliminate values outside this range \n",
    "from being used in the computation of the [trimmed mean]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-09T13:34:40.459107Z",
     "start_time": "2018-11-09T13:34:40.443624Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Trimmed Mean = 18.5894 with bounds = 12.50 : 27.50\n"
     ]
    }
   ],
   "source": [
    "low = 12.5\n",
    "up = 27.5\n",
    "tm  = sp.stats.tmean(data, (low,up))\n",
    "print( \" Trimmed Mean = {0:6.4f} with bounds = {1:4.2f} : {2:4.2f}\".format(tm,low,up))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-09T13:36:12.861474Z",
     "start_time": "2018-11-09T13:36:12.855381Z"
    }
   },
   "source": [
    "##################################################"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Q. compute the mean and median for the total_bill column, separately for those rows corresponding to lunch and dinner. What do these values suggest about the typical lunch or dinner party?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-09T13:40:57.269931Z",
     "start_time": "2018-11-09T13:40:57.257906Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lunch = tips[tips['time'] == 'Lunch']\n",
    "dinner = tips[tips[\"time\"] == 'Dinner']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-09T13:42:21.026412Z",
     "start_time": "2018-11-09T13:42:21.017872Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "total_bill_lunch = lunch.total_bill.values\n",
    "total_bill_dinner = dinner.total_bill.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-09T13:46:44.990164Z",
     "start_time": "2018-11-09T13:46:44.974115Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MEDIAN\n",
      "Median Value for Lunch = 15.9650\n",
      "Median Value for Dinner = 18.3900\n",
      "MEAN\n",
      "Mean Value for Lunch = 17.1687\n",
      "Mean Value for Dinner = 20.7972\n"
     ]
    }
   ],
   "source": [
    "print(\"MEDIAN\")\n",
    "print(\"Median Value for Lunch = {:6.4f}\".format(np.median(total_bill_lunch)))\n",
    "print(\"Median Value for Dinner = {:6.4f}\".format(np.median(total_bill_dinner)))\n",
    "print(\"MEAN\")\n",
    "print(\"Mean Value for Lunch = {:6.4f}\".format(np.mean(total_bill_lunch)))\n",
    "print(\"Mean Value for Dinner = {:6.4f}\".format(np.mean(total_bill_dinner)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}