020 Classification - Measuring Classifier Performance with Confusion Matrix, Error Metrics and ROC Curve.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step #1 Load the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean radius</th>\n",
       "      <th>mean texture</th>\n",
       "      <th>mean perimeter</th>\n",
       "      <th>mean area</th>\n",
       "      <th>mean smoothness</th>\n",
       "      <th>mean compactness</th>\n",
       "      <th>mean concavity</th>\n",
       "      <th>mean concave points</th>\n",
       "      <th>mean symmetry</th>\n",
       "      <th>mean fractal dimension</th>\n",
       "      <th>...</th>\n",
       "      <th>worst texture</th>\n",
       "      <th>worst perimeter</th>\n",
       "      <th>worst area</th>\n",
       "      <th>worst smoothness</th>\n",
       "      <th>worst compactness</th>\n",
       "      <th>worst concavity</th>\n",
       "      <th>worst concave points</th>\n",
       "      <th>worst symmetry</th>\n",
       "      <th>worst fractal dimension</th>\n",
       "      <th>cancer_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.3001</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>0.2419</td>\n",
       "      <td>0.07871</td>\n",
       "      <td>...</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.1622</td>\n",
       "      <td>0.6656</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.0869</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>0.1812</td>\n",
       "      <td>0.05667</td>\n",
       "      <td>...</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.1238</td>\n",
       "      <td>0.1866</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.1974</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>0.2069</td>\n",
       "      <td>0.05999</td>\n",
       "      <td>...</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.1444</td>\n",
       "      <td>0.4245</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.2414</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>0.2597</td>\n",
       "      <td>0.09744</td>\n",
       "      <td>...</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.8663</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.1980</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>0.1809</td>\n",
       "      <td>0.05883</td>\n",
       "      <td>...</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.1374</td>\n",
       "      <td>0.2050</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean radius  mean texture  mean perimeter  mean area  mean smoothness  \\\n",
       "0        17.99         10.38          122.80     1001.0          0.11840   \n",
       "1        20.57         17.77          132.90     1326.0          0.08474   \n",
       "2        19.69         21.25          130.00     1203.0          0.10960   \n",
       "3        11.42         20.38           77.58      386.1          0.14250   \n",
       "4        20.29         14.34          135.10     1297.0          0.10030   \n",
       "\n",
       "   mean compactness  mean concavity  mean concave points  mean symmetry  \\\n",
       "0           0.27760          0.3001              0.14710         0.2419   \n",
       "1           0.07864          0.0869              0.07017         0.1812   \n",
       "2           0.15990          0.1974              0.12790         0.2069   \n",
       "3           0.28390          0.2414              0.10520         0.2597   \n",
       "4           0.13280          0.1980              0.10430         0.1809   \n",
       "\n",
       "   mean fractal dimension  ...  worst texture  worst perimeter  worst area  \\\n",
       "0                 0.07871  ...          17.33           184.60      2019.0   \n",
       "1                 0.05667  ...          23.41           158.80      1956.0   \n",
       "2                 0.05999  ...          25.53           152.50      1709.0   \n",
       "3                 0.09744  ...          26.50            98.87       567.7   \n",
       "4                 0.05883  ...          16.67           152.20      1575.0   \n",
       "\n",
       "   worst smoothness  worst compactness  worst concavity  worst concave points  \\\n",
       "0            0.1622             0.6656           0.7119                0.2654   \n",
       "1            0.1238             0.1866           0.2416                0.1860   \n",
       "2            0.1444             0.4245           0.4504                0.2430   \n",
       "3            0.2098             0.8663           0.6869                0.2575   \n",
       "4            0.1374             0.2050           0.4000                0.1625   \n",
       "\n",
       "   worst symmetry  worst fractal dimension  cancer_type  \n",
       "0          0.4601                  0.11890            0  \n",
       "1          0.2750                  0.08902            0  \n",
       "2          0.3613                  0.08758            0  \n",
       "3          0.6638                  0.17300            0  \n",
       "4          0.2364                  0.07678            0  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6YAAACqCAYAAAC+qFINAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAR4UlEQVR4nO3dfbBkd1kn8O+TDCPBDZA3KczbRA26iYsJZikilhKUNYm1BiLZSpYFSlICa2DBFdZAWbrsiitFAS67CgaJCSkgJiiSlSxIQYyWBsiMJpDXIsRAEmJi3iBAVcgMz/7RZ/A6mzu3M/f2nHu7P5+qrtvn16dPP11P/XrqO+fXp6u7AwAAAGPZZ+wCAAAAWGyCKQAAAKMSTAEAABiVYAoAAMCoBFMAAABGJZgCAAAwKsEUAPZQVd1WVT89xX5dVT+wh6+xx88FgI1CMAUAAGBUgikAAACjEkwBYJWq6plVdVVVPVhVd1XV/66qzbvsdmpV3VpV91bVW6tqnyXPf1lV3VhVD1TVx6vqyGVe59SquqGqHqqqO6vqdTN9YwCwlwimALB6O5L8cpKDk5yY5KeS/NIu+7wgyQlJnpHktCQvS5KqOi3JG5OcnuSQJH+V5IPLvM57k7yiu/dP8sNJPrWm7wIARiKYAsAqdfe27v50d2/v7tuS/H6Sn9xlt7d09/3d/eUkv5PkrGH8lUn+R3ff2N3bk/xWkuOWOWv6SJJjquqJ3f1Ad//tTN4QAOxlgikArFJVPa2q/qyq/qGqvpZJuDx4l91uX3L/S0m+d7h/ZJL/OSwDfjDJ/UkqyaGP8lI/n+TUJF+qqiur6sS1fB8AMBbBFABW711JbkpydHc/MZOlubXLPocvuX9Ekq8M92/PZHnuk5fc9uvuv9n1Rbr76u4+Lcn3JPnTJJes8fsAgFEIpgCwevsn+VqSr1fVDyX5j4+yz+ur6oCqOjzJa5L80TD+7iRvqKpjk6SqnlRVZ+z65KraXFUvqqondfcjw+t9exZvBgD2NsEUAFbvdUn+fZKHkrwn/xQ6l/pIkm1Jrkny0UwuZJTu/nCStyS5eFgGfF2SU5Z5nRcnuW3Y75VJXrR2bwEAxlPdPXYNAAAALDBnTAEAABiVYAoAAMCoBFMAAABGJZgCAAAwKsEUAACAUW0au4ClDj744N6yZcvYZQAAADAD27Ztu7e7D9l1fF0F0y1btmTr1q1jlwEAAMAMVNWXHm3cUl4AAABGJZgCAAAwKsEUAACAUQmmAAAAjGpdXfzoxjvuy4++/n1jlwEArLFtb33J2CUAsI45YwoAAMCoBFMAAABGJZgCAAAwKsEUAACAUQmmAAAAjEowBQAAYFSCKQAAAKMSTAEAABiVYAoAAMCoBFMAAABGJZgCAAAwKsEUAACAUQmmAAAAjEowBQAAYFSCKQAAAKMSTAEAABjVTINpVZ1cVTdX1S1Vde4sXwsAAICNaWbBtKr2TfK7SU5JckySs6rqmFm9HgAAABvTLM+YPjPJLd19a3d/K8nFSU6b4esBAACwAU0VTKvqyKr66eH+flW1/xRPOzTJ7Uu27xjGAAAA4DtWDKZV9YtJPpTk94ehw5L86VoVUFUvr6qtVbV1+zcfWqvDAgAAsEFMc8b0nCTPTvK1JOnuLyT5nimed2eSw5dsHzaM/TPdfV53n9DdJ2x6wjQnYgEAAJgn0wTTh4fviCZJqmpTkp7ieVcnObqqjqqqzUnOTHLZnpUJAADAvNo0xT5XVtUbk+xXVc9L8ktJ/s9KT+ru7VX1qiQfT7JvkvO7+/pVVQsAAMDcmSaYnpvk7CSfT/KKJJcn+YNpDt7dlw/7AwAAwKNaMZh297er6sIkn8lkCe/N3T3NUl4AAABY0YrBtKp+Nsm7k3wxSSU5qqpe0d3/d9bFAQAAMP+mWcr7tiQndfctSVJV35/ko0kEUwAAAFZtmqvyPrQzlA5uTeIHRwEAAFgT05wx3VpVlye5JJPvmJ6R5OqqOj1JuvtPZlgfAAAAc26aYPr4JHcn+clh+x+T7Jfk32YSVAVTAAAA9tg0wfR13X3fzCsBAABgIU3zHdNPV9WlVXVqVdXMKwIAAGChTBNMn5bkvCQvTvKFqvqtqnrabMsCAABgUawYTHviE919VpJfTPLSJJ+tqiur6sSZVwgAAMBcW/E7plV1UJL/kMkZ07uTvDrJZUmOS3JpkqNmWB8AAABzbpqLH12V5KIkz+/uO5aMb62qd8+mLAAAABbFNN8x/bXu/u9LQ2lVnZEk3f2WmVUGAADAQpgmmJ77KGNvWOtCAAAAWEzLLuWtqlOSnJrk0Kp655KHnphk+6wLAwAAYDHs7jumX0myNcnPJdm2ZPyhJL88y6IAAABYHMsG0+6+Nsm1VfWB7n5kuf2q6o+7++dnUh0AAABzb8Wr8u4ulA6+b41qyb887KBsfetL1upwAAAAbADTXPxoJb0GxwAAAGBBrUUwBQAAgD22FsG01uAYAAAALKjdBtOq2req3r/CMX51DesBAABgwew2mHb3jiRHVtXm3ezz52teFQAAAAtjxavyJrk1yV9X1WVJvrFzsLvfPrOqAAAAWBjTBNMvDrd9kuw/23IAAABYNNP8jumbkqSqntDd35x9SQAAACySFa/KW1UnVtUNSW4atn+kqn5v5pUBAACwEKb5uZjfSfIzSe5Lku6+NslPzLAmAAAAFsg03zFNd99e9c9+rnTHLIr51l3X58v/7V/N4tAAAABz6Yhf//zYJazaNMH09qr6sSRdVY9L8pokN862LAAAABbFNEt5X5nknCSHJrkzyXHDNgAAAKzaNFflvTfJi/ZCLQAAACygaa7Ke2FVPXnJ9gFVdf5MqwIAAGBhTLOU9+nd/eDOje5+IMnxM6sIAACAhTJNMN2nqg7YuVFVB2bKq/kCAADASqYJmG9LclVVXZqkkrwwyZtnWhUAAAALY5qLH72vqrYlOWkYOr27b5htWQAAACyKaZfk3pTkgZ37V9UR3f3lmVUFAADAwlgxmFbVq5P8RpK7k+zIZDlvJ3n6bEsDAABgEUxzxvQ1SX6wu++bdTEAAAAsnmmuynt7kq/OuhAAAAAW0zRnTG9N8hdV9dEkD+8c7O63z6wqAAAAFsY0wfTLw23zcAMAAIA1M83PxbxpbxQCAADAYprmqryHJPkvSY5N8vid49393BnWBQAAwIKY5uJH78/kd0yPSvKmJLcluXqGNQEAALBApgmmB3X3e5M80t1XdvfLkkx1trSqzq+qe6rqulVVCQAAwNyaJpg+Mvy9q6p+tqqOT3LglMe/IMnJe1IYAAAAi2Gaq/L+ZlU9KcmvJPlfSZ6Y5LXTHLy7/7KqtuxxdQAAAMy9ac6YnpGkuvu67j4pyfOSvGC2ZQEAALAopgmmT+/uB3dudPf9SY5fqwKq6uVVtbWqtt7/jR1rdVgAAAA2iGmC6T5VdcDOjao6MNMtAZ5Kd5/X3Sd09wkHfve+a3VYAAAANohpAubbklxVVZcO22ckefPsSgIAAGCRrHjGtLvfl+T0JHcPt9O7+6JpDl5VH0xyVZIfrKo7qurs1RQLAADA/JlqSW5335Dkhsd68O4+6zFXBAAAwEKZ5jumAAAAMDOCKQAAAKMSTAEAABiVYAoAAMCoBFMAAABGJZgCAAAwKsEUAACAUQmmAAAAjEowBQAAYFSCKQAAAKMSTAEAABiVYAoAAMCoBFMAAABGJZgCAAAwKsEUAACAUQmmAAAAjGrT2AUstfmpx+aIX986dhkAAADsRc6YAgAAMCrBFAAAgFEJpgAAAIxKMAUAAGBUgikAAACjqu4eu4bvqKqHktw8dh3MxMFJ7h27CGZCb+eX3s4vvZ1fejvf9Hd+LVJvj+zuQ3YdXFc/F5Pk5u4+YewiWHtVtVVv55Pezi+9nV96O7/0dr7p7/zSW0t5AQAAGJlgCgAAwKjWWzA9b+wCmBm9nV96O7/0dn7p7fzS2/mmv/Nr4Xu7ri5+BAAAwOJZb2dMAQAAWDDrIphW1clVdXNV3VJV545dD6tXVbdV1eer6pqq2jqMHVhVn6iqLwx/Dxi7TlZWVedX1T1Vdd2SsUftZU28c5jLn6uqZ4xXOStZprf/taruHObuNVV16pLH3jD09uaq+plxqmYaVXV4VV1RVTdU1fVV9Zph3Nzd4HbTW3N3g6uqx1fVZ6vq2qG3bxrGj6qqzww9/KOq2jyMf9ewfcvw+JZR3wDL2k1vL6iqv18yb48bxhfyM3n0YFpV+yb53SSnJDkmyVlVdcy4VbFGTuru45Zc+vrcJJ/s7qOTfHLYZv27IMnJu4wt18tTkhw93F6e5F17qUb2zAX5/3ubJO8Y5u5x3X15kgyfy2cmOXZ4zu8Nn9+sT9uT/Ep3H5PkWUnOGXpo7m58y/U2MXc3uoeTPLe7fyTJcUlOrqpnJXlLJr39gSQPJDl72P/sJA8M4+8Y9mN9Wq63SfL6JfP2mmFsIT+TRw+mSZ6Z5JbuvrW7v5Xk4iSnjVwTs3FakguH+xcmef54pTCt7v7LJPfvMrxcL09L8r6e+HSSJ1fVU/dKoTxmy/R2Oaclubi7H+7uv09ySyaf36xD3X1Xd//tcP+hJDcmOTTm7oa3m94ux9zdIIb59/Vh83HDrZM8N8mHhvFd5+3O+fyhJD9VVbV3quWx2E1vl7OQn8nrIZgemuT2Jdt3ZPcfsGwMneTPq2pbVb18GHtKd9813P+HJE8ZpzTWwHK9NJ/nw6uGpUPnL1lyr7cb1LC87/gkn4m5O1d26W1i7m54VbVvVV2T5J4kn0jyxSQPdvf2YZel/ftOb4fHv5rkoL1aMFPbtbfdvXPevnmYt++oqu8axhZy3q6HYMp8+vHufkYmSxHOqaqfWPpgTy4H7ZLQc0Av5867knx/JkuN7krytlGrYVWq6l8k+eMkr+3ury19zNzd2B6lt+buHOjuHd19XJLDMjmz/UPjVsRa2bW3VfXDSd6QSY//dZIDk/zqeBWObz0E0zuTHL5k+7BhjA2su+8c/t6T5MOZfLjevXMZwvD3nvEqZJWW66X5vMF1993DP57fTvKe/NOSP73dYKrqcZkEl/d3958Mw+buHHi03pq786W7H0xyRZITM1nGuWl4aGn/vtPb4fEnJblv71bKY7WktycPS/O7ux9O8odZ8Hm7HoLp1UmOHq44tjmTL+hfNnJNrEJVfXdV7b/zfpJ/k+S6TPr60mG3lyb5yDgVsgaW6+VlSV4yXE3uWUm+umTZIBvALt9heUEmczeZ9PbM4SqQR2VyQYbP7u36mM7wPbP3Jrmxu9++5CFzd4Nbrrfm7sZXVYdU1ZOH+/sleV4m3yG+IskLh912nbc75/MLk3xqWAnBOrNMb29a8h+Flcl3h5fO24X7TN608i6z1d3bq+pVST6eZN8k53f39SOXxeo8JcmHh+/fb0ryge7+WFVdneSSqjo7yZeS/LsRa2RKVfXBJM9JcnBV3ZHkN5L8dh69l5cnOTWTi2t8M8kv7PWCmdoyvX3OcLn6TnJbklckSXdfX1WXJLkhk6uCntPdO0Yom+k8O8mLk3x++E5Tkrwx5u48WK63Z5m7G95Tk1w4XDV5nySXdPefVdUNSS6uqt9M8neZ/MdEhr8XVdUtmVzI7swximYqy/X2U1V1SJJKck2SVw77L+RncvmPFQAAAMa0HpbyAgAAsMAEUwAAAEYlmAIAADAqwRQAAIBRCaYAAACMSjAFgA2gql5bVU8Yuw4AmAU/FwMAG0BV3ZbkhO6+d+xaAGCtOWMKAGukql5SVZ+rqmur6qKq2jL8gPrnquqTVXXEsN8FVfXCJc/7+vD3OVX1F1X1oaq6qareXxP/Kcn3Jrmiqq4Y590BwOxsGrsAAJgHVXVskl9L8mPdfW9VHZjkwiQXdveFVfWyJO9M8vwVDnV8kmOTfCXJXyd5dne/s6r+c5KTnDEFYB45YwoAa+O5SS7dGRy7+/4kJyb5wPD4RUl+fIrjfLa77+jubye5JsmWtS8VANYXwRQA9r7tGf4Nrqp9kmxe8tjDS+7viNVNACwAwRQA1sankpxRVQclybCU92+SnDk8/qIkfzXcvy3Jjw73fy7J46Y4/kNJ9l+rYgFgPfG/sACwBrr7+qp6c5Irq2pHkr9L8uokf1hVr0/yj0l+Ydj9PUk+UlXXJvlYkm9M8RLnJflYVX2lu09a+3cAAOPxczEAAACMylJeAAAARiWYAgAAMCrBFAAAgFEJpgAAAIxKMAUAAGBUgikAAACjEkwBAAAYlWAKAADAqP4f0PgkiuHn8GwAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1152x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# A tutorial for this file is available at www.relataly.com\n",
    "\n",
    "import pandas as pd \n",
    "import matplotlib.pyplot as plt \n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix, classification_report, roc_auc_score, RocCurveDisplay\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn import datasets\n",
    "import seaborn as sns\n",
    "sns.set_style('white', { 'axes.spines.right': False, 'axes.spines.top': False})\n",
    "\n",
    "df = datasets.load_breast_cancer(as_frame=True)\n",
    "\n",
    "df_dia = df.data\n",
    "df_dia['cancer_type'] = df.target\n",
    "\n",
    "plt.figure(figsize=(16,2))\n",
    "plt.title(f'labels: 0=maligant, 1=benign')\n",
    "fig = sns.countplot(y=\"cancer_type\", data=df_dia)\n",
    "\n",
    "df_dia.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step #2 Data Preparation and Model Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separate labels from training data\n",
    "# X = df_dia[['mean symmetry', 'mean area']] #Training data\n",
    "X = df_dia.drop('cancer_type', axis=1) #Training data\n",
    "y = df_dia['cancer_type'] #Prediction label\n",
    "\n",
    "# Split the data into x_train and y_train data sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.5, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the Random Forest Classifier\n",
    "dfrst = RandomForestClassifier(n_estimators=3, max_depth=4, min_samples_split=6, class_weight='balanced')\n",
    "ranfor = dfrst.fit(X_train, y_train)\n",
    "y_pred = ranfor.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step #3 Confusion Matrix and Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaMAAAGBCAYAAADL8d7JAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAhb0lEQVR4nO3debwddX3/8dc7IJvKEtCAgBIUtEBdQKlKRZSquLTBigoupUibakWraAXUimhRahc3WmlkEZeyuFRx+aFIVbQVBFyQVSO4hB3CEgEhJJ/fH2cCh5jce3O3uTN5PXnMI+d8Z+7M98Q87tvPd75nvqkqJElq06y2OyBJkmEkSWqdYSRJap1hJElqnWEkSWqdYSRJap1hpBkryYZJvpzktiSfncB5XpnkG5PZt7YkeUaSK9ruhzTZ4veMNFFJXgEcCjwOWAL8GDi6qr43wfO+GngD8PSqunei/ZzpkhSwQ1UtbLsv0nSzMtKEJDkU+BDwPmAO8EjgP4B5k3D6RwE/WxuCaCySrNt2H6SpYhhp3JJsArwHeH1VfaGq7qiqpVX15ar6++aY9ZN8KMk1zfahJOs3+/ZKsijJW5LckOTaJAc1+44C3gW8PMlvkxyc5N1JPj10/e2S1Ipf0kn+MsmVSZYkuSrJK4favzf0c09Pcn4z/Hd+kqcP7ft2kvcm+d/mPN9IssVqPv+K/r9tqP/7JnlBkp8lWZzk7UPH757k+0lubY49Nsl6zb5zmsN+0nzelw+d/7Ak1wEnrWhrfubRzTV2bd4/IsmNSfaayP+uUhsMI03E04ANgP8e4Zh3AE8Fngg8AdgdeOfQ/i2BTYCtgYOBf0+yWVUdyaDaOq2qHlJVJ4zUkSQPBj4CPL+qHgo8ncFw4crHzQa+2hy7OfBvwFeTbD502CuAg4CHA+sBbx3h0lsy+DvYmkF4fhx4FbAb8AzgH5LMbY5dBrwZ2ILB393ewN8CVNWezTFPaD7vaUPnn82gSpw/fOGq+gVwGPDpJBsBJwEnV9W3R+ivNCMZRpqIzYGbRhlGeyXwnqq6oapuBI4CXj20f2mzf2lVfQ34LfDYcfZnObBLkg2r6tqqumQVx7wQ+HlVfaqq7q2qU4DLgT8dOuakqvpZVd0FnM4gSFdnKYP7Y0uBUxkEzYeraklz/UsZhDBVdWFVndtc95fAfwLPHMNnOrKq7m768wBV9XFgIXAesBWD8Jc6xzDSRNwMbDHKvYxHAL8aev+rpu2+c6wUZncCD1nTjlTVHcDLgdcC1yb5apLHjaE/K/q09dD769agPzdX1bLm9YqwuH5o/10rfj7Jjkm+kuS6JLczqPxWOQQ45Maq+t0ox3wc2AX4aFXdPcqx0oxkGGkivg/cDew7wjHXMBhiWuGRTdt43AFsNPR+y+GdVfX1qnoOgwrhcga/pEfrz4o+XT3OPq2JjzHo1w5VtTHwdiCj/MyI012TPITBBJITgHc3w5BS5xhGGrequo3BfZJ/b27cb5TkQUmen+QDzWGnAO9M8rBmIsC7gE+v7pyj+DGwZ5JHNpMnjlixI8mcJPOae0d3MxjuW76Kc3wN2DHJK5Ksm+TlwE7AV8bZpzXxUOB24LdN1fa6lfZfD2y/huf8MHBBVf0Vg3thx024l1ILDCNNSFX9K4PvGL0TuBH4DXAI8MXmkH8ELgAuAn4K/LBpG8+1zgJOa851IQ8MkFlNP64BFjO4F7PyL3uq6mbgRcBbGAwzvg14UVXdNJ4+raG3MpgcsYRB1XbaSvvfDZzczLZ72WgnSzIP2If7P+ehwK4rZhFKXeKXXiVJrbMykiS1zjCSJLXOMJIktc4wkiS1zjCSJLXOMFIrkixL8uMkFyf5bPNstfGe6xNJ9mteH59kpxGO3Wv4wahrcI1fruqBqatrX+mY367htd6dZKTn4Um9YxipLXdV1ROrahfgHgaP8bnPeJdLqKq/qqpLRzhkLwYPUZU0gxhGmgm+CzymqVq+m+QM4NIk6yT552aZh4uS/A1ABo5NckWSbzJ4ujbNvm8neXLzep8kP0zykyRnJ9mOQei9uanKntE8GeLzzTXOT7JH87ObN8tHXJLkeEZ/bA9JvpjkwuZn5q+074NN+9lJHta0PTrJmc3PfHc1z9KT1gou1qVWNRXQ84Ezm6ZdgV2q6qrmF/ptVfWUDNZA+t8Mlg9/EoMne+/EYEG/S4ETVzrvwxg85WDP5lyzq2pxkuOA31bVvzTH/Rfwwar6XpJHAl8H/gA4EvheVb0nyQsZLG8xmtc019gQOD/J55snPjyYwSN73pzkXc25DwEWAK+tqp8n+SMGixI+exx/jVLnGUZqy4ZJfty8/i6DB30+HfhBVV3VtD8XePyK+0EM1j3aAdgTOKV5WvY1Sf5nFed/KnDOinNV1eLV9ONPgJ2S+wqfjZuHj+4J/Hnzs19NcssYPtMbk7y4eb1t09ebGTwjb8Wjfz4NfKG5xtOBzw5de/0xXEPqJcNIbbmrqp443ND8Ur5juAl4Q1V9faXjXjCJ/ZgFPHXlZRqGAmJMMlhd9U+Ap1XVnUm+zWDRvVWp5rq3rvx3IK2tvGekmezrwOuSPAjuWw/owcA5DJYjXyfJVsCzVvGz5zJ4wvfc5mdXLK2whMHTs1f4BvCGFW+SPLF5eQ6Dh5qS5PnAZqP0dRPgliaIHsegMlthFrCiunsFg+G/24Grkry0uUaSPGGUa0i9ZRhpJjuewf2gHya5mMHKqOsyWOb8582+TzJYV+kBmlVl5zMYEvsJ9w+TfRl48YoJDMAbgSc3EyQu5f5ZfUcxCLNLGAzX/XqUvp4JrJvkMuAYBmG4wh3A7s1neDbwnqb9lcDBTf8uAeaN4e9E6iWf2i1Jap2VkSSpdYaRJKl1hpEkqXUzdmr3Qaf+1JtZmjYf2+8P2+6C1jIbrDv6Uz3GasMnHTLh35d3/ejYSevPeMzYMJIkjVG6P8hlGElS163hl7RnIsNIkrquB5VR9z+BJKnzrIwkqescppMkta4Hw3SGkSR1XQ8qo+7HqSSp86yMJKnrHKaTJLWuB8N0hpEkdZ2VkSSpdT2ojLofp5KkzrMykqSuc5hOktS6HgzTGUaS1HVWRpKk1vUgjLr/CSRJnWdlJEldN8t7RpKktvVgmM4wkqSu68Fsuu7HqSRpyiU5MckNSS5eqf0NSS5PckmSDwy1H5FkYZIrkjxvtPNbGUlS103PMN0ngGOBT9532eRZwDzgCVV1d5KHN+07AfsDOwOPAL6ZZMeqWra6k1sZSVLXJRPfRlFV5wCLV2p+HXBMVd3dHHND0z4POLWq7q6qq4CFwO4jnd8wkqSuy6wJb0nmJ7lgaJs/hivvCDwjyXlJvpPkKU371sBvho5b1LStlsN0ktR1kzCBoaoWAAvW8MfWBWYDTwWeApyeZPvxXN/KSJI0XouAL9TAD4DlwBbA1cC2Q8dt07StlmEkSV03CcN04/RF4FkASXYE1gNuAs4A9k+yfpK5wA7AD0Y6kcN0ktR10/A9oySnAHsBWyRZBBwJnAic2Ez3vgc4sKoKuCTJ6cClwL3A60eaSQeGkSR13zRM7a6qA1az61WrOf5o4Oixnt8wkqSu8wkMkiRNnJWRJHWdD0qVJLXOMJIktc57RpIkTZyVkSR1ncN0kqTW9WCYzjCSpK6zMpIkta4HlVH341SS1HlWRpLUcelBZWQYSVLHGUaSpPZ1P4u8ZyRJap+VkSR1nMN0kqTWGUaSpNYZRpKk1vUhjJzAIElqnZWRJHVd9wsjw0iSuq4Pw3SGkSR1nGEkSWpdH8LICQySpNZZGUlSx/WhMjKMJKnrup9FhpEkdV0fKiPvGUmSWmdlJEkdZ2UkSWpdkglvY7jGiUluSHLxKva9JUkl2aJ5nyQfSbIwyUVJdh3t/IaRJHVdJmEb3SeAfX7v0sm2wHOBXw81Px/YodnmAx8b7eSGkSR13HRURlV1DrB4Fbs+CLwNqKG2ecAna+BcYNMkW410fsNIkkSS+UkuGNrmj+Fn5gFXV9VPVtq1NfCbofeLmrbVcgKDJHXcZExgqKoFwII1uOZGwNsZDNFNmGEkSR3X0my6RwNzgZ80198G+GGS3YGrgW2Hjt2maVstw0iSOq6NMKqqnwIPH+rDL4EnV9VNSc4ADklyKvBHwG1Vde1I5/OekSR13TTMpktyCvB94LFJFiU5eITDvwZcCSwEPg787WjntzKSJI2qqg4YZf92Q68LeP2anN8wkqSO68MTGAwjSeo4w0iS1Lo+hJETGCRJrbMykqSu635hZBhJUtf1YZjOMJKkjjOM1Lrn7Lg5e24/mwS+84vFnPWzm5m3y8N55vazWXL3vQB8/qLruejaJS33VH1z++23c9S73snChT8jCUe993084YlPartbayXDSK3aepP12XP72bz3rIXcu7w49Jlz+ck1g9D5xhU3ceYVN7XcQ/XZB95/NHv88TP41w99hKX33MNdv/td211ShzmbrsO22nh9rlx8J/csK5YXXHHjHey2zcZtd0trgSVLlnDhhefz4pfsB8CD1luPjTf2315bpmM9o6k2ZZVRkscxWGBpxRoWVwNnVNVlU3XNtc3Vt93NS/5wSx683josXbacx2/1UH65+E5+e88y9t5xc54+d1N+ufguTv3Rtdy5dHnb3VWPXL1oEZttNpt3veMIrrjicnbaeWfedvg72Gijjdru2tqp/SyZsCmpjJIcBpzK4K/oB80W4JQkh4/wc/ct7nTF2Z+biq71yrW3383XLr+Rt+41l0OfOZdf33IXywu+9fPFvO0rV3DkmQu59a572f9JIy6wKK2xZcvu5fLLLuWl+x/A6Z//IhtuuCEnHj/mpXA0yayMVu9gYOeqWjrcmOTfgEuAY1b1Q8OLOx106k9rVcfogb575S1898pbAHjJ4+ew+M6l3N5MXAD4zpWLedMztmupd+qrOXO2ZM6cLXn8458AwHOeu49h1KKZECYTNVX3jJYDj1hF+1bNPk2Sh66/DgCzN3oQu22zMef+6lY22eD+/4+x29Ybc/Vt3ljW5NriYQ9jzpZb8surrgTgvHO/z/aPfnTLvVKXTVVl9Cbg7CQ/5/510B8JPAY4ZIquuVY65I8fxYPXW4dly4tPXXgNdy1dzqt2ewSP3HRDiuKmO5Zy8vkjLrAojcvhb/8HjjjsrSxdupRtttmW9/zj+9vu0lqrB4XR1IRRVZ2ZZEdgdx44geH8qlo2FddcW73/7Ct/r+3j5y5qoSda2zzuD/6AU07/QtvdEP0Yppuy2XRVtRw4d6rOL0ka6EEW+T0jSVL7fAKDJHWcw3SSpNb1IIsMI0nqulmzup9GhpEkdVwfKiMnMEiSWmdlJEkd5wQGSVLrepBFhpEkdZ2VkSSpdX0IIycwSJJaZ2UkSR3Xg8LIMJKkrnOYTpLUumTi2+jXyIlJbkhy8VDbPye5PMlFSf47yaZD+45IsjDJFUmeN9r5DSNJ0lh8AthnpbazgF2q6vHAz4AjAJLsBOwP7Nz8zH8kWWekkxtGktRxSSa8jaaqzgEWr9T2jaq6t3l7LrBN83oecGpV3V1VVwELGSy2ulqGkSR13GQM0yWZn+SCoW3+GnbjNcD/a15vDfxmaN8i7l/1e5WcwCBJHTcZExiqagGwYJzXfwdwL/CZ8V7fMJKkjmtzMl2SvwReBOxdVdU0Xw1sO3TYNk3bajlMJ0kalyT7AG8D/qyq7hzadQawf5L1k8wFdgB+MNK5rIwkqeOm43tGSU4B9gK2SLIIOJLB7Ln1gbOaPpxbVa+tqkuSnA5cymD47vVVtWyk8xtGktRx0zFMV1UHrKL5hBGOPxo4eqznN4wkqeP68AQGw0iSOq4HWeQEBklS+6yMJKnjHKaTJLWuB1lkGElS1/WhMvKekSSpdVZGktRxfaiMDCNJ6rgeZJFhJEldZ2UkSWpdD7LICQySpPZZGUlSxzlMJ0lqXQ+yyDCSpK6b1YM0MowkqeN6kEVOYJAktc/KSJI6zgkMkqTWzep+FhlGktR1faiMvGckSWqdlZEkdVwPCiPDSJK6LnQ/jQwjSeo4JzBIklrnBAZJkiaBlZEkdVwPCqPVh1GSJUCteNv8Wc3rqqqNp7hvkqQx6PWDUqvqodPZEUnS+PQgi8Z2zyjJHyc5qHm9RZK5U9stSdLaZNQwSnIkcBhwRNO0HvDpqeyUJGnskkx4G8M1TkxyQ5KLh9pmJzkryc+bPzdr2pPkI0kWJrkoya6jnX8sldGLgT8D7gCoqmsAh/AkaYZIJr6NwSeAfVZqOxw4u6p2AM5u3gM8H9ih2eYDHxvt5GMJo3uqqmgmMyR58Ji6LUmaFrOSCW+jqapzgMUrNc8DTm5enwzsO9T+yRo4F9g0yVYjfoYxfM7Tk/xnc7K/Br4JfHwMPydJmgaZjC2Zn+SCoW3+GC49p6qubV5fB8xpXm8N/GbouEVN22qN+j2jqvqXJM8Bbgd2BN5VVWeNoZOSpI6oqgXAggn8fCWp0Y9ctbF+6fWnwIYMhup+Ot6LSZImX4uPA7o+yVZVdW0zDHdD0341sO3Qcds0bas1ltl0fwX8APhzYD/g3CSvGVe3JUmTblYmvo3TGcCBzesDgS8Ntf9FM6vuqcBtQ8N5qzSWyujvgSdV1c0ASTYH/g84cTw9lyRNrumojJKcAuwFbJFkEXAkcAyDeQUHA78CXtYc/jXgBcBC4E7goNHOP5YwuhlYMvR+SdMmSZoBpmOUrqoOWM2uvVdxbAGvX5Pzj/RsukOblwuB85J8icE9o3nARWtyEUmSRjJSZbTii62/aLYVvrSKYyVJLenDekYjPSj1qOnsiCRpfNaKlV6TPAx4G7AzsMGK9qp69hT2S5I0Rn2ojMbyBIbPAJcDc4GjgF8C509hnyRJa5mxhNHmVXUCsLSqvlNVrwGsiiRphpiMxwG1bSxTu5c2f16b5IXANcDsqeuSJGlN9Hql1yH/mGQT4C3AR4GNgTdPaa8kSWPWgywa04NSv9K8vA141tR2R5K0pvowgWGkL71+lGYNo1WpqjdOSY8kSWudkSqjC6atF5KkcetBYTTil15PXt0+SdLMsbZMYJAkzWA9yCLDSJK6rg8TGMbypVdJkqbUjJ1N95F9d57K00sPsNlTDmm7C1rL3PWjYyftXH2oKpxNJ0kd14dhOmfTSVLHrU1LSBwG7IRLSEiSpsBYl5C4DJeQkKQZaVYmvrXNJSQkqeOSTHhrm0tISFLHzYTKZqJcQkKSOm4GFDYT5hISkqTWjWU23Ums4suvzb0jSVLL1pYHpX5l6PUGwIsZ3DeSJM0AfX8CAwBV9fnh90lOAb43ZT2SJK2RHhRG43pq9w7Awye7I5Kk8VkrhumSLOGB94yuY/BEBkmSJsVYhukeOh0dkSSNTw8Ko9HveyU5eyxtkqR2TMfjgJK8OcklSS5OckqSDZLMTXJekoVJTkuy3rg/wwgX3iDJbGCLJJslmd1s2wFbj/eCkqTJNSuZ8DaSJFsDbwSeXFW7AOsA+wP/BHywqh4D3AIcPO7PMMK+vwEuBB7X/Lli+xIweatCSZK6YF1gwyTrAhsB1zJ4Tunnmv0nA/tO5OSrVFUfBj6c5A1V9dHxXkCSNLUm455RkvnA/KGmBVW1AKCqrk7yL8CvgbuAbzAoTm6tqnub4xcxgVGzsUztXp5k06q6tenwZsABVfUf472oJGnyTMaDUpvgWbCqfc3v/XkMlhK6FfgssM/Er3q/sXxx969XBBFAVd0C/PVkdkKSNH6ZhP9G8SfAVVV1Y1UtBb4A7AFs2gzbAWwDXD3ezzCWMFonQ4tdJFkHGPeMCUnS5JqG2XS/Bp6aZKMmD/YGLgW+BezXHHMggzkF4/sMYzjmTOC0JHsn2Rs4pWmTJK0Fquo8BhMVfgj8lEF2LGDwAIRDkywENgdOGO81xnLP6DAGN7Ve17w/C/j4eC8oSZpc07G4XlUdCRy5UvOVwO6Tcf5RK6OqWl5Vx1XVflW1H4PSzNl1kjRDrC3LjpPkScABwMuAqxjcvJIkzQC9XnY8yY4MAugA4CbgNCBV5WqvkjSDzIDCZsJGqowuB74LvKiqFsLg2UTT0itJ0lplpDD6cwbPHvpWkjOBU2H0yeiSpOnVh/WMVjuBoaq+WFX7M3g23beANwEPT/KxJM+dpv5JkkYxHU/tnvLPMNoBVXVHVf1XVf0pg2/Y/ggX15OkGSOZ+Na2sXzp9T5VdUtVLaiqvaeqQ5Kktc+YpnZLkmauWT24nW8YSVLHzYRhtokyjCSp42bCBISJMowkqeN6PbVbkqTpYmUkSR3Xg8LIMJKkruvDMJ1hJEkd14MsMowkqev6cPO/D59BktRxVkaS1HEzYaXWiTKMJKnjuh9FhpEkdV4fZtN5z0iS1DorI0nquO7XRYaRJHVeD0bpDCNJ6jpn00mSWteHm/99+AySpI6zMpKkjnOYTpLUuu5HkWEkSZ3Xh8rIe0aSpFEl2TTJ55JcnuSyJE9LMjvJWUl+3vy52XjPbxhJUsfNmoRtDD4MnFlVjwOeAFwGHA6cXVU7AGc378f9GSRJHZZkwtso598E2BM4AaCq7qmqW4F5wMnNYScD+473MxhGktRxmYwtmZ/kgqFt/tAl5gI3Aicl+VGS45M8GJhTVdc2x1wHzBnvZ3ACgyR13GTMX6iqBcCC1exeF9gVeENVnZfkw6w0JFdVlaTGe30rI0nSaBYBi6rqvOb95xiE0/VJtgJo/rxhvBcwjCSp42aRCW8jqarrgN8keWzTtDdwKXAGcGDTdiDwpfF+BofpJKnjpulrRm8APpNkPeBK4CAGBc3pSQ4GfgW8bLwnN4wkqeMyDc9gqKofA09exa69J+P8hpEkdVwPHsDgPSNJUvusjCSp40abgNAFhpEkdVwfhukMI0nquD6EkfeMJEmtszKSpI6bjqndU80wkqSOm9X9LDKMJKnrrIwkSa1zAoMkSZPAykiSOs5hOklS65zAIElqnZWRWnXUu97B9875NpvNns3pX/gyAN/8xpks+NixXHXVlZz8mdPZaeddWu6luuy4I1/J8/fchRsXL+HJL30fAJ865iB22G4OAJs+dENuXXIXT93/GJ6886M49h8OAAY31I8+7muc8a2LWuv72qQPExgMow7703n78vIDXsG73nH/UvSPfswOfOCDH+V97z2yxZ6pLz715XM57rTvcPx7/+K+tlcfftJ9r4859MXc9tu7ALjkF9ewxys/wLJly9lyi40577Qj+Oo5F7Ns2fJp77e6x9l0Hbbrbk9h4403fUDb3O0fzXbbzW2nQ+qd//3hL1h8252r3f+S5+zK6WdeCMBdv1t6X/Csv96DqKpp6aMgk7C1zcpI0rjsseujuX7xEn7x6xvva3vKLo/iuHe/ikduNZuD33myVdE0mdWDcbppr4ySHDTCvvlJLkhywUknLJjObklaQy/b58l89swLHtB2/sW/Yrf9juaPX/UB/v41z2X99fz/u9PBymh8jgJOWtWOqloALABY8rvl1vjSDLXOOrOY9+wnsMcrPrDK/VdcdT2/vfNudn7MI/jhpb+e5t6pi6YkjJKsbgpNgDlTcU1J0+fZf/RYfvbL67n6hlvva3vUIzZn0fW3sGzZch651WY8du6W/Oqam9vr5NpkJpQ2EzRVldEc4HnALSu1B/i/KbrmWufth72FCy/4AbfeeisveM5ezH/dIWyyySb88zFHc8sti3nTIa9lx8c+jmOPO77trqqjTn7/X/KM3XZgi00fwsIz38t7j/saJ3/x+7z0ebvdN3Fhhac/aXveetBzWXrvMpYvL/7ufadx8613tNTztUsfvmeUqZjxkuQE4KSq+t4q9v1XVb1itHM4TKfp9PCnvbHtLmgtc9ePjp20BPnBlbdN+Pfl7ttv0mqiTUllVFUHj7Bv1CCSJI1d9+siv2ckSZoBnHcpSV3Xg9LIMJKkjuvDBAbDSJI6rgcPYDCMJKnrepBFTmCQJLXPMJKkrpumh9MlWSfJj5J8pXk/N8l5SRYmOS3JeuP9CIaRJHVcJuG/Mfo74LKh9/8EfLCqHsPgiTur/Y7paAwjSeq4ZOLb6NfINsALgeOb9wGeDXyuOeRkYN/xfgbDSJL0gCV8mm3+Sod8CHgbsGKRqs2BW6vq3ub9ImDr8V7f2XSS1HGTMZtueAmf3zt/8iLghqq6MMlek3C532MYSVLXTf3c7j2AP0vyAmADYGPgw8CmSdZtqqNtgKvHewGH6SSp46Z6AkNVHVFV21TVdsD+wP9U1SuBbwH7NYcdCHxpvJ/BMJKkjpuOCQyrcRhwaJKFDO4hnTDeEzlMJ0kas6r6NvDt5vWVwO6TcV7DSJI6rg+PAzKMJKnrepBGhpEkdZxLSEiSWteHJSScTSdJap2VkSR1XA8KI8NIkjqvB2lkGElSx/VhAoP3jCRJrbMykqSO68NsOsNIkjquB1lkGElS5/UgjQwjSeo4JzBIkjQJrIwkqeOcwCBJal0PssgwkqTO60EaGUaS1HFOYJAkaRJYGUlSxzmBQZLUuh5kkWEkSZ3XgzTynpEkqXVWRpLUcX2YTWcYSVLHOYFBktS6HmSRYSRJXdeHysgJDJKk1lkZSVLndb80MowkqeMcppMktS6TsI14/mTbJN9KcmmSS5L8XdM+O8lZSX7e/LnZeD+DYSRJHZdMfBvFvcBbqmon4KnA65PsBBwOnF1VOwBnN+/HxTCSJI2oqq6tqh82r5cAlwFbA/OAk5vDTgb2He81DCNJ6rhMxn/J/CQXDG3zV3mtZDvgScB5wJyqurbZdR0wZ7yfwQkMktR1kzCBoaoWAAtGvEzyEODzwJuq6vYMje9VVSWp8V7fykiSOm6qJzAAJHkQgyD6TFV9oWm+PslWzf6tgBvG+xkMI0nSiDIogU4ALquqfxvadQZwYPP6QOBL472Gw3SS1HHT8D2jPYBXAz9N8uOm7e3AMcDpSQ4GfgW8bLwXMIwkqeOmegmJqvoeqx/N23syrmEYSVLX9eAJDIaRJHVcD7LICQySpPZZGUlSx/XhQamGkSR13FRPYJgOhpEkdVwfKiPvGUmSWmcYSZJa5zCdJHVcH4bpDCNJ6jgnMEiSWteHysh7RpKk1lkZSVLH9aAwMowkqfN6kEaGkSR1nBMYJEmtcwKDJEmTwMpIkjquB4WRYSRJndeDNDKMJKnj+jCBwXtGkqTWWRlJUsf1YTZdqqrtPmgSJZlfVQva7ofWHv6b02RwmK5/5rfdAa11/DenCTOMJEmtM4wkSa0zjPrHsXtNN//NacKcwCBJap2VkSSpdYZRjyTZJ8kVSRYmObzt/qi/kpyY5IYkF7fdF/WDYdQTSdYB/h14PrATcECSndrtlXrsE8A+bXdC/WEY9cfuwMKqurKq7gFOBea13Cf1VFWdAyxuux/qD8OoP7YGfjP0flHTJkkznmEkSWqdYdQfVwPbDr3fpmmTpBnPMOqP84EdksxNsh6wP3BGy32SpDExjHqiqu4FDgG+DlwGnF5Vl7TbK/VVklOA7wOPTbIoycFt90nd5hMYJEmtszKSJLXOMJIktc4wkiS1zjCSJLXOMJIktc4wkiS1zjCSJLXOMJIkte7/A5OsEUgP3WKIAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 504x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create heatmap from the confusion matrix\n",
    "def createConfMatrix(class_names, matrix):\n",
    "    class_names=[0, 1] \n",
    "    tick_marks = [0.5, 1.5]\n",
    "    fig, ax = plt.subplots(figsize=(7, 6))\n",
    "    sns.heatmap(pd.DataFrame(matrix), annot=True, cmap=\"Blues\", fmt='g')\n",
    "    ax.xaxis.set_label_position(\"top\")\n",
    "    plt.title('Confusion matrix')\n",
    "    plt.ylabel('Actual label'); plt.xlabel('Predicted label')\n",
    "    plt.yticks(tick_marks, class_names); plt.xticks(tick_marks, class_names)\n",
    "    \n",
    "# Create a confusion matrix\n",
    "cnf_matrix = confusion_matrix(y_test, y_pred)\n",
    "createConfMatrix(matrix=cnf_matrix, class_names=[0, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.94\n",
      "precision: 0.97\n",
      "recall: 0.94\n",
      "f1_score: 0.95\n",
      "              precision    recall  f1-score     support\n",
      "0              0.896226  0.940594  0.917874  101.000000\n",
      "1              0.966480  0.940217  0.953168  184.000000\n",
      "accuracy       0.940351  0.940351  0.940351    0.940351\n",
      "macro avg      0.931353  0.940406  0.935521  285.000000\n",
      "weighted avg   0.941583  0.940351  0.940660  285.000000\n"
     ]
    }
   ],
   "source": [
    "# Calculate Standard Error Metrics\n",
    "print('accuracy: {:.2f}'.format(accuracy_score(y_test, y_pred)))\n",
    "print('precision: {:.2f}'.format(precision_score(y_test, y_pred)))\n",
    "print('recall: {:.2f}'.format(recall_score(y_test, y_pred)))\n",
    "print('f1_score: {:.2f}'.format(f1_score(y_test, y_pred)))\n",
    "\n",
    "# Classification Report (Alternative)\n",
    "results_log = classification_report(y_test, y_pred, output_dict=True)\n",
    "results_df_log = pd.DataFrame(results_log).transpose()\n",
    "print(results_df_log)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step #4 ROC Curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmEAAAGDCAYAAABjkcdfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABE/0lEQVR4nO3deXxU9fX/8ddJCCRhVUREkYKKsimoVIpbte5WsdVWxGq12p/futS2WlutrVpqra1WW63Wuq8V1G5YrVat+w4SEXBDigVFZRMhEzJZzu+PeydMhknmJpmbGcj7+Xjkkbn7ufdOZk4+n3PvNXdHRERERDpXSaEDEBEREemKlISJiIiIFICSMBEREZECUBImIiIiUgBKwkREREQKQEmYiIiISAEoCRORFplZhZk9aGarzez+mLZxiZndHce6pfOZ2VAzczPrFtP6f2JmN6cNf9XMFpvZWjPb1czmmdl+cWxbJN+UhImEzGyRmdWEH+YfmdntZtYrY549zew/ZrYmTEweNLNRGfP0MbPfmdn/wnW9Fw5v0cJ2zczONrO5ZlZtZkvM7H4z2znO/Y3oa8BAoL+7f72jKzOz/cxsScfDanH9t5tZMjzua8xslpl9Ma7ttRLHyWb2XIT5DjGzZ8JYl5nZ02Y2qTNi7Agz2zF8jy4P/w7mmNk5ZlYa97bd/TJ3/3baqCuBs9y9l7vPdvfR7v5U3HGI5IOSMJHmjnT3XsA4YFfggtQEM5sI/Bv4B7A1MAx4HXjezLYL5+kOPAGMBg4F+gATgRXAHi1s8/fA94Czgc2BHYG/A19ua/AxtD58DnjH3euLIJaofhOewz7AH4G/tpQcFDBGzOxrwP3AncBggmT3IuDIQsWUKdvxMbPtgZeBxcDO7t4X+DowHujduRECwXt0XkdXUsj3gnRh7q4f/ejHHWARcGDa8G+Ah9KGnwWuz7Lcv4A7w9ffBj4GekXc5nCgAdijlXmeAr6dNnwy8FzasANnAu8C/yVIPK7MWMc/gHPC11sDfwGWhfOf3cJ2fw4kgTpgLXAqwT9uPwXeBz4hSCD6hvMPDWM5Ffgf8EzG+noCNUBjuL61YSyXAPeF61pD8IU6Pm25SPGG894OXJo2XBnGtHXasXseuJogMb4U6EHQmvK/8NzdAFSE828G/DPc9qrw9eCMc7EwjPu/wDeAkcC68LyuBT7NEqeF2zuvlX3ZHvhPGOdy4B6gX8b79YfAHGA1MB0oT5t+FFAFfAa8Bxwaju8L3AIsBT4Ij0FpS8cnS1x3k/Z3kWV66n3QLRz+FvBmeIwWAv+XNu8W4TH9FFhJ8DdWEk77cRjfGuBt4IBw/CVhDD3C4+tANfBe5t8xwfv1/HD/VxC8zzaP8n7Vj34640ctYSJZmNlg4DBgQThcCexJ0HKR6T7goPD1gcAj7r424qYOAJa4+ysdi5ivABOAUcC9wGQzMwAz2ww4GJhmZiXAgwQteNuE2/++mR2SuUJ3vxi4DJjuQVfPLQRf0icD+wPbAb2AP2Qs+kWCRKTZOt29muCYfhiur5e7fxhOngRMA/oBM1LrbEu8mcLWr28SJEcfp02aQJAMDAR+CVxO0Po4Dtgh3M5F4bwlwG0ErS1DCJLIVGw9gWuAw9y9N8H7o8rd3wS+A7wY7mO/LOHtBGwLPNDaLgC/IkhCR4bzX5Ixz7EELa7DgF0Izg1mtgdBUnsewTHdlyA5gSBRrQ/3dVeC90Z6917m8cl0YI64M30CHEHQMvkt4Goz2y2cdi6wBBgQbu8ngJvZTsBZwOfDY3tIWvwAuHutBy2eAGPdffss2/4uwd/GFwmO4yrguox5sr5fRTqDkjCR5v5uZmsIulo+AS4Ox29O8PeyNMsySwn+owfo38I8LWnr/C35lbuvdPcagtYEB/YJp32NICH4EPg8MMDdp7p70t0XAjcBx0XczjeAq9x9YZhoXgAcl9GVc4m7V4exRPWcuz/s7g3AXcDYcHx74v2hmX1K0EryO+Bn4XpTPnT3az3oYl0HnAb8IDx+awgSz+MA3H2Fu//F3RPhtF8SfGmnNAJjzKzC3Ze6e9Rusf7h7xbPvbsvcPfHwmRjGXBVxrYBrnH3D919JUGyOi4cfypwa7h8o7t/4O5vmdlA4HDg++E5+oSg1Sv9eDYdnxbOYZves+7+kLu/54GnCbr0U+/NOmAQ8Dl3r3P3Z93dCVoRewCjzKzM3Re5+3tRt5nmO8CF7r7E3WsJktiv5eH9KpIXSsJEmvtK+J/3fsAI1idXqwi+cAdlWWYQQXcRBF0e2eZpSVvnb8ni1IvwS2waMCUcdTxBVxYELTpbm9mnqR+C1oeBEbezNUFXZMr7QLeM5RfTdh+lvU4A5eEXZXvivTJsfaokqFO6wswOayG+AeF8s9LW/0g4HjOrNLM/mdn7ZvYZ8AzQz8xKw5a9yQRf9EvN7CEzGxFxf1eEv1s892Y20MymmdkH4bbvZv37MSXzuKVahrYl6ILL9DmgLIw3tb9/ArZMmyfX+WvTe9bMDjOzl8xsZbi9w1m/H1cQtDb/28wWmtn5ECSgwPcJkqZPwuOwddRtpvkc8Le0fX2TIMHr6PtVJC+UhIlkEf7HfjtBrVCqK+1FggLkTMcSFOMDPA4cEnZVRfEEMNjMxrcyTzVBopCyVbaQM4bvJfiP/3ME3Ut/CccvBv7r7v3Sfnq7++ER4/2Q4IstZQhB11Z6d19mLK3FmUu74w1bXuYS1DilX+SQHsNygi7G0Wnr75vWzXUuQdfhBHfvQ9CtB0FXIe7+qLsfRJCUvEXQShdlP98O9+2YVua5LFzPzuG2T0htN4LFBDVl2cbXAluk7W8fdx+dNk+u2B/PEXcTM+tB8N67EhgYJscPs/74rXH3c919O4Iu6XPM7IBw2p/dfW+C95sDv46yzQyLCbqL098/5e7+Qdo8bX1PiuSNkjCRlv0OOMjMUl1j5wMnhbeT6G1mm5nZpQRXP/48nOcugg/+v5jZCDMrMbP+FtzbaIPEwd3fBa4H7g1v39DdzMrN7LhUqwBBcfXRYavMDgRdTa1y99kECcbNwKPu/mk46RVgjZn92IJ7gJWa2Rgz+3zEY3Iv8AMzG2bB7TtSNWNRr578GOhvZn0jzt+heMOWqb1p4eo5d28kSJyuNrMtw2W2Sas5602QpH1qZpuzvns61VJ1VJhw1xJ0fzam7efg8GrZbNt14BzgZ2b2LQtua1JiZnub2Y1p214LrDazbQjqu6K6BfiWmR0QrncbMxvh7ksJugN/m7bN7a1tt/G4GNjTzK4ws63CY7GDmd1tZv0y5u1O0K24DKgPWyQPTk00syPCZY3g4oIGoNHMdjKzL4VJ3DrWX9DRVjcAvwz/GcHMBpjZUe1Yj0gslISJtCCsw7mTsEjb3Z8jKN49mqAm5n2Cwua9w2SKsO7kQIJWkccIrkx7haD75eUWNnU2QbH3dQRXib0HfJWgxgeCmp0kwRf7HazvWszlz2Esf07bpwaCIulxBAXrqUQtalJ0K0Gi+Uy4/DqC4udI3P0tgkRuYdhF1GoXUzvj/ZEF9wmrJkg4biPocmvJjwm6xF4Ku/0eJ2j9giARrwi3+xJBV2VKCUEi9SHBlX1fBE4Pp/2HIPH7yMyWk4W7P0DQnXlKuI6PCa5U/Ec4y8+B3QiSk4eAv7ayD5nrfoWwCD5c/mnWt2B+kyA5mk/Qzf4AbeheDGuzJhJcXTjPzFYTtHbNJLiSMX3eNQTv7/vCbR1PcOFFynCC472WoKX5end/kiBxu5zguH9E0F16AW33+3B7/w5rPV8iaBkWKQoW/EMmIiIiIp1JLWEiIiIiBaAkTERERKQAlISJiIiIFICSMBEREZECUBImIiIiUgAb3VPjt9hiCx86dGihwxARERHJadasWcvdfUC2aRtdEjZ06FBmzpxZ6DBEREREcjKz91uapu5IERERkQJQEiYiIiJSAErCRERERApASZiIiIhIASgJExERESkAJWEiIiIiBaAkTERERKQAlISJiIiIFICSMBEREZECiC0JM7NbzewTM5vbwnQzs2vMbIGZzTGz3eKKRURERKTYxNkSdjtwaCvTDwOGhz+nAX+MMRYRERGRohLbsyPd/RkzG9rKLEcBd7q7Ay+ZWT8zG+TuS+OKSURERMDdcQdPvYZwOByf/jpznlaWxaEx27Ke2i40Ni2zflnS5m/0DWMgbRuN3vKyTetPiydz31Lbx2H7Ab0Y0r+yk476hgr5AO9tgMVpw0vCcRskYWZ2GkFrGUOGDOmU4EQk/1r68G4MP2Vb/OBvbOULobUvjRxfJjR9oGdfD6R/oGf/4M/8Qsj84M/8wiFjW41Z1pm+rtT6Nzg+aV9YzdaZsSyZX57QfH9aWHbDL9vm8+Ge5cu2+f43Nm54nkg7Fo0Z66fZvjU/N6THnHFsM4/p+i/bzGOa4z2X6/3U2rFLG9/S+yZ9HxozlqWFfdvwfeHNjm2247L+3Lf23heA8w8bwXe+uH3Btl/IJCwyd78RuBFg/PjxevtIp2hsdNbU1vNZTR2ra+r4bF0dn9XU8VlNfbPh1TV1rKtrzP2h2eqXae4PzZYTj5a+TNd/SDe2sCxk+ULIsg/r/zPN/sGf/oWTbR8a9Ve7yTEDA8ws/A1GMNKAErMN5iFtuMSaLwvp84fLh/M2bS/cRvp0WojDMtbfbHxq/lb2oaQEjJINlqWFfUvfh5JmMTQ/Lk37vsFxWb/OpvVnOS7G+mNH5v6mHbvMdbZ4XDLnadq3KMs2P4407duGy27wvsg4pqn1Nj+2mfuf2vcNj2nm+2bD/V9/btLXuU2/io78GXRYIZOwD4Bt04YHh+NE8qa+oZGV1cmmpGl1ehKVbVzTcB1rautb/Y/RDPqUl9G3oozyspLgg4/sH/wl4YgNPrya5gtft/DBn/WDj/UfUiUbzN/CB1/asukfUplfGukfUpnLQvNtpX+ZZvvgX/+hmP2Dv6Sk5S+TKB/8rX1pYOlf5tk/+LN9aWTd/ywf/JnLbvBlWtJSktH8C2eD/U9737S0bNOXbeZ7KMt6sr7nsPD9ln1ZWnjfpLYlIh1XyCRsBnCWmU0DJgCrVQ8m+VCTbODpd5bx6LyPePzNj1mzrr7FeSu7lzYlUn0qurFVn3J2GtibPhVlwU95N/pUhNPT5utTUUav7t2aEggREZG2ii0JM7N7gf2ALcxsCXAxUAbg7jcADwOHAwuABPCtuGKR4rOyOsmTb33Cyupk3tbZ6E7V4k956u1l1NQ10K+yjENGb8W4bfvRN5VINSVU3ehdXkb3brpVnoiIFEacV0dOyTHdgTPj2r4Un49Wr+Pf8z/ikbkf8fJ/V9IQQ6HQgN49OGb3bThszCD2GLY5ZaVKskREpDhtFIX5Ulzcnc9q6ln6WQ0frV7Hx5+tY2n4+6PV6/jos1qS9Q3NlmlodBatSACw/YCenP7F7Tlk9FYMG9Azr7FVlpWqi1BERDYKSsKkmYZGZ9maWj76bB0fra5pSqo+Wl0TjlvHR5+tY11d4wbLbtGrO1v1LWfrvuWUdy/dYPoxuw3msJ23Yocte3fGroiIiBQ1JWFdSE2yoSmRymy9WvrZOj5evY5P1qzb4HYCZaXGwD7lbNWnnNHb9OXAkQPZqm958NOnnIHhj+qrREREolMStolyd37z6NvM//CzpoRrdU3dBvP17tGNgX3LGdS3nOFbbsFWfdYnV6lEa/PK7uriExERyTMlYZuoFdVJ/vjUewzerIIRW/Vh/NDNGNS3goF9goRrYJhk9eqht4CIiEgh6Bt4E/d/+27HiROHFjoMERERyaAiHhEREZECUBImIiIiUgDqjtwIuTvVyQZWrk2yMpFkZXUtK6vrmv3++LPaQocpIiIirVASVkSWrEqweGUNqxJJVlQnWbk22fR6VfX63ysTSZL1G96nC6B7aQmb9+zOZj27s++OA5i4ff9O3gsRERGJQklYkfjg0xr2/vWTG4zvXd6N/mFStU2/csZs3YfNe3UPxlV2p3+v8HfPHmzWs4xePbphpttJiIiIFDslYUWgvqGRlxeuAOAHB+7IwaMH0r9nd/pVdtcNUEVERDZRSsIKpLa+gecXLOeRuR/x2PyPWZWoo6KslANHbcnIQX0KHZ6IiIjETElYAXzy2ToO/t0zfJqoo3ePbnxp5JYcOnorvrjTACq765SIiIh0BfrGL4D/rUzwaaKOHx26E6fuPYwe3TZ82LWIiIhs2pSEdZLla2t5bP7H/GvuR7ywYDkAnx+6uRIwERGRLkpJWMxWJ+o4/Z5ZvLRwBY0OQzav5NS9h3HYzoMYt22/QocnIiIiBaIkLGYLlq3lhfdWMHn8tpy811BGbNVbt5AQERERJWGd5fBdBumqRxEREWmim1DFzN0LHYKIiIgUISVhMVq4bC0//sscSkuMbTerKHQ4IiIiUkTUHRmTJ978mO9Pq6KsWwl3nzqB7Qb0KnRIIiIiUkSUhOVZY6Nz3ZMLuOrxdxi9dR/+dOJ4tumnVjARERFpTklYHq2trefc+6p4dN7HfHXXbfjV0TtTXqb7gImIiMiGlITlyX+XV3PanTNZuLyanx0xilP2GqpbUYiIiEiLlITlwZNvf8LZ986mW4lx1yl7sOcOWxQ6JBERESlySsI6wN25/qn3uPLfbzNyqz786cTd2XbzykKHJSIiIhsBJWHtVN/QyPemVfHQG0uZNHZrfn3MLlR0V/2XiIiIRKMkrJ1mvr+Kh95Yyne/tAPnHLSj6r9ERESkTXSz1naqbwjuhL/vjgOUgImIiEibKQkTERERKQAlYSIiIiIFoCRMREREpACUhImIiIgUgJIwERERkQJQEiYiIiJSAErCRERERApASZiIiIhIASgJExERESkAJWEiIiIiBaAkTERERKQAlISJiIiIFICSMBEREZECUBImIiIiUgBKwkREREQKQEmYiIiISAEoCRMREREpACVhIiIiIgWgJExERESkAJSEiYiIiBRArEmYmR1qZm+b2QIzOz/L9CFm9qSZzTazOWZ2eJzxiIiIiBSL2JIwMysFrgMOA0YBU8xsVMZsPwXuc/ddgeOA6+OKR0RERKSYxNkStgewwN0XunsSmAYclTGPA33C132BD2OMR0RERKRodItx3dsAi9OGlwATMua5BPi3mX0X6AkcGGM8IiIiIkWj0IX5U4Db3X0wcDhwl5ltEJOZnWZmM81s5rJlyzo9SBEREZF8i5SEmVmJme1qZl82sy+Z2ZYRFvsA2DZteHA4Lt2pwH0A7v4iUA5skbkid7/R3ce7+/gBAwZECVlERESkqLXaHWlm2wM/JugmfBdYRpAo7WhmCeBPwB3u3phl8VeB4WY2jCD5Og44PmOe/wEHALeb2chw3WrqEhERkU1erpqwS4E/Av/n7p4+IWwNOx44Ebgjc0F3rzezs4BHgVLgVnefZ2ZTgZnuPgM4F7jJzH5AUKR/cuZ2RERERDZFrSZh7j6llWmfAL/LsfzDwMMZ4y5Kez0f2CtKoCIiIiKbknYX5pvZQfkMRERERKQr6cjVkbfkLQoRERGRLiZXYf6MliYB/fMfjoiIiEjXkKswfx/gBGBtxngjuCO+iIiIiLRDriTsJSDh7k9nTjCzt+MJSURERGTTl+vqyMNambZv/sMRERER6RoK/dgiERERkS5JSZiIiIhIASgJExERESkAJWEiIiIiBRA5CTOzS1obFhEREZHo2tISNivHsIiIiIhEFDkJc/cHWxvuahwvdAgiIiKyEcv12KJroeVsw93PzntEG4naukYAenRTWZ2IiIi0Xa475s/slCg2QtXJegAqu+c6hCIiIiIbynXH/DvSh82s0t0T8Ya0cahJNgDQs0dpgSMRERGRjVGkvjQzm2hm84G3wuGxZnZ9rJEVueowCassU0uYiIiItF3UgqbfAYcAKwDc/XWgSz87MlEbdEdWdFdLmIiIiLRdW66OXJwxqiHPsWxUqpMNdC8tobsK80VERKQdovalLTazPQE3szLge8Cb8YVV/GqS9VSqHkxERETaKWozzneAM4FtgA+BceFwl1WdbKCyTEmYiIiItE+kljB3Xw58I+ZYNiqJZD2VPVSULyIiIu0T9erI7czsQTNbZmafmNk/zGy7uIMrZolkAz1VlC8iIiLtFLU78s/AfcAgYGvgfuDeuILaGCRqG3SjVhEREWm3qElYpbvf5e714c/dQHmcgRW76mQ9lWoJExERkXbK9ezIzcOX/zKz84FpBM+SnAw8HHNsRS2RbFBNmIiIiLRbrixiFkHSZeHw/6VNc+CCOILaGCSS9aoJExERkXbL9ezIYZ0VyMZGNWEiIiLSEZGzCDMbA4wirRbM3e+MI6hi5+6qCRMREZEOiZSEmdnFwH4ESdjDwGHAc0CXTMJq6xtpdHTHfBEREWm3qFdHfg04APjI3b8FjAX6xhZVkUskg8dm9lR3pIiIiLRT1CSsxt0bgXoz6wN8AmwbX1jFrbq2HoAKdUeKiIhIO0VtyplpZv2AmwiumFwLvBhXUMVOLWEiIiLSUVGfHXlG+PIGM3sE6OPuc+ILq7hVJ4OWMNWEiYiISHvlulnrbq1Nc/fX8h9S8atRS5iIiIh0UK4s4retTHPgS3mMZaORqgnTLSpERESkvXLdrHX/zgpkY5KqCVMSJiIiIu0V9epISdNUmK9nR4qIiEg7KQlrh0RS3ZEiIiLSMUrC2qG6NtUdqZYwERERaZ9ISZgFTjCzi8LhIWa2R7yhFa9Esp4e3UooLbFChyIiIiIbqagtYdcDE4Ep4fAa4LpYItoIJJINqgcTERGRDomaSUxw993MbDaAu68ys+4xxlXUqpP1qgcTERGRDonaElZnZqUE9wbDzAYAjbFFVeQStQ1KwkRERKRDoiZh1wB/A7Y0s18CzwGXxRZVkQtawtQdKSIiIu0X9dmR95jZLOAAwICvuPubsUZWxGqSDfTUcyNFRESkAyIlYWZ2DTDN3btsMX666mQD/Sq7bEmciIiI5EHU7shZwE/N7D0zu9LMxscZVLFLJOvVEiYiIiIdEikJc/c73P1w4PPA28CvzezdWCMrYtW1DaoJExERkQ5p6x3zdwBGAJ8D3sp/OBuHmmQ9PXV1pIiIiHRA1Dvm/yZs+ZoKzAXGu/uRsUZWpBobnUSdblEhIiIiHRO1Jew9YKK7H+rut7n7p1EWMrNDzextM1tgZue3MM+xZjbfzOaZ2Z8jxlMw6+obcIdK3TFfREREOqDVTMLMRrj7W8CrwBAzG5I+3d1fa2XZUoJHGx0ELAFeNbMZ7j4/bZ7hwAXAXuFd+Lds/650jkQyeHi3uiNFRESkI3I155wDnAb8Nss0B77UyrJ7AAvcfSGAmU0DjgLmp83z/4Dr3H0VgLt/EjHugknUBkmYCvNFRESkI1rNJNz9tPDlYe6+Ln2amZXnWPc2wOK04SXAhIx5dgzX9TxQClzi7o9krsjMTiNIBhkyZEjm5E5VnawHUE2YiIiIdEjUmrAXIo5rq27AcGA/YApwk5n1y5zJ3W909/HuPn7AgAF52Gz7JVJJmGrCREREpANy1YRtRdCiVWFmuxI8sgigD1CZY90fANumDQ8Ox6VbArzs7nXAf83sHYKk7NVo4Xc+1YSJiIhIPuRqzjkEOJkggboqbfwa4Cc5ln0VGG5mwwiSr+OA4zPm+TtBC9htZrYFQffkwiiBF0q1asJEREQkD3LVhN0B3GFmx7j7X9qyYnevN7OzgEcJ6r1udfd5ZjYVmOnuM8JpB5vZfKABOM/dV7RrTzpJQjVhIiIikge5uiNPcPe7gaFmdk7mdHe/Ksti6dMfBh7OGHdR2msnuAJzg3UXq+qwO7JSz44UERGRDsjVp9Yz/N0r7kA2FjVhS1hPdUeKiIhIB+TqjvxT+PvnnRNO8UvVhFWUqSVMRERE2q8tz47sY2ZlZvaEmS0zsxPiDq4YJZL1VJSVUlJiuWcWERERaUHU+4Qd7O6fAUcAi4AdgPPiCqqYVScb6Kl6MBEREemgqElYqtvyy8D97r46pniKXk2yQbenEBERkQ6Lmk3808zeAmqA081sALAuxzKbpOraet2eQkRERDosUkuYu58P7AmMD+9uX03wMO4uJ5FsUBImIiIiHRapJczMyoATgH3NDOBp4IYY4ypaiWQ9PfXcSBEREemgqDVhfwR2B64Pf3YLx3U5agkTERGRfIjapPN5dx+bNvwfM3s9joCKXXWyXoX5IiIi0mFRW8IazGz71ICZbUfwrMcuJ1GrljARERHpuKhNOucBT5rZQsCAzwHfii2qIpZINqgmTERERDosZzYR3o5iNbAHsGU4+m13r40zsGLU0OjU1KklTERERDqu1e5IM/s2MA+4FqgChrr7nK6YgAHU1AU9sErCREREpKNytYR9Hxjt7svCOrB7gBmxR1WkErX1ACrMFxERkQ7LVZifdPdlAO6+EOgRf0jFK5EMWsL07EgRERHpqFxNOoPN7JqWht397HjCKk7VSbWEiYiISH7kyibOyxieFVcgG4NUS5hqwkRERKSjWk3C3P2OzgpkY1CtmjARERHJk1xXR95kZmNamNbTzE4xs2/EE1rxqVFNmIiIiORJriad64CLzGxnYC6wDCgHhgN9gFsJrpjsEqpTSZhawkRERKSDcnVHVgHHmlkvYDwwCKgB3nT3t+MPr7gkwsL8CtWEiYiISAdFatJx97XAU/GGUvwSagkTERGRPIn6AG8huFmrGZSX6bCJiIhIxyibaIPqZAOVZaWYWaFDERERkY1cm5IwM6uMK5CNQSJZT2UPdUWKiIhIx0VKwsxsTzObD7wVDo81s+tjjawIJZIN9FRRvoiIiORB1Jawq4FDgBUA7v46sG9cQRWr6toG3ahVRERE8iJyd6S7L84Y1ZDnWIpeIlmvRxaJiIhIXkRNwhab2Z6Am1mZmf0QeDPGuIpSdbJBNWEiIiKSF1GTsO8AZwLbAB8A44AzYoqpaNUk61UTJiIiInkRtVlnJ3dv9oxIM9sLeD7/IRUv1YSJiIhIvkRtCbs24rhNmmrCREREJF9abdYxs4nAnsAAMzsnbVIfoMtlI0FNWJfbbREREYlBrr617kCvcL7eaeM/A74WV1DFqL6hkWR9o54bKSIiInnRakbh7k8DT5vZ7e7+fifFVJQSdcEdOdQdKSIiIvkQtVknYWZXAKOB8tRId/9SLFEVoURtKglTS5iIiIh0XNTC/HsIHlk0DPg5sAh4NaaYilIiWQ9AT9WEiYiISB5ETcL6u/stQJ27P+3upwBdphUMgudGglrCREREJD+iZhR14e+lZvZl4ENg83hCKk7VtUFLmGrCREREJB+iJmGXmllf4FyC+4P1Ab4fV1DFaH1LmJIwERER6bhISZi7/zN8uRrYH5rumN9lpJKwnnp2pIiIiORBrpu1lgLHEjwz8hF3n2tmRwA/ASqAXeMPsThUJ9UdKSIiIvmTq1nnFmBb4BXgGjP7EBgPnO/uf485tqKSaKoJU0uYiIiIdFyujGI8sIu7N5pZOfARsL27r4g/tOJSrZowERERyaNct6hIunsjgLuvAxZ2xQQMoCbZQGmJ0aNb1Lt6iIiIiLQsV0vYCDObE742YPtw2AB3911ija6IVCfrqexeipkVOhQRERHZBORKwkZ2ShQbgURtg7oiRUREJG9yPcC7Sz+0O111sp6eKsoXERGRPFGBU0Q1yQYq9dxIERERyZNYkzAzO9TM3jazBWZ2fivzHWNmbmbj44ynI4KaMLWEiYiISH5ETsLMrMLMdmrD/KXAdcBhwChgipmNyjJfb+B7wMtR110IiaRqwkRERCR/IiVhZnYkUAU8Eg6PM7MZORbbA1jg7gvdPQlMA47KMt8vgF8D66IGXQiJZINqwkRERCRvoraEXUKQVH0K4O5VwLAcy2wDLE4bXhKOa2JmuwHbuvtDra3IzE4zs5lmNnPZsmURQ86vRG29WsJEREQkb6ImYXXuvjpjnHdkw2ZWAlwFnJtrXne/0d3Hu/v4AQMGdGSz7VadbNDDu0VERCRvoiZh88zseKDUzIab2bXACzmW+YDguZMpg8NxKb2BMcBTZrYI+AIwo1iL8xPJeirUEiYiIiJ5EjUJ+y4wGqgF/gysBr6fY5lXgeFmNszMugPHAU11ZO6+2t23cPeh7j4UeAmY5O4z27YL8UvWN1LX4PRUEiYiIiJ5ErV/bYS7XwhcGHXF7l5vZmcBjwKlwK3uPs/MpgIz3T1XYX/RqGl6eLe6I0VERCQ/omYVvzWzrYAHgOnuPjfKQu7+MPBwxriLWph3v4ixdLrqZD2ACvNFREQkbyJ1R7r7/sD+wDLgT2b2hpn9NNbIikgilYSpMF9ERETyJPLNWt39I3e/BvgOwT3DsrZobYoSYXekasJEREQkX6LerHWkmV1iZm8AqSsjB8caWRGprlVNmIiIiORX1KziVmA6cIi7fxhjPEUpoZowERERybNISZi7T4w7kGJWneqO7KEkTERERPKj1STMzO5z92PDbsj0O+Qb4O6+S6zRFYmappYwdUeKiIhIfuTKKr4X/j4i7kCKWaomTA/wFhERkXxptTDf3ZeGL89w9/fTf4Az4g+vOKRqwvTYIhEREcmXqLeoOCjLuMPyGUgxSyQbKCs1uneLfEcPERERkVblqgk7naDFazszm5M2qTfwfJyBFZNEskH1YCIiIpJXuTKLPwP/An4FnJ82fo27r4wtqiJTXVuvG7WKiIhIXuVKwtzdF5nZmZkTzGzzrpKIJZINqgcTERGRvIrSEnYEMIvgFhWWNs2B7WKKq6gkkvX01HMjRUREJI9azSzc/Yjw97DOCac4VScbdLd8ERERyauoz47cy8x6hq9PMLOrzGxIvKEVj0SyXvcIExERkbyKes+FPwIJMxsLnAu8B9wVW1RFJlGrmjARERHJr6hJWL27O3AU8Ad3v47gNhVdQiLZoJYwERERyauomcUaM7sAOBHYx8xKgLL4wiou1cl6KvXwbhEREcmjqC1hk4Fa4BR3/wgYDFwRW1RFxN3Dm7UqCRMREZH8iZSEhYnXPUBfMzsCWOfud8YaWZGorW+kodF1x3wRERHJq6hXRx4LvAJ8HTgWeNnMvhZnYMWiJtkAoDvmi4iISF5Fbd65EPi8u38CYGYDgMeBB+IKrFhUJ+sBqNTNWkVERCSPotaElaQSsNCKNiy7UUuELWGqCRMREZF8itq884iZPQrcGw5PBh6OJ6TikmjqjlRLmIiIiORPpMzC3c8zs6OBvcNRN7r73+ILq3gkasPuSLWEiYiISB61moSZ2XDgSmB74A3gh+7+QWcEViyqUy1hqgkTERGRPMpV13Ur8E/gGGAWcG3sERWZRFiYr8cWiYiISD7lat7p7e43ha/fNrPX4g6o2KgmTEREROKQK7MoN7NdAQuHK9KH3X2TT8qqUzVhemyRiIiI5FGuJGwpcFXa8Edpww58KY6giknTLSrKlISJiIhI/rSahLn7/p0VSLGqTtbTvVsJ3Uq7xG3RREREpJMos8ihJtmgRxaJiIhI3ikJy6G6tkEP7xYREZG8UxKWQyJZrxu1ioiISN5FSsIscIKZXRQODzGzPeINrThUJxv08G4RERHJu6gtYdcDE4Ep4fAa4LpYIioyNcl61YSJiIhI3kVNwia4+5nAOgB3XwV0jy2qIqKaMBEREYlD1CSszsxKCe4NhpkNABpji6qIqCZMRERE4hA1CbsG+BuwpZn9EngOuCy2qIpIItlAT90tX0RERPIsUj+bu99jZrOAAwgeWfQVd38z1siKRCKp7kgRERHJv0jZhZkNARLAg+nj3P1/cQVWDNydahXmi4iISAyiNvE8RFAPZkA5MAx4GxgdU1xFYV1dI+5QoZYwERERybOo3ZE7pw+b2W7AGbFEVEQSyXoA1YSJiIhI3rXrjvnu/howIc+xFJ1EsgFANWEiIiKSd1Frws5JGywBdgM+jCWiIlKdaglTTZiIiIjkWdQmnt5pr+sJasT+kv9wikt1bdASVqEkTERERPIsZxIW3qS1t7v/sBPiKSo1YXdkTz07UkRERPKs1ZowM+vm7g3AXp0UT1FJdUfqjvkiIiKSb7maeF4hqP+qMrMZwP1AdWqiu/81xtgKrunqSBXmi4iISJ5FzS7KgRXAl1h/vzAHNukkLFUTppYwERERybdcSdiW4ZWRc1mffKV4rpWb2aHA74FS4GZ3vzxj+jnAtwmK/ZcBp7j7+9HDj1eqJqxSNWEiIiKSZ7nuE1YK9Ap/eqe9Tv20KCzovw44DBgFTDGzURmzzQbGu/suwAPAb9q6A3FK1YRVlKklTERERPIrVxPPUnef2s517wEscPeFAGY2DTgKmJ+awd2fTJv/JeCEdm4rFolkA+VlJZSWWO6ZRURERNogV0tYR7KPbYDFacNLwnEtORX4Vwe2l3eJZL2K8kVERCQWuTKMAzojCDM7ARgPfLGF6acBpwEMGTKkM0ICIFHbQKWeGykiIiIxaLUlzN1XdmDdHwDbpg0PDsc1Y2YHAhcCk9y9toU4bnT38e4+fsCAAR0IqW2q1RImIiIiMWnXA7wjehUYbmbDzKw7cBwwI30GM9sV+BNBAvZJjLG0SyLZoEcWiYiISCxiS8LcvR44C3gUeBO4z93nmdlUM5sUznYFwVWW95tZ6oawRSORbFBLmIiIiMQi1gzD3R8GHs4Yd1Ha6wPj3H5HVdfW079n90KHISIiIpugOLsjN3qJZIMe3i0iIiKxUBLWikSyXjVhIiIiEgslYa0IasKUhImIiEj+KQlrQWOjk0g2UKnCfBEREYmBkrAW1NQFD+/uqZu1ioiISAyUhLWg6eHdagkTERGRGCgJa0FNMmwJU02YiIiIxEBJWAuqa4MkTDVhIiIiEgclYS1IhN2RlWoJExERkRgoCWtBIqnCfBEREYmPkrAWrG8JU3ekiIiI5J+SsBakasL0AG8RERGJg5KwFiSablGh7kgRERHJPyVhLVBNmIiIiMRJSVgLqpMNmEF5NyVhIiIikn9KwlqQqK2nsqyUkhIrdCgiIiKyCVIS1oLqZIMeWSQiIiKxURLWgppkverBREREJDZKwlpQnWzQPcJEREQkNkrCWpBI1uvh3SIiIhIbJWEtqK5t0D3CREREJDZKwlpQk2zQ3fJFREQkNkrCWlCdrKdShfkiIiISEyVhLUioJUxERERipCSsBYlkPZWqCRMREZGYKAnLoqHRWVfXqFtUiIiISGyUhGWRSNYDeni3iIiIxEdJWBaJZAOAblEhIiIisVESlkUqCVNhvoiIiMRFSVgW1bVBd6QK80VERCQuSsKyaGoJ66GWMBEREYmHkrAsqsPCfNWEiYiISFyUhGVRo5owERERiZmSsCxUEyYiIiJxUxKWhWrCREREJG5KwrJI1YSpJUxERETioiQsi5pkAyUGPbrp8IiIiEg8lGVkUV3bQM/u3TCzQociIiIimyglYVkkkvVU6rmRIiIiEiMlYVkkkg1U6vYUIiIiEiMlYVkkkvUqyhcREZFYKQnLIlUTJiIiIhIXJWFZqCZMRERE4qYkLIugJkxJmIiIiMRHSVgWKswXERGRuCkJy6I6WU9PtYSJiIhIjNTck0WitoEKtYSJyEasrq6OJUuWsG7dukKHItIllJeXM3jwYMrKyiIvo0wjQ11DI8mGRrWEichGbcmSJfTu3ZuhQ4fq6R8iMXN3VqxYwZIlSxg2bFjk5dQdmSGRbACgsofyUxHZeK1bt47+/fsrARPpBGZG//7929zyrCQsQyJZD6CWMBHZ6CkBE+k87fl7UxKWobo2aAmrUBImIiIiMYo1CTOzQ83sbTNbYGbnZ5new8ymh9NfNrOhccYTRU3YHak75ouIdExpaSnjxo1jzJgxHHnkkXz66ad5We/tt9/OWWedlZd1DR06lJ133plx48Yxbtw4XnjhhbysN1NVVRUPP/xws3H/+te/GD9+PKNGjWLXXXfl3HPPBeCSSy7hyiuvzNu299xzz6bX5513HqNHj+a8887jhhtu4M477+zQumfPns2pp57abNxXvvIVvvCFLzQbd/LJJ/PAAw80G9erV6+m1++88w6HH344w4cPZ7fdduPYY4/l448/7lBsK1eu5KCDDmL48OEcdNBBrFq1Kut8P/7xjxkzZgxjxoxh+vTpTeOfeOIJdtttN8aNG8fee+/NggULAPjDH/7Arbfe2qHYUmJLwsysFLgOOAwYBUwxs1EZs50KrHL3HYCrgV/HFU9U1WF3pO6YLyLSMRUVFVRVVTF37lw233xzrrvuukKHlNWTTz5JVVUVVVVVzRKW1tTX17dpG5lJ2Ny5cznrrLO4++67mT9/PjNnzmSHHXZo0zqjSk8sb7zxRubMmcMVV1zBd77zHb75zW9GXk+2fb7ssss4++yzm4Y//fRTZs2axerVq1m4cGGk9a5bt44vf/nLnH766bz77ru89tprnHHGGSxbtixybNlcfvnlHHDAAbz77rsccMABXH755RvM89BDD/Haa69RVVXFyy+/zJVXXslnn30GwOmnn84999xDVVUVxx9/PJdeeikAp5xyCtdee22HYkuJs7lnD2CBuy8EMLNpwFHA/LR5jgIuCV8/APzBzMzdPca4WrW+JkwtYSKyafj5g/OY/+FneV3nqK37cPGRoyPPP3HiRObMmQPAK6+8wve+9z3WrVtHRUUFt912GzvttBO33347M2bMIJFI8N577/HVr36V3/zmNwDcdttt/OpXv6Jfv36MHTuWHj16ALBo0SJOOeUUli9fzoABA7jtttsYMmQIJ598MhUVFcyePZtPPvmEW2+9lTvvvJMXX3yRCRMmcPvtt7cYa2vrLC8vZ/bs2ey1116ceeaZnHnmmSxbtozKykpuuukmRowYwf3338/Pf/5zSktL6du3L48//jgXXXQRNTU1PPfcc1xwwQU89NBDXHjhhYwYMQIIWg1PP/30DWK56aabuPHGG0kmk+ywww7cddddVFZWbrCNZ555hnnz5vGtb32LZDJJY2Mjf/nLXxg+fDi9evVi7dq1TJo0ibVr17L77rtzwQUX8Oabb9KrVy9++MMf8t5772Xdl8x9vuqqq5piW7NmDXPmzGHs2LFN4/76179y5JFHMnDgQKZNm8ZPfvKTnO+NP//5z0ycOJEjjzyyadx+++2Xc7lc/vGPf/DUU08BcNJJJ7Hffvvx6183b+uZP38+++67L926daNbt27ssssuPPLIIxx77LGYWVNCtnr1arbeemsAKisrGTp0KK+88gp77LFHh2KMsztyG2Bx2vCScFzWedy9HlgN9M9ckZmdZmYzzWxmRzPjXPqUl7HP8C3YvGf3WLcjItJVNDQ08MQTTzBp0iQARowYwbPPPsvs2bOZOnVqsy/qqqoqpk+fzhtvvMH06dNZvHgxS5cu5eKLL+b555/nueeeY/789f/Lf/e73+Wkk05izpw5fOMb32jWKrNq1SpefPFFrr76aiZNmsQPfvAD5s2bxxtvvEFVVVXTfPvvvz/jxo1jwoQJOde5ZMkSXnjhBa666ipOO+00rr32WmbNmsWVV17JGWecAcDUqVN59NFHef3115kxYwbdu3dn6tSpTJ48maqqKiZPnszcuXPZfffdcx67o48+mldffZXXX3+dkSNHcsstt2TdBsANN9zA9773Paqqqpg5cyaDBw9utq4ZM2Y0tU5Onjy52bSW9iVzn9PNnDmTMWPGNBt37733MmXKFKZMmcK9996bc/+AyMdizZo1Td3GmT/p74mUjz/+mEGDBgGw1VZbZe3eHDt2LI888giJRILly5fz5JNPsnhxkLrcfPPNHH744QwePJi77rqL889fX1U1fvx4nn322Uj715qNornH3W8EbgQYP358rK1k44duzl2nTohzEyIinaotLVb5VFNTw7hx4/jggw8YOXIkBx10EBC0Kpx00km8++67mBl1dXVNyxxwwAH07dsXgFGjRvH++++zfPly9ttvPwYMGADA5MmTeeeddwB48cUX+etf/wrAiSeeyI9+9KOmdR155JGYGTvvvDMDBw5k5513BmD06NEsWrSIcePGAUF35BZbbNG0XGvr/PrXv05paSlr167lhRde4Otf/3rTtNraWgD22msvTj75ZI499liOPvroDh3DuXPn8tOf/pRPP/2UtWvXcsghh7S4jYkTJ/LLX/6SJUuWcPTRRzN8+PBI22htX9L3OdPSpUubzgkESc+7777L3nvvjZlRVlbG3LlzGTNmTNYrB9t6NWHv3r2bJc9tYWZZt3fwwQfz6quvsueeezJgwAAmTpzYtK9XX301Dz/8MBMmTOCKK67gnHPO4eabbwZgyy235K233mpXLOnibAn7ANg2bXhwOC7rPGbWDegLrIgxJhER6SSpVpf3338fd2+qCfvZz37G/vvvz9y5c3nwwQeb3Vsp1c0IQRddW2uv0qXWVVJS0my9JSUl7V5vz549AWhsbKRfv35NtWRVVVW8+eabQNAidemll7J48WJ23313VqzY8Gtt9OjRzJo1K+f2Tj75ZP7whz/wxhtvcPHFFzcdq2zbOP7445tauw4//HD+85//RNqn1vYlfZ8zVVRUNDt39913H6tWrWLYsGEMHTqURYsWNbWG9e/fv1lh/MqVK5sS36jHoq0tYQMHDmTp0qVAkDBuueWWWdd74YUXUlVVxWOPPYa7s+OOO7Js2TJef/31ptbRyZMnN6utS3Wld1ScSdirwHAzG2Zm3YHjgBkZ88wATgpffw34TyHrwUREJP8qKyu55ppr+O1vf0t9fT2rV69mm22C6pTWarNSJkyYwNNPP82KFSuoq6vj/vvvb5q25557Mm3aNADuuece9tlnnw7HG2Wdffr0YdiwYU2xuDuvv/46AO+99x4TJkxg6tSpDBgwgMWLF9O7d2/WrFnTtPx5553HZZdd1tSi19jYyA033LDBdtasWcOgQYOoq6vjnnvuaRqfbRsLFy5ku+224+yzz+aoo45qqsHLpbV9ac3IkSObrhiEoCvykUceYdGiRSxatIhZs2Y1Hcf99tuP6dOnk0wmgeC877///gAcf/zxvPDCCzz00ENN63rmmWeYO3dus+2lWsKy/YwalXndH0yaNIk77rgDgDvuuIOjjjpqg3kaGhqakuQ5c+YwZ84cDj74YDbbbDNWr17ddH4ee+wxRo4c2bTcO++8s0FXbHvEloSFNV5nAY8CbwL3ufs8M5tqZpPC2W4B+pvZAuAcYIPbWIiIyMZv1113ZZddduHee+/lRz/6ERdccAG77rprpBapQYMGcckllzBx4kT22muvZl+G1157Lbfddhu77LILd911F7///e87HGvUdd5zzz3ccsstjB07ltGjR/OPf/wDCBKsnXfemTFjxrDnnnsyduxY9t9/f+bPn8+4ceOYPn06u+yyC7/73e+YMmUKI0eOZMyYMVmvJvzFL37BhAkT2GuvvZqK+Fvaxn333ceYMWMYN24cc+fObdOVjy3tS2tGjBjB6tWrWbNmDYsWLeL9999vdmuKYcOG0bdvX15++WWOOOII9tlnH3bffXfGjRvH888/31QkX1FRwT//+U+uvfZahg8fzqhRo7j++uubdXW2x/nnn89jjz3G8OHDefzxx5tqumbOnMm3v/1tIHjG6j777MOoUaM47bTTuPvuu5uK9G+66SaOOeYYxo4dy1133cUVV1zRtO7nn3++qXu9I2xja3gaP368z5w5s9BhiIgUtTfffLNZsiISh6uvvprevXs3JTVdwezZs7nqqqu46667NpiW7e/OzGa5+/hs69Id80VERKRdTj/99Gb1dl3B8uXL+cUvfpGXdW0UV0eKiIhI8SkvL+fEE08sdBidKh/dkClqCRMR2URtbOUmIhuz9vy9KQkTEdkElZeXs2LFCiViIp3A3VmxYgXl5eVtWk7dkSIim6DBgwezZMmSDj9/T0SiKS8v3+ApBbkoCRMR2QSVlZUxbNiwQochIq1Qd6SIiIhIASgJExERESkAJWEiIiIiBbDR3THfzJYB78e8mS2A5TFvQ9pO56X46JwUJ52X4qNzUpw647x8zt2zPoNpo0vCOoOZzWzpEQNSODovxUfnpDjpvBQfnZPiVOjzou5IERERkQJQEiYiIiJSAErCsrux0AFIVjovxUfnpDjpvBQfnZPiVNDzopowERERkQJQS5iIiIhIAXTpJMzMDjWzt81sgZmdn2V6DzObHk5/2cyGFiDMLifCeTnHzOab2Rwze8LMPleIOLuSXOckbb5jzMzNTFeBxSzKOTGzY8O/lXlm9ufOjrErivD5NcTMnjSz2eFn2OGFiLMrMbNbzewTM5vbwnQzs2vCczbHzHbrrNi6bBJmZqXAdcBhwChgipmNypjtVGCVu+8AXA38unOj7HoinpfZwHh33wV4APhN50bZtUQ8J5hZb+B7wMudG2HXE+WcmNlw4AJgL3cfDXy/s+PsaiL+rfwUuM/ddwWOA67v3Ci7pNuBQ1uZfhgwPPw5DfhjJ8QEdOEkDNgDWODuC909CUwDjsqY5yjgjvD1A8ABZmadGGNXlPO8uPuT7p4IB18C2vbYemmrKH8rAL8g+EdlXWcG10VFOSf/D7jO3VcBuPsnnRxjVxTlvDjQJ3zdF/iwE+Prktz9GWBlK7McBdzpgZeAfmY2qDNi68pJ2DbA4rThJeG4rPO4ez2wGujfKdF1XVHOS7pTgX/FGpHkPCdh8/227v5QZwbWhUX5O9kR2NHMnjezl8ystZYAyY8o5+US4AQzWwI8DHy3c0KTVrT1eydvunXGRkTiYGYnAOOBLxY6lq7MzEqAq4CTCxyKNNeNoHtlP4LW4mfMbGd3/7SQQQlTgNvd/bdmNhG4y8zGuHtjoQOTzteVW8I+ALZNGx4cjss6j5l1I2g6XtEp0XVdUc4LZnYgcCEwyd1rOym2rirXOekNjAGeMrNFwBeAGSrOj1WUv5MlwAx3r3P3/wLvECRlEp8o5+VU4D4Ad38RKCd4fqEUTqTvnTh05STsVWC4mQ0zs+4EBZIzMuaZAZwUvv4a8B/XjdXilvO8mNmuwJ8IEjDVucSv1XPi7qvdfQt3H+ruQwnq9Ca5+8zChNslRPn8+jtBKxhmtgVB9+TCToyxK4pyXv4HHABgZiMJkrBlnRqlZJoBfDO8SvILwGp3X9oZG+6y3ZHuXm9mZwGPAqXAre4+z8ymAjPdfQZwC0FT8QKCor7jChdx1xDxvFwB9ALuD6+T+J+7TypY0Ju4iOdEOlHEc/IocLCZzQcagPPcXS35MYp4Xs4FbjKzHxAU6Z+sf+7jZWb3EvxDskVYi3cxUAbg7jcQ1OYdDiwAEsC3Oi02nXsRERGRzteVuyNFRERECkZJmIiIiEgBKAkTERERKQAlYSIiIiIFoCRMREREpACUhImIiIgUgJIwkSJnZg1mVpX2M7SVedfmYXu3m9l/w229Fj5apa3ruNnMRoWvf5Ix7YWOxhiuJ3Vc5prZg2bWL8f848zs8HZsZ5CZ/TN8vZ+ZrQ63+6aZXdyO9U0ys/PD119JHadweGr4NIgOCc/h13LM81RbnmoQ7vs/I8x3q5l9YmZzM8ZfaWZfiro9ka5ASZhI8atx93FpP4s6YZvnufs44HyCpxO0ibt/293nh4M/yZi2Z8fDA9YflzEEN1M+M8f84whuyNhW5wA3pQ0/Gx6b8QQPYt6tLStz9xnufnk4+BVgVNq0i9z98XbEWExuB7I9LPxagveTiISUhIlsZMysl5k9EbZSvWFmR2WZZ5CZPZPWUrRPOP5gM3sxXPZ+M+uVY3PPADuEy54TrmuumX0/HNfTzB4ys9fD8ZPD8U+Z2XgzuxyoCOO4J5y2Nvw9zcy+nBbz7Wb2NTMrNbMrzOxVM5tjZv8X4bC8CGwTrmePcB9nm9kLZrZT+AiZqcDkMJbJYey3mtkr4bwbHMfQMcAjmSPdvRqYBewQtrK9FMb7NzPbLIzlbDObH46fFo472cz+YGZ7ApOAK8KYtk87Boea2f1px6apFaqt59DMLgqP5Vwzu9EseMxE6MS098ge4fxRj0tW7v4MQVKcOf59oL+ZbdWW9YlsypSEiRS/VBJTZWZ/A9YBX3X33YD9gd9mfLECHA88GrbYjAWqLHh+4E+BA8NlZxK08rTmSOANM9ud4FEeEwge0P3/LHiG56HAh+4+NmyRapasuPv5rG+x+kbGuqcDxwKESdIBwEMEDzhe7e6fBz4fbmtYSwGaWWm4bOrxSW8B+7j7rsBFwGXungxfTw9jmU7wAPj/uPseBMfxCjPrmbHuYcCqbA+JN7P+4bGYB9wJ/NjddwHeIHgsCgQtP7uG47+TcWxeCGM+L4zpvbTJjwMT0uKZDExr5zn8g7t/Pjw/FcARadMqw/fIGcCt4bgox2W8md2cY7vZvAbs1Y7lRDZJXfbZkSIbkZrwixIAMysDLjOzfYFGghaggcBHacu8Ctwazvt3d68ysy8SdH09H+Zs3QlakLK5wsx+SvBg4VMJkpy/ha0/mNlfgX0Ikq7fmtmvgX+6+7Nt2K9/Ab83sx4Eydwz7l5jZgcDu9j6mqa+wHDgvxnLV5hZVbj/bwKPpc1/h5kNJ3g2X1kL2z8YmGRmPwyHy4Eh4bpSBrHhw5X3MbPZBMf+cmAJ0M/dnw6n3wGkWrHmAPeY2d8JHqgdSfgMwkeAI83sAeDLwI+AtpzDlP3N7EdAJbA5QdL4YDjt3nB7z5hZHwvq6lo6LunxzQS+HXV/0nwCbN2O5UQ2SUrCRDY+3wAGALu7e52ZLSL4omwSfqnuS/DlfbuZXQWsAh5z9ykRtnGeuz+QGjCzA7LN5O7vWFATdThwqZk94e5To+yEu68zs6eAQwhbelKbA77r7o/mWEWNu48zs0qCByafCVwD/AJ40t2/asFFDE+1sLwBx7j7261tg4xjS1AT1tSaZGZ9W1n+y8C+BC2KF5rZzq3Mm2kacBZB195Md18TtnhGPYeYWTlwPTDe3Reb2SU035/Mhwc7LRwXMxvYhthbUk5wTEUEdUeKbIz6Ap+ECdj+wOcyZzCzzwEfu/tNwM3AbsBLwF5mlqrx6mlmO0bc5rPAV8ysMuya+irwrJltDSTc/W7ginA7merCFrlsphN0c6Za1SBIqE5PLWNmO2Z2h6Vz9wRwNnCumXUjOD4fhJNPTpt1DdA7bfhR4LuprtywezXTO8DQlrYdbn81sMrCujvgROBpMysBtnX3J4Efh3Fl1m9lxpTuaYLj+f9Yn6C29RymEq7lYe1Y5hWTqRq+vQm6gFcT7bi0147A3JxziXQRSsJENj73AOPN7A3gmwQ1UJn2A14Pu80mA79392UEScm9ZjaHoBtrRJQNuvtrBFe9vQK8DNzs7rOBnYFXwm7Bi4FLsyx+IzDHwsL8DP8m6GJ7PKzbgiBpnA+8ZsFtDv5Ejlb7MJY5wBTgN8Cvwn1PX+5JYFRYWzeZoMWsLIxtXjicud5q4L1U0tOKkwi6cOcQXIU5FSgF7g7P02zgGnf/NGO5acB5YQH89hnbbgD+CRwW/qat5zDc3k0Eic+jBN3U6daFx+kGgm5niHBcWqsJM7N7w7h2MrMlZnZqOL6M4CKPmS3FK9LVmHtma7SIiKSY2VcJun5/WuhYNmbhcdzN3X9W6FhEioVqwkREWuHufwuvhJSO6Qb8ttBBiBQTtYSJiIiIFIBqwkREREQKQEmYiIiISAEoCRMREREpACVhIiIiIgWgJExERESkAP4/+KFB8iNdebwAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Compute ROC curve\n",
    "fig, ax = plt.subplots(figsize=(10, 6))\n",
    "RocCurveDisplay.from_estimator(ranfor, X_test, y_test, ax=ax)\n",
    "plt.title('ROC Curve for the Breast Cancer Classifier')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9035191562634525"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate probability scores \n",
    "y_scores = cross_val_predict(ranfor, X_test, y_test, cv=3, method='predict_proba')\n",
    "# Because of the structure of how the model returns the y_scores, we need to convert them into binary values\n",
    "y_scores_binary = [1 if x[0] < 0.5 else 0 for x in y_scores]\n",
    "\n",
    "# Now, we can calculate the area under the ROC curve\n",
    "auc = roc_auc_score(y_test, y_scores_binary, average=\"macro\")\n",
    "auc # Be aware that due to the random nature of cross validation, the results will change when you run the code"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "2563e9ff0724e60438e7d4bc88917de92dced889a710d60bd5f36a42c74c3553"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 ('sparkenv')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.11"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}