Random forest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>job</th>\n",
       "      <th>marital</th>\n",
       "      <th>education</th>\n",
       "      <th>default</th>\n",
       "      <th>balance</th>\n",
       "      <th>housing</th>\n",
       "      <th>loan</th>\n",
       "      <th>contact</th>\n",
       "      <th>day</th>\n",
       "      <th>month</th>\n",
       "      <th>duration</th>\n",
       "      <th>campaign</th>\n",
       "      <th>pdays</th>\n",
       "      <th>previous</th>\n",
       "      <th>poutcome</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>58</td>\n",
       "      <td>management</td>\n",
       "      <td>married</td>\n",
       "      <td>tertiary</td>\n",
       "      <td>no</td>\n",
       "      <td>2143</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>may</td>\n",
       "      <td>261</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>44</td>\n",
       "      <td>technician</td>\n",
       "      <td>single</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>29</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>may</td>\n",
       "      <td>151</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>33</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>2</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>may</td>\n",
       "      <td>76</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>47</td>\n",
       "      <td>blue-collar</td>\n",
       "      <td>married</td>\n",
       "      <td>unknown</td>\n",
       "      <td>no</td>\n",
       "      <td>1506</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>may</td>\n",
       "      <td>92</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>33</td>\n",
       "      <td>unknown</td>\n",
       "      <td>single</td>\n",
       "      <td>unknown</td>\n",
       "      <td>no</td>\n",
       "      <td>1</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>may</td>\n",
       "      <td>198</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age           job  marital  education default  balance housing loan  \\\n",
       "0   58    management  married   tertiary      no     2143     yes   no   \n",
       "1   44    technician   single  secondary      no       29     yes   no   \n",
       "2   33  entrepreneur  married  secondary      no        2     yes  yes   \n",
       "3   47   blue-collar  married    unknown      no     1506     yes   no   \n",
       "4   33       unknown   single    unknown      no        1      no   no   \n",
       "\n",
       "   contact  day month  duration  campaign  pdays  previous poutcome   y  \n",
       "0  unknown    5   may       261         1     -1         0  unknown  no  \n",
       "1  unknown    5   may       151         1     -1         0  unknown  no  \n",
       "2  unknown    5   may        76         1     -1         0  unknown  no  \n",
       "3  unknown    5   may        92         1     -1         0  unknown  no  \n",
       "4  unknown    5   may       198         1     -1         0  unknown  no  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn import *\n",
    "import numpy as np\n",
    "\n",
    "df=pd.read_csv(\"/data/bank-full.csv\",\";\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "unknown    36959\n",
       "failure     4901\n",
       "other       1840\n",
       "success     1511\n",
       "Name: poutcome, dtype: int64"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.poutcome.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    " #dependent variable/target varible/label\n",
    "label = \"y\"\n",
    "\n",
    "# independepent variables/features/predictors\n",
    "X = df.drop(columns=[label,\"day\",\"pdays\"])\n",
    "\n",
    "# vector for target variable\n",
    "y = df[label]\n",
    "\n",
    "# one hot encoding to conver categorical features into numeric\n",
    "# drop_first: remove first value of a categorical feature ... it is redundant\n",
    "X = pd.get_dummies(X, drop_first=True)\n",
    "\n",
    "# Creating trainnig and test sets\n",
    "# test_size is 30% of the whole\n",
    "# random_state: to reprduce the same combination of training and test records\n",
    "X_train, X_test, y_train, y_test = model_selection.train_test_split(X.values,                       \n",
    "                                    y,  test_size = 0.3, random_state = 1)\n",
    "\n",
    "\n",
    "pipe = pipeline.Pipeline([\n",
    "    (\"poly\", preprocessing.PolynomialFeatures(degree=1, include_bias=False)),\n",
    "    (\"scaler\", preprocessing.StandardScaler()),\n",
    "    (\"est\", tree.DecisionTreeClassifier(max_depth=5, random_state=1))\n",
    "])\n",
    "\n",
    "pipe.fit(X_train, y_train)\n",
    "\n",
    "# prediction on training and test data\n",
    "y_train_pred = pipe.predict(X_train)\n",
    "y_test_pred = pipe.predict(X_test)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[11725   288]\n",
      " [ 1029   522]]\n",
      "Training accuracy 0.904098334755269\n",
      "Test accuracy 0.9029047478619876\n"
     ]
    }
   ],
   "source": [
    "print(metrics.confusion_matrix(y_test,y_test_pred))\n",
    "print(\"Training accuracy\", metrics.accuracy_score(y_train, y_train_pred))\n",
    "print(\"Test accuracy\", metrics.accuracy_score(y_test, y_test_pred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=5,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=1,\n",
       "            splitter='best')"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "est = pipe.steps[-1][-1]\n",
    "est"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dot: graph is too large for cairo-renderer bitmaps. Scaling by 0.78949 to fit\r\n"
     ]
    }
   ],
   "source": [
    "from sklearn.tree import export_graphviz\n",
    "export_graphviz(est, out_file = \"tree.dot\", feature_names = X.columns, filled=True)\n",
    "!dot -Tpng tree.dot -o tree.png\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>duration</td>\n",
       "      <td>0.557299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>poutcome_success</td>\n",
       "      <td>0.288661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>age</td>\n",
       "      <td>0.047522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>month_mar</td>\n",
       "      <td>0.034552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>month_oct</td>\n",
       "      <td>0.032319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>contact_unknown</td>\n",
       "      <td>0.022191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>month_may</td>\n",
       "      <td>0.005642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>housing_yes</td>\n",
       "      <td>0.004479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>education_unknown</td>\n",
       "      <td>0.001582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>month_dec</td>\n",
       "      <td>0.001511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>month_jun</td>\n",
       "      <td>0.001353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>campaign</td>\n",
       "      <td>0.001096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>month_nov</td>\n",
       "      <td>0.000923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>balance</td>\n",
       "      <td>0.000871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>job_retired</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>job_housemaid</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>poutcome_other</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>month_sep</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>previous</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>job_blue-collar</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>job_entrepreneur</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>month_jul</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>month_jan</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>month_feb</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>month_aug</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>contact_telephone</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>job_self-employed</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>loan_yes</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>job_management</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>default_yes</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>education_tertiary</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>education_secondary</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>marital_single</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>marital_married</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>job_unknown</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>job_unemployed</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>job_technician</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>job_student</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>job_services</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>poutcome_unknown</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                feature  importance\n",
       "2              duration    0.557299\n",
       "38     poutcome_success    0.288661\n",
       "0                   age    0.047522\n",
       "32            month_mar    0.034552\n",
       "35            month_oct    0.032319\n",
       "25      contact_unknown    0.022191\n",
       "33            month_may    0.005642\n",
       "22          housing_yes    0.004479\n",
       "20    education_unknown    0.001582\n",
       "27            month_dec    0.001511\n",
       "31            month_jun    0.001353\n",
       "3              campaign    0.001096\n",
       "34            month_nov    0.000923\n",
       "1               balance    0.000871\n",
       "9           job_retired    0.000000\n",
       "7         job_housemaid    0.000000\n",
       "37       poutcome_other    0.000000\n",
       "36            month_sep    0.000000\n",
       "4              previous    0.000000\n",
       "5       job_blue-collar    0.000000\n",
       "6      job_entrepreneur    0.000000\n",
       "30            month_jul    0.000000\n",
       "29            month_jan    0.000000\n",
       "28            month_feb    0.000000\n",
       "26            month_aug    0.000000\n",
       "24    contact_telephone    0.000000\n",
       "10    job_self-employed    0.000000\n",
       "23             loan_yes    0.000000\n",
       "8        job_management    0.000000\n",
       "21          default_yes    0.000000\n",
       "19   education_tertiary    0.000000\n",
       "18  education_secondary    0.000000\n",
       "17       marital_single    0.000000\n",
       "16      marital_married    0.000000\n",
       "15          job_unknown    0.000000\n",
       "14       job_unemployed    0.000000\n",
       "13       job_technician    0.000000\n",
       "12          job_student    0.000000\n",
       "11         job_services    0.000000\n",
       "39     poutcome_unknown    0.000000"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "importance = pd.DataFrame({\"feature\": X.columns\n",
    "                , \"importance\":  est.feature_importances_})\n",
    "importance.sort_values(\"importance\", ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    " #dependent variable/target varible/label\n",
    "label = \"y\"\n",
    "\n",
    "# independepent variables/features/predictors\n",
    "X = df.drop(columns=[label,\"day\",\"pdays\"])\n",
    "\n",
    "# vector for target variable\n",
    "y = df[label]\n",
    "\n",
    "# one hot encoding to conver categorical features into numeric\n",
    "# drop_first: remove first value of a categorical feature ... it is redundant\n",
    "X = pd.get_dummies(X, drop_first=True)\n",
    "\n",
    "# Creating trainnig and test sets\n",
    "# test_size is 30% of the whole\n",
    "# random_state: to reprduce the same combination of training and test records\n",
    "X_train, X_test, y_train, y_test = model_selection.train_test_split(X.values,                       \n",
    "                                    y,  test_size = 0.3, random_state = 1)\n",
    "\n",
    "\n",
    "pipe = pipeline.Pipeline([\n",
    "    (\"poly\", preprocessing.PolynomialFeatures(degree=1, include_bias=False)),\n",
    "    (\"scaler\", preprocessing.StandardScaler()),\n",
    "    (\"est\", ensemble.RandomForestClassifier(n_estimators=20,max_features=0.8\n",
    "                                        , max_depth=10, random_state=1))\n",
    "])\n",
    "\n",
    "pipe.fit(X_train, y_train)\n",
    "\n",
    "# prediction on training and test data\n",
    "y_train_pred = pipe.predict(X_train)\n",
    "y_test_pred = pipe.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=10,\n",
       "             max_features=0.8, max_leaf_nodes=None,\n",
       "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "             min_samples_leaf=1, min_samples_split=2,\n",
       "             min_weight_fraction_leaf=0.0, presort=False,\n",
       "             random_state=1791095845, splitter='best'),\n",
       " DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=10,\n",
       "             max_features=0.8, max_leaf_nodes=None,\n",
       "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "             min_samples_leaf=1, min_samples_split=2,\n",
       "             min_weight_fraction_leaf=0.0, presort=False,\n",
       "             random_state=2135392491, splitter='best'),\n",
       " DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=10,\n",
       "             max_features=0.8, max_leaf_nodes=None,\n",
       "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "             min_samples_leaf=1, min_samples_split=2,\n",
       "             min_weight_fraction_leaf=0.0, presort=False,\n",
       "             random_state=946286476, splitter='best'),\n",
       " DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=10,\n",
       "             max_features=0.8, max_leaf_nodes=None,\n",
       "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "             min_samples_leaf=1, min_samples_split=2,\n",
       "             min_weight_fraction_leaf=0.0, presort=False,\n",
       "             random_state=1857819720, splitter='best'),\n",
       " DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=10,\n",
       "             max_features=0.8, max_leaf_nodes=None,\n",
       "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "             min_samples_leaf=1, min_samples_split=2,\n",
       "             min_weight_fraction_leaf=0.0, presort=False,\n",
       "             random_state=491263, splitter='best'),\n",
       " DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=10,\n",
       "             max_features=0.8, max_leaf_nodes=None,\n",
       "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "             min_samples_leaf=1, min_samples_split=2,\n",
       "             min_weight_fraction_leaf=0.0, presort=False,\n",
       "             random_state=550290313, splitter='best'),\n",
       " DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=10,\n",
       "             max_features=0.8, max_leaf_nodes=None,\n",
       "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "             min_samples_leaf=1, min_samples_split=2,\n",
       "             min_weight_fraction_leaf=0.0, presort=False,\n",
       "             random_state=1298508491, splitter='best'),\n",
       " DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=10,\n",
       "             max_features=0.8, max_leaf_nodes=None,\n",
       "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "             min_samples_leaf=1, min_samples_split=2,\n",
       "             min_weight_fraction_leaf=0.0, presort=False,\n",
       "             random_state=2143362693, splitter='best'),\n",
       " DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=10,\n",
       "             max_features=0.8, max_leaf_nodes=None,\n",
       "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "             min_samples_leaf=1, min_samples_split=2,\n",
       "             min_weight_fraction_leaf=0.0, presort=False,\n",
       "             random_state=630311759, splitter='best'),\n",
       " DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=10,\n",
       "             max_features=0.8, max_leaf_nodes=None,\n",
       "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "             min_samples_leaf=1, min_samples_split=2,\n",
       "             min_weight_fraction_leaf=0.0, presort=False,\n",
       "             random_state=1013994432, splitter='best'),\n",
       " DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=10,\n",
       "             max_features=0.8, max_leaf_nodes=None,\n",
       "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "             min_samples_leaf=1, min_samples_split=2,\n",
       "             min_weight_fraction_leaf=0.0, presort=False,\n",
       "             random_state=396591248, splitter='best'),\n",
       " DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=10,\n",
       "             max_features=0.8, max_leaf_nodes=None,\n",
       "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "             min_samples_leaf=1, min_samples_split=2,\n",
       "             min_weight_fraction_leaf=0.0, presort=False,\n",
       "             random_state=1703301249, splitter='best'),\n",
       " DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=10,\n",
       "             max_features=0.8, max_leaf_nodes=None,\n",
       "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "             min_samples_leaf=1, min_samples_split=2,\n",
       "             min_weight_fraction_leaf=0.0, presort=False,\n",
       "             random_state=799981516, splitter='best'),\n",
       " DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=10,\n",
       "             max_features=0.8, max_leaf_nodes=None,\n",
       "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "             min_samples_leaf=1, min_samples_split=2,\n",
       "             min_weight_fraction_leaf=0.0, presort=False,\n",
       "             random_state=1666063943, splitter='best'),\n",
       " DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=10,\n",
       "             max_features=0.8, max_leaf_nodes=None,\n",
       "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "             min_samples_leaf=1, min_samples_split=2,\n",
       "             min_weight_fraction_leaf=0.0, presort=False,\n",
       "             random_state=1484172013, splitter='best'),\n",
       " DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=10,\n",
       "             max_features=0.8, max_leaf_nodes=None,\n",
       "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "             min_samples_leaf=1, min_samples_split=2,\n",
       "             min_weight_fraction_leaf=0.0, presort=False,\n",
       "             random_state=729053692, splitter='best'),\n",
       " DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=10,\n",
       "             max_features=0.8, max_leaf_nodes=None,\n",
       "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "             min_samples_leaf=1, min_samples_split=2,\n",
       "             min_weight_fraction_leaf=0.0, presort=False,\n",
       "             random_state=1704103302, splitter='best'),\n",
       " DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=10,\n",
       "             max_features=0.8, max_leaf_nodes=None,\n",
       "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "             min_samples_leaf=1, min_samples_split=2,\n",
       "             min_weight_fraction_leaf=0.0, presort=False,\n",
       "             random_state=1870626073, splitter='best'),\n",
       " DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=10,\n",
       "             max_features=0.8, max_leaf_nodes=None,\n",
       "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "             min_samples_leaf=1, min_samples_split=2,\n",
       "             min_weight_fraction_leaf=0.0, presort=False,\n",
       "             random_state=166716594, splitter='best'),\n",
       " DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=10,\n",
       "             max_features=0.8, max_leaf_nodes=None,\n",
       "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "             min_samples_leaf=1, min_samples_split=2,\n",
       "             min_weight_fraction_leaf=0.0, presort=False,\n",
       "             random_state=1487394068, splitter='best')]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "est = pipe.steps[-1][-1]\n",
    "est\n",
    "est.estimators_\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>duration</td>\n",
       "      <td>0.420481</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>poutcome_success</td>\n",
       "      <td>0.172472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>age</td>\n",
       "      <td>0.079567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>balance</td>\n",
       "      <td>0.057749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>housing_yes</td>\n",
       "      <td>0.027378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>contact_unknown</td>\n",
       "      <td>0.023816</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>campaign</td>\n",
       "      <td>0.021752</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>month_oct</td>\n",
       "      <td>0.020552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>previous</td>\n",
       "      <td>0.020345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>month_mar</td>\n",
       "      <td>0.020303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>month_jun</td>\n",
       "      <td>0.018509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>month_sep</td>\n",
       "      <td>0.015222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>month_may</td>\n",
       "      <td>0.010117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>marital_married</td>\n",
       "      <td>0.006450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>month_nov</td>\n",
       "      <td>0.006198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>education_tertiary</td>\n",
       "      <td>0.006029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>job_technician</td>\n",
       "      <td>0.005790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>job_management</td>\n",
       "      <td>0.005501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>loan_yes</td>\n",
       "      <td>0.004901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>education_secondary</td>\n",
       "      <td>0.004429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>contact_telephone</td>\n",
       "      <td>0.004386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>marital_single</td>\n",
       "      <td>0.004133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>job_blue-collar</td>\n",
       "      <td>0.003875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>month_jul</td>\n",
       "      <td>0.003822</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>poutcome_unknown</td>\n",
       "      <td>0.003670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>month_aug</td>\n",
       "      <td>0.003497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>month_feb</td>\n",
       "      <td>0.003337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>job_student</td>\n",
       "      <td>0.003105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>month_jan</td>\n",
       "      <td>0.002968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>education_unknown</td>\n",
       "      <td>0.002858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>job_services</td>\n",
       "      <td>0.002414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>job_self-employed</td>\n",
       "      <td>0.002381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>month_dec</td>\n",
       "      <td>0.002166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>job_entrepreneur</td>\n",
       "      <td>0.001667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>job_unemployed</td>\n",
       "      <td>0.001666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>job_housemaid</td>\n",
       "      <td>0.001620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>poutcome_other</td>\n",
       "      <td>0.001610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>job_retired</td>\n",
       "      <td>0.001589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>default_yes</td>\n",
       "      <td>0.001072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>job_unknown</td>\n",
       "      <td>0.000603</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                feature  importance\n",
       "2              duration    0.420481\n",
       "38     poutcome_success    0.172472\n",
       "0                   age    0.079567\n",
       "1               balance    0.057749\n",
       "22          housing_yes    0.027378\n",
       "25      contact_unknown    0.023816\n",
       "3              campaign    0.021752\n",
       "35            month_oct    0.020552\n",
       "4              previous    0.020345\n",
       "32            month_mar    0.020303\n",
       "31            month_jun    0.018509\n",
       "36            month_sep    0.015222\n",
       "33            month_may    0.010117\n",
       "16      marital_married    0.006450\n",
       "34            month_nov    0.006198\n",
       "19   education_tertiary    0.006029\n",
       "13       job_technician    0.005790\n",
       "8        job_management    0.005501\n",
       "23             loan_yes    0.004901\n",
       "18  education_secondary    0.004429\n",
       "24    contact_telephone    0.004386\n",
       "17       marital_single    0.004133\n",
       "5       job_blue-collar    0.003875\n",
       "30            month_jul    0.003822\n",
       "39     poutcome_unknown    0.003670\n",
       "26            month_aug    0.003497\n",
       "28            month_feb    0.003337\n",
       "12          job_student    0.003105\n",
       "29            month_jan    0.002968\n",
       "20    education_unknown    0.002858\n",
       "11         job_services    0.002414\n",
       "10    job_self-employed    0.002381\n",
       "27            month_dec    0.002166\n",
       "6      job_entrepreneur    0.001667\n",
       "14       job_unemployed    0.001666\n",
       "7         job_housemaid    0.001620\n",
       "37       poutcome_other    0.001610\n",
       "9           job_retired    0.001589\n",
       "21          default_yes    0.001072\n",
       "15          job_unknown    0.000603"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "importance = pd.DataFrame({\"feature\": X.columns\n",
    "                , \"importance\":  est.feature_importances_})\n",
    "importance.sort_values(\"importance\", ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}