examples/04_sklearn_api_regression_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# StackingTransformer (scikit-learn API for stacking)\n",
    "# Regression\n",
    "# 2-level stacking: step-by-step and using Pipeline\n",
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.datasets import load_boston\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.ensemble import ExtraTreesRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.externals import joblib\n",
    "from xgboost import XGBRegressor\n",
    "from vecstack import StackingTransformer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "boston = load_boston()\n",
    "X, y = boston.data, boston.target\n",
    "\n",
    "# Make train/test split\n",
    "# As usual in machine learning task we have X_train, y_train, and X_test\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialize 1st level estimators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Caution! All estimators and parameter values are just \n",
    "# demonstrational and shouldn't be considered as recommended.\n",
    "\n",
    "# This is list of tuples\n",
    "# Each tuple contains arbitrary unique name and estimator object\n",
    "estimators_L1 = [\n",
    "    ('et', ExtraTreesRegressor(random_state=0, n_jobs=-1, \n",
    "                               n_estimators=100, max_depth=3)),\n",
    "        \n",
    "    ('rf', RandomForestRegressor(random_state=0, n_jobs=-1, \n",
    "                                 n_estimators=100, max_depth=3)),\n",
    "        \n",
    "    ('xgb', XGBRegressor(random_state=0, n_jobs=-1, learning_rate=0.1, \n",
    "                         n_estimators=100, max_depth=3))\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialize StackingTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "stack = StackingTransformer(estimators=estimators_L1,   # base estimators\n",
    "                            regression=True,            # regression task (if you need \n",
    "                                                        #     classification - set to False)\n",
    "                            variant='A',                # oof for train set, predict test \n",
    "                                                        #     set in each fold and find mean\n",
    "                            metric=mean_absolute_error, # metric: callable\n",
    "                            n_folds=4,                  # number of folds\n",
    "                            shuffle=True,               # shuffle the data\n",
    "                            random_state=0,             # ensure reproducibility\n",
    "                            verbose=2)                  # print all info"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Step-by-step approach"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fit StackingTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "task:         [regression]\n",
      "metric:       [mean_absolute_error]\n",
      "variant:      [A]\n",
      "n_estimators: [3]\n",
      "\n",
      "estimator  0: [et: ExtraTreesRegressor]\n",
      "    fold  0:  [3.20733439]\n",
      "    fold  1:  [2.87943130]\n",
      "    fold  2:  [2.53026486]\n",
      "    fold  3:  [2.83618694]\n",
      "    ----\n",
      "    MEAN:     [2.86330437] + [0.23993093]\n",
      "\n",
      "estimator  1: [rf: RandomForestRegressor]\n",
      "    fold  0:  [3.11110485]\n",
      "    fold  1:  [2.78404210]\n",
      "    fold  2:  [2.55707729]\n",
      "    fold  3:  [2.32209992]\n",
      "    ----\n",
      "    MEAN:     [2.69358104] + [0.29117900]\n",
      "\n",
      "estimator  2: [xgb: XGBRegressor]\n",
      "    fold  0:  [2.40318942]\n",
      "    fold  1:  [2.37286943]\n",
      "    fold  2:  [1.89121526]\n",
      "    fold  3:  [1.95382805]\n",
      "    ----\n",
      "    MEAN:     [2.15527554] + [0.23404984]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "stack = stack.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get stacked features: transform (predict) train set and test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set was detected.\n",
      "Transforming...\n",
      "\n",
      "estimator  0: [et: ExtraTreesRegressor]\n",
      "    model from fold  0: done\n",
      "    model from fold  1: done\n",
      "    model from fold  2: done\n",
      "    model from fold  3: done\n",
      "    ----\n",
      "    DONE\n",
      "\n",
      "estimator  1: [rf: RandomForestRegressor]\n",
      "    model from fold  0: done\n",
      "    model from fold  1: done\n",
      "    model from fold  2: done\n",
      "    model from fold  3: done\n",
      "    ----\n",
      "    DONE\n",
      "\n",
      "estimator  2: [xgb: XGBRegressor]\n",
      "    model from fold  0: done\n",
      "    model from fold  1: done\n",
      "    model from fold  2: done\n",
      "    model from fold  3: done\n",
      "    ----\n",
      "    DONE\n",
      "\n"
     ]
    }
   ],
   "source": [
    "S_train = stack.transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transforming...\n",
      "\n",
      "estimator  0: [et: ExtraTreesRegressor]\n",
      "    model from fold  0: done\n",
      "    model from fold  1: done\n",
      "    model from fold  2: done\n",
      "    model from fold  3: done\n",
      "    ----\n",
      "    DONE\n",
      "\n",
      "estimator  1: [rf: RandomForestRegressor]\n",
      "    model from fold  0: done\n",
      "    model from fold  1: done\n",
      "    model from fold  2: done\n",
      "    model from fold  3: done\n",
      "    ----\n",
      "    DONE\n",
      "\n",
      "estimator  2: [xgb: XGBRegressor]\n",
      "    model from fold  0: done\n",
      "    model from fold  1: done\n",
      "    model from fold  2: done\n",
      "    model from fold  3: done\n",
      "    ----\n",
      "    DONE\n",
      "\n"
     ]
    }
   ],
   "source": [
    "S_test = stack.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Look at the result\n",
    "\n",
    "Let's look at our stacked features.  \n",
    "We have three 1st level estimators, so we expect to get three columns in `S_train` and `S_test`.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[27.21782522, 28.23561508, 27.78520966],\n",
       "       [22.25443115, 22.32927929, 22.57203102],\n",
       "       [26.03879794, 25.80114661, 26.27923012],\n",
       "       [21.82927308, 21.30478775, 21.39201546],\n",
       "       [13.02143285, 12.04667683,  8.88440514]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "S_train[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[24.89602382, 23.85490698, 24.85046005],\n",
       "       [20.85135955, 25.05068336, 26.30952454],\n",
       "       [23.13164045, 21.56864103, 23.67526102],\n",
       "       [13.47709586, 11.81606315, 11.02050447],\n",
       "       [21.93179664, 21.30652111, 21.75125122]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "S_test[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Apply 2nd level estimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final prediction score: [2.78409081]\n"
     ]
    }
   ],
   "source": [
    "# Initialize 2nd level estimator\n",
    "final_estimator = XGBRegressor(random_state=0, n_jobs=-1, learning_rate=0.1, \n",
    "                               n_estimators=100, max_depth=3)\n",
    "\n",
    "# Fit\n",
    "final_estimator = final_estimator.fit(S_train, y_train)\n",
    "\n",
    "# Predict\n",
    "y_pred = final_estimator.predict(S_test)\n",
    "\n",
    "# Final prediction score\n",
    "print('Final prediction score: [%.8f]' % mean_absolute_error(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Some useful StackingTransformer attributes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of base estimators\n",
    "# Type: int\n",
    "stack.n_estimators_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3.20733439, 2.8794313 , 2.53026486, 2.83618694],\n",
       "       [3.11110485, 2.7840421 , 2.55707729, 2.32209992],\n",
       "       [2.40318942, 2.37286943, 1.89121526, 1.95382805]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Scores for each estimator (rows) in each fold (columns)\n",
    "# Type: 2d numpy array\n",
    "stack.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('et', 2.8633043735634116, 0.23993092887498238),\n",
       " ('rf', 2.6935810393014306, 0.2911789973137302),\n",
       " ('xgb', 2.15527553747196, 0.23404984189134637)]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Mean and std for each estimator\n",
    "# Type: list of tuples\n",
    "stack.mean_std_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>xgb</td>\n",
       "      <td>2.155276</td>\n",
       "      <td>0.234050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>rf</td>\n",
       "      <td>2.693581</td>\n",
       "      <td>0.291179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>et</td>\n",
       "      <td>2.863304</td>\n",
       "      <td>0.239931</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  name      mean       std\n",
       "2  xgb  2.155276  0.234050\n",
       "1   rf  2.693581  0.291179\n",
       "0   et  2.863304  0.239931"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Mean and std convenient representation using pandas.DataFrame\n",
    "df = pd.DataFrame.from_records(stack.mean_std_, columns=['name', 'mean', 'std'])\n",
    "# Sort by column 'mean' (best on the top)\n",
    "df.sort_values('mean', ascending=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Pipeline\n",
    "\n",
    "StackingTransformer is fully scikit-learn compatible so we can easily implement **arbitrary number of stacking levels** using Pipeline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specify steps of Pipeline\n",
    "steps = [('stack', stack),\n",
    "         ('final_estimator', final_estimator)]\n",
    "\n",
    "# Init Pipeline\n",
    "pipe = Pipeline(steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If we have several stacking levels our Pipeline steps would be:\n",
    "# steps = [('stack_L1', stack_L1),\n",
    "#          ('stack_L2', stack_L2),\n",
    "#          ('stack_L99', stack_L99), # :-)\n",
    "#          ('final_estimator', final_estimator)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ability to set parameters of nested estimators and transformers\n",
    "\n",
    "Following scikit-learn naming convention we can access parameters of nested estimators and transformers.  \n",
    "Each nested level (should not be confused with stacking level) is separated by double underscore `__`.  \n",
    "For example using Pipeline we want StackingTransformer to be silent.  \n",
    "We can access StackingTransformer parameter `verbose` through `pipe` instance as `stack__verbose`.  \n",
    "We can also set any parameter of 1st level estimators inside StackingTransformer.  \n",
    "We can access XGBoost parameter `learning_rate` through `pipe` instance as `stack__xgb__learning_rate`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipe = pipe.set_params(stack__verbose=0)\n",
    "# pipe = pipe.set_params(stack__xgb__learning_rate=0.555)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fit and predict using Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final prediction score using Pipeline: [2.78409081]\n"
     ]
    }
   ],
   "source": [
    "# Fit\n",
    "pipe = pipe.fit(X_train, y_train)\n",
    "\n",
    "# Predict\n",
    "y_pred_pipe = pipe.predict(X_test)\n",
    "\n",
    "# Final prediction score\n",
    "print('Final prediction score using Pipeline: [%.8f]' % mean_absolute_error(y_test, y_pred_pipe))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ability to save fitted Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save Pipeline\n",
    "_ = joblib.dump(pipe, 'pipe_with_stack.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final prediction score using loaded Pipeline: [2.78409081]\n"
     ]
    }
   ],
   "source": [
    "# Load Pipeline\n",
    "pipe_loaded = joblib.load('pipe_with_stack.pkl')\n",
    "\n",
    "# Predict using loaded Pipeline\n",
    "y_pred_pipe_loaded = pipe_loaded.predict(X_test)\n",
    "\n",
    "# Final prediction score\n",
    "print('Final prediction score using loaded Pipeline: [%.8f]' % mean_absolute_error(y_test, y_pred_pipe_loaded))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Conclusion\n",
    "\n",
    "Step-by-step approach is useful when we need direct access to stacked features `S_train` and `S_test`.  \n",
    "Pipeline approach is useful when we need to represent many steps in a single object.  \n",
    "Both approaches give exactly the same score."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}