Machine Learning Pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Machine Learning PipeLine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.pipeline import make_pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import RobustScaler\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.svm import LinearSVR\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.ensemble import GradientBoostingRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>MSZoning</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>Street</th>\n",
       "      <th>Alley</th>\n",
       "      <th>LotShape</th>\n",
       "      <th>LandContour</th>\n",
       "      <th>Utilities</th>\n",
       "      <th>...</th>\n",
       "      <th>PoolArea</th>\n",
       "      <th>PoolQC</th>\n",
       "      <th>Fence</th>\n",
       "      <th>MiscFeature</th>\n",
       "      <th>MiscVal</th>\n",
       "      <th>MoSold</th>\n",
       "      <th>YrSold</th>\n",
       "      <th>SaleType</th>\n",
       "      <th>SaleCondition</th>\n",
       "      <th>SalePrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>65.0</td>\n",
       "      <td>8450</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>208500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>RL</td>\n",
       "      <td>80.0</td>\n",
       "      <td>9600</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2007</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>181500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>68.0</td>\n",
       "      <td>11250</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>223500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>70</td>\n",
       "      <td>RL</td>\n",
       "      <td>60.0</td>\n",
       "      <td>9550</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2006</td>\n",
       "      <td>WD</td>\n",
       "      <td>Abnorml</td>\n",
       "      <td>140000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>84.0</td>\n",
       "      <td>14260</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>250000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 81 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id  MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape  \\\n",
       "0   1          60       RL         65.0     8450   Pave   NaN      Reg   \n",
       "1   2          20       RL         80.0     9600   Pave   NaN      Reg   \n",
       "2   3          60       RL         68.0    11250   Pave   NaN      IR1   \n",
       "3   4          70       RL         60.0     9550   Pave   NaN      IR1   \n",
       "4   5          60       RL         84.0    14260   Pave   NaN      IR1   \n",
       "\n",
       "  LandContour Utilities  ... PoolArea PoolQC Fence MiscFeature MiscVal MoSold  \\\n",
       "0         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      2   \n",
       "1         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      5   \n",
       "2         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      9   \n",
       "3         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      2   \n",
       "4         Lvl    AllPub  ...        0    NaN   NaN         NaN       0     12   \n",
       "\n",
       "  YrSold  SaleType  SaleCondition  SalePrice  \n",
       "0   2008        WD         Normal     208500  \n",
       "1   2007        WD         Normal     181500  \n",
       "2   2008        WD         Normal     223500  \n",
       "3   2006        WD        Abnorml     140000  \n",
       "4   2008        WD         Normal     250000  \n",
       "\n",
       "[5 rows x 81 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1460, 81)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def missing_values(df):\n",
    "    missing_columns = [column for column in df.columns if df[column].isnull().sum() > 0]\n",
    "    no_of_missing_value = [missing for missing in df.isnull().sum() if missing > 0]\n",
    "    return list(zip(missing_columns, no_of_missing_value))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('LotFrontage', 259),\n",
       " ('Alley', 1369),\n",
       " ('MasVnrType', 8),\n",
       " ('MasVnrArea', 8),\n",
       " ('BsmtQual', 37),\n",
       " ('BsmtCond', 37),\n",
       " ('BsmtExposure', 38),\n",
       " ('BsmtFinType1', 37),\n",
       " ('BsmtFinType2', 38),\n",
       " ('Electrical', 1),\n",
       " ('FireplaceQu', 690),\n",
       " ('GarageType', 81),\n",
       " ('GarageYrBlt', 81),\n",
       " ('GarageFinish', 81),\n",
       " ('GarageQual', 81),\n",
       " ('GarageCond', 81),\n",
       " ('PoolQC', 1453),\n",
       " ('Fence', 1179),\n",
       " ('MiscFeature', 1406)]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "missing_values(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sklearn PipeLine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifiers = [               \n",
    "               LinearRegression(n_jobs=-1), \n",
    "               Ridge(alpha=0.003, max_iter=30), \n",
    "               Lasso(alpha=.0005),                                              \n",
    "               SVR(kernel=\"linear\"),\n",
    "               LinearSVR(),\n",
    "               RandomForestRegressor(n_jobs=-1, n_estimators=350, \n",
    "                                     max_depth=12, random_state=1),\n",
    "               GradientBoostingRegressor(n_estimators=500, max_depth=2),               \n",
    "]\n",
    "\n",
    "clf_names = [            \n",
    "            \"linear\", \n",
    "            \"ridge\",\n",
    "            \"lasso\",                        \n",
    "            \"svr\",\n",
    "            \"linearsvr\",\n",
    "            \"randomforest\",                         \n",
    "            \"gbm\",\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_data(data, is_train_data=True):\n",
    "    for column in data.columns:\n",
    "        if data[column].isnull().sum() > 500:\n",
    "            data.drop(column,1,inplace=True)\n",
    "    data['TotalSF'] = data['TotalBsmtSF'] + data['1stFlrSF'] + data['2ndFlrSF']\n",
    "     # add here the code that you only want to apply to your training data and not the test set\n",
    "    # e.g. removing outliers from the training data works... \n",
    "    # ...but you cannot remove samples from your test set.\n",
    "    if is_train_data == True:\n",
    "        data = data[data.GrLivArea < 4000]\n",
    "        \n",
    "    return data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_data(df, is_train_data=True):\n",
    "    # split data into numerical & categorical in order to process seperately in the pipeline\n",
    "    numerical   = df.select_dtypes(\"number\").copy()\n",
    "    categorical = df.select_dtypes(\"object\").copy()\n",
    "    \n",
    "    # for training data only...\n",
    "    # ...convert SalePrice to log values and drop \"Id\" and \"SalePrice\" columns\n",
    "    if is_train_data == True :\n",
    "        SalePrice = numerical.SalePrice\n",
    "        y = np.log1p(SalePrice)\n",
    "        numerical.drop([\"Id\", \"SalePrice\"], axis=1, inplace=True)\n",
    "        \n",
    "    # for the test data: just drop \"Id\" and set \"y\" to None\n",
    "    else:\n",
    "        numerical.drop([\"Id\"], axis=1, inplace=True)\n",
    "        y = None\n",
    "        \n",
    "     # concatenate numerical and categorical data to X (our final training data)\n",
    "    X = pd.concat([numerical, categorical], axis=1)\n",
    "    \n",
    "    return X, y, numerical.columns, categorical.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_pipeline(classifier, num_cols, cat_cols):\n",
    "    # the numeric transformer gets the numerical data acording to num_cols\n",
    "    # the first step is the imputer which imputes all missing values to the mean\n",
    "    # in the second step all numerical data gets scaled by the StandardScaler()\n",
    "    numeric_transformer = Pipeline(steps=[\n",
    "        ('imputer', make_pipeline(SimpleImputer(strategy='mean'))),\n",
    "        ('scaler', StandardScaler())])\n",
    "    \n",
    "    # the categorical transformer gets all categorical data according to cat_cols\n",
    "    # again: first step is imputing missing values and one hot encoding the categoricals\n",
    "    categorical_transformer = Pipeline(steps=[\n",
    "        ('imputer', SimpleImputer(strategy='most_frequent', fill_value='missing')),\n",
    "        ('onehot', OneHotEncoder(handle_unknown='ignore'))])\n",
    "    \n",
    "    # the column transformer creates one Pipeline for categorical and numerical data each\n",
    "    preprocessor = ColumnTransformer(\n",
    "        transformers=[\n",
    "            ('num', numeric_transformer, num_cols),\n",
    "            ('cat', categorical_transformer, cat_cols)])\n",
    "    \n",
    "    # return the whole pipeline with the classifier provided in the function call    \n",
    "    return Pipeline(steps=[('preprocessor', preprocessor), ('classifier', classifier)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def score_models(df):\n",
    "    # retrieve X, y and the seperate columns names\n",
    "    X, y, num_cols, cat_cols = prepare_data(df)\n",
    "    \n",
    "    # since we converted SalePrice to log values, we use neg_mean_squared_error... \n",
    "    # ...rather than *neg_mean_squared_log_error* \n",
    "    scoring_metric = \"neg_mean_squared_error\"\n",
    "    scores = []\n",
    "    \n",
    "    for clf_name, classifier in zip(clf_names, classifiers):\n",
    "        # create a pipeline for each classifier\n",
    "        clf = get_pipeline(classifier, num_cols, cat_cols)\n",
    "        # set a kfold with 3 splits to get more robust scores. \n",
    "        # increase to 5 or 10 to get more precise estimations on models score\n",
    "        kfold = KFold(n_splits=3, shuffle=True, random_state=1)  \n",
    "        # crossvalidate and return the square root of the results\n",
    "        results = np.sqrt(-cross_val_score(clf, X, y, cv=kfold, scoring=scoring_metric))\n",
    "        scores.append([clf_name, results.mean()])\n",
    "\n",
    "    scores = pd.DataFrame(scores, columns=[\"classifier\", \"rmse\"]).sort_values(\"rmse\", ascending=False)\n",
    "    # just for good measure: add the mean of all scores to dataframe\n",
    "    scores.loc[len(scores) + 1, :] = [\"mean_all\", scores.rmse.mean()]\n",
    "    return scores.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fit Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_models(df): \n",
    "    X, y, num_cols, cat_cols = prepare_data(df)\n",
    "    pipelines = []\n",
    "    \n",
    "    for clf_name, classifier in zip(clf_names, classifiers):\n",
    "        clf = get_pipeline(classifier, num_cols, cat_cols)\n",
    "        clf.fit(X, y)\n",
    "        pipelines.append(clf)\n",
    "    \n",
    "    return pipelines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_from_models(df_test, pipelines):\n",
    "    X_test, _ , _, _ = prepare_data(df_test, is_train_data=False)\n",
    "    predictions = []\n",
    "    \n",
    "    for pipeline in pipelines:\n",
    "        preds = pipeline.predict(X_test)\n",
    "        # we return the exponent of the predictions since we have log converted y for training\n",
    "        predictions.append(np.expm1(preds))\n",
    "    \n",
    "    return predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"data/train.csv\")\n",
    "df_test = pd.read_csv(\"data/test.csv\")\n",
    "\n",
    "# We clean the data\n",
    "df = clean_data(df)\n",
    "df_test = clean_data(df_test, is_train_data=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We score the models on the preprocessed training data\n",
    "my_scores = score_models(df)\n",
    "display(my_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = train_models(df)\n",
    "predictions = predict_from_models(df_test, models)\n",
    "# We average over the results of all 12 classifiers (simple ensembling)\n",
    "# we exclude the DummyRegressor and the SGDRegressor: they perform worst...\n",
    "prediction_final = pd.DataFrame(predictions[2:]).mean().T.values\n",
    "\n",
    "submission = pd.DataFrame({'Id': df_test.Id.values, 'SalePrice': prediction_final})\n",
    "submission.to_csv(f\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}