Ready_Codes.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# My Useful Codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# to select columns based on dtypes\n",
    "from sklearn.compose import make_column_selector as selector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run ./Ready_Codes.ipynb\n",
    "import classif_models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# import pandas_bokeh\n",
    "# pandas_bokeh.output_notebook()\n",
    "\n",
    "import scikitplot as skplt\n",
    "import seaborn as sns\n",
    "sns.set()\n",
    "\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import cross_validate\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "plt.rcParams['axes.labelsize'] = 15\n",
    "plt.rcParams['axes.titlesize'] = 15\n",
    "\n",
    "#Importing Classification algorithms\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "from sklearn.linear_model import (\n",
    "    LogisticRegression,\n",
    "    LogisticRegressionCV,\n",
    "    SGDClassifier,\n",
    "    Perceptron,\n",
    "    PassiveAggressiveClassifier,\n",
    "    RidgeClassifier, \n",
    "    RidgeClassifierCV\n",
    ")\n",
    "\n",
    "from sklearn.svm import LinearSVC, SVC, NuSVC\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.naive_bayes import  GaussianNB, BernoulliNB\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.gaussian_process import GaussianProcessClassifier\n",
    "\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from catboost import CatBoostClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "from rgf.sklearn import RGFClassifier, FastRGFClassifier\n",
    "from gpboost import GPBoostClassifier\n",
    "from ngboost import NGBClassifier\n",
    "from sklearn.experimental import enable_hist_gradient_boosting\n",
    "\n",
    "from sklearn.ensemble import (\n",
    "    RandomForestClassifier, \n",
    "    AdaBoostClassifier, \n",
    "    GradientBoostingClassifier, \n",
    "    ExtraTreesClassifier, \n",
    "    IsolationForest, \n",
    "    BaggingClassifier, \n",
    "    HistGradientBoostingClassifier\n",
    ")\n",
    "\n",
    "from imblearn.ensemble import (\n",
    "    EasyEnsembleClassifier, \n",
    "    RUSBoostClassifier, \n",
    "    BalancedBaggingClassifier, \n",
    "    BalancedRandomForestClassifier \n",
    ")\n",
    "\n",
    "from numpy import interp\n",
    "\n",
    "class Progress:\n",
    "    def __init__(self, value, end, title='Progress',buffer=100):\n",
    "        self.title = title\n",
    "        #when calling in a for loop it doesn't include the last number\n",
    "        self.end = end\n",
    "        self.buffer = buffer\n",
    "        self.value = value + 1\n",
    "        self.progress()\n",
    "\n",
    "    def progress(self):\n",
    "        maped = int(interp(self.value, [0, self.end], [0, self.buffer]))\n",
    "        print(f'{self.title}: [{\"#\"*maped}{\"-\"*(self.buffer - maped)}]{self.value}/{self.end} {((self.value/self.end)*100):.2f}%', end='\\r')\n",
    "\n",
    "from sklearn import set_config\n",
    "set_config(display='diagram')        \n",
    "\n",
    "seed = #seed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "class_weights = compute_class_weight('balanced', np.unique(y_train.Loan_Status), y_train.Loan_Status)\n",
    "\n",
    "print(class_weights)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stacking Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "from mlxtend.classifier import StackingClassifier\n",
    "from sklearn import metrics\n",
    "import scikitplot as skplt\n",
    "\n",
    "sclf = StackingClassifier(classifiers=[cat, bag_clf, gbc, et_clf], \n",
    "                          meta_classifier=LogisticRegression(random_state = seed), \n",
    "                          use_probas = True)\n",
    "\n",
    "sclf.fit(X_train_pars, y_train)\n",
    "\n",
    "y_pred = sclf.predict(X_test_pars)\n",
    "y_probs = sclf.predict_proba(X_test_pars)\n",
    "\n",
    "print()\n",
    "print(f'Stacking Classifier F1 score on TEST set: {metrics.f1_score(y_test, y_pred)*100:.4f} %')\n",
    "print() \n",
    "print(f'Stacking Classifier Accuracy on TEST set: {metrics.accuracy_score(y_test, y_pred)*100:.4f} %')\n",
    "print()\n",
    "print(f'Stacking Classifier ROC AUC Score: {metrics.roc_auc_score(y_test, y_probs[:,1])*100:.4f} %')\n",
    "print()\n",
    "print(metrics.classification_report(y_test, y_pred))\n",
    "skplt.metrics.plot_roc(y_test, y_probs);\n",
    "skplt.metrics.plot_confusion_matrix(y_test,y_pred, text_fontsize = 'large', cmap='YlGn');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ensemble Voting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "from mlxtend.classifier import EnsembleVoteClassifier\n",
    "\n",
    "evc = EnsembleVoteClassifier(clfs=calibrated_models, voting = 'soft')\n",
    "\n",
    "evc.fit(X_train_pars, y_train)\n",
    "\n",
    "y_pred = evc.predict(X_test_pars)\n",
    "y_probs = evc.predict_proba(X_test_pars)\n",
    "\n",
    "print()\n",
    "print(f'Voting Classifier F1 score on TEST set: {metrics.f1_score(y_test, y_pred)*100:.4f} %')\n",
    "print() \n",
    "print(f'Voting Classifier Accuracy on TEST set: {metrics.accuracy_score(y_test, y_pred)*100:.4f} %')\n",
    "print()\n",
    "print(f'Voting Classifier ROC AUC Score: {metrics.roc_auc_score(y_test, y_probs[:,1])*100:.4f} %')\n",
    "print()\n",
    "print(metrics.classification_report(y_test, y_pred))\n",
    "skplt.metrics.plot_roc(y_test, y_probs);\n",
    "skplt.metrics.plot_confusion_matrix(y_test,y_pred, text_fontsize = 'large', cmap='YlGn');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Custom Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Compare all models on the Train Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Function for selecting the right model\n",
    "\n",
    "class classif_models:\n",
    "    \n",
    "    def __init__(self, Xtrain, ytrain, preprocessor, cv):\n",
    "        self.Xtrain = Xtrain\n",
    "        self.ytrain = ytrain\n",
    "        self.preprocessor = preprocessor \n",
    "        self.cv = cv\n",
    "        \n",
    "    def check_clf_models(self):\n",
    "        \n",
    "        models = [\n",
    "            LogisticRegression(random_state = seed),\n",
    "            LogisticRegressionCV(cv=10, random_state = seed),\n",
    "            SGDClassifier(tol = 0.1, early_stopping = True, validation_fraction = 0.2, random_state = seed),\n",
    "            Perceptron(tol = 0.1, early_stopping = True, validation_fraction = 0.2, random_state = seed),\n",
    "            PassiveAggressiveClassifier(tol = 0.1, early_stopping = True, validation_fraction = 0.2, random_state = seed),\n",
    "            RidgeClassifier(random_state = seed),\n",
    "            RidgeClassifierCV(cv=10),\n",
    "            LinearSVC(loss = 'hinge', random_state = seed),\n",
    "            SVC(kernel = 'rbf', random_state = seed),\n",
    "            NuSVC(random_state = seed),\n",
    "            KNeighborsClassifier(n_neighbors = 5, metric = 'minkowski', p = 2),\n",
    "            GaussianNB(), \n",
    "            BernoulliNB(),\n",
    "            MLPClassifier(tol=0.1, early_stopping = True, validation_fraction = 0.2, random_state = seed),\n",
    "            GaussianProcessClassifier(random_state = seed)\n",
    "        ]\n",
    "        \n",
    "        acc = []\n",
    "        acc_std = []\n",
    "        f1 = []\n",
    "        roc_auc = []\n",
    "        prec = []\n",
    "        recall = []\n",
    "        bal_acc = []\n",
    "        model_names = []\n",
    "        \n",
    "        print('Training using Non-Tree based models...')\n",
    "        for model, i in zip(models, range(len(models))):\n",
    "            pipe = Pipeline(steps = [('preprocessor', preprocessor), (type(model).__name__, model)])\n",
    "            \n",
    "            scores = cross_validate(pipe, \n",
    "                                    self.Xtrain, \n",
    "                                    self.ytrain,\n",
    "                                    scoring = ['accuracy', 'f1', 'roc_auc', 'precision', 'recall', \n",
    "                                               'balanced_accuracy'],\n",
    "                                    cv = self.cv, \n",
    "                                    n_jobs = -1)\n",
    "            \n",
    "            acc.append(list(scores.values())[2].mean()*100)\n",
    "            acc_std.append(list(scores.values())[2].std()*100)\n",
    "            f1.append(list(scores.values())[3].mean()*100)\n",
    "            roc_auc.append(list(scores.values())[4].mean()*100)\n",
    "            prec.append(list(scores.values())[5].mean()*100)\n",
    "            recall.append(list(scores.values())[6].mean()*100)\n",
    "            bal_acc.append(list(scores.values())[7].mean()*100)\n",
    "            \n",
    "            model_names.append(type(model).__name__)\n",
    "            Progress(i, len(models))\n",
    "            \n",
    "        print()\n",
    "\n",
    "    #Using Ensemble Models\n",
    "\n",
    "        tree_models = [\n",
    "            DecisionTreeClassifier(criterion = 'entropy', max_depth = 6, random_state = seed),\n",
    "            RandomForestClassifier(criterion='entropy', max_depth=6, class_weight='balanced', n_jobs=-1, random_state = seed), \n",
    "            XGBClassifier(use_label_encoder=False, eval_metric = 'error', seed = seed), \n",
    "            CatBoostClassifier(verbose = False, loss_function='CrossEntropy', eval_metric='TotalF1', random_seed = seed), \n",
    "            LGBMClassifier(random_state = seed), \n",
    "            AdaBoostClassifier(random_state = seed), \n",
    "            GradientBoostingClassifier(random_state = seed), \n",
    "            BaggingClassifier(random_state = seed), \n",
    "            ExtraTreesClassifier(criterion='entropy', max_depth=6, class_weight='balanced', n_jobs=-1, random_state = seed), \n",
    "            HistGradientBoostingClassifier(random_state = seed), \n",
    "            EasyEnsembleClassifier(random_state = seed), \n",
    "            RUSBoostClassifier(random_state = seed), \n",
    "            BalancedBaggingClassifier(random_state = seed), \n",
    "            BalancedRandomForestClassifier(n_estimators = 100, criterion = 'entropy', max_depth = 6, random_state = seed), \n",
    "            RGFClassifier(loss = 'Log', algorithm='RGF_Sib'), \n",
    "            FastRGFClassifier(loss='LOGISTIC'),\n",
    "            GPBoostClassifier(random_state = seed), \n",
    "            NGBClassifier(random_state = seed)\n",
    "        ]\n",
    "        \n",
    "        print()\n",
    "        print('Training Using Ensemble models...')\n",
    "        for model, i in zip(tree_models, range(len(tree_models))):\n",
    "            \n",
    "            scores = cross_validate(model, \n",
    "                                    self.Xtrain, \n",
    "                                    self.ytrain,\n",
    "                                    scoring = ['accuracy', 'f1', 'roc_auc', 'precision', 'recall', \n",
    "                                               'balanced_accuracy'],\n",
    "                                    cv = self.cv, \n",
    "                                    n_jobs = -1)\n",
    "            \n",
    "            acc.append(list(scores.values())[2].mean()*100)\n",
    "            acc_std.append(list(scores.values())[2].std()*100)\n",
    "            f1.append(list(scores.values())[3].mean()*100)\n",
    "            roc_auc.append(list(scores.values())[4].mean()*100)\n",
    "            prec.append(list(scores.values())[5].mean()*100)\n",
    "            recall.append(list(scores.values())[6].mean()*100)\n",
    "            bal_acc.append(list(scores.values())[7].mean()*100)\n",
    "            \n",
    "            model_names.append(type(model).__name__)\n",
    "            Progress(i, len(tree_models))\n",
    "        \n",
    "        print()\n",
    "        self.df = pd.DataFrame({\n",
    "            'Models': model_names, \n",
    "            'Acc %': acc, \n",
    "            'Acc STD %': acc_std, \n",
    "            'f1 %':f1,\n",
    "            'ROC_AUC %' :roc_auc, \n",
    "            'Precision %': prec,\n",
    "            'Recall %': recall, \n",
    "            'Balanced_Acc %': bal_acc \n",
    "        }).sort_values('f1 %', ascending = False, ignore_index = True).style.highlight_max(color = 'green')\n",
    "\n",
    "        return self.df\n",
    "    \n",
    "# classif_models(Xtrain, ytrain, preprocessor, cv)\n",
    "#.check_clf_models()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Validation with Threshold Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for evaluating model\n",
    "\n",
    "class validate_on_test:\n",
    "    \n",
    "    def __init__(self, model, Xtrain, ytrain, Xtest, ytest, cv, beta, optimize):\n",
    "        self.model = model\n",
    "        self.Xtrain = Xtrain\n",
    "        self.ytrain = ytrain\n",
    "        self.Xtest = Xtest\n",
    "        self.ytest = ytest\n",
    "        self.cv = cv\n",
    "        self.beta = beta\n",
    "        self.optimize = optimize\n",
    "        \n",
    "        self.Xtrain = self.Xtrain.values\n",
    "        self.ytrain = self.ytrain[self.ytrain.columns[0]].values\n",
    "        self.Xtest = self.Xtest.values\n",
    "        self.ytest = self.ytest[self.ytest.columns[0]].values\n",
    "        \n",
    "    def evaluate_model(self):\n",
    "        \n",
    "        from texttable import Texttable\n",
    "        t = Texttable()\n",
    "        \n",
    "        if type(self.model).__name__ == 'CalibratedClassifierCV':\n",
    "            if type(self.model.base_estimator).__name__ == 'Pipeline':\n",
    "                model_name = 'Cal_' + type(list(self.model.base_estimator.named_steps.items())[1][1]).__name__\n",
    "            else:\n",
    "                model_name = 'Cal_' + type(self.model.base_estimator).__name__\n",
    "        else:\n",
    "            try: \n",
    "                type(list(self.model.named_steps.items())[1][1]).__name__\n",
    "                \n",
    "            except AttributeError:\n",
    "                model_name = type(self.model).__name__\n",
    "                \n",
    "            else:\n",
    "                model_name = type(list(self.model.named_steps.items())[1][1]).__name__\n",
    "            \n",
    "        print('+' * len(f' {model_name} '))\n",
    "        print(f' {model_name} ')\n",
    "        print('+' * len(f' {model_name} '))\n",
    "        print()\n",
    "\n",
    "        print('Performing Cross-Validation...')\n",
    "        print('------------------------------------------')\n",
    "        t.set_deco(t.VLINES)\n",
    "        t.add_rows([['CV#', 'Accuracy %', 'f1 Score %', 'ROC-AUC %']], header = False)\n",
    "        print(t.draw())\n",
    "        print('------------------------------------------')\n",
    "        t.reset()\n",
    "        t.set_deco(t.HLINES)\n",
    "        \n",
    "        thresholds = np.arange(0, 1, 0.001)\n",
    "        \n",
    "        # apply threshold to positive probabilities to create labels\n",
    "        def to_labels(pos_probs, threshold):\n",
    "            return (pos_probs >= threshold).astype('int64')\n",
    "\n",
    "        accuracy = []\n",
    "        f1_score = []\n",
    "        roc_auc_score = []\n",
    "        fold_no = 1\n",
    "\n",
    "        for train_index, test_index in self.cv.split(self.Xtrain,self.ytrain):\n",
    "            X_train_kfold, X_val_kfold = self.Xtrain[train_index], self.Xtrain[test_index]\n",
    "            y_train_kfold, y_val_kfold = self.ytrain[train_index], self.ytrain[test_index]\n",
    "\n",
    "            classifier = self.model\n",
    "\n",
    "            classifier.fit(X_train_kfold, y_train_kfold) \n",
    "            y_pred = classifier.predict(X_val_kfold)\n",
    "            y_probs = classifier.predict_proba(X_val_kfold)\n",
    "           \n",
    "            accuracy.append(np.round(metrics.accuracy_score(y_val_kfold, y_pred)*100,3))\n",
    "            f1_score.append(np.round(metrics.f1_score(y_val_kfold, y_pred)*100,3))\n",
    "            roc_auc_score.append(np.round(metrics.roc_auc_score(y_val_kfold, y_probs[:,1])*100,3))\n",
    "            \n",
    "            t.set_cols_align([\"c\", \"c\", \"c\", \"c\"])\n",
    "            t.add_row([fold_no, accuracy[fold_no-1],  f1_score[fold_no-1],  roc_auc_score[fold_no-1]])\n",
    "            print(t.draw())\n",
    "            t.reset()\n",
    "        \n",
    "            fold_no += 1\n",
    "                \n",
    "        print()\n",
    "        print(f'*** {model_name} Mean CV Scores ***')\n",
    "        print('=' * len(f'*** {model_name} Mean CV Scores ***'))\n",
    "        print(f'ROC AUC   : {np.mean(roc_auc_score):.3f} ± {np.std(roc_auc_score):.1f} %')\n",
    "        print(f'f1        : {np.mean(f1_score):.3f} ± {np.std(f1_score):.1f} %')\n",
    "        print(f'Accuracy  : {np.mean(accuracy):.3f} ± {np.std(accuracy):.1f} %')\n",
    "        print()\n",
    "        print(\"---\" * 40)\n",
    "\n",
    "        y_pred = classifier.predict(self.Xtest)\n",
    "        y_probs = classifier.predict_proba(self.Xtest)\n",
    "        \n",
    "        self.y_probs = y_probs\n",
    "\n",
    "        print()\n",
    "        print('====================================')\n",
    "        print(\"Classification report on Test set:\")\n",
    "        print('====================================')\n",
    "        print()\n",
    "        print(metrics.classification_report(self.ytest, y_pred))\n",
    "        print()\n",
    "        print(\"---\" * 40)\n",
    "        print()\n",
    "        \n",
    "        print(f'*** {model_name} scores on TEST set ***')\n",
    "        print('=' * len(f'*** {model_name} scores on TEST set ***'))\n",
    "        print(f'ROC AUC   : {metrics.roc_auc_score(self.ytest, y_probs[:,1])*100:.4f} %')\n",
    "        print(f'f1        : {metrics.f1_score(self.ytest, y_pred)*100:.4f} %')\n",
    "        print(f'Accuracy  : {metrics.accuracy_score(self.ytest, y_pred)*100:.4f} %')\n",
    "        print()\n",
    "        \n",
    "        print(\"---\" * 40)\n",
    "        \n",
    "        if self.beta == 'auto':\n",
    "            if self.optimize == 'fbeta':\n",
    "                beta_array = np.arange(0.5,2.05,0.05)\n",
    "\n",
    "                # evaluate each threshold\n",
    "                beta_val_array = []\n",
    "                best_f1 = []\n",
    "                best_threshold = []\n",
    "                best_acc = []\n",
    "                print()\n",
    "                print('Estimating New Probability Threshold...')\n",
    "\n",
    "                for b, i in zip(beta_array, range(len(beta_array))):\n",
    "\n",
    "                    beta_val_array.append(b)\n",
    "\n",
    "                    fbeta_score = [metrics.fbeta_score(self.ytest, \n",
    "                                                       to_labels(self.y_probs[:,1], t), \n",
    "                                                       beta = b) for t in thresholds]\n",
    "\n",
    "                    acc = [metrics.accuracy_score(self.ytest,\n",
    "                                                  to_labels(self.y_probs[:,1], t)) for t in thresholds]\n",
    "                    \n",
    "                    ix = np.argmax(fbeta_score)\n",
    "                        \n",
    "                    best_f1.append(fbeta_score[ix])\n",
    "                    best_acc.append(acc[ix])\n",
    "                    best_threshold.append(thresholds[ix])\n",
    "\n",
    "                    Progress(i, len(beta_array))\n",
    "            \n",
    "                print()\n",
    "                score_df = pd.DataFrame({\n",
    "                    'threshold': best_threshold,\n",
    "                    'Beta' : beta_val_array,\n",
    "                    'fbeta': best_f1, \n",
    "                    'accuracy' : best_acc\n",
    "                }).sort_values(self.optimize, ascending = False, ignore_index = True)\n",
    "\n",
    "                self.score_df = score_df\n",
    "                \n",
    "                print()\n",
    "                print('Best F-beta, Accuracy, Beta & Threshold:')\n",
    "                print('========================================')\n",
    "                print(f'Beta      : {score_df.iloc[0, 1]:.2f}')\n",
    "                print(f'Threshold : {score_df.iloc[0, 0]*100:.2f} %')\n",
    "                print(f'F-beta    : {score_df.iloc[0, 2]*100:.4f} %')\n",
    "                print(f'Accuracy  : {score_df.iloc[0, 3]*100:.4f} %')\n",
    "                print()\n",
    "                \n",
    "                thresh_val = score_df.iloc[0,0]\n",
    "                self.tuned_pred = np.where(self.y_probs[:,1] >= thresh_val, 1, 0).astype('int64')\n",
    "                self.new_prob_threshold = thresh_val\n",
    "                \n",
    "                plot_df = pd.DataFrame({\n",
    "                    'threshold' : thresholds, \n",
    "                    'fbeta' : [metrics.fbeta_score(self.ytest,\n",
    "                                                   to_labels(self.y_probs[:,1], t),\n",
    "                                                   beta = score_df.iloc[0, 1]) for t in thresholds], \n",
    "                    'accuracy': acc\n",
    "                })\n",
    "                \n",
    "            else:\n",
    "                fbeta_score = [metrics.fbeta_score(self.ytest, \n",
    "                                                       to_labels(self.y_probs[:,1], t), \n",
    "                                                       beta = 1) for t in thresholds]\n",
    "                \n",
    "                acc = [metrics.accuracy_score(self.ytest,\n",
    "                                                  to_labels(self.y_probs[:,1], t)) for t in thresholds]\n",
    "                \n",
    "                ix = np.argmax(acc)\n",
    "                \n",
    "                score_df = pd.DataFrame({\n",
    "                    'threshold': thresholds,\n",
    "                    'fbeta': fbeta_score, \n",
    "                    'accuracy' : acc\n",
    "                }).sort_values(self.optimize, ascending = False, ignore_index = True)\n",
    "\n",
    "                self.score_df = score_df\n",
    "                plot_df = score_df\n",
    "            \n",
    "                print()\n",
    "                print('Best F1, Accuracy, & Threshold:')\n",
    "                print('===============================')\n",
    "                print(f'Threshold : {score_df.iloc[0, 0]*100:.2f} %')\n",
    "                print(f'F1        : {score_df.iloc[0, 1]*100:.4f} %')\n",
    "                print(f'Accuracy  : {score_df.iloc[0, 2]*100:.4f} %')\n",
    "                print()\n",
    "                \n",
    "                thresh_val = score_df.iloc[0, 0]\n",
    "                self.tuned_pred = np.where(self.y_probs[:,1] >= thresh_val, 1, 0).astype('int64')\n",
    "                self.new_prob_threshold = thresh_val\n",
    "            \n",
    "        else:\n",
    "            beta_val = self.beta\n",
    "\n",
    "            # evaluate each threshold\n",
    "            fbeta_score = [metrics.fbeta_score(self.ytest, \n",
    "                                               to_labels(self.y_probs[:,1], t), \n",
    "                                               beta = beta_val) for t in thresholds]\n",
    "            \n",
    "            acc = [metrics.accuracy_score(self.ytest,\n",
    "                                          to_labels(self.y_probs[:,1], t)) for t in thresholds]\n",
    "                    \n",
    "            score_df = pd.DataFrame({\n",
    "                'threshold': thresholds,\n",
    "                'fbeta': fbeta_score, \n",
    "                'accuracy' : acc\n",
    "            }).sort_values(self.optimize, ascending = False, ignore_index = True)\n",
    "            \n",
    "            self.score_df = score_df\n",
    "            plot_df = score_df\n",
    "            \n",
    "            print('Best F-beta, Accuracy, Beta & Threshold:')\n",
    "            print('========================================')\n",
    "            print(f'Beta      : {self.beta}')\n",
    "            print(f'Threshold : {score_df.iloc[0,0]*100:.4f} %')\n",
    "            print(f'F-beta    : {score_df.iloc[0,1]*100:.4f} %')\n",
    "            print(f'Accuracy  : {score_df.iloc[0,2]*100:.4f} %')\n",
    "            print()\n",
    "            \n",
    "            thresh_val = score_df.iloc[0,0]\n",
    "            self.new_prob_threshold = thresh_val\n",
    "            self.tuned_pred = np.where(self.y_probs[:,1] >= thresh_val, 1, 0).astype('int64')\n",
    "            \n",
    "        print(\"---\" * 40)\n",
    "        \n",
    "        print('=================================================================================')\n",
    "        print(' Threshold-Tuning Curve, ROC-AUC Plot, Precision-Recall Curve & Confusion Matrix ')\n",
    "        print('=================================================================================')\n",
    "        \n",
    "        #All Train set\n",
    "        \n",
    "        train_probs = classifier.predict_proba(self.Xtrain)\n",
    "        \n",
    "        all_f = []\n",
    "        all_acc = []\n",
    "        \n",
    "        for t, i in zip(thresholds, range(len(thresholds))):\n",
    "            all_f.append(metrics.f1_score(self.ytrain, \n",
    "                                          to_labels(train_probs[:,1], t)))\n",
    "            \n",
    "            all_acc.append(metrics.accuracy_score(self.ytrain, \n",
    "                                                  to_labels(train_probs[:,1], t)))\n",
    "            \n",
    "            Progress(i, len(thresholds))\n",
    "        \n",
    "        print()\n",
    "        \n",
    "        fig = plt.figure(figsize = (11.5,5));\n",
    "        ax1 = fig.add_subplot(121);\n",
    "        ax2 = fig.add_subplot(122);\n",
    "        \n",
    "        #Train set\n",
    "        sns.lineplot(x = thresholds, y = all_f, ax = ax1, label = 'Train F1 scores').\\\n",
    "            set_title('Threshold for Max F-score');\n",
    "        \n",
    "        ax1.axvline(thresholds[np.argmax(all_f)], color = 'red', linestyle = '--', \n",
    "           lw = 2, label = f'Train Thresh = {thresholds[np.argmax(all_f)]*100:.2f} %');\n",
    "        \n",
    "        sns.lineplot(x = thresholds, y = all_acc, ax = ax2, label = 'Train Accuracy').\\\n",
    "            set_title('Threshold for Max Accuracy');\n",
    "        \n",
    "        ax2.axvline(thresholds[np.argmax(all_acc)], color = 'red', linestyle = '--', \n",
    "           lw = 2, label = f'Train Thresh = {thresholds[np.argmax(all_acc)]*100:.2f} %');\n",
    "\n",
    "        plt.tight_layout();\n",
    "        \n",
    "        #Test set\n",
    "        f_df = plot_df.sort_values('fbeta', ascending = False, ignore_index = True)\n",
    "        acc_df = plot_df.sort_values('accuracy', ascending = False, ignore_index = True)\n",
    "        \n",
    "        sns.lineplot(x = 'threshold', y = 'fbeta', data = plot_df, \n",
    "                     label = 'Test F-score', ax = ax1).set_title('Threshold for Max F-score');\n",
    "        \n",
    "        ax1.axvline(f_df.iloc[0,0], color = 'black', linestyle = '--', \n",
    "                   label = f' Test F-score Thresh = {f_df.iloc[0,0]*100:.2f} %');\n",
    "        \n",
    "        ax1.legend(loc='lower right');\n",
    "        \n",
    "        sns.lineplot(x = 'threshold', y = 'accuracy', data = plot_df, \n",
    "                     label = 'Test Accuracy', ax = ax2).set_title('Threshold for Max Accuracy');\n",
    "        \n",
    "        ax2.axvline(acc_df.iloc[0,0], color = 'black', linestyle = '--', \n",
    "                   label = f' Test Accuracy Thresh = {acc_df.iloc[0,0]*100:.2f} %');\n",
    "\n",
    "        ax2.legend(loc='lower right');\n",
    "        plt.tight_layout()\n",
    "        \n",
    "        #roc-auc, precision-recall curve\n",
    "        fig = plt.figure(figsize = (13,4.5));\n",
    "        ax1 = fig.add_subplot(121);\n",
    "        ax2 = fig.add_subplot(122);\n",
    "        \n",
    "        #roc curve\n",
    "        fpr, tpr, thresh = metrics.roc_curve(self.ytest, y_probs[:,1])\n",
    "        J = tpr - fpr\n",
    "        ix = np.argmax(J)\n",
    "\n",
    "        ax1.plot([0,1], [0,1], linestyle='--', label='No Skill')\n",
    "        ax1.plot(fpr, tpr, marker='.', \n",
    "                 label=f'roc-auc = {metrics.roc_auc_score(self.ytest, y_probs[:,1])*100:.2f} %')\n",
    "        \n",
    "        ax1.scatter(fpr[ix], tpr[ix], marker='o', color='black', \n",
    "                    label=f'Best threshold = {thresh[ix]*100:.2f} %')\n",
    "        ax1.set_xlabel('False Positive Rate')\n",
    "        ax1.set_ylabel('True Positive Rate')\n",
    "        ax1.legend();\n",
    "        \n",
    "        #precision-recall curve\n",
    "        precision, recall, thresh = metrics.precision_recall_curve(self.ytest, y_probs[:,1])\n",
    "        fscore = (2 * precision * recall) / (precision + recall)\n",
    "        ix = np.argmax(fscore)\n",
    "        ax2.plot(recall, precision, marker='.', label=f'F1 Score = {fscore[ix]*100:.2f} %')\n",
    "        ax2.scatter(recall[ix], precision[ix], marker='o', color='black', \n",
    "                       label=f'Best Thresh = {thresh[ix]*100:.2f} %')\n",
    "        ax2.set_xlabel('Recall')\n",
    "        ax2.set_ylabel('Precision')\n",
    "        ax2.legend()\n",
    "\n",
    "        #Confusion Matrix\n",
    "        fig = plt.figure(figsize = (13,4));\n",
    "        ax1 = fig.add_subplot(121);\n",
    "        ax2 = fig.add_subplot(122);\n",
    "        \n",
    "        skplt.metrics.plot_confusion_matrix(self.ytest, y_pred, ax = ax1, text_fontsize = 'large', \n",
    "                                            cmap='YlGn');\n",
    "    \n",
    "        \n",
    "        skplt.metrics.plot_confusion_matrix(self.ytest, self.tuned_pred, ax = ax2, text_fontsize = 'large',\n",
    "                                            cmap='YlGn');\n",
    "        \n",
    "        ax1.title.set_text('With Prob. Threshold = 50 %');\n",
    "        ax2.title.set_text(f'With New Prob. Threshold = {thresh_val*100:.2f} %');\n",
    "        \n",
    "# validate_on_test(model, Xtrain, ytrain, Xtest, ytest, cv, beta = 'auto', optimize = 'fbeta')\n",
    "#.evaluate_model()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test on New/Unseen Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class predict_unseen_data:\n",
    "    def __init__(self, model, Xtrain, ytrain, Xtest, cv, set_threshold = 0.5):\n",
    "        self.model = model\n",
    "        self.Xtrain = Xtrain\n",
    "        self.ytrain = ytrain\n",
    "        self.Xtest = Xtest\n",
    "        self.cv = cv\n",
    "        self.set_threshold = set_threshold\n",
    "        \n",
    "        self.Xtrain = self.Xtrain.values\n",
    "        self.ytrain = self.ytrain[self.ytrain.columns[0]].values\n",
    "        self.Xtest = self.Xtest.values\n",
    "        \n",
    "    def prediction(self):\n",
    "        from texttable import Texttable\n",
    "        t = Texttable()\n",
    "        \n",
    "        if type(self.model).__name__ == 'CalibratedClassifierCV':\n",
    "            if type(self.model.base_estimator).__name__ == 'Pipeline':\n",
    "                model_name = 'Cal_' + type(list(self.model.base_estimator.named_steps.items())[1][1]).__name__\n",
    "            else:\n",
    "                model_name = 'Cal_' + type(self.model.base_estimator).__name__\n",
    "        else:\n",
    "            try: \n",
    "                type(list(self.model.named_steps.items())[1][1]).__name__\n",
    "                \n",
    "            except AttributeError:\n",
    "                model_name = type(self.model).__name__\n",
    "                \n",
    "            else:\n",
    "                model_name = type(list(self.model.named_steps.items())[1][1]).__name__\n",
    "            \n",
    "        print('+' * len(f' {model_name} '))\n",
    "        print(f' {model_name} ')\n",
    "        print('+' * len(f' {model_name} '))\n",
    "        print()\n",
    "\n",
    "        print('Performing Cross-Validation...')\n",
    "        print('------------------------------------------')\n",
    "        t.set_deco(t.VLINES)\n",
    "        t.add_rows([['CV#', 'Accuracy %', 'f1 Score %', 'ROC-AUC %']], header = False)\n",
    "        print(t.draw())\n",
    "        print('------------------------------------------')\n",
    "        t.reset()\n",
    "        t.set_deco(t.HLINES)\n",
    "        \n",
    "        # apply threshold to positive probabilities to create labels\n",
    "        thresholds = np.arange(0, 1, 0.001)\n",
    "        \n",
    "        # apply threshold to positive probabilities to create labels\n",
    "        def to_labels(pos_probs, threshold):\n",
    "            return (pos_probs >= threshold).astype('int64')\n",
    "\n",
    "        accuracy = []\n",
    "        f1_score = []\n",
    "        roc_auc_score = []\n",
    "        fold_no = 1\n",
    "        \n",
    "        for train_index, test_index in self.cv.split(self.Xtrain,self.ytrain):\n",
    "            X_train_kfold, X_val_kfold = self.Xtrain[train_index], self.Xtrain[test_index]\n",
    "            y_train_kfold, y_val_kfold = self.ytrain[train_index], self.ytrain[test_index]\n",
    "\n",
    "            classifier = self.model\n",
    "\n",
    "            classifier.fit(X_train_kfold, y_train_kfold) \n",
    "            y_pred = classifier.predict(X_val_kfold)\n",
    "            y_probs = classifier.predict_proba(X_val_kfold)\n",
    "            \n",
    "            accuracy.append(np.round(metrics.accuracy_score(y_val_kfold, y_pred)*100,3))\n",
    "            f1_score.append(np.round(metrics.f1_score(y_val_kfold, y_pred)*100,3))\n",
    "            roc_auc_score.append(np.round(metrics.roc_auc_score(y_val_kfold, y_probs[:,1])*100,3))\n",
    "            \n",
    "            t.set_cols_align([\"c\", \"c\", \"c\", \"c\"])\n",
    "            t.add_row([fold_no, accuracy[fold_no-1],  f1_score[fold_no-1],  roc_auc_score[fold_no-1]])\n",
    "            print(t.draw())\n",
    "            t.reset()\n",
    "            \n",
    "            fold_no += 1\n",
    "        \n",
    "        print()\n",
    "        \n",
    "        train_preds = classifier.predict_proba(self.Xtrain)\n",
    "        \n",
    "        all_f = []\n",
    "        all_acc = []\n",
    "        \n",
    "        for t, i in zip(thresholds, range(len(thresholds))):\n",
    "            all_f.append(metrics.f1_score(self.ytrain, \n",
    "                                          to_labels(train_preds[:,1], t)))\n",
    "            \n",
    "            all_acc.append(metrics.accuracy_score(self.ytrain, \n",
    "                                                  to_labels(train_preds[:,1], t)))\n",
    "            \n",
    "            Progress(i, len(thresholds))\n",
    "            \n",
    "        print()\n",
    "    \n",
    "        fig = plt.figure(figsize = (12,5));\n",
    "        ax1 = fig.add_subplot(121);\n",
    "        ax2 = fig.add_subplot(122);\n",
    "        \n",
    "        #F1\n",
    "        sns.lineplot(x = thresholds, y = all_f, ax = ax1).set_title('Threshold for Max F-score');\n",
    "        \n",
    "        ax1.axvline(thresholds[np.argmax(all_f)], color = 'red', linestyle = '--', \n",
    "           lw = 2, label = f'Train Thresh = {thresholds[np.argmax(all_f)]*100:.2f} %');\n",
    "\n",
    "        ax1.legend(loc='upper right');\n",
    "        \n",
    "        #Accuracy\n",
    "        sns.lineplot(x = thresholds, y = all_acc, ax = ax2).set_title('Threshold for Max Accuracy');\n",
    "        \n",
    "        ax2.axvline(thresholds[np.argmax(all_acc)], color = 'red', linestyle = '--', \n",
    "           lw = 2, label = f'Train Thresh = {thresholds[np.argmax(all_acc)]*100:.2f} %');\n",
    "\n",
    "        ax2.legend(loc='upper right');\n",
    "        plt.tight_layout();\n",
    "        \n",
    "        print()\n",
    "        print(f'*** {model_name} Mean CV Scores ***')\n",
    "        print('=' * len(f'*** {model_name} Mean CV Scores ***'))\n",
    "        print(f'ROC AUC   : {np.mean(roc_auc_score):.3f} ± {np.std(roc_auc_score):.1f} %')\n",
    "        print(f'f1        : {np.mean(f1_score):.3f} ± {np.std(f1_score):.1f} %')\n",
    "        print(f'Accuracy  : {np.mean(accuracy):.3f} ± {np.std(accuracy):.1f} %')\n",
    "        print()\n",
    "        print(\"---\" * 40)\n",
    "\n",
    "        self.thresh_val = self.set_threshold\n",
    "        print()\n",
    "        print(f'Chosen Probability Threshold: {self.thresh_val*100:.2f} %')\n",
    "        \n",
    "        self.y_probs = classifier.predict_proba(self.Xtest)[:,1]\n",
    "        self.final_pred =  np.where(self.y_probs >= self.thresh_val, 1, 0)\n",
    "        \n",
    "        print()\n",
    "        print('Finish!')\n",
    "        \n",
    "#predict_unseen_data(model, Xtrain, ytrain, Xtest, cv, set_threshold)\n",
    "#.prediction()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Calibration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class calibrate_model:\n",
    "    \n",
    "    def __init__(self, model, Xtrain, ytrain, Xtest, ytest, cv):\n",
    "        self.model = model\n",
    "        self.Xtrain = Xtrain\n",
    "        self.ytrain = ytrain\n",
    "        self.Xtest = Xtest\n",
    "        self.ytest = ytest\n",
    "        self.cv = cv\n",
    "        \n",
    "        self.Xtrain = self.Xtrain.values\n",
    "        self.ytrain = self.ytrain[self.ytrain.columns[0]].values\n",
    "        self.Xtest = self.Xtest.values\n",
    "        self.ytest = self.ytest[self.ytest.columns[0]].values\n",
    "        \n",
    "    def calibrate_probability(self):\n",
    "        \n",
    "        from sklearn.model_selection import train_test_split\n",
    "        from sklearn.calibration import CalibratedClassifierCV\n",
    "        from sklearn.calibration import calibration_curve\n",
    "        \n",
    "        train_X, val_X, train_y, val_y = train_test_split(self.Xtrain, \n",
    "                                                          self.ytrain, \n",
    "                                                          test_size = 0.2, \n",
    "                                                          random_state = seed)\n",
    "        \n",
    "        \n",
    "        #uncalibrated model\n",
    "        \n",
    "        for train_index, test_index in self.cv.split(train_X, train_y): \n",
    "            X_train_kfold, X_val_kfold = train_X[train_index], train_X[test_index] \n",
    "            y_train_kfold, y_val_kfold = train_y[train_index], train_y[test_index] \n",
    "            self.model.fit(X_train_kfold, y_train_kfold)\n",
    "            \n",
    "        uc_probs = self.model.predict_proba(self.Xtest)[:, 1]\n",
    "        uc_fop, uc_mpv = calibration_curve(self.ytest, uc_probs, n_bins=10, normalize=True, \n",
    "                                           strategy = 'quantile')\n",
    "        \n",
    "        print()\n",
    "        print(f'Uncalibrated Brier Score: {metrics.brier_score_loss(self.ytest, uc_probs):.4f}')\n",
    "        print(f'Uncalibrated ROC-AUC: {metrics.roc_auc_score(self.ytest, uc_probs)*100:.3f}')\n",
    "    \n",
    "\n",
    "        #Calibrating Sigmoid Model\n",
    "        self.sigmoid_cal_model = CalibratedClassifierCV(self.model, method='sigmoid', cv=self.cv)\n",
    "        self.sigmoid_cal_model.fit(val_X, val_y)\n",
    "        \n",
    "        # predict probabilities\n",
    "        c_sig_probs = self.sigmoid_cal_model.predict_proba(self.Xtest)[:, 1]\n",
    "        \n",
    "        print()\n",
    "        print(f'Calibrated Sigmoid ROC-AUC: {metrics.roc_auc_score(self.ytest, c_sig_probs)*100:.3f} %')\n",
    "        print()\n",
    "        \n",
    "        #Calibrating isotonic Model\n",
    "        self.isotonic_cal_model = CalibratedClassifierCV(self.model, method='isotonic', cv=self.cv)\n",
    "        self.isotonic_cal_model.fit(val_X, val_y)\n",
    "        \n",
    "        # predict probabilities\n",
    "        c_iso_probs = self.isotonic_cal_model.predict_proba(self.Xtest)[:, 1]\n",
    "        \n",
    "        print(f'Calibrated isotonic ROC-AUC: {metrics.roc_auc_score(self.ytest, c_iso_probs)*100:.3f} %')\n",
    "        print()\n",
    "        \n",
    "        fig = plt.figure(figsize = (12,5));\n",
    "        ax1 = fig.add_subplot(121);\n",
    "        ax2 = fig.add_subplot(122);\n",
    "        # reliability diagram for sigmoid\n",
    "        c_fop, c_mpv = calibration_curve(self.ytest, c_sig_probs, n_bins=10, normalize=True,\n",
    "                                         strategy = 'quantile')\n",
    "\n",
    "        # plot calibrated\n",
    "        ax1.plot([0, 1], [0, 1], linestyle='--');\n",
    "\n",
    "        # plot un calibrated model reliability \n",
    "        ax1.plot(uc_mpv, uc_fop, marker='.', label = 'Uncalibrated');\n",
    "\n",
    "        # plot calibrated reliability\n",
    "        ax1.plot(c_mpv, c_fop, marker='.', \n",
    "                 label = f'Brier Score = {metrics.brier_score_loss(self.ytest, c_sig_probs):.4f}');\n",
    "\n",
    "        ax1.set_title('Calibration using Sigmoid')\n",
    "        ax1.set_ylabel('Fraction of Positives (fop)')\n",
    "        ax1.set_xlabel('Mean Predicted Value (mpv)')\n",
    "        ax1.legend();\n",
    "        plt.tight_layout()\n",
    "        \n",
    "        # reliability diagram for isotonic\n",
    "        c_fop, c_mpv = calibration_curve(self.ytest, c_iso_probs, n_bins=10, normalize=True,\n",
    "                                         strategy = 'quantile')\n",
    "\n",
    "        # plot CATBOOST calibrated\n",
    "        ax2.plot([0, 1], [0, 1], linestyle='--');\n",
    "\n",
    "        # plot un calibrated model reliability \n",
    "        ax2.plot(uc_mpv, uc_fop, marker='.', label = 'Uncalibrated');\n",
    "\n",
    "        # plot calibrated reliability\n",
    "        ax2.plot(c_mpv, c_fop, marker='.', \n",
    "                 label = f'Brier Score = {metrics.brier_score_loss(self.ytest, c_iso_probs):.4f}');\n",
    "\n",
    "        ax2.set_title('Calibration using Isotonic')\n",
    "        ax2.set_ylabel('Fraction of Positives (fop)')\n",
    "        ax2.set_xlabel('Mean Predicted Value (mpv)')\n",
    "        ax2.legend();\n",
    "        \n",
    "        plt.tight_layout()\n",
    "        \n",
    "#calibrate_model(model, Xtrain, ytrain, Xtest, ytest, cv)\n",
    "#.calibrate_probability()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "#BAckward Elimination\n",
    "\n",
    "from mlxtend.feature_selection import SequentialFeatureSelector as SFS\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "back = SFS(#model, k_features=(1,4),forward=False, floating=False, scoring = 'f1', \n",
    "            cv=StratifiedKFold(n_splits=10, random_state=seed, shuffle = True))\n",
    "\n",
    "back.fit(#X_train, y_train)\n",
    "\n",
    " \n",
    "print() \n",
    "print(f'Best Score: {back.k_score_*100:.3f} %')\n",
    "    \n",
    "back.k_feature_names_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "#Forward Selection\n",
    "\n",
    "from mlxtend.feature_selection import SequentialFeatureSelector as SFS\n",
    "\n",
    "front = SFS(#model, k_features=(1,4), scoring = 'f1', forward=True, \n",
    "            floating=False, cv=StratifiedKFold(n_splits=10, random_state=seed, shuffle = True))\n",
    "\n",
    "front.fit(#X_train, y_train)\n",
    "\n",
    " \n",
    "print()    \n",
    "print(f'Best Score: {front.k_score_*100:.3f} %')\n",
    "    \n",
    "front.k_feature_names_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "#Bidirectional Elimination\n",
    "\n",
    "from mlxtend.feature_selection import SequentialFeatureSelector as SFS\n",
    "\n",
    "bi = SFS(#model, k_features=(1,4), scoring = 'f1', forward=True, \n",
    "         floating=True, cv=StratifiedKFold(n_splits=10, random_state=seed, shuffle = True))\n",
    "\n",
    "bi.fit(#X_train, y_train)\n",
    "\n",
    " \n",
    "print() \n",
    "print(f'Best Score: {bi.k_score_*100:.3f} %')\n",
    "    \n",
    "bi.k_feature_names_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##########################################################################################################\n",
    "\"\"\"\n",
    "TAKES TOO MUCH TIME!!! AVOID AT ALL COST!\n",
    "\"\"\"\n",
    "%%time\n",
    "from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS\n",
    "\n",
    "efs = EFS(#model, min_features=1,max_features=2,scoring='f1',print_progress=True,\n",
    "          cv=StratifiedKFold(n_splits=5, random_state=seed, shuffle = True))\n",
    "\n",
    "efs.fit(#X_train, y_train)\n",
    "\n",
    " \n",
    "print()     \n",
    "print(f'Best Score: {efs.best_score_*100:.3f} %')    \n",
    "       \n",
    "efs.best_feature_names_ \n",
    "    \n",
    "###########################################################################################################"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Embedded Methods"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Lasso Regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Using regularization traceplots\n",
    "%%time\n",
    "from sklearn import linear_model\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "# #############################################################################\n",
    "# Compute paths\n",
    "\n",
    "n_alphas = 100\n",
    "alphas = np.linspace(0.0,1,n_alphas)\n",
    "\n",
    "coefs = []\n",
    "for a in alphas:\n",
    "    lasso = Pipeline(steps = [\n",
    "        ('scaler', StandardScaler()), \n",
    "        ('lasso', linear_model.Lasso(alpha=a, fit_intercept=False))\n",
    "    ])\n",
    "    \n",
    "    lasso.fit(#X_train_df, y_train_df)\n",
    "    coefs.append(lasso.named_steps.lasso.coef_)\n",
    "\n",
    "# #############################################################################\n",
    "# Display results\n",
    "label = #X_train_df.columns\n",
    "\n",
    "plt.figure(figsize = (10,7))\n",
    "ax = plt.gca()\n",
    "ax.plot(alphas, coefs);\n",
    "plt.axhline(y=0, color='black', linestyle='-')\n",
    "\n",
    "plt.xlabel('alpha')\n",
    "plt.ylabel('weights')\n",
    "plt.title('Lasso coefficients as a function of the regularization')\n",
    "plt.axis('tight')\n",
    "plt.legend(label, bbox_to_anchor=(1.05, 1), loc='upper left');\n",
    "\n",
    "coef_df = pd.DataFrame({'Features': label, 'Score':np.mean(coefs,0)})\n",
    "\n",
    "lasso_feat = [i for i in coef_df[coef_df['Score']>0]['Features']]\n",
    "print(f'Features Selected from Lasso Regularization: {lasso_feat}')\n",
    "coef_df.sort_values('Score', ascending = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Recursive Feature Eliminaion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "from yellowbrick.model_selection import RFECV\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "\n",
    " \n",
    " \n",
    "\n",
    "visualizer = RFECV(ExtraTreesClassifier(criterion = 'entropy', max_depth = 6), \n",
    "                   cv = StratifiedKFold(n_splits=5, random_state=seed, shuffle = True), \n",
    "                   scoring = 'f1')\n",
    "\n",
    "visualizer.fit(X_train_df, y_train_df)        # Fit the data to the visualizer\n",
    "visualizer.show();           # Finalize and render the figure\n",
    "\n",
    " \n",
    "print()    \n",
    "print(f'Feature Rankings: {visualizer.ranking_}')\n",
    "\n",
    "rfe_feat = [ind for ind, x in enumerate(visualizer.ranking_) if x==1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BORUTA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "from boruta import BorutaPy\n",
    "\n",
    "model = ExtraTreesClassifier(criterion = 'entropy', max_depth = 6, class_weight='balanced', n_jobs = -1)\n",
    "\n",
    "feat_selector = BorutaPy(model, n_estimators='auto', verbose=0, random_state=seed)\n",
    "\n",
    "feat_selector.fit(X_train.values, y_train.Loan_Status.values)\n",
    "\n",
    "print(f'Selected Features: {feat_selector.support_}')\n",
    "print()\n",
    "print(f'Feature Ranking: {feat_selector.ranking_}')\n",
    "print()\n",
    "boruta_feat = [X_train.columns[i] for i, feat in enumerate(feat_selector.ranking_) if feat==1]\n",
    "print(boruta_feat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ALL Feature Selection method Dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_features = {\n",
    "    'mutual_information': mi_feat, \n",
    "    'chi_squared': chi2_feat, \n",
    "    'ANOVA': anova_feat, \n",
    "    'back_elimination': list(back.k_feature_names_), \n",
    "    'front_elimination': list(front.k_feature_names_),\n",
    "    'bidirectional_elimination': list(bi.k_feature_names_),\n",
    "    'LASSO_regularization': lasso_feat, \n",
    "    'ExtraTrees_feature_imp': et_fi_feat, \n",
    "    'recursive_feature_elimination': rfe_feat, \n",
    "    'recursive_feature_addition': rfa_feat, \n",
    "    'select_by_shuffle': sel_shuff_feat, \n",
    "    'Boruta': boruta_feat\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Try all feature selection models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def best_features(train_X, train_Y, feature_dict, preprocessor, cv):\n",
    "    \n",
    "    models = [\n",
    "        LogisticRegression(random_state = seed),\n",
    "        LogisticRegressionCV(cv=10, random_state = seed),\n",
    "        SGDClassifier(tol = 0.1, early_stopping = True, validation_fraction = 0.2, random_state = seed),\n",
    "        Perceptron(tol = 0.1, early_stopping = True, validation_fraction = 0.2, random_state = seed),\n",
    "        PassiveAggressiveClassifier(tol = 0.1, early_stopping = True, validation_fraction = 0.2, random_state = seed),\n",
    "        RidgeClassifier(random_state = seed),\n",
    "        RidgeClassifierCV(cv=10),\n",
    "        LinearSVC(loss = 'hinge', random_state = seed),\n",
    "        SVC(kernel = 'rbf', random_state = seed),\n",
    "        NuSVC(random_state = seed),\n",
    "        KNeighborsClassifier(n_neighbors = 5, metric = 'minkowski', p = 2),\n",
    "        GaussianNB(), \n",
    "        BernoulliNB(),\n",
    "        MLPClassifier(tol=0.1, early_stopping = True, validation_fraction = 0.2, random_state = seed),\n",
    "        GaussianProcessClassifier(random_state = seed)\n",
    "    ]\n",
    "\n",
    "    tree_models = [\n",
    "        DecisionTreeClassifier(criterion = 'entropy', max_depth = 6, random_state = seed),\n",
    "        RandomForestClassifier(criterion='entropy', max_depth=6, class_weight='balanced', n_jobs=-1, random_state = seed), \n",
    "        XGBClassifier(use_label_encoder=False, eval_metric = 'error', seed = seed), \n",
    "        CatBoostClassifier(verbose = False, loss_function='CrossEntropy', eval_metric='TotalF1', random_seed = seed), \n",
    "        LGBMClassifier(random_state = seed), \n",
    "        AdaBoostClassifier(random_state = seed), \n",
    "        GradientBoostingClassifier(random_state = seed), \n",
    "        BaggingClassifier(random_state = seed), \n",
    "        ExtraTreesClassifier(criterion='entropy', max_depth=6, class_weight='balanced', n_jobs=-1, random_state = seed), \n",
    "        HistGradientBoostingClassifier(random_state = seed), \n",
    "        EasyEnsembleClassifier(random_state = seed), \n",
    "        RUSBoostClassifier(random_state = seed), \n",
    "        BalancedBaggingClassifier(random_state = seed), \n",
    "        BalancedRandomForestClassifier(n_estimators = 100, criterion = 'entropy', max_depth = 6, random_state = seed, class_weight = 'balanced'), \n",
    "        RGFClassifier(loss = 'Log', algorithm='RGF_Sib'), \n",
    "        FastRGFClassifier(loss='LOGISTIC'),\n",
    "        GPBoostClassifier(random_state = seed), \n",
    "        NGBClassifier(random_state = seed)\n",
    "    ]\n",
    "\n",
    "    select = []\n",
    "    top_acc = []\n",
    "    top_f1 = []\n",
    "    top_roc_auc = []\n",
    "    top_prec = []\n",
    "    top_recall = []\n",
    "    top_bal_acc = []\n",
    "    top_model = []\n",
    "\n",
    "    for name, feat, i in zip(feature_dict.keys(), \n",
    "                             feature_dict.values(), \n",
    "                             range(len(feature_dict.keys()))):\n",
    "        \n",
    "        Xtrain = train_X.loc[:,feat].values\n",
    "        ytrain = train_Y.values\n",
    "\n",
    "        acc = []\n",
    "        f1 = []\n",
    "        roc_auc = []\n",
    "        prec = []\n",
    "        recall = []\n",
    "        bal_acc = []\n",
    "        model_names = []\n",
    "\n",
    "        for model in models:\n",
    "            pipe = Pipeline(steps = [('preprocessor', preprocessor), (type(model).__name__, model)])\n",
    "\n",
    "            scores = cross_validate(pipe, \n",
    "                                    Xtrain, \n",
    "                                    ytrain,\n",
    "                                    scoring = ['accuracy', 'f1', 'roc_auc', 'precision', 'recall', \n",
    "                                               'balanced_accuracy'],\n",
    "                                    cv = cv, \n",
    "                                    n_jobs = -1)\n",
    "\n",
    "            acc.append(list(scores.values())[2].mean()*100)\n",
    "            f1.append(list(scores.values())[3].mean()*100)\n",
    "            roc_auc.append(list(scores.values())[4].mean()*100)\n",
    "            prec.append(list(scores.values())[5].mean()*100)\n",
    "            recall.append(list(scores.values())[6].mean()*100)\n",
    "            bal_acc.append(list(scores.values())[7].mean()*100)\n",
    "\n",
    "            model_names.append(type(model).__name__)\n",
    "\n",
    "        #Using Ensemble Models\n",
    "        for model in tree_models:\n",
    "\n",
    "            scores = cross_validate(model, \n",
    "                                    Xtrain, \n",
    "                                    ytrain,\n",
    "                                    scoring = ['accuracy', 'f1', 'roc_auc', 'precision', 'recall', \n",
    "                                               'balanced_accuracy'],\n",
    "                                    cv = cv, \n",
    "                                    n_jobs = -1)\n",
    "\n",
    "            acc.append(list(scores.values())[2].mean()*100)\n",
    "            f1.append(list(scores.values())[3].mean()*100)\n",
    "            roc_auc.append(list(scores.values())[4].mean()*100)\n",
    "            prec.append(list(scores.values())[5].mean()*100)\n",
    "            recall.append(list(scores.values())[6].mean()*100)\n",
    "            bal_acc.append(list(scores.values())[7].mean()*100)\n",
    "            model_names.append(type(model).__name__)\n",
    "\n",
    "        temp_df = pd.DataFrame({\n",
    "            'Models': model_names, \n",
    "            'Acc %': acc, \n",
    "            'f1 %':f1,\n",
    "            'ROC_AUC %' :roc_auc, \n",
    "            'Precision %': prec,\n",
    "            'Recall %': recall, \n",
    "            'Balanced_Acc %': bal_acc \n",
    "        }).sort_values('f1 %', ascending = False, ignore_index = True)\n",
    "\n",
    "        select.append(name)\n",
    "        top_model.append(temp_df.iloc[0,0])\n",
    "        top_acc.append(temp_df.iloc[0,1])\n",
    "        top_f1.append(temp_df.iloc[0,2])\n",
    "        top_roc_auc.append(temp_df.iloc[0,3])\n",
    "        top_prec.append(temp_df.iloc[0,4])\n",
    "        top_recall.append(temp_df.iloc[0,5])\n",
    "        top_bal_acc.append(temp_df.iloc[0,6])\n",
    "        \n",
    "        Progress(i, len(feature_dict.keys()))\n",
    "    print()\n",
    "\n",
    "    df = pd.DataFrame({\n",
    "        'Feature_Selection': select,\n",
    "        'Model': top_model, \n",
    "        'Acc': top_acc, \n",
    "        'f1': top_f1,\n",
    "        'ROC_AUC' : top_roc_auc, \n",
    "        'Prec': top_prec,\n",
    "        'Recall': top_recall, \n",
    "        'Bal_Acc': top_bal_acc \n",
    "    }).sort_values('f1', ascending = False, ignore_index = True).style.\\\n",
    "                                                    highlight_max(color = 'green')\n",
    "    \n",
    "    return df\n",
    "\n",
    "# best_features(train_X, train_Y, feature_dict, preprocessor, cv)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Variable Transformation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class num_transformer:\n",
    "    def __init__(self, x, variable):\n",
    "        self.x = x\n",
    "        self.variable = variable\n",
    "\n",
    "    def var_transform_plots(self):\n",
    "    \n",
    "        import scipy.stats as stats\n",
    "\n",
    "        fig, ax = plt.subplots(1,3, figsize = (15,5))\n",
    "\n",
    "        sns.histplot(self.x[self.variable], kde = True, ax = ax[0]);\n",
    "        sns.boxplot(self.x[self.variable], ax = ax[1]);\n",
    "        stats.probplot(self.x[self.variable], dist=\"norm\", plot=plt);\n",
    "        plt.suptitle('Current Distribution', fontsize = 20)\n",
    "\n",
    "        import feature_engine.transformation as vt\n",
    "        \n",
    "        self.log = vt.LogTransformer()\n",
    "        self.recipr = vt.ReciprocalTransformer()\n",
    "        self.exp = vt.PowerTransformer()\n",
    "        self.boxcox = vt.BoxCoxTransformer()\n",
    "        self.yeojohn = vt.YeoJohnsonTransformer()\n",
    "        \n",
    "        transformation_dict = dict(\n",
    "            log = self.log, \n",
    "            reciper = self.recipr, \n",
    "            exp = self.exp, \n",
    "            boxcox = self.boxcox, \n",
    "            yeojohn = self.yeojohn \n",
    "        )\n",
    "\n",
    "        for name, trnfm in transformation_dict.items():\n",
    "            try:\n",
    "                self.x[name + '_' + self.variable] = trnfm.fit_transform(self.x[self.variable].to_frame())\n",
    "\n",
    "            except ValueError:\n",
    "                print(f\"\"\"\n",
    "                      Some variables contain zero or negative values, can't apply {type(trnfm).__name__}\n",
    "                \"\"\")\n",
    "                continue\n",
    "\n",
    "            else:\n",
    "                fig, ax = plt.subplots(1,3, figsize = (15,5))\n",
    "                sns.histplot(self.x[name + '_' + self.variable], \n",
    "                             kde = True, ax = ax[0]).set(xlabel=self.variable);\n",
    "                \n",
    "                sns.boxplot(self.x[name + '_' + self.variable], \n",
    "                            ax = ax[1]).set(xlabel=self.variable);\n",
    "                \n",
    "                stats.probplot(self.x[name + '_' + self.variable], \n",
    "                               dist=\"norm\", \n",
    "                               plot=plt);\n",
    "                \n",
    "                plt.suptitle(type(trnfm).__name__, fontsize = 20)\n",
    "                plt.show()\n",
    "         \n",
    "        print(\"#\" * len('Normality Tests:'))\n",
    "        print('Normality Tests:')        \n",
    "        print(\"#\" * len('Normality Tests:'))\n",
    "        print()\n",
    "        print(\"Null hypothesis: Variable comes from a normal distribution\")\n",
    "        print()\n",
    "        \n",
    "        cols = self.x.iloc[:, -5:].columns\n",
    "        alpha = 1e-3\n",
    "        \n",
    "        for col in cols:\n",
    "            k2, p = stats.normaltest(self.x[col].values)\n",
    "            print(f'{col} P-Val: {p:.4f}')\n",
    "            if p < alpha: \n",
    "                print(f\"Reject H0, {col} is not normally distributed\")\n",
    "            else:\n",
    "                print(f\"Cannot Reject H0, {col} is normally distributed\")\n",
    "            print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}