Grid_Random_Search.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Class weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#class weights\n",
    "[{0:x, 1:1.0-x} for x in np.linspace(0.0,0.99,100)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# KNN\n",
    "%%time\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    " \n",
    "\n",
    "\n",
    "params = [\n",
    "    {'knn__algorithm': ['ball_tree', 'kd_tree', 'brute'], \n",
    "     'knn__leaf_size': np.arange(5,55,5),\n",
    "     'knn__n_neighbors': np.arange(1,11,1), \n",
    "     'knn__p': [1,2], \n",
    "     'knn__weights': ['uniform', 'distance'],\n",
    "     'knn__metric': ['euclidean', 'manhattan', 'chebyshev', 'minkowski']}\n",
    "]\n",
    "\n",
    "knn = Pipeline(steps = [\n",
    "    ('scaler', StandardScaler()), \n",
    "    ('knn', KNeighborsClassifier())\n",
    "])\n",
    "\n",
    "tuning = 1\n",
    "\n",
    "while tuning <=3:\n",
    "    grid_search = GridSearchCV(estimator = knn, \n",
    "                               param_grid = params, \n",
    "                               cv = StratifiedKFold(n_splits=10, random_state=seed, shuffle = True), \n",
    "                               scoring = 'f1', n_jobs = -1) #To use all processors\n",
    "\n",
    "    grid_search.fit(X_train_pars, y_train)\n",
    "    best_accuracy = grid_search.best_score_\n",
    "    best_param = grid_search.best_params_\n",
    "    print(f'Best Accuracy: {best_accuracy*100:.4f} %')\n",
    "    print(f'Best Parameters: {best_param}')\n",
    "    print()\n",
    "    tuning +=1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Support Vector Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time \n",
    "#KERNAL SVC\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "params = [\n",
    "    {'svc__C': np.arange(1.0, 11.1,1), \n",
    "     'svc__kernel': ['rbf'],\n",
    "     'svc__gamma': ['scale', 'auto'],\n",
    "     'svc__probability': [True], \n",
    "     'svc__class_weight': ['balanced']}, \n",
    "#     {'svc__C': np.arange(1.0, 11.1,1), \n",
    "#      'svc__kernel': ['poly'], \n",
    "#      'svc__degree': np.arange(1,7,1), \n",
    "#      'svc__gamma': ['scale', 'auto'], \n",
    "#      'svc__coef0': [0.0, 0.01, 0.1, 0.2], \n",
    "#      'svc__class_weight': ['balanced'], \n",
    "#      'svc__probability': [True]},\n",
    "    {'svc__C': np.arange(1.0, 11.1,1), \n",
    "     'svc__kernel': ['sigmoid'], \n",
    "     'svc__gamma': ['scale', 'auto'], \n",
    "     'svc__coef0': [0.0, 0.01, 0.1, 0.2], \n",
    "     'svc__probability': [True], \n",
    "     'svc__class_weight': ['balanced']}, \n",
    "    {'svc__C': np.arange(1.0, 11.1,1), \n",
    "     'svc__kernel': ['linear'], \n",
    "     'svc__probability': [True], \n",
    "     'svc__class_weight': ['balanced']}\n",
    "]\n",
    "\n",
    "svc = Pipeline(steps = [\n",
    "    ('scaler', StandardScaler()), \n",
    "    ('svc', SVC())\n",
    "])\n",
    "\n",
    "grid_search = RandomizedSearchCV(estimator = svc, \n",
    "                                 param_distributions = params, \n",
    "                                 cv = StratifiedKFold(n_splits=10, random_state=seed, shuffle = True), \n",
    "                                 scoring = 'roc_auc', \n",
    "                                 n_jobs = -1) #To use all processors\n",
    "\n",
    "tuning = 1\n",
    "\n",
    "while tuning <=5:\n",
    "    grid_search.fit(X_train_pars, y_train)\n",
    "    best_accuracy = grid_search.best_score_\n",
    "    best_param = grid_search.best_params_\n",
    "    print(f'tuning: {tuning}')\n",
    "    print(f'Best Accuracy: {best_accuracy*100:.4f} %')\n",
    "    print(f'Best Parameters: {best_param}')\n",
    "    print()\n",
    "    tuning += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# For Logistic Regression\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV, StratifiedKFold\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "params = [\n",
    "    {'logit_C': np.arange(0.1,10,0.1), \n",
    "     'logit_class_weight': [{0:x, 1:1.0-x} for x in np.linspace(0.0,0.99,100)], \n",
    "     'logit_penalty': ['l1', 'l2', 'elasticnet', 'none'], \n",
    "     'logit_solver': ['newton-cg', 'lbfgs', 'sag', 'saga']},\n",
    "    {'logit_C': np.arange(0.1,10,0.1),\n",
    "     'logit_class_weight': [{0:x, 1:1.0-x} for x in np.linspace(0.0,0.99,100)],\n",
    "     'logit_penalty': ['l1', 'l2', 'elasticnet'], \n",
    "     'logit_solver': ['liblinear']}\n",
    "]\n",
    "\n",
    "logit = Pipeline(steps = [\n",
    "    ('scaler', StandardScaler()), \n",
    "    ('logit', LogisticRegression())\n",
    "])\n",
    "\n",
    "grid_search = GridSearchCV(estimator = logit, \n",
    "                                 param_grid= params, \n",
    "                                 cv = StratifiedKFold(n_splits=10, random_state=seed, shuffle = True), \n",
    "                                 scoring = 'f1', \n",
    "                                 n_jobs = -1) #To use all processors\n",
    "tuning = 1\n",
    "while tuning <=5:\n",
    "    \n",
    "    grid_search.fit(X_train_pars, y_train)\n",
    "    best_accuracy = grid_search.best_score_\n",
    "    best_param = grid_search.best_params_\n",
    "    print(f'tuning: {tuning}')\n",
    "    print('Logistic Regression Params:')\n",
    "    print(f'Best Accuracy: {best_accuracy*100:.4f} %')\n",
    "    print(f'Best Parameters: {best_param}')\n",
    "    print()\n",
    "    tuning +=1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GaussianProcessClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# For Logistic Regression\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV, StratifiedKFold\n",
    "\n",
    "from sklearn.gaussian_process.kernels import (\n",
    "    RBF,\n",
    "    DotProduct, \n",
    "    Matern, \n",
    "    RationalQuadratic,\n",
    "    WhiteKernel, \n",
    "    ExpSineSquared \n",
    ")\n",
    "\n",
    "params = [\n",
    "    {'gpc__kernel': [1*RBF(), 1*DotProduct(), 1*Matern(),  1*RationalQuadratic(), 1*WhiteKernel(),\n",
    "                     1*ExpSineSquared()]}\n",
    "]\n",
    "\n",
    "gpc = Pipeline(steps = [\n",
    "    ('scaler', StandardScaler()), \n",
    "    ('gpc', GaussianProcessClassifier())\n",
    "])\n",
    "\n",
    "grid_search = GridSearchCV(estimator = gpc, \n",
    "                                 param_grid= params, \n",
    "                                 cv = StratifiedKFold(n_splits=10, random_state=seed, shuffle = True), \n",
    "                                 scoring = 'roc_auc', n_jobs = -1) #To use all processors\n",
    "tuning = 1\n",
    "while tuning <2:\n",
    "    \n",
    "    grid_search.fit(X_train_pars, y_train)\n",
    "    best_accuracy = grid_search.best_score_\n",
    "    best_param = grid_search.best_params_\n",
    "    print(f'tuning: {tuning}')\n",
    "    print('Logistic Regression Params:')\n",
    "    print(f'Best Accuracy: {best_accuracy*100:.4f} %')\n",
    "    print(f'Best Parameters: {best_param}')\n",
    "    print()\n",
    "    tuning +=1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LogisticRegressionCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# For Logistic Regression\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV, StratifiedKFold\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "params = [\n",
    "    {'logit_CV__Cs': np.arange(0,30,1), \n",
    "     'logit_CV__cv': [10],\n",
    "     'logit_CV__class_weight': ['balanced'], \n",
    "     'logit_CV__penalty': ['l1'], \n",
    "     'logit_CV__solver': ['liblinear', 'saga']},\n",
    "    {'logit_CV__Cs': np.arange(0,30,1),\n",
    "     'logit_CV__cv': [10],\n",
    "     'logit_CV__class_weight': ['balanced'],\n",
    "     'logit_CV__penalty': ['l2'], \n",
    "     'logit_CV__solver': ['newton-cg', 'lbfgs', 'sag']}, \n",
    "    {'logit_CV__Cs': np.arange(0,30,1),\n",
    "     'logit_CV__cv': [10],\n",
    "     'logit_CV__class_weight': ['balanced'],\n",
    "     'logit_CV__penalty': ['elasticnet'], \n",
    "     'logit_CV__solver': ['saga'], \n",
    "     'logit_CV__l1_ratios': [0.5]}\n",
    "]\n",
    "\n",
    "logit_cv = Pipeline(steps = [\n",
    "    ('scaler', StandardScaler()), \n",
    "    ('logit_CV', LogisticRegressionCV(scoring = 'roc_auc'))\n",
    "])\n",
    "\n",
    "\n",
    "grid_search = RandomizedSearchCV(estimator = logit_cv, \n",
    "                                 param_distributions= params, \n",
    "                                 cv = StratifiedKFold(n_splits=10, random_state=seed, shuffle = True), \n",
    "                                 scoring = 'roc_auc', n_jobs = -1) #To use all processors\n",
    "tuning = 1\n",
    "while tuning <=5:\n",
    "    \n",
    "    grid_search.fit(X_train_pars, y_train)\n",
    "    best_accuracy = grid_search.best_score_\n",
    "    best_param = grid_search.best_params_\n",
    "    print(f'tuning: {tuning}')\n",
    "    print('Logistic Regression Params:')\n",
    "    print(f'Best Accuracy: {best_accuracy*100:.4f} %')\n",
    "    print(f'Best Parameters: {best_param}')\n",
    "    print()\n",
    "    tuning +=1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gaussian NB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# For Logistic Regression\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV, StratifiedKFold\n",
    "\n",
    "params = [\n",
    "    {'gnb__var_smoothing': np.logspace(0,-9, num=100)}\n",
    "]\n",
    "\n",
    "gnb = Pipeline(steps = [\n",
    "    ('scaler', StandardScaler()), \n",
    "    ('gnb', GaussianNB())\n",
    "])\n",
    "\n",
    "grid_search = GridSearchCV(estimator = gnb, \n",
    "                                 param_grid= params, \n",
    "                                 cv = StratifiedKFold(n_splits=10, random_state=seed, shuffle = True), \n",
    "                                 scoring = 'roc_auc', n_jobs = -1) #To use all processors\n",
    "tuning = 1\n",
    "while tuning <2:\n",
    "    \n",
    "    grid_search.fit(X_train_pars, y_train)\n",
    "    best_accuracy = grid_search.best_score_\n",
    "    best_param = grid_search.best_params_\n",
    "    print(f'tuning: {tuning}')\n",
    "    print('Logistic Regression Params:')\n",
    "    print(f'Best Accuracy: {best_accuracy*100:.4f} %')\n",
    "    print(f'Best Parameters: {best_param}')\n",
    "    print()\n",
    "    tuning +=1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Random Forest\n",
    "%%time\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "params = [\n",
    "    {'n_estimators': np.arange(50,1050,50), \n",
    "     'criterion': ['gini', 'entropy'], \n",
    "     'max_depth': np.arange(1,7,1), \n",
    "     'min_samples_split': [2,3,10], \n",
    "     'min_samples_leaf': [1,2,10], \n",
    "     'max_features': ['auto', 'sqrt', 'log2'], \n",
    "     'ccp_alpha': [0.0, 0.001, 0.015,0.035], \n",
    "     'bootstrap':[True, False], \n",
    "     'class_weight': [\"balanced\", \"balanced_subsample\"]}\n",
    "]\n",
    "\n",
    "grid_search = RandomizedSearchCV(estimator = RandomForestClassifier(), \n",
    "                           param_distributions = params, cv = skf, \n",
    "                           scoring = 'f1', n_jobs = -1) #To use all processors\n",
    "\n",
    "tuning = 1\n",
    "\n",
    "while tuning <=5:\n",
    "    grid_search.fit(#X_train, y_train)\n",
    "    best_accuracy = grid_search.best_score_\n",
    "    best_param = grid_search.best_params_\n",
    "    print(f'tuning: {tuning}')\n",
    "    print(f'Best Accuracy: {best_accuracy*100:.4f} %')\n",
    "    print(f'Best Parameters: {best_param}')\n",
    "    print()\n",
    "    tuning += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Balanced Random Forest Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Balanced Random Forest Classifier\n",
    "%%time\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n",
    "from imblearn.ensemble import BalancedRandomForestClassifier\n",
    "\n",
    "params = [\n",
    "    {'n_estimators': np.arange(50,1050,50), \n",
    "     'criterion': ['gini', 'entropy'], \n",
    "     'max_depth': np.arange(1,7,1), \n",
    "     'min_samples_split': [1,2,3,4,5,10], \n",
    "     'min_samples_leaf': [1,2,3,4,5,10], \n",
    "     'max_features': ['auto', 'sqrt', 'log2'],\n",
    "     'max_leaf_nodes': [None,1,2,3,4],\n",
    "     'ccp_alpha': [0.0, 0.001, 0.015,0.035], \n",
    "     'bootstrap':[True, False], \n",
    "     'replacement': [True, False],\n",
    "     'class_weight': ['balanced', 'balanced_subsample'], \n",
    "     'max_samples': [1.0,0.8,0.5,0.3]}\n",
    "]\n",
    "\n",
    "grid_search = RandomizedSearchCV(estimator = BalancedRandomForestClassifier(), \n",
    "                           param_distributions = params, cv = skf, \n",
    "                           scoring = 'f1', n_jobs = -1) #To use all processors\n",
    "\n",
    "tuning = 1\n",
    "\n",
    "while tuning <=5:\n",
    "    grid_search.fit(X_train_pars_df, y_train_df)\n",
    "    best_accuracy = grid_search.best_score_\n",
    "    best_param = grid_search.best_params_\n",
    "    print(f'tuning: {tuning}')\n",
    "    print(f'Best Accuracy: {best_accuracy*100:.4f} %')\n",
    "    print(f'Best Parameters: {best_param}')\n",
    "    print()\n",
    "    tuning += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gradient Boosting Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Gradient Boosting Classifier\n",
    "%%time\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "params = [\n",
    "    {'loss': ['deviance', 'exponential'], \n",
    "     'learning_rate': [0.3, 0.2,0.15,0.1,0.05,0.01,0.005,0.001], \n",
    "     'n_estimators': [100,250,500,750,1000,1250,1500,1750],\n",
    "     'min_samples_split': [2,4,6,8,10,20,40,60,100], \n",
    "     'min_samples_leaf': [1,3,5,7,9], \n",
    "     'max_depth':[3,4,5,6,7], \n",
    "     'max_features': ['sqrt', 'log2'], \n",
    "     'ccp_alpha': [0.0,0.015,0.035], \n",
    "     'subsample':[0.5,0.7,0.75,0.8,0.85,0.9,0.95,1]}\n",
    "]\n",
    "\n",
    "skf = StratifiedKFold(n_splits=5, random_state=seed, shuffle = True)\n",
    "\n",
    "grid_search = RandomizedSearchCV(estimator = GradientBoostingClassifier(), \n",
    "                           param_distributions = params, cv = skf, \n",
    "                           scoring = 'accuracy', n_jobs = -1) #To use all processors\n",
    "\n",
    "tuning = 1\n",
    "\n",
    "while tuning <=5:\n",
    "    grid_search.fit(#X_train, y_train)\n",
    "    best_accuracy = grid_search.best_score_\n",
    "    best_param = grid_search.best_params_\n",
    "    print(f'tuning: {tuning}')\n",
    "    print(f'Best Accuracy: {best_accuracy*100:.4f} %')\n",
    "    print(f'Best Parameters: {best_param}')\n",
    "    print()\n",
    "    tuning += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Decision Tree\n",
    "%%time\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "params = [{\n",
    "    'criterion': ['gini', 'entropy'],\n",
    "    'splitter': ['best', 'random'],\n",
    "    'max_depth': np.arange(1,7,1),\n",
    "    'min_samples_split': [2,3,10],\n",
    "    'min_samples_leaf': [1,2,10],\n",
    "    'max_features': ['auto', 'sqrt', 'log2'],\n",
    "    'class_weight': ['balanced', None],\n",
    "    'ccp_alpha': [0.0, 0.001, 0.015,0.035]\n",
    "}]\n",
    "\n",
    "skf = StratifiedKFold(n_splits=10, random_State=seed, shuffle = True)\n",
    "\n",
    "grid_search = RandomizedSearchCV(estimator = DecisionTreeClassifier(), \n",
    "                           param_distributions = params, cv = skf, \n",
    "                           scoring = 'f1', n_jobs = -1) #To use all processors\n",
    "\n",
    "tuning = 1\n",
    "\n",
    "while tuning <=5:\n",
    "    grid_search.fit(#X_train, y_train)\n",
    "    best_accuracy = grid_search.best_score_\n",
    "    best_param = grid_search.best_params_\n",
    "    print(f'tuning: {tuning}')\n",
    "    print(f'Best Accuracy: {best_accuracy*100:.4f} %')\n",
    "    print(f'Best Parameters: {best_param}')\n",
    "    print()\n",
    "    tuning += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ExtraTree Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ExtraTree Classifier\n",
    "%%time\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "\n",
    "params = [\n",
    "    {'n_estimators': np.arange(50,1050,50), \n",
    "     'criterion': ['gini', 'entropy'],\n",
    "     'max_depth': np.arange(1,7,1), \n",
    "     'min_samples_split':[2,3,10], \n",
    "     'min_samples_leaf': [1,2,10], \n",
    "     'max_features': ['auto', 'sqrt', 'log2'], \n",
    "     'bootstrap': [True, False], \n",
    "     'n_jobs': [-1], \n",
    "     'class_weight': ['balanced', 'balanced_subsample'], \n",
    "     'ccp_alpha':[0.0,0.01,0.015,0.035]}\n",
    "]\n",
    "\n",
    "grid_search = RandomizedSearchCV(estimator = ExtraTreesClassifier(), \n",
    "                           param_distributions = params, cv = skf, \n",
    "                           scoring = 'f1', n_jobs = -1) #To use all processors\n",
    "\n",
    "tuning = 1\n",
    "\n",
    "while tuning <=5:\n",
    "    grid_search.fit(#X_train, y_train)\n",
    "    best_accuracy = grid_search.best_score_\n",
    "    best_param = grid_search.best_params_\n",
    "    print(f'tuning: {tuning}')\n",
    "    print(f'Best Accuracy: {best_accuracy*100:.4f} %')\n",
    "    print(f'Best Parameters: {best_param}')\n",
    "    print()\n",
    "    tuning += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBOOST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "params = [\n",
    "    {'booster': ['dart', 'gbtree', 'gblinear'], \n",
    "     'learning_rate': [0.15,0.1,0.05,0.01,0.005,0.001],\n",
    "     'max_depth': [3, 4, 5, 6, 7],\n",
    "     \"min_child_weight\" : [1, 3, 5, 7, 10, 15, 20],\n",
    "     \"gamma\" : [ 0.0, 0.1, 0.2 , 0.3, 0.4, 1, 1.5, 2, 5],\n",
    "     'subsample': [0.5, 0.6, 0.7, 0.8, 1.0],\n",
    "     'colsample_bytree': [0.3, 0.4, 0.5 , 0.6, 0.7, 0.8, 1.0],\n",
    "     'n_estimators' : [100, 200, 500,800,1000], \n",
    "     'base_score': [0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6, 0.65], \n",
    "     'max_delta_step': [0, 1, 2, 3, 5, 10],\n",
    "     'reg_alpha': [0, 0.5, 1, 1.5, 2],\n",
    "     'objective': ['binary:logistic']}\n",
    "]\n",
    "\n",
    "grid_search = RandomizedSearchCV(estimator = XGBClassifier(use_label_encoder=False, eval_metric = 'error'),\n",
    "                                 param_distributions = params, \n",
    "                                 cv = skf, \n",
    "                                 scoring = 'f1', \n",
    "                                 n_jobs = -1) #To use all processors\n",
    "\n",
    "tuning = 1\n",
    "\n",
    "while tuning <=5:\n",
    "    grid_search.fit(#X_train, y_train)\n",
    "    best_accuracy = grid_search.best_score_\n",
    "    best_param = grid_search.best_params_\n",
    "    print(f'tuning: {tuning}')\n",
    "    print(f'Best Accuracy: {best_accuracy*100} %')\n",
    "    print(f'Best Parameters: {best_param}')\n",
    "    print()\n",
    "    tuning += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CatBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n",
    "from catboost import CatBoostClassifier\n",
    "\n",
    "params = [{\n",
    "    'depth':[1,2,3,4,5,6,7],\n",
    "    'iterations':[250,100,500,1000],\n",
    "    'learning_rate':[0.001,0.005,0.03,0.02,0.01,0.1,0.2,0.3], \n",
    "    'l2_leaf_reg':[3.0,1.0,5.0,10.0,100.0],\n",
    "    'border_count':[32,5,10,20,50,100,200], \n",
    "    'boosting_type': ['Ordered', 'Plain']\n",
    "}]\n",
    "\n",
    "cat = CatBoostClassifier(verbose = False, loss_function='CrossEntropy', eval_metric='TotalF1')\n",
    "\n",
    "grid_search = RandomizedSearchCV(estimator = cat, param_distributions = params, cv = skf, scoring = 'f1', \n",
    "                                 n_jobs = -1) #To use all processors\n",
    "\n",
    "tuning = 1\n",
    "\n",
    "while tuning <=5:\n",
    "    grid_search.fit(#X_train, y_train)\n",
    "    best_accuracy = grid_search.best_score_\n",
    "    best_param = grid_search.best_params_\n",
    "    print(f'tuning: {tuning}')\n",
    "    print(f'Best Accuracy: {best_accuracy*100:.4f} %')\n",
    "    print(f'Best Parameters: {best_param}')\n",
    "    print()\n",
    "    tuning += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bagging Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "\n",
    "params = [{\n",
    "    'n_estimators': np.arange(5,1005,5), \n",
    "    'max_samples': [1.0, 0.8, 0.5, 0.3], \n",
    "    'max_features': [1.0, 0.8, 0.5, 0.3], \n",
    "    'bootstrap' : [True, False], \n",
    "    'bootstrap_features' : [True, False], \n",
    "    'warm_start': [True, False], \n",
    "    'n_jobs': [-1]\n",
    "}]\n",
    "\n",
    "grid_search = RandomizedSearchCV(estimator = BaggingClassifier(), \n",
    "                                 param_distributions = params, cv = skf, scoring = 'f1', \n",
    "                                 n_jobs = -1) #To use all processors\n",
    "\n",
    "tuning = 1\n",
    "\n",
    "while tuning <=5:\n",
    "    grid_search.fit(#X_train, y_train)\n",
    "    best_accuracy = grid_search.best_score_\n",
    "    best_param = grid_search.best_params_\n",
    "    print(f'tuning: {tuning}')\n",
    "    print(f'Best Accuracy: {best_accuracy*100:.4f} %')\n",
    "    print(f'Best Parameters: {best_param}')\n",
    "    print()\n",
    "    tuning += 1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}