mushroom_models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import LabelEncoder, StandardScaler\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, precision_score, f1_score, log_loss, roc_curve, roc_auc_score, ConfusionMatrixDisplay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read in mushrooms dataset\n",
    "mushrooms = pd.read_csv(\"secondary_data.csv\", sep = \";\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Label encoding the response variable and categorical features\n",
    "le = LabelEncoder()\n",
    "mask = mushrooms.isna()\n",
    "cols_to_encode = mushrooms.columns.drop([\"cap-diameter\", \"stem-width\", \"stem-height\"])\n",
    "\n",
    "for col in cols_to_encode:\n",
    "    mushrooms[col] = le.fit_transform(mushrooms[col])\n",
    "\n",
    "mushrooms = mushrooms.where(~ mask, np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove NaN values from dataset\n",
    "mushrooms = mushrooms.drop(columns = [\"veil-type\", \"veil-color\", \"spore-print-color\", \"stem-root\", \"stem-surface\", \"gill-spacing\"])\n",
    "mushrooms = mushrooms[mushrooms[\"cap-surface\"].notnull() & mushrooms[\"gill-attachment\"].notnull() & mushrooms[\"ring-type\"].notnull()]\n",
    "mushrooms.reset_index(drop = True, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the dataset into X (predictors) and y (response)\n",
    "X = mushrooms.drop('class', axis=1)\n",
    "y = mushrooms['class']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Square root transform numeric features and drop \"veil-type\" feature\n",
    "X[\"stem-height\"] = np.sqrt(X[\"stem-height\"])\n",
    "X[\"stem-width\"] = np.sqrt(X[\"stem-width\"])\n",
    "X[\"cap-diameter\"] = np.sqrt(X[\"cap-diameter\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Z-transform the numeric features\n",
    "quantitative_vars = X[[\"cap-diameter\", \"stem-width\", \"stem-height\"]]\n",
    "X.drop([\"cap-diameter\", \"stem-width\", \"stem-height\"], axis=1, inplace=True)\n",
    "sc = StandardScaler()\n",
    "sc.fit(quantitative_vars)\n",
    "x_scaled=sc.transform(quantitative_vars)\n",
    "\n",
    "quant_scaled=pd.DataFrame(data=x_scaled,columns=[\"cap-diameter\", \"stem-width\", \"stem-height\"])\n",
    "X_scaled = pd.concat([quant_scaled, X], axis = 1)\n",
    "X_scaled[[\"cap-surface\", \"gill-attachment\", \"ring-type\"]] = X_scaled[[\"cap-surface\", \"gill-attachment\", \"ring-type\"]].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train-test split dataset\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Running Logistic Regression model\n",
    "\n",
    "# Initialize logistic regression model\n",
    "model_lr = LogisticRegression()\n",
    "\n",
    "# param grid for Logistic Regression\n",
    "param_grid = {\n",
    "    'penalty': ['l1', 'l2'],\n",
    "    'C': [0.001, 0.01, 0.1, 1, 10, 100],\n",
    "    'solver': ['liblinear', 'saga'],\n",
    "    'max_iter': [100, 200, 300, 500]\n",
    "}\n",
    "lr_grid = GridSearchCV(model_lr, param_grid, cv=5)\n",
    "lr_grid.fit(X_train, y_train)\n",
    "print(lr_grid.best_params_)\n",
    "\n",
    "# Predict on the test data using the best model\n",
    "lr_grid_predictions = lr_grid.predict(X_test)\n",
    "\n",
    "# Calculate model performance metrics\n",
    "lr_acc = accuracy_score(y_test, lr_grid_predictions)\n",
    "lr_prec = precision_score(y_test, lr_grid_predictions)\n",
    "lr_f1 = f1_score(y_test, lr_grid_predictions)\n",
    "lr_logloss = log_loss(y_test, lr_grid_predictions)\n",
    "\n",
    "# Print the performance metrics\n",
    "print(\"Accuracy:\", round(lr_acc, ndigits = 3))\n",
    "print(\"Precision:\", round(lr_prec, ndigits = 3))\n",
    "print(\"F1 Score:\", round(lr_f1, ndigits = 3))\n",
    "print(\"Log Loss:\", round(lr_logloss, ndigits = 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Running Linear Discriminant Analysis model\n",
    "\n",
    "# Initialize LDA model\n",
    "model_lda = LinearDiscriminantAnalysis()\n",
    "\n",
    "# param grid for LDA\n",
    "param_grid = {\n",
    "    'solver': ['lsqr', 'eigen'],\n",
    "    'shrinkage': [None, 'auto'] + list(np.linspace(0, 1, 50)),\n",
    "    'n_components': [None, 1],\n",
    "    'store_covariance': [True, False]\n",
    "}\n",
    "\n",
    "lda_grid = GridSearchCV(model_lda, param_grid, cv=5)\n",
    "lda_grid.fit(X_train, y_train)\n",
    "print(lda_grid.best_params_)\n",
    "\n",
    "# Predict on the test data using the best model\n",
    "lda_grid_predictions = lda_grid.predict(X_test)\n",
    "\n",
    "print('--------------------------------------------------')\n",
    "\n",
    "# Calculate model performance metrics\n",
    "lda_acc = accuracy_score(y_test, lda_grid_predictions)\n",
    "lda_prec = precision_score(y_test, lda_grid_predictions)\n",
    "lda_f1 = f1_score(y_test, lda_grid_predictions)\n",
    "lda_logloss = log_loss(y_test, lda_grid_predictions)\n",
    "\n",
    "# Print the performance metrics\n",
    "print(\"Accuracy:\", round(lda_acc, ndigits = 3))\n",
    "print(\"Precision:\", round(lda_prec, ndigits = 3))\n",
    "print(\"F1 Score:\", round(lda_f1, ndigits = 3))\n",
    "print(\"Log Loss:\", round(lda_logloss, ndigits = 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Running k-Nearest Neighbours model\n",
    "\n",
    "# Initialize kNN model\n",
    "model_knn = KNeighborsClassifier()\n",
    "\n",
    "# param_grid for k-Nearest Neighbours\n",
    "param_grid = {\n",
    "    'n_neighbors': [3, 5, 7, 9],\n",
    "    'weights': ['uniform', 'distance'],\n",
    "    'algorithm': ['auto', 'ball_tree', 'kd_tree', 'brute'],\n",
    "    'p': [1, 2]\n",
    "}\n",
    "knn_grid = GridSearchCV(model_knn, param_grid, cv=5)\n",
    "knn_grid.fit(X_train, y_train)\n",
    "print(knn_grid.best_params_)\n",
    "\n",
    "# Predict the model using the best parameters\n",
    "knn_grid_predictions = knn_grid.predict(X_test)\n",
    "\n",
    "print('--------------------------------------------------')\n",
    "\n",
    "knn_acc = accuracy_score(y_test, knn_grid_predictions)\n",
    "knn_prec = precision_score(y_test, knn_grid_predictions)\n",
    "knn_f1 = f1_score(y_test, knn_grid_predictions)\n",
    "knn_logloss = log_loss(y_test, knn_grid_predictions)\n",
    "\n",
    "print(\"Accuracy:\", round(knn_acc, ndigits = 3))\n",
    "print(\"Precision:\", round(knn_prec, ndigits = 3))\n",
    "print(\"F1 Score:\", round(knn_f1, ndigits = 3))\n",
    "print(\"Log Loss:\", round(knn_logloss, ndigits = 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Running Random Forest model\n",
    "\n",
    "# Initialize random forest classifier\n",
    "model_rf = RandomForestClassifier()\n",
    "\n",
    "# param_grid for random forest classifier\n",
    "param_grid = {\n",
    "    'n_estimators': [100, 200, 300],\n",
    "    'max_depth': [1, 5, 10],\n",
    "    'min_samples_split': [2, 5, 10]\n",
    "}\n",
    "rf_grid = GridSearchCV(model_rf, param_grid, cv=5)\n",
    "rf_grid.fit(X_train, y_train)\n",
    "print(rf_grid.best_params_) \n",
    "\n",
    "# Predict the model using the best parameters\n",
    "rf_grid_predictions = rf_grid.predict(X_test)\n",
    "\n",
    "print('--------------------------------------------------')\n",
    "\n",
    "rf_acc = accuracy_score(y_test, rf_grid_predictions)\n",
    "rf_prec = precision_score(y_test, rf_grid_predictions)\n",
    "rf_f1 = f1_score(y_test, rf_grid_predictions)\n",
    "rf_logloss = log_loss(y_test, rf_grid_predictions)\n",
    "\n",
    "print(\"Accuracy:\", round(rf_acc, ndigits = 3))\n",
    "print(\"Precision:\", round(rf_prec, ndigits = 3))\n",
    "print(\"F1 Score:\", round(rf_f1, ndigits = 3))\n",
    "print(\"Log Loss:\", round(rf_logloss, ndigits = 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Running Boosting model\n",
    "\n",
    "# Initialize boosting classifier\n",
    "gbc_model = GradientBoostingClassifier()\n",
    "\n",
    "# param_grid for boosting classifier\n",
    "param_grid = {\n",
    "    'loss': ['log_loss', 'exponential'],\n",
    "    'learning_rate': [0.01, 0.5, 1],\n",
    "    'max_depth': [1, 3, 5],\n",
    "}\n",
    "gbc_grid = GridSearchCV(gbc_model, param_grid, cv=5)\n",
    "gbc_grid.fit(X_train, y_train)\n",
    "print(gbc_grid.best_params_) \n",
    "\n",
    "# Predict the model using the best parameters\n",
    "gbc_grid_predictions = gbc_grid.predict(X_test)\n",
    "\n",
    "print('--------------------------------------------------')\n",
    "\n",
    "gbc_acc = accuracy_score(y_test, gbc_grid_predictions)\n",
    "gbc_prec = precision_score(y_test, gbc_grid_predictions)\n",
    "gbc_f1 = f1_score(y_test, gbc_grid_predictions)\n",
    "gbc_logloss = log_loss(y_test, gbc_grid_predictions)\n",
    "\n",
    "print(\"Accuracy:\", round(gbc_acc, ndigits = 3))\n",
    "print(\"Precision:\", round(gbc_prec, ndigits = 3))\n",
    "print(\"F1 Score:\", round(gbc_f1, ndigits = 3))\n",
    "print(\"Log Loss:\", round(gbc_logloss, ndigits = 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Confusion Matrices\n",
    "\n",
    "fig, axes = plt.subplots(3, 2, figsize=(15, 20))\n",
    "\n",
    "# Logistic Regression\n",
    "cm_lr = confusion_matrix(y_test, lr_grid_predictions)\n",
    "disp_lr = ConfusionMatrixDisplay(cm_lr)\n",
    "disp_lr.plot(ax=axes[0, 0])\n",
    "axes[0, 0].set_title('Confusion Matrix - Logistic Regression', fontsize = 16)\n",
    "\n",
    "# LDA\n",
    "cm_lda = confusion_matrix(y_test, lda_grid_predictions)\n",
    "disp_lda = ConfusionMatrixDisplay(cm_lda)\n",
    "disp_lda.plot(ax=axes[0, 1])\n",
    "axes[0, 1].set_title('Confusion Matrix - Linear Discriminant Analysis', fontsize = 16)\n",
    "\n",
    "# kNN\n",
    "cm_knn = confusion_matrix(y_test, knn_grid_predictions)\n",
    "disp_knn = ConfusionMatrixDisplay(cm_knn)\n",
    "disp_knn.plot(ax=axes[1, 0])\n",
    "axes[1, 0].set_title('Confusion Matrix - K Nearest Neighbours', fontsize = 16)\n",
    "\n",
    "# Random Forest\n",
    "cm_rf = confusion_matrix(y_test, rf_grid_predictions)\n",
    "disp_rf = ConfusionMatrixDisplay(cm_rf)\n",
    "disp_rf.plot(ax=axes[1, 1])\n",
    "axes[1, 1].set_title('Confusion Matrix - Random Forest', fontsize = 16)\n",
    "\n",
    "# Boosting\n",
    "cm_gbc = confusion_matrix(y_test, gbc_grid_predictions)\n",
    "disp_gbc = ConfusionMatrixDisplay(cm_gbc)\n",
    "disp_gbc.plot(ax=axes[2, 0])\n",
    "axes[2, 0].set_title('Confusion Matrix - Boosting', fontsize = 16)\n",
    "\n",
    "# Hide empty subplot\n",
    "axes[2, 1].axis('off')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ROC curves\n",
    "lr_grid_probabilities = lr_grid.predict_proba(X_test)[:, 1]\n",
    "fpr_lr, tpr_lr, thresholds_lr = roc_curve(y_test, lr_grid_probabilities)\n",
    "roc_auc_lr = roc_auc_score(y_test, lr_grid_probabilities)\n",
    "\n",
    "lda_grid_probabilities = lda_grid.predict_proba(X_test)[:, 1]\n",
    "fpr_lda, tpr_lda, thresholds_lda = roc_curve(y_test, lda_grid_probabilities)\n",
    "roc_auc_lda = roc_auc_score(y_test, lda_grid_probabilities)\n",
    "\n",
    "knn_grid_probabilities = knn_grid.predict_proba(X_test)[:, 1]\n",
    "fpr_knn, tpr_knn, thresholds_knn = roc_curve(y_test, knn_grid_probabilities)\n",
    "roc_auc_knn = roc_auc_score(y_test, knn_grid_probabilities)\n",
    "\n",
    "rf_grid_probabilities = rf_grid.predict_proba(X_test)[:, 1]\n",
    "fpr_rf, tpr_rf, thresholds_rf = roc_curve(y_test, rf_grid_probabilities)\n",
    "roc_auc_rf = roc_auc_score(y_test, rf_grid_probabilities)\n",
    "\n",
    "gbc_grid_probabilities = gbc_grid.predict_proba(X_test)[:, 1]\n",
    "fpr_gbc, tpr_gbc, thresholds_gbc = roc_curve(y_test, gbc_grid_probabilities)\n",
    "roc_auc_gbc = roc_auc_score(y_test, gbc_grid_probabilities)\n",
    "\n",
    "plt.figure(figsize=(14, 18))\n",
    "\n",
    "# Logistic Regression\n",
    "plt.subplot(3, 2, 1)\n",
    "plt.plot(fpr_lr, tpr_lr, color='blue', lw=2, label=f'ROC Curve (AUC = {roc_auc_lr:.2f})')\n",
    "plt.plot([0, 1], [0, 1], color='gray', linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC Curve - Logistic Regression', fontsize = 16)\n",
    "plt.legend(loc='lower right')\n",
    "\n",
    "# Linear Discriminant Analysis\n",
    "plt.subplot(3, 2, 2)\n",
    "plt.plot(fpr_lda, tpr_lda, color='blue', lw=2, label=f'ROC Curve (AUC = {roc_auc_lda:.2f})')\n",
    "plt.plot([0, 1], [0, 1], color='gray', linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC Curve - Linear Discriminant Analysis', fontsize = 16)\n",
    "plt.legend(loc='lower right')\n",
    "\n",
    "# K Nearest Neighbours\n",
    "plt.subplot(3, 2, 3)\n",
    "plt.plot(fpr_knn, tpr_knn, color='blue', lw=2, label=f'ROC Curve (AUC = {roc_auc_knn:.2f})')\n",
    "plt.plot([0, 1], [0, 1], color='gray', linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC Curve - K Nearest Neighbours',  fontsize = 16)\n",
    "plt.legend(loc='lower right')\n",
    "\n",
    "# Random Forest\n",
    "plt.subplot(3, 2, 4)\n",
    "plt.plot(fpr_rf, tpr_rf, color='blue', lw=2, label=f'ROC Curve (AUC = {roc_auc_rf:.2f})')\n",
    "plt.plot([0, 1], [0, 1], color='gray', linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC Curve - Random Forest',  fontsize = 16)\n",
    "plt.legend(loc='lower right')\n",
    "\n",
    "# Boosting\n",
    "plt.subplot(3, 2, 5)\n",
    "plt.plot(fpr_gbc, tpr_gbc, color='blue', lw=2, label=f'ROC Curve (AUC = {roc_auc_gbc:.2f})')\n",
    "plt.plot([0, 1], [0, 1], color='gray', linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC Curve - Gradient Boost',  fontsize = 16)\n",
    "plt.legend(loc='lower right')\n",
    "\n",
    "# Adjust layout\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# feature importance in random forest and boosting models\n",
    "\n",
    "\n",
    "feature_importances_rf = rf_grid.best_estimator_.feature_importances_\n",
    "feature_importance_df_rf = pd.DataFrame({'Feature': X_scaled.columns, 'Importance': feature_importances_rf})\n",
    "feature_importance_df_rf = feature_importance_df_rf.sort_values(by='Importance', ascending=False).reset_index(drop = True)\n",
    "\n",
    "feature_importances_gbc = gbc_grid.best_estimator_.feature_importances_\n",
    "feature_importance_df_gbc = pd.DataFrame({'Feature': X_scaled.columns, 'Importance': feature_importances_gbc})\n",
    "feature_importance_df_gbc = feature_importance_df_gbc.sort_values(by='Importance', ascending=False).reset_index(drop = True)\n",
    "\n",
    "fig, ax = plt.subplots(1,2, figsize = (16,8))\n",
    "ax[0].barh(feature_importance_df_rf['Feature'], feature_importance_df_rf['Importance'])\n",
    "ax[0].set_xlabel('Importance')\n",
    "ax[0].set_ylabel(None)\n",
    "ax[0].set_title('Feature Importance in Random Forest Model', fontsize = 16)\n",
    "ax[0].invert_yaxis()\n",
    "\n",
    "ax[1].barh(feature_importance_df_gbc['Feature'], feature_importance_df_gbc['Importance'])\n",
    "ax[1].set_xlabel('Importance')\n",
    "ax[1].set_ylabel(None)\n",
    "ax[1].set_title('Feature Importance in Boosting Model', fontsize = 16)\n",
    "ax[1].invert_yaxis()\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}