final_project_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import layers, models\n",
    "from keras.callbacks import Callback\n",
    "\n",
    "from sklearn.model_selection import cross_validate, StratifiedKFold, KFold\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import make_scorer, accuracy_score, precision_score, recall_score, f1_score, classification_report\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from PIL import Image\n",
    "\n",
    "SEED = 42\n",
    "EPOCHS = 100\n",
    "BATCH_SIZE = 25\n",
    "SCORING = {\n",
    "    'accuracy': make_scorer(accuracy_score),\n",
    "    'f1': make_scorer(f1_score, average='weighted', zero_division=0),\n",
    "    'recall': make_scorer(recall_score, average='weighted', zero_division=0),\n",
    "    'precision': make_scorer(precision_score, average='weighted', zero_division=0)\n",
    "}\n",
    "STRATIFIED_K_FOLD = StratifiedKFold(n_splits=5) \n",
    "\n",
    "\n",
    "### ###########################################\n",
    "### Metric for CNN model\n",
    "### ###########################################\n",
    "\n",
    "class MetricsCallback(Callback):\n",
    "    def __init__(self, validation_data):\n",
    "        super(MetricsCallback, self).__init__()\n",
    "        self.validation_data = validation_data\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        x_val, y_val = self.validation_data\n",
    "        y_pred = self.model.predict(x_val)\n",
    "        \n",
    "        accuracy = accuracy_score(y_val, y_pred.argmax(axis=1))\n",
    "        f1 = f1_score(y_val, y_pred.argmax(axis=1), average='weighted', zero_division=0)\n",
    "        recall = recall_score(y_val, y_pred.argmax(axis=1), average='weighted', zero_division=0)\n",
    "        precision = precision_score(y_val, y_pred.argmax(axis=1), average='weighted', zero_division=0)\n",
    "\n",
    "        logs['val_accuracy'] = accuracy\n",
    "        logs['val_f1'] = f1\n",
    "        logs['val_recall'] = recall\n",
    "        logs['val_precision'] = precision\n",
    "        print(\" - val_accuracy: {:.4f} - val_f1: {:.4f} - val_recall: {:.4f} - val_precision: {:.4f}\".format(accuracy, f1, recall, precision))\n",
    "\n",
    "\n",
    "### ###########################################\n",
    "### Model creation functions\n",
    "### ###########################################\n",
    "\n",
    "def create_cnn(height=256, width=256, channels=1, num_classes=4):\n",
    "    cnn_model = models.Sequential()\n",
    "\n",
    "    cnn_model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(height, width, channels)))\n",
    "    cnn_model.add(layers.MaxPooling2D((2, 2)))\n",
    "    cnn_model.add(layers.Dropout(0.25))\n",
    "\n",
    "    cnn_model.add(layers.Conv2D(64, (3, 3), activation='relu'))\n",
    "    cnn_model.add(layers.MaxPooling2D((2, 2)))\n",
    "    cnn_model.add(layers.Dropout(0.25))\n",
    "\n",
    "    cnn_model.add(layers.Conv2D(128, (3, 3), activation='relu'))\n",
    "    cnn_model.add(layers.MaxPooling2D((2, 2)))\n",
    "    cnn_model.add(layers.Dropout(0.25))\n",
    "\n",
    "    # Flatten the output of the convolutional layers\n",
    "    cnn_model.add(layers.Flatten())\n",
    "\n",
    "    # Fully connected layer for classification\n",
    "    cnn_model.add(layers.Dense(512, activation='relu'))\n",
    "    cnn_model.add(layers.Dropout(0.5))\n",
    "    cnn_model.add(layers.Dense(num_classes, activation='softmax'))\n",
    "\n",
    "    # Compile the model\n",
    "    cnn_model.compile(optimizer='adam',\n",
    "                loss='sparse_categorical_crossentropy',\n",
    "                metrics=['accuracy'])\n",
    "    \n",
    "    return cnn_model\n",
    "\n",
    "\n",
    "def create_feature_extraction_cnn(input_shape):\n",
    "    model = models.Sequential()\n",
    "    \n",
    "    model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=input_shape))\n",
    "    model.add(layers.MaxPooling2D((2, 2)))\n",
    "    \n",
    "    model.add(layers.Conv2D(64, (3, 3), activation='relu'))\n",
    "    model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "    model.add(layers.Conv2D(128, (3, 3), activation='relu'))\n",
    "    model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "    model.add(layers.Flatten())\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "### ###########################################\n",
    "### Data loading and processing functions\n",
    "### ###########################################\n",
    "    \n",
    "def load_data(data_directory):\n",
    "    images_list = []\n",
    "    labels_list = []\n",
    "\n",
    "    # iterate over the list of subfolders (i.e. categories)\n",
    "    categories = os.listdir(data_directory)\n",
    "    for category in categories:\n",
    "        category_path = os.path.join(data_directory, category)\n",
    "        \n",
    "        for image_name in os.listdir(category_path):\n",
    "            image_path = os.path.join(category_path, image_name)\n",
    "            \n",
    "            # load and preprocess the image\n",
    "            img = Image.open(image_path).convert('L')  # convert to greyscale\n",
    "            img = img.resize((256, 256))               # resize to 256x256\n",
    "            img = np.array(img)                        # convert to numpy array\n",
    "            \n",
    "            # append the image and its label to the lists\n",
    "            images_list.append(img)\n",
    "            labels_list.append(category)\n",
    "\n",
    "    # stack the images into a single numpy array to get the dimensions (num_images, 256, 256)\n",
    "    images_array = np.stack(images_list)\n",
    "\n",
    "    # conver the labels list to a numpy array\n",
    "    labels_array = np.array(labels_list)\n",
    "\n",
    "    return images_array, labels_array\n",
    "\n",
    "def preprocess_images(images_array):\n",
    "    # normalise pixel values to be between [0, 1]\n",
    "    images_array = images_array.astype('float32') / 255.0\n",
    "    # reshape the images array to include a single channel (greyscale)\n",
    "    images_array = np.expand_dims(images_array, axis=-1)\n",
    "\n",
    "    return images_array\n",
    "\n",
    "\n",
    "### ###########################################\n",
    "### Summary and detail utility functions\n",
    "### ###########################################\n",
    "\n",
    "def print_cross_validation_scores(cv_scores):\n",
    "    for score in SCORING:\n",
    "        scores_per_fold = [round(x, 2) for x in cv_scores[f'test_{score}']]\n",
    "        mean_score = round(cv_scores[f'test_{score}'].mean(), 2)\n",
    "        print(f\"{score.capitalize()} scores for each fold: {scores_per_fold}\")\n",
    "        print(f\"Mean {score}: {mean_score}\\n\")\n",
    "\n",
    "\n",
    "def print_summary_of_array(array, name):\n",
    "    print(f\"{name} length: {len(array)}\")\n",
    "    print(f\"Shape of {name}:\", array.shape)\n",
    "\n",
    "\n",
    "def plot_history(history):\n",
    "    # plot training & validation accuracy values\n",
    "    plt.plot(history.history['accuracy'])\n",
    "    plt.plot(history.history['val_accuracy'])\n",
    "    plt.title('Model accuracy')\n",
    "    plt.ylabel('Accuracy')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.legend(['Train', 'Validation'], loc='upper left')\n",
    "    plt.show()\n",
    "\n",
    "    # plot training & validation loss values\n",
    "    plt.plot(history.history['loss'])\n",
    "    plt.plot(history.history['val_loss'])\n",
    "    plt.title('Model loss')\n",
    "    plt.ylabel('Loss')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.legend(['Train', 'Validation'], loc='upper left')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data before split:\n",
      "X_train length: 5120\n",
      "Shape of X_train: (5120, 256, 256)\n",
      "y_train length: 5120\n",
      "Shape of y_train: (5120,)\n",
      "\n",
      "Test data:\n",
      "X_test length: 40\n",
      "Shape of X_test: (40, 256, 256)\n",
      "y_test length: 40\n",
      "Shape of y_test: (40,)\n"
     ]
    }
   ],
   "source": [
    "# Load data\n",
    "X_train, y_train = load_data(\"data/train\")\n",
    "print(\"Training data before split:\")\n",
    "print_summary_of_array(X_train, \"X_train\")\n",
    "print_summary_of_array(y_train, \"y_train\")\n",
    "\n",
    "X_test, y_test = load_data(\"data/test\")\n",
    "print(\"\\nTest data:\")\n",
    "print_summary_of_array(X_test, \"X_test\")\n",
    "print_summary_of_array(y_test, \"y_test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preprocess images\n",
    "X_train = preprocess_images(X_train)\n",
    "X_test = preprocess_images(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "160/160 [==============================] - 15s 95ms/step\n",
      "train_features length: 5120\n",
      "Shape of train_features: (5120, 115200)\n",
      "2/2 [==============================] - 0s 27ms/step\n",
      "test_features length: 40\n",
      "Shape of test_features: (40, 115200)\n"
     ]
    }
   ],
   "source": [
    "input_shape = (256, 256, 1)\n",
    "feature_extraction_model = create_feature_extraction_cnn(input_shape)\n",
    "\n",
    "train_features = feature_extraction_model.predict(X_train)\n",
    "print_summary_of_array(train_features, \"train_features\")\n",
    "\n",
    "test_features = feature_extraction_model.predict(X_test)\n",
    "print_summary_of_array(test_features, \"test_features\")\n",
    "\n",
    "num_features = train_features.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define a dictionary to map string labels to numerical labels\n",
    "label_mapping = {label: index for index, label in enumerate(np.unique(y_train))}\n",
    "\n",
    "# encode labels\n",
    "y_train_encoded = np.array([label_mapping[label] for label in y_train])\n",
    "y_test_encoded = np.array([label_mapping[label] for label in y_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "reverse_label_mapping = {index: label for label, index in label_mapping.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## KNN Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross-Validation Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy scores for each fold: [1.0, 0.99, 0.99, 1.0, 0.99]\n",
      "Mean accuracy: 0.99\n",
      "\n",
      "F1 scores for each fold: [1.0, 0.99, 0.99, 1.0, 0.99]\n",
      "Mean f1: 0.99\n",
      "\n",
      "Recall scores for each fold: [1.0, 0.99, 0.99, 1.0, 0.99]\n",
      "Mean recall: 0.99\n",
      "\n",
      "Precision scores for each fold: [1.0, 0.99, 0.99, 1.0, 0.99]\n",
      "Mean precision: 0.99\n",
      "\n"
     ]
    }
   ],
   "source": [
    "n_neighbors = 3\n",
    "knn_model = KNeighborsClassifier(n_neighbors=n_neighbors)\n",
    "\n",
    "# flatten the extracted features for KNN input\n",
    "X_train_knn = train_features.reshape((train_features.shape[0], -1))\n",
    "X_test_knn = test_features.reshape((test_features.shape[0], -1))\n",
    "\n",
    "cv_scores_knn = cross_validate(knn_model, X_train_knn, y_train_encoded, cv=STRATIFIED_K_FOLD, scoring=SCORING)\n",
    "\n",
    "print_cross_validation_scores(cv_scores_knn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation on Unseen Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-4 {color: black;background-color: white;}#sk-container-id-4 pre{padding: 0;}#sk-container-id-4 div.sk-toggleable {background-color: white;}#sk-container-id-4 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-4 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-4 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-4 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-4 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-4 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-4 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-4 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-4 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-4 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-4 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-4 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-4 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-4 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-4 div.sk-item {position: relative;z-index: 1;}#sk-container-id-4 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-4 div.sk-item::before, #sk-container-id-4 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-4 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-4 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-4 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-4 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-4 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-4 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-4 div.sk-label-container {text-align: center;}#sk-container-id-4 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-4 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-4\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>KNeighborsClassifier(n_neighbors=3)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" checked><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">KNeighborsClassifier</label><div class=\"sk-toggleable__content\"><pre>KNeighborsClassifier(n_neighbors=3)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "KNeighborsClassifier(n_neighbors=3)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train the KNN model\n",
    "knn_model.fit(X_train_knn, y_train_encoded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0\n",
      "Precision: 1.0\n",
      "Recall: 1.0\n",
      "F1 Score: 1.0\n",
      "                  precision    recall  f1-score   support\n",
      "\n",
      "    MildDemented       1.00      1.00      1.00        10\n",
      "ModerateDemented       1.00      1.00      1.00        10\n",
      "     NonDemented       1.00      1.00      1.00        10\n",
      "VeryMildDemented       1.00      1.00      1.00        10\n",
      "\n",
      "        accuracy                           1.00        40\n",
      "       macro avg       1.00      1.00      1.00        40\n",
      "    weighted avg       1.00      1.00      1.00        40\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Make predictions on the unseen test data\n",
    "predictions_knn = knn_model.predict(test_features)\n",
    "\n",
    "# Evaluate the model's performance\n",
    "accuracy_knn = accuracy_score(y_test_encoded, predictions_knn)\n",
    "precision_knn = precision_score(y_test_encoded, predictions_knn, average='weighted', zero_division=0)\n",
    "recall_knn = recall_score(y_test_encoded, predictions_knn, average='weighted', zero_division=0)\n",
    "f1_knn = f1_score(y_test_encoded, predictions_knn, average='weighted', zero_division=0)\n",
    "\n",
    "print(f\"Accuracy: {round(accuracy_knn, 2)}\")\n",
    "print(f\"Precision: {round(precision_knn, 2)}\")\n",
    "print(f\"Recall: {round(recall_knn, 2)}\")\n",
    "print(f\"F1 Score: {round(f1_knn, 2)}\")\n",
    "\n",
    "report_knn = classification_report(y_test_encoded, predictions_knn, target_names=reverse_label_mapping.values())\n",
    "print(report_knn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross-Validation Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy scores for each fold: [0.89, 0.88, 0.89, 0.89, 0.89]\n",
      "Mean accuracy: 0.89\n",
      "\n",
      "F1 scores for each fold: [0.88, 0.88, 0.88, 0.89, 0.88]\n",
      "Mean f1: 0.88\n",
      "\n",
      "Recall scores for each fold: [0.89, 0.88, 0.89, 0.89, 0.89]\n",
      "Mean recall: 0.89\n",
      "\n",
      "Precision scores for each fold: [0.89, 0.88, 0.89, 0.89, 0.89]\n",
      "Mean precision: 0.89\n",
      "\n"
     ]
    }
   ],
   "source": [
    "n_estimators = 100\n",
    "rf_model = RandomForestClassifier(n_estimators=n_estimators, random_state=SEED)\n",
    "\n",
    "# Flatten the extracted features for Random Forest input\n",
    "X_train_rf = train_features.reshape((train_features.shape[0], -1))\n",
    "X_test_rf = test_features.reshape((test_features.shape[0], -1))\n",
    "\n",
    "cv_scores_rf = cross_validate(rf_model, X_train_rf, y_train_encoded, cv=STRATIFIED_K_FOLD, scoring=SCORING)\n",
    "\n",
    "print_cross_validation_scores(cv_scores_rf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation on Unseen Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-5 {color: black;background-color: white;}#sk-container-id-5 pre{padding: 0;}#sk-container-id-5 div.sk-toggleable {background-color: white;}#sk-container-id-5 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-5 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-5 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-5 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-5 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-5 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-5 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-5 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-5 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-5 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-5 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-5 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-5 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-5 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-5 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-5 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-5 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-5 div.sk-item {position: relative;z-index: 1;}#sk-container-id-5 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-5 div.sk-item::before, #sk-container-id-5 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-5 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-5 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-5 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-5 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-5 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-5 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-5 div.sk-label-container {text-align: center;}#sk-container-id-5 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-5 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-5\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>RandomForestClassifier(random_state=42)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" checked><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomForestClassifier</label><div class=\"sk-toggleable__content\"><pre>RandomForestClassifier(random_state=42)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "RandomForestClassifier(random_state=42)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train the Random Forest model\n",
    "rf_model.fit(X_train_rf, y_train_encoded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.72\n",
      "Precision: 0.8\n",
      "Recall: 0.72\n",
      "F1 Score: 0.66\n",
      "                  precision    recall  f1-score   support\n",
      "\n",
      "    MildDemented       0.83      1.00      0.91        10\n",
      "ModerateDemented       1.00      0.10      0.18        10\n",
      "     NonDemented       0.83      1.00      0.91        10\n",
      "VeryMildDemented       0.53      0.80      0.64        10\n",
      "\n",
      "        accuracy                           0.73        40\n",
      "       macro avg       0.80      0.73      0.66        40\n",
      "    weighted avg       0.80      0.72      0.66        40\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Make predictions on the unseen test data\n",
    "predictions_rf = rf_model.predict(test_features)\n",
    "\n",
    "# Evaluate the model's performance\n",
    "accuracy_rf = accuracy_score(y_test_encoded, predictions_rf)\n",
    "precision_rf = precision_score(y_test_encoded, predictions_rf, average='weighted', zero_division=0)\n",
    "recall_rf = recall_score(y_test_encoded, predictions_rf, average='weighted', zero_division=0)\n",
    "f1_rf = f1_score(y_test_encoded, predictions_rf, average='weighted', zero_division=0)\n",
    "\n",
    "print(f\"Accuracy: {round(accuracy_rf, 2)}\")\n",
    "print(f\"Precision: {round(precision_rf, 2)}\")\n",
    "print(f\"Recall: {round(recall_rf, 2)}\")\n",
    "print(f\"F1 Score: {round(f1_rf, 2)}\")\n",
    "\n",
    "report_rf = classification_report(y_test_encoded, predictions_rf, target_names=reverse_label_mapping.values())\n",
    "print(report_rf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN Model\n",
    "\n",
    "In this section, we will train and evaluate the performance of a CNN to predict on the test dataset. This will not make use of the feature extraction."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross-Validation Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnn_epochs = 10\n",
    "cnn_model = create_cnn()\n",
    "\n",
    "num_folds = 5\n",
    "kf = KFold(n_splits=num_folds, shuffle=True)\n",
    "\n",
    "accuracy_scores = []\n",
    "f1_scores = []\n",
    "recall_scores = []\n",
    "precision_scores = []\n",
    "\n",
    "# Iterate over the folds\n",
    "for train_index, val_index in kf.split(X_train):\n",
    "    # Split data into training and validation sets\n",
    "    fold_X_train, fold_X_val = X_train[train_index], X_train[val_index]\n",
    "    fold_y_train, fold_y_val = y_train_encoded[train_index], y_train_encoded[val_index]\n",
    "\n",
    "    metrics_callback = MetricsCallback(validation_data=(fold_X_val, fold_y_val))\n",
    "\n",
    "    # Train the model on the training set for this fold\n",
    "    history = cnn_model.fit(fold_X_train, fold_y_train, epochs=cnn_epochs, batch_size=BATCH_SIZE, validation_data=(fold_X_val, fold_y_val), verbose=0, callbacks=[metrics_callback])\n",
    "\n",
    "    # After training, you can access the accuracy, F1, recall, and precision scores from the history object\n",
    "    val_accuracy = history.history['val_accuracy']\n",
    "    val_f1 = history.history['val_f1']\n",
    "    val_recall = history.history['val_recall']\n",
    "    val_precision = history.history['val_precision']\n",
    "\n",
    "    # Store evaluation metrics for this fold\n",
    "    accuracy_scores.append(val_accuracy)\n",
    "    f1_scores.append(val_f1)\n",
    "    recall_scores.append(val_recall)\n",
    "    precision_scores.append(val_precision)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Accuracy: 0.96\n",
      "Average F1 Score: 0.95\n",
      "Average Recall: 0.96\n",
      "Average Precision: 0.95\n"
     ]
    }
   ],
   "source": [
    "# Calculate average scores across all folds\n",
    "avg_accuracy_cnn = np.mean(accuracy_scores)\n",
    "avg_f1_cnn = np.mean(f1_scores)\n",
    "avg_recall_cnn = np.mean(recall_scores)\n",
    "avg_precision_cnn = np.mean(precision_scores)\n",
    "\n",
    "print(\"Average Accuracy:\", round(avg_accuracy_cnn, 2))\n",
    "print(\"Average F1 Score:\", round(avg_f1_cnn, 2))\n",
    "print(\"Average Recall:\", round(avg_recall_cnn, 2))\n",
    "print(\"Average Precision:\", round(avg_precision_cnn, 2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation on Unseen Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2/2 [==============================] - 0s 32ms/step\n",
      "CNN test Loss: 0.0\n",
      "CNN test Accuracy: 1\n",
      "CNN Metrics:\n",
      "                  precision    recall  f1-score   support\n",
      "\n",
      "    MildDemented       1.00      1.00      1.00        10\n",
      "ModerateDemented       1.00      1.00      1.00        10\n",
      "     NonDemented       1.00      1.00      1.00        10\n",
      "VeryMildDemented       1.00      1.00      1.00        10\n",
      "\n",
      "        accuracy                           1.00        40\n",
      "       macro avg       1.00      1.00      1.00        40\n",
      "    weighted avg       1.00      1.00      1.00        40\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test_loss_cnn, test_accuracy_cnn = cnn_model.evaluate(X_test, y_test_encoded, verbose=0)\n",
    "\n",
    "# Predict classes for test data\n",
    "predictions_cnn = cnn_model.predict(X_test)\n",
    "y_pred_classes_cnn = np.argmax(predictions_cnn, axis=1)\n",
    "\n",
    "# Decode labels\n",
    "y_pred_classes_decoded = np.array([reverse_label_mapping[index] for index in y_pred_classes_cnn])\n",
    "y_test_decoded = np.array([reverse_label_mapping[index] for index in y_test_encoded])\n",
    "\n",
    "report_cnn = classification_report(y_test_decoded, y_pred_classes_decoded)\n",
    "\n",
    "print(\"CNN test Loss:\", round(test_loss_cnn, 2))\n",
    "print(\"CNN test Accuracy:\", round(test_accuracy_cnn))\n",
    "print(\"CNN Metrics:\")\n",
    "print(report_cnn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overall Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                  precision  recall  f1-score          Model\n",
      "MildDemented       0.833333   1.000  0.909091  Random Forest\n",
      "ModerateDemented   1.000000   0.100  0.181818  Random Forest\n",
      "NonDemented        0.833333   1.000  0.909091  Random Forest\n",
      "VeryMildDemented   0.533333   0.800  0.640000  Random Forest\n",
      "accuracy           0.725000   0.725  0.725000  Random Forest\n",
      "macro avg          0.800000   0.725  0.660000  Random Forest\n",
      "weighted avg       0.800000   0.725  0.660000  Random Forest\n",
      "MildDemented       1.000000   1.000  1.000000            KNN\n",
      "ModerateDemented   1.000000   1.000  1.000000            KNN\n",
      "NonDemented        1.000000   1.000  1.000000            KNN\n",
      "VeryMildDemented   1.000000   1.000  1.000000            KNN\n",
      "accuracy           1.000000   1.000  1.000000            KNN\n",
      "macro avg          1.000000   1.000  1.000000            KNN\n",
      "weighted avg       1.000000   1.000  1.000000            KNN\n",
      "MildDemented       1.000000   1.000  1.000000            CNN\n",
      "ModerateDemented   1.000000   1.000  1.000000            CNN\n",
      "NonDemented        1.000000   1.000  1.000000            CNN\n",
      "VeryMildDemented   1.000000   1.000  1.000000            CNN\n",
      "accuracy           1.000000   1.000  1.000000            CNN\n",
      "macro avg          1.000000   1.000  1.000000            CNN\n",
      "weighted avg       1.000000   1.000  1.000000            CNN\n"
     ]
    }
   ],
   "source": [
    "def classification_report_df(y_true, y_pred, labels, target_names):\n",
    "    report = classification_report(y_true, y_pred, labels=labels, target_names=target_names, output_dict=True)\n",
    "    report_df = pd.DataFrame(report).transpose()\n",
    "    return report_df\n",
    "\n",
    "labels = np.unique(y_test_encoded)\n",
    "report_rf = classification_report_df(y_test_encoded, predictions_rf, labels=labels, target_names=reverse_label_mapping.values())\n",
    "report_knn = classification_report_df(y_test_encoded, predictions_knn, labels=labels, target_names=reverse_label_mapping.values())\n",
    "report_cnn = classification_report_df(y_test_encoded, y_pred_classes_cnn, labels=labels, target_names=reverse_label_mapping.values())\n",
    "\n",
    "# add the name of the model as a column\n",
    "report_rf['Model'] = 'Random Forest'\n",
    "report_knn['Model'] = 'KNN'\n",
    "report_cnn['Model'] = 'CNN'\n",
    "\n",
    "comparison_df = pd.concat([report_rf, report_knn, report_cnn])\n",
    "comparison_df.drop(columns=['support'], inplace=True)\n",
    "\n",
    "print(comparison_df)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ml-env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}