code_02_hyperparameters_tuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -------------------------------------------------%\n",
    "# Created by Safwan Mahmood Al-Selwi - 22/08/2024  %\n",
    "# Email:     saf1.alselwi@gmail.com                %\n",
    "# Github:    https://github.com/SafwanAlselwi      %\n",
    "# -------------------------------------------------%\n",
    "\n",
    "# Please consider citing our work if you find it useful:\n",
    "# Al-Selwi, S. M., Hassan, M. F., Abdulkadir, S. J., & et al. (2024). \n",
    "# Smart Grid Stability Prediction Using Adaptive Aquila Optimizer and Ensemble Stacked BiLSTM. Results in Engineering, 24, 103261. \n",
    "# doi: https://doi.org/10.1016/j.rineng.2024.103261."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Hyperparameter Tuning with the HParams Dashboard and Mealpy\n",
    "https://www.tensorflow.org/tensorboard/hyperparameter_tuning_with_hparams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tensorflow as tf\n",
    "from tensorboard.plugins.hparams import api as hp\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import LSTM, Dense, Bidirectional, Dropout, BatchNormalization, InputLayer\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.optimizers import Adam, Nadam, SGD, RMSprop, Adadelta, Adagrad, Adamax\n",
    "\n",
    "from mealpy import FloatVar, StringVar, IntegerVar, Problem\n",
    "from mealpy.swarm_based import AO\n",
    "\n",
    "import time\n",
    "from datetime import datetime\n",
    "from collections import Counter\n",
    "from imblearn.over_sampling import SMOTE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# **Load the data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original class distribution: Counter({'unstable': 6380, 'stable': 3620})\n",
      "After SMOTE class distribution: Counter({'unstable': 6380, 'stable': 6380})\n"
     ]
    }
   ],
   "source": [
    "# Load the dataset\n",
    "data = pd.read_csv('data_org.csv')\n",
    "\n",
    "# dropping some columns \n",
    "data = data.drop(['p1', 'p2', 'p3', 'p4','stab'], axis=1)\n",
    "\n",
    "# Select all columns except the last one for X\n",
    "X = data.iloc[:, :-1]\n",
    "y = data.iloc[:, -1].values\n",
    "\n",
    "print(\"Original class distribution:\", Counter(y))\n",
    "X, y = SMOTE(random_state=42).fit_resample(X, y)\n",
    "X = np.array(X)\n",
    "y = np.array(y)\n",
    "print(\"After SMOTE class distribution:\", Counter(y))\n",
    "\n",
    "# Encode the categorical target variable\n",
    "encoder = OneHotEncoder(sparse_output=False)\n",
    "y_encoded = encoder.fit_transform(y.reshape(-1, 1))\n",
    "\n",
    "# Split the dataset into training and testing sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y_encoded, test_size=0.2, random_state=42)\n",
    "\n",
    "# Normalize the features\n",
    "# scaler = StandardScaler()\n",
    "scaler = MinMaxScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "X_test_scaled = scaler.transform(X_test)\n",
    "\n",
    "# Reshape input to be 3D [samples, timesteps, features] for LSTM\n",
    "X_train_scaled = X_train_scaled.reshape((X_train_scaled.shape[0], 1, X_train_scaled.shape[1]))\n",
    "X_test_scaled = X_test_scaled.reshape((X_test_scaled.shape[0], 1, X_test_scaled.shape[1]))\n",
    "\n",
    "data = [X_train_scaled, X_test_scaled, y_train, y_test]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# **Define the bounds for hyperparameters**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# These are mealpy's hyperparameters for model's tunninig \n",
    "my_bounds = [\n",
    "    IntegerVar(lb=5, ub=40, name=\"num_units\"),\n",
    "    IntegerVar(lb=5, ub=100, name=\"epochs\"),\n",
    "    IntegerVar(lb=16, ub=64, name=\"batch_size\"),\n",
    "    FloatVar(lb=0.0, ub=0.2, name=\"dropout\"),\n",
    "    StringVar(valid_sets=('softmax', 'relu', 'sigmoid', 'tanh', 'elu', 'LeakyReLU'), name=\"activation\"),\n",
    "    StringVar(valid_sets=('Adam', 'Nadam', 'SGD', 'RMSprop', 'Adadelta', 'Adagrad', 'Adamax'), name=\"optimizer\"),\n",
    "    # FloatVar(lb=0.0001, ub=0.1, name=\"learning_rate\"),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the logs to use them later with TensorBoard's Dashboard\n",
    "SAVE_LOGS_TO_PATH = f\"logs/hparam_tuning_{datetime.now().strftime('%Y_%m_%d_%H_%M_%S')}\"\n",
    "\n",
    "if SAVE_LOGS_TO_PATH is not None and not os.path.exists(SAVE_LOGS_TO_PATH):\n",
    "    os.makedirs(SAVE_LOGS_TO_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensorboard --logdir logs/hparam_tuning_2024_12_12_11_56_22\n"
     ]
    }
   ],
   "source": [
    "# Visualize the results in TensorBoard's HParams plugin\n",
    "\n",
    "print(f\"tensorboard --logdir {SAVE_LOGS_TO_PATH}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# These are HParams hyperparameters for logs\n",
    "HP_NUM_UNITS = hp.HParam('num_units', hp.IntInterval(5, 40))\n",
    "HP_EPOCHS = hp.HParam('epochs', hp.IntInterval(5, 100))\n",
    "HP_BATCH_SIZE = hp.HParam('batch_size', hp.IntInterval(16, 64))\n",
    "HP_DROPOUT = hp.HParam('dropout', hp.RealInterval(0.0, 0.2))\n",
    "HP_ACTIVATION = hp.HParam('activation', hp.Discrete(['softmax', 'relu', 'sigmoid', 'tanh', 'elu', 'LeakyReLU']))\n",
    "HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['Adam', 'Nadam', 'SGD', 'RMSprop', 'Adadelta', 'Adagrad', 'Adamax']))\n",
    "\n",
    "METRIC_ACCURACY = 'accuracy'\n",
    "\n",
    "with tf.summary.create_file_writer(SAVE_LOGS_TO_PATH).as_default():\n",
    "    hp.hparams_config(\n",
    "        hparams=[HP_NUM_UNITS, HP_EPOCHS, HP_BATCH_SIZE, HP_DROPOUT, HP_ACTIVATION, HP_OPTIMIZER],\n",
    "        metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')],\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# **Define the custom problem for optimization**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LstmOptimizationProblem(Problem):    \n",
    "    def __init__(self, bounds=None, minmax=\"max\", data=None, **kwargs):\n",
    "        self.data = data\n",
    "        super().__init__(bounds, minmax, obj_weights=(1.0, 0.), **kwargs)\n",
    "\n",
    "    def obj_func(self, x):\n",
    "        x_decoded = self.decode_solution(x)\n",
    "    \n",
    "        v_num_units = x_decoded[\"num_units\"]\n",
    "        v_epochs = x_decoded[\"epochs\"]\n",
    "        v_batch_size = x_decoded[\"batch_size\"]\n",
    "        v_dropout = x_decoded[\"dropout\"]\n",
    "        v_activation = x_decoded[\"activation\"]\n",
    "        v_optimizer = x_decoded[\"optimizer\"]\n",
    "        v_loss_function = \"categorical_crossentropy\"\n",
    "        \n",
    "\n",
    "        model = Sequential([\n",
    "            InputLayer(input_shape=(X_train_scaled.shape[1], X_train_scaled.shape[2])),\n",
    "            \n",
    "            Bidirectional(LSTM(units=v_num_units, return_sequences=True), merge_mode='sum'),\n",
    "            Dropout(v_dropout),\n",
    "            \n",
    "            Bidirectional(LSTM(units=v_num_units, return_sequences=True), merge_mode='sum'),\n",
    "            Dropout(v_dropout),\n",
    "            \n",
    "            Bidirectional(LSTM(units=v_num_units, return_sequences=False), merge_mode='sum'),\n",
    "            Dropout(v_dropout),\n",
    "            \n",
    "            Dense(units=y_encoded.shape[1], activation=v_activation)\n",
    "        ])\n",
    "\n",
    "        model.compile(optimizer=v_optimizer, loss=v_loss_function, metrics=['accuracy'])\n",
    "\n",
    "        early_stopping = EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True)\n",
    "\n",
    "        # Directory for TensorBoard logs specific to each trial\n",
    "        trial_log_dir = f'{SAVE_LOGS_TO_PATH}/{datetime.now().strftime(\"%Y_%m_%d_%H_%M_%S\")}'\n",
    "\n",
    "        hparams = {\n",
    "            HP_NUM_UNITS: v_num_units,\n",
    "            HP_EPOCHS: v_epochs,\n",
    "            HP_BATCH_SIZE: v_batch_size,\n",
    "            HP_DROPOUT: v_dropout,\n",
    "            HP_ACTIVATION: v_activation,\n",
    "            HP_OPTIMIZER: v_optimizer,\n",
    "        }\n",
    "        \n",
    "        history = model.fit(X_train_scaled, \n",
    "                    y_train, \n",
    "                    epochs=v_epochs, \n",
    "                    batch_size=v_batch_size,\n",
    "                    validation_data=(X_test_scaled, y_test), \n",
    "                    callbacks=[\n",
    "                        early_stopping,\n",
    "                        tf.keras.callbacks.TensorBoard(trial_log_dir),  # Log metrics\n",
    "                        # hp.KerasCallback(trial_log_dir, hparams),  # log hparams\n",
    "                    ], \n",
    "                    verbose=0)\n",
    "        \n",
    "        # val_loss, accuracy = model.evaluate(X_test_scaled, y_test, verbose=0)\n",
    "         \n",
    "        # Predict the test set results\n",
    "        y_pred = model.predict(X_test_scaled)\n",
    "        y_pred_classes = np.argmax(y_pred, axis=1)\n",
    "        y_true = np.argmax(y_test, axis=1)\n",
    "        accuracy = accuracy_score(y_true, y_pred_classes)\n",
    "        \n",
    "        print(f\"Accuracy = {accuracy}, num_units = {v_num_units}, activation = {v_activation}, optimizer = {v_optimizer}, epochs = {v_epochs}, batch_size = {v_batch_size}\")\n",
    "        \n",
    "        with tf.summary.create_file_writer(trial_log_dir).as_default():\n",
    "            hp.hparams(hparams)  # record the values used in this trial\n",
    "            tf.summary.scalar(\"accuracy\", accuracy, step=1)\n",
    "            \n",
    "        return accuracy "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# **Run the Experiment**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    # Define the optimization problem\n",
    "    problem = LstmOptimizationProblem(bounds=my_bounds, minmax=\"max\", data=data)\n",
    "    \n",
    "    # Define the Adaptive Aquila Optimizer (AAO) model for hyperparameter tuning with both k and m\n",
    "    model = AO.AdaptiveAO(epoch=20, pop_size=15, sharpness=10, sigmoid_midpoint=0.5)\n",
    " \n",
    "    start_time = datetime.now()\n",
    "    print('Time Start', start_time)\n",
    "    model.solve(problem)\n",
    "    end_time = datetime.now()\n",
    "    print('Time elapsed', end_time - start_time)\n",
    "    \n",
    "    best_agent = model.g_best\n",
    "    best_solution = model.g_best.solution\n",
    "    best_accuracy = model.g_best.target.fitness\n",
    "    best_parameters = model.problem.decode_solution(model.g_best.solution)\n",
    "    \n",
    "    print(f\"Best agent: {best_agent}\")\n",
    "    print(f\"Best solution: {best_solution}\")\n",
    "    print(f\"Best accuracy: {best_accuracy}\")\n",
    "    print(f\"Best parameters: {best_parameters}\")\n",
    "    \n",
    "except Exception as e:\n",
    "    print(f\"Error occured': {str(e)}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "FS_META",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}