3_AE_performance_metrics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"-1\" \n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from utils.preprocessing import preprocess_df\n",
    "from utils.df_loader import load_adult_df, load_compas_df, load_german_df, load_diabetes_df, load_breast_cancer_df\n",
    "from utils.evaluation import get_evaluations, EvaluationMatrix\n",
    "\n",
    "from utils.load import load_result_from_csv, load_datapoints_from_npy\n",
    "from utils.models import load_models\n",
    "from utils.models import save_model_performance\n",
    "\n",
    "seed = 42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "pd.options.mode.chained_assignment = None # suppress \"SettingWithCopyWarning\" warning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets = [\n",
    "        \"adult\",\n",
    "        \"german\",\n",
    "        \"compas\",\n",
    "        \"diabetes\",\n",
    "        \"breast_cancer\",\n",
    "        ]\n",
    "\n",
    "models = [\"lr\",\"svc\",\"nn_2\"] # \"dt\",\"gbc\",\"lr\",\"svc\",\n",
    "\n",
    "\n",
    "attack_list = [\n",
    "        'deepfool', \n",
    "        'carlini_l_2', \n",
    "        # 'carlini_l_inf', \n",
    "        'lowprofool_l_2', \n",
    "        # 'lowprofool_l_inf', \n",
    "        # 'fgsm_l_1',\n",
    "        # 'fgsm_l_2',\n",
    "        'fgsm_l_inf',\n",
    "        # 'bim',\n",
    "        # 'mim',\n",
    "        # 'pgd_l_1',\n",
    "        # 'pgd_l_2',\n",
    "        'pgd_l_inf',\n",
    "\n",
    "        # 'boundary', \n",
    "        # 'hopskipjump_l_2', 'hopskipjump_l_inf'\n",
    "        ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_loading_fn(dataset_name):\n",
    "    if dataset_name == 'adult':\n",
    "        dataset_loading_fn = load_adult_df\n",
    "    elif dataset_name == 'german':\n",
    "        dataset_loading_fn = load_german_df\n",
    "    elif dataset_name == 'compas':\n",
    "        dataset_loading_fn = load_compas_df\n",
    "    elif dataset_name == 'diabetes':\n",
    "        dataset_loading_fn = load_diabetes_df\n",
    "    elif dataset_name == 'breast_cancer':\n",
    "        dataset_loading_fn = load_breast_cancer_df\n",
    "    else:\n",
    "        raise Exception(\"Unsupported dataset\")\n",
    "    return dataset_loading_fn\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combine dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for dataset_name in datasets:\n",
    "#     for attack in attack_list:\n",
    "#         folder_name = f\"{attack}_{dataset_name}\"\n",
    "\n",
    "#         ## check if the folder exist\n",
    "\n",
    "#         if os.path.isdir(f'./results/{folder_name}'):\n",
    "#             for model_name in models:\n",
    "\n",
    "#                 dfs = []\n",
    "#                 file_name = f'{folder_name}_{model_name}_result.csv'\n",
    "#                 destination_path = f'./results/{folder_name}/{file_name}'\n",
    "\n",
    "#                 if os.path.isfile(f'./results/{folder_name}/{folder_name}_{model_name}_result_1.csv'):\n",
    "#                     for i in range(0,10):\n",
    "#                         dataset_path = (\n",
    "#                             f\"{attack}_{dataset_name}_{model_name}_result_{i}.csv\"\n",
    "#                         )\n",
    "#                         dfs.append(pd.read_csv(f\"./results/{folder_name}/{dataset_path}\"))\n",
    "\n",
    "#                     ### Combine dfs\n",
    "#                     complete_df = pd.DataFrame([], columns=dfs[0].columns)\n",
    "#                     for l in range(len(dfs[0])):\n",
    "#                         for df in dfs:\n",
    "#                             complete_df = complete_df.append(df.iloc[l : l + 1])\n",
    "\n",
    "#                     ### Save dfs\n",
    "#                     complete_df.to_csv(destination_path)\n",
    "#                     print(f\"Have saved combined sheet to {destination_path}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check wheather white-box attack output same results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def are_ndarrays_same(ndarrays):\n",
    "#   # Create an empty matrix of size len(ndarrays) x len(ndarrays)\n",
    "#   results_matrix = np.empty((len(ndarrays), len(ndarrays)))\n",
    "\n",
    "#   # Iterate through each pair of ndarrays and check if they are the same\n",
    "#   for i in range(len(ndarrays)):\n",
    "#     for j in range(len(ndarrays)):\n",
    "#       if i == j:\n",
    "#         # If the indices are the same, mark it as True in the results matrix\n",
    "#         results_matrix[i][j] = True\n",
    "#       else:\n",
    "#         # Compare the ndarrays using the numpy.array_equal function\n",
    "#         results_matrix[i][j] = np.array_equal(ndarrays[i], ndarrays[j])\n",
    "#   return results_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for dataset_name in datasets:\n",
    "#     for attack in attack_list:\n",
    "#         for model_name in models:\n",
    "#             ndarrays = []\n",
    "#             for running_times in range(0,10):\n",
    "#                 ndarrays.append(load_datapoints_from_npy(attack, dataset_name, model_name, running_times, adv=True))\n",
    "#             print(f'{dataset_name} - {attack} - {model_name}')\n",
    "#             print(are_ndarrays_same(ndarrays).min())\n",
    "\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dic_from_metric(all_metric):\n",
    "\n",
    "    dataset_arr = []\n",
    "    attack_arr = []\n",
    "    model_arr = []\n",
    "    metric_arr = []\n",
    "    value_arr = []\n",
    "\n",
    "    for dataset, dic1 in all_metric.items():\n",
    "        for attack, dic2 in dic1.items():\n",
    "            for model, dic3 in dic2.items():\n",
    "                for metric, value in dic3.items():\n",
    "                    dataset_arr.append(dataset)\n",
    "                    attack_arr.append(attack)\n",
    "                    model_arr.append(model)\n",
    "                    metric_arr.append(metric)\n",
    "                    value_arr.append(value)\n",
    "\n",
    "    table = {\n",
    "            'Dataset': dataset_arr,\n",
    "            'Attack': attack_arr,\n",
    "            'Model': model_arr,\n",
    "            'Metric': metric_arr,\n",
    "            'Value': value_arr,\n",
    "        }\n",
    "\n",
    "    return table\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Have saved file to results/deepfool_adult/eval_deepfool_adult_lr_result_0.csv\n",
      "Have saved file to results/deepfool_adult/eval_deepfool_adult_svc_result_0.csv\n",
      "Have saved file to results/deepfool_adult/eval_deepfool_adult_nn_2_result_0.csv\n",
      "Have saved file to results/carlini_l_2_adult/eval_carlini_l_2_adult_lr_result_0.csv\n",
      "Have saved file to results/carlini_l_2_adult/eval_carlini_l_2_adult_svc_result_0.csv\n",
      "Have saved file to results/carlini_l_2_adult/eval_carlini_l_2_adult_nn_2_result_0.csv\n",
      "Have saved file to results/carlini_l_inf_adult/eval_carlini_l_inf_adult_lr_result_0.csv\n",
      "Have saved file to results/carlini_l_inf_adult/eval_carlini_l_inf_adult_svc_result_0.csv\n",
      "Have saved file to results/carlini_l_inf_adult/eval_carlini_l_inf_adult_nn_2_result_0.csv\n",
      "Have saved file to results/fgsm_l_1_adult/eval_fgsm_l_1_adult_lr_result_0.csv\n",
      "Have saved file to results/fgsm_l_1_adult/eval_fgsm_l_1_adult_svc_result_0.csv\n",
      "Have saved file to results/fgsm_l_1_adult/eval_fgsm_l_1_adult_nn_2_result_0.csv\n",
      "Have saved file to results/fgsm_l_2_adult/eval_fgsm_l_2_adult_lr_result_0.csv\n",
      "Have saved file to results/fgsm_l_2_adult/eval_fgsm_l_2_adult_svc_result_0.csv\n",
      "Have saved file to results/fgsm_l_2_adult/eval_fgsm_l_2_adult_nn_2_result_0.csv\n",
      "Have saved file to results/fgsm_l_inf_adult/eval_fgsm_l_inf_adult_lr_result_0.csv\n",
      "Have saved file to results/fgsm_l_inf_adult/eval_fgsm_l_inf_adult_svc_result_0.csv\n",
      "Have saved file to results/fgsm_l_inf_adult/eval_fgsm_l_inf_adult_nn_2_result_0.csv\n",
      "Have saved file to results/bim_adult/eval_bim_adult_lr_result_0.csv\n",
      "Have saved file to results/bim_adult/eval_bim_adult_svc_result_0.csv\n",
      "Have saved file to results/bim_adult/eval_bim_adult_nn_2_result_0.csv\n",
      "Have saved file to results/mim_adult/eval_mim_adult_lr_result_0.csv\n",
      "Have saved file to results/mim_adult/eval_mim_adult_svc_result_0.csv\n",
      "Have saved file to results/mim_adult/eval_mim_adult_nn_2_result_0.csv\n",
      "Have saved file to results/pgd_l_1_adult/eval_pgd_l_1_adult_lr_result_0.csv\n",
      "Have saved file to results/pgd_l_1_adult/eval_pgd_l_1_adult_svc_result_0.csv\n",
      "Have saved file to results/pgd_l_1_adult/eval_pgd_l_1_adult_nn_2_result_0.csv\n",
      "Have saved file to results/pgd_l_2_adult/eval_pgd_l_2_adult_lr_result_0.csv\n",
      "Have saved file to results/pgd_l_2_adult/eval_pgd_l_2_adult_svc_result_0.csv\n",
      "Have saved file to results/pgd_l_2_adult/eval_pgd_l_2_adult_nn_2_result_0.csv\n",
      "Have saved file to results/pgd_l_inf_adult/eval_pgd_l_inf_adult_lr_result_0.csv\n",
      "Have saved file to results/pgd_l_inf_adult/eval_pgd_l_inf_adult_svc_result_0.csv\n",
      "Have saved file to results/pgd_l_inf_adult/eval_pgd_l_inf_adult_nn_2_result_0.csv\n"
     ]
    }
   ],
   "source": [
    "#### Select dataset ####\n",
    "\n",
    "all_metric = {}\n",
    "\n",
    "for dataset_name in datasets:\n",
    "    all_metric[dataset_name]={}\n",
    "\n",
    "    df_info = preprocess_df(get_loading_fn(dataset_name))\n",
    "    for attack in attack_list:\n",
    "        all_metric[dataset_name][attack]={}\n",
    "\n",
    "        folder_name = f'{attack}_{dataset_name}'\n",
    "        for model_name in models:\n",
    "\n",
    "            file_name = f'{folder_name}_{model_name}_result_0.csv'\n",
    "            result_path = f'./results/{folder_name}/{file_name}'\n",
    "            if os.path.isfile(result_path):\n",
    "                result_df = pd.read_csv(result_path)\n",
    "                evaluation_df, metric = get_evaluations(result_df=result_df, \n",
    "                    df_info=df_info, \n",
    "                    matrix = [\n",
    "                        EvaluationMatrix.L1, \n",
    "                        EvaluationMatrix.L2, \n",
    "                        EvaluationMatrix.Linf,\n",
    "                        EvaluationMatrix.Sen, \n",
    "                        EvaluationMatrix.Mahalanobis,\n",
    "                        EvaluationMatrix.Sparsity, \n",
    "                        EvaluationMatrix.Neighbour_Distance,\n",
    "                        ])\n",
    "                \n",
    "                all_metric[dataset_name][attack][model_name] = metric\n",
    "\n",
    "                csv_save_result_path = f'results/{folder_name}/eval_{file_name}'\n",
    "                evaluation_df.to_csv(csv_save_result_path)\n",
    "                print(f\"Have saved file to {csv_save_result_path}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "im_results = pd.DataFrame.from_dict(get_dic_from_metric(all_metric))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "im_results.to_csv(f\"./results/{datasets[0]}_evaluation_results.csv\",index=False)\n",
    "# im_results.to_csv(f\"./results/diabetes_evaluation_results.csv\",index=False)\n",
    "# im_results.to_csv(f\"./results/carlini_l2_adult_table.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dataset</th>\n",
       "      <th>Attack</th>\n",
       "      <th>Model</th>\n",
       "      <th>Metric</th>\n",
       "      <th>Value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>adult</td>\n",
       "      <td>deepfool</td>\n",
       "      <td>lr</td>\n",
       "      <td>eval_L1</td>\n",
       "      <td>1.438910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>adult</td>\n",
       "      <td>deepfool</td>\n",
       "      <td>lr</td>\n",
       "      <td>eval_L2</td>\n",
       "      <td>0.635517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>adult</td>\n",
       "      <td>deepfool</td>\n",
       "      <td>lr</td>\n",
       "      <td>eval_Linf</td>\n",
       "      <td>0.418918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>adult</td>\n",
       "      <td>deepfool</td>\n",
       "      <td>lr</td>\n",
       "      <td>eval_Sen</td>\n",
       "      <td>2.392331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>adult</td>\n",
       "      <td>deepfool</td>\n",
       "      <td>lr</td>\n",
       "      <td>eval_Mahalanobis</td>\n",
       "      <td>0.127444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>adult</td>\n",
       "      <td>deepfool</td>\n",
       "      <td>lr</td>\n",
       "      <td>eval_Sparsity</td>\n",
       "      <td>3.949722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>adult</td>\n",
       "      <td>deepfool</td>\n",
       "      <td>lr</td>\n",
       "      <td>eval_Neighbour_Distance</td>\n",
       "      <td>0.544389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>adult</td>\n",
       "      <td>deepfool</td>\n",
       "      <td>svc</td>\n",
       "      <td>eval_L1</td>\n",
       "      <td>0.139075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>adult</td>\n",
       "      <td>deepfool</td>\n",
       "      <td>svc</td>\n",
       "      <td>eval_L2</td>\n",
       "      <td>0.110818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>adult</td>\n",
       "      <td>deepfool</td>\n",
       "      <td>svc</td>\n",
       "      <td>eval_Linf</td>\n",
       "      <td>0.108636</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Dataset    Attack Model                   Metric     Value\n",
       "0   adult  deepfool    lr                  eval_L1  1.438910\n",
       "1   adult  deepfool    lr                  eval_L2  0.635517\n",
       "2   adult  deepfool    lr                eval_Linf  0.418918\n",
       "3   adult  deepfool    lr                 eval_Sen  2.392331\n",
       "4   adult  deepfool    lr         eval_Mahalanobis  0.127444\n",
       "5   adult  deepfool    lr            eval_Sparsity  3.949722\n",
       "6   adult  deepfool    lr  eval_Neighbour_Distance  0.544389\n",
       "7   adult  deepfool   svc                  eval_L1  0.139075\n",
       "8   adult  deepfool   svc                  eval_L2  0.110818\n",
       "9   adult  deepfool   svc                eval_Linf  0.108636"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "im_results.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "im_results = pd.read_csv(f\"./results/{datasets[0]}_evaluation_results.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "title_list = [\"Dataset\", \"Attack\", \"Model\"] + list(im_results.iloc[0:7,3])\n",
    "\n",
    "all_eva = []\n",
    "for dataset in datasets:\n",
    "    im_results = pd.read_csv(f\"./results/{dataset}_evaluation_results.csv\")\n",
    "\n",
    "\n",
    "    for m in range(0,im_results.shape[0],7):\n",
    "\n",
    "        data_list = list(im_results.iloc[m,0:3]) + list(im_results.iloc[m:m+7,4])\n",
    "        all_eva.append(data_list)\n",
    "\n",
    "df_all_eva = pd.DataFrame(all_eva,columns=title_list)\n",
    "\n",
    "df_all_eva.Dataset = df_all_eva.Dataset.astype(\"category\")\n",
    "df_all_eva.Dataset = df_all_eva.Dataset.cat.set_categories(datasets)\n",
    "df_all_eva.Model = df_all_eva.Model.astype(\"category\")\n",
    "df_all_eva.Model = df_all_eva.Model.cat.set_categories(models)\n",
    "df_all_eva.Attack = df_all_eva.Attack.astype(\"category\")\n",
    "df_all_eva.Attack = df_all_eva.Attack.cat.set_categories(attack_list)\n",
    "\n",
    "df_all_eva = df_all_eva.sort_values(by=[\"Dataset\", \"Model\", \"Attack\", ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all_eva.to_csv(f\"./results/all_evaluation_results.csv\",index=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "def compare_ndarrays(arr1, arr2):\n",
    "    if arr1.shape != arr2.shape:\n",
    "        raise ValueError(\"Input arrays have different shapes\")\n",
    "    return np.where(arr1 == arr2, 0, 1)\n",
    "\n",
    "def get_attack_success_accuracy(models, model, input_array, adv_array, groundtruth):\n",
    "\n",
    "    if model == 'dt':\n",
    "        predictions = models['dt'].predict(input_array)\n",
    "        adv_predictions = models['dt'].predict(adv_array)\n",
    "    if model == 'rfc':\n",
    "        predictions = models['rfc'].predict(input_array)\n",
    "        adv_predictions = models['rfc'].predict(adv_array)\n",
    "    if model == 'svc':\n",
    "        predictions = models['svc'].predict(input_array)\n",
    "        adv_predictions = models['svc'].predict(adv_array)\n",
    "    if model == 'lr':\n",
    "        predictions = models['lr'].predict(input_array)\n",
    "        adv_predictions = models['lr'].predict(adv_array)\n",
    "    if model == 'gbc':\n",
    "        predictions = models['gbc'].predict(input_array)\n",
    "        adv_predictions = models['gbc'].predict(adv_array)\n",
    "    if model == 'nn':\n",
    "        predictions = (models['nn'].predict(input_array) > 0.5).flatten().astype(int)\n",
    "        adv_predictions = (models['nn'].predict(adv_array) > 0.5).flatten().astype(int)\n",
    "    if model == 'nn_2':\n",
    "        predictions = models['nn_2'].predict(input_array).argmax(axis=1).flatten().astype(int)\n",
    "        adv_predictions = models['nn_2'].predict(adv_array).argmax(axis=1).flatten().astype(int)\n",
    "\n",
    "\n",
    "    pred_attack_success = compare_ndarrays(predictions, adv_predictions).mean()\n",
    "    groundtruth_attack_success = compare_ndarrays(groundtruth, adv_predictions).mean()\n",
    "    original_accuracy = accuracy_score(groundtruth, predictions)\n",
    "    robust_accuracy = accuracy_score(groundtruth, adv_predictions)\n",
    "\n",
    "\n",
    "\n",
    "    dict = {\n",
    "        'groundtruth_attack_success': groundtruth_attack_success,\n",
    "        'pred_attack_success':pred_attack_success, \n",
    "            'original_accuracy': original_accuracy, \n",
    "            'robust_accuracy': robust_accuracy}\n",
    "\n",
    "    print(dict)\n",
    "    return dict\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.models import save_model_performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### Select dataset ####\n",
    "\n",
    "all_performance = {}\n",
    "\n",
    "for dataset_name in datasets:\n",
    "    all_performance[dataset_name]={}\n",
    "\n",
    "    df_info = preprocess_df(get_loading_fn(dataset_name))\n",
    "    train_df, test_df = train_test_split(\n",
    "        df_info.dummy_df, train_size=0.8, random_state=seed, shuffle=True\n",
    "    )\n",
    "    X_train = np.array(train_df[df_info.ohe_feature_names])\n",
    "    y_train = np.array(train_df[df_info.target_name])\n",
    "    X_test = np.array(test_df[df_info.ohe_feature_names])\n",
    "    y_test = np.array(test_df[df_info.target_name])\n",
    "\n",
    "    X_test_num = len(X_test) - (len(X_test)%64)\n",
    "    X_test_re=X_test[0:X_test_num]\n",
    "    y_test_num = len(y_test) - (len(y_test)%64)\n",
    "    y_test_re=y_test[0:y_test_num]\n",
    "\n",
    "    predict_model = load_models(X_train.shape[-1], dataset_name)\n",
    "\n",
    "    for attack in attack_list:\n",
    "        all_performance[dataset_name][attack]={}\n",
    "\n",
    "        folder_name = f'{attack}_{dataset_name}'\n",
    "        for model_name in models:\n",
    "\n",
    "            adv_arr_name = f'{folder_name}_{model_name}_arr_adv_0.npy'\n",
    "            adv_arr_path = f'./datapoints/{folder_name}/{adv_arr_name}'\n",
    "            \n",
    "            if os.path.isfile(adv_arr_path):\n",
    "                adv_arr = load_datapoints_from_npy(attack, dataset_name, model_name, 0, adv=True)\n",
    "\n",
    "                all_performance[dataset_name][attack][model_name] = get_attack_success_accuracy(predict_model, model_name, X_test_re, adv_arr, y_test_re)\n",
    "\n",
    "                # m = save_model_performance(predict_model,dataset_name, adv_arr, y_test_re)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "performance_df = pd.DataFrame.from_dict(get_dic_from_metric(all_performance))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "performance_df.to_csv(f\"./results/{datasets[0]}_performance_results.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "performance_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "xai",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "350746cdd709c967d80ca4cb0f1d2cbf04d079be136a05ad7342c703ce6b7c0e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}