final_project_codes_results.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "### Project Members\n",
        "* Onur Şero\n",
        "* Yunus Emre Aydın\n",
        "* Yunus Emre Türkoğlu\n",
        "* Ful Belin Korukoğlu\n"
      ],
      "metadata": {
        "id": "mOo42lnfYBmF"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aN9PLT4M1tmK"
      },
      "source": [
        "### 1. Initialize"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OuKCZCXO7kvW",
        "outputId": "2de85c69-de10-46d2-e3c7-d98749d481cc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "csv_dir = \"/content/drive/MyDrive/Colab Notebooks/CMPE58P/Project/petct_data.csv\"\n",
        "xlsx_dir = \"/content/petct_data.xlsx\"\n",
        "image_dir = \"/content/drive/MyDrive/Colab Notebooks/CMPE58P/Project/PETCT_data\"\n"
      ],
      "metadata": {
        "id": "8wbRdXLcMGs2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rFppEDSb7gas"
      },
      "outputs": [],
      "source": [
        "%pip install pynrrd\n",
        "%pip install torch\n",
        "%pip install numpy\n",
        "%pip install SimpleITK\n",
        "%pip install matplotlib\n",
        "%pip install pandas\n",
        "%pip install torchio\n",
        "%pip install torchmetrics"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kl1LVUJ410BD"
      },
      "source": [
        "### 2. ImageDataset"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import numpy as np\n",
        "import torch \n",
        "import torchvision.transforms as T\n",
        "import torchio as tio\n",
        "import nrrd\n",
        "import SimpleITK as sitk\n",
        "import matplotlib.pyplot as plt\n",
        "from datetime import datetime\n",
        "import gc\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "\n",
        "class ImageDatasetClinical(torch.utils.data.Dataset):\n",
        "    \n",
        "    def __init__(self, partition, data_dir, df, split):\n",
        "        self.partition = partition\n",
        "        self.data_dir = data_dir\n",
        "        self.df = df\n",
        "        self.split = split\n",
        "        self.ct_images = []\n",
        "        self.pet_images = []\n",
        "        self.seg_images = []\n",
        "        self.tabular_data = []\n",
        "        self.labels = []\n",
        "\n",
        "        if data_dir.endswith(\"/\") == False:\n",
        "            self.data_dir = self.data_dir+ \"/\"\n",
        "\n",
        "        list_dir = os.listdir(self.data_dir)\n",
        "        # development_image_number = 50\n",
        "        # list_dir = list_dir[0:development_image_number]\n",
        "\n",
        "        if self.split == \"train\":\n",
        "          split_dir = list_dir[:int(len(list_dir)*0.7)]\n",
        "          print(\"Train size: \", len(split_dir))\n",
        "        elif self.split == \"val\":    \n",
        "          split_dir = list_dir[int(len(list_dir)*0.7):] \n",
        "          print(\"Val size: \", len(split_dir))\n",
        "\n",
        "        unordered_clinical_data = self.get_clinical_data(xlsx_dir)\n",
        "        for patient_dir in split_dir:\n",
        "          clinical_data = unordered_clinical_data[(unordered_clinical_data[:,0] == int(patient_dir)).nonzero(as_tuple=True)[0],1:][0]\n",
        "          img_ct, img_pet, img_seg, label = self.normalize_item(patient_dir)\n",
        "          self.ct_images.append(img_ct)\n",
        "          self.pet_images.append(img_pet)\n",
        "          self.seg_images.append(img_seg)\n",
        "          self.labels.append(label)\n",
        "          self.tabular_data.append(clinical_data)\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n",
        "\n",
        "    def __getitem__(self,idx):\n",
        "        img_ct = self.ct_images[idx]\n",
        "        img_pet = self.pet_images[idx]\n",
        "        img_seg = self.seg_images[idx]\n",
        "        \n",
        "        if self.split == \"train\":\n",
        "          if self.partition == \"ct\":\n",
        "              img = img_ct\n",
        "              img = self.augmentation(img)\n",
        "          if self.partition == \"pet\":\n",
        "              img = img_pet\n",
        "              img = self.augmentation(img)\n",
        "          if self.partition == \"seg\":\n",
        "              img = img_seg\n",
        "          if self.partition == \"ct_seg\":\n",
        "              img = img_ct + img_seg\n",
        "          if self.partition == \"ct_mult_seg\":\n",
        "              img = np.multiply(img_ct, 0.1 + (img_seg / 1.2))\n",
        "          if self.partition == \"pet_seg\":\n",
        "              img = img_pet + img_seg\n",
        "          if self.partition == \"pet_ct\":\n",
        "              img = img_pet + img_ct\n",
        "              img = self.augmentation(img)\n",
        "          if self.partition == \"mixed\":\n",
        "              img = img_pet + img_ct + img_seg\n",
        "\n",
        "        if self.split == \"val\":\n",
        "          if self.partition == \"ct\" or self.partition == \"ct_seg\" or self.partition == \"ct_mult_seg\":\n",
        "            img = img_ct      \n",
        "          if self.partition == \"pet\" or self.partition == \"pet_seg\":\n",
        "            img = img_pet\n",
        "          if self.partition == \"mixed\" or self.partition == \"pet_ct\":\n",
        "            img = img_pet + img_ct\n",
        "        \n",
        "        img = img.permute(2,0,1).unsqueeze(0)\n",
        "        return img, self.tabular_data[idx], self.labels[idx]\n",
        "    \n",
        "    def normalize_item(self,path):\n",
        "        print(\"Read image for \", path, \", Partition: \", self.partition)\n",
        "        img_ct, img_pet, img_seg = self.read_image(path)\n",
        "\n",
        "        if self.df.loc[self.df['Number'] == int(path), 'node'].values[0] == 1:\n",
        "            label = np.ones(1)\n",
        "        if self.df.loc[self.df['Number'] == int(path), 'node'].values[0] == 0:\n",
        "            label= np.zeros(1)\n",
        "\n",
        "        return img_ct, img_pet, img_seg, label\n",
        "\n",
        "    def read_image(self, path):\n",
        "       \n",
        "        img_ct_raw, _ = nrrd.read(self.data_dir + path + f\"/{path}_CT.nrrd\")   \n",
        "        img_ct_raw = (img_ct_raw-np.min(img_ct_raw))/(np.max(img_ct_raw)- np.min(img_ct_raw)) #normalization\n",
        "        img_ct_raw = img_ct_raw.astype('float32')\n",
        "        img_pet_raw, _ = nrrd.read(self.data_dir + path + f\"/{path}_PET.nrrd\")\n",
        "        img_pet_raw = (img_pet_raw-np.min(img_pet_raw))/(np.max(img_pet_raw)- np.min(img_pet_raw)) #normalization\n",
        "        img_pet_raw = img_pet_raw.astype('float32')\n",
        "        img_seg_raw, _ = nrrd.read(self.data_dir + path + f\"/{path}_SEG.nrrd\") \n",
        "        img_seg_raw = img_seg_raw.astype('float32')\n",
        "        \n",
        "        z_mask =np.nonzero(img_seg_raw)[2]\n",
        "        z_avg = ((min(z_mask) + max(z_mask)) / 2) / img_seg_raw.shape[2]\n",
        "        z_start = int((z_avg - 0.1) * img_seg_raw.shape[2])\n",
        "        z_end = int((z_avg + 0.1) * img_seg_raw.shape[2])\n",
        "        \n",
        "        if z_start < 0:\n",
        "          z_start = 0\n",
        "        if z_end > (img_seg_raw.shape[2] - 1):\n",
        "          z_end = img_seg_raw.shape[2] - 1\n",
        "\n",
        "        img_ct = img_ct_raw[ int(img_ct_raw.shape[0] * 0.4) : int(img_ct_raw.shape[0] * 0.6) ,  int(img_ct_raw.shape[1] * 0.4) : int(img_ct_raw.shape[1] * 0.6), z_start: z_end]\n",
        "        img_pet = img_pet_raw[ int(img_pet_raw.shape[0] * 0.4) : int(img_pet_raw.shape[0] * 0.6) ,  int(img_pet_raw.shape[1] * 0.4) : int(img_pet_raw.shape[1] * 0.6), z_start: z_end]\n",
        "        img_seg = img_seg_raw[ int(img_seg_raw.shape[0] * 0.4) : int(img_seg_raw.shape[0] * 0.6) ,  int(img_seg_raw.shape[1] * 0.4) : int(img_seg_raw.shape[1] * 0.6), z_start: z_end]\n",
        "        print(\"Shape: \", img_ct.shape)\n",
        "        del img_ct_raw\n",
        "        del img_pet_raw\n",
        "        del img_seg_raw\n",
        "        gc.collect()\n",
        "        #registraion with ct images\n",
        "        \n",
        "        img_pet = self.registration(img_ct, img_pet)\n",
        "        img_seg = self.registration(img_ct, img_seg)\n",
        "\n",
        "\n",
        "        img_ct = torch.tensor(img_ct)\n",
        "        img_pet = torch.tensor(img_pet)\n",
        "        img_seg = torch.tensor(img_seg)\n",
        "        img_ct = torch.nn.functional.interpolate(img_ct, 128)\n",
        "        img_pet = torch.nn.functional.interpolate(img_pet, 128)\n",
        "        img_seg = torch.nn.functional.interpolate(img_seg, 128)       \n",
        "        return img_ct, img_pet, img_seg\n",
        "    \n",
        "    def registration(self, img1, img2):\n",
        "    \n",
        "        start = datetime.now()\n",
        "        img1 = sitk.GetImageFromArray(img1)\n",
        "        img2 = sitk.GetImageFromArray(img2)\n",
        "\n",
        "        initial_transform = sitk.CenteredTransformInitializer(\n",
        "            img1, \n",
        "            img2,  \n",
        "            sitk.Euler3DTransform(),\n",
        "            sitk.CenteredTransformInitializerFilter.GEOMETRY,\n",
        "        )\n",
        "     \n",
        "        registration_method = sitk.ImageRegistrationMethod()\n",
        "\n",
        "        # Similarity metric settings.\n",
        "        registration_method.SetMetricAsCorrelation()\n",
        "        registration_method.SetMetricSamplingStrategy(registration_method.RANDOM)\n",
        "        registration_method.SetMetricSamplingPercentage(0.01)\n",
        "\n",
        "        registration_method.SetInterpolator(sitk.sitkLinear)\n",
        "\n",
        "        # Optimizer settings.\n",
        "        registration_method.SetOptimizerAsGradientDescent(learningRate=1.0, numberOfIterations=10)\n",
        "        registration_method.SetOptimizerScalesFromPhysicalShift()\n",
        "        \n",
        "        # Don't optimize in-place, we would possibly like to run this cell multiple times.\n",
        "        registration_method.SetInitialTransform(initial_transform, inPlace= False)\n",
        "        final_transform = registration_method.Execute(img1,img2)\n",
        "        moving_resampled = sitk.Resample(\n",
        "            img2,\n",
        "            img1,\n",
        "            final_transform,\n",
        "            sitk.sitkLinear,\n",
        "            0.0,\n",
        "            img2.GetPixelID(),\n",
        "        ) \n",
        "\n",
        "        final_image = sitk.GetArrayFromImage(moving_resampled)\n",
        "        stop = datetime.now()\n",
        "        #print('Diff Time in sec: ', (stop - start).total_seconds())\n",
        "        return final_image\n",
        "    \n",
        "    \n",
        "    def augmentation(self,data):\n",
        "        \n",
        "        transform = T.Compose([\n",
        "        T.RandomApply([\n",
        "            tio.transforms.RandomBiasField(p=0.25),\n",
        "            tio.transforms.RandomGhosting(p=0.25),\n",
        "            tio.transforms.RandomSpike(p=0.25),\n",
        "            tio.transforms.RandomAffine(degrees=10, scales=0., translation=0., p=0.25)],\n",
        "            p=0.8)\n",
        "        ])\n",
        "\n",
        "        x = data.unsqueeze(-1) # x is a random variable to store data\n",
        "        x = transform(x)\n",
        "        data = x.squeeze(-1)\n",
        "        \n",
        "        return data\n",
        "\n",
        "    def get_clinical_data(self,path):\n",
        "        \n",
        "        # Load clinical data from Excel file\n",
        "        clinical_data = pd.read_excel(path)  # Replace with the path to your Excel file\n",
        "        clinical_data = clinical_data.drop(columns=['node'])\n",
        "        # Separate numerical and categorical columns\n",
        "        \n",
        "        cat_colnames = ['Number', 'metastasis', 'T_yale', 'SUV_prostate', 'psa', 'SUV_prostate']\n",
        "        clinical_columns = clinical_data.loc[:, cat_colnames]\n",
        "\n",
        "        # Convert numerical columns to a numpy array\n",
        "        numerical_data = clinical_columns.to_numpy()\n",
        "\n",
        "        # Impute missing values in the numerical data\n",
        "        imputer = KNNImputer(n_neighbors=5)\n",
        "        numerical_data_imputed = imputer.fit_transform(numerical_data)\n",
        "\n",
        "        # Convert categorical columns to a numpy array\n",
        "        categorical_data = clinical_columns.values\n",
        "        categorical_data = categorical_data.astype(str)\n",
        "\n",
        "        # Encode categorical columns using one-hot encoding\n",
        "        encoder = OneHotEncoder(sparse=False, handle_unknown='ignore')\n",
        "        categorical_data_encoded = encoder.fit_transform(categorical_data)\n",
        "\n",
        "        # Concatenate the imputed numerical data with the categorical data\n",
        "        clinical_data_processed = numerical_data_imputed\n",
        "\n",
        "        # Convert the processed clinical data to a tensor\n",
        "        x_clinical = torch.Tensor(clinical_data_processed)\n",
        "        \n",
        "        return x_clinical"
      ],
      "metadata": {
        "id": "B4ZXdApT_6QD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "siMds5Z-7gaz"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import numpy as np\n",
        "import torch \n",
        "import torchvision.transforms as T\n",
        "import torchio as tio\n",
        "import nrrd\n",
        "import SimpleITK as sitk\n",
        "import matplotlib.pyplot as plt\n",
        "from datetime import datetime\n",
        "import gc\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "\n",
        "class ImageDataset(torch.utils.data.Dataset):\n",
        "    \n",
        "    def __init__(self, partition, data_dir, df, split):\n",
        "        self.partition = partition\n",
        "        self.data_dir = data_dir\n",
        "        self.df = df\n",
        "        self.split = split\n",
        "        self.ct_images = []\n",
        "        self.pet_images = []\n",
        "        self.seg_images = []\n",
        "        self.tabular_data = []\n",
        "        self.labels = []\n",
        "\n",
        "        if data_dir.endswith(\"/\") == False:\n",
        "            self.data_dir = self.data_dir+ \"/\"\n",
        "\n",
        "        list_dir = os.listdir(self.data_dir)\n",
        "\n",
        "        if self.split == \"train\":\n",
        "          split_dir = list_dir[:int(len(list_dir)*0.7)]\n",
        "          print(\"Train size: \", len(split_dir))\n",
        "        elif self.split == \"val\":    \n",
        "          split_dir = list_dir[int(len(list_dir)*0.7):] \n",
        "          print(\"Val size: \", len(split_dir))\n",
        "\n",
        "        unordered_clinical_data = self.get_tabular_data(xlsx_dir)\n",
        "        for patient_dir in split_dir:\n",
        "          clinical_data = unordered_clinical_data[(unordered_clinical_data[:,0] == int(patient_dir)).nonzero(as_tuple=True)[0],1:][0]\n",
        "          img_ct, img_pet, img_seg, label = self.normalize_item(patient_dir)\n",
        "          self.ct_images.append(img_ct)\n",
        "          self.pet_images.append(img_pet)\n",
        "          self.seg_images.append(img_seg)\n",
        "          self.labels.append(label)\n",
        "          self.tabular_data.append(clinical_data)\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n",
        "\n",
        "    def __getitem__(self,idx):\n",
        "        img_ct = self.ct_images[idx]\n",
        "        img_pet = self.pet_images[idx]\n",
        "        img_seg = self.seg_images[idx]\n",
        "        \n",
        "        if self.split == \"train\":\n",
        "          if self.partition == \"ct\":\n",
        "              img = img_ct\n",
        "              img = self.augmentation(img)\n",
        "          if self.partition == \"pet\":\n",
        "              img = img_pet\n",
        "              img = self.augmentation(img)\n",
        "          if self.partition == \"seg\":\n",
        "              img = img_seg\n",
        "          if self.partition == \"ct_seg\":\n",
        "              img = img_ct + img_seg\n",
        "          if self.partition == \"ct_mult_seg\":\n",
        "              img = np.multiply(img_ct, 0.1 + (img_seg / 1.2))\n",
        "          if self.partition == \"pet_seg\":\n",
        "              img = img_pet + img_seg\n",
        "          if self.partition == \"pet_ct\":\n",
        "              img = img_pet + img_ct\n",
        "              img = self.augmentation(img)\n",
        "          if self.partition == \"mixed\":\n",
        "              img = img_pet + img_ct + img_seg\n",
        "\n",
        "        if self.split == \"val\":\n",
        "          if self.partition == \"ct\" or self.partition == \"ct_seg\" or self.partition == \"ct_mult_seg\":\n",
        "            img = img_ct      \n",
        "          if self.partition == \"pet\" or self.partition == \"pet_seg\":\n",
        "            img = img_pet\n",
        "          if self.partition == \"mixed\" or self.partition == \"pet_ct\":\n",
        "            img = img_pet + img_ct\n",
        "        \n",
        "        img = img.permute(2,0,1).unsqueeze(0)\n",
        "        return img, self.tabular_data[idx], self.labels[idx]\n",
        "    \n",
        "    def normalize_item(self,path):\n",
        "        print(\"Read image for \", path, \", Partition: \", self.partition)\n",
        "        img_ct, img_pet, img_seg = self.read_image(path)\n",
        "\n",
        "        if self.df.loc[self.df['Number'] == int(path), 'node'].values[0] == 1:\n",
        "            label = np.ones(1)\n",
        "        if self.df.loc[self.df['Number'] == int(path), 'node'].values[0] == 0:\n",
        "            label= np.zeros(1)\n",
        "\n",
        "        return img_ct, img_pet, img_seg, label\n",
        "\n",
        "    def read_image(self, path):\n",
        "       \n",
        "        img_ct_raw, _ = nrrd.read(self.data_dir + path + f\"/{path}_CT.nrrd\")   \n",
        "        img_ct_raw = (img_ct_raw-np.min(img_ct_raw))/(np.max(img_ct_raw)- np.min(img_ct_raw)) #normalization\n",
        "        img_ct_raw = img_ct_raw.astype('float32')\n",
        "        img_pet_raw, _ = nrrd.read(self.data_dir + path + f\"/{path}_PET.nrrd\")\n",
        "        img_pet_raw = (img_pet_raw-np.min(img_pet_raw))/(np.max(img_pet_raw)- np.min(img_pet_raw)) #normalization\n",
        "        img_pet_raw = img_pet_raw.astype('float32')\n",
        "        img_seg_raw, _ = nrrd.read(self.data_dir + path + f\"/{path}_SEG.nrrd\") \n",
        "        img_seg_raw = img_seg_raw.astype('float32')\n",
        "        \n",
        "        z_mask =np.nonzero(img_seg_raw)[2]\n",
        "        z_avg = ((min(z_mask) + max(z_mask)) / 2) / img_seg_raw.shape[2]\n",
        "        z_start = int((z_avg - 0.1) * img_seg_raw.shape[2])\n",
        "        z_end = int((z_avg + 0.1) * img_seg_raw.shape[2])\n",
        "        \n",
        "        if z_start < 0:\n",
        "          z_start = 0\n",
        "        if z_end > (img_seg_raw.shape[2] - 1):\n",
        "          z_end = img_seg_raw.shape[2] - 1\n",
        "\n",
        "        img_ct = img_ct_raw[ int(img_ct_raw.shape[0] * 0.4) : int(img_ct_raw.shape[0] * 0.6) ,  int(img_ct_raw.shape[1] * 0.4) : int(img_ct_raw.shape[1] * 0.6), z_start: z_end]\n",
        "        img_pet = img_pet_raw[ int(img_pet_raw.shape[0] * 0.4) : int(img_pet_raw.shape[0] * 0.6) ,  int(img_pet_raw.shape[1] * 0.4) : int(img_pet_raw.shape[1] * 0.6), z_start: z_end]\n",
        "        img_seg = img_seg_raw[ int(img_seg_raw.shape[0] * 0.4) : int(img_seg_raw.shape[0] * 0.6) ,  int(img_seg_raw.shape[1] * 0.4) : int(img_seg_raw.shape[1] * 0.6), z_start: z_end]\n",
        "        print(\"Shape: \", img_ct.shape)\n",
        "        del img_ct_raw\n",
        "        del img_pet_raw\n",
        "        del img_seg_raw\n",
        "        gc.collect()\n",
        "        #registraion with ct images\n",
        "        \n",
        "        img_pet = self.registration(img_ct, img_pet)\n",
        "        img_seg = self.registration(img_ct, img_seg)\n",
        "\n",
        "\n",
        "        img_ct = torch.tensor(img_ct)\n",
        "        img_pet = torch.tensor(img_pet)\n",
        "        img_seg = torch.tensor(img_seg)\n",
        "        img_ct = torch.nn.functional.interpolate(img_ct, 128)\n",
        "        img_pet = torch.nn.functional.interpolate(img_pet, 128)\n",
        "        img_seg = torch.nn.functional.interpolate(img_seg, 128)       \n",
        "        return img_ct, img_pet, img_seg\n",
        "    \n",
        "    def registration(self, img1, img2):\n",
        "    \n",
        "        start = datetime.now()\n",
        "        img1 = sitk.GetImageFromArray(img1)\n",
        "        img2 = sitk.GetImageFromArray(img2)\n",
        "\n",
        "        initial_transform = sitk.CenteredTransformInitializer(\n",
        "            img1, \n",
        "            img2,  \n",
        "            sitk.Euler3DTransform(),\n",
        "            sitk.CenteredTransformInitializerFilter.GEOMETRY,\n",
        "        )\n",
        "     \n",
        "        registration_method = sitk.ImageRegistrationMethod()\n",
        "\n",
        "        # Similarity metric settings.\n",
        "        registration_method.SetMetricAsCorrelation()\n",
        "        registration_method.SetMetricSamplingStrategy(registration_method.RANDOM)\n",
        "        registration_method.SetMetricSamplingPercentage(0.01)\n",
        "\n",
        "        registration_method.SetInterpolator(sitk.sitkLinear)\n",
        "\n",
        "        # Optimizer settings.\n",
        "        registration_method.SetOptimizerAsGradientDescent(learningRate=1.0, numberOfIterations=10)\n",
        "        registration_method.SetOptimizerScalesFromPhysicalShift()\n",
        "        \n",
        "        # Don't optimize in-place, we would possibly like to run this cell multiple times.\n",
        "        registration_method.SetInitialTransform(initial_transform, inPlace= False)\n",
        "        final_transform = registration_method.Execute(img1,img2)\n",
        "        moving_resampled = sitk.Resample(\n",
        "            img2,\n",
        "            img1,\n",
        "            final_transform,\n",
        "            sitk.sitkLinear,\n",
        "            0.0,\n",
        "            img2.GetPixelID(),\n",
        "        ) \n",
        "\n",
        "        final_image = sitk.GetArrayFromImage(moving_resampled)\n",
        "        stop = datetime.now()\n",
        "        return final_image\n",
        "    \n",
        "    \n",
        "    def augmentation(self,data):\n",
        "        \n",
        "        transform = T.Compose([\n",
        "        T.RandomApply([\n",
        "            tio.transforms.RandomBiasField(p=0.25),\n",
        "            tio.transforms.RandomGhosting(p=0.25),\n",
        "            tio.transforms.RandomSpike(p=0.25),\n",
        "            tio.transforms.RandomAffine(degrees=10, scales=0., translation=0., p=0.25)],\n",
        "            p=0.8)\n",
        "        ])\n",
        "\n",
        "        x = data.unsqueeze(-1) # x is a random variable to store data\n",
        "        x = transform(x)\n",
        "        data = x.squeeze(-1)\n",
        "        \n",
        "        return data\n",
        "\n",
        "      \n",
        "    def get_tabular_data(self,path):\n",
        "        \n",
        "        # Load clinical data from Excel file\n",
        "        clinical_data = pd.read_excel(path)  # Replace with the path to your Excel file\n",
        "        clinical_data = clinical_data.drop(columns=['node'])\n",
        "        # Separate numerical and categorical columns\n",
        "        numerical_columns = clinical_data.select_dtypes(include='number')\n",
        "        categorical_columns = clinical_data.select_dtypes(exclude='number')\n",
        "\n",
        "        # Convert numerical columns to a numpy array\n",
        "        numerical_data = numerical_columns.to_numpy()\n",
        "\n",
        "        # Impute missing values in the numerical data\n",
        "        imputer = KNNImputer(n_neighbors=5)\n",
        "        numerical_data_imputed = imputer.fit_transform(numerical_data)\n",
        "\n",
        "        # Convert categorical columns to a numpy array\n",
        "        categorical_data = categorical_columns.values\n",
        "        categorical_data = categorical_data.astype(str)\n",
        "\n",
        "        # Encode categorical columns using one-hot encoding\n",
        "        encoder = OneHotEncoder(sparse=False, handle_unknown='ignore')\n",
        "        categorical_data_encoded = encoder.fit_transform(categorical_data)\n",
        "\n",
        "        # Concatenate the imputed numerical data with the categorical data\n",
        "        clinical_data_processed = np.concatenate([numerical_data_imputed, categorical_data_encoded], axis=1)\n",
        "\n",
        "        # Convert the processed clinical data to a tensor\n",
        "        x_clinical = torch.Tensor(clinical_data_processed)\n",
        "        \n",
        "        return x_clinical\n",
        "\n",
        "    def get_clinical_data(self,path):\n",
        "        \n",
        "        # Load clinical data from Excel file\n",
        "        clinical_data = pd.read_excel(path)  # Replace with the path to your Excel file\n",
        "        clinical_data = clinical_data.drop(columns=['node'])\n",
        "        # Separate numerical and categorical columns\n",
        "        \n",
        "        cat_colnames = ['Number', 'metastasis', 'T_yale', 'SUV_prostate', 'psa', 'SUV_prostate']\n",
        "        clinical_columns = clinical_data.loc[:, cat_colnames]\n",
        "\n",
        "        # Convert numerical columns to a numpy array\n",
        "        numerical_data = clinical_columns.to_numpy()\n",
        "\n",
        "        # Impute missing values in the numerical data\n",
        "        imputer = KNNImputer(n_neighbors=5)\n",
        "        numerical_data_imputed = imputer.fit_transform(numerical_data)\n",
        "\n",
        "        # Convert categorical columns to a numpy array\n",
        "        categorical_data = clinical_columns.values\n",
        "        categorical_data = categorical_data.astype(str)\n",
        "\n",
        "        # Encode categorical columns using one-hot encoding\n",
        "        encoder = OneHotEncoder(sparse=False, handle_unknown='ignore')\n",
        "        categorical_data_encoded = encoder.fit_transform(categorical_data)\n",
        "\n",
        "        # Concatenate the imputed numerical data with the categorical data\n",
        "        clinical_data_processed = np.concatenate([numerical_data_imputed, categorical_data_encoded], axis=1)\n",
        "\n",
        "        # Convert the processed clinical data to a tensor\n",
        "        x_clinical = torch.Tensor(clinical_data_processed)\n",
        "        \n",
        "        return x_clinical"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lR7jnzKR16Ru"
      },
      "source": [
        "### 3. Network"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3oNkJH017ga2"
      },
      "outputs": [],
      "source": [
        "\n",
        "import math\n",
        "from functools import partial\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "#reference: https://github.com/kenshohara/3D-ResNets-PyTorch/blob/master/models/resnet.py#L31\n",
        "\n",
        "\n",
        "def get_inplanes():\n",
        "    return [64, 128, 256, 512]\n",
        "\n",
        "\n",
        "def conv3x3x3(in_planes, out_planes, stride=1):\n",
        "    return nn.Conv3d(in_planes,\n",
        "                     out_planes,\n",
        "                     kernel_size=3,\n",
        "                     stride=stride,\n",
        "                     padding=1,\n",
        "                     bias=False)\n",
        "\n",
        "\n",
        "def conv1x1x1(in_planes, out_planes, stride=1):\n",
        "    return nn.Conv3d(in_planes,\n",
        "                     out_planes,\n",
        "                     kernel_size=1,\n",
        "                     stride=stride,\n",
        "                     bias=False)\n",
        "\n",
        "\n",
        "class BasicBlock(nn.Module):\n",
        "    expansion = 1\n",
        "\n",
        "    def __init__(self, in_planes, planes, stride=1, downsample=None):\n",
        "        super().__init__()\n",
        "\n",
        "        self.conv1 = conv3x3x3(in_planes, planes, stride)\n",
        "        self.bn1 = nn.BatchNorm3d(planes)\n",
        "        self.relu = nn.ReLU(inplace=True)\n",
        "        self.conv2 = conv3x3x3(planes, planes)\n",
        "        self.bn2 = nn.BatchNorm3d(planes)\n",
        "        self.downsample = downsample\n",
        "        self.stride = stride\n",
        "\n",
        "    def forward(self, x):\n",
        "        residual = x\n",
        "\n",
        "        out = self.conv1(x)\n",
        "        out = self.bn1(out)\n",
        "        out = self.relu(out)\n",
        "\n",
        "        out = self.conv2(out)\n",
        "        out = self.bn2(out)\n",
        "\n",
        "        if self.downsample is not None:\n",
        "            residual = self.downsample(x)\n",
        "\n",
        "        out += residual\n",
        "        out = self.relu(out)\n",
        "\n",
        "        return out\n",
        "\n",
        "\n",
        "class Bottleneck(nn.Module):\n",
        "    expansion = 4\n",
        "\n",
        "    def __init__(self, in_planes, planes, stride=1, downsample=None):\n",
        "        super().__init__()\n",
        "\n",
        "        self.conv1 = conv1x1x1(in_planes, planes)\n",
        "        self.bn1 = nn.BatchNorm3d(planes)\n",
        "        self.conv2 = conv3x3x3(planes, planes, stride)\n",
        "        self.bn2 = nn.BatchNorm3d(planes)\n",
        "        self.conv3 = conv1x1x1(planes, planes * self.expansion)\n",
        "        self.bn3 = nn.BatchNorm3d(planes * self.expansion)\n",
        "        self.relu = nn.ReLU(inplace=True)\n",
        "        self.downsample = downsample\n",
        "        self.stride = stride\n",
        "\n",
        "    def forward(self, x):\n",
        "        residual = x\n",
        "\n",
        "        out = self.conv1(x)\n",
        "        out = self.bn1(out)\n",
        "        out = self.relu(out)\n",
        "\n",
        "        out = self.conv2(out)\n",
        "        out = self.bn2(out)\n",
        "        out = self.relu(out)\n",
        "\n",
        "        out = self.conv3(out)\n",
        "        out = self.bn3(out)\n",
        "\n",
        "        if self.downsample is not None:\n",
        "            residual = self.downsample(x)\n",
        "\n",
        "        out += residual\n",
        "        out = self.relu(out)\n",
        "\n",
        "        return out\n",
        "\n",
        "\n",
        "class ResNet(nn.Module):\n",
        "\n",
        "    def __init__(self,\n",
        "                 block,\n",
        "                 layers,\n",
        "                 block_inplanes,\n",
        "                 drop_rate,\n",
        "                 n_input_channels=1,\n",
        "                 conv1_t_size=7,\n",
        "                 conv1_t_stride=1,\n",
        "                 no_max_pool=False,\n",
        "                 shortcut_type='B',\n",
        "                 widen_factor=1.0,\n",
        "                 n_classes=1):\n",
        "        super().__init__()\n",
        "        self.drop_rate = drop_rate\n",
        "        self.sigmoid = nn.Sigmoid()\n",
        "        block_inplanes = [int(x * widen_factor) for x in block_inplanes]\n",
        "\n",
        "        self.in_planes = block_inplanes[0]\n",
        "        self.no_max_pool = no_max_pool\n",
        "\n",
        "        self.conv1 = nn.Conv3d(n_input_channels,\n",
        "                               self.in_planes,\n",
        "                               kernel_size=(conv1_t_size, 7, 7),\n",
        "                               stride=(conv1_t_stride, 2, 2),\n",
        "                               padding=(conv1_t_size // 2, 3, 3),\n",
        "                               bias=False)\n",
        "        self.bn1 = nn.BatchNorm3d(self.in_planes)\n",
        "        self.relu = nn.ReLU(inplace=True)\n",
        "        self.maxpool = nn.MaxPool3d(kernel_size=3, stride=2, padding=1)\n",
        "        self.layer1 = self._make_layer(block, block_inplanes[0], layers[0],\n",
        "                                       shortcut_type)\n",
        "        self.layer2 = self._make_layer(block,\n",
        "                                       block_inplanes[1],\n",
        "                                       layers[1],\n",
        "                                       shortcut_type,\n",
        "                                       stride=2)\n",
        "        self.layer3 = self._make_layer(block,\n",
        "                                       block_inplanes[2],\n",
        "                                       layers[2],\n",
        "                                       shortcut_type,\n",
        "                                       stride=2)\n",
        "        self.layer4 = self._make_layer(block,\n",
        "                                       block_inplanes[3],\n",
        "                                       layers[3],\n",
        "                                       shortcut_type,\n",
        "                                       stride=2)\n",
        "\n",
        "        self.avgpool = nn.AdaptiveAvgPool3d((1, 1, 1))\n",
        "        self.fc = nn.Linear(block_inplanes[3] * block.expansion, n_classes)\n",
        "        self.dropout = torch.nn.Dropout(drop_rate)\n",
        "\n",
        "        for m in self.modules():\n",
        "            if isinstance(m, nn.Conv3d):\n",
        "                nn.init.kaiming_normal_(m.weight,\n",
        "                                        mode='fan_out',\n",
        "                                        nonlinearity='relu')\n",
        "            elif isinstance(m, nn.BatchNorm3d):\n",
        "                nn.init.constant_(m.weight, 1)\n",
        "                nn.init.constant_(m.bias, 0)\n",
        "\n",
        "    def _downsample_basic_block(self, x, planes, stride):\n",
        "        out = F.avg_pool3d(x, kernel_size=1, stride=stride)\n",
        "        zero_pads = torch.zeros(out.size(0), planes - out.size(1), out.size(2),\n",
        "                                out.size(3), out.size(4))\n",
        "        if isinstance(out.data, torch.cuda.FloatTensor):\n",
        "            zero_pads = zero_pads.cuda()\n",
        "\n",
        "        out = torch.cat([out.data, zero_pads], dim=1)\n",
        "\n",
        "        return out\n",
        "\n",
        "    def _make_layer(self, block, planes, blocks, shortcut_type, stride=1):\n",
        "        downsample = None\n",
        "        if stride != 1 or self.in_planes != planes * block.expansion:\n",
        "            if shortcut_type == 'A':\n",
        "                downsample = partial(self._downsample_basic_block,\n",
        "                                     planes=planes * block.expansion,\n",
        "                                     stride=stride)\n",
        "            else:\n",
        "                downsample = nn.Sequential(\n",
        "                    conv1x1x1(self.in_planes, planes * block.expansion, stride),\n",
        "                    nn.BatchNorm3d(planes * block.expansion))\n",
        "\n",
        "        layers = []\n",
        "        layers.append(\n",
        "            block(in_planes=self.in_planes,\n",
        "                  planes=planes,\n",
        "                  stride=stride,\n",
        "                  downsample=downsample))\n",
        "        self.in_planes = planes * block.expansion\n",
        "        for i in range(1, blocks):\n",
        "            layers.append(block(self.in_planes, planes))\n",
        "\n",
        "        return nn.Sequential(*layers)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.conv1(x)\n",
        "        x = self.bn1(x)\n",
        "        x = self.relu(x)\n",
        "        if not self.no_max_pool:\n",
        "            x = self.maxpool(x)\n",
        "\n",
        "        x = self.layer1(x)\n",
        "        x = self.layer2(x)\n",
        "        x = self.layer3(x)\n",
        "        x = self.layer4(x)\n",
        "        x = self.avgpool(x)\n",
        "        x = self.dropout(x)\n",
        "\n",
        "        x = x.view(x.size(0), -1)\n",
        "        x = self.fc(x)\n",
        "        x = torch.sigmoid(x)\n",
        "\n",
        "        return x\n",
        "\n",
        "\n",
        "def generate_model(model_depth, drop_rate, **kwargs):\n",
        "    assert model_depth in [10, 18, 34, 50, 101, 152, 200]\n",
        "\n",
        "    if model_depth == 10:\n",
        "        model = ResNet(BasicBlock, [1, 1, 1, 1], get_inplanes(), drop_rate, **kwargs)\n",
        "    elif model_depth == 18:\n",
        "        model = ResNet(BasicBlock, [2, 2, 2, 2], get_inplanes(), drop_rate, **kwargs)\n",
        "    elif model_depth == 34:\n",
        "        model = ResNet(BasicBlock, [3, 4, 6, 3], get_inplanes(), drop_rate, **kwargs)\n",
        "    elif model_depth == 50:\n",
        "        model = ResNet(Bottleneck, [3, 4, 6, 3], get_inplanes(), drop_rate, **kwargs)\n",
        "    elif model_depth == 101:\n",
        "        model = ResNet(Bottleneck, [3, 4, 23, 3], get_inplanes(), drop_rate, **kwargs)\n",
        "    elif model_depth == 152:\n",
        "        model = ResNet(Bottleneck, [3, 8, 36, 3], get_inplanes(), drop_rate, **kwargs)\n",
        "    elif model_depth == 200:\n",
        "        model = ResNet(Bottleneck, [3, 24, 36, 3], get_inplanes(), drop_rate, **kwargs)\n",
        "\n",
        "    return model"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "from torchvision.models import resnet18\n",
        "import pandas as pd\n",
        "from sklearn.preprocessing import OneHotEncoder\n",
        "from sklearn.impute import KNNImputer\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "\n",
        "class MultimodalFusionModel(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(MultimodalFusionModel, self).__init__()\n",
        "\n",
        "        # Part 1: ResNet for feature extraction from COVID-19 images\n",
        "        self.resnet = generate_model(50, 0.2)\n",
        "        \n",
        "\n",
        "        # Part 2: FC layers for clinical information and feature concatenation\n",
        "        self.fc_clinical = nn.Sequential(\n",
        "            nn.Linear(20, 64),\n",
        "            nn.ReLU(),\n",
        "            nn.Dropout(0.1)\n",
        "        )\n",
        "        self.fc_concat = nn.Sequential(\n",
        "            nn.Linear(2048 + 64, 128),\n",
        "            nn.ReLU(),\n",
        "            nn.Dropout(0.1)\n",
        "        )\n",
        "\n",
        "        # Part 3: Final FC layer for COVID-19 prediction\n",
        "        self.fc_final = nn.Linear(128, 1)\n",
        "        self.sigmoid = nn.Sigmoid()\n",
        "\n",
        "    def forward(self, x_image, x_clinical):\n",
        "        x_image, x_clinical\n",
        "        # Part 1: 3-D ResNet for feature extraction \n",
        "        x_image = self.resnet.conv1(x_image)  \n",
        "        x_image = self.resnet.bn1(x_image)\n",
        "        x_image = self.resnet.relu(x_image)\n",
        "        x_image = self.resnet.maxpool(x_image)\n",
        "\n",
        "        x_image = self.resnet.layer1(x_image)\n",
        "        x_image = self.resnet.layer2(x_image)\n",
        "        x_image = self.resnet.layer3(x_image)\n",
        "        x_image = self.resnet.layer4(x_image)\n",
        "\n",
        "        x_image = self.resnet.avgpool(x_image)\n",
        "        x_image = self.resnet.dropout(x_image)\n",
        "        x_image = torch.flatten(x_image, 1)\n",
        "        # Part 2: FC layers for clinical information and feature concatenation\n",
        "        x_clinical = self.fc_clinical(x_clinical)\n",
        "        x_fusion = torch.cat((x_image, x_clinical), dim=1)\n",
        "        x_fusion = self.fc_concat(x_fusion)\n",
        "\n",
        "        # Part 3: Final FC layer for COVID-19 prediction\n",
        "        output = self.fc_final(x_fusion)\n",
        "        output = self.sigmoid(output)\n",
        "\n",
        "        return output"
      ],
      "metadata": {
        "id": "_wSEuqWis7jB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YP_6Tux9195A"
      },
      "source": [
        "### 4. AverageMeter"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8Mjf5JAR7ga3"
      },
      "outputs": [],
      "source": [
        "class AverageMeter:\n",
        "    \"\"\"Computes and stores the average and current value\"\"\"\n",
        "\n",
        "    def __init__(self):\n",
        "      ##### START OF YOUR CODE #####\n",
        "      self.sum = []\n",
        "      self.size = []\n",
        "      self.avg = 0\n",
        "      self.count = 0\n",
        "\n",
        "      self.reset()\n",
        "\n",
        "    def reset(self): \n",
        "      self.sum = []\n",
        "      self.size = []\n",
        "      self.avg = 0\n",
        "      self.count = 0\n",
        "\n",
        "    def update(self, value, size):\n",
        "   \n",
        "      self.sum.append(value*size)   \n",
        "      self.size.append(size)\n",
        "\n",
        "      sum1 = 0  \n",
        "      size1 = 0    \n",
        "      for i in range(0, len(self.sum)):\n",
        "        sum1 += self.sum[i]\n",
        "        size1 += self.size[i]\n",
        "        self.count = size1\n",
        "        self.avg = sum1/self.count\n",
        "\n",
        "\n",
        "      ##### END OF YOUR CODE #####"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "im6k1FR-7ga3",
        "outputId": "98918237-929b-4a11-d5e4-e9eaba22af19"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "75.0 10\n"
          ]
        }
      ],
      "source": [
        "avg_meter = AverageMeter()\n",
        "avg_meter.update(100, 5)\n",
        "avg_meter.update(50, 5)\n",
        "\n",
        "print(avg_meter.avg, avg_meter.count)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5vSB8yZt2GdF"
      },
      "source": [
        "### 5. Training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "w0LWxY7J7ga4"
      },
      "outputs": [],
      "source": [
        "device = torch.device('cuda')\n",
        "from torchmetrics import MetricCollection, Accuracy, Precision, Recall, F1Score\n",
        "from torchmetrics.classification import BinaryAUROC \n",
        "\n",
        "def training(train_loader, model, criterion, optimizer):\n",
        "  # Let's start by initializing our AverageMeters.\n",
        "  avg_meters = {'loss': AverageMeter(),\n",
        "                'acc': AverageMeter()}\n",
        "\n",
        "  metric_collection = MetricCollection({ #reference:https://pub.towardsai.net/improve-your-model-validation-with-torchmetrics-b457d3954dcd\n",
        "    'prec': Precision(task='binary', average='macro').to(device),\n",
        "    'rec': Recall(task='binary', average='macro').to(device),\n",
        "    'f1': F1Score(task='binary').to(device),\n",
        "    'auc': BinaryAUROC().to(device)})              \n",
        "\n",
        "    #load data\n",
        "  for data in train_loader:\n",
        "      \n",
        "    inputs, tabular_data, labels = data\n",
        "    inputs, tabular_data, labels = inputs.to(device), tabular_data.to(device), labels.to(device)\n",
        "\n",
        "    #zero gradients\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "    output = model(inputs, tabular_data).to(device)\n",
        "\n",
        "\n",
        "    #Metrics  \n",
        "    loss = criterion(output.float(), labels.float())\n",
        "\n",
        "    loss_value = loss.item()\n",
        "    total = labels.size(0)\n",
        "    correct_count = (output.round() == labels).sum().item()\n",
        "    accuracy = correct_count / total\n",
        "\n",
        "    avg_meters['loss'].update(loss_value, len(data))\n",
        "    avg_meters['acc'].update(accuracy, len(data))\n",
        "\n",
        "    metric_collection.update(output, labels)\n",
        "\n",
        "    #gradients and learning weights\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "  \n",
        "  val_metrics = metric_collection.compute()\n",
        "  metric_collection.reset()  \n",
        "\n",
        "  return dict([('loss', avg_meters['loss'].avg),\n",
        "                ('acc', avg_meters['acc'].avg),\n",
        "                ('f1', val_metrics['f1'].item()),\n",
        "                ('prec', val_metrics['prec'].item()),\n",
        "                ('rec', val_metrics['rec'].item()),\n",
        "                ('auc', val_metrics['auc'].item())])\n",
        "\n",
        "def validation(val_loader, model, criterion):\n",
        "  avg_meters = {'loss': AverageMeter(),\n",
        "                'acc': AverageMeter()}\n",
        "  metric_collection = MetricCollection({\n",
        "    'prec': Precision(task='binary', average='macro').to(device),\n",
        "    'rec': Recall(task='binary', average='macro').to(device),\n",
        "    'f1': F1Score(task='binary').to(device),\n",
        "    'auc': BinaryAUROC().to(device)})                \n",
        "\n",
        "  model.eval()\n",
        " \n",
        "  with torch.no_grad():\n",
        "     #load data\n",
        "    for data in val_loader:\n",
        "      \n",
        "      inputs, tabular_data, labels = data\n",
        "      inputs, tabular_data, labels = inputs.to(device), tabular_data.to(device), labels.to(device)\n",
        "\n",
        "      \n",
        "      output = model(inputs, tabular_data).to(device)\n",
        "\n",
        "      #Metrics  \n",
        "      loss = criterion(output.float(), labels.float())\n",
        "\n",
        "      loss_value = loss.item()\n",
        "      total = labels.size(0)\n",
        "      correct_count = (output.round() == labels).sum().item()\n",
        "      accuracy = correct_count / total\n",
        "\n",
        "      avg_meters['loss'].update(loss_value, len(data))\n",
        "      avg_meters['acc'].update(accuracy, len(data))\n",
        "\n",
        "      metric_collection.update(output, labels)\n",
        "\n",
        "  val_metrics = metric_collection.compute()\n",
        "  metric_collection.reset()\n",
        "\n",
        "  return dict([('loss', avg_meters['loss'].avg),\n",
        "                ('acc', avg_meters['acc'].avg),\n",
        "                ('f1', val_metrics['f1'].item()),\n",
        "                ('prec', val_metrics['prec'].item()),\n",
        "                ('rec', val_metrics['rec'].item()),\n",
        "                ('auc', val_metrics['auc'].item())])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "55Mylom0-6LR"
      },
      "source": [
        "### 6. Run test"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iOT6-Hpg97UG"
      },
      "outputs": [],
      "source": [
        "## Define Global\n",
        "initial_config = {\n",
        "    \"data_dir\": image_dir,\n",
        "    \"image_size\": 128,\n",
        "    \"train_batch_size\": 64,\n",
        "    \"val_batch_size\": 32,\n",
        "    \"test_batch_size\": 1,\n",
        "    \"activation\": \"relu\",\n",
        "    \"drop_rate\": .2,\n",
        "    \"optimizer\": \"Adam\",\n",
        "    \"learning_rate\": 1e-5,\n",
        "    \"l2_reg\": 1e-6, # Weight decay\n",
        "    \"nb_epoch\": 10,\n",
        "    \"early_stopping\": 8, # trigger value for early stopping\n",
        "}\n",
        "run_dir = '/content/drive/MyDrive/CMPE_runs/cmpe58p_project'\n",
        "\n",
        "learning_rate  =  initial_config[\"learning_rate\"]\n",
        "drop_rate  =  initial_config[\"drop_rate\"]\n",
        "weight_decay  =  initial_config[\"l2_reg\"]\n",
        "activation  =  initial_config[\"activation\"]\n",
        "optimizer_name  =  initial_config[\"optimizer\"]\n",
        "patience = initial_config[\"early_stopping\"]\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "df = pd.read_csv(csv_dir)\n",
        "data_dir = initial_config['data_dir']\n",
        "data_dict_train = ImageDataset('ct_seg', data_dir, df, \"train\")\n",
        "data_dict_val = ImageDataset('ct_seg', data_dir, df, \"val\")\n",
        "print(\"Data is loaded\")  "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cdmXvJYqykYB",
        "outputId": "c0bb03e1-37cc-4f6f-9967-878e70841aa6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train size:  187\n",
            "Read image for  0268 , Partition:  ct_seg\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shape:  (103, 103, 74)\n",
            "Read image for  0005 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0100 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0210 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0211 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 83)\n",
            "Read image for  0104 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0041 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0231 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0077 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0128 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0182 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0016 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0283 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 107)\n",
            "Read image for  0168 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0101 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0176 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0045 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0185 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 83)\n",
            "Read image for  0024 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0067 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0265 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0154 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0221 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0148 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0086 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0143 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 98)\n",
            "Read image for  0056 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0189 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0258 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0237 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0195 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0285 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0018 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0274 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0057 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0049 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 114)\n",
            "Read image for  0242 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0161 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0190 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0229 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 99)\n",
            "Read image for  0060 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0053 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0107 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0277 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 64)\n",
            "Read image for  0034 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0147 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0288 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0112 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0224 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 18)\n",
            "Read image for  0284 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0269 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0096 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0203 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0266 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0127 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0225 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0214 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0291 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0126 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0082 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0103 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0243 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0213 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 83)\n",
            "Read image for  0157 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0070 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0090 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0178 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0046 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0042 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0017 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0097 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0188 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0200 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0252 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0072 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0073 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0068 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0155 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 79)\n",
            "Read image for  0106 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0289 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0163 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0121 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0088 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0272 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0215 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0297 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0087 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0293 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0286 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 99)\n",
            "Read image for  0153 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 71)\n",
            "Read image for  0022 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0227 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0010 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0117 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 27)\n",
            "Read image for  0187 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0216 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0164 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0084 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0137 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0062 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0238 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0035 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0223 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0219 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0122 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0180 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0146 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0099 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0159 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 18)\n",
            "Read image for  0296 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0208 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0059 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0158 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 83)\n",
            "Read image for  0175 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0241 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0256 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0244 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 98)\n",
            "Read image for  0306 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0071 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 88)\n",
            "Read image for  0044 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0270 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0240 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0156 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 18)\n",
            "Read image for  0261 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 83)\n",
            "Read image for  0116 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0282 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0105 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0186 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0119 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0118 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0232 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0102 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0020 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0011 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0009 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0021 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0110 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0115 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0064 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0149 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0171 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0039 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0179 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 123)\n",
            "Read image for  0222 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0076 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0134 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0303 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0177 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0048 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0302 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0220 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0263 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 83)\n",
            "Read image for  0083 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0065 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0280 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 67)\n",
            "Read image for  0245 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0015 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0031 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 81)\n",
            "Read image for  0198 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0140 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0144 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0033 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0169 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0209 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0165 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0027 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0233 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0130 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0196 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0078 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0292 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0267 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0264 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0139 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0023 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0036 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0043 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0276 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0234 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 71)\n",
            "Read image for  0069 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0136 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0173 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0003 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0025 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0120 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0184 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0132 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Val size:  81\n",
            "Read image for  0275 , Partition:  ct_seg\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shape:  (103, 103, 74)\n",
            "Read image for  0114 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0278 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0081 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 67)\n",
            "Read image for  0032 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0152 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0040 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0123 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0281 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0125 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0085 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0307 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0253 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0013 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0194 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0098 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0058 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0108 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0167 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0304 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0051 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0205 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 83)\n",
            "Read image for  0249 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 83)\n",
            "Read image for  0201 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0193 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0089 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0174 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0309 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0236 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0255 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0028 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0151 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 67)\n",
            "Read image for  0162 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 122)\n",
            "Read image for  0079 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 97)\n",
            "Read image for  0257 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0150 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0038 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0007 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0294 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0047 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0228 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0166 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0259 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0055 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0075 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0206 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0006 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0218 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0295 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0133 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0250 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0030 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0181 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0235 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0308 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0192 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0052 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0212 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0290 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0001 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0207 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0287 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0191 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0008 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0246 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0301 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0202 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 83)\n",
            "Read image for  0037 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0251 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 99)\n",
            "Read image for  0204 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0183 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0305 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0271 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0138 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0135 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0273 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0004 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0199 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 123)\n",
            "Read image for  0239 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0247 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0080 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Data is loaded\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Fusion Test\n",
        "It can be changed dynamically. Also network can be added dynamically. Create ImageDataset just once."
      ],
      "metadata": {
        "id": "mUr94gLTBw1m"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GCraJLaB88wu"
      },
      "outputs": [],
      "source": [
        "from torch import optim\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "from torchvision.models import ResNet18_Weights, ResNet50_Weights\n",
        "from torch import nn\n",
        "import os\n",
        "from torch.utils.data import DataLoader\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import KFold\n",
        "from torch.utils.data import Subset\n",
        "\n",
        "def run_test(seg_type):\n",
        "  from torch.utils.data import DataLoader\n",
        "  import torch.optim as optim\n",
        "  import os\n",
        "  from google.colab import drive\n",
        "  profile_dir = '/content/drive/MyDrive/CMPE_runs/'\n",
        "  run_name = \"cmpe58p_project\"\n",
        "  full_dir = profile_dir + run_name\n",
        "\n",
        "  if os.path.exists(full_dir) == False:\n",
        "    os.makedirs(full_dir)\n",
        "\n",
        "  print(\"learning_rate: \", learning_rate, \", drop_rate: \", drop_rate, \", weight_decay: \", weight_decay, \", activation: \", activation, \", optimizer: \", optimizer_name)\n",
        "  \n",
        "  model = MultimodalFusionModel().to(device)\n",
        "  model = torch.nn.DataParallel(model)\n",
        "\n",
        "  optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay = weight_decay)\n",
        "  scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min',factor=0.1, patience=2,min_lr=0.001)\n",
        "  criterion = nn.BCELoss()\n",
        "  trigger_times = 0\n",
        "\n",
        "  #dataloader dictionary\n",
        "  data_dict_train.partition = seg_type\n",
        "  # dataloader_full = DataLoader(data_dict_train, 1)\n",
        "  print(\"Data is loaded, training starting\")\n",
        "  kfold = KFold(n_splits=5, shuffle=True)\n",
        "  scores = []\n",
        "  f1s = []\n",
        "  precs = []\n",
        "  recs = []\n",
        "  for fold, (train, test) in enumerate(kfold.split(data_dict_train)):\n",
        "    print('Fold [%d/%d]' % (fold+1, 5))\n",
        "\n",
        "    # Create train and test subsets\n",
        "    train_subset = Subset(data_dict_train, train)\n",
        "    test_subset = Subset(data_dict_train, test)\n",
        "    train_loader = DataLoader(train_subset, 1)\n",
        "    test_loader = DataLoader(test_subset, 1)\n",
        "    # Reset the model for each fold\n",
        "    model = None\n",
        "    print(\"learning_rate: \", learning_rate, \", drop_rate: \", drop_rate, \", weight_decay: \", weight_decay, \", activation: \", activation, \", optimizer: \", optimizer_name)\n",
        "  \n",
        "    model = MultimodalFusionModel().to(device)\n",
        "    model = torch.nn.DataParallel(model)\n",
        "\n",
        "    optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay = weight_decay)\n",
        "    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min',factor=0.1, patience=2,min_lr=0.001)\n",
        "    criterion = nn.BCELoss()\n",
        "    trigger_times = 0\n",
        "    best_accuracy = 0\n",
        "    f1 = 0\n",
        "    prec = 0\n",
        "    rec = 0\n",
        "    for e in range(initial_config[\"nb_epoch\"]):  # loop over the dataset multiple times    \n",
        "      train_dict = training(train_loader, model, criterion, optimizer)\n",
        "      val_dict = validation(test_loader, model, criterion)\n",
        "      print(\"Epoch: \", e, ', train_loss: ', train_dict['loss'], \", train_acc: \",  train_dict['acc'], ',val_loss: ', val_dict['loss'], \", val_acc: \",  val_dict['acc'])\n",
        "      print(\"Epoch: \", e, ', train_f1_score: ', train_dict['f1'], \", train_precision: \",  train_dict['prec'], ',train_recall: ', train_dict['rec'], \", train_auc: \",  train_dict['auc'])\n",
        "      print(\"Epoch: \", e, ', val_f1_score: ', val_dict['f1'], \", val_precision: \",  val_dict['prec'], ',val_recall: ', val_dict['rec'], \", val_auc: \",  val_dict['auc'])\n",
        "\n",
        "\n",
        "      scheduler.step(val_dict['loss'])\n",
        "      if(((val_dict['acc']) - 0.01) >= best_accuracy):\n",
        "          \n",
        "          rec = val_dict['rec']\n",
        "          prec = val_dict['prec']\n",
        "          f1 = val_dict['f1']\n",
        "          best_accuracy = (val_dict['acc'])\n",
        "          print(\"Save Model, best_accuracy: \", best_accuracy)\n",
        "          torch.save(model, os.path.join(run_dir, \"model.pt\"))\n",
        "      \n",
        "      if val_dict['acc'] > best_accuracy + 0.01:\n",
        "        trigger_times = 0  \n",
        "\n",
        "\n",
        "\n",
        "    \n",
        "      if(((val_dict['acc']) - 0.01) < best_accuracy):\n",
        "        trigger_times += 1\n",
        "        if trigger_times == patience:\n",
        "          print(\"Early stopping in epoch: \", e, \"best_accuracy: \", best_accuracy, \"current_acc: \", val_dict['acc'])\n",
        "          break\n",
        "      scores.append(best_accuracy)\n",
        "      f1s.append(f1)\n",
        "      precs.append(prec)\n",
        "      recs.append(rec)\n",
        "\n",
        "  load_model = torch.load(os.path.join(run_dir, \"model.pt\"))\n",
        "  # test_dict = validation(dataloader_val[], load_model, criterion)\n",
        "  # print(\"test_acc: \", test_dict['acc'], \"test_loss: \", test_dict['loss'])\n",
        "  mean_f1 = np.array(f1s).mean()\n",
        "  std_f1 = np.array(f1s).std()\n",
        "  mean_prec = np.array(precs).mean()\n",
        "  std_prec = np.array(precs).std()\n",
        "  mean_rec = np.array(recs).mean()\n",
        "  std_rec = np.array(recs).std()\n",
        "  mean_auc = np.array(scores).mean()\n",
        "  std_auc = np.array(scores).std()\n",
        "  print(\"F1 Score Mean: \", mean_f1, ', F1 Score Std: ', std_f1, \", Precision Mean: \",  mean_prec, ',Precision Std: ', std_prec)\n",
        "  print(\"Recall Mean: \", mean_rec, \"Recall Std: \", std_rec, \"AUC Mean: \", mean_auc, \"AUC Std: \", std_auc)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "i5gfxM-Q-pzl"
      },
      "outputs": [],
      "source": [
        "run_test('ct')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0gnJoH7XHtbH"
      },
      "outputs": [],
      "source": [
        "run_test('pet')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DGtPnYICHtw_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "69aedc79-620f-472f-efe4-6fcff119b33b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Data is loaded, training starting\n",
            "Fold [1/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6904420698649131 , train_acc:  0.6845637583892618 ,val_loss:  0.6503450799929468 , val_acc:  0.7368421052631579\n",
            "Epoch:  0 , train_f1_score:  0.11320754885673523 , train_precision:  1.0 ,train_recall:  0.05999999865889549 , train_auc:  0.5002020001411438\n",
            "Epoch:  0 , val_f1_score:  0.375 , val_precision:  1.0 ,val_recall:  0.23076923191547394 , val_auc:  0.4738461375236511\n",
            "Save Model, best_accuracy:  0.7368421052631579\n",
            "Epoch:  1 , train_loss:  0.6146966489219425 , train_acc:  0.7181208053691275 ,val_loss:  0.4652065403918785 , val_acc:  0.8421052631578947\n",
            "Epoch:  1 , train_f1_score:  0.4324324429035187 , train_precision:  0.6666666865348816 ,train_recall:  0.3199999928474426 , train_auc:  0.704242467880249\n",
            "Epoch:  1 , val_f1_score:  0.75 , val_precision:  0.8181818127632141 ,val_recall:  0.692307710647583 , val_auc:  0.9015384912490845\n",
            "Save Model, best_accuracy:  0.8421052631578947\n",
            "Epoch:  2 , train_loss:  0.5098995062602284 , train_acc:  0.7718120805369127 ,val_loss:  0.4366504579712024 , val_acc:  0.8157894736842105\n",
            "Epoch:  2 , train_f1_score:  0.5641025900840759 , train_precision:  0.7857142686843872 ,train_recall:  0.4399999976158142 , train_auc:  0.8020201921463013\n",
            "Epoch:  2 , val_f1_score:  0.7200000286102295 , val_precision:  0.75 ,val_recall:  0.692307710647583 , val_auc:  0.9046154022216797\n",
            "Epoch:  3 , train_loss:  0.4815476031877466 , train_acc:  0.7718120805369127 ,val_loss:  0.43006505325167 , val_acc:  0.8157894736842105\n",
            "Epoch:  3 , train_f1_score:  0.5853658318519592 , train_precision:  0.75 ,train_recall:  0.47999998927116394 , train_auc:  0.820605993270874\n",
            "Epoch:  3 , val_f1_score:  0.7200000286102295 , val_precision:  0.75 ,val_recall:  0.692307710647583 , val_auc:  0.9138461351394653\n",
            "Epoch:  4 , train_loss:  0.4617727599776703 , train_acc:  0.785234899328859 ,val_loss:  0.42008000070391543 , val_acc:  0.8421052631578947\n",
            "Epoch:  4 , train_f1_score:  0.6279069781303406 , train_precision:  0.75 ,train_recall:  0.5400000214576721 , train_auc:  0.8440403938293457\n",
            "Epoch:  4 , val_f1_score:  0.7692307829856873 , val_precision:  0.7692307829856873 ,val_recall:  0.7692307829856873 , val_auc:  0.9107692241668701\n",
            "Epoch:  5 , train_loss:  0.4518709512933446 , train_acc:  0.7919463087248322 ,val_loss:  0.4169206546303988 , val_acc:  0.8421052631578947\n",
            "Epoch:  5 , train_f1_score:  0.6352941393852234 , train_precision:  0.7714285850524902 ,train_recall:  0.5400000214576721 , train_auc:  0.851111114025116\n",
            "Epoch:  5 , val_f1_score:  0.7692307829856873 , val_precision:  0.7692307829856873 ,val_recall:  0.7692307829856873 , val_auc:  0.9107692241668701\n",
            "Epoch:  6 , train_loss:  0.43748555972042413 , train_acc:  0.7919463087248322 ,val_loss:  0.4161068659603571 , val_acc:  0.8421052631578947\n",
            "Epoch:  6 , train_f1_score:  0.6352941393852234 , train_precision:  0.7714285850524902 ,train_recall:  0.5400000214576721 , train_auc:  0.8585858345031738\n",
            "Epoch:  6 , val_f1_score:  0.75 , val_precision:  0.8181818127632141 ,val_recall:  0.692307710647583 , val_auc:  0.9169230461120605\n",
            "Epoch:  7 , train_loss:  0.42059424214827634 , train_acc:  0.8053691275167785 ,val_loss:  0.40602392657341335 , val_acc:  0.8421052631578947\n",
            "Epoch:  7 , train_f1_score:  0.658823549747467 , train_precision:  0.800000011920929 ,train_recall:  0.5600000023841858 , train_auc:  0.8749495148658752\n",
            "Epoch:  7 , val_f1_score:  0.7692307829856873 , val_precision:  0.7692307829856873 ,val_recall:  0.7692307829856873 , val_auc:  0.9046154022216797\n",
            "Epoch:  8 , train_loss:  0.4040971577847056 , train_acc:  0.8322147651006712 ,val_loss:  0.40620373804306026 , val_acc:  0.8157894736842105\n",
            "Epoch:  8 , train_f1_score:  0.7191011309623718 , train_precision:  0.8205128312110901 ,train_recall:  0.6399999856948853 , train_auc:  0.8864645957946777\n",
            "Epoch:  8 , val_f1_score:  0.7200000286102295 , val_precision:  0.75 ,val_recall:  0.692307710647583 , val_auc:  0.9046154022216797\n",
            "Epoch:  9 , train_loss:  0.38485172382616845 , train_acc:  0.8523489932885906 ,val_loss:  0.4053685019240028 , val_acc:  0.8157894736842105\n",
            "Epoch:  9 , train_f1_score:  0.760869562625885 , train_precision:  0.8333333134651184 ,train_recall:  0.699999988079071 , train_auc:  0.897777795791626\n",
            "Epoch:  9 , val_f1_score:  0.7200000286102295 , val_precision:  0.75 ,val_recall:  0.692307710647583 , val_auc:  0.8984615802764893\n",
            "Epoch:  10 , train_loss:  0.36887560087848364 , train_acc:  0.8590604026845637 ,val_loss:  0.40583992291450205 , val_acc:  0.8157894736842105\n",
            "Epoch:  10 , train_f1_score:  0.774193525314331 , train_precision:  0.8372092843055725 ,train_recall:  0.7200000286102295 , train_auc:  0.90505051612854\n",
            "Epoch:  10 , val_f1_score:  0.7200000286102295 , val_precision:  0.75 ,val_recall:  0.692307710647583 , val_auc:  0.8861538171768188\n",
            "Epoch:  11 , train_loss:  0.3181117125097334 , train_acc:  0.8859060402684564 ,val_loss:  0.43287976246363924 , val_acc:  0.8157894736842105\n",
            "Epoch:  11 , train_f1_score:  0.821052610874176 , train_precision:  0.8666666746139526 ,train_recall:  0.7799999713897705 , train_auc:  0.9234343767166138\n",
            "Epoch:  11 , val_f1_score:  0.7200000286102295 , val_precision:  0.75 ,val_recall:  0.692307710647583 , val_auc:  0.8830769062042236\n",
            "Epoch:  12 , train_loss:  0.2891492002555197 , train_acc:  0.9194630872483222 ,val_loss:  0.45396595100541354 , val_acc:  0.7631578947368421\n",
            "Epoch:  12 , train_f1_score:  0.8695651888847351 , train_precision:  0.9523809552192688 ,train_recall:  0.800000011920929 , train_auc:  0.9337373971939087\n",
            "Epoch:  12 , val_f1_score:  0.6086956262588501 , val_precision:  0.699999988079071 ,val_recall:  0.5384615659713745 , val_auc:  0.8646153211593628\n",
            "Epoch:  13 , train_loss:  0.2629190055400955 , train_acc:  0.9261744966442953 ,val_loss:  0.5287742578841158 , val_acc:  0.7105263157894737\n",
            "Epoch:  13 , train_f1_score:  0.8817204236984253 , train_precision:  0.9534883499145508 ,train_recall:  0.8199999928474426 , train_auc:  0.9446464776992798\n",
            "Epoch:  13 , val_f1_score:  0.42105263471603394 , val_precision:  0.6666666865348816 ,val_recall:  0.3076923191547394 , val_auc:  0.8030769228935242\n",
            "Epoch:  14 , train_loss:  0.2996509071768778 , train_acc:  0.8859060402684564 ,val_loss:  0.45343749682146345 , val_acc:  0.7894736842105263\n",
            "Epoch:  14 , train_f1_score:  0.8172042965888977 , train_precision:  0.8837209343910217 ,train_recall:  0.7599999904632568 , train_auc:  0.936565637588501\n",
            "Epoch:  14 , val_f1_score:  0.6666666865348816 , val_precision:  0.7272727489471436 ,val_recall:  0.6153846383094788 , val_auc:  0.8646153211593628\n",
            "Early stopping in epoch:  14 best_accuracy:  0.8421052631578947 current_acc:  0.7894736842105263\n",
            "Fold [2/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6094239585151608 , train_acc:  0.7046979865771812 ,val_loss:  0.7943150930498776 , val_acc:  0.6578947368421053\n",
            "Epoch:  0 , train_f1_score:  0.3529411852359772 , train_precision:  0.5454545617103577 ,train_recall:  0.260869562625885 , train_auc:  0.634233832359314\n",
            "Epoch:  0 , val_f1_score:  0.380952388048172 , val_precision:  1.0 ,val_recall:  0.23529411852359772 , val_auc:  0.6778711676597595\n",
            "Save Model, best_accuracy:  0.6578947368421053\n",
            "Epoch:  1 , train_loss:  0.5606649974048538 , train_acc:  0.7315436241610739 ,val_loss:  0.48157781811038913 , val_acc:  0.7894736842105263\n",
            "Epoch:  1 , train_f1_score:  0.45945945382118225 , train_precision:  0.6071428656578064 ,train_recall:  0.3695652186870575 , train_auc:  0.7211903929710388\n",
            "Epoch:  1 , val_f1_score:  0.692307710647583 , val_precision:  1.0 ,val_recall:  0.529411792755127 , val_auc:  0.9047619104385376\n",
            "Save Model, best_accuracy:  0.7894736842105263\n",
            "Epoch:  2 , train_loss:  0.4739048202778369 , train_acc:  0.7718120805369127 ,val_loss:  0.47404192527756095 , val_acc:  0.8157894736842105\n",
            "Epoch:  2 , train_f1_score:  0.5405405163764954 , train_precision:  0.7142857313156128 ,train_recall:  0.43478259444236755 , train_auc:  0.8201773166656494\n",
            "Epoch:  2 , val_f1_score:  0.7407407164573669 , val_precision:  1.0 ,val_recall:  0.5882353186607361 , val_auc:  0.9103641510009766\n",
            "Save Model, best_accuracy:  0.8157894736842105\n",
            "Epoch:  3 , train_loss:  0.4607077926028545 , train_acc:  0.7785234899328859 ,val_loss:  0.46982483232491895 , val_acc:  0.8157894736842105\n",
            "Epoch:  3 , train_f1_score:  0.5714285969734192 , train_precision:  0.7096773982048035 ,train_recall:  0.47826087474823 , train_auc:  0.8355845808982849\n",
            "Epoch:  3 , val_f1_score:  0.7407407164573669 , val_precision:  1.0 ,val_recall:  0.5882353186607361 , val_auc:  0.924369752407074\n",
            "Epoch:  4 , train_loss:  0.457655176089064 , train_acc:  0.7718120805369127 ,val_loss:  0.46898239831391136 , val_acc:  0.7894736842105263\n",
            "Epoch:  4 , train_f1_score:  0.5641025900840759 , train_precision:  0.6875 ,train_recall:  0.47826087474823 , train_auc:  0.8338961601257324\n",
            "Epoch:  4 , val_f1_score:  0.692307710647583 , val_precision:  1.0 ,val_recall:  0.529411792755127 , val_auc:  0.918767511844635\n",
            "Epoch:  5 , train_loss:  0.44638585058922237 , train_acc:  0.785234899328859 ,val_loss:  0.46119442028238583 , val_acc:  0.7631578947368421\n",
            "Epoch:  5 , train_f1_score:  0.5897436141967773 , train_precision:  0.71875 ,train_recall:  0.5 , train_auc:  0.8486703038215637\n",
            "Epoch:  5 , val_f1_score:  0.6666666865348816 , val_precision:  0.8999999761581421 ,val_recall:  0.529411792755127 , val_auc:  0.913165271282196\n",
            "Epoch:  6 , train_loss:  0.43503032039519673 , train_acc:  0.8053691275167785 ,val_loss:  0.46704971290340547 , val_acc:  0.7894736842105263\n",
            "Epoch:  6 , train_f1_score:  0.6419752836227417 , train_precision:  0.7428571581840515 ,train_recall:  0.5652173757553101 , train_auc:  0.8562684655189514\n",
            "Epoch:  6 , val_f1_score:  0.692307710647583 , val_precision:  1.0 ,val_recall:  0.529411792755127 , val_auc:  0.918767511844635\n",
            "Epoch:  7 , train_loss:  0.4195018062655558 , train_acc:  0.7919463087248322 ,val_loss:  0.4642013603037125 , val_acc:  0.7894736842105263\n",
            "Epoch:  7 , train_f1_score:  0.6172839403152466 , train_precision:  0.7142857313156128 ,train_recall:  0.54347825050354 , train_auc:  0.8678767681121826\n",
            "Epoch:  7 , val_f1_score:  0.692307710647583 , val_precision:  1.0 ,val_recall:  0.529411792755127 , val_auc:  0.9299719929695129\n",
            "Epoch:  8 , train_loss:  0.4033847134951002 , train_acc:  0.8053691275167785 ,val_loss:  0.4636846624902989 , val_acc:  0.8421052631578947\n",
            "Epoch:  8 , train_f1_score:  0.650602400302887 , train_precision:  0.7297297120094299 ,train_recall:  0.5869565010070801 , train_auc:  0.8763191103935242\n",
            "Epoch:  8 , val_f1_score:  0.7857142686843872 , val_precision:  1.0 ,val_recall:  0.6470588445663452 , val_auc:  0.9215686321258545\n",
            "Save Model, best_accuracy:  0.8421052631578947\n",
            "Epoch:  9 , train_loss:  0.38028031331877654 , train_acc:  0.8322147651006712 ,val_loss:  0.4666531422224484 , val_acc:  0.8157894736842105\n",
            "Epoch:  9 , train_f1_score:  0.6987951993942261 , train_precision:  0.7837837934494019 ,train_recall:  0.6304348111152649 , train_auc:  0.8902490139007568\n",
            "Epoch:  9 , val_f1_score:  0.7407407164573669 , val_precision:  1.0 ,val_recall:  0.5882353186607361 , val_auc:  0.918767511844635\n",
            "Epoch:  10 , train_loss:  0.36041527721896494 , train_acc:  0.8456375838926175 ,val_loss:  0.45590269457744925 , val_acc:  0.8157894736842105\n",
            "Epoch:  10 , train_f1_score:  0.7160493731498718 , train_precision:  0.8285714387893677 ,train_recall:  0.6304348111152649 , train_auc:  0.9003798961639404\n",
            "Epoch:  10 , val_f1_score:  0.7407407164573669 , val_precision:  1.0 ,val_recall:  0.5882353186607361 , val_auc:  0.9215686321258545\n",
            "Epoch:  11 , train_loss:  0.3286589143982145 , train_acc:  0.8456375838926175 ,val_loss:  0.4788193472685586 , val_acc:  0.8157894736842105\n",
            "Epoch:  11 , train_f1_score:  0.7228915691375732 , train_precision:  0.8108108043670654 ,train_recall:  0.6521739363670349 , train_auc:  0.9145208597183228\n",
            "Epoch:  11 , val_f1_score:  0.7407407164573669 , val_precision:  1.0 ,val_recall:  0.5882353186607361 , val_auc:  0.913165271282196\n",
            "Epoch:  12 , train_loss:  0.283819072654638 , train_acc:  0.8926174496644296 ,val_loss:  0.49109193688797714 , val_acc:  0.8157894736842105\n",
            "Epoch:  12 , train_f1_score:  0.800000011920929 , train_precision:  0.9411764740943909 ,train_recall:  0.695652186870575 , train_auc:  0.9278176426887512\n",
            "Epoch:  12 , val_f1_score:  0.7407407164573669 , val_precision:  1.0 ,val_recall:  0.5882353186607361 , val_auc:  0.9019607901573181\n",
            "Epoch:  13 , train_loss:  0.2545160273367773 , train_acc:  0.8926174496644296 ,val_loss:  0.48124421151403923 , val_acc:  0.8157894736842105\n",
            "Epoch:  13 , train_f1_score:  0.8048780560493469 , train_precision:  0.9166666865348816 ,train_recall:  0.717391312122345 , train_auc:  0.9413254261016846\n",
            "Epoch:  13 , val_f1_score:  0.7407407164573669 , val_precision:  1.0 ,val_recall:  0.5882353186607361 , val_auc:  0.8935573697090149\n",
            "Epoch:  14 , train_loss:  0.2453677629779708 , train_acc:  0.8859060402684564 ,val_loss:  0.4890028621034856 , val_acc:  0.7894736842105263\n",
            "Epoch:  14 , train_f1_score:  0.790123462677002 , train_precision:  0.9142857193946838 ,train_recall:  0.695652186870575 , train_auc:  0.9474461674690247\n",
            "Epoch:  14 , val_f1_score:  0.692307710647583 , val_precision:  1.0 ,val_recall:  0.529411792755127 , val_auc:  0.8767507076263428\n",
            "Early stopping in epoch:  14 best_accuracy:  0.8421052631578947 current_acc:  0.7894736842105263\n",
            "Fold [3/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6782781619764865 , train_acc:  0.6666666666666666 ,val_loss:  0.6015924968429514 , val_acc:  0.7297297297297297\n",
            "Epoch:  0 , train_f1_score:  0.2647058963775635 , train_precision:  0.6000000238418579 ,train_recall:  0.16981132328510284 , train_auc:  0.5876288414001465\n",
            "Epoch:  0 , val_f1_score:  0.1666666716337204 , val_precision:  0.5 ,val_recall:  0.10000000149011612 , val_auc:  0.5185185670852661\n",
            "Save Model, best_accuracy:  0.7297297297297297\n",
            "Epoch:  1 , train_loss:  0.5245919662745049 , train_acc:  0.7666666666666667 ,val_loss:  0.6041179009763574 , val_acc:  0.7567567567567568\n",
            "Epoch:  1 , train_f1_score:  0.6153846383094788 , train_precision:  0.7368420958518982 ,train_recall:  0.5283018946647644 , train_auc:  0.7755300998687744\n",
            "Epoch:  1 , val_f1_score:  0.5714285969734192 , val_precision:  0.5454545617103577 ,val_recall:  0.6000000238418579 , val_auc:  0.7666666507720947\n",
            "Save Model, best_accuracy:  0.7567567567567568\n",
            "Epoch:  2 , train_loss:  0.44590441179306556 , train_acc:  0.8333333333333334 ,val_loss:  0.5821220991783775 , val_acc:  0.7297297297297297\n",
            "Epoch:  2 , train_f1_score:  0.7311828136444092 , train_precision:  0.8500000238418579 ,train_recall:  0.6415094137191772 , train_auc:  0.847305953502655\n",
            "Epoch:  2 , val_f1_score:  0.4444444477558136 , val_precision:  0.5 ,val_recall:  0.4000000059604645 , val_auc:  0.7851851582527161\n",
            "Epoch:  3 , train_loss:  0.43200099612702614 , train_acc:  0.82 ,val_loss:  0.5869023831658745 , val_acc:  0.7567567567567568\n",
            "Epoch:  3 , train_f1_score:  0.7096773982048035 , train_precision:  0.824999988079071 ,train_recall:  0.6226415038108826 , train_auc:  0.853335976600647\n",
            "Epoch:  3 , val_f1_score:  0.5263158082962036 , val_precision:  0.5555555820465088 ,val_recall:  0.5 , val_auc:  0.8037037253379822\n",
            "Epoch:  4 , train_loss:  0.4152811454746795 , train_acc:  0.82 ,val_loss:  0.5917433988935364 , val_acc:  0.7567567567567568\n",
            "Epoch:  4 , train_f1_score:  0.7216494679450989 , train_precision:  0.7954545617103577 ,train_recall:  0.6603773832321167 , train_auc:  0.8685081005096436\n",
            "Epoch:  4 , val_f1_score:  0.5263158082962036 , val_precision:  0.5555555820465088 ,val_recall:  0.5 , val_auc:  0.8074074387550354\n",
            "Epoch:  5 , train_loss:  0.40528050762193746 , train_acc:  0.8333333333333334 ,val_loss:  0.6281608627995828 , val_acc:  0.7837837837837838\n",
            "Epoch:  5 , train_f1_score:  0.7422680258750916 , train_precision:  0.8181818127632141 ,train_recall:  0.6792452931404114 , train_auc:  0.8751215934753418\n",
            "Epoch:  5 , val_f1_score:  0.6000000238418579 , val_precision:  0.6000000238418579 ,val_recall:  0.6000000238418579 , val_auc:  0.7925925850868225\n",
            "Save Model, best_accuracy:  0.7837837837837838\n",
            "Epoch:  6 , train_loss:  0.40190021251790314 , train_acc:  0.84 ,val_loss:  0.6229635025610568 , val_acc:  0.7567567567567568\n",
            "Epoch:  6 , train_f1_score:  0.7551020383834839 , train_precision:  0.8222222328186035 ,train_recall:  0.698113203048706 , train_auc:  0.8758996725082397\n",
            "Epoch:  6 , val_f1_score:  0.5263158082962036 , val_precision:  0.5555555820465088 ,val_recall:  0.5 , val_auc:  0.8074074387550354\n",
            "Epoch:  7 , train_loss:  0.3892846334646553 , train_acc:  0.8333333333333334 ,val_loss:  0.5878364250262117 , val_acc:  0.7297297297297297\n",
            "Epoch:  7 , train_f1_score:  0.7422680258750916 , train_precision:  0.8181818127632141 ,train_recall:  0.6792452931404114 , train_auc:  0.8856253623962402\n",
            "Epoch:  7 , val_f1_score:  0.4444444477558136 , val_precision:  0.5 ,val_recall:  0.4000000059604645 , val_auc:  0.7962962985038757\n",
            "Epoch:  8 , train_loss:  0.37946377092749267 , train_acc:  0.8466666666666667 ,val_loss:  0.6249614640487503 , val_acc:  0.7297297297297297\n",
            "Epoch:  8 , train_f1_score:  0.7722772359848022 , train_precision:  0.8125 ,train_recall:  0.7358490824699402 , train_auc:  0.8875705003738403\n",
            "Epoch:  8 , val_f1_score:  0.4444444477558136 , val_precision:  0.5 ,val_recall:  0.4000000059604645 , val_auc:  0.7925925850868225\n",
            "Epoch:  9 , train_loss:  0.3678195537554954 , train_acc:  0.86 ,val_loss:  0.5883977217757067 , val_acc:  0.7297297297297297\n",
            "Epoch:  9 , train_f1_score:  0.7835051417350769 , train_precision:  0.8636363744735718 ,train_recall:  0.7169811129570007 , train_auc:  0.8959346413612366\n",
            "Epoch:  9 , val_f1_score:  0.4444444477558136 , val_precision:  0.5 ,val_recall:  0.4000000059604645 , val_auc:  0.8037037253379822\n",
            "Epoch:  10 , train_loss:  0.35743211498410044 , train_acc:  0.8533333333333334 ,val_loss:  0.598681973943147 , val_acc:  0.7297297297297297\n",
            "Epoch:  10 , train_f1_score:  0.7799999713897705 , train_precision:  0.8297872543334961 ,train_recall:  0.7358490824699402 , train_auc:  0.9004085659980774\n",
            "Epoch:  10 , val_f1_score:  0.4444444477558136 , val_precision:  0.5 ,val_recall:  0.4000000059604645 , val_auc:  0.800000011920929\n",
            "Epoch:  11 , train_loss:  0.34694797786995574 , train_acc:  0.86 ,val_loss:  0.5681584529402489 , val_acc:  0.7027027027027027\n",
            "Epoch:  11 , train_f1_score:  0.7835051417350769 , train_precision:  0.8636363744735718 ,train_recall:  0.7169811129570007 , train_auc:  0.908189058303833\n",
            "Epoch:  11 , val_f1_score:  0.3529411852359772 , val_precision:  0.4285714328289032 ,val_recall:  0.30000001192092896 , val_auc:  0.8074073791503906\n",
            "Epoch:  12 , train_loss:  0.33158918111297075 , train_acc:  0.86 ,val_loss:  0.6226754041602584 , val_acc:  0.7297297297297297\n",
            "Epoch:  12 , train_f1_score:  0.7835051417350769 , train_precision:  0.8636363744735718 ,train_recall:  0.7169811129570007 , train_auc:  0.916942298412323\n",
            "Epoch:  12 , val_f1_score:  0.4444444477558136 , val_precision:  0.5 ,val_recall:  0.4000000059604645 , val_auc:  0.7888889312744141\n",
            "Epoch:  13 , train_loss:  0.30601709306546276 , train_acc:  0.8666666666666667 ,val_loss:  0.6467691602931056 , val_acc:  0.7297297297297297\n",
            "Epoch:  13 , train_f1_score:  0.795918345451355 , train_precision:  0.8666666746139526 ,train_recall:  0.7358490824699402 , train_auc:  0.9276405572891235\n",
            "Epoch:  13 , val_f1_score:  0.4444444477558136 , val_precision:  0.5 ,val_recall:  0.4000000059604645 , val_auc:  0.796296238899231\n",
            "Epoch:  14 , train_loss:  0.29101246804466274 , train_acc:  0.8933333333333333 ,val_loss:  0.6149231340937518 , val_acc:  0.7027027027027027\n",
            "Epoch:  14 , train_f1_score:  0.8461538553237915 , train_precision:  0.8627451062202454 ,train_recall:  0.8301886916160583 , train_auc:  0.9328924417495728\n",
            "Epoch:  14 , val_f1_score:  0.3529411852359772 , val_precision:  0.4285714328289032 ,val_recall:  0.30000001192092896 , val_auc:  0.7925925850868225\n",
            "Early stopping in epoch:  14 best_accuracy:  0.7837837837837838 current_acc:  0.7027027027027027\n",
            "Fold [4/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6627540491273006 , train_acc:  0.66 ,val_loss:  1.0272346805479076 , val_acc:  0.7567567567567568\n",
            "Epoch:  0 , train_f1_score:  0.5405405163764954 , train_precision:  0.508474588394165 ,train_recall:  0.5769230723381042 , train_auc:  0.6736656427383423\n",
            "Epoch:  0 , val_f1_score:  0.3076923191547394 , val_precision:  1.0 ,val_recall:  0.1818181872367859 , val_auc:  0.43356645107269287\n",
            "Save Model, best_accuracy:  0.7567567567567568\n",
            "Epoch:  1 , train_loss:  0.537931842058897 , train_acc:  0.7733333333333333 ,val_loss:  0.6985012121699952 , val_acc:  0.7837837837837838\n",
            "Epoch:  1 , train_f1_score:  0.6304348111152649 , train_precision:  0.7250000238418579 ,train_recall:  0.557692289352417 , train_auc:  0.7615777254104614\n",
            "Epoch:  1 , val_f1_score:  0.5 , val_precision:  0.800000011920929 ,val_recall:  0.3636363744735718 , val_auc:  0.6293706297874451\n",
            "Save Model, best_accuracy:  0.7837837837837838\n",
            "Epoch:  2 , train_loss:  0.46876330732988813 , train_acc:  0.8066666666666666 ,val_loss:  0.6487032922135817 , val_acc:  0.7837837837837838\n",
            "Epoch:  2 , train_f1_score:  0.6813187003135681 , train_precision:  0.7948718070983887 ,train_recall:  0.5961538553237915 , train_auc:  0.8388932347297668\n",
            "Epoch:  2 , val_f1_score:  0.5 , val_precision:  0.800000011920929 ,val_recall:  0.3636363744735718 , val_auc:  0.6503496170043945\n",
            "Epoch:  3 , train_loss:  0.44311733325943353 , train_acc:  0.8266666666666667 ,val_loss:  0.6092977392915133 , val_acc:  0.7837837837837838\n",
            "Epoch:  3 , train_f1_score:  0.717391312122345 , train_precision:  0.824999988079071 ,train_recall:  0.6346153616905212 , train_auc:  0.861067533493042\n",
            "Epoch:  3 , val_f1_score:  0.5 , val_precision:  0.800000011920929 ,val_recall:  0.3636363744735718 , val_auc:  0.6678321361541748\n",
            "Epoch:  4 , train_loss:  0.42727296692163996 , train_acc:  0.8266666666666667 ,val_loss:  0.5893210895560883 , val_acc:  0.7837837837837838\n",
            "Epoch:  4 , train_f1_score:  0.7234042286872864 , train_precision:  0.8095238208770752 ,train_recall:  0.6538461446762085 , train_auc:  0.8720565438270569\n",
            "Epoch:  4 , val_f1_score:  0.5 , val_precision:  0.800000011920929 ,val_recall:  0.3636363744735718 , val_auc:  0.6398601531982422\n",
            "Epoch:  5 , train_loss:  0.418332271167698 , train_acc:  0.8266666666666667 ,val_loss:  0.5596428009102473 , val_acc:  0.7837837837837838\n",
            "Epoch:  5 , train_f1_score:  0.7234042286872864 , train_precision:  0.8095238208770752 ,train_recall:  0.6538461446762085 , train_auc:  0.8773548007011414\n",
            "Epoch:  5 , val_f1_score:  0.5 , val_precision:  0.800000011920929 ,val_recall:  0.3636363744735718 , val_auc:  0.6468531489372253\n",
            "Epoch:  6 , train_loss:  0.4097029353274653 , train_acc:  0.82 ,val_loss:  0.5404994850223129 , val_acc:  0.7837837837837838\n",
            "Epoch:  6 , train_f1_score:  0.7216494679450989 , train_precision:  0.7777777910232544 ,train_recall:  0.6730769276618958 , train_auc:  0.883437991142273\n",
            "Epoch:  6 , val_f1_score:  0.5 , val_precision:  0.800000011920929 ,val_recall:  0.3636363744735718 , val_auc:  0.692307710647583\n",
            "Epoch:  7 , train_loss:  0.3987101596718033 , train_acc:  0.8133333333333334 ,val_loss:  0.5335070134417431 , val_acc:  0.8108108108108109\n",
            "Epoch:  7 , train_f1_score:  0.7142857313156128 , train_precision:  0.760869562625885 ,train_recall:  0.6730769276618958 , train_auc:  0.8910911083221436\n",
            "Epoch:  7 , val_f1_score:  0.5882353186607361 , val_precision:  0.8333333134651184 ,val_recall:  0.4545454680919647 , val_auc:  0.7272727489471436\n",
            "Save Model, best_accuracy:  0.8108108108108109\n",
            "Epoch:  8 , train_loss:  0.3902476928631465 , train_acc:  0.8133333333333334 ,val_loss:  0.5262405099095525 , val_acc:  0.8108108108108109\n",
            "Epoch:  8 , train_f1_score:  0.7142857313156128 , train_precision:  0.760869562625885 ,train_recall:  0.6730769276618958 , train_auc:  0.8958005905151367\n",
            "Epoch:  8 , val_f1_score:  0.5882353186607361 , val_precision:  0.8333333134651184 ,val_recall:  0.4545454680919647 , val_auc:  0.7377622127532959\n",
            "Epoch:  9 , train_loss:  0.3767192137318974 , train_acc:  0.8133333333333334 ,val_loss:  0.5333356966038008 , val_acc:  0.7837837837837838\n",
            "Epoch:  9 , train_f1_score:  0.7200000286102295 , train_precision:  0.75 ,train_recall:  0.692307710647583 , train_auc:  0.9018838405609131\n",
            "Epoch:  9 , val_f1_score:  0.6000000238418579 , val_precision:  0.6666666865348816 ,val_recall:  0.5454545617103577 , val_auc:  0.748251736164093\n",
            "Epoch:  10 , train_loss:  0.3542760097538121 , train_acc:  0.8266666666666667 ,val_loss:  0.5244101591005519 , val_acc:  0.8108108108108109\n",
            "Epoch:  10 , train_f1_score:  0.7400000095367432 , train_precision:  0.7708333134651184 ,train_recall:  0.7115384340286255 , train_auc:  0.9158163070678711\n",
            "Epoch:  10 , val_f1_score:  0.6315789222717285 , val_precision:  0.75 ,val_recall:  0.5454545617103577 , val_auc:  0.7692307829856873\n",
            "Epoch:  11 , train_loss:  0.3262974071964466 , train_acc:  0.8466666666666667 ,val_loss:  0.5349601349113761 , val_acc:  0.8378378378378378\n",
            "Epoch:  11 , train_f1_score:  0.7766990065574646 , train_precision:  0.7843137383460999 ,train_recall:  0.7692307829856873 , train_auc:  0.9266090989112854\n",
            "Epoch:  11 , val_f1_score:  0.6666666865348816 , val_precision:  0.8571428656578064 ,val_recall:  0.5454545617103577 , val_auc:  0.751748263835907\n",
            "Save Model, best_accuracy:  0.8378378378378378\n",
            "Epoch:  12 , train_loss:  0.3089770925588285 , train_acc:  0.8533333333333334 ,val_loss:  0.5456020826020757 , val_acc:  0.8108108108108109\n",
            "Epoch:  12 , train_f1_score:  0.7843137383460999 , train_precision:  0.800000011920929 ,train_recall:  0.7692307829856873 , train_auc:  0.9338696599006653\n",
            "Epoch:  12 , val_f1_score:  0.5882353186607361 , val_precision:  0.8333333134651184 ,val_recall:  0.4545454680919647 , val_auc:  0.7272727489471436\n",
            "Epoch:  13 , train_loss:  0.2769431875931332 , train_acc:  0.8866666666666667 ,val_loss:  0.58562834757204 , val_acc:  0.7837837837837838\n",
            "Epoch:  13 , train_f1_score:  0.8316831588745117 , train_precision:  0.8571428656578064 ,train_recall:  0.807692289352417 , train_auc:  0.9458398818969727\n",
            "Epoch:  13 , val_f1_score:  0.6000000238418579 , val_precision:  0.6666666865348816 ,val_recall:  0.5454545617103577 , val_auc:  0.751748263835907\n",
            "Epoch:  14 , train_loss:  0.26563218578265757 , train_acc:  0.8866666666666667 ,val_loss:  0.6127043607065806 , val_acc:  0.7837837837837838\n",
            "Epoch:  14 , train_f1_score:  0.8380952477455139 , train_precision:  0.8301886916160583 ,train_recall:  0.8461538553237915 , train_auc:  0.9532966613769531\n",
            "Epoch:  14 , val_f1_score:  0.6363636255264282 , val_precision:  0.6363636255264282 ,val_recall:  0.6363636255264282 , val_auc:  0.7412587404251099\n",
            "Early stopping in epoch:  14 best_accuracy:  0.8378378378378378 current_acc:  0.7837837837837838\n",
            "Fold [5/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6979732697907214 , train_acc:  0.6333333333333333 ,val_loss:  0.5870259045749098 , val_acc:  0.6756756756756757\n",
            "Epoch:  0 , train_f1_score:  0.035087719559669495 , train_precision:  0.1666666716337204 ,train_recall:  0.019607843831181526 , train_auc:  0.4727668762207031\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.7133333683013916\n",
            "Save Model, best_accuracy:  0.6756756756756757\n",
            "Epoch:  1 , train_loss:  0.5918750528370341 , train_acc:  0.7133333333333334 ,val_loss:  0.4476786324301282 , val_acc:  0.8108108108108109\n",
            "Epoch:  1 , train_f1_score:  0.4691357910633087 , train_precision:  0.6333333253860474 ,train_recall:  0.37254902720451355 , train_auc:  0.673202633857727\n",
            "Epoch:  1 , val_f1_score:  0.6666666865348816 , val_precision:  0.7777777910232544 ,val_recall:  0.5833333134651184 , val_auc:  0.9133333563804626\n",
            "Save Model, best_accuracy:  0.8108108108108109\n",
            "Epoch:  2 , train_loss:  0.5090168332494796 , train_acc:  0.76 ,val_loss:  0.42964971276956637 , val_acc:  0.8108108108108109\n",
            "Epoch:  2 , train_f1_score:  0.5609756112098694 , train_precision:  0.7419354915618896 ,train_recall:  0.45098039507865906 , train_auc:  0.7857002019882202\n",
            "Epoch:  2 , val_f1_score:  0.7200000286102295 , val_precision:  0.692307710647583 ,val_recall:  0.75 , val_auc:  0.9133333563804626\n",
            "Epoch:  3 , train_loss:  0.48312513236422094 , train_acc:  0.78 ,val_loss:  0.41257309797849206 , val_acc:  0.8108108108108109\n",
            "Epoch:  3 , train_f1_score:  0.6117647290229797 , train_precision:  0.7647058963775635 ,train_recall:  0.5098039507865906 , train_auc:  0.8082789182662964\n",
            "Epoch:  3 , val_f1_score:  0.7200000286102295 , val_precision:  0.692307710647583 ,val_recall:  0.75 , val_auc:  0.9100000262260437\n",
            "Epoch:  4 , train_loss:  0.4699753560211199 , train_acc:  0.8 ,val_loss:  0.41850866484329907 , val_acc:  0.8108108108108109\n",
            "Epoch:  4 , train_f1_score:  0.6428571343421936 , train_precision:  0.8181818127632141 ,train_recall:  0.529411792755127 , train_auc:  0.8158051371574402\n",
            "Epoch:  4 , val_f1_score:  0.7200000286102295 , val_precision:  0.692307710647583 ,val_recall:  0.75 , val_auc:  0.8966667056083679\n",
            "Epoch:  5 , train_loss:  0.4563954672294979 , train_acc:  0.8 ,val_loss:  0.4062000817644435 , val_acc:  0.8108108108108109\n",
            "Epoch:  5 , train_f1_score:  0.6590909361839294 , train_precision:  0.7837837934494019 ,train_recall:  0.5686274766921997 , train_auc:  0.8318479061126709\n",
            "Epoch:  5 , val_f1_score:  0.7200000286102295 , val_precision:  0.692307710647583 ,val_recall:  0.75 , val_auc:  0.8966667056083679\n",
            "Epoch:  6 , train_loss:  0.4511419950441147 , train_acc:  0.8266666666666667 ,val_loss:  0.40699620278099097 , val_acc:  0.8108108108108109\n",
            "Epoch:  6 , train_f1_score:  0.7045454382896423 , train_precision:  0.837837815284729 ,train_recall:  0.6078431606292725 , train_auc:  0.8336304426193237\n",
            "Epoch:  6 , val_f1_score:  0.7200000286102295 , val_precision:  0.692307710647583 ,val_recall:  0.75 , val_auc:  0.9066666960716248\n",
            "Epoch:  7 , train_loss:  0.44258043621084653 , train_acc:  0.8133333333333334 ,val_loss:  0.4048550954919208 , val_acc:  0.8108108108108109\n",
            "Epoch:  7 , train_f1_score:  0.6888889074325562 , train_precision:  0.7948718070983887 ,train_recall:  0.6078431606292725 , train_auc:  0.8439295291900635\n",
            "Epoch:  7 , val_f1_score:  0.7200000286102295 , val_precision:  0.692307710647583 ,val_recall:  0.75 , val_auc:  0.9066666960716248\n",
            "Epoch:  8 , train_loss:  0.4329467751760967 , train_acc:  0.8266666666666667 ,val_loss:  0.39859858117808866 , val_acc:  0.8108108108108109\n",
            "Epoch:  8 , train_f1_score:  0.7111111283302307 , train_precision:  0.8205128312110901 ,train_recall:  0.6274510025978088 , train_auc:  0.8482868671417236\n",
            "Epoch:  8 , val_f1_score:  0.7200000286102295 , val_precision:  0.692307710647583 ,val_recall:  0.75 , val_auc:  0.9033333659172058\n",
            "Epoch:  9 , train_loss:  0.4193993795282828 , train_acc:  0.8333333333333334 ,val_loss:  0.38710725067711965 , val_acc:  0.8108108108108109\n",
            "Epoch:  9 , train_f1_score:  0.7252747416496277 , train_precision:  0.824999988079071 ,train_recall:  0.6470588445663452 , train_auc:  0.8657160401344299\n",
            "Epoch:  9 , val_f1_score:  0.7200000286102295 , val_precision:  0.692307710647583 ,val_recall:  0.75 , val_auc:  0.9066666960716248\n",
            "Epoch:  10 , train_loss:  0.4110258251554721 , train_acc:  0.8333333333333334 ,val_loss:  0.3958324708698972 , val_acc:  0.8108108108108109\n",
            "Epoch:  10 , train_f1_score:  0.7311828136444092 , train_precision:  0.8095238208770752 ,train_recall:  0.6666666865348816 , train_auc:  0.8730441331863403\n",
            "Epoch:  10 , val_f1_score:  0.7200000286102295 , val_precision:  0.692307710647583 ,val_recall:  0.75 , val_auc:  0.8966667056083679\n",
            "Epoch:  11 , train_loss:  0.40219722721745105 , train_acc:  0.8333333333333334 ,val_loss:  0.3987313640945767 , val_acc:  0.8108108108108109\n",
            "Epoch:  11 , train_f1_score:  0.7311828136444092 , train_precision:  0.8095238208770752 ,train_recall:  0.6666666865348816 , train_auc:  0.8793820738792419\n",
            "Epoch:  11 , val_f1_score:  0.7200000286102295 , val_precision:  0.692307710647583 ,val_recall:  0.75 , val_auc:  0.9000000357627869\n",
            "Epoch:  12 , train_loss:  0.38765740260171394 , train_acc:  0.8466666666666667 ,val_loss:  0.39862228275396283 , val_acc:  0.7837837837837838\n",
            "Epoch:  12 , train_f1_score:  0.7526881694793701 , train_precision:  0.8333333134651184 ,train_recall:  0.686274528503418 , train_auc:  0.8878986239433289\n",
            "Epoch:  12 , val_f1_score:  0.6666666865348816 , val_precision:  0.6666666865348816 ,val_recall:  0.6666666865348816 , val_auc:  0.8966667056083679\n",
            "Epoch:  13 , train_loss:  0.3493721734268668 , train_acc:  0.8666666666666667 ,val_loss:  0.4174838257903183 , val_acc:  0.7837837837837838\n",
            "Epoch:  13 , train_f1_score:  0.7916666865348816 , train_precision:  0.8444444537162781 ,train_recall:  0.7450980544090271 , train_auc:  0.902554988861084\n",
            "Epoch:  13 , val_f1_score:  0.6666666865348816 , val_precision:  0.6666666865348816 ,val_recall:  0.6666666865348816 , val_auc:  0.90666663646698\n",
            "Epoch:  14 , train_loss:  0.3181433212266226 , train_acc:  0.8933333333333333 ,val_loss:  0.48791491051166747 , val_acc:  0.8108108108108109\n",
            "Epoch:  14 , train_f1_score:  0.8367347121238708 , train_precision:  0.8723404407501221 ,train_recall:  0.8039215803146362 , train_auc:  0.9164191484451294\n",
            "Epoch:  14 , val_f1_score:  0.695652186870575 , val_precision:  0.7272727489471436 ,val_recall:  0.6666666865348816 , val_auc:  0.9000000357627869\n",
            "Early stopping in epoch:  14 best_accuracy:  0.8108108108108109 current_acc:  0.8108108108108109\n",
            "F1 Score Mean:  0.6666666865348816 , F1 Score Std:  0.0 , Precision Mean:  0.7777777910232544 ,Precision Std:  0.0\n",
            "Recall Mean:  0.5833333134651184 Recall Std:  0.0 AUC Mean:  0.8046027230237759 AUC Std:  0.03740922568728924\n"
          ]
        }
      ],
      "source": [
        "run_test('ct_seg')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "run_test('ct_mult_seg')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M1nXc_xi_sDQ",
        "outputId": "2b33eadf-4378-4160-beb7-c947b8c5aa31"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Data is loaded, training starting\n",
            "Fold [1/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6296973705616773 , train_acc:  0.6577181208053692 ,val_loss:  0.8605739148823839 , val_acc:  0.7368421052631579\n",
            "Epoch:  0 , train_f1_score:  0.4137931168079376 , train_precision:  0.4864864945411682 ,train_recall:  0.36000001430511475 , train_auc:  0.6412121057510376\n",
            "Epoch:  0 , val_f1_score:  0.375 , val_precision:  1.0 ,val_recall:  0.23076923191547394 , val_auc:  0.5169230699539185\n",
            "Save Model, best_accuracy:  0.7368421052631579\n",
            "Epoch:  1 , train_loss:  0.5700314129208959 , train_acc:  0.6912751677852349 ,val_loss:  0.41058373790369124 , val_acc:  0.8421052631578947\n",
            "Epoch:  1 , train_f1_score:  0.4390243887901306 , train_precision:  0.5625 ,train_recall:  0.36000001430511475 , train_auc:  0.6993939280509949\n",
            "Epoch:  1 , val_f1_score:  0.7272727489471436 , val_precision:  0.8888888955116272 ,val_recall:  0.6153846383094788 , val_auc:  0.944615364074707\n",
            "Save Model, best_accuracy:  0.8421052631578947\n",
            "Epoch:  2 , train_loss:  0.5040905873803735 , train_acc:  0.7315436241610739 ,val_loss:  0.4000239398722586 , val_acc:  0.868421052631579\n",
            "Epoch:  2 , train_f1_score:  0.523809552192688 , train_precision:  0.6470588445663452 ,train_recall:  0.4399999976158142 , train_auc:  0.7856565713882446\n",
            "Epoch:  2 , val_f1_score:  0.761904776096344 , val_precision:  1.0 ,val_recall:  0.6153846383094788 , val_auc:  0.9569230079650879\n",
            "Save Model, best_accuracy:  0.868421052631579\n",
            "Epoch:  3 , train_loss:  0.4902249403038506 , train_acc:  0.7449664429530202 ,val_loss:  0.38926595029739763 , val_acc:  0.868421052631579\n",
            "Epoch:  3 , train_f1_score:  0.5476190447807312 , train_precision:  0.6764705777168274 ,train_recall:  0.46000000834465027 , train_auc:  0.7973737120628357\n",
            "Epoch:  3 , val_f1_score:  0.761904776096344 , val_precision:  1.0 ,val_recall:  0.6153846383094788 , val_auc:  0.9569231271743774\n",
            "Epoch:  4 , train_loss:  0.47583236237409704 , train_acc:  0.7785234899328859 ,val_loss:  0.3831180213478786 , val_acc:  0.868421052631579\n",
            "Epoch:  4 , train_f1_score:  0.6117647290229797 , train_precision:  0.7428571581840515 ,train_recall:  0.5199999809265137 , train_auc:  0.8179798126220703\n",
            "Epoch:  4 , val_f1_score:  0.761904776096344 , val_precision:  1.0 ,val_recall:  0.6153846383094788 , val_auc:  0.9569231271743774\n",
            "Epoch:  5 , train_loss:  0.4640990288165592 , train_acc:  0.7785234899328859 ,val_loss:  0.37866559719335974 , val_acc:  0.8421052631578947\n",
            "Epoch:  5 , train_f1_score:  0.6117647290229797 , train_precision:  0.7428571581840515 ,train_recall:  0.5199999809265137 , train_auc:  0.8262625932693481\n",
            "Epoch:  5 , val_f1_score:  0.699999988079071 , val_precision:  1.0 ,val_recall:  0.5384615659713745 , val_auc:  0.9600000381469727\n",
            "Epoch:  6 , train_loss:  0.44689619843691786 , train_acc:  0.8053691275167785 ,val_loss:  0.3725063893178135 , val_acc:  0.8421052631578947\n",
            "Epoch:  6 , train_f1_score:  0.6666666865348816 , train_precision:  0.7837837934494019 ,train_recall:  0.5799999833106995 , train_auc:  0.8438383340835571\n",
            "Epoch:  6 , val_f1_score:  0.7272727489471436 , val_precision:  0.8888888955116272 ,val_recall:  0.6153846383094788 , val_auc:  0.947692334651947\n",
            "Epoch:  7 , train_loss:  0.4287989538166072 , train_acc:  0.8187919463087249 ,val_loss:  0.37208125322764607 , val_acc:  0.868421052631579\n",
            "Epoch:  7 , train_f1_score:  0.6823529601097107 , train_precision:  0.8285714387893677 ,train_recall:  0.5799999833106995 , train_auc:  0.8585858345031738\n",
            "Epoch:  7 , val_f1_score:  0.761904776096344 , val_precision:  1.0 ,val_recall:  0.6153846383094788 , val_auc:  0.947692334651947\n",
            "Early stopping in epoch:  7 best_accuracy:  0.868421052631579 current_acc:  0.868421052631579\n",
            "Fold [2/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.7260107378031584 , train_acc:  0.6778523489932886 ,val_loss:  0.6948721620597338 , val_acc:  0.6578947368421053\n",
            "Epoch:  0 , train_f1_score:  0.03999999910593033 , train_precision:  1.0 ,train_recall:  0.020408162847161293 , train_auc:  0.4114285707473755\n",
            "Epoch:  0 , val_f1_score:  0.13333334028720856 , val_precision:  1.0 ,val_recall:  0.0714285746216774 , val_auc:  0.6815476417541504\n",
            "Save Model, best_accuracy:  0.6578947368421053\n",
            "Epoch:  1 , train_loss:  0.5879818162187034 , train_acc:  0.697986577181208 ,val_loss:  0.6207598067427936 , val_acc:  0.6842105263157895\n",
            "Epoch:  1 , train_f1_score:  0.3478260934352875 , train_precision:  0.6000000238418579 ,train_recall:  0.2448979616165161 , train_auc:  0.7055101990699768\n",
            "Epoch:  1 , val_f1_score:  0.4000000059604645 , val_precision:  0.6666666865348816 ,val_recall:  0.2857142984867096 , val_auc:  0.726190447807312\n",
            "Save Model, best_accuracy:  0.6842105263157895\n",
            "Epoch:  2 , train_loss:  0.4778562476694421 , train_acc:  0.7651006711409396 ,val_loss:  0.6075429212498037 , val_acc:  0.7105263157894737\n",
            "Epoch:  2 , train_f1_score:  0.5454545617103577 , train_precision:  0.75 ,train_recall:  0.4285714328289032 , train_auc:  0.8151020407676697\n",
            "Epoch:  2 , val_f1_score:  0.4761904776096344 , val_precision:  0.7142857313156128 ,val_recall:  0.3571428656578064 , val_auc:  0.7380952835083008\n",
            "Save Model, best_accuracy:  0.7105263157894737\n",
            "Epoch:  3 , train_loss:  0.4520075639847341 , train_acc:  0.7919463087248322 ,val_loss:  0.5981236411944816 , val_acc:  0.7105263157894737\n",
            "Epoch:  3 , train_f1_score:  0.6265060305595398 , train_precision:  0.7647058963775635 ,train_recall:  0.5306122303009033 , train_auc:  0.8287755250930786\n",
            "Epoch:  3 , val_f1_score:  0.4761904776096344 , val_precision:  0.7142857313156128 ,val_recall:  0.3571428656578064 , val_auc:  0.7827380895614624\n",
            "Epoch:  4 , train_loss:  0.43366375385202316 , train_acc:  0.8120805369127517 ,val_loss:  0.5863615309721545 , val_acc:  0.7105263157894737\n",
            "Epoch:  4 , train_f1_score:  0.6666666865348816 , train_precision:  0.800000011920929 ,train_recall:  0.5714285969734192 , train_auc:  0.8479591608047485\n",
            "Epoch:  4 , val_f1_score:  0.4761904776096344 , val_precision:  0.7142857313156128 ,val_recall:  0.3571428656578064 , val_auc:  0.800595223903656\n",
            "Epoch:  5 , train_loss:  0.42259349168986143 , train_acc:  0.8187919463087249 ,val_loss:  0.580830855393096 , val_acc:  0.7105263157894737\n",
            "Epoch:  5 , train_f1_score:  0.6896551847457886 , train_precision:  0.7894737124443054 ,train_recall:  0.6122449040412903 , train_auc:  0.8563265204429626\n",
            "Epoch:  5 , val_f1_score:  0.4761904776096344 , val_precision:  0.7142857313156128 ,val_recall:  0.3571428656578064 , val_auc:  0.8035714030265808\n",
            "Epoch:  6 , train_loss:  0.4019804325602455 , train_acc:  0.8322147651006712 ,val_loss:  0.5777895097669802 , val_acc:  0.7105263157894737\n",
            "Epoch:  6 , train_f1_score:  0.7058823704719543 , train_precision:  0.8333333134651184 ,train_recall:  0.6122449040412903 , train_auc:  0.8710204362869263\n",
            "Epoch:  6 , val_f1_score:  0.4761904776096344 , val_precision:  0.7142857313156128 ,val_recall:  0.3571428656578064 , val_auc:  0.8065475821495056\n",
            "Epoch:  7 , train_loss:  0.3830610106220811 , train_acc:  0.8523489932885906 ,val_loss:  0.5804972394712662 , val_acc:  0.7368421052631579\n",
            "Epoch:  7 , train_f1_score:  0.75 , train_precision:  0.8461538553237915 ,train_recall:  0.6734693646430969 , train_auc:  0.8826530575752258\n",
            "Epoch:  7 , val_f1_score:  0.5454545617103577 , val_precision:  0.75 ,val_recall:  0.4285714328289032 , val_auc:  0.8125\n",
            "Save Model, best_accuracy:  0.7368421052631579\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7368421052631579 current_acc:  0.7368421052631579\n",
            "Fold [3/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6995464288194975 , train_acc:  0.6866666666666666 ,val_loss:  0.6891013371783334 , val_acc:  0.6216216216216216\n",
            "Epoch:  0 , train_f1_score:  0.20338982343673706 , train_precision:  0.6666666865348816 ,train_recall:  0.11999999731779099 , train_auc:  0.6019999980926514\n",
            "Epoch:  0 , val_f1_score:  0.2222222238779068 , val_precision:  0.4000000059604645 ,val_recall:  0.1538461595773697 , val_auc:  0.5224359035491943\n",
            "Save Model, best_accuracy:  0.6216216216216216\n",
            "Epoch:  1 , train_loss:  0.610587463180224 , train_acc:  0.7866666666666666 ,val_loss:  0.5160931437905576 , val_acc:  0.7837837837837838\n",
            "Epoch:  1 , train_f1_score:  0.6097561120986938 , train_precision:  0.78125 ,train_recall:  0.5 , train_auc:  0.7283999919891357\n",
            "Epoch:  1 , val_f1_score:  0.6666666865348816 , val_precision:  0.7272727489471436 ,val_recall:  0.6153846383094788 , val_auc:  0.8108974695205688\n",
            "Save Model, best_accuracy:  0.7837837837837838\n",
            "Epoch:  2 , train_loss:  0.4964658938441426 , train_acc:  0.82 ,val_loss:  0.5205779605217882 , val_acc:  0.7567567567567568\n",
            "Epoch:  2 , train_f1_score:  0.6746987700462341 , train_precision:  0.8484848737716675 ,train_recall:  0.5600000023841858 , train_auc:  0.8134000301361084\n",
            "Epoch:  2 , val_f1_score:  0.6086956262588501 , val_precision:  0.699999988079071 ,val_recall:  0.5384615659713745 , val_auc:  0.807692289352417\n",
            "Epoch:  3 , train_loss:  0.45595667182312655 , train_acc:  0.82 ,val_loss:  0.5257175523737395 , val_acc:  0.7297297297297297\n",
            "Epoch:  3 , train_f1_score:  0.6896551847457886 , train_precision:  0.8108108043670654 ,train_recall:  0.6000000238418579 , train_auc:  0.8312000036239624\n",
            "Epoch:  3 , val_f1_score:  0.5454545617103577 , val_precision:  0.6666666865348816 ,val_recall:  0.4615384638309479 , val_auc:  0.807692289352417\n",
            "Epoch:  4 , train_loss:  0.4290879977152993 , train_acc:  0.8333333333333334 ,val_loss:  0.5281489107103364 , val_acc:  0.7297297297297297\n",
            "Epoch:  4 , train_f1_score:  0.7126436829566956 , train_precision:  0.837837815284729 ,train_recall:  0.6200000047683716 , train_auc:  0.8578000068664551\n",
            "Epoch:  4 , val_f1_score:  0.5454545617103577 , val_precision:  0.6666666865348816 ,val_recall:  0.4615384638309479 , val_auc:  0.8012820482254028\n",
            "Epoch:  5 , train_loss:  0.41857663452004396 , train_acc:  0.8466666666666667 ,val_loss:  0.5277085510409765 , val_acc:  0.7567567567567568\n",
            "Epoch:  5 , train_f1_score:  0.7356321811676025 , train_precision:  0.8648648858070374 ,train_recall:  0.6399999856948853 , train_auc:  0.8637999892234802\n",
            "Epoch:  5 , val_f1_score:  0.6086956262588501 , val_precision:  0.699999988079071 ,val_recall:  0.5384615659713745 , val_auc:  0.8012820482254028\n",
            "Epoch:  6 , train_loss:  0.40880132544940956 , train_acc:  0.8466666666666667 ,val_loss:  0.5283071227069642 , val_acc:  0.7297297297297297\n",
            "Epoch:  6 , train_f1_score:  0.7356321811676025 , train_precision:  0.8648648858070374 ,train_recall:  0.6399999856948853 , train_auc:  0.8711999654769897\n",
            "Epoch:  6 , val_f1_score:  0.5454545617103577 , val_precision:  0.6666666865348816 ,val_recall:  0.4615384638309479 , val_auc:  0.8012820482254028\n",
            "Epoch:  7 , train_loss:  0.39653180084191264 , train_acc:  0.8466666666666667 ,val_loss:  0.5279753905897205 , val_acc:  0.7297297297297297\n",
            "Epoch:  7 , train_f1_score:  0.7356321811676025 , train_precision:  0.8648648858070374 ,train_recall:  0.6399999856948853 , train_auc:  0.8782000541687012\n",
            "Epoch:  7 , val_f1_score:  0.5454545617103577 , val_precision:  0.6666666865348816 ,val_recall:  0.4615384638309479 , val_auc:  0.8044871687889099\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7837837837837838 current_acc:  0.7297297297297297\n",
            "Fold [4/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.5878475284291198 , train_acc:  0.7133333333333334 ,val_loss:  0.746711929926548 , val_acc:  0.7567567567567568\n",
            "Epoch:  0 , train_f1_score:  0.5057471394538879 , train_precision:  0.6285714507102966 ,train_recall:  0.42307692766189575 , train_auc:  0.6866169571876526\n",
            "Epoch:  0 , val_f1_score:  0.4000000059604645 , val_precision:  0.75 ,val_recall:  0.27272728085517883 , val_auc:  0.5139859914779663\n",
            "Save Model, best_accuracy:  0.7567567567567568\n",
            "Epoch:  1 , train_loss:  0.5395955094498155 , train_acc:  0.7733333333333333 ,val_loss:  0.469522184830763 , val_acc:  0.7567567567567568\n",
            "Epoch:  1 , train_f1_score:  0.6222222447395325 , train_precision:  0.7368420958518982 ,train_recall:  0.5384615659713745 , train_auc:  0.779434859752655\n",
            "Epoch:  1 , val_f1_score:  0.6086956262588501 , val_precision:  0.5833333134651184 ,val_recall:  0.6363636255264282 , val_auc:  0.884615421295166\n",
            "Epoch:  2 , train_loss:  0.47623512448364635 , train_acc:  0.8 ,val_loss:  0.45512780271520886 , val_acc:  0.7567567567567568\n",
            "Epoch:  2 , train_f1_score:  0.6590909361839294 , train_precision:  0.8055555820465088 ,train_recall:  0.557692289352417 , train_auc:  0.8228021860122681\n",
            "Epoch:  2 , val_f1_score:  0.6086956262588501 , val_precision:  0.5833333134651184 ,val_recall:  0.6363636255264282 , val_auc:  0.884615421295166\n",
            "Epoch:  3 , train_loss:  0.4569024067222684 , train_acc:  0.82 ,val_loss:  0.4517442893387901 , val_acc:  0.7837837837837838\n",
            "Epoch:  3 , train_f1_score:  0.7032967209815979 , train_precision:  0.8205128312110901 ,train_recall:  0.6153846383094788 , train_auc:  0.8390895128250122\n",
            "Epoch:  3 , val_f1_score:  0.6363636255264282 , val_precision:  0.6363636255264282 ,val_recall:  0.6363636255264282 , val_auc:  0.881118893623352\n",
            "Save Model, best_accuracy:  0.7837837837837838\n",
            "Epoch:  4 , train_loss:  0.4448765531661532 , train_acc:  0.82 ,val_loss:  0.4548153069776458 , val_acc:  0.7567567567567568\n",
            "Epoch:  4 , train_f1_score:  0.7032967209815979 , train_precision:  0.8205128312110901 ,train_recall:  0.6153846383094788 , train_auc:  0.8434064984321594\n",
            "Epoch:  4 , val_f1_score:  0.6086956262588501 , val_precision:  0.5833333134651184 ,val_recall:  0.6363636255264282 , val_auc:  0.8706293702125549\n",
            "Epoch:  5 , train_loss:  0.42970900872470036 , train_acc:  0.8266666666666667 ,val_loss:  0.4497455573967985 , val_acc:  0.7837837837837838\n",
            "Epoch:  5 , train_f1_score:  0.717391312122345 , train_precision:  0.824999988079071 ,train_recall:  0.6346153616905212 , train_auc:  0.8500784635543823\n",
            "Epoch:  5 , val_f1_score:  0.6363636255264282 , val_precision:  0.6363636255264282 ,val_recall:  0.6363636255264282 , val_auc:  0.8671329021453857\n",
            "Epoch:  6 , train_loss:  0.4175689744495321 , train_acc:  0.8333333333333334 ,val_loss:  0.4576802499265083 , val_acc:  0.7837837837837838\n",
            "Epoch:  6 , train_f1_score:  0.7311828136444092 , train_precision:  0.8292682766914368 ,train_recall:  0.6538461446762085 , train_auc:  0.8608712553977966\n",
            "Epoch:  6 , val_f1_score:  0.6363636255264282 , val_precision:  0.6363636255264282 ,val_recall:  0.6363636255264282 , val_auc:  0.8636363744735718\n",
            "Epoch:  7 , train_loss:  0.39990612647748397 , train_acc:  0.8333333333333334 ,val_loss:  0.459431885540284 , val_acc:  0.7567567567567568\n",
            "Epoch:  7 , train_f1_score:  0.7311828136444092 , train_precision:  0.8292682766914368 ,train_recall:  0.6538461446762085 , train_auc:  0.8759811520576477\n",
            "Epoch:  7 , val_f1_score:  0.6086956262588501 , val_precision:  0.5833333134651184 ,val_recall:  0.6363636255264282 , val_auc:  0.8706294298171997\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7837837837837838 current_acc:  0.7567567567567568\n",
            "Fold [5/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6978215788801511 , train_acc:  0.68 ,val_loss:  0.5608971133022695 , val_acc:  0.8108108108108109\n",
            "Epoch:  0 , train_f1_score:  0.17241379618644714 , train_precision:  0.7142857313156128 ,train_recall:  0.09803921729326248 , train_auc:  0.6064567565917969\n",
            "Epoch:  0 , val_f1_score:  0.5882353186607361 , val_precision:  1.0 ,val_recall:  0.4166666567325592 , val_auc:  0.8100000023841858\n",
            "Save Model, best_accuracy:  0.8108108108108109\n",
            "Epoch:  1 , train_loss:  0.6252687210465471 , train_acc:  0.7333333333333333 ,val_loss:  0.5011705014464521 , val_acc:  0.8378378378378378\n",
            "Epoch:  1 , train_f1_score:  0.5348837375640869 , train_precision:  0.6571428775787354 ,train_recall:  0.45098039507865906 , train_auc:  0.7320261001586914\n",
            "Epoch:  1 , val_f1_score:  0.7692307829856873 , val_precision:  0.7142857313156128 ,val_recall:  0.8333333134651184 , val_auc:  0.8233333826065063\n",
            "Save Model, best_accuracy:  0.8378378378378378\n",
            "Epoch:  2 , train_loss:  0.503127711241444 , train_acc:  0.7666666666666667 ,val_loss:  0.47759589820634574 , val_acc:  0.8378378378378378\n",
            "Epoch:  2 , train_f1_score:  0.6153846383094788 , train_precision:  0.699999988079071 ,train_recall:  0.5490196347236633 , train_auc:  0.8072885870933533\n",
            "Epoch:  2 , val_f1_score:  0.7692307829856873 , val_precision:  0.7142857313156128 ,val_recall:  0.8333333134651184 , val_auc:  0.84333336353302\n",
            "Epoch:  3 , train_loss:  0.4626065775814156 , train_acc:  0.7866666666666666 ,val_loss:  0.4723123349894367 , val_acc:  0.8378378378378378\n",
            "Epoch:  3 , train_f1_score:  0.6521739363670349 , train_precision:  0.7317073345184326 ,train_recall:  0.5882353186607361 , train_auc:  0.8286789655685425\n",
            "Epoch:  3 , val_f1_score:  0.7692307829856873 , val_precision:  0.7142857313156128 ,val_recall:  0.8333333134651184 , val_auc:  0.8500000238418579\n",
            "Epoch:  4 , train_loss:  0.43770258125228184 , train_acc:  0.8 ,val_loss:  0.4726474776657965 , val_acc:  0.8378378378378378\n",
            "Epoch:  4 , train_f1_score:  0.6808510422706604 , train_precision:  0.7441860437393188 ,train_recall:  0.6274510025978088 , train_auc:  0.8516538143157959\n",
            "Epoch:  4 , val_f1_score:  0.7692307829856873 , val_precision:  0.7142857313156128 ,val_recall:  0.8333333134651184 , val_auc:  0.8500000238418579\n",
            "Epoch:  5 , train_loss:  0.4247482724813744 , train_acc:  0.8066666666666666 ,val_loss:  0.46051980949847054 , val_acc:  0.8378378378378378\n",
            "Epoch:  5 , train_f1_score:  0.6947368383407593 , train_precision:  0.75 ,train_recall:  0.6470588445663452 , train_auc:  0.8621509075164795\n",
            "Epoch:  5 , val_f1_score:  0.7692307829856873 , val_precision:  0.7142857313156128 ,val_recall:  0.8333333134651184 , val_auc:  0.8400000333786011\n",
            "Epoch:  6 , train_loss:  0.4128664060030133 , train_acc:  0.8066666666666666 ,val_loss:  0.46882130132478383 , val_acc:  0.8378378378378378\n",
            "Epoch:  6 , train_f1_score:  0.6947368383407593 , train_precision:  0.75 ,train_recall:  0.6470588445663452 , train_auc:  0.8673005104064941\n",
            "Epoch:  6 , val_f1_score:  0.7692307829856873 , val_precision:  0.7142857313156128 ,val_recall:  0.8333333134651184 , val_auc:  0.84333336353302\n",
            "Epoch:  7 , train_loss:  0.3961663111470019 , train_acc:  0.8066666666666666 ,val_loss:  0.463972263812163 , val_acc:  0.8378378378378378\n",
            "Epoch:  7 , train_f1_score:  0.6947368383407593 , train_precision:  0.75 ,train_recall:  0.6470588445663452 , train_auc:  0.8805704116821289\n",
            "Epoch:  7 , val_f1_score:  0.7692307829856873 , val_precision:  0.7142857313156128 ,val_recall:  0.8333333134651184 , val_auc:  0.8400000333786011\n",
            "Early stopping in epoch:  7 best_accuracy:  0.8378378378378378 current_acc:  0.8378378378378378\n",
            "F1 Score Mean:  0.7692307829856873 , F1 Score Std:  0.0 , Precision Mean:  0.7142857313156128 ,Precision Std:  0.0\n",
            "Recall Mean:  0.8333333134651184 Recall Std:  0.0 AUC Mean:  0.7823816297500509 AUC Std:  0.06340325864841052\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "run_test('pet_seg')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rc2g_q7c_r8c",
        "outputId": "8143d4b2-5804-4f7e-8ec1-2f3113e2b76e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Data is loaded, training starting\n",
            "Fold [1/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.5804490283281611 , train_acc:  0.7583892617449665 ,val_loss:  0.5892073521880727 , val_acc:  0.7105263157894737\n",
            "Epoch:  0 , train_f1_score:  0.6326530575752258 , train_precision:  0.6458333134651184 ,train_recall:  0.6200000047683716 , train_auc:  0.7612121105194092\n",
            "Epoch:  0 , val_f1_score:  0.4761904776096344 , val_precision:  0.625 ,val_recall:  0.38461539149284363 , val_auc:  0.7230769395828247\n",
            "Save Model, best_accuracy:  0.7105263157894737\n",
            "Epoch:  1 , train_loss:  0.5018892890004903 , train_acc:  0.7785234899328859 ,val_loss:  0.5005916923186496 , val_acc:  0.7631578947368421\n",
            "Epoch:  1 , train_f1_score:  0.6117647290229797 , train_precision:  0.7428571581840515 ,train_recall:  0.5199999809265137 , train_auc:  0.803636372089386\n",
            "Epoch:  1 , val_f1_score:  0.6086956262588501 , val_precision:  0.699999988079071 ,val_recall:  0.5384615659713745 , val_auc:  0.8215384483337402\n",
            "Save Model, best_accuracy:  0.7631578947368421\n",
            "Epoch:  2 , train_loss:  0.4560015915749257 , train_acc:  0.8322147651006712 ,val_loss:  0.49815665651112795 , val_acc:  0.7631578947368421\n",
            "Epoch:  2 , train_f1_score:  0.7126436829566956 , train_precision:  0.837837815284729 ,train_recall:  0.6200000047683716 , train_auc:  0.8488888740539551\n",
            "Epoch:  2 , val_f1_score:  0.6086956262588501 , val_precision:  0.699999988079071 ,val_recall:  0.5384615659713745 , val_auc:  0.8338460922241211\n",
            "Epoch:  3 , train_loss:  0.4391215657065279 , train_acc:  0.8322147651006712 ,val_loss:  0.5005536575458551 , val_acc:  0.7631578947368421\n",
            "Epoch:  3 , train_f1_score:  0.7126436829566956 , train_precision:  0.837837815284729 ,train_recall:  0.6200000047683716 , train_auc:  0.862828254699707\n",
            "Epoch:  3 , val_f1_score:  0.6086956262588501 , val_precision:  0.699999988079071 ,val_recall:  0.5384615659713745 , val_auc:  0.8369230628013611\n",
            "Epoch:  4 , train_loss:  0.42631037416086603 , train_acc:  0.825503355704698 ,val_loss:  0.5026691567740942 , val_acc:  0.7631578947368421\n",
            "Epoch:  4 , train_f1_score:  0.7045454382896423 , train_precision:  0.8157894611358643 ,train_recall:  0.6200000047683716 , train_auc:  0.8735353946685791\n",
            "Epoch:  4 , val_f1_score:  0.6086956262588501 , val_precision:  0.699999988079071 ,val_recall:  0.5384615659713745 , val_auc:  0.8338460922241211\n",
            "Epoch:  5 , train_loss:  0.4187463864023116 , train_acc:  0.8322147651006712 ,val_loss:  0.5050130680595574 , val_acc:  0.7631578947368421\n",
            "Epoch:  5 , train_f1_score:  0.7191011309623718 , train_precision:  0.8205128312110901 ,train_recall:  0.6399999856948853 , train_auc:  0.8789898753166199\n",
            "Epoch:  5 , val_f1_score:  0.6086956262588501 , val_precision:  0.699999988079071 ,val_recall:  0.5384615659713745 , val_auc:  0.8338460922241211\n",
            "Epoch:  6 , train_loss:  0.39121576917401496 , train_acc:  0.8389261744966443 ,val_loss:  0.5082978151346508 , val_acc:  0.7631578947368421\n",
            "Epoch:  6 , train_f1_score:  0.7333333492279053 , train_precision:  0.824999988079071 ,train_recall:  0.6600000262260437 , train_auc:  0.9008080363273621\n",
            "Epoch:  6 , val_f1_score:  0.6086956262588501 , val_precision:  0.699999988079071 ,val_recall:  0.5384615659713745 , val_auc:  0.8399999141693115\n",
            "Epoch:  7 , train_loss:  0.33544293160420596 , train_acc:  0.8859060402684564 ,val_loss:  0.5071992429071351 , val_acc:  0.7368421052631579\n",
            "Epoch:  7 , train_f1_score:  0.821052610874176 , train_precision:  0.8666666746139526 ,train_recall:  0.7799999713897705 , train_auc:  0.9268686771392822\n",
            "Epoch:  7 , val_f1_score:  0.5454545617103577 , val_precision:  0.6666666865348816 ,val_recall:  0.4615384638309479 , val_auc:  0.8523076772689819\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7631578947368421 current_acc:  0.7368421052631579\n",
            "Fold [2/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6345499778578946 , train_acc:  0.6644295302013423 ,val_loss:  0.5311634842815501 , val_acc:  0.8421052631578947\n",
            "Epoch:  0 , train_f1_score:  0.375 , train_precision:  0.5555555820465088 ,train_recall:  0.2830188572406769 , train_auc:  0.5841194987297058\n",
            "Epoch:  0 , val_f1_score:  0.5714285969734192 , val_precision:  1.0 ,val_recall:  0.4000000059604645 , val_auc:  0.7142857313156128\n",
            "Save Model, best_accuracy:  0.8421052631578947\n",
            "Epoch:  1 , train_loss:  0.5819437973468483 , train_acc:  0.6644295302013423 ,val_loss:  0.38534271640894174 , val_acc:  0.8421052631578947\n",
            "Epoch:  1 , train_f1_score:  0.375 , train_precision:  0.5555555820465088 ,train_recall:  0.2830188572406769 , train_auc:  0.6992924213409424\n",
            "Epoch:  1 , val_f1_score:  0.5714285969734192 , val_precision:  1.0 ,val_recall:  0.4000000059604645 , val_auc:  0.9821428060531616\n",
            "Epoch:  2 , train_loss:  0.5292573427256422 , train_acc:  0.7181208053691275 ,val_loss:  0.3689404406532527 , val_acc:  0.868421052631579\n",
            "Epoch:  2 , train_f1_score:  0.4878048896789551 , train_precision:  0.6896551847457886 ,train_recall:  0.37735849618911743 , train_auc:  0.7838050127029419\n",
            "Epoch:  2 , val_f1_score:  0.7058823704719543 , val_precision:  0.8571428656578064 ,val_recall:  0.6000000238418579 , val_auc:  0.9714285135269165\n",
            "Save Model, best_accuracy:  0.868421052631579\n",
            "Epoch:  3 , train_loss:  0.5090906759841217 , train_acc:  0.7181208053691275 ,val_loss:  0.3532426424020454 , val_acc:  0.8421052631578947\n",
            "Epoch:  3 , train_f1_score:  0.4878048896789551 , train_precision:  0.6896551847457886 ,train_recall:  0.37735849618911743 , train_auc:  0.8089622259140015\n",
            "Epoch:  3 , val_f1_score:  0.625 , val_precision:  0.8333333134651184 ,val_recall:  0.5 , val_auc:  0.9714285731315613\n",
            "Epoch:  4 , train_loss:  0.48818735984363437 , train_acc:  0.7449664429530202 ,val_loss:  0.33994186356626177 , val_acc:  0.8421052631578947\n",
            "Epoch:  4 , train_f1_score:  0.5581395626068115 , train_precision:  0.7272727489471436 ,train_recall:  0.4528301954269409 , train_auc:  0.8268475532531738\n",
            "Epoch:  4 , val_f1_score:  0.625 , val_precision:  0.8333333134651184 ,val_recall:  0.5 , val_auc:  0.9678571224212646\n",
            "Epoch:  5 , train_loss:  0.4611166603761669 , train_acc:  0.7919463087248322 ,val_loss:  0.3302885095996604 , val_acc:  0.8157894736842105\n",
            "Epoch:  5 , train_f1_score:  0.6436781883239746 , train_precision:  0.8235294222831726 ,train_recall:  0.5283018946647644 , train_auc:  0.8527909517288208\n",
            "Epoch:  5 , val_f1_score:  0.5333333611488342 , val_precision:  0.800000011920929 ,val_recall:  0.4000000059604645 , val_auc:  0.9678571224212646\n",
            "Epoch:  6 , train_loss:  0.41564198849635237 , train_acc:  0.8456375838926175 ,val_loss:  0.3313915788423018 , val_acc:  0.8421052631578947\n",
            "Epoch:  6 , train_f1_score:  0.7526881694793701 , train_precision:  0.875 ,train_recall:  0.6603773832321167 , train_auc:  0.8810927867889404\n",
            "Epoch:  6 , val_f1_score:  0.6666666865348816 , val_precision:  0.75 ,val_recall:  0.6000000238418579 , val_auc:  0.9357142448425293\n",
            "Epoch:  7 , train_loss:  0.36089906780500997 , train_acc:  0.8657718120805369 ,val_loss:  0.349167328243742 , val_acc:  0.8157894736842105\n",
            "Epoch:  7 , train_f1_score:  0.7916666865348816 , train_precision:  0.8837209343910217 ,train_recall:  0.7169811129570007 , train_auc:  0.902515709400177\n",
            "Epoch:  7 , val_f1_score:  0.5882353186607361 , val_precision:  0.7142857313156128 ,val_recall:  0.5 , val_auc:  0.9142857193946838\n",
            "Early stopping in epoch:  7 best_accuracy:  0.868421052631579 current_acc:  0.8157894736842105\n",
            "Fold [3/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.5923658561054617 , train_acc:  0.6666666666666666 ,val_loss:  0.4960328275400743 , val_acc:  0.8378378378378378\n",
            "Epoch:  0 , train_f1_score:  0.4444444477558136 , train_precision:  0.5128205418586731 ,train_recall:  0.3921568691730499 , train_auc:  0.7169736623764038\n",
            "Epoch:  0 , val_f1_score:  0.699999988079071 , val_precision:  0.875 ,val_recall:  0.5833333134651184 , val_auc:  0.7100000381469727\n",
            "Save Model, best_accuracy:  0.8378378378378378\n",
            "Epoch:  1 , train_loss:  0.5397699354071422 , train_acc:  0.7333333333333333 ,val_loss:  0.39428779297068595 , val_acc:  0.8378378378378378\n",
            "Epoch:  1 , train_f1_score:  0.5555555820465088 , train_precision:  0.6410256624221802 ,train_recall:  0.4901960790157318 , train_auc:  0.7381659746170044\n",
            "Epoch:  1 , val_f1_score:  0.75 , val_precision:  0.75 ,val_recall:  0.75 , val_auc:  0.9466665983200073\n",
            "Epoch:  2 , train_loss:  0.4844902029003424 , train_acc:  0.7933333333333333 ,val_loss:  0.40134962232450316 , val_acc:  0.8918918918918919\n",
            "Epoch:  2 , train_f1_score:  0.6352941393852234 , train_precision:  0.7941176295280457 ,train_recall:  0.529411792755127 , train_auc:  0.8070905208587646\n",
            "Epoch:  2 , val_f1_score:  0.8461538553237915 , val_precision:  0.7857142686843872 ,val_recall:  0.9166666865348816 , val_auc:  0.9399999380111694\n",
            "Save Model, best_accuracy:  0.8918918918918919\n",
            "Epoch:  3 , train_loss:  0.4649585017009425 , train_acc:  0.8066666666666666 ,val_loss:  0.40118228581962506 , val_acc:  0.8918918918918919\n",
            "Epoch:  3 , train_f1_score:  0.6741573214530945 , train_precision:  0.7894737124443054 ,train_recall:  0.5882353186607361 , train_auc:  0.8270944356918335\n",
            "Epoch:  3 , val_f1_score:  0.8461538553237915 , val_precision:  0.7857142686843872 ,val_recall:  0.9166666865348816 , val_auc:  0.9399999380111694\n",
            "Epoch:  4 , train_loss:  0.44443825921756797 , train_acc:  0.82 ,val_loss:  0.3987626174635512 , val_acc:  0.8918918918918919\n",
            "Epoch:  4 , train_f1_score:  0.6966292262077332 , train_precision:  0.8157894611358643 ,train_recall:  0.6078431606292725 , train_auc:  0.846108078956604\n",
            "Epoch:  4 , val_f1_score:  0.8461538553237915 , val_precision:  0.7857142686843872 ,val_recall:  0.9166666865348816 , val_auc:  0.9399999380111694\n",
            "Epoch:  5 , train_loss:  0.4324503573078255 , train_acc:  0.8133333333333334 ,val_loss:  0.3831950638480988 , val_acc:  0.8648648648648649\n",
            "Epoch:  5 , train_f1_score:  0.695652186870575 , train_precision:  0.7804877758026123 ,train_recall:  0.6274510025978088 , train_auc:  0.8571994304656982\n",
            "Epoch:  5 , val_f1_score:  0.800000011920929 , val_precision:  0.7692307829856873 ,val_recall:  0.8333333134651184 , val_auc:  0.9433332681655884\n",
            "Epoch:  6 , train_loss:  0.4195000486089945 , train_acc:  0.8266666666666667 ,val_loss:  0.3705372692540452 , val_acc:  0.8648648648648649\n",
            "Epoch:  6 , train_f1_score:  0.7234042286872864 , train_precision:  0.7906976938247681 ,train_recall:  0.6666666865348816 , train_auc:  0.8645276427268982\n",
            "Epoch:  6 , val_f1_score:  0.800000011920929 , val_precision:  0.7692307829856873 ,val_recall:  0.8333333134651184 , val_auc:  0.9366666078567505\n",
            "Epoch:  7 , train_loss:  0.3986109652562785 , train_acc:  0.84 ,val_loss:  0.35292664803784796 , val_acc:  0.8378378378378378\n",
            "Epoch:  7 , train_f1_score:  0.7333333492279053 , train_precision:  0.8461538553237915 ,train_recall:  0.6470588445663452 , train_auc:  0.8694791793823242\n",
            "Epoch:  7 , val_f1_score:  0.75 , val_precision:  0.75 ,val_recall:  0.75 , val_auc:  0.949999988079071\n",
            "Early stopping in epoch:  7 best_accuracy:  0.8918918918918919 current_acc:  0.8378378378378378\n",
            "Fold [4/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.8179742309947808 , train_acc:  0.6333333333333333 ,val_loss:  0.5935092810037974 , val_acc:  0.7027027027027027\n",
            "Epoch:  0 , train_f1_score:  0.1269841343164444 , train_precision:  0.3636363744735718 ,train_recall:  0.07692307978868484 , train_auc:  0.34262168407440186\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.6048951148986816\n",
            "Save Model, best_accuracy:  0.7027027027027027\n",
            "Epoch:  1 , train_loss:  0.5886388884764164 , train_acc:  0.6933333333333334 ,val_loss:  0.5264587982280834 , val_acc:  0.8108108108108109\n",
            "Epoch:  1 , train_f1_score:  0.42500001192092896 , train_precision:  0.6071428656578064 ,train_recall:  0.32692307233810425 , train_auc:  0.7315541505813599\n",
            "Epoch:  1 , val_f1_score:  0.5882353186607361 , val_precision:  0.8333333134651184 ,val_recall:  0.4545454680919647 , val_auc:  0.7762237787246704\n",
            "Save Model, best_accuracy:  0.8108108108108109\n",
            "Epoch:  2 , train_loss:  0.48815652295015755 , train_acc:  0.7733333333333333 ,val_loss:  0.5160520026409948 , val_acc:  0.7837837837837838\n",
            "Epoch:  2 , train_f1_score:  0.604651153087616 , train_precision:  0.7647058963775635 ,train_recall:  0.5 , train_auc:  0.8161302804946899\n",
            "Epoch:  2 , val_f1_score:  0.5 , val_precision:  0.800000011920929 ,val_recall:  0.3636363744735718 , val_auc:  0.7762237787246704\n",
            "Epoch:  3 , train_loss:  0.45001682312227786 , train_acc:  0.8066666666666666 ,val_loss:  0.5213994623438732 , val_acc:  0.7837837837837838\n",
            "Epoch:  3 , train_f1_score:  0.658823549747467 , train_precision:  0.8484848737716675 ,train_recall:  0.5384615659713745 , train_auc:  0.8549842834472656\n",
            "Epoch:  3 , val_f1_score:  0.5 , val_precision:  0.800000011920929 ,val_recall:  0.3636363744735718 , val_auc:  0.7727272510528564\n",
            "Epoch:  4 , train_loss:  0.42907146133870505 , train_acc:  0.8 ,val_loss:  0.5216824817093643 , val_acc:  0.7567567567567568\n",
            "Epoch:  4 , train_f1_score:  0.6590909361839294 , train_precision:  0.8055555820465088 ,train_recall:  0.557692289352417 , train_auc:  0.8706828355789185\n",
            "Epoch:  4 , val_f1_score:  0.47058823704719543 , val_precision:  0.6666666865348816 ,val_recall:  0.3636363744735718 , val_auc:  0.7622377872467041\n",
            "Epoch:  5 , train_loss:  0.40270128538560435 , train_acc:  0.84 ,val_loss:  0.5265802068484796 , val_acc:  0.7567567567567568\n",
            "Epoch:  5 , train_f1_score:  0.7446808218955994 , train_precision:  0.8333333134651184 ,train_recall:  0.6730769276618958 , train_auc:  0.8895211815834045\n",
            "Epoch:  5 , val_f1_score:  0.47058823704719543 , val_precision:  0.6666666865348816 ,val_recall:  0.3636363744735718 , val_auc:  0.7552447319030762\n",
            "Epoch:  6 , train_loss:  0.3609335365272515 , train_acc:  0.8733333333333333 ,val_loss:  0.5382763413158623 , val_acc:  0.7567567567567568\n",
            "Epoch:  6 , train_f1_score:  0.800000011920929 , train_precision:  0.8837209343910217 ,train_recall:  0.7307692170143127 , train_auc:  0.906593382358551\n",
            "Epoch:  6 , val_f1_score:  0.47058823704719543 , val_precision:  0.6666666865348816 ,val_recall:  0.3636363744735718 , val_auc:  0.751748263835907\n",
            "Epoch:  7 , train_loss:  0.2927954933337848 , train_acc:  0.9133333333333333 ,val_loss:  0.5794078931212425 , val_acc:  0.7297297297297297\n",
            "Epoch:  7 , train_f1_score:  0.8659793734550476 , train_precision:  0.9333333373069763 ,train_recall:  0.807692289352417 , train_auc:  0.9319073557853699\n",
            "Epoch:  7 , val_f1_score:  0.375 , val_precision:  0.6000000238418579 ,val_recall:  0.27272728085517883 , val_auc:  0.7377622127532959\n",
            "Early stopping in epoch:  7 best_accuracy:  0.8108108108108109 current_acc:  0.7297297297297297\n",
            "Fold [5/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.5864955484867096 , train_acc:  0.7266666666666667 ,val_loss:  0.8066512523470698 , val_acc:  0.5675675675675675\n",
            "Epoch:  0 , train_f1_score:  0.2545454502105713 , train_precision:  0.7777777910232544 ,train_recall:  0.15217390656471252 , train_auc:  0.6045150756835938\n",
            "Epoch:  0 , val_f1_score:  0.1111111119389534 , val_precision:  1.0 ,val_recall:  0.05882352963089943 , val_auc:  0.39411765336990356\n",
            "Save Model, best_accuracy:  0.5675675675675675\n",
            "Epoch:  1 , train_loss:  0.5093200615234673 , train_acc:  0.7733333333333333 ,val_loss:  0.5918780059704708 , val_acc:  0.7027027027027027\n",
            "Epoch:  1 , train_f1_score:  0.5142857432365417 , train_precision:  0.75 ,train_recall:  0.3913043439388275 , train_auc:  0.781354546546936\n",
            "Epoch:  1 , val_f1_score:  0.5925925970077515 , val_precision:  0.800000011920929 ,val_recall:  0.47058823704719543 , val_auc:  0.7852941155433655\n",
            "Save Model, best_accuracy:  0.7027027027027027\n",
            "Epoch:  2 , train_loss:  0.4264792478006954 , train_acc:  0.8266666666666667 ,val_loss:  0.6088660989724402 , val_acc:  0.7027027027027027\n",
            "Epoch:  2 , train_f1_score:  0.6578947305679321 , train_precision:  0.8333333134651184 ,train_recall:  0.54347825050354 , train_auc:  0.8543059825897217\n",
            "Epoch:  2 , val_f1_score:  0.5925925970077515 , val_precision:  0.800000011920929 ,val_recall:  0.47058823704719543 , val_auc:  0.7941176891326904\n",
            "Epoch:  3 , train_loss:  0.40063190444372593 , train_acc:  0.84 ,val_loss:  0.6160572157681894 , val_acc:  0.7027027027027027\n",
            "Epoch:  3 , train_f1_score:  0.699999988079071 , train_precision:  0.8235294222831726 ,train_recall:  0.6086956262588501 , train_auc:  0.8727006912231445\n",
            "Epoch:  3 , val_f1_score:  0.5925925970077515 , val_precision:  0.800000011920929 ,val_recall:  0.47058823704719543 , val_auc:  0.7823529243469238\n",
            "Epoch:  4 , train_loss:  0.3802555384968097 , train_acc:  0.8666666666666667 ,val_loss:  0.6333453602604316 , val_acc:  0.7027027027027027\n",
            "Epoch:  4 , train_f1_score:  0.761904776096344 , train_precision:  0.8421052694320679 ,train_recall:  0.695652186870575 , train_auc:  0.8871237635612488\n",
            "Epoch:  4 , val_f1_score:  0.5925925970077515 , val_precision:  0.800000011920929 ,val_recall:  0.47058823704719543 , val_auc:  0.7764705419540405\n",
            "Epoch:  5 , train_loss:  0.3547299529782807 , train_acc:  0.86 ,val_loss:  0.6480277846000988 , val_acc:  0.7027027027027027\n",
            "Epoch:  5 , train_f1_score:  0.7529411911964417 , train_precision:  0.8205128312110901 ,train_recall:  0.695652186870575 , train_auc:  0.903637170791626\n",
            "Epoch:  5 , val_f1_score:  0.5925925970077515 , val_precision:  0.800000011920929 ,val_recall:  0.47058823704719543 , val_auc:  0.7823529243469238\n",
            "Epoch:  6 , train_loss:  0.3197706345375627 , train_acc:  0.8733333333333333 ,val_loss:  0.6854632376737239 , val_acc:  0.7027027027027027\n",
            "Epoch:  6 , train_f1_score:  0.7816091775894165 , train_precision:  0.8292682766914368 ,train_recall:  0.739130437374115 , train_auc:  0.917015016078949\n",
            "Epoch:  6 , val_f1_score:  0.5925925970077515 , val_precision:  0.800000011920929 ,val_recall:  0.47058823704719543 , val_auc:  0.7852941155433655\n",
            "Epoch:  7 , train_loss:  0.26774318762589244 , train_acc:  0.9 ,val_loss:  0.7718417985220062 , val_acc:  0.6486486486486487\n",
            "Epoch:  7 , train_f1_score:  0.8314606547355652 , train_precision:  0.8604651093482971 ,train_recall:  0.804347813129425 , train_auc:  0.9452341198921204\n",
            "Epoch:  7 , val_f1_score:  0.47999998927116394 , val_precision:  0.75 ,val_recall:  0.3529411852359772 , val_auc:  0.7882353067398071\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7027027027027027 current_acc:  0.6486486486486487\n",
            "F1 Score Mean:  0.5925925970077515 , F1 Score Std:  0.0 , Precision Mean:  0.800000011920929 ,Precision Std:  0.0\n",
            "Recall Mean:  0.47058823704719543 Recall Std:  0.0 AUC Mean:  0.7943507417191628 AUC Std:  0.07710158932688323\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "run_test('pet_ct')"
      ],
      "metadata": {
        "id": "dx2wnte__r2S"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "run_test('mixed')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gq7yT5oy_rv2",
        "outputId": "dc5d128f-54c8-476d-d2f1-15f37d82c66b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Data is loaded, training starting\n",
            "Fold [1/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.597491244366705 , train_acc:  0.7114093959731543 ,val_loss:  0.5184268156115553 , val_acc:  0.7631578947368421\n",
            "Epoch:  0 , train_f1_score:  0.5376344323158264 , train_precision:  0.5813953280448914 ,train_recall:  0.5 , train_auc:  0.6913130879402161\n",
            "Epoch:  0 , val_f1_score:  0.5263158082962036 , val_precision:  0.8333333134651184 ,val_recall:  0.38461539149284363 , val_auc:  0.7723076939582825\n",
            "Save Model, best_accuracy:  0.7631578947368421\n",
            "Epoch:  1 , train_loss:  0.5521226927693527 , train_acc:  0.7315436241610739 ,val_loss:  0.5417314867271145 , val_acc:  0.6578947368421053\n",
            "Epoch:  1 , train_f1_score:  0.5121951103210449 , train_precision:  0.65625 ,train_recall:  0.41999998688697815 , train_auc:  0.7276767492294312\n",
            "Epoch:  1 , val_f1_score:  0.6060606241226196 , val_precision:  0.5 ,val_recall:  0.7692307829856873 , val_auc:  0.8153846263885498\n",
            "Epoch:  2 , train_loss:  0.5073641656412378 , train_acc:  0.785234899328859 ,val_loss:  0.39912404754125563 , val_acc:  0.868421052631579\n",
            "Epoch:  2 , train_f1_score:  0.6097561120986938 , train_precision:  0.78125 ,train_recall:  0.5 , train_auc:  0.8012121319770813\n",
            "Epoch:  2 , val_f1_score:  0.782608687877655 , val_precision:  0.8999999761581421 ,val_recall:  0.692307710647583 , val_auc:  0.9138461351394653\n",
            "Save Model, best_accuracy:  0.868421052631579\n",
            "Epoch:  3 , train_loss:  0.49328923149732873 , train_acc:  0.785234899328859 ,val_loss:  0.39370973717029156 , val_acc:  0.8421052631578947\n",
            "Epoch:  3 , train_f1_score:  0.6190476417541504 , train_precision:  0.7647058963775635 ,train_recall:  0.5199999809265137 , train_auc:  0.8137373924255371\n",
            "Epoch:  3 , val_f1_score:  0.7272727489471436 , val_precision:  0.8888888955116272 ,val_recall:  0.6153846383094788 , val_auc:  0.926153838634491\n",
            "Epoch:  4 , train_loss:  0.4786008404721756 , train_acc:  0.7919463087248322 ,val_loss:  0.3906700481464596 , val_acc:  0.8421052631578947\n",
            "Epoch:  4 , train_f1_score:  0.6352941393852234 , train_precision:  0.7714285850524902 ,train_recall:  0.5400000214576721 , train_auc:  0.8258585929870605\n",
            "Epoch:  4 , val_f1_score:  0.7272727489471436 , val_precision:  0.8888888955116272 ,val_recall:  0.6153846383094788 , val_auc:  0.926153838634491\n",
            "Epoch:  5 , train_loss:  0.46018304595195975 , train_acc:  0.8053691275167785 ,val_loss:  0.3889950841028047 , val_acc:  0.868421052631579\n",
            "Epoch:  5 , train_f1_score:  0.6666666865348816 , train_precision:  0.7837837934494019 ,train_recall:  0.5799999833106995 , train_auc:  0.846666693687439\n",
            "Epoch:  5 , val_f1_score:  0.782608687877655 , val_precision:  0.8999999761581421 ,val_recall:  0.692307710647583 , val_auc:  0.9138461351394653\n",
            "Epoch:  6 , train_loss:  0.44193934710806115 , train_acc:  0.8322147651006712 ,val_loss:  0.3910386574111487 , val_acc:  0.8421052631578947\n",
            "Epoch:  6 , train_f1_score:  0.7311828136444092 , train_precision:  0.7906976938247681 ,train_recall:  0.6800000071525574 , train_auc:  0.8585858941078186\n",
            "Epoch:  6 , val_f1_score:  0.7272727489471436 , val_precision:  0.8888888955116272 ,val_recall:  0.6153846383094788 , val_auc:  0.9138461351394653\n",
            "Epoch:  7 , train_loss:  0.42979374926839253 , train_acc:  0.825503355704698 ,val_loss:  0.3926642613828574 , val_acc:  0.8421052631578947\n",
            "Epoch:  7 , train_f1_score:  0.7234042286872864 , train_precision:  0.7727272510528564 ,train_recall:  0.6800000071525574 , train_auc:  0.8676767349243164\n",
            "Epoch:  7 , val_f1_score:  0.7272727489471436 , val_precision:  0.8888888955116272 ,val_recall:  0.6153846383094788 , val_auc:  0.9138461351394653\n",
            "Early stopping in epoch:  7 best_accuracy:  0.868421052631579 current_acc:  0.8421052631578947\n",
            "Fold [2/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.7257067379135413 , train_acc:  0.6308724832214765 ,val_loss:  0.6480055929798829 , val_acc:  0.7631578947368421\n",
            "Epoch:  0 , train_f1_score:  0.09836065769195557 , train_precision:  0.3333333432674408 ,train_recall:  0.057692307978868484 , train_auc:  0.5255749225616455\n",
            "Epoch:  0 , val_f1_score:  0.4000000059604645 , val_precision:  0.75 ,val_recall:  0.27272728085517883 , val_auc:  0.5252525210380554\n",
            "Save Model, best_accuracy:  0.7631578947368421\n",
            "Epoch:  1 , train_loss:  0.6054631535638899 , train_acc:  0.7181208053691275 ,val_loss:  0.6295424708945287 , val_acc:  0.7105263157894737\n",
            "Epoch:  1 , train_f1_score:  0.4878048896789551 , train_precision:  0.6666666865348816 ,train_recall:  0.38461539149284363 , train_auc:  0.7075733542442322\n",
            "Epoch:  1 , val_f1_score:  0.4761904776096344 , val_precision:  0.5 ,val_recall:  0.4545454680919647 , val_auc:  0.7912458181381226\n",
            "Epoch:  2 , train_loss:  0.532451396490816 , train_acc:  0.7651006711409396 ,val_loss:  0.6254538584170836 , val_acc:  0.7368421052631579\n",
            "Epoch:  2 , train_f1_score:  0.5783132314682007 , train_precision:  0.774193525314331 ,train_recall:  0.4615384638309479 , train_auc:  0.7781522274017334\n",
            "Epoch:  2 , val_f1_score:  0.5 , val_precision:  0.5555555820465088 ,val_recall:  0.4545454680919647 , val_auc:  0.7744108438491821\n",
            "Epoch:  3 , train_loss:  0.4901791445281194 , train_acc:  0.7919463087248322 ,val_loss:  0.6297426165600835 , val_acc:  0.7368421052631579\n",
            "Epoch:  3 , train_f1_score:  0.6265060305595398 , train_precision:  0.8387096524238586 ,train_recall:  0.5 , train_auc:  0.7942109107971191\n",
            "Epoch:  3 , val_f1_score:  0.5 , val_precision:  0.5555555820465088 ,val_recall:  0.4545454680919647 , val_auc:  0.7777777910232544\n",
            "Epoch:  4 , train_loss:  0.45797731584655244 , train_acc:  0.8053691275167785 ,val_loss:  0.6244954914632457 , val_acc:  0.7105263157894737\n",
            "Epoch:  4 , train_f1_score:  0.6666666865348816 , train_precision:  0.8285714387893677 ,train_recall:  0.557692289352417 , train_auc:  0.8370341062545776\n",
            "Epoch:  4 , val_f1_score:  0.4761904776096344 , val_precision:  0.5 ,val_recall:  0.4545454680919647 , val_auc:  0.7845118641853333\n",
            "Epoch:  5 , train_loss:  0.44352076355904657 , train_acc:  0.8187919463087249 ,val_loss:  0.6309148745131815 , val_acc:  0.7105263157894737\n",
            "Epoch:  5 , train_f1_score:  0.6896551847457886 , train_precision:  0.8571428656578064 ,train_recall:  0.5769230723381042 , train_auc:  0.847343385219574\n",
            "Epoch:  5 , val_f1_score:  0.52173912525177 , val_precision:  0.5 ,val_recall:  0.5454545617103577 , val_auc:  0.7878788113594055\n",
            "Epoch:  6 , train_loss:  0.43920160523981433 , train_acc:  0.8120805369127517 ,val_loss:  0.6060886865817147 , val_acc:  0.7368421052631579\n",
            "Epoch:  6 , train_f1_score:  0.6744186282157898 , train_precision:  0.8529411554336548 ,train_recall:  0.557692289352417 , train_auc:  0.8578509092330933\n",
            "Epoch:  6 , val_f1_score:  0.5 , val_precision:  0.5555555820465088 ,val_recall:  0.4545454680919647 , val_auc:  0.7811447978019714\n",
            "Epoch:  7 , train_loss:  0.422479317490138 , train_acc:  0.8322147651006712 ,val_loss:  0.6023646390689297 , val_acc:  0.7368421052631579\n",
            "Epoch:  7 , train_f1_score:  0.7191011309623718 , train_precision:  0.8648648858070374 ,train_recall:  0.6153846383094788 , train_auc:  0.8707375526428223\n",
            "Epoch:  7 , val_f1_score:  0.5454545617103577 , val_precision:  0.5454545617103577 ,val_recall:  0.5454545617103577 , val_auc:  0.7845118045806885\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7631578947368421 current_acc:  0.7368421052631579\n",
            "Fold [3/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6196460309329753 , train_acc:  0.72 ,val_loss:  0.6329592097349264 , val_acc:  0.7837837837837838\n",
            "Epoch:  0 , train_f1_score:  0.5 , train_precision:  0.7241379022598267 ,train_recall:  0.38181817531585693 , train_auc:  0.7353110313415527\n",
            "Epoch:  0 , val_f1_score:  0.20000000298023224 , val_precision:  0.5 ,val_recall:  0.125 , val_auc:  0.5\n",
            "Save Model, best_accuracy:  0.7837837837837838\n",
            "Epoch:  1 , train_loss:  0.6183961108435566 , train_acc:  0.7133333333333334 ,val_loss:  0.5363232745109378 , val_acc:  0.7567567567567568\n",
            "Epoch:  1 , train_f1_score:  0.5274725556373596 , train_precision:  0.6666666865348816 ,train_recall:  0.4363636374473572 , train_auc:  0.7134928703308105\n",
            "Epoch:  1 , val_f1_score:  0.3076923191547394 , val_precision:  0.4000000059604645 ,val_recall:  0.25 , val_auc:  0.732758641242981\n",
            "Epoch:  2 , train_loss:  0.5196882811553466 , train_acc:  0.7733333333333333 ,val_loss:  0.5533797702676541 , val_acc:  0.7567567567567568\n",
            "Epoch:  2 , train_f1_score:  0.6222222447395325 , train_precision:  0.800000011920929 ,train_recall:  0.5090909004211426 , train_auc:  0.8133971691131592\n",
            "Epoch:  2 , val_f1_score:  0.4000000059604645 , val_precision:  0.4285714328289032 ,val_recall:  0.375 , val_auc:  0.7370690107345581\n",
            "Epoch:  3 , train_loss:  0.48661391007908 , train_acc:  0.8 ,val_loss:  0.5875778350022596 , val_acc:  0.7027027027027027\n",
            "Epoch:  3 , train_f1_score:  0.6875 , train_precision:  0.8048780560493469 ,train_recall:  0.6000000238418579 , train_auc:  0.8359808921813965\n",
            "Epoch:  3 , val_f1_score:  0.42105263471603394 , val_precision:  0.3636363744735718 ,val_recall:  0.5 , val_auc:  0.7155172824859619\n",
            "Epoch:  4 , train_loss:  0.4657673008883527 , train_acc:  0.8066666666666666 ,val_loss:  0.5463618371081916 , val_acc:  0.7837837837837838\n",
            "Epoch:  4 , train_f1_score:  0.7010309100151062 , train_precision:  0.8095238208770752 ,train_recall:  0.6181818246841431 , train_auc:  0.860287070274353\n",
            "Epoch:  4 , val_f1_score:  0.4285714328289032 , val_precision:  0.5 ,val_recall:  0.375 , val_auc:  0.7413793206214905\n",
            "Epoch:  5 , train_loss:  0.4538122217054479 , train_acc:  0.82 ,val_loss:  0.551715132692275 , val_acc:  0.8108108108108109\n",
            "Epoch:  5 , train_f1_score:  0.7216494679450989 , train_precision:  0.8333333134651184 ,train_recall:  0.6363636255264282 , train_auc:  0.8646889925003052\n",
            "Epoch:  5 , val_f1_score:  0.5882353186607361 , val_precision:  0.5555555820465088 ,val_recall:  0.625 , val_auc:  0.7543103694915771\n",
            "Save Model, best_accuracy:  0.8108108108108109\n",
            "Epoch:  6 , train_loss:  0.4357545165445966 , train_acc:  0.82 ,val_loss:  0.5835883748289701 , val_acc:  0.8108108108108109\n",
            "Epoch:  6 , train_f1_score:  0.7216494679450989 , train_precision:  0.8333333134651184 ,train_recall:  0.6363636255264282 , train_auc:  0.8777033090591431\n",
            "Epoch:  6 , val_f1_score:  0.5882353186607361 , val_precision:  0.5555555820465088 ,val_recall:  0.625 , val_auc:  0.7413792610168457\n",
            "Epoch:  7 , train_loss:  0.4227117019550254 , train_acc:  0.8333333333333334 ,val_loss:  0.5719857969340189 , val_acc:  0.8108108108108109\n",
            "Epoch:  7 , train_f1_score:  0.747474730014801 , train_precision:  0.8409090638160706 ,train_recall:  0.6727272868156433 , train_auc:  0.8819138407707214\n",
            "Epoch:  7 , val_f1_score:  0.5882353186607361 , val_precision:  0.5555555820465088 ,val_recall:  0.625 , val_auc:  0.7370690107345581\n",
            "Early stopping in epoch:  7 best_accuracy:  0.8108108108108109 current_acc:  0.8108108108108109\n",
            "Fold [4/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.7122671990593274 , train_acc:  0.6866666666666666 ,val_loss:  1.1130489080238182 , val_acc:  0.5405405405405406\n",
            "Epoch:  0 , train_f1_score:  0.145454540848732 , train_precision:  0.4000000059604645 ,train_recall:  0.08888889104127884 , train_auc:  0.5750264525413513\n",
            "Epoch:  0 , val_f1_score:  0.10526315867900848 , val_precision:  1.0 ,val_recall:  0.0555555559694767 , val_auc:  0.4736841917037964\n",
            "Save Model, best_accuracy:  0.5405405405405406\n",
            "Epoch:  1 , train_loss:  0.6062586426859101 , train_acc:  0.7266666666666667 ,val_loss:  0.6435295087662903 , val_acc:  0.5405405405405406\n",
            "Epoch:  1 , train_f1_score:  0.3692307770252228 , train_precision:  0.6000000238418579 ,train_recall:  0.2666666805744171 , train_auc:  0.6402116417884827\n",
            "Epoch:  1 , val_f1_score:  0.190476194024086 , val_precision:  0.6666666865348816 ,val_recall:  0.1111111119389534 , val_auc:  0.8771929740905762\n",
            "Epoch:  2 , train_loss:  0.5279628017389526 , train_acc:  0.78 ,val_loss:  0.6342962688288173 , val_acc:  0.6216216216216216\n",
            "Epoch:  2 , train_f1_score:  0.5074626803398132 , train_precision:  0.7727272510528564 ,train_recall:  0.3777777850627899 , train_auc:  0.7456084489822388\n",
            "Epoch:  2 , val_f1_score:  0.4166666567325592 , val_precision:  0.8333333134651184 ,val_recall:  0.2777777910232544 , val_auc:  0.8538011908531189\n",
            "Save Model, best_accuracy:  0.6216216216216216\n",
            "Epoch:  3 , train_loss:  0.5077699996655186 , train_acc:  0.78 ,val_loss:  0.6083204659155091 , val_acc:  0.6756756756756757\n",
            "Epoch:  3 , train_f1_score:  0.5074626803398132 , train_precision:  0.7727272510528564 ,train_recall:  0.3777777850627899 , train_auc:  0.754709005355835\n",
            "Epoch:  3 , val_f1_score:  0.5384615659713745 , val_precision:  0.875 ,val_recall:  0.3888888955116272 , val_auc:  0.8625730276107788\n",
            "Save Model, best_accuracy:  0.6756756756756757\n",
            "Epoch:  4 , train_loss:  0.4906066807700942 , train_acc:  0.8 ,val_loss:  0.5876637638319988 , val_acc:  0.7027027027027027\n",
            "Epoch:  4 , train_f1_score:  0.5454545617103577 , train_precision:  0.8571428656578064 ,train_recall:  0.4000000059604645 , train_auc:  0.7610582113265991\n",
            "Epoch:  4 , val_f1_score:  0.6206896305084229 , val_precision:  0.8181818127632141 ,val_recall:  0.5 , val_auc:  0.8830409049987793\n",
            "Save Model, best_accuracy:  0.7027027027027027\n",
            "Epoch:  5 , train_loss:  0.48628600446196896 , train_acc:  0.7933333333333333 ,val_loss:  0.5886155574305637 , val_acc:  0.7027027027027027\n",
            "Epoch:  5 , train_f1_score:  0.5507246255874634 , train_precision:  0.7916666865348816 ,train_recall:  0.42222222685813904 , train_auc:  0.7881481647491455\n",
            "Epoch:  5 , val_f1_score:  0.6206896305084229 , val_precision:  0.8181818127632141 ,val_recall:  0.5 , val_auc:  0.8625730276107788\n",
            "Epoch:  6 , train_loss:  0.4707104468718171 , train_acc:  0.8133333333333334 ,val_loss:  0.5710472438383747 , val_acc:  0.7027027027027027\n",
            "Epoch:  6 , train_f1_score:  0.5882353186607361 , train_precision:  0.8695651888847351 ,train_recall:  0.4444444477558136 , train_auc:  0.7881481647491455\n",
            "Epoch:  6 , val_f1_score:  0.6206896305084229 , val_precision:  0.8181818127632141 ,val_recall:  0.5 , val_auc:  0.8654971122741699\n",
            "Epoch:  7 , train_loss:  0.45861571868571144 , train_acc:  0.8133333333333334 ,val_loss:  0.5619887389947433 , val_acc:  0.7027027027027027\n",
            "Epoch:  7 , train_f1_score:  0.6000000238418579 , train_precision:  0.8399999737739563 ,train_recall:  0.46666666865348816 , train_auc:  0.807195782661438\n",
            "Epoch:  7 , val_f1_score:  0.6206896305084229 , val_precision:  0.8181818127632141 ,val_recall:  0.5 , val_auc:  0.8654969930648804\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7027027027027027 current_acc:  0.7027027027027027\n",
            "Fold [5/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6261784129754718 , train_acc:  0.68 ,val_loss:  0.7189224467084214 , val_acc:  0.7567567567567568\n",
            "Epoch:  0 , train_f1_score:  0.22580644488334656 , train_precision:  0.5833333134651184 ,train_recall:  0.14000000059604645 , train_auc:  0.6114000082015991\n",
            "Epoch:  0 , val_f1_score:  0.47058823704719543 , val_precision:  1.0 ,val_recall:  0.3076923191547394 , val_auc:  0.6314102411270142\n",
            "Save Model, best_accuracy:  0.7567567567567568\n",
            "Epoch:  1 , train_loss:  0.5818698923507084 , train_acc:  0.6933333333333334 ,val_loss:  0.5159502168362206 , val_acc:  0.8108108108108109\n",
            "Epoch:  1 , train_f1_score:  0.37837839126586914 , train_precision:  0.5833333134651184 ,train_recall:  0.2800000011920929 , train_auc:  0.6907999515533447\n",
            "Epoch:  1 , val_f1_score:  0.6315789222717285 , val_precision:  1.0 ,val_recall:  0.4615384638309479 , val_auc:  0.7628204822540283\n",
            "Save Model, best_accuracy:  0.8108108108108109\n",
            "Epoch:  2 , train_loss:  0.5210585212366035 , train_acc:  0.7466666666666667 ,val_loss:  0.5322911157398611 , val_acc:  0.8108108108108109\n",
            "Epoch:  2 , train_f1_score:  0.5 , train_precision:  0.7307692170143127 ,train_recall:  0.3799999952316284 , train_auc:  0.7879999876022339\n",
            "Epoch:  2 , val_f1_score:  0.6315789222717285 , val_precision:  1.0 ,val_recall:  0.4615384638309479 , val_auc:  0.7852563858032227\n",
            "Epoch:  3 , train_loss:  0.5027139979911347 , train_acc:  0.7666666666666667 ,val_loss:  0.545190349120546 , val_acc:  0.8108108108108109\n",
            "Epoch:  3 , train_f1_score:  0.5454545617103577 , train_precision:  0.7777777910232544 ,train_recall:  0.41999998688697815 , train_auc:  0.8076000213623047\n",
            "Epoch:  3 , val_f1_score:  0.6315789222717285 , val_precision:  1.0 ,val_recall:  0.4615384638309479 , val_auc:  0.7884615659713745\n",
            "Epoch:  4 , train_loss:  0.48388923482969404 , train_acc:  0.7666666666666667 ,val_loss:  0.5534567701051364 , val_acc:  0.8378378378378378\n",
            "Epoch:  4 , train_f1_score:  0.5679012537002563 , train_precision:  0.7419354915618896 ,train_recall:  0.46000000834465027 , train_auc:  0.8209999799728394\n",
            "Epoch:  4 , val_f1_score:  0.7272727489471436 , val_precision:  0.8888888955116272 ,val_recall:  0.6153846383094788 , val_auc:  0.807692289352417\n",
            "Save Model, best_accuracy:  0.8378378378378378\n",
            "Epoch:  5 , train_loss:  0.47464149330432215 , train_acc:  0.7933333333333333 ,val_loss:  0.5617069144909447 , val_acc:  0.8378378378378378\n",
            "Epoch:  5 , train_f1_score:  0.6265060305595398 , train_precision:  0.7878788113594055 ,train_recall:  0.5199999809265137 , train_auc:  0.828000009059906\n",
            "Epoch:  5 , val_f1_score:  0.7272727489471436 , val_precision:  0.8888888955116272 ,val_recall:  0.6153846383094788 , val_auc:  0.8044871687889099\n",
            "Epoch:  6 , train_loss:  0.46707072811201217 , train_acc:  0.8066666666666666 ,val_loss:  0.5722211174465515 , val_acc:  0.8648648648648649\n",
            "Epoch:  6 , train_f1_score:  0.6741573214530945 , train_precision:  0.7692307829856873 ,train_recall:  0.6000000238418579 , train_auc:  0.8331999778747559\n",
            "Epoch:  6 , val_f1_score:  0.761904776096344 , val_precision:  1.0 ,val_recall:  0.6153846383094788 , val_auc:  0.8108974695205688\n",
            "Save Model, best_accuracy:  0.8648648648648649\n",
            "Epoch:  7 , train_loss:  0.4529641808072726 , train_acc:  0.8 ,val_loss:  0.5706820290636372 , val_acc:  0.8378378378378378\n",
            "Epoch:  7 , train_f1_score:  0.6666666865348816 , train_precision:  0.75 ,train_recall:  0.6000000238418579 , train_auc:  0.847000002861023\n",
            "Epoch:  7 , val_f1_score:  0.7272727489471436 , val_precision:  0.8888888955116272 ,val_recall:  0.6153846383094788 , val_auc:  0.817307710647583\n",
            "Early stopping in epoch:  7 best_accuracy:  0.8648648648648649 current_acc:  0.8378378378378378\n",
            "F1 Score Mean:  0.761904776096344 , F1 Score Std:  0.0 , Precision Mean:  1.0 ,Precision Std:  0.0\n",
            "Recall Mean:  0.6153846383094788 Recall Std:  0.0 AUC Mean:  0.7704938020727494 AUC Std:  0.08041070393101744\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### CNN Test"
      ],
      "metadata": {
        "id": "_jFjxTA8_OWc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "device = torch.device('cuda')\n",
        "from torchmetrics import MetricCollection, Accuracy, Precision, Recall, F1Score\n",
        "from torchmetrics.classification import BinaryAUROC \n",
        "\n",
        "def training(train_loader, model, criterion, optimizer):\n",
        "  # Let's start by initializing our AverageMeters.\n",
        "  avg_meters = {'loss': AverageMeter(),\n",
        "                'acc': AverageMeter()}\n",
        "\n",
        "  metric_collection = MetricCollection({ #reference:https://pub.towardsai.net/improve-your-model-validation-with-torchmetrics-b457d3954dcd\n",
        "    'prec': Precision(task='binary', average='macro').to(device),\n",
        "    'rec': Recall(task='binary', average='macro').to(device),\n",
        "    'f1': F1Score(task='binary').to(device),\n",
        "    'auc': BinaryAUROC().to(device)})              \n",
        "\n",
        "    #load data\n",
        "  for data in train_loader:\n",
        "      \n",
        "    inputs, tabular_data, labels = data\n",
        "    inputs, labels = inputs.to(device), labels.to(device)\n",
        "\n",
        "    #zero gradients\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "    output = model(inputs).to(device)\n",
        "\n",
        "\n",
        "    #Metrics  \n",
        "    loss = criterion(output.float(), labels.float())\n",
        "\n",
        "    loss_value = loss.item()\n",
        "    total = labels.size(0)\n",
        "    correct_count = (output.round() == labels).sum().item()\n",
        "    accuracy = correct_count / total\n",
        "\n",
        "    avg_meters['loss'].update(loss_value, len(data))\n",
        "    avg_meters['acc'].update(accuracy, len(data))\n",
        "\n",
        "    metric_collection.update(output, labels)\n",
        "\n",
        "    #gradients and learning weights\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "  \n",
        "  val_metrics = metric_collection.compute()\n",
        "  metric_collection.reset()  \n",
        "\n",
        "  return dict([('loss', avg_meters['loss'].avg),\n",
        "                ('acc', avg_meters['acc'].avg),\n",
        "                ('f1', val_metrics['f1'].item()),\n",
        "                ('prec', val_metrics['prec'].item()),\n",
        "                ('rec', val_metrics['rec'].item()),\n",
        "                ('auc', val_metrics['auc'].item())])\n",
        "\n",
        "def validation(val_loader, model, criterion):\n",
        "  avg_meters = {'loss': AverageMeter(),\n",
        "                'acc': AverageMeter()}\n",
        "  metric_collection = MetricCollection({\n",
        "    'prec': Precision(task='binary', average='macro').to(device),\n",
        "    'rec': Recall(task='binary', average='macro').to(device),\n",
        "    'f1': F1Score(task='binary').to(device),\n",
        "    'auc': BinaryAUROC().to(device)})                \n",
        "\n",
        "  model.eval()\n",
        " \n",
        "  with torch.no_grad():\n",
        "     #load data\n",
        "    for data in val_loader:\n",
        "      \n",
        "      inputs, tabular_data, labels = data\n",
        "      inputs, labels = inputs.to(device), labels.to(device)\n",
        "\n",
        "      \n",
        "      output = model(inputs).to(device)\n",
        "\n",
        "      #Metrics  \n",
        "      loss = criterion(output.float(), labels.float())\n",
        "\n",
        "      loss_value = loss.item()\n",
        "      total = labels.size(0)\n",
        "      correct_count = (output.round() == labels).sum().item()\n",
        "      accuracy = correct_count / total\n",
        "\n",
        "      avg_meters['loss'].update(loss_value, len(data))\n",
        "      avg_meters['acc'].update(accuracy, len(data))\n",
        "\n",
        "      metric_collection.update(output, labels)\n",
        "\n",
        "  val_metrics = metric_collection.compute()\n",
        "  metric_collection.reset()\n",
        "\n",
        "  return dict([('loss', avg_meters['loss'].avg),\n",
        "                ('acc', avg_meters['acc'].avg),\n",
        "                ('f1', val_metrics['f1'].item()),\n",
        "                ('prec', val_metrics['prec'].item()),\n",
        "                ('rec', val_metrics['rec'].item()),\n",
        "                ('auc', val_metrics['auc'].item())])"
      ],
      "metadata": {
        "id": "NSKEDriv_QwZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from torch import optim\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "from torchvision.models import ResNet18_Weights, ResNet50_Weights\n",
        "from torch import nn\n",
        "import os\n",
        "from torch.utils.data import DataLoader\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import KFold\n",
        "from torch.utils.data import Subset\n",
        "\n",
        "def run_test(seg_type):\n",
        "  from torch.utils.data import DataLoader\n",
        "  import torch.optim as optim\n",
        "  import os\n",
        "  from google.colab import drive\n",
        "  profile_dir = '/content/drive/MyDrive/CMPE_runs/'\n",
        "  run_name = \"cmpe58p_project\"\n",
        "  full_dir = profile_dir + run_name\n",
        "\n",
        "  if os.path.exists(full_dir) == False:\n",
        "    os.makedirs(full_dir)\n",
        "\n",
        "  print(\"learning_rate: \", learning_rate, \", drop_rate: \", drop_rate, \", weight_decay: \", weight_decay, \", activation: \", activation, \", optimizer: \", optimizer_name)\n",
        "  \n",
        "  model = generate_model(50, 0.2).to(device)\n",
        "  model = torch.nn.DataParallel(model)\n",
        "\n",
        "  optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay = weight_decay)\n",
        "  scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min',factor=0.1, patience=2,min_lr=0.001)\n",
        "  criterion = nn.BCELoss()\n",
        "  trigger_times = 0\n",
        "\n",
        "  #dataloader dictionary\n",
        "  data_dict_train.partition = seg_type\n",
        "  # dataloader_full = DataLoader(data_dict_train, 1)\n",
        "  print(\"Data is loaded, training starting\")\n",
        "  kfold = KFold(n_splits=5, shuffle=True)\n",
        "  scores = []\n",
        "  f1s = []\n",
        "  precs = []\n",
        "  recs = []\n",
        "  for fold, (train, test) in enumerate(kfold.split(data_dict_train)):\n",
        "    print('Fold [%d/%d]' % (fold+1, 5))\n",
        "\n",
        "    # Create train and test subsets\n",
        "    train_subset = Subset(data_dict_train, train)\n",
        "    test_subset = Subset(data_dict_train, test)\n",
        "    train_loader = DataLoader(train_subset, 1)\n",
        "    test_loader = DataLoader(test_subset, 1)\n",
        "    # Reset the model for each fold\n",
        "    model = None\n",
        "    print(\"learning_rate: \", learning_rate, \", drop_rate: \", drop_rate, \", weight_decay: \", weight_decay, \", activation: \", activation, \", optimizer: \", optimizer_name)\n",
        "  \n",
        "    model = generate_model(50, 0.2).to(device)\n",
        "    model = torch.nn.DataParallel(model)\n",
        "\n",
        "    optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay = weight_decay)\n",
        "    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min',factor=0.1, patience=2,min_lr=0.001)\n",
        "    criterion = nn.BCELoss()\n",
        "    trigger_times = 0\n",
        "    best_accuracy = 0\n",
        "    f1 = 0\n",
        "    prec = 0\n",
        "    rec = 0\n",
        "    for e in range(initial_config[\"nb_epoch\"]):  # loop over the dataset multiple times    \n",
        "      train_dict = training(train_loader, model, criterion, optimizer)\n",
        "      val_dict = validation(test_loader, model, criterion)\n",
        "      print(\"Epoch: \", e, ', train_loss: ', train_dict['loss'], \", train_acc: \",  train_dict['acc'], ',val_loss: ', val_dict['loss'], \", val_acc: \",  val_dict['acc'])\n",
        "      print(\"Epoch: \", e, ', train_f1_score: ', train_dict['f1'], \", train_precision: \",  train_dict['prec'], ',train_recall: ', train_dict['rec'], \", train_auc: \",  train_dict['auc'])\n",
        "      print(\"Epoch: \", e, ', val_f1_score: ', val_dict['f1'], \", val_precision: \",  val_dict['prec'], ',val_recall: ', val_dict['rec'], \", val_auc: \",  val_dict['auc'])\n",
        "\n",
        "\n",
        "      scheduler.step(val_dict['loss'])\n",
        "      if(((val_dict['acc']) - 0.01) >= best_accuracy):\n",
        "          \n",
        "          rec = val_dict['rec']\n",
        "          prec = val_dict['prec']\n",
        "          f1 = val_dict['f1']\n",
        "          best_accuracy = (val_dict['acc'])\n",
        "          print(\"Save Model, best_accuracy: \", best_accuracy)\n",
        "          torch.save(model, os.path.join(run_dir, \"model.pt\"))\n",
        "      \n",
        "      if val_dict['acc'] > best_accuracy + 0.01:\n",
        "        trigger_times = 0  \n",
        "\n",
        "\n",
        "\n",
        "    \n",
        "      if(((val_dict['acc']) - 0.01) < best_accuracy):\n",
        "        trigger_times += 1\n",
        "        if trigger_times == patience:\n",
        "          print(\"Early stopping in epoch: \", e, \"best_accuracy: \", best_accuracy, \"current_acc: \", val_dict['acc'])\n",
        "          break\n",
        "      scores.append(best_accuracy)\n",
        "      f1s.append(f1)\n",
        "      precs.append(prec)\n",
        "      recs.append(rec)\n",
        "\n",
        "  load_model = torch.load(os.path.join(run_dir, \"model.pt\"))\n",
        "  # test_dict = validation(dataloader_val[], load_model, criterion)\n",
        "  # print(\"test_acc: \", test_dict['acc'], \"test_loss: \", test_dict['loss'])\n",
        "  mean_f1 = np.array(f1s).mean()\n",
        "  std_f1 = np.array(f1s).std()\n",
        "  mean_prec = np.array(precs).mean()\n",
        "  std_prec = np.array(precs).std()\n",
        "  mean_rec = np.array(recs).mean()\n",
        "  std_rec = np.array(recs).std()\n",
        "  mean_auc = np.array(scores).mean()\n",
        "  std_auc = np.array(scores).std()\n",
        "  print(\"F1 Score Mean: \", mean_f1, ', F1 Score Std: ', std_f1, \", Precision Mean: \",  mean_prec, ',Precision Std: ', std_prec)\n",
        "  print(\"Recall Mean: \", mean_rec, \"Recall Std: \", std_rec, \"AUC Mean: \", mean_auc, \"AUC Std: \", std_auc)"
      ],
      "metadata": {
        "id": "P94Kz-Bw_tq0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "run_test('ct_seg')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6JnWWn4n_tEf",
        "outputId": "68a78659-2c6a-46ee-a448-3e992f1494a7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Data is loaded, training starting\n",
            "Fold [1/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6968355592865272 , train_acc:  0.5704697986577181 ,val_loss:  0.6709546807565188 , val_acc:  0.7631578947368421\n",
            "Epoch:  0 , train_f1_score:  0.20000000298023224 , train_precision:  0.3076923191547394 ,train_recall:  0.14814814925193787 , train_auc:  0.446393758058548\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.26819923520088196\n",
            "Save Model, best_accuracy:  0.7631578947368421\n",
            "Epoch:  1 , train_loss:  0.7307991857496684 , train_acc:  0.6375838926174496 ,val_loss:  0.4965349327540025 , val_acc:  0.8421052631578947\n",
            "Epoch:  1 , train_f1_score:  0.3863636255264282 , train_precision:  0.5 ,train_recall:  0.31481480598449707 , train_auc:  0.5943470001220703\n",
            "Epoch:  1 , val_f1_score:  0.5714285969734192 , val_precision:  0.800000011920929 ,val_recall:  0.4444444477558136 , val_auc:  0.6858236789703369\n",
            "Save Model, best_accuracy:  0.8421052631578947\n",
            "Epoch:  2 , train_loss:  0.6605810750800891 , train_acc:  0.6442953020134228 ,val_loss:  0.4582883839274904 , val_acc:  0.868421052631579\n",
            "Epoch:  2 , train_f1_score:  0.34567901492118835 , train_precision:  0.5185185074806213 ,train_recall:  0.25925925374031067 , train_auc:  0.5980507135391235\n",
            "Epoch:  2 , val_f1_score:  0.6153846383094788 , val_precision:  1.0 ,val_recall:  0.4444444477558136 , val_auc:  0.7164750695228577\n",
            "Save Model, best_accuracy:  0.868421052631579\n",
            "Epoch:  3 , train_loss:  0.627457132575496 , train_acc:  0.6644295302013423 ,val_loss:  0.45242174930478396 , val_acc:  0.868421052631579\n",
            "Epoch:  3 , train_f1_score:  0.34210526943206787 , train_precision:  0.5909090638160706 ,train_recall:  0.24074074625968933 , train_auc:  0.6280701756477356\n",
            "Epoch:  3 , val_f1_score:  0.6153846383094788 , val_precision:  1.0 ,val_recall:  0.4444444477558136 , val_auc:  0.7471264600753784\n",
            "Epoch:  4 , train_loss:  0.6284977339658161 , train_acc:  0.6711409395973155 ,val_loss:  0.4539768243527138 , val_acc:  0.8421052631578947\n",
            "Epoch:  4 , train_f1_score:  0.37974682450294495 , train_precision:  0.6000000238418579 ,train_recall:  0.2777777910232544 , train_auc:  0.6263157725334167\n",
            "Epoch:  4 , val_f1_score:  0.5 , val_precision:  1.0 ,val_recall:  0.3333333432674408 , val_auc:  0.7011493444442749\n",
            "Epoch:  5 , train_loss:  0.605245491477267 , train_acc:  0.7046979865771812 ,val_loss:  0.4429477281427305 , val_acc:  0.8421052631578947\n",
            "Epoch:  5 , train_f1_score:  0.46341463923454285 , train_precision:  0.6785714030265808 ,train_recall:  0.35185185074806213 , train_auc:  0.6575048565864563\n",
            "Epoch:  5 , val_f1_score:  0.5 , val_precision:  1.0 ,val_recall:  0.3333333432674408 , val_auc:  0.7624521255493164\n",
            "Epoch:  6 , train_loss:  0.5954261333275361 , train_acc:  0.7046979865771812 ,val_loss:  0.4388552743598427 , val_acc:  0.8421052631578947\n",
            "Epoch:  6 , train_f1_score:  0.46341463923454285 , train_precision:  0.6785714030265808 ,train_recall:  0.35185185074806213 , train_auc:  0.6678363084793091\n",
            "Epoch:  6 , val_f1_score:  0.5 , val_precision:  1.0 ,val_recall:  0.3333333432674408 , val_auc:  0.7432950139045715\n",
            "Epoch:  7 , train_loss:  0.5855036138513144 , train_acc:  0.7248322147651006 ,val_loss:  0.4376486503568135 , val_acc:  0.8421052631578947\n",
            "Epoch:  7 , train_f1_score:  0.5176470875740051 , train_precision:  0.7096773982048035 ,train_recall:  0.40740740299224854 , train_auc:  0.6719298362731934\n",
            "Epoch:  7 , val_f1_score:  0.5 , val_precision:  1.0 ,val_recall:  0.3333333432674408 , val_auc:  0.7279693484306335\n",
            "Early stopping in epoch:  7 best_accuracy:  0.868421052631579 current_acc:  0.8421052631578947\n",
            "Fold [2/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.696674712152289 , train_acc:  0.5503355704697986 ,val_loss:  0.6884867011716491 , val_acc:  0.6578947368421053\n",
            "Epoch:  0 , train_f1_score:  0.24719101190567017 , train_precision:  0.28205129504203796 ,train_recall:  0.2199999988079071 , train_auc:  0.4765656590461731\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.4000000059604645\n",
            "Save Model, best_accuracy:  0.6578947368421053\n",
            "Epoch:  1 , train_loss:  0.7136148093830819 , train_acc:  0.6577181208053692 ,val_loss:  0.5637055297725295 , val_acc:  0.7105263157894737\n",
            "Epoch:  1 , train_f1_score:  0.3544303774833679 , train_precision:  0.48275861144065857 ,train_recall:  0.2800000011920929 , train_auc:  0.5816161632537842\n",
            "Epoch:  1 , val_f1_score:  0.42105263471603394 , val_precision:  0.6666666865348816 ,val_recall:  0.3076923191547394 , val_auc:  0.6984615325927734\n",
            "Save Model, best_accuracy:  0.7105263157894737\n",
            "Epoch:  2 , train_loss:  0.6085842759567219 , train_acc:  0.6912751677852349 ,val_loss:  0.5717545920296719 , val_acc:  0.6842105263157895\n",
            "Epoch:  2 , train_f1_score:  0.41025641560554504 , train_precision:  0.5714285969734192 ,train_recall:  0.3199999928474426 , train_auc:  0.6446464657783508\n",
            "Epoch:  2 , val_f1_score:  0.25 , val_precision:  0.6666666865348816 ,val_recall:  0.1538461595773697 , val_auc:  0.6984615325927734\n",
            "Epoch:  3 , train_loss:  0.6019894599714535 , train_acc:  0.7046979865771812 ,val_loss:  0.5926077431557994 , val_acc:  0.7105263157894737\n",
            "Epoch:  3 , train_f1_score:  0.42105263471603394 , train_precision:  0.6153846383094788 ,train_recall:  0.3199999928474426 , train_auc:  0.6505050659179688\n",
            "Epoch:  3 , val_f1_score:  0.2666666805744171 , val_precision:  1.0 ,val_recall:  0.1538461595773697 , val_auc:  0.6523076891899109\n",
            "Epoch:  4 , train_loss:  0.57566023311709 , train_acc:  0.738255033557047 ,val_loss:  0.6039395297044202 , val_acc:  0.7105263157894737\n",
            "Epoch:  4 , train_f1_score:  0.47999998927116394 , train_precision:  0.7200000286102295 ,train_recall:  0.36000001430511475 , train_auc:  0.6727272868156433\n",
            "Epoch:  4 , val_f1_score:  0.2666666805744171 , val_precision:  1.0 ,val_recall:  0.1538461595773697 , val_auc:  0.6861538887023926\n",
            "Epoch:  5 , train_loss:  0.5649336885235854 , train_acc:  0.7449664429530202 ,val_loss:  0.611575908762844 , val_acc:  0.6842105263157895\n",
            "Epoch:  5 , train_f1_score:  0.5 , train_precision:  0.7307692170143127 ,train_recall:  0.3799999952316284 , train_auc:  0.6761616468429565\n",
            "Epoch:  5 , val_f1_score:  0.1428571492433548 , val_precision:  1.0 ,val_recall:  0.07692307978868484 , val_auc:  0.7138462066650391\n",
            "Epoch:  6 , train_loss:  0.5439996926376484 , train_acc:  0.7718120805369127 ,val_loss:  0.6277385279536247 , val_acc:  0.6842105263157895\n",
            "Epoch:  6 , train_f1_score:  0.5526315569877625 , train_precision:  0.807692289352417 ,train_recall:  0.41999998688697815 , train_auc:  0.7012121677398682\n",
            "Epoch:  6 , val_f1_score:  0.1428571492433548 , val_precision:  1.0 ,val_recall:  0.07692307978868484 , val_auc:  0.6676923036575317\n",
            "Epoch:  7 , train_loss:  0.5281067779225191 , train_acc:  0.7785234899328859 ,val_loss:  0.6392636801067152 , val_acc:  0.6578947368421053\n",
            "Epoch:  7 , train_f1_score:  0.5714285969734192 , train_precision:  0.8148148059844971 ,train_recall:  0.4399999976158142 , train_auc:  0.7119191884994507\n",
            "Epoch:  7 , val_f1_score:  0.13333334028720856 , val_precision:  0.5 ,val_recall:  0.07692307978868484 , val_auc:  0.6646153926849365\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7105263157894737 current_acc:  0.6578947368421053\n",
            "Fold [3/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6785669507582982 , train_acc:  0.5866666666666667 ,val_loss:  0.6712717140043104 , val_acc:  0.6216216216216216\n",
            "Epoch:  0 , train_f1_score:  0.22499999403953552 , train_precision:  0.29032257199287415 ,train_recall:  0.18367347121238708 , train_auc:  0.4970701038837433\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.41925469040870667\n",
            "Save Model, best_accuracy:  0.6216216216216216\n",
            "Epoch:  1 , train_loss:  0.6867629204775828 , train_acc:  0.6466666666666666 ,val_loss:  0.6301892962105371 , val_acc:  0.7027027027027027\n",
            "Epoch:  1 , train_f1_score:  0.3291139304637909 , train_precision:  0.4333333373069763 ,train_recall:  0.26530611515045166 , train_auc:  0.6023439168930054\n",
            "Epoch:  1 , val_f1_score:  0.52173912525177 , val_precision:  0.6666666865348816 ,val_recall:  0.4285714328289032 , val_auc:  0.5559006333351135\n",
            "Save Model, best_accuracy:  0.7027027027027027\n",
            "Epoch:  2 , train_loss:  0.6345622062670494 , train_acc:  0.7 ,val_loss:  0.636839000759898 , val_acc:  0.7027027027027027\n",
            "Epoch:  2 , train_f1_score:  0.3478260934352875 , train_precision:  0.6000000238418579 ,train_recall:  0.2448979616165161 , train_auc:  0.5829460620880127\n",
            "Epoch:  2 , val_f1_score:  0.52173912525177 , val_precision:  0.6666666865348816 ,val_recall:  0.4285714328289032 , val_auc:  0.5559006333351135\n",
            "Epoch:  3 , train_loss:  0.6030361950824348 , train_acc:  0.74 ,val_loss:  0.6396330204364415 , val_acc:  0.7027027027027027\n",
            "Epoch:  3 , train_f1_score:  0.43478259444236755 , train_precision:  0.75 ,train_recall:  0.30612245202064514 , train_auc:  0.6269953846931458\n",
            "Epoch:  3 , val_f1_score:  0.4761904776096344 , val_precision:  0.7142857313156128 ,val_recall:  0.3571428656578064 , val_auc:  0.5527950525283813\n",
            "Epoch:  4 , train_loss:  0.5679025567350133 , train_acc:  0.7533333333333333 ,val_loss:  0.6467164805209314 , val_acc:  0.6486486486486487\n",
            "Epoch:  4 , train_f1_score:  0.47887325286865234 , train_precision:  0.7727272510528564 ,train_recall:  0.3469387888908386 , train_auc:  0.6613457202911377\n",
            "Epoch:  4 , val_f1_score:  0.47999998927116394 , val_precision:  0.5454545617103577 ,val_recall:  0.4285714328289032 , val_auc:  0.5496894717216492\n",
            "Epoch:  5 , train_loss:  0.5640216967845724 , train_acc:  0.76 ,val_loss:  0.6400837219647459 , val_acc:  0.6216216216216216\n",
            "Epoch:  5 , train_f1_score:  0.5 , train_precision:  0.782608687877655 ,train_recall:  0.36734694242477417 , train_auc:  0.6791270971298218\n",
            "Epoch:  5 , val_f1_score:  0.4615384638309479 , val_precision:  0.5 ,val_recall:  0.4285714328289032 , val_auc:  0.5559006333351135\n",
            "Epoch:  6 , train_loss:  0.5402659297346448 , train_acc:  0.7666666666666667 ,val_loss:  0.6627483083992391 , val_acc:  0.6216216216216216\n",
            "Epoch:  6 , train_f1_score:  0.5205479264259338 , train_precision:  0.7916666865348816 ,train_recall:  0.3877550959587097 , train_auc:  0.7126692533493042\n",
            "Epoch:  6 , val_f1_score:  0.4615384638309479 , val_precision:  0.5 ,val_recall:  0.4285714328289032 , val_auc:  0.5496894717216492\n",
            "Epoch:  7 , train_loss:  0.5177373234776315 , train_acc:  0.7733333333333333 ,val_loss:  0.6809349440642305 , val_acc:  0.5945945945945946\n",
            "Epoch:  7 , train_f1_score:  0.5405405163764954 , train_precision:  0.800000011920929 ,train_recall:  0.40816327929496765 , train_auc:  0.7532835006713867\n",
            "Epoch:  7 , val_f1_score:  0.4444444477558136 , val_precision:  0.4615384638309479 ,val_recall:  0.4285714328289032 , val_auc:  0.5465838313102722\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7027027027027027 current_acc:  0.5945945945945946\n",
            "Fold [4/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.701617912153403 , train_acc:  0.5533333333333333 ,val_loss:  0.6567739352986619 , val_acc:  0.7297297297297297\n",
            "Epoch:  0 , train_f1_score:  0.3366336524486542 , train_precision:  0.3541666567325592 ,train_recall:  0.3207547068595886 , train_auc:  0.4839525520801544\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.32962965965270996\n",
            "Save Model, best_accuracy:  0.7297297297297297\n",
            "Epoch:  1 , train_loss:  0.743914070489506 , train_acc:  0.6533333333333333 ,val_loss:  0.5723885599825833 , val_acc:  0.7567567567567568\n",
            "Epoch:  1 , train_f1_score:  0.380952388048172 , train_precision:  0.5161290168762207 ,train_recall:  0.30188679695129395 , train_auc:  0.60494065284729\n",
            "Epoch:  1 , val_f1_score:  0.47058823704719543 , val_precision:  0.5714285969734192 ,val_recall:  0.4000000059604645 , val_auc:  0.7259259223937988\n",
            "Save Model, best_accuracy:  0.7567567567567568\n",
            "Epoch:  2 , train_loss:  0.6337024506864448 , train_acc:  0.6666666666666666 ,val_loss:  0.524234414503381 , val_acc:  0.8108108108108109\n",
            "Epoch:  2 , train_f1_score:  0.3589743673801422 , train_precision:  0.5600000023841858 ,train_recall:  0.2641509473323822 , train_auc:  0.5956039428710938\n",
            "Epoch:  2 , val_f1_score:  0.5333333611488342 , val_precision:  0.800000011920929 ,val_recall:  0.4000000059604645 , val_auc:  0.7740740776062012\n",
            "Save Model, best_accuracy:  0.8108108108108109\n",
            "Epoch:  3 , train_loss:  0.6135726436724266 , train_acc:  0.6933333333333334 ,val_loss:  0.5254217563448725 , val_acc:  0.8108108108108109\n",
            "Epoch:  3 , train_f1_score:  0.3947368562221527 , train_precision:  0.6521739363670349 ,train_recall:  0.2830188572406769 , train_auc:  0.6162225008010864\n",
            "Epoch:  3 , val_f1_score:  0.5333333611488342 , val_precision:  0.800000011920929 ,val_recall:  0.4000000059604645 , val_auc:  0.7333333492279053\n",
            "Epoch:  4 , train_loss:  0.5986559626304855 , train_acc:  0.7133333333333334 ,val_loss:  0.5242751506534783 , val_acc:  0.7837837837837838\n",
            "Epoch:  4 , train_f1_score:  0.4691357910633087 , train_precision:  0.6785714030265808 ,train_recall:  0.35849055647850037 , train_auc:  0.638008177280426\n",
            "Epoch:  4 , val_f1_score:  0.4285714328289032 , val_precision:  0.75 ,val_recall:  0.30000001192092896 , val_auc:  0.7370370030403137\n",
            "Epoch:  5 , train_loss:  0.5826590429153293 , train_acc:  0.7266666666666667 ,val_loss:  0.5185456292049305 , val_acc:  0.7837837837837838\n",
            "Epoch:  5 , train_f1_score:  0.5060241222381592 , train_precision:  0.699999988079071 ,train_recall:  0.3962264060974121 , train_auc:  0.6496790647506714\n",
            "Epoch:  5 , val_f1_score:  0.4285714328289032 , val_precision:  0.75 ,val_recall:  0.30000001192092896 , val_auc:  0.7444444298744202\n",
            "Epoch:  6 , train_loss:  0.5654689987003804 , train_acc:  0.7533333333333333 ,val_loss:  0.50746958884033 , val_acc:  0.8108108108108109\n",
            "Epoch:  6 , train_f1_score:  0.5647059082984924 , train_precision:  0.75 ,train_recall:  0.4528301954269409 , train_auc:  0.6765221357345581\n",
            "Epoch:  6 , val_f1_score:  0.5333333611488342 , val_precision:  0.800000011920929 ,val_recall:  0.4000000059604645 , val_auc:  0.8111110925674438\n",
            "Epoch:  7 , train_loss:  0.5565267031112064 , train_acc:  0.7533333333333333 ,val_loss:  0.5063075113135416 , val_acc:  0.7837837837837838\n",
            "Epoch:  7 , train_f1_score:  0.5647059082984924 , train_precision:  0.75 ,train_recall:  0.4528301954269409 , train_auc:  0.6843026876449585\n",
            "Epoch:  7 , val_f1_score:  0.4285714328289032 , val_precision:  0.75 ,val_recall:  0.30000001192092896 , val_auc:  0.7814815044403076\n",
            "Early stopping in epoch:  7 best_accuracy:  0.8108108108108109 current_acc:  0.7837837837837838\n",
            "Fold [5/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6314131314555804 , train_acc:  0.6933333333333334 ,val_loss:  0.671449713610314 , val_acc:  0.5675675675675675\n",
            "Epoch:  0 , train_f1_score:  0.11538461595773697 , train_precision:  0.5 ,train_recall:  0.06521739065647125 , train_auc:  0.4770066738128662\n",
            "Epoch:  0 , val_f1_score:  0.3333333432674408 , val_precision:  0.5714285969734192 ,val_recall:  0.23529411852359772 , val_auc:  0.6882352828979492\n",
            "Save Model, best_accuracy:  0.5675675675675675\n",
            "Epoch:  1 , train_loss:  0.7591991175152362 , train_acc:  0.6666666666666666 ,val_loss:  0.6430571610863144 , val_acc:  0.6216216216216216\n",
            "Epoch:  1 , train_f1_score:  0.3055555522441864 , train_precision:  0.42307692766189575 ,train_recall:  0.239130437374115 , train_auc:  0.5687708854675293\n",
            "Epoch:  1 , val_f1_score:  0.3636363744735718 , val_precision:  0.800000011920929 ,val_recall:  0.23529411852359772 , val_auc:  0.6852940917015076\n",
            "Save Model, best_accuracy:  0.6216216216216216\n",
            "Epoch:  2 , train_loss:  0.6130369828393062 , train_acc:  0.7133333333333334 ,val_loss:  0.6461212038993835 , val_acc:  0.6216216216216216\n",
            "Epoch:  2 , train_f1_score:  0.3174603283405304 , train_precision:  0.5882353186607361 ,train_recall:  0.21739129722118378 , train_auc:  0.5974080562591553\n",
            "Epoch:  2 , val_f1_score:  0.3636363744735718 , val_precision:  0.800000011920929 ,val_recall:  0.23529411852359772 , val_auc:  0.7911764979362488\n",
            "Epoch:  3 , train_loss:  0.5998307343324025 , train_acc:  0.7266666666666667 ,val_loss:  0.6832920385373605 , val_acc:  0.6216216216216216\n",
            "Epoch:  3 , train_f1_score:  0.3692307770252228 , train_precision:  0.6315789222717285 ,train_recall:  0.260869562625885 , train_auc:  0.5978260636329651\n",
            "Epoch:  3 , val_f1_score:  0.3636363744735718 , val_precision:  0.800000011920929 ,val_recall:  0.23529411852359772 , val_auc:  0.7676470875740051\n",
            "Epoch:  4 , train_loss:  0.5815527984468887 , train_acc:  0.7466666666666667 ,val_loss:  0.6864082040013494 , val_acc:  0.6486486486486487\n",
            "Epoch:  4 , train_f1_score:  0.44117647409439087 , train_precision:  0.6818181872367859 ,train_recall:  0.32608696818351746 , train_auc:  0.6017976403236389\n",
            "Epoch:  4 , val_f1_score:  0.380952388048172 , val_precision:  1.0 ,val_recall:  0.23529411852359772 , val_auc:  0.75\n",
            "Save Model, best_accuracy:  0.6486486486486487\n",
            "Epoch:  5 , train_loss:  0.570691306645361 , train_acc:  0.74 ,val_loss:  0.6913118169114396 , val_acc:  0.6486486486486487\n",
            "Epoch:  5 , train_f1_score:  0.41791045665740967 , train_precision:  0.6666666865348816 ,train_recall:  0.30434781312942505 , train_auc:  0.6091136932373047\n",
            "Epoch:  5 , val_f1_score:  0.380952388048172 , val_precision:  1.0 ,val_recall:  0.23529411852359772 , val_auc:  0.7441176176071167\n",
            "Epoch:  6 , train_loss:  0.561874762836378 , train_acc:  0.7466666666666667 ,val_loss:  0.6771236887654742 , val_acc:  0.6216216216216216\n",
            "Epoch:  6 , train_f1_score:  0.44117647409439087 , train_precision:  0.6818181872367859 ,train_recall:  0.32608696818351746 , train_auc:  0.625\n",
            "Epoch:  6 , val_f1_score:  0.3636363744735718 , val_precision:  0.800000011920929 ,val_recall:  0.23529411852359772 , val_auc:  0.779411792755127\n",
            "Epoch:  7 , train_loss:  0.5558220495640611 , train_acc:  0.7466666666666667 ,val_loss:  0.6876927481309788 , val_acc:  0.6216216216216216\n",
            "Epoch:  7 , train_f1_score:  0.44117647409439087 , train_precision:  0.6818181872367859 ,train_recall:  0.32608696818351746 , train_auc:  0.637123703956604\n",
            "Epoch:  7 , val_f1_score:  0.3636363744735718 , val_precision:  0.800000011920929 ,val_recall:  0.23529411852359772 , val_auc:  0.7617647051811218\n",
            "Early stopping in epoch:  7 best_accuracy:  0.6486486486486487 current_acc:  0.6216216216216216\n",
            "F1 Score Mean:  0.380952388048172 , F1 Score Std:  0.0 , Precision Mean:  1.0 ,Precision Std:  0.0\n",
            "Recall Mean:  0.23529411852359772 Recall Std:  0.0 AUC Mean:  0.7321479374110953 AUC Std:  0.0841499875761339\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "run_test('ct_mult_seg')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tFdRMrsFGqAL",
        "outputId": "84c8f8ad-d9bb-435e-9694-ed3090547b38"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Data is loaded, training starting\n",
            "Fold [1/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6569252082165455 , train_acc:  0.6644295302013423 ,val_loss:  0.8412543899918857 , val_acc:  0.6052631578947368\n",
            "Epoch:  0 , train_f1_score:  0.0 , train_precision:  0.0 ,train_recall:  0.0 , train_auc:  0.40820956230163574\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.3246377110481262\n",
            "Save Model, best_accuracy:  0.6052631578947368\n",
            "Epoch:  1 , train_loss:  0.7367859329815779 , train_acc:  0.6577181208053692 ,val_loss:  0.6971753746628039 , val_acc:  0.631578947368421\n",
            "Epoch:  1 , train_f1_score:  0.3199999928474426 , train_precision:  0.4444444477558136 ,train_recall:  0.25 , train_auc:  0.5693069696426392\n",
            "Epoch:  1 , val_f1_score:  0.5 , val_precision:  0.5384615659713745 ,val_recall:  0.46666666865348816 , val_auc:  0.6637681722640991\n",
            "Save Model, best_accuracy:  0.631578947368421\n",
            "Epoch:  2 , train_loss:  0.6381914410615127 , train_acc:  0.7181208053691275 ,val_loss:  0.6818430869683844 , val_acc:  0.6052631578947368\n",
            "Epoch:  2 , train_f1_score:  0.38235294818878174 , train_precision:  0.6499999761581421 ,train_recall:  0.2708333432674408 , train_auc:  0.5759075880050659\n",
            "Epoch:  2 , val_f1_score:  0.4444444477558136 , val_precision:  0.5 ,val_recall:  0.4000000059604645 , val_auc:  0.6115942001342773\n",
            "Epoch:  3 , train_loss:  0.5892261359715621 , train_acc:  0.7315436241610739 ,val_loss:  0.6758475251645952 , val_acc:  0.5789473684210527\n",
            "Epoch:  3 , train_f1_score:  0.4117647111415863 , train_precision:  0.699999988079071 ,train_recall:  0.2916666567325592 , train_auc:  0.6000412702560425\n",
            "Epoch:  3 , val_f1_score:  0.38461539149284363 , val_precision:  0.4545454680919647 ,val_recall:  0.3333333432674408 , val_auc:  0.6202898621559143\n",
            "Epoch:  4 , train_loss:  0.5766527127199526 , train_acc:  0.738255033557047 ,val_loss:  0.6764346902307711 , val_acc:  0.5789473684210527\n",
            "Epoch:  4 , train_f1_score:  0.41791045665740967 , train_precision:  0.7368420958518982 ,train_recall:  0.2916666567325592 , train_auc:  0.6155115365982056\n",
            "Epoch:  4 , val_f1_score:  0.38461539149284363 , val_precision:  0.4545454680919647 ,val_recall:  0.3333333432674408 , val_auc:  0.6115942001342773\n",
            "Epoch:  5 , train_loss:  0.565446766243385 , train_acc:  0.7651006711409396 ,val_loss:  0.6936190653314521 , val_acc:  0.5789473684210527\n",
            "Epoch:  5 , train_f1_score:  0.49275362491607666 , train_precision:  0.8095238208770752 ,train_recall:  0.3541666567325592 , train_auc:  0.628506600856781\n",
            "Epoch:  5 , val_f1_score:  0.38461539149284363 , val_precision:  0.4545454680919647 ,val_recall:  0.3333333432674408 , val_auc:  0.6202899217605591\n",
            "Epoch:  6 , train_loss:  0.5748589151232635 , train_acc:  0.7583892617449665 ,val_loss:  0.6902023381083525 , val_acc:  0.6052631578947368\n",
            "Epoch:  6 , train_f1_score:  0.47058823704719543 , train_precision:  0.800000011920929 ,train_recall:  0.3333333432674408 , train_auc:  0.625\n",
            "Epoch:  6 , val_f1_score:  0.4444444477558136 , val_precision:  0.5 ,val_recall:  0.4000000059604645 , val_auc:  0.626086950302124\n",
            "Epoch:  7 , train_loss:  0.5452770718542121 , train_acc:  0.7651006711409396 ,val_loss:  0.6903872938686809 , val_acc:  0.631578947368421\n",
            "Epoch:  7 , train_f1_score:  0.49275362491607666 , train_precision:  0.8095238208770752 ,train_recall:  0.3541666567325592 , train_auc:  0.6474834680557251\n",
            "Epoch:  7 , val_f1_score:  0.4615384638309479 , val_precision:  0.5454545617103577 ,val_recall:  0.4000000059604645 , val_auc:  0.6289855241775513\n",
            "Early stopping in epoch:  7 best_accuracy:  0.631578947368421 current_acc:  0.631578947368421\n",
            "Fold [2/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6569493769999318 , train_acc:  0.6442953020134228 ,val_loss:  0.6707280113508827 , val_acc:  0.6052631578947368\n",
            "Epoch:  0 , train_f1_score:  0.10169491171836853 , train_precision:  0.27272728085517883 ,train_recall:  0.0625 , train_auc:  0.481435626745224\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.5855072736740112\n",
            "Save Model, best_accuracy:  0.6052631578947368\n",
            "Epoch:  1 , train_loss:  0.7359091064723946 , train_acc:  0.6711409395973155 ,val_loss:  0.6238262398462546 , val_acc:  0.631578947368421\n",
            "Epoch:  1 , train_f1_score:  0.28985506296157837 , train_precision:  0.4761904776096344 ,train_recall:  0.2083333283662796 , train_auc:  0.5371286869049072\n",
            "Epoch:  1 , val_f1_score:  0.3636363744735718 , val_precision:  0.5714285969734192 ,val_recall:  0.2666666805744171 , val_auc:  0.5768116116523743\n",
            "Save Model, best_accuracy:  0.631578947368421\n",
            "Epoch:  2 , train_loss:  0.597537007432856 , train_acc:  0.7181208053691275 ,val_loss:  0.6284608025299875 , val_acc:  0.7105263157894737\n",
            "Epoch:  2 , train_f1_score:  0.38235294818878174 , train_precision:  0.6499999761581421 ,train_recall:  0.2708333432674408 , train_auc:  0.5924092531204224\n",
            "Epoch:  2 , val_f1_score:  0.42105263471603394 , val_precision:  1.0 ,val_recall:  0.2666666805744171 , val_auc:  0.5594202876091003\n",
            "Save Model, best_accuracy:  0.7105263157894737\n",
            "Epoch:  3 , train_loss:  0.5786010909375529 , train_acc:  0.738255033557047 ,val_loss:  0.6298937005432028 , val_acc:  0.6842105263157895\n",
            "Epoch:  3 , train_f1_score:  0.41791045665740967 , train_precision:  0.7368420958518982 ,train_recall:  0.2916666567325592 , train_auc:  0.6190181374549866\n",
            "Epoch:  3 , val_f1_score:  0.3333333432674408 , val_precision:  1.0 ,val_recall:  0.20000000298023224 , val_auc:  0.582608699798584\n",
            "Epoch:  4 , train_loss:  0.5622602541174665 , train_acc:  0.7516778523489933 ,val_loss:  0.6277491622849515 , val_acc:  0.7105263157894737\n",
            "Epoch:  4 , train_f1_score:  0.4637681245803833 , train_precision:  0.761904776096344 ,train_recall:  0.3333333432674408 , train_auc:  0.646864652633667\n",
            "Epoch:  4 , val_f1_score:  0.42105263471603394 , val_precision:  1.0 ,val_recall:  0.2666666805744171 , val_auc:  0.591304361820221\n",
            "Epoch:  5 , train_loss:  0.5484611689419354 , train_acc:  0.7516778523489933 ,val_loss:  0.6299941300561553 , val_acc:  0.6578947368421053\n",
            "Epoch:  5 , train_f1_score:  0.4637681245803833 , train_precision:  0.761904776096344 ,train_recall:  0.3333333432674408 , train_auc:  0.667904257774353\n",
            "Epoch:  5 , val_f1_score:  0.31578946113586426 , val_precision:  0.75 ,val_recall:  0.20000000298023224 , val_auc:  0.582608699798584\n",
            "Epoch:  6 , train_loss:  0.5318842278418425 , train_acc:  0.7516778523489933 ,val_loss:  0.6264992611188638 , val_acc:  0.6578947368421053\n",
            "Epoch:  6 , train_f1_score:  0.4637681245803833 , train_precision:  0.761904776096344 ,train_recall:  0.3333333432674408 , train_auc:  0.6918317079544067\n",
            "Epoch:  6 , val_f1_score:  0.380952388048172 , val_precision:  0.6666666865348816 ,val_recall:  0.2666666805744171 , val_auc:  0.5855072736740112\n",
            "Epoch:  7 , train_loss:  0.5018738604867761 , train_acc:  0.785234899328859 ,val_loss:  0.6348071808093473 , val_acc:  0.631578947368421\n",
            "Epoch:  7 , train_f1_score:  0.529411792755127 , train_precision:  0.8999999761581421 ,train_recall:  0.375 , train_auc:  0.7285478115081787\n",
            "Epoch:  7 , val_f1_score:  0.30000001192092896 , val_precision:  0.6000000238418579 ,val_recall:  0.20000000298023224 , val_auc:  0.6000000238418579\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7105263157894737 current_acc:  0.631578947368421\n",
            "Fold [3/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.7360562871893247 , train_acc:  0.49333333333333335 ,val_loss:  0.5877635003747167 , val_acc:  0.8378378378378378\n",
            "Epoch:  0 , train_f1_score:  0.3214285671710968 , train_precision:  0.3272727131843567 ,train_recall:  0.31578946113586426 , train_auc:  0.4555744528770447\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.39247313141822815\n",
            "Save Model, best_accuracy:  0.8378378378378378\n",
            "Epoch:  1 , train_loss:  0.818017704715797 , train_acc:  0.6133333333333333 ,val_loss:  0.4935470271754909 , val_acc:  0.8648648648648649\n",
            "Epoch:  1 , train_f1_score:  0.3695652186870575 , train_precision:  0.48571428656578064 ,train_recall:  0.2982456088066101 , train_auc:  0.5465006828308105\n",
            "Epoch:  1 , val_f1_score:  0.4444444477558136 , val_precision:  0.6666666865348816 ,val_recall:  0.3333333432674408 , val_auc:  0.7688171863555908\n",
            "Save Model, best_accuracy:  0.8648648648648649\n",
            "Epoch:  2 , train_loss:  0.6462280707309643 , train_acc:  0.6666666666666666 ,val_loss:  0.4949260144620328 , val_acc:  0.8378378378378378\n",
            "Epoch:  2 , train_f1_score:  0.375 , train_precision:  0.6521739363670349 ,train_recall:  0.2631579041481018 , train_auc:  0.5427278280258179\n",
            "Epoch:  2 , val_f1_score:  0.25 , val_precision:  0.5 ,val_recall:  0.1666666716337204 , val_auc:  0.8118279576301575\n",
            "Epoch:  3 , train_loss:  0.6315985803554456 , train_acc:  0.68 ,val_loss:  0.49271651619189494 , val_acc:  0.8378378378378378\n",
            "Epoch:  3 , train_f1_score:  0.38461539149284363 , train_precision:  0.7142857313156128 ,train_recall:  0.2631579041481018 , train_auc:  0.5695151686668396\n",
            "Epoch:  3 , val_f1_score:  0.25 , val_precision:  0.5 ,val_recall:  0.1666666716337204 , val_auc:  0.8440860509872437\n",
            "Epoch:  4 , train_loss:  0.6221357617899775 , train_acc:  0.6933333333333334 ,val_loss:  0.4890804467974482 , val_acc:  0.8378378378378378\n",
            "Epoch:  4 , train_f1_score:  0.41025641560554504 , train_precision:  0.761904776096344 ,train_recall:  0.28070175647735596 , train_auc:  0.5863044857978821\n",
            "Epoch:  4 , val_f1_score:  0.25 , val_precision:  0.5 ,val_recall:  0.1666666716337204 , val_auc:  0.827957034111023\n",
            "Epoch:  5 , train_loss:  0.6120197698846459 , train_acc:  0.7066666666666667 ,val_loss:  0.495035413954709 , val_acc:  0.8378378378378378\n",
            "Epoch:  5 , train_f1_score:  0.43589743971824646 , train_precision:  0.8095238208770752 ,train_recall:  0.2982456088066101 , train_auc:  0.5868704319000244\n",
            "Epoch:  5 , val_f1_score:  0.25 , val_precision:  0.5 ,val_recall:  0.1666666716337204 , val_auc:  0.8225806355476379\n",
            "Epoch:  6 , train_loss:  0.594016095213592 , train_acc:  0.7133333333333334 ,val_loss:  0.4897473994139078 , val_acc:  0.8378378378378378\n",
            "Epoch:  6 , train_f1_score:  0.4556961953639984 , train_precision:  0.8181818127632141 ,train_recall:  0.31578946113586426 , train_auc:  0.622146725654602\n",
            "Epoch:  6 , val_f1_score:  0.25 , val_precision:  0.5 ,val_recall:  0.1666666716337204 , val_auc:  0.801075279712677\n",
            "Epoch:  7 , train_loss:  0.5879466771831116 , train_acc:  0.72 ,val_loss:  0.490537936623032 , val_acc:  0.8378378378378378\n",
            "Epoch:  7 , train_f1_score:  0.4615384638309479 , train_precision:  0.8571428656578064 ,train_recall:  0.31578946113586426 , train_auc:  0.626674234867096\n",
            "Epoch:  7 , val_f1_score:  0.25 , val_precision:  0.5 ,val_recall:  0.1666666716337204 , val_auc:  0.7419354915618896\n",
            "Early stopping in epoch:  7 best_accuracy:  0.8648648648648649 current_acc:  0.8378378378378378\n",
            "Fold [4/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6772830158472061 , train_acc:  0.62 ,val_loss:  0.7869436313977113 , val_acc:  0.6756756756756757\n",
            "Epoch:  0 , train_f1_score:  0.17391304671764374 , train_precision:  0.3333333432674408 ,train_recall:  0.11764705926179886 , train_auc:  0.4375123977661133\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.4266666769981384\n",
            "Save Model, best_accuracy:  0.6756756756756757\n",
            "Epoch:  1 , train_loss:  0.8270723985570172 , train_acc:  0.6333333333333333 ,val_loss:  0.583847272234994 , val_acc:  0.7297297297297297\n",
            "Epoch:  1 , train_f1_score:  0.3529411852359772 , train_precision:  0.44117647409439087 ,train_recall:  0.29411765933036804 , train_auc:  0.5654585361480713\n",
            "Epoch:  1 , val_f1_score:  0.2857142984867096 , val_precision:  1.0 ,val_recall:  0.1666666716337204 , val_auc:  0.5699999332427979\n",
            "Save Model, best_accuracy:  0.7297297297297297\n",
            "Epoch:  2 , train_loss:  0.6216888653238615 , train_acc:  0.7 ,val_loss:  0.5799232548153078 , val_acc:  0.7297297297297297\n",
            "Epoch:  2 , train_f1_score:  0.3661971688270569 , train_precision:  0.6499999761581421 ,train_recall:  0.2549019753932953 , train_auc:  0.5836799144744873\n",
            "Epoch:  2 , val_f1_score:  0.2857142984867096 , val_precision:  1.0 ,val_recall:  0.1666666716337204 , val_auc:  0.5733333826065063\n",
            "Epoch:  3 , train_loss:  0.6081391152491172 , train_acc:  0.7133333333333334 ,val_loss:  0.5810060400415111 , val_acc:  0.7027027027027027\n",
            "Epoch:  3 , train_f1_score:  0.39436620473861694 , train_precision:  0.699999988079071 ,train_recall:  0.27450981736183167 , train_auc:  0.6080412268638611\n",
            "Epoch:  3 , val_f1_score:  0.1538461595773697 , val_precision:  1.0 ,val_recall:  0.0833333358168602 , val_auc:  0.5799999833106995\n",
            "Epoch:  4 , train_loss:  0.5998143958548705 , train_acc:  0.7266666666666667 ,val_loss:  0.5824500712993983 , val_acc:  0.7027027027027027\n",
            "Epoch:  4 , train_f1_score:  0.4383561611175537 , train_precision:  0.7272727489471436 ,train_recall:  0.3137255012989044 , train_auc:  0.6191325187683105\n",
            "Epoch:  4 , val_f1_score:  0.1538461595773697 , val_precision:  1.0 ,val_recall:  0.0833333358168602 , val_auc:  0.6466666460037231\n",
            "Epoch:  5 , train_loss:  0.5893579656630754 , train_acc:  0.7266666666666667 ,val_loss:  0.5790350912390528 , val_acc:  0.7027027027027027\n",
            "Epoch:  5 , train_f1_score:  0.4383561611175537 , train_precision:  0.7272727489471436 ,train_recall:  0.3137255012989044 , train_auc:  0.6276490688323975\n",
            "Epoch:  5 , val_f1_score:  0.1538461595773697 , val_precision:  1.0 ,val_recall:  0.0833333358168602 , val_auc:  0.6200000047683716\n",
            "Epoch:  6 , train_loss:  0.5794417110209664 , train_acc:  0.7266666666666667 ,val_loss:  0.5798621330712292 , val_acc:  0.7027027027027027\n",
            "Epoch:  6 , train_f1_score:  0.4383561611175537 , train_precision:  0.7272727489471436 ,train_recall:  0.3137255012989044 , train_auc:  0.6430976390838623\n",
            "Epoch:  6 , val_f1_score:  0.1538461595773697 , val_precision:  1.0 ,val_recall:  0.0833333358168602 , val_auc:  0.6266666650772095\n",
            "Epoch:  7 , train_loss:  0.5672607704624534 , train_acc:  0.7266666666666667 ,val_loss:  0.5805298484660484 , val_acc:  0.7027027027027027\n",
            "Epoch:  7 , train_f1_score:  0.4383561611175537 , train_precision:  0.7272727489471436 ,train_recall:  0.3137255012989044 , train_auc:  0.6575559377670288\n",
            "Epoch:  7 , val_f1_score:  0.1538461595773697 , val_precision:  1.0 ,val_recall:  0.0833333358168602 , val_auc:  0.6700000166893005\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7297297297297297 current_acc:  0.7027027027027027\n",
            "Fold [5/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6093288290500641 , train_acc:  0.68 ,val_loss:  0.7239642630557757 , val_acc:  0.5945945945945946\n",
            "Epoch:  0 , train_f1_score:  0.03999999910593033 , train_precision:  0.5 ,train_recall:  0.02083333395421505 , train_auc:  0.6390931606292725\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.3242424428462982\n",
            "Save Model, best_accuracy:  0.5945945945945946\n",
            "Epoch:  1 , train_loss:  0.7279209587536752 , train_acc:  0.6533333333333333 ,val_loss:  0.6087821411925394 , val_acc:  0.6756756756756757\n",
            "Epoch:  1 , train_f1_score:  0.2777777910232544 , train_precision:  0.4166666567325592 ,train_recall:  0.2083333283662796 , train_auc:  0.5089869499206543\n",
            "Epoch:  1 , val_f1_score:  0.4545454680919647 , val_precision:  0.7142857313156128 ,val_recall:  0.3333333432674408 , val_auc:  0.6424242258071899\n",
            "Save Model, best_accuracy:  0.6756756756756757\n",
            "Epoch:  2 , train_loss:  0.6254263694491238 , train_acc:  0.68 ,val_loss:  0.6154037645539722 , val_acc:  0.7027027027027027\n",
            "Epoch:  2 , train_f1_score:  0.27272728085517883 , train_precision:  0.5 ,train_recall:  0.1875 , train_auc:  0.5553512573242188\n",
            "Epoch:  2 , val_f1_score:  0.42105263471603394 , val_precision:  1.0 ,val_recall:  0.2666666805744171 , val_auc:  0.678787887096405\n",
            "Save Model, best_accuracy:  0.7027027027027027\n",
            "Epoch:  3 , train_loss:  0.6092038399539887 , train_acc:  0.7133333333333334 ,val_loss:  0.6209012874074884 , val_acc:  0.6756756756756757\n",
            "Epoch:  3 , train_f1_score:  0.3174603283405304 , train_precision:  0.6666666865348816 ,train_recall:  0.2083333283662796 , train_auc:  0.5819035768508911\n",
            "Epoch:  3 , val_f1_score:  0.4000000059604645 , val_precision:  0.800000011920929 ,val_recall:  0.2666666805744171 , val_auc:  0.6969696879386902\n",
            "Epoch:  4 , train_loss:  0.5937889396368216 , train_acc:  0.7266666666666667 ,val_loss:  0.6220494645672876 , val_acc:  0.6756756756756757\n",
            "Epoch:  4 , train_f1_score:  0.3692307770252228 , train_precision:  0.7058823704719543 ,train_recall:  0.25 , train_auc:  0.6113153696060181\n",
            "Epoch:  4 , val_f1_score:  0.4000000059604645 , val_precision:  0.800000011920929 ,val_recall:  0.2666666805744171 , val_auc:  0.7212121486663818\n",
            "Epoch:  5 , train_loss:  0.5777596604793022 , train_acc:  0.7333333333333333 ,val_loss:  0.6513332531258866 , val_acc:  0.6486486486486487\n",
            "Epoch:  5 , train_f1_score:  0.39393940567970276 , train_precision:  0.7222222089767456 ,train_recall:  0.2708333432674408 , train_auc:  0.6292892694473267\n",
            "Epoch:  5 , val_f1_score:  0.23529411852359772 , val_precision:  1.0 ,val_recall:  0.13333334028720856 , val_auc:  0.6969696879386902\n",
            "Epoch:  6 , train_loss:  0.5813688421125214 , train_acc:  0.74 ,val_loss:  0.6383111001671972 , val_acc:  0.6756756756756757\n",
            "Epoch:  6 , train_f1_score:  0.43478259444236755 , train_precision:  0.7142857313156128 ,train_recall:  0.3125 , train_auc:  0.6386846303939819\n",
            "Epoch:  6 , val_f1_score:  0.4000000059604645 , val_precision:  0.800000011920929 ,val_recall:  0.2666666805744171 , val_auc:  0.6727272868156433\n",
            "Epoch:  7 , train_loss:  0.5689413345232606 , train_acc:  0.7533333333333333 ,val_loss:  0.6709529701116923 , val_acc:  0.6216216216216216\n",
            "Epoch:  7 , train_f1_score:  0.4637681245803833 , train_precision:  0.761904776096344 ,train_recall:  0.3333333432674408 , train_auc:  0.657475471496582\n",
            "Epoch:  7 , val_f1_score:  0.2222222238779068 , val_precision:  0.6666666865348816 ,val_recall:  0.13333334028720856 , val_auc:  0.678787887096405\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7027027027027027 current_acc:  0.6216216216216216\n",
            "F1 Score Mean:  0.42105263471603394 , F1 Score Std:  0.0 , Precision Mean:  1.0 ,Precision Std:  0.0\n",
            "Recall Mean:  0.2666666805744171 Recall Std:  0.0 AUC Mean:  0.7156878683194472 AUC Std:  0.08321667695476623\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "run_test('pet_seg')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vn_rvyVxGqN3",
        "outputId": "89f4eae4-e72f-4037-c37a-78b32acbc5a9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Data is loaded, training starting\n",
            "Fold [1/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6575936291041791 , train_acc:  0.6644295302013423 ,val_loss:  0.7284844235370034 , val_acc:  0.6578947368421053\n",
            "Epoch:  0 , train_f1_score:  0.0 , train_precision:  0.0 ,train_recall:  0.0 , train_auc:  0.44727271795272827\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.23692309856414795\n",
            "Save Model, best_accuracy:  0.6578947368421053\n",
            "Epoch:  1 , train_loss:  0.7307806586559187 , train_acc:  0.6174496644295302 ,val_loss:  0.5636892855951661 , val_acc:  0.7105263157894737\n",
            "Epoch:  1 , train_f1_score:  0.3132530152797699 , train_precision:  0.39393940567970276 ,train_recall:  0.25999999046325684 , train_auc:  0.5373737215995789\n",
            "Epoch:  1 , val_f1_score:  0.2666666805744171 , val_precision:  1.0 ,val_recall:  0.1538461595773697 , val_auc:  0.763076901435852\n",
            "Save Model, best_accuracy:  0.7105263157894737\n",
            "Epoch:  2 , train_loss:  0.6144346946337879 , train_acc:  0.7046979865771812 ,val_loss:  0.5708583863942247 , val_acc:  0.7105263157894737\n",
            "Epoch:  2 , train_f1_score:  0.37142857909202576 , train_precision:  0.6499999761581421 ,train_recall:  0.25999999046325684 , train_auc:  0.6309090852737427\n",
            "Epoch:  2 , val_f1_score:  0.2666666805744171 , val_precision:  1.0 ,val_recall:  0.1538461595773697 , val_auc:  0.7507692575454712\n",
            "Epoch:  3 , train_loss:  0.5962299653547722 , train_acc:  0.7181208053691275 ,val_loss:  0.5744067779497096 , val_acc:  0.7105263157894737\n",
            "Epoch:  3 , train_f1_score:  0.4000000059604645 , train_precision:  0.699999988079071 ,train_recall:  0.2800000011920929 , train_auc:  0.6597979664802551\n",
            "Epoch:  3 , val_f1_score:  0.2666666805744171 , val_precision:  1.0 ,val_recall:  0.1538461595773697 , val_auc:  0.7384615540504456\n",
            "Epoch:  4 , train_loss:  0.5774999439916355 , train_acc:  0.738255033557047 ,val_loss:  0.5811341150026572 , val_acc:  0.7105263157894737\n",
            "Epoch:  4 , train_f1_score:  0.4507042169570923 , train_precision:  0.761904776096344 ,train_recall:  0.3199999928474426 , train_auc:  0.6850504875183105\n",
            "Epoch:  4 , val_f1_score:  0.2666666805744171 , val_precision:  1.0 ,val_recall:  0.1538461595773697 , val_auc:  0.7169231176376343\n",
            "Epoch:  5 , train_loss:  0.5450259687366502 , train_acc:  0.7583892617449665 ,val_loss:  0.5840367565029546 , val_acc:  0.7105263157894737\n",
            "Epoch:  5 , train_f1_score:  0.5135135054588318 , train_precision:  0.7916666865348816 ,train_recall:  0.3799999952316284 , train_auc:  0.7250505685806274\n",
            "Epoch:  5 , val_f1_score:  0.2666666805744171 , val_precision:  1.0 ,val_recall:  0.1538461595773697 , val_auc:  0.7107692956924438\n",
            "Epoch:  6 , train_loss:  0.49405508644678847 , train_acc:  0.7919463087248322 ,val_loss:  0.5962735320392408 , val_acc:  0.7105263157894737\n",
            "Epoch:  6 , train_f1_score:  0.607594907283783 , train_precision:  0.8275862336158752 ,train_recall:  0.47999998927116394 , train_auc:  0.7725252509117126\n",
            "Epoch:  6 , val_f1_score:  0.3529411852359772 , val_precision:  0.75 ,val_recall:  0.23076923191547394 , val_auc:  0.6615384817123413\n",
            "Epoch:  7 , train_loss:  0.41458539652869403 , train_acc:  0.8523489932885906 ,val_loss:  0.6187548770716316 , val_acc:  0.7368421052631579\n",
            "Epoch:  7 , train_f1_score:  0.7317073345184326 , train_precision:  0.9375 ,train_recall:  0.6000000238418579 , train_auc:  0.8317171335220337\n",
            "Epoch:  7 , val_f1_score:  0.4444444477558136 , val_precision:  0.800000011920929 ,val_recall:  0.3076923191547394 , val_auc:  0.5938462018966675\n",
            "Save Model, best_accuracy:  0.7368421052631579\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7368421052631579 current_acc:  0.7368421052631579\n",
            "Fold [2/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.670771530110564 , train_acc:  0.6040268456375839 ,val_loss:  0.6647245146726307 , val_acc:  0.6578947368421053\n",
            "Epoch:  0 , train_f1_score:  0.14492753148078918 , train_precision:  0.2631579041481018 ,train_recall:  0.10000000149011612 , train_auc:  0.47272729873657227\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.4646154046058655\n",
            "Save Model, best_accuracy:  0.6578947368421053\n",
            "Epoch:  1 , train_loss:  0.7142518742022498 , train_acc:  0.6308724832214765 ,val_loss:  0.5157680789891043 , val_acc:  0.7894736842105263\n",
            "Epoch:  1 , train_f1_score:  0.32098764181137085 , train_precision:  0.4193548262119293 ,train_recall:  0.25999999046325684 , train_auc:  0.5402020215988159\n",
            "Epoch:  1 , val_f1_score:  0.5555555820465088 , val_precision:  1.0 ,val_recall:  0.38461539149284363 , val_auc:  0.803076982498169\n",
            "Save Model, best_accuracy:  0.7894736842105263\n",
            "Epoch:  2 , train_loss:  0.6294896149595312 , train_acc:  0.6510067114093959 ,val_loss:  0.5254914207678092 , val_acc:  0.7631578947368421\n",
            "Epoch:  2 , train_f1_score:  0.2777777910232544 , train_precision:  0.4545454680919647 ,train_recall:  0.20000000298023224 , train_auc:  0.5983838438987732\n",
            "Epoch:  2 , val_f1_score:  0.47058823704719543 , val_precision:  1.0 ,val_recall:  0.3076923191547394 , val_auc:  0.8061538934707642\n",
            "Epoch:  3 , train_loss:  0.6114527934149607 , train_acc:  0.6711409395973155 ,val_loss:  0.5296664892842895 , val_acc:  0.7631578947368421\n",
            "Epoch:  3 , train_f1_score:  0.28985506296157837 , train_precision:  0.5263158082962036 ,train_recall:  0.20000000298023224 , train_auc:  0.6296969652175903\n",
            "Epoch:  3 , val_f1_score:  0.47058823704719543 , val_precision:  1.0 ,val_recall:  0.3076923191547394 , val_auc:  0.8061538934707642\n",
            "Epoch:  4 , train_loss:  0.5876436848130903 , train_acc:  0.697986577181208 ,val_loss:  0.5145743634355696 , val_acc:  0.7631578947368421\n",
            "Epoch:  4 , train_f1_score:  0.3661971688270569 , train_precision:  0.6190476417541504 ,train_recall:  0.25999999046325684 , train_auc:  0.6658585667610168\n",
            "Epoch:  4 , val_f1_score:  0.47058823704719543 , val_precision:  1.0 ,val_recall:  0.3076923191547394 , val_auc:  0.8307692408561707\n",
            "Epoch:  5 , train_loss:  0.5661389949706527 , train_acc:  0.738255033557047 ,val_loss:  0.515800747040071 , val_acc:  0.7631578947368421\n",
            "Epoch:  5 , train_f1_score:  0.47999998927116394 , train_precision:  0.7200000286102295 ,train_recall:  0.36000001430511475 , train_auc:  0.6921212673187256\n",
            "Epoch:  5 , val_f1_score:  0.47058823704719543 , val_precision:  1.0 ,val_recall:  0.3076923191547394 , val_auc:  0.8246153593063354\n",
            "Epoch:  6 , train_loss:  0.527626674274614 , train_acc:  0.7651006711409396 ,val_loss:  0.5129159610522421 , val_acc:  0.7631578947368421\n",
            "Epoch:  6 , train_f1_score:  0.5569620132446289 , train_precision:  0.7586206793785095 ,train_recall:  0.4399999976158142 , train_auc:  0.7353535890579224\n",
            "Epoch:  6 , val_f1_score:  0.47058823704719543 , val_precision:  1.0 ,val_recall:  0.3076923191547394 , val_auc:  0.8215384483337402\n",
            "Epoch:  7 , train_loss:  0.46807285491406414 , train_acc:  0.8120805369127517 ,val_loss:  0.4924746483172241 , val_acc:  0.7631578947368421\n",
            "Epoch:  7 , train_f1_score:  0.6585366129875183 , train_precision:  0.84375 ,train_recall:  0.5400000214576721 , train_auc:  0.7866666316986084\n",
            "Epoch:  7 , val_f1_score:  0.5263158082962036 , val_precision:  0.8333333134651184 ,val_recall:  0.38461539149284363 , val_auc:  0.8276922702789307\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7894736842105263 current_acc:  0.7631578947368421\n",
            "Fold [3/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6700985269745191 , train_acc:  0.6 ,val_loss:  0.8413384240624067 , val_acc:  0.7027027027027027\n",
            "Epoch:  0 , train_f1_score:  0.23076923191547394 , train_precision:  0.3461538553237915 ,train_recall:  0.17307692766189575 , train_auc:  0.5156986117362976\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.440559446811676\n",
            "Save Model, best_accuracy:  0.7027027027027027\n",
            "Epoch:  1 , train_loss:  0.7070237126946449 , train_acc:  0.6333333333333333 ,val_loss:  0.6769344620154919 , val_acc:  0.6486486486486487\n",
            "Epoch:  1 , train_f1_score:  0.3820224702358246 , train_precision:  0.45945945382118225 ,train_recall:  0.32692307233810425 , train_auc:  0.6151884198188782\n",
            "Epoch:  1 , val_f1_score:  0.380952388048172 , val_precision:  0.4000000059604645 ,val_recall:  0.3636363744735718 , val_auc:  0.5664335489273071\n",
            "Epoch:  2 , train_loss:  0.5834030818815032 , train_acc:  0.7066666666666667 ,val_loss:  0.6821450280478677 , val_acc:  0.6486486486486487\n",
            "Epoch:  2 , train_f1_score:  0.4761904776096344 , train_precision:  0.625 ,train_recall:  0.38461539149284363 , train_auc:  0.6846545934677124\n",
            "Epoch:  2 , val_f1_score:  0.380952388048172 , val_precision:  0.4000000059604645 ,val_recall:  0.3636363744735718 , val_auc:  0.5734266042709351\n",
            "Epoch:  3 , train_loss:  0.5548754554800689 , train_acc:  0.74 ,val_loss:  0.679034753073309 , val_acc:  0.6486486486486487\n",
            "Epoch:  3 , train_f1_score:  0.5301204919815063 , train_precision:  0.7096773982048035 ,train_recall:  0.42307692766189575 , train_auc:  0.7191915512084961\n",
            "Epoch:  3 , val_f1_score:  0.380952388048172 , val_precision:  0.4000000059604645 ,val_recall:  0.3636363744735718 , val_auc:  0.5454545617103577\n",
            "Epoch:  4 , train_loss:  0.5309059826284647 , train_acc:  0.7666666666666667 ,val_loss:  0.6773467581165401 , val_acc:  0.6486486486486487\n",
            "Epoch:  4 , train_f1_score:  0.5977011322975159 , train_precision:  0.7428571581840515 ,train_recall:  0.5 , train_auc:  0.7494113445281982\n",
            "Epoch:  4 , val_f1_score:  0.380952388048172 , val_precision:  0.4000000059604645 ,val_recall:  0.3636363744735718 , val_auc:  0.5559440851211548\n",
            "Epoch:  5 , train_loss:  0.48961140231539807 , train_acc:  0.7933333333333333 ,val_loss:  0.6675133389622174 , val_acc:  0.7297297297297297\n",
            "Epoch:  5 , train_f1_score:  0.6436781883239746 , train_precision:  0.800000011920929 ,train_recall:  0.5384615659713745 , train_auc:  0.7833595275878906\n",
            "Epoch:  5 , val_f1_score:  0.4444444477558136 , val_precision:  0.5714285969734192 ,val_recall:  0.3636363744735718 , val_auc:  0.5454545617103577\n",
            "Save Model, best_accuracy:  0.7297297297297297\n",
            "Epoch:  6 , train_loss:  0.43106784294359385 , train_acc:  0.8333333333333334 ,val_loss:  0.6663891598299095 , val_acc:  0.7027027027027027\n",
            "Epoch:  6 , train_f1_score:  0.7311828136444092 , train_precision:  0.8292682766914368 ,train_recall:  0.6538461446762085 , train_auc:  0.8241758346557617\n",
            "Epoch:  6 , val_f1_score:  0.3529411852359772 , val_precision:  0.5 ,val_recall:  0.27272728085517883 , val_auc:  0.5489510297775269\n",
            "Epoch:  7 , train_loss:  0.357576241127681 , train_acc:  0.8933333333333333 ,val_loss:  0.6238216621222327 , val_acc:  0.6756756756756757\n",
            "Epoch:  7 , train_f1_score:  0.8181818127632141 , train_precision:  1.0 ,train_recall:  0.692307710647583 , train_auc:  0.8608712553977966\n",
            "Epoch:  7 , val_f1_score:  0.3333333432674408 , val_precision:  0.4285714328289032 ,val_recall:  0.27272728085517883 , val_auc:  0.5489510893821716\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7297297297297297 current_acc:  0.6756756756756757\n",
            "Fold [4/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6704566380381585 , train_acc:  0.6266666666666667 ,val_loss:  0.6654547678457724 , val_acc:  0.6486486486486487\n",
            "Epoch:  0 , train_f1_score:  0.2631579041481018 , train_precision:  0.38461539149284363 ,train_recall:  0.20000000298023224 , train_auc:  0.4926000237464905\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.4391025900840759\n",
            "Save Model, best_accuracy:  0.6486486486486487\n",
            "Epoch:  1 , train_loss:  0.6386508818832226 , train_acc:  0.64 ,val_loss:  0.6334838472508095 , val_acc:  0.6756756756756757\n",
            "Epoch:  1 , train_f1_score:  0.32499998807907104 , train_precision:  0.4333333373069763 ,train_recall:  0.25999999046325684 , train_auc:  0.6708000302314758\n",
            "Epoch:  1 , val_f1_score:  0.3333333432674408 , val_precision:  0.6000000238418579 ,val_recall:  0.23076923191547394 , val_auc:  0.6185897588729858\n",
            "Save Model, best_accuracy:  0.6756756756756757\n",
            "Epoch:  2 , train_loss:  0.5702548637605893 , train_acc:  0.72 ,val_loss:  0.6389382787652917 , val_acc:  0.7027027027027027\n",
            "Epoch:  2 , train_f1_score:  0.4749999940395355 , train_precision:  0.6333333253860474 ,train_recall:  0.3799999952316284 , train_auc:  0.7067999839782715\n",
            "Epoch:  2 , val_f1_score:  0.2666666805744171 , val_precision:  1.0 ,val_recall:  0.1538461595773697 , val_auc:  0.615384578704834\n",
            "Save Model, best_accuracy:  0.7027027027027027\n",
            "Epoch:  3 , train_loss:  0.5467318296269513 , train_acc:  0.74 ,val_loss:  0.6474170737169884 , val_acc:  0.6756756756756757\n",
            "Epoch:  3 , train_f1_score:  0.5185185074806213 , train_precision:  0.6774193644523621 ,train_recall:  0.41999998688697815 , train_auc:  0.7253999710083008\n",
            "Epoch:  3 , val_f1_score:  0.1428571492433548 , val_precision:  1.0 ,val_recall:  0.07692307978868484 , val_auc:  0.6025640964508057\n",
            "Epoch:  4 , train_loss:  0.517351891513293 , train_acc:  0.7733333333333333 ,val_loss:  0.6460280889594877 , val_acc:  0.6756756756756757\n",
            "Epoch:  4 , train_f1_score:  0.574999988079071 , train_precision:  0.7666666507720947 ,train_recall:  0.46000000834465027 , train_auc:  0.754800021648407\n",
            "Epoch:  4 , val_f1_score:  0.1428571492433548 , val_precision:  1.0 ,val_recall:  0.07692307978868484 , val_auc:  0.6057692170143127\n",
            "Epoch:  5 , train_loss:  0.4684711466310546 , train_acc:  0.8266666666666667 ,val_loss:  0.6478924747254398 , val_acc:  0.7027027027027027\n",
            "Epoch:  5 , train_f1_score:  0.6904761791229248 , train_precision:  0.8529411554336548 ,train_recall:  0.5799999833106995 , train_auc:  0.8021999597549438\n",
            "Epoch:  5 , val_f1_score:  0.2666666805744171 , val_precision:  1.0 ,val_recall:  0.1538461595773697 , val_auc:  0.6089743375778198\n",
            "Epoch:  6 , train_loss:  0.4290240446174721 , train_acc:  0.8266666666666667 ,val_loss:  0.6533337839551874 , val_acc:  0.7027027027027027\n",
            "Epoch:  6 , train_f1_score:  0.6904761791229248 , train_precision:  0.8529411554336548 ,train_recall:  0.5799999833106995 , train_auc:  0.8267999887466431\n",
            "Epoch:  6 , val_f1_score:  0.2666666805744171 , val_precision:  1.0 ,val_recall:  0.1538461595773697 , val_auc:  0.6025640964508057\n",
            "Epoch:  7 , train_loss:  0.36677558547547356 , train_acc:  0.8866666666666667 ,val_loss:  0.6711702999230977 , val_acc:  0.7027027027027027\n",
            "Epoch:  7 , train_f1_score:  0.800000011920929 , train_precision:  0.9714285731315613 ,train_recall:  0.6800000071525574 , train_auc:  0.8593999743461609\n",
            "Epoch:  7 , val_f1_score:  0.2666666805744171 , val_precision:  1.0 ,val_recall:  0.1538461595773697 , val_auc:  0.567307710647583\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7027027027027027 current_acc:  0.7027027027027027\n",
            "Fold [5/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6848753723502159 , train_acc:  0.58 ,val_loss:  0.8188263733644743 , val_acc:  0.6486486486486487\n",
            "Epoch:  0 , train_f1_score:  0.20253165066242218 , train_precision:  0.27586206793785095 ,train_recall:  0.1599999964237213 , train_auc:  0.508400022983551\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.2788461446762085\n",
            "Save Model, best_accuracy:  0.6486486486486487\n",
            "Epoch:  1 , train_loss:  0.6971964117636283 , train_acc:  0.6333333333333333 ,val_loss:  0.5978867043111775 , val_acc:  0.7297297297297297\n",
            "Epoch:  1 , train_f1_score:  0.32098764181137085 , train_precision:  0.4193548262119293 ,train_recall:  0.25999999046325684 , train_auc:  0.6115999817848206\n",
            "Epoch:  1 , val_f1_score:  0.6428571343421936 , val_precision:  0.6000000238418579 ,val_recall:  0.692307710647583 , val_auc:  0.759615421295166\n",
            "Save Model, best_accuracy:  0.7297297297297297\n",
            "Epoch:  2 , train_loss:  0.6085024680631856 , train_acc:  0.6866666666666666 ,val_loss:  0.5725110723964266 , val_acc:  0.7837837837837838\n",
            "Epoch:  2 , train_f1_score:  0.3896103799343109 , train_precision:  0.5555555820465088 ,train_recall:  0.30000001192092896 , train_auc:  0.6438000202178955\n",
            "Epoch:  2 , val_f1_score:  0.692307710647583 , val_precision:  0.692307710647583 ,val_recall:  0.692307710647583 , val_auc:  0.7628205418586731\n",
            "Save Model, best_accuracy:  0.7837837837837838\n",
            "Epoch:  3 , train_loss:  0.5906082791524628 , train_acc:  0.7 ,val_loss:  0.5635599607752787 , val_acc:  0.7297297297297297\n",
            "Epoch:  3 , train_f1_score:  0.41558441519737244 , train_precision:  0.5925925970077515 ,train_recall:  0.3199999928474426 , train_auc:  0.6633999943733215\n",
            "Epoch:  3 , val_f1_score:  0.5454545617103577 , val_precision:  0.6666666865348816 ,val_recall:  0.4615384638309479 , val_auc:  0.759615421295166\n",
            "Epoch:  4 , train_loss:  0.5643015264999122 , train_acc:  0.72 ,val_loss:  0.5506468372973236 , val_acc:  0.7297297297297297\n",
            "Epoch:  4 , train_f1_score:  0.4615384638309479 , train_precision:  0.6428571343421936 ,train_recall:  0.36000001430511475 , train_auc:  0.6934000253677368\n",
            "Epoch:  4 , val_f1_score:  0.5454545617103577 , val_precision:  0.6666666865348816 ,val_recall:  0.4615384638309479 , val_auc:  0.7660256624221802\n",
            "Epoch:  5 , train_loss:  0.536154199897622 , train_acc:  0.74 ,val_loss:  0.5428984485365249 , val_acc:  0.7567567567567568\n",
            "Epoch:  5 , train_f1_score:  0.4935064911842346 , train_precision:  0.7037037014961243 ,train_recall:  0.3799999952316284 , train_auc:  0.7278000116348267\n",
            "Epoch:  5 , val_f1_score:  0.6086956262588501 , val_precision:  0.699999988079071 ,val_recall:  0.5384615659713745 , val_auc:  0.7724359035491943\n",
            "Epoch:  6 , train_loss:  0.5030099981743842 , train_acc:  0.74 ,val_loss:  0.5510711416954527 , val_acc:  0.7567567567567568\n",
            "Epoch:  6 , train_f1_score:  0.5063291192054749 , train_precision:  0.6896551847457886 ,train_recall:  0.4000000059604645 , train_auc:  0.767799973487854\n",
            "Epoch:  6 , val_f1_score:  0.6399999856948853 , val_precision:  0.6666666865348816 ,val_recall:  0.6153846383094788 , val_auc:  0.7756410837173462\n",
            "Epoch:  7 , train_loss:  0.4554561161698075 , train_acc:  0.7866666666666666 ,val_loss:  0.5575247139770638 , val_acc:  0.7297297297297297\n",
            "Epoch:  7 , train_f1_score:  0.6000000238418579 , train_precision:  0.800000011920929 ,train_recall:  0.47999998927116394 , train_auc:  0.8122000098228455\n",
            "Epoch:  7 , val_f1_score:  0.6153846383094788 , val_precision:  0.6153846383094788 ,val_recall:  0.6153846383094788 , val_auc:  0.7756410837173462\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7837837837837838 current_acc:  0.7297297297297297\n",
            "F1 Score Mean:  0.692307710647583 , F1 Score Std:  0.0 , Precision Mean:  0.692307710647583 ,Precision Std:  0.0\n",
            "Recall Mean:  0.692307710647583 Recall Std:  0.0 AUC Mean:  0.7263970737654947 AUC Std:  0.04521200809322972\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "run_test('mixed')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KOw8Uo3UGql7",
        "outputId": "caadd9a4-3204-4574-e611-35fe1d84c07c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Data is loaded, training starting\n",
            "Fold [1/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.7617426696639733 , train_acc:  0.4697986577181208 ,val_loss:  1.0132714196255332 , val_acc:  0.631578947368421\n",
            "Epoch:  0 , train_f1_score:  0.2752293646335602 , train_precision:  0.25 ,train_recall:  0.30612245202064514 , train_auc:  0.4306122660636902\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.1607142835855484\n",
            "Save Model, best_accuracy:  0.631578947368421\n",
            "Epoch:  1 , train_loss:  0.781084462800282 , train_acc:  0.6174496644295302 ,val_loss:  0.6253805442860252 , val_acc:  0.7368421052631579\n",
            "Epoch:  1 , train_f1_score:  0.19718310236930847 , train_precision:  0.3181818127632141 ,train_recall:  0.1428571492433548 , train_auc:  0.5\n",
            "Epoch:  1 , val_f1_score:  0.5 , val_precision:  0.8333333134651184 ,val_recall:  0.3571428656578064 , val_auc:  0.7827380895614624\n",
            "Save Model, best_accuracy:  0.7368421052631579\n",
            "Epoch:  2 , train_loss:  0.6841798699742195 , train_acc:  0.6040268456375839 ,val_loss:  0.5818868236322152 , val_acc:  0.7105263157894737\n",
            "Epoch:  2 , train_f1_score:  0.14492753148078918 , train_precision:  0.25 ,train_recall:  0.10204081982374191 , train_auc:  0.4908163249492645\n",
            "Epoch:  2 , val_f1_score:  0.3529411852359772 , val_precision:  1.0 ,val_recall:  0.2142857164144516 , val_auc:  0.7738094925880432\n",
            "Epoch:  3 , train_loss:  0.6523327192404126 , train_acc:  0.6510067114093959 ,val_loss:  0.5941016807367927 , val_acc:  0.6842105263157895\n",
            "Epoch:  3 , train_f1_score:  0.1875 , train_precision:  0.4000000059604645 ,train_recall:  0.12244898080825806 , train_auc:  0.5267347097396851\n",
            "Epoch:  3 , val_f1_score:  0.25 , val_precision:  1.0 ,val_recall:  0.1428571492433548 , val_auc:  0.726190447807312\n",
            "Epoch:  4 , train_loss:  0.646669930909704 , train_acc:  0.6711409395973155 ,val_loss:  0.5745755606576016 , val_acc:  0.7105263157894737\n",
            "Epoch:  4 , train_f1_score:  0.2222222238779068 , train_precision:  0.5 ,train_recall:  0.1428571492433548 , train_auc:  0.5375510454177856\n",
            "Epoch:  4 , val_f1_score:  0.3529411852359772 , val_precision:  1.0 ,val_recall:  0.2142857164144516 , val_auc:  0.788690447807312\n",
            "Epoch:  5 , train_loss:  0.6427919194102287 , train_acc:  0.6711409395973155 ,val_loss:  0.5787938676382366 , val_acc:  0.7105263157894737\n",
            "Epoch:  5 , train_f1_score:  0.19672131538391113 , train_precision:  0.5 ,train_recall:  0.12244898080825806 , train_auc:  0.5514285564422607\n",
            "Epoch:  5 , val_f1_score:  0.3529411852359772 , val_precision:  1.0 ,val_recall:  0.2142857164144516 , val_auc:  0.7648809552192688\n",
            "Epoch:  6 , train_loss:  0.6275359118644823 , train_acc:  0.6845637583892618 ,val_loss:  0.5701382020978552 , val_acc:  0.7105263157894737\n",
            "Epoch:  6 , train_f1_score:  0.2539682686328888 , train_precision:  0.5714285969734192 ,train_recall:  0.16326530277729034 , train_auc:  0.5687755346298218\n",
            "Epoch:  6 , val_f1_score:  0.3529411852359772 , val_precision:  1.0 ,val_recall:  0.2142857164144516 , val_auc:  0.803571343421936\n",
            "Epoch:  7 , train_loss:  0.6310743205999368 , train_acc:  0.697986577181208 ,val_loss:  0.6187186790140051 , val_acc:  0.6578947368421053\n",
            "Epoch:  7 , train_f1_score:  0.2857142984867096 , train_precision:  0.6428571343421936 ,train_recall:  0.18367347121238708 , train_auc:  0.5942857265472412\n",
            "Epoch:  7 , val_f1_score:  0.13333334028720856 , val_precision:  1.0 ,val_recall:  0.0714285746216774 , val_auc:  0.6815476417541504\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7368421052631579 current_acc:  0.6578947368421053\n",
            "Fold [2/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6363145637032169 , train_acc:  0.6308724832214765 ,val_loss:  0.7940070091893798 , val_acc:  0.6052631578947368\n",
            "Epoch:  0 , train_f1_score:  0.035087719559669495 , train_precision:  0.1111111119389534 ,train_recall:  0.02083333395421505 , train_auc:  0.5396039485931396\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.3855072855949402\n",
            "Save Model, best_accuracy:  0.6052631578947368\n",
            "Epoch:  1 , train_loss:  0.7247315924927192 , train_acc:  0.6442953020134228 ,val_loss:  0.7380407298302376 , val_acc:  0.5789473684210527\n",
            "Epoch:  1 , train_f1_score:  0.2933333218097687 , train_precision:  0.40740740299224854 ,train_recall:  0.2291666716337204 , train_auc:  0.5437293648719788\n",
            "Epoch:  1 , val_f1_score:  0.46666666865348816 , val_precision:  0.46666666865348816 ,val_recall:  0.46666666865348816 , val_auc:  0.5623188614845276\n",
            "Epoch:  2 , train_loss:  0.6361403845550629 , train_acc:  0.7114093959731543 ,val_loss:  0.7537358365952969 , val_acc:  0.5526315789473685\n",
            "Epoch:  2 , train_f1_score:  0.3384615480899811 , train_precision:  0.6470588445663452 ,train_recall:  0.2291666716337204 , train_auc:  0.5701320171356201\n",
            "Epoch:  2 , val_f1_score:  0.37037035822868347 , val_precision:  0.4166666567325592 ,val_recall:  0.3333333432674408 , val_auc:  0.5246376991271973\n",
            "Epoch:  3 , train_loss:  0.6187040417911422 , train_acc:  0.7181208053691275 ,val_loss:  0.6852891330460185 , val_acc:  0.6578947368421053\n",
            "Epoch:  3 , train_f1_score:  0.3636363744735718 , train_precision:  0.6666666865348816 ,train_recall:  0.25 , train_auc:  0.5930280685424805\n",
            "Epoch:  3 , val_f1_score:  0.47999998927116394 , val_precision:  0.6000000238418579 ,val_recall:  0.4000000059604645 , val_auc:  0.5420290231704712\n",
            "Save Model, best_accuracy:  0.6578947368421053\n",
            "Epoch:  4 , train_loss:  0.5808911981192362 , train_acc:  0.738255033557047 ,val_loss:  0.6923654690678966 , val_acc:  0.6578947368421053\n",
            "Epoch:  4 , train_f1_score:  0.41791045665740967 , train_precision:  0.7368420958518982 ,train_recall:  0.2916666567325592 , train_auc:  0.6307755708694458\n",
            "Epoch:  4 , val_f1_score:  0.47999998927116394 , val_precision:  0.6000000238418579 ,val_recall:  0.4000000059604645 , val_auc:  0.5420290231704712\n",
            "Epoch:  5 , train_loss:  0.5716749231327926 , train_acc:  0.7449664429530202 ,val_loss:  0.673279222444092 , val_acc:  0.6578947368421053\n",
            "Epoch:  5 , train_f1_score:  0.44117647409439087 , train_precision:  0.75 ,train_recall:  0.3125 , train_auc:  0.6619223952293396\n",
            "Epoch:  5 , val_f1_score:  0.47999998927116394 , val_precision:  0.6000000238418579 ,val_recall:  0.4000000059604645 , val_auc:  0.539130449295044\n",
            "Epoch:  6 , train_loss:  0.5577094749801077 , train_acc:  0.7583892617449665 ,val_loss:  0.6473140355787779 , val_acc:  0.6578947368421053\n",
            "Epoch:  6 , train_f1_score:  0.48571428656578064 , train_precision:  0.7727272510528564 ,train_recall:  0.3541666567325592 , train_auc:  0.6751237511634827\n",
            "Epoch:  6 , val_f1_score:  0.43478259444236755 , val_precision:  0.625 ,val_recall:  0.3333333432674408 , val_auc:  0.5623188614845276\n",
            "Epoch:  7 , train_loss:  0.5434274361814582 , train_acc:  0.7583892617449665 ,val_loss:  0.6423663881264234 , val_acc:  0.6578947368421053\n",
            "Epoch:  7 , train_f1_score:  0.5135135054588318 , train_precision:  0.7307692170143127 ,train_recall:  0.3958333432674408 , train_auc:  0.6879125237464905\n",
            "Epoch:  7 , val_f1_score:  0.43478259444236755 , val_precision:  0.625 ,val_recall:  0.3333333432674408 , val_auc:  0.5623188614845276\n",
            "Early stopping in epoch:  7 best_accuracy:  0.6578947368421053 current_acc:  0.6578947368421053\n",
            "Fold [3/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6954894493023555 , train_acc:  0.5733333333333334 ,val_loss:  0.6840461262980023 , val_acc:  0.7297297297297297\n",
            "Epoch:  0 , train_f1_score:  0.2888889014720917 , train_precision:  0.3513513505458832 ,train_recall:  0.24528302252292633 , train_auc:  0.4732542634010315\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.3037036955356598\n",
            "Save Model, best_accuracy:  0.7297297297297297\n",
            "Epoch:  1 , train_loss:  0.7681340760116776 , train_acc:  0.62 ,val_loss:  0.5715127960350868 , val_acc:  0.7567567567567568\n",
            "Epoch:  1 , train_f1_score:  0.3132530152797699 , train_precision:  0.4333333373069763 ,train_recall:  0.24528302252292633 , train_auc:  0.5281074047088623\n",
            "Epoch:  1 , val_f1_score:  0.3076923191547394 , val_precision:  0.6666666865348816 ,val_recall:  0.20000000298023224 , val_auc:  0.6000000238418579\n",
            "Save Model, best_accuracy:  0.7567567567567568\n",
            "Epoch:  2 , train_loss:  0.6433154869576295 , train_acc:  0.66 ,val_loss:  0.5613246877249833 , val_acc:  0.7567567567567568\n",
            "Epoch:  2 , train_f1_score:  0.3544303774833679 , train_precision:  0.5384615659713745 ,train_recall:  0.2641509473323822 , train_auc:  0.580042839050293\n",
            "Epoch:  2 , val_f1_score:  0.3076923191547394 , val_precision:  0.6666666865348816 ,val_recall:  0.20000000298023224 , val_auc:  0.5962963104248047\n",
            "Epoch:  3 , train_loss:  0.6355729535718759 , train_acc:  0.7 ,val_loss:  0.5655169017814301 , val_acc:  0.7567567567567568\n",
            "Epoch:  3 , train_f1_score:  0.4000000059604645 , train_precision:  0.6818181872367859 ,train_recall:  0.2830188572406769 , train_auc:  0.5985217094421387\n",
            "Epoch:  3 , val_f1_score:  0.3076923191547394 , val_precision:  0.6666666865348816 ,val_recall:  0.20000000298023224 , val_auc:  0.5962963104248047\n",
            "Epoch:  4 , train_loss:  0.6423981734365225 , train_acc:  0.6733333333333333 ,val_loss:  0.5554538928979152 , val_acc:  0.7567567567567568\n",
            "Epoch:  4 , train_f1_score:  0.3636363744735718 , train_precision:  0.5833333134651184 ,train_recall:  0.2641509473323822 , train_auc:  0.58412766456604\n",
            "Epoch:  4 , val_f1_score:  0.3076923191547394 , val_precision:  0.6666666865348816 ,val_recall:  0.20000000298023224 , val_auc:  0.5481481552124023\n",
            "Epoch:  5 , train_loss:  0.6085841468349099 , train_acc:  0.7 ,val_loss:  0.5493734122530834 , val_acc:  0.7837837837837838\n",
            "Epoch:  5 , train_f1_score:  0.41558441519737244 , train_precision:  0.6666666865348816 ,train_recall:  0.30188679695129395 , train_auc:  0.6448162198066711\n",
            "Epoch:  5 , val_f1_score:  0.3333333432674408 , val_precision:  1.0 ,val_recall:  0.20000000298023224 , val_auc:  0.5555555820465088\n",
            "Save Model, best_accuracy:  0.7837837837837838\n",
            "Epoch:  6 , train_loss:  0.5961425561582049 , train_acc:  0.7066666666666667 ,val_loss:  0.5475651464752249 , val_acc:  0.7567567567567568\n",
            "Epoch:  6 , train_f1_score:  0.43589743971824646 , train_precision:  0.6800000071525574 ,train_recall:  0.3207547068595886 , train_auc:  0.6786617636680603\n",
            "Epoch:  6 , val_f1_score:  0.3076923191547394 , val_precision:  0.6666666865348816 ,val_recall:  0.20000000298023224 , val_auc:  0.555555522441864\n",
            "Epoch:  7 , train_loss:  0.5908068600545326 , train_acc:  0.72 ,val_loss:  0.5494680068380123 , val_acc:  0.7297297297297297\n",
            "Epoch:  7 , train_f1_score:  0.4749999940395355 , train_precision:  0.7037037014961243 ,train_recall:  0.35849055647850037 , train_auc:  0.6876093745231628\n",
            "Epoch:  7 , val_f1_score:  0.2857142984867096 , val_precision:  0.5 ,val_recall:  0.20000000298023224 , val_auc:  0.5777778029441833\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7837837837837838 current_acc:  0.7297297297297297\n",
            "Fold [4/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6658080185453097 , train_acc:  0.6333333333333333 ,val_loss:  0.6648115644583831 , val_acc:  0.7297297297297297\n",
            "Epoch:  0 , train_f1_score:  0.1538461595773697 , train_precision:  0.4166666567325592 ,train_recall:  0.09433962404727936 , train_auc:  0.49970823526382446\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.3333333432674408\n",
            "Save Model, best_accuracy:  0.7297297297297297\n",
            "Epoch:  1 , train_loss:  0.7367003738010923 , train_acc:  0.6266666666666667 ,val_loss:  0.5689853470067721 , val_acc:  0.7297297297297297\n",
            "Epoch:  1 , train_f1_score:  0.31707316637039185 , train_precision:  0.4482758641242981 ,train_recall:  0.24528302252292633 , train_auc:  0.5728457570075989\n",
            "Epoch:  1 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.7666667103767395\n",
            "Epoch:  2 , train_loss:  0.6662785411160439 , train_acc:  0.6533333333333333 ,val_loss:  0.5958318178718155 , val_acc:  0.7567567567567568\n",
            "Epoch:  2 , train_f1_score:  0.380952388048172 , train_precision:  0.5161290168762207 ,train_recall:  0.30188679695129395 , train_auc:  0.5769305229187012\n",
            "Epoch:  2 , val_f1_score:  0.1818181872367859 , val_precision:  1.0 ,val_recall:  0.10000000149011612 , val_auc:  0.6259258985519409\n",
            "Save Model, best_accuracy:  0.7567567567567568\n",
            "Epoch:  3 , train_loss:  0.6437828643868367 , train_acc:  0.6266666666666667 ,val_loss:  0.5513459661522427 , val_acc:  0.7567567567567568\n",
            "Epoch:  3 , train_f1_score:  0.28205129504203796 , train_precision:  0.4399999976158142 ,train_recall:  0.2075471729040146 , train_auc:  0.5856837034225464\n",
            "Epoch:  3 , val_f1_score:  0.1818181872367859 , val_precision:  1.0 ,val_recall:  0.10000000149011612 , val_auc:  0.7592592239379883\n",
            "Epoch:  4 , train_loss:  0.6432063065903882 , train_acc:  0.6866666666666666 ,val_loss:  0.5406766682863235 , val_acc:  0.7567567567567568\n",
            "Epoch:  4 , train_f1_score:  0.3380281627178192 , train_precision:  0.6666666865348816 ,train_recall:  0.22641509771347046 , train_auc:  0.611359715461731\n",
            "Epoch:  4 , val_f1_score:  0.1818181872367859 , val_precision:  1.0 ,val_recall:  0.10000000149011612 , val_auc:  0.7333333492279053\n",
            "Epoch:  5 , train_loss:  0.6225196452143913 , train_acc:  0.6933333333333334 ,val_loss:  0.5443435045512947 , val_acc:  0.7297297297297297\n",
            "Epoch:  5 , train_f1_score:  0.3947368562221527 , train_precision:  0.6521739363670349 ,train_recall:  0.2830188572406769 , train_auc:  0.6267263293266296\n",
            "Epoch:  5 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.7518518567085266\n",
            "Epoch:  6 , train_loss:  0.6078807439810286 , train_acc:  0.7066666666666667 ,val_loss:  0.5361433246651212 , val_acc:  0.7567567567567568\n",
            "Epoch:  6 , train_f1_score:  0.44999998807907104 , train_precision:  0.6666666865348816 ,train_recall:  0.3396226465702057 , train_auc:  0.6440380811691284\n",
            "Epoch:  6 , val_f1_score:  0.1818181872367859 , val_precision:  1.0 ,val_recall:  0.10000000149011612 , val_auc:  0.7777777910232544\n",
            "Epoch:  7 , train_loss:  0.5950283512333407 , train_acc:  0.7266666666666667 ,val_loss:  0.5471245827707084 , val_acc:  0.7297297297297297\n",
            "Epoch:  7 , train_f1_score:  0.4810126721858978 , train_precision:  0.7307692170143127 ,train_recall:  0.35849055647850037 , train_auc:  0.6644622087478638\n",
            "Epoch:  7 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.7592592835426331\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7567567567567568 current_acc:  0.7297297297297297\n",
            "Fold [5/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6757698422670364 , train_acc:  0.6533333333333333 ,val_loss:  0.6894376310142311 , val_acc:  0.6216216216216216\n",
            "Epoch:  0 , train_f1_score:  0.1034482792019844 , train_precision:  0.3333333432674408 ,train_recall:  0.06122449040412903 , train_auc:  0.40957769751548767\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.3819875717163086\n",
            "Save Model, best_accuracy:  0.6216216216216216\n",
            "Epoch:  1 , train_loss:  0.7188205673545599 , train_acc:  0.6333333333333333 ,val_loss:  0.6691537765232293 , val_acc:  0.6486486486486487\n",
            "Epoch:  1 , train_f1_score:  0.2666666805744171 , train_precision:  0.38461539149284363 ,train_recall:  0.20408163964748383 , train_auc:  0.5641543865203857\n",
            "Epoch:  1 , val_f1_score:  0.13333334028720856 , val_precision:  1.0 ,val_recall:  0.0714285746216774 , val_auc:  0.5403726696968079\n",
            "Save Model, best_accuracy:  0.6486486486486487\n",
            "Epoch:  2 , train_loss:  0.623243655351301 , train_acc:  0.6933333333333334 ,val_loss:  0.6687961567092586 , val_acc:  0.6486486486486487\n",
            "Epoch:  2 , train_f1_score:  0.34285715222358704 , train_precision:  0.5714285969734192 ,train_recall:  0.2448979616165161 , train_auc:  0.5803192853927612\n",
            "Epoch:  2 , val_f1_score:  0.13333334028720856 , val_precision:  1.0 ,val_recall:  0.0714285746216774 , val_auc:  0.52173912525177\n",
            "Epoch:  3 , train_loss:  0.6076193013880402 , train_acc:  0.7133333333333334 ,val_loss:  0.6784303514538584 , val_acc:  0.6216216216216216\n",
            "Epoch:  3 , train_f1_score:  0.35820895433425903 , train_precision:  0.6666666865348816 ,train_recall:  0.2448979616165161 , train_auc:  0.6021418571472168\n",
            "Epoch:  3 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.5527950525283813\n",
            "Epoch:  4 , train_loss:  0.5920421944589664 , train_acc:  0.72 ,val_loss:  0.6627956652963484 , val_acc:  0.6486486486486487\n",
            "Epoch:  4 , train_f1_score:  0.38235294818878174 , train_precision:  0.6842105388641357 ,train_recall:  0.26530611515045166 , train_auc:  0.6261870861053467\n",
            "Epoch:  4 , val_f1_score:  0.13333334028720856 , val_precision:  1.0 ,val_recall:  0.0714285746216774 , val_auc:  0.5869565010070801\n",
            "Epoch:  5 , train_loss:  0.5882433575391769 , train_acc:  0.7133333333333334 ,val_loss:  0.6588205139379244 , val_acc:  0.6486486486486487\n",
            "Epoch:  5 , train_f1_score:  0.39436620473861694 , train_precision:  0.6363636255264282 ,train_recall:  0.2857142984867096 , train_auc:  0.6294200420379639\n",
            "Epoch:  5 , val_f1_score:  0.13333334028720856 , val_precision:  1.0 ,val_recall:  0.0714285746216774 , val_auc:  0.6180124282836914\n",
            "Epoch:  6 , train_loss:  0.5726269469161829 , train_acc:  0.7466666666666667 ,val_loss:  0.6657134878474313 , val_acc:  0.6486486486486487\n",
            "Epoch:  6 , train_f1_score:  0.4722222089767456 , train_precision:  0.739130437374115 ,train_recall:  0.3469387888908386 , train_auc:  0.6623560190200806\n",
            "Epoch:  6 , val_f1_score:  0.13333334028720856 , val_precision:  1.0 ,val_recall:  0.0714285746216774 , val_auc:  0.6024844646453857\n",
            "Epoch:  7 , train_loss:  0.5621772712639843 , train_acc:  0.7533333333333333 ,val_loss:  0.6701844254055539 , val_acc:  0.6486486486486487\n",
            "Epoch:  7 , train_f1_score:  0.4931506812572479 , train_precision:  0.75 ,train_recall:  0.36734694242477417 , train_auc:  0.6767023801803589\n",
            "Epoch:  7 , val_f1_score:  0.13333334028720856 , val_precision:  1.0 ,val_recall:  0.0714285746216774 , val_auc:  0.6024844646453857\n",
            "Early stopping in epoch:  7 best_accuracy:  0.6486486486486487 current_acc:  0.6486486486486487\n",
            "F1 Score Mean:  0.13333334028720856 , F1 Score Std:  0.0 , Precision Mean:  1.0 ,Precision Std:  0.0\n",
            "Recall Mean:  0.0714285746216774 Recall Std:  0.0 AUC Mean:  0.7023166023166024 AUC Std:  0.057146976130813425\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Clinical"
      ],
      "metadata": {
        "id": "K8Ez2QeMq-yF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "from torchvision.models import resnet18\n",
        "import pandas as pd\n",
        "from sklearn.preprocessing import OneHotEncoder\n",
        "from sklearn.impute import KNNImputer\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "\n",
        "class MultimodalFusionModel(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(MultimodalFusionModel, self).__init__()\n",
        "\n",
        "        # Part 1: ResNet for feature extraction from COVID-19 images\n",
        "        self.resnet = generate_model(50, 0.2)\n",
        "        \n",
        "\n",
        "        # Part 2: FC layers for clinical information and feature concatenation\n",
        "        self.fc_clinical = nn.Sequential(\n",
        "            nn.Linear(5, 64),\n",
        "            nn.ReLU(),\n",
        "            nn.Dropout(0.1)\n",
        "        )\n",
        "        self.fc_concat = nn.Sequential(\n",
        "            nn.Linear(2048 + 64, 128),\n",
        "            nn.ReLU(),\n",
        "            nn.Dropout(0.1)\n",
        "        )\n",
        "\n",
        "        # Part 3: Final FC layer for COVID-19 prediction\n",
        "        self.fc_final = nn.Linear(128, 1)\n",
        "        self.sigmoid = nn.Sigmoid()\n",
        "\n",
        "    def forward(self, x_image, x_clinical):\n",
        "        x_image, x_clinical\n",
        "        # Part 1: 3-D ResNet for feature extraction \n",
        "        x_image = self.resnet.conv1(x_image)  \n",
        "        x_image = self.resnet.bn1(x_image)\n",
        "        x_image = self.resnet.relu(x_image)\n",
        "        x_image = self.resnet.maxpool(x_image)\n",
        "\n",
        "        x_image = self.resnet.layer1(x_image)\n",
        "        x_image = self.resnet.layer2(x_image)\n",
        "        x_image = self.resnet.layer3(x_image)\n",
        "        x_image = self.resnet.layer4(x_image)\n",
        "\n",
        "        x_image = self.resnet.avgpool(x_image)\n",
        "        x_image = self.resnet.dropout(x_image)\n",
        "        x_image = torch.flatten(x_image, 1)\n",
        "        # Part 2: FC layers for clinical information and feature concatenation\n",
        "        x_clinical = self.fc_clinical(x_clinical)\n",
        "        x_fusion = torch.cat((x_image, x_clinical), dim=1)\n",
        "        x_fusion = self.fc_concat(x_fusion)\n",
        "\n",
        "        # Part 3: Final FC layer for COVID-19 prediction\n",
        "        output = self.fc_final(x_fusion)\n",
        "        output = self.sigmoid(output)\n",
        "\n",
        "        return output"
      ],
      "metadata": {
        "id": "3oIscmjvq8ZG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "df = pd.read_csv(csv_dir)\n",
        "data_dir = initial_config['data_dir']\n",
        "data_dict_train = ImageDatasetClinical('ct_seg', data_dir, df, \"train\")\n",
        "data_dict_val = ImageDatasetClinical('ct_seg', data_dir, df, \"val\")\n",
        "print(\"Data is loaded\")  "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jEm09c9nrlay",
        "outputId": "bb66eca2-f711-4428-fc42-12310046db8d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train size:  187\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Read image for  0268 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0005 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0100 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0210 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0211 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 83)\n",
            "Read image for  0104 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0041 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0231 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0077 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0128 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0182 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0016 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0283 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 107)\n",
            "Read image for  0168 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0101 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0176 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0045 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0185 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 83)\n",
            "Read image for  0024 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0067 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0265 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0154 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0221 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0148 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0086 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0143 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 98)\n",
            "Read image for  0056 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0189 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0258 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0237 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0195 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0285 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0018 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0274 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0057 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0049 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 114)\n",
            "Read image for  0242 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0161 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0190 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0229 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 99)\n",
            "Read image for  0060 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0053 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0107 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0277 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 64)\n",
            "Read image for  0034 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0147 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0288 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0112 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0224 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 18)\n",
            "Read image for  0284 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0269 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0096 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0203 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0266 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0127 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0225 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0214 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0291 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0126 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0082 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0103 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0243 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0213 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 83)\n",
            "Read image for  0157 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0070 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0090 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0178 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0046 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0042 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0017 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0097 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0188 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0200 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0252 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0072 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0073 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0068 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0155 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 79)\n",
            "Read image for  0106 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0289 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0163 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0121 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0088 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0272 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0215 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0297 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0087 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0293 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0286 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 99)\n",
            "Read image for  0153 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 71)\n",
            "Read image for  0022 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0227 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0010 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0117 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 27)\n",
            "Read image for  0187 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0216 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0164 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0084 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0137 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0062 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0238 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0035 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0223 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0219 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0122 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0180 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0146 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0099 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0159 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 18)\n",
            "Read image for  0296 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0208 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0059 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0158 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 83)\n",
            "Read image for  0175 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0241 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0256 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0244 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 98)\n",
            "Read image for  0306 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0071 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 88)\n",
            "Read image for  0044 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0270 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0240 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0156 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 18)\n",
            "Read image for  0261 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 83)\n",
            "Read image for  0116 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0282 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0105 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0186 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0119 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0118 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0232 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0102 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0020 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0011 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0009 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0021 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0110 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0115 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0064 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0149 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0171 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0039 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0179 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 123)\n",
            "Read image for  0222 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0076 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0134 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0303 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0177 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0048 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0302 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0220 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0263 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 83)\n",
            "Read image for  0083 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0065 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0280 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 67)\n",
            "Read image for  0245 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0015 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0031 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 81)\n",
            "Read image for  0198 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0140 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0144 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0033 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0169 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0209 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0165 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0027 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0233 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0130 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0196 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0078 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0292 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0267 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0264 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0139 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0023 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0036 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0043 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0276 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0234 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 71)\n",
            "Read image for  0069 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0136 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0173 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0003 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0025 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0120 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0184 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0132 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Val size:  81\n",
            "Read image for  0275 , Partition:  ct_seg\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shape:  (103, 103, 74)\n",
            "Read image for  0114 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0278 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0081 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 67)\n",
            "Read image for  0032 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0152 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0040 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0123 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0281 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0125 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0085 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0307 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0253 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0013 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0194 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0098 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0058 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0108 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0167 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0304 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0051 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0205 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 83)\n",
            "Read image for  0249 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 83)\n",
            "Read image for  0201 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0193 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0089 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0174 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0309 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0236 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0255 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0028 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0151 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 67)\n",
            "Read image for  0162 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 122)\n",
            "Read image for  0079 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 97)\n",
            "Read image for  0257 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0150 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0038 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0007 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0294 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0047 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0228 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0166 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0259 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0055 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0075 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0206 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0006 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 72)\n",
            "Read image for  0218 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0295 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0133 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0250 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0030 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Read image for  0181 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0235 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0308 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0192 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0052 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0212 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0290 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0001 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0207 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0287 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0191 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0008 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0246 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0301 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0202 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 83)\n",
            "Read image for  0037 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 73)\n",
            "Read image for  0251 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 99)\n",
            "Read image for  0204 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0183 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0305 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0271 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 91)\n",
            "Read image for  0138 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0135 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 90)\n",
            "Read image for  0273 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 82)\n",
            "Read image for  0004 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 66)\n",
            "Read image for  0199 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 123)\n",
            "Read image for  0239 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 75)\n",
            "Read image for  0247 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 74)\n",
            "Read image for  0080 , Partition:  ct_seg\n",
            "Shape:  (103, 103, 65)\n",
            "Data is loaded\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "run_test('ct_seg')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fNPLaGJy2PpZ",
        "outputId": "2b52a04d-e5a0-46ba-9299-b1bf5e700845"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Data is loaded, training starting\n",
            "Fold [1/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.5576787336498359 , train_acc:  0.697986577181208 ,val_loss:  0.8435406260453563 , val_acc:  0.6578947368421053\n",
            "Epoch:  0 , train_f1_score:  0.5161290168762207 , train_precision:  0.5333333611488342 ,train_recall:  0.5 , train_auc:  0.7320544719696045\n",
            "Epoch:  0 , val_f1_score:  0.43478259444236755 , val_precision:  0.625 ,val_recall:  0.3333333432674408 , val_auc:  0.5246376991271973\n",
            "Save Model, best_accuracy:  0.6578947368421053\n",
            "Epoch:  1 , train_loss:  0.48948470205847044 , train_acc:  0.8120805369127517 ,val_loss:  0.762664062513283 , val_acc:  0.7105263157894737\n",
            "Epoch:  1 , train_f1_score:  0.6410256624221802 , train_precision:  0.8333333134651184 ,train_recall:  0.5208333134651184 , train_auc:  0.7908415794372559\n",
            "Epoch:  1 , val_f1_score:  0.6206896305084229 , val_precision:  0.6428571343421936 ,val_recall:  0.6000000238418579 , val_auc:  0.7536232471466064\n",
            "Save Model, best_accuracy:  0.7105263157894737\n",
            "Epoch:  2 , train_loss:  0.44389643334414736 , train_acc:  0.8053691275167785 ,val_loss:  0.7473017192274758 , val_acc:  0.7105263157894737\n",
            "Epoch:  2 , train_f1_score:  0.6419752836227417 , train_precision:  0.7878788113594055 ,train_recall:  0.5416666865348816 , train_auc:  0.8378713130950928\n",
            "Epoch:  2 , val_f1_score:  0.6206896305084229 , val_precision:  0.6428571343421936 ,val_recall:  0.6000000238418579 , val_auc:  0.7565217018127441\n",
            "Epoch:  3 , train_loss:  0.4268377206516099 , train_acc:  0.8322147651006712 ,val_loss:  0.7695029522445255 , val_acc:  0.6842105263157895\n",
            "Epoch:  3 , train_f1_score:  0.6987951993942261 , train_precision:  0.8285714387893677 ,train_recall:  0.6041666865348816 , train_auc:  0.8477723002433777\n",
            "Epoch:  3 , val_f1_score:  0.5714285969734192 , val_precision:  0.6153846383094788 ,val_recall:  0.5333333611488342 , val_auc:  0.7449275255203247\n",
            "Epoch:  4 , train_loss:  0.41748068270923117 , train_acc:  0.825503355704698 ,val_loss:  0.7775667398704842 , val_acc:  0.6578947368421053\n",
            "Epoch:  4 , train_f1_score:  0.6976743936538696 , train_precision:  0.7894737124443054 ,train_recall:  0.625 , train_auc:  0.8533415794372559\n",
            "Epoch:  4 , val_f1_score:  0.5185185074806213 , val_precision:  0.5833333134651184 ,val_recall:  0.46666666865348816 , val_auc:  0.7536231875419617\n",
            "Epoch:  5 , train_loss:  0.4120186289538437 , train_acc:  0.8322147651006712 ,val_loss:  0.7929616856137823 , val_acc:  0.7105263157894737\n",
            "Epoch:  5 , train_f1_score:  0.7126436829566956 , train_precision:  0.7948718070983887 ,train_recall:  0.6458333134651184 , train_auc:  0.8584983348846436\n",
            "Epoch:  5 , val_f1_score:  0.5600000023841858 , val_precision:  0.699999988079071 ,val_recall:  0.46666666865348816 , val_auc:  0.7536231875419617\n",
            "Epoch:  6 , train_loss:  0.4139452598223298 , train_acc:  0.8456375838926175 ,val_loss:  0.8041668327575137 , val_acc:  0.7105263157894737\n",
            "Epoch:  6 , train_f1_score:  0.729411780834198 , train_precision:  0.837837815284729 ,train_recall:  0.6458333134651184 , train_auc:  0.8584983348846436\n",
            "Epoch:  6 , val_f1_score:  0.5600000023841858 , val_precision:  0.699999988079071 ,val_recall:  0.46666666865348816 , val_auc:  0.7420289516448975\n",
            "Epoch:  7 , train_loss:  0.4027090432034486 , train_acc:  0.8456375838926175 ,val_loss:  0.8030789079482594 , val_acc:  0.7105263157894737\n",
            "Epoch:  7 , train_f1_score:  0.729411780834198 , train_precision:  0.837837815284729 ,train_recall:  0.6458333134651184 , train_auc:  0.867161750793457\n",
            "Epoch:  7 , val_f1_score:  0.5600000023841858 , val_precision:  0.699999988079071 ,val_recall:  0.46666666865348816 , val_auc:  0.747826099395752\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7105263157894737 current_acc:  0.7105263157894737\n",
            "Fold [2/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6402172909412008 , train_acc:  0.7248322147651006 ,val_loss:  0.8055636655343207 , val_acc:  0.6578947368421053\n",
            "Epoch:  0 , train_f1_score:  0.3692307770252228 , train_precision:  0.800000011920929 ,train_recall:  0.23999999463558197 , train_auc:  0.6224242448806763\n",
            "Epoch:  0 , val_f1_score:  0.0 , val_precision:  0.0 ,val_recall:  0.0 , val_auc:  0.6061538457870483\n",
            "Save Model, best_accuracy:  0.6578947368421053\n",
            "Epoch:  1 , train_loss:  0.6139826779597558 , train_acc:  0.7046979865771812 ,val_loss:  0.48983412549683925 , val_acc:  0.7894736842105263\n",
            "Epoch:  1 , train_f1_score:  0.4883720874786377 , train_precision:  0.5833333134651184 ,train_recall:  0.41999998688697815 , train_auc:  0.7054545879364014\n",
            "Epoch:  1 , val_f1_score:  0.6000000238418579 , val_precision:  0.8571428656578064 ,val_recall:  0.4615384638309479 , val_auc:  0.8430769443511963\n",
            "Save Model, best_accuracy:  0.7894736842105263\n",
            "Epoch:  2 , train_loss:  0.5133165454604481 , train_acc:  0.7785234899328859 ,val_loss:  0.4208549286581968 , val_acc:  0.7894736842105263\n",
            "Epoch:  2 , train_f1_score:  0.6024096608161926 , train_precision:  0.7575757503509521 ,train_recall:  0.5 , train_auc:  0.7690908908843994\n",
            "Epoch:  2 , val_f1_score:  0.6000000238418579 , val_precision:  0.8571428656578064 ,val_recall:  0.4615384638309479 , val_auc:  0.9138460755348206\n",
            "Epoch:  3 , train_loss:  0.5021381113715719 , train_acc:  0.785234899328859 ,val_loss:  0.4143999065027425 , val_acc:  0.8157894736842105\n",
            "Epoch:  3 , train_f1_score:  0.6279069781303406 , train_precision:  0.75 ,train_recall:  0.5400000214576721 , train_auc:  0.7830302715301514\n",
            "Epoch:  3 , val_f1_score:  0.6666666865348816 , val_precision:  0.875 ,val_recall:  0.5384615659713745 , val_auc:  0.9169231057167053\n",
            "Save Model, best_accuracy:  0.8157894736842105\n",
            "Epoch:  4 , train_loss:  0.4937848627345399 , train_acc:  0.785234899328859 ,val_loss:  0.4036578290213488 , val_acc:  0.8157894736842105\n",
            "Epoch:  4 , train_f1_score:  0.6279069781303406 , train_precision:  0.75 ,train_recall:  0.5400000214576721 , train_auc:  0.7963635921478271\n",
            "Epoch:  4 , val_f1_score:  0.6666666865348816 , val_precision:  0.875 ,val_recall:  0.5384615659713745 , val_auc:  0.8953846096992493\n",
            "Epoch:  5 , train_loss:  0.4867832237349092 , train_acc:  0.785234899328859 ,val_loss:  0.40643332113119723 , val_acc:  0.8157894736842105\n",
            "Epoch:  5 , train_f1_score:  0.6279069781303406 , train_precision:  0.75 ,train_recall:  0.5400000214576721 , train_auc:  0.8054545521736145\n",
            "Epoch:  5 , val_f1_score:  0.6666666865348816 , val_precision:  0.875 ,val_recall:  0.5384615659713745 , val_auc:  0.892307698726654\n",
            "Epoch:  6 , train_loss:  0.4838886581113814 , train_acc:  0.7919463087248322 ,val_loss:  0.3961413025390357 , val_acc:  0.8157894736842105\n",
            "Epoch:  6 , train_f1_score:  0.6436781883239746 , train_precision:  0.7567567825317383 ,train_recall:  0.5600000023841858 , train_auc:  0.8046464920043945\n",
            "Epoch:  6 , val_f1_score:  0.6666666865348816 , val_precision:  0.875 ,val_recall:  0.5384615659713745 , val_auc:  0.9169230461120605\n",
            "Epoch:  7 , train_loss:  0.47894514784822007 , train_acc:  0.7919463087248322 ,val_loss:  0.4032698086413898 , val_acc:  0.8157894736842105\n",
            "Epoch:  7 , train_f1_score:  0.6436781883239746 , train_precision:  0.7567567825317383 ,train_recall:  0.5600000023841858 , train_auc:  0.8111111521720886\n",
            "Epoch:  7 , val_f1_score:  0.6666666865348816 , val_precision:  0.875 ,val_recall:  0.5384615659713745 , val_auc:  0.892307698726654\n",
            "Early stopping in epoch:  7 best_accuracy:  0.8157894736842105 current_acc:  0.8157894736842105\n",
            "Fold [3/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.5724257807434878 , train_acc:  0.72 ,val_loss:  0.7727814261436563 , val_acc:  0.7027027027027027\n",
            "Epoch:  0 , train_f1_score:  0.4615384638309479 , train_precision:  0.6206896305084229 ,train_recall:  0.36734694242477417 , train_auc:  0.7023640871047974\n",
            "Epoch:  0 , val_f1_score:  0.42105263471603394 , val_precision:  0.800000011920929 ,val_recall:  0.2857142984867096 , val_auc:  0.6149068474769592\n",
            "Save Model, best_accuracy:  0.7027027027027027\n",
            "Epoch:  1 , train_loss:  0.5659951438941198 , train_acc:  0.7 ,val_loss:  0.4450673639012903 , val_acc:  0.8108108108108109\n",
            "Epoch:  1 , train_f1_score:  0.4303797483444214 , train_precision:  0.5666666626930237 ,train_recall:  0.3469387888908386 , train_auc:  0.7157001495361328\n",
            "Epoch:  1 , val_f1_score:  0.7200000286102295 , val_precision:  0.8181818127632141 ,val_recall:  0.6428571343421936 , val_auc:  0.8850932121276855\n",
            "Save Model, best_accuracy:  0.8108108108108109\n",
            "Epoch:  2 , train_loss:  0.5198357010031276 , train_acc:  0.76 ,val_loss:  0.4394630049040692 , val_acc:  0.7837837837837838\n",
            "Epoch:  2 , train_f1_score:  0.5384615659713745 , train_precision:  0.7241379022598267 ,train_recall:  0.4285714328289032 , train_auc:  0.7969287037849426\n",
            "Epoch:  2 , val_f1_score:  0.6666666865348816 , val_precision:  0.800000011920929 ,val_recall:  0.5714285969734192 , val_auc:  0.8819875717163086\n",
            "Epoch:  3 , train_loss:  0.506262568909054 , train_acc:  0.76 ,val_loss:  0.4303225799010614 , val_acc:  0.7837837837837838\n",
            "Epoch:  3 , train_f1_score:  0.5263158082962036 , train_precision:  0.7407407164573669 ,train_recall:  0.40816327929496765 , train_auc:  0.8011719584465027\n",
            "Epoch:  3 , val_f1_score:  0.6666666865348816 , val_precision:  0.800000011920929 ,val_recall:  0.5714285969734192 , val_auc:  0.8819876313209534\n",
            "Epoch:  4 , train_loss:  0.49919073104547956 , train_acc:  0.7666666666666667 ,val_loss:  0.4311764841848934 , val_acc:  0.7837837837837838\n",
            "Epoch:  4 , train_f1_score:  0.5569620132446289 , train_precision:  0.7333333492279053 ,train_recall:  0.44897958636283875 , train_auc:  0.8054152727127075\n",
            "Epoch:  4 , val_f1_score:  0.6666666865348816 , val_precision:  0.800000011920929 ,val_recall:  0.5714285969734192 , val_auc:  0.8788820505142212\n",
            "Epoch:  5 , train_loss:  0.48951744709629563 , train_acc:  0.7666666666666667 ,val_loss:  0.435740580001997 , val_acc:  0.7837837837837838\n",
            "Epoch:  5 , train_f1_score:  0.5679012537002563 , train_precision:  0.71875 ,train_recall:  0.4693877696990967 , train_auc:  0.8104667663574219\n",
            "Epoch:  5 , val_f1_score:  0.6666666865348816 , val_precision:  0.800000011920929 ,val_recall:  0.5714285969734192 , val_auc:  0.8788819313049316\n",
            "Epoch:  6 , train_loss:  0.4803493215655908 , train_acc:  0.7933333333333333 ,val_loss:  0.43055918858059355 , val_acc:  0.7837837837837838\n",
            "Epoch:  6 , train_f1_score:  0.6172839403152466 , train_precision:  0.78125 ,train_recall:  0.5102040767669678 , train_auc:  0.8227924704551697\n",
            "Epoch:  6 , val_f1_score:  0.6666666865348816 , val_precision:  0.800000011920929 ,val_recall:  0.5714285969734192 , val_auc:  0.8788819909095764\n",
            "Epoch:  7 , train_loss:  0.4733557548591246 , train_acc:  0.7866666666666666 ,val_loss:  0.44824510792622696 , val_acc:  0.7837837837837838\n",
            "Epoch:  7 , train_f1_score:  0.6097561120986938 , train_precision:  0.7575757503509521 ,train_recall:  0.5102040767669678 , train_auc:  0.8276419639587402\n",
            "Epoch:  7 , val_f1_score:  0.6666666865348816 , val_precision:  0.800000011920929 ,val_recall:  0.5714285969734192 , val_auc:  0.8571428060531616\n",
            "Early stopping in epoch:  7 best_accuracy:  0.8108108108108109 current_acc:  0.7837837837837838\n",
            "Fold [4/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.5668269058080236 , train_acc:  0.72 ,val_loss:  0.8300784485968383 , val_acc:  0.7297297297297297\n",
            "Epoch:  0 , train_f1_score:  0.5531914830207825 , train_precision:  0.6190476417541504 ,train_recall:  0.5 , train_auc:  0.7339089512825012\n",
            "Epoch:  0 , val_f1_score:  0.2857142984867096 , val_precision:  0.6666666865348816 ,val_recall:  0.1818181872367859 , val_auc:  0.4755244851112366\n",
            "Save Model, best_accuracy:  0.7297297297297297\n",
            "Epoch:  1 , train_loss:  0.5651083665830083 , train_acc:  0.76 ,val_loss:  0.4372631930997847 , val_acc:  0.8378378378378378\n",
            "Epoch:  1 , train_f1_score:  0.5714285969734192 , train_precision:  0.75 ,train_recall:  0.4615384638309479 , train_auc:  0.7337127327919006\n",
            "Epoch:  1 , val_f1_score:  0.6666666865348816 , val_precision:  0.8571428656578064 ,val_recall:  0.5454545617103577 , val_auc:  0.8706294298171997\n",
            "Save Model, best_accuracy:  0.8378378378378378\n",
            "Epoch:  2 , train_loss:  0.5043266934823866 , train_acc:  0.78 ,val_loss:  0.42877534766228415 , val_acc:  0.8378378378378378\n",
            "Epoch:  2 , train_f1_score:  0.6117647290229797 , train_precision:  0.7878788113594055 ,train_recall:  0.5 , train_auc:  0.7998429536819458\n",
            "Epoch:  2 , val_f1_score:  0.6666666865348816 , val_precision:  0.8571428656578064 ,val_recall:  0.5454545617103577 , val_auc:  0.8776224255561829\n",
            "Epoch:  3 , train_loss:  0.4918660796361898 , train_acc:  0.7933333333333333 ,val_loss:  0.42763498353746693 , val_acc:  0.8378378378378378\n",
            "Epoch:  3 , train_f1_score:  0.6516854166984558 , train_precision:  0.7837837934494019 ,train_recall:  0.557692289352417 , train_auc:  0.8104395866394043\n",
            "Epoch:  3 , val_f1_score:  0.6666666865348816 , val_precision:  0.8571428656578064 ,val_recall:  0.5454545617103577 , val_auc:  0.8741258382797241\n",
            "Epoch:  4 , train_loss:  0.4834874129978319 , train_acc:  0.7866666666666666 ,val_loss:  0.41952291532495134 , val_acc:  0.8378378378378378\n",
            "Epoch:  4 , train_f1_score:  0.644444465637207 , train_precision:  0.7631579041481018 ,train_recall:  0.557692289352417 , train_auc:  0.8208398818969727\n",
            "Epoch:  4 , val_f1_score:  0.6666666865348816 , val_precision:  0.8571428656578064 ,val_recall:  0.5454545617103577 , val_auc:  0.881118893623352\n",
            "Epoch:  5 , train_loss:  0.47526067431938523 , train_acc:  0.7866666666666666 ,val_loss:  0.4142537119112456 , val_acc:  0.8378378378378378\n",
            "Epoch:  5 , train_f1_score:  0.6363636255264282 , train_precision:  0.7777777910232544 ,train_recall:  0.5384615659713745 , train_auc:  0.8284929394721985\n",
            "Epoch:  5 , val_f1_score:  0.6666666865348816 , val_precision:  0.8571428656578064 ,val_recall:  0.5454545617103577 , val_auc:  0.881118893623352\n",
            "Epoch:  6 , train_loss:  0.47049856050560873 , train_acc:  0.7866666666666666 ,val_loss:  0.4142620886260694 , val_acc:  0.8378378378378378\n",
            "Epoch:  6 , train_f1_score:  0.644444465637207 , train_precision:  0.7631579041481018 ,train_recall:  0.557692289352417 , train_auc:  0.8322213292121887\n",
            "Epoch:  6 , val_f1_score:  0.6666666865348816 , val_precision:  0.8571428656578064 ,val_recall:  0.5454545617103577 , val_auc:  0.884615421295166\n",
            "Epoch:  7 , train_loss:  0.4630762113382419 , train_acc:  0.7933333333333333 ,val_loss:  0.409790236679082 , val_acc:  0.8378378378378378\n",
            "Epoch:  7 , train_f1_score:  0.6516854166984558 , train_precision:  0.7837837934494019 ,train_recall:  0.557692289352417 , train_auc:  0.8385008573532104\n",
            "Epoch:  7 , val_f1_score:  0.6666666865348816 , val_precision:  0.8571428656578064 ,val_recall:  0.5454545617103577 , val_auc:  0.881118893623352\n",
            "Early stopping in epoch:  7 best_accuracy:  0.8378378378378378 current_acc:  0.8378378378378378\n",
            "Fold [5/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6841300314106532 , train_acc:  0.6333333333333333 ,val_loss:  0.5611121763248701 , val_acc:  0.7837837837837838\n",
            "Epoch:  0 , train_f1_score:  0.3820224702358246 , train_precision:  0.4722222089767456 ,train_recall:  0.3207547068595886 , train_auc:  0.576541543006897\n",
            "Epoch:  0 , val_f1_score:  0.4285714328289032 , val_precision:  0.75 ,val_recall:  0.30000001192092896 , val_auc:  0.6592592597007751\n",
            "Save Model, best_accuracy:  0.7837837837837838\n",
            "Epoch:  1 , train_loss:  0.5471036912980587 , train_acc:  0.7266666666666667 ,val_loss:  0.5058974769671221 , val_acc:  0.8108108108108109\n",
            "Epoch:  1 , train_f1_score:  0.5494505763053894 , train_precision:  0.6578947305679321 ,train_recall:  0.4716981053352356 , train_auc:  0.73293137550354\n",
            "Epoch:  1 , val_f1_score:  0.5882353186607361 , val_precision:  0.7142857313156128 ,val_recall:  0.5 , val_auc:  0.7740740776062012\n",
            "Save Model, best_accuracy:  0.8108108108108109\n",
            "Epoch:  2 , train_loss:  0.47227675567352284 , train_acc:  0.8 ,val_loss:  0.5037295690460785 , val_acc:  0.8108108108108109\n",
            "Epoch:  2 , train_f1_score:  0.6666666865348816 , train_precision:  0.8108108043670654 ,train_recall:  0.5660377144813538 , train_auc:  0.8276599645614624\n",
            "Epoch:  2 , val_f1_score:  0.5882353186607361 , val_precision:  0.7142857313156128 ,val_recall:  0.5 , val_auc:  0.7666666507720947\n",
            "Epoch:  3 , train_loss:  0.4691331162730815 , train_acc:  0.7866666666666666 ,val_loss:  0.5002196630513346 , val_acc:  0.8108108108108109\n",
            "Epoch:  3 , train_f1_score:  0.6521739363670349 , train_precision:  0.7692307829856873 ,train_recall:  0.5660377144813538 , train_auc:  0.8471115231513977\n",
            "Epoch:  3 , val_f1_score:  0.5882353186607361 , val_precision:  0.7142857313156128 ,val_recall:  0.5 , val_auc:  0.770370364189148\n",
            "Epoch:  4 , train_loss:  0.4485471191695782 , train_acc:  0.8066666666666666 ,val_loss:  0.5093325975577574 , val_acc:  0.8108108108108109\n",
            "Epoch:  4 , train_f1_score:  0.6881720423698425 , train_precision:  0.800000011920929 ,train_recall:  0.6037735939025879 , train_auc:  0.8519742488861084\n",
            "Epoch:  4 , val_f1_score:  0.5882353186607361 , val_precision:  0.7142857313156128 ,val_recall:  0.5 , val_auc:  0.7555555105209351\n",
            "Epoch:  5 , train_loss:  0.4387883242487525 , train_acc:  0.8066666666666666 ,val_loss:  0.5167281375442808 , val_acc:  0.8108108108108109\n",
            "Epoch:  5 , train_f1_score:  0.6881720423698425 , train_precision:  0.800000011920929 ,train_recall:  0.6037735939025879 , train_auc:  0.8558646440505981\n",
            "Epoch:  5 , val_f1_score:  0.5882353186607361 , val_precision:  0.7142857313156128 ,val_recall:  0.5 , val_auc:  0.7518517971038818\n",
            "Epoch:  6 , train_loss:  0.42429910832357864 , train_acc:  0.8066666666666666 ,val_loss:  0.5277041389632064 , val_acc:  0.7837837837837838\n",
            "Epoch:  6 , train_f1_score:  0.6881720423698425 , train_precision:  0.800000011920929 ,train_recall:  0.6037735939025879 , train_auc:  0.8688970804214478\n",
            "Epoch:  6 , val_f1_score:  0.5555555820465088 , val_precision:  0.625 ,val_recall:  0.5 , val_auc:  0.7444444298744202\n",
            "Epoch:  7 , train_loss:  0.41059382466418054 , train_acc:  0.8133333333333334 ,val_loss:  0.5223455176361509 , val_acc:  0.7837837837837838\n",
            "Epoch:  7 , train_f1_score:  0.7083333134651184 , train_precision:  0.7906976938247681 ,train_recall:  0.6415094137191772 , train_auc:  0.8757051229476929\n",
            "Epoch:  7 , val_f1_score:  0.5555555820465088 , val_precision:  0.625 ,val_recall:  0.5 , val_auc:  0.7518518567085266\n",
            "Early stopping in epoch:  7 best_accuracy:  0.8108108108108109 current_acc:  0.7837837837837838\n",
            "F1 Score Mean:  0.6002953640052251 , F1 Score Std:  0.13956283545645917 , Precision Mean:  0.7500216484069824 ,Precision Std:  0.15550189237777887\n",
            "Recall Mean:  0.5116474100521633 Recall Std:  0.1377273773340029 AUC Mean:  0.7826864458443404 AUC Std:  0.05446064232762959\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "run_test('pet_seg')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NbaZTAjDLtw1",
        "outputId": "7aa5b5b3-9ea6-4f3e-c7ad-96175d9a28dc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Data is loaded, training starting\n",
            "Fold [1/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.5973182066725602 , train_acc:  0.7516778523489933 ,val_loss:  0.6723664538248589 , val_acc:  0.6842105263157895\n",
            "Epoch:  0 , train_f1_score:  0.602150559425354 , train_precision:  0.6829268336296082 ,train_recall:  0.5384615659713745 , train_auc:  0.7242268323898315\n",
            "Epoch:  0 , val_f1_score:  0.3333333432674408 , val_precision:  0.4285714328289032 ,val_recall:  0.27272728085517883 , val_auc:  0.6666666865348816\n",
            "Save Model, best_accuracy:  0.6842105263157895\n",
            "Epoch:  1 , train_loss:  0.5320723220869331 , train_acc:  0.7785234899328859 ,val_loss:  0.6320238262460885 , val_acc:  0.7631578947368421\n",
            "Epoch:  1 , train_f1_score:  0.6206896305084229 , train_precision:  0.7714285850524902 ,train_recall:  0.5192307829856873 , train_auc:  0.7910388708114624\n",
            "Epoch:  1 , val_f1_score:  0.5263158082962036 , val_precision:  0.625 ,val_recall:  0.4545454680919647 , val_auc:  0.6835017204284668\n",
            "Save Model, best_accuracy:  0.7631578947368421\n",
            "Epoch:  2 , train_loss:  0.4832581398680179 , train_acc:  0.7986577181208053 ,val_loss:  0.6294423889745591 , val_acc:  0.7631578947368421\n",
            "Epoch:  2 , train_f1_score:  0.6511628031730652 , train_precision:  0.8235294222831726 ,train_recall:  0.5384615659713745 , train_auc:  0.8453608751296997\n",
            "Epoch:  2 , val_f1_score:  0.5263158082962036 , val_precision:  0.625 ,val_recall:  0.4545454680919647 , val_auc:  0.6902357339859009\n",
            "Epoch:  3 , train_loss:  0.4656862033154278 , train_acc:  0.7986577181208053 ,val_loss:  0.6343990065184986 , val_acc:  0.7631578947368421\n",
            "Epoch:  3 , train_f1_score:  0.6511628031730652 , train_precision:  0.8235294222831726 ,train_recall:  0.5384615659713745 , train_auc:  0.8578509092330933\n",
            "Epoch:  3 , val_f1_score:  0.5263158082962036 , val_precision:  0.625 ,val_recall:  0.4545454680919647 , val_auc:  0.6936026811599731\n",
            "Epoch:  4 , train_loss:  0.4459641144702016 , train_acc:  0.7986577181208053 ,val_loss:  0.635034804158893 , val_acc:  0.7631578947368421\n",
            "Epoch:  4 , train_f1_score:  0.6511628031730652 , train_precision:  0.8235294222831726 ,train_recall:  0.5384615659713745 , train_auc:  0.8752973079681396\n",
            "Epoch:  4 , val_f1_score:  0.5263158082962036 , val_precision:  0.625 ,val_recall:  0.4545454680919647 , val_auc:  0.6868686676025391\n",
            "Epoch:  5 , train_loss:  0.4157587086543273 , train_acc:  0.8120805369127517 ,val_loss:  0.6390029261957878 , val_acc:  0.7894736842105263\n",
            "Epoch:  5 , train_f1_score:  0.6818181872367859 , train_precision:  0.8333333134651184 ,train_recall:  0.5769230723381042 , train_auc:  0.8996827602386475\n",
            "Epoch:  5 , val_f1_score:  0.6000000238418579 , val_precision:  0.6666666865348816 ,val_recall:  0.5454545617103577 , val_auc:  0.6902356743812561\n",
            "Save Model, best_accuracy:  0.7894736842105263\n",
            "Epoch:  6 , train_loss:  0.37310129505120787 , train_acc:  0.8590604026845637 ,val_loss:  0.6648541883279042 , val_acc:  0.7631578947368421\n",
            "Epoch:  6 , train_f1_score:  0.7789473533630371 , train_precision:  0.8604651093482971 ,train_recall:  0.7115384340286255 , train_auc:  0.9234734177589417\n",
            "Epoch:  6 , val_f1_score:  0.5714285969734192 , val_precision:  0.6000000238418579 ,val_recall:  0.5454545617103577 , val_auc:  0.6902356743812561\n",
            "Epoch:  7 , train_loss:  0.3192024566572601 , train_acc:  0.87248322147651 ,val_loss:  0.7537631135922933 , val_acc:  0.6578947368421053\n",
            "Epoch:  7 , train_f1_score:  0.8118811845779419 , train_precision:  0.8367347121238708 ,train_recall:  0.7884615659713745 , train_auc:  0.9419112205505371\n",
            "Epoch:  7 , val_f1_score:  0.5185185074806213 , val_precision:  0.4375 ,val_recall:  0.6363636255264282 , val_auc:  0.6700336933135986\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7894736842105263 current_acc:  0.6578947368421053\n",
            "Fold [2/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.6185974053555796 , train_acc:  0.6711409395973155 ,val_loss:  0.5891242491412222 , val_acc:  0.7631578947368421\n",
            "Epoch:  0 , train_f1_score:  0.4842105209827423 , train_precision:  0.5111111402511597 ,train_recall:  0.46000000834465027 , train_auc:  0.6604040265083313\n",
            "Epoch:  0 , val_f1_score:  0.47058823704719543 , val_precision:  1.0 ,val_recall:  0.3076923191547394 , val_auc:  0.6338461637496948\n",
            "Save Model, best_accuracy:  0.7631578947368421\n",
            "Epoch:  1 , train_loss:  0.5469040681198615 , train_acc:  0.697986577181208 ,val_loss:  0.4588180837101965 , val_acc:  0.7368421052631579\n",
            "Epoch:  1 , train_f1_score:  0.4444444477558136 , train_precision:  0.5806451439857483 ,train_recall:  0.36000001430511475 , train_auc:  0.762222170829773\n",
            "Epoch:  1 , val_f1_score:  0.4444444477558136 , val_precision:  0.800000011920929 ,val_recall:  0.3076923191547394 , val_auc:  0.873846173286438\n",
            "Epoch:  2 , train_loss:  0.5124180220987885 , train_acc:  0.7785234899328859 ,val_loss:  0.447525216202944 , val_acc:  0.7631578947368421\n",
            "Epoch:  2 , train_f1_score:  0.5925925970077515 , train_precision:  0.774193525314331 ,train_recall:  0.47999998927116394 , train_auc:  0.803636372089386\n",
            "Epoch:  2 , val_f1_score:  0.5263158082962036 , val_precision:  0.8333333134651184 ,val_recall:  0.38461539149284363 , val_auc:  0.8830769062042236\n",
            "Epoch:  3 , train_loss:  0.49636795254730487 , train_acc:  0.7785234899328859 ,val_loss:  0.43410957326475336 , val_acc:  0.7631578947368421\n",
            "Epoch:  3 , train_f1_score:  0.6024096608161926 , train_precision:  0.7575757503509521 ,train_recall:  0.5 , train_auc:  0.8123232126235962\n",
            "Epoch:  3 , val_f1_score:  0.5263158082962036 , val_precision:  0.8333333134651184 ,val_recall:  0.38461539149284363 , val_auc:  0.8923077583312988\n",
            "Epoch:  4 , train_loss:  0.482923463363538 , train_acc:  0.8053691275167785 ,val_loss:  0.4344062595842651 , val_acc:  0.7631578947368421\n",
            "Epoch:  4 , train_f1_score:  0.658823549747467 , train_precision:  0.800000011920929 ,train_recall:  0.5600000023841858 , train_auc:  0.8248485326766968\n",
            "Epoch:  4 , val_f1_score:  0.5263158082962036 , val_precision:  0.8333333134651184 ,val_recall:  0.38461539149284363 , val_auc:  0.8984615802764893\n",
            "Epoch:  5 , train_loss:  0.4723335121486386 , train_acc:  0.8053691275167785 ,val_loss:  0.4321496834561817 , val_acc:  0.7631578947368421\n",
            "Epoch:  5 , train_f1_score:  0.658823549747467 , train_precision:  0.800000011920929 ,train_recall:  0.5600000023841858 , train_auc:  0.8333333730697632\n",
            "Epoch:  5 , val_f1_score:  0.5263158082962036 , val_precision:  0.8333333134651184 ,val_recall:  0.38461539149284363 , val_auc:  0.9015384912490845\n",
            "Epoch:  6 , train_loss:  0.4571253329202151 , train_acc:  0.8187919463087249 ,val_loss:  0.4314887323049142 , val_acc:  0.7631578947368421\n",
            "Epoch:  6 , train_f1_score:  0.6896551847457886 , train_precision:  0.8108108043670654 ,train_recall:  0.6000000238418579 , train_auc:  0.8470706939697266\n",
            "Epoch:  6 , val_f1_score:  0.5263158082962036 , val_precision:  0.8333333134651184 ,val_recall:  0.38461539149284363 , val_auc:  0.9015384912490845\n",
            "Epoch:  7 , train_loss:  0.43511362223690236 , train_acc:  0.825503355704698 ,val_loss:  0.4202819996832037 , val_acc:  0.7631578947368421\n",
            "Epoch:  7 , train_f1_score:  0.7111111283302307 , train_precision:  0.800000011920929 ,train_recall:  0.6399999856948853 , train_auc:  0.8626262545585632\n",
            "Epoch:  7 , val_f1_score:  0.5263158082962036 , val_precision:  0.8333333134651184 ,val_recall:  0.38461539149284363 , val_auc:  0.9046154022216797\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7631578947368421 current_acc:  0.7631578947368421\n",
            "Fold [3/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.695146221816655 , train_acc:  0.6866666666666666 ,val_loss:  1.0505929301316674 , val_acc:  0.7837837837837838\n",
            "Epoch:  0 , train_f1_score:  0.2769230902194977 , train_precision:  0.692307710647583 ,train_recall:  0.17307692766189575 , train_auc:  0.46114596724510193\n",
            "Epoch:  0 , val_f1_score:  0.5 , val_precision:  0.800000011920929 ,val_recall:  0.3636363744735718 , val_auc:  0.7027971744537354\n",
            "Save Model, best_accuracy:  0.7837837837837838\n",
            "Epoch:  1 , train_loss:  0.5992316287507613 , train_acc:  0.66 ,val_loss:  0.7528302147795723 , val_acc:  0.8918918918918919\n",
            "Epoch:  1 , train_f1_score:  0.42696627974510193 , train_precision:  0.5135135054588318 ,train_recall:  0.36538460850715637 , train_auc:  0.6819073557853699\n",
            "Epoch:  1 , val_f1_score:  0.8181818127632141 , val_precision:  0.8181818127632141 ,val_recall:  0.8181818127632141 , val_auc:  0.8636363744735718\n",
            "Save Model, best_accuracy:  0.8918918918918919\n",
            "Epoch:  2 , train_loss:  0.5344073911973586 , train_acc:  0.7533333333333333 ,val_loss:  0.6467842704528389 , val_acc:  0.8918918918918919\n",
            "Epoch:  2 , train_f1_score:  0.5647059082984924 , train_precision:  0.7272727489471436 ,train_recall:  0.4615384638309479 , train_auc:  0.7709968686103821\n",
            "Epoch:  2 , val_f1_score:  0.8181818127632141 , val_precision:  0.8181818127632141 ,val_recall:  0.8181818127632141 , val_auc:  0.8601399064064026\n",
            "Epoch:  3 , train_loss:  0.5131566454004496 , train_acc:  0.7666666666666667 ,val_loss:  0.5793403983493713 , val_acc:  0.8918918918918919\n",
            "Epoch:  3 , train_f1_score:  0.5977011322975159 , train_precision:  0.7428571581840515 ,train_recall:  0.5 , train_auc:  0.793759822845459\n",
            "Epoch:  3 , val_f1_score:  0.8181818127632141 , val_precision:  0.8181818127632141 ,val_recall:  0.8181818127632141 , val_auc:  0.8566433787345886\n",
            "Epoch:  4 , train_loss:  0.4945908405538648 , train_acc:  0.7733333333333333 ,val_loss:  0.5245192209887948 , val_acc:  0.8918918918918919\n",
            "Epoch:  4 , train_f1_score:  0.6136363744735718 , train_precision:  0.75 ,train_recall:  0.5192307829856873 , train_auc:  0.8143641948699951\n",
            "Epoch:  4 , val_f1_score:  0.8181818127632141 , val_precision:  0.8181818127632141 ,val_recall:  0.8181818127632141 , val_auc:  0.8566433787345886\n",
            "Epoch:  5 , train_loss:  0.47334568814374506 , train_acc:  0.7866666666666666 ,val_loss:  0.4745252967585583 , val_acc:  0.8918918918918919\n",
            "Epoch:  5 , train_f1_score:  0.644444465637207 , train_precision:  0.7631579041481018 ,train_recall:  0.557692289352417 , train_auc:  0.8385007381439209\n",
            "Epoch:  5 , val_f1_score:  0.8181818127632141 , val_precision:  0.8181818127632141 ,val_recall:  0.8181818127632141 , val_auc:  0.8566433787345886\n",
            "Epoch:  6 , train_loss:  0.4443525963121404 , train_acc:  0.8066666666666666 ,val_loss:  0.45712199449740554 , val_acc:  0.8918918918918919\n",
            "Epoch:  6 , train_f1_score:  0.6813187003135681 , train_precision:  0.7948718070983887 ,train_recall:  0.5961538553237915 , train_auc:  0.8634222745895386\n",
            "Epoch:  6 , val_f1_score:  0.8181818127632141 , val_precision:  0.8181818127632141 ,val_recall:  0.8181818127632141 , val_auc:  0.8531468510627747\n",
            "Epoch:  7 , train_loss:  0.4053752686300625 , train_acc:  0.84 ,val_loss:  0.4868198942292381 , val_acc:  0.8648648648648649\n",
            "Epoch:  7 , train_f1_score:  0.75 , train_precision:  0.8181818127632141 ,train_recall:  0.692307710647583 , train_auc:  0.8861852288246155\n",
            "Epoch:  7 , val_f1_score:  0.782608687877655 , val_precision:  0.75 ,val_recall:  0.8181818127632141 , val_auc:  0.8496503233909607\n",
            "Early stopping in epoch:  7 best_accuracy:  0.8918918918918919 current_acc:  0.8648648648648649\n",
            "Fold [4/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.658948892277355 , train_acc:  0.64 ,val_loss:  0.7475827104739241 , val_acc:  0.7027027027027027\n",
            "Epoch:  0 , train_f1_score:  0.20588235557079315 , train_precision:  0.3499999940395355 ,train_recall:  0.1458333283662796 , train_auc:  0.5825163125991821\n",
            "Epoch:  0 , val_f1_score:  0.4761904776096344 , val_precision:  0.8333333134651184 ,val_recall:  0.3333333432674408 , val_auc:  0.5212121605873108\n",
            "Save Model, best_accuracy:  0.7027027027027027\n",
            "Epoch:  1 , train_loss:  0.5523898199697336 , train_acc:  0.7 ,val_loss:  0.5032260392515643 , val_acc:  0.7567567567567568\n",
            "Epoch:  1 , train_f1_score:  0.41558441519737244 , train_precision:  0.5517241358757019 ,train_recall:  0.3333333432674408 , train_auc:  0.7081290483474731\n",
            "Epoch:  1 , val_f1_score:  0.6666666865348816 , val_precision:  0.75 ,val_recall:  0.6000000238418579 , val_auc:  0.836363673210144\n",
            "Save Model, best_accuracy:  0.7567567567567568\n",
            "Epoch:  2 , train_loss:  0.4918253988213837 , train_acc:  0.7733333333333333 ,val_loss:  0.49253136317270835 , val_acc:  0.7567567567567568\n",
            "Epoch:  2 , train_f1_score:  0.5526315569877625 , train_precision:  0.75 ,train_recall:  0.4375 , train_auc:  0.7984068393707275\n",
            "Epoch:  2 , val_f1_score:  0.6666666865348816 , val_precision:  0.75 ,val_recall:  0.6000000238418579 , val_auc:  0.8333333730697632\n",
            "Epoch:  3 , train_loss:  0.4773500334719817 , train_acc:  0.7866666666666666 ,val_loss:  0.4934377778271163 , val_acc:  0.7567567567567568\n",
            "Epoch:  3 , train_f1_score:  0.5897436141967773 , train_precision:  0.7666666507720947 ,train_recall:  0.4791666567325592 , train_auc:  0.8145425319671631\n",
            "Epoch:  3 , val_f1_score:  0.6666666865348816 , val_precision:  0.75 ,val_recall:  0.6000000238418579 , val_auc:  0.8424242734909058\n",
            "Epoch:  4 , train_loss:  0.4612756330582003 , train_acc:  0.8 ,val_loss:  0.48844851180911064 , val_acc:  0.7567567567567568\n",
            "Epoch:  4 , train_f1_score:  0.625 , train_precision:  0.78125 ,train_recall:  0.5208333134651184 , train_auc:  0.8292484283447266\n",
            "Epoch:  4 , val_f1_score:  0.6666666865348816 , val_precision:  0.75 ,val_recall:  0.6000000238418579 , val_auc:  0.836363673210144\n",
            "Epoch:  5 , train_loss:  0.44421031008784967 , train_acc:  0.8133333333333334 ,val_loss:  0.4892987497528461 , val_acc:  0.7567567567567568\n",
            "Epoch:  5 , train_f1_score:  0.6585366129875183 , train_precision:  0.7941176295280457 ,train_recall:  0.5625 , train_auc:  0.8435457348823547\n",
            "Epoch:  5 , val_f1_score:  0.6666666865348816 , val_precision:  0.75 ,val_recall:  0.6000000238418579 , val_auc:  0.8393939733505249\n",
            "Epoch:  6 , train_loss:  0.4279369374892364 , train_acc:  0.8066666666666666 ,val_loss:  0.4984226245433092 , val_acc:  0.7027027027027027\n",
            "Epoch:  6 , train_f1_score:  0.650602400302887 , train_precision:  0.7714285850524902 ,train_recall:  0.5625 , train_auc:  0.858047366142273\n",
            "Epoch:  6 , val_f1_score:  0.6206896305084229 , val_precision:  0.6428571343421936 ,val_recall:  0.6000000238418579 , val_auc:  0.8333333730697632\n",
            "Epoch:  7 , train_loss:  0.3956147731995831 , train_acc:  0.8333333333333334 ,val_loss:  0.5149670678719476 , val_acc:  0.7297297297297297\n",
            "Epoch:  7 , train_f1_score:  0.7058823704719543 , train_precision:  0.8108108043670654 ,train_recall:  0.625 , train_auc:  0.8792892098426819\n",
            "Epoch:  7 , val_f1_score:  0.6666666865348816 , val_precision:  0.6666666865348816 ,val_recall:  0.6666666865348816 , val_auc:  0.8424242734909058\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7567567567567568 current_acc:  0.7297297297297297\n",
            "Fold [5/5]\n",
            "learning_rate:  1e-05 , drop_rate:  0.2 , weight_decay:  1e-06 , activation:  relu , optimizer:  Adam\n",
            "Epoch:  0 , train_loss:  0.5409044082049695 , train_acc:  0.7266666666666667 ,val_loss:  0.6786474224834426 , val_acc:  0.7567567567567568\n",
            "Epoch:  0 , train_f1_score:  0.5176470875740051 , train_precision:  0.6285714507102966 ,train_recall:  0.4399999976158142 , train_auc:  0.748199999332428\n",
            "Epoch:  0 , val_f1_score:  0.5263158082962036 , val_precision:  0.8333333134651184 ,val_recall:  0.38461539149284363 , val_auc:  0.6153846383094788\n",
            "Save Model, best_accuracy:  0.7567567567567568\n",
            "Epoch:  1 , train_loss:  0.5040919947884929 , train_acc:  0.7466666666666667 ,val_loss:  0.6245664116813222 , val_acc:  0.7297297297297297\n",
            "Epoch:  1 , train_f1_score:  0.5249999761581421 , train_precision:  0.699999988079071 ,train_recall:  0.41999998688697815 , train_auc:  0.7940000295639038\n",
            "Epoch:  1 , val_f1_score:  0.4444444477558136 , val_precision:  0.800000011920929 ,val_recall:  0.3076923191547394 , val_auc:  0.8044871687889099\n",
            "Epoch:  2 , train_loss:  0.46260352467704935 , train_acc:  0.7866666666666666 ,val_loss:  0.6267228555563535 , val_acc:  0.7567567567567568\n",
            "Epoch:  2 , train_f1_score:  0.6190476417541504 , train_precision:  0.7647058963775635 ,train_recall:  0.5199999809265137 , train_auc:  0.8350000381469727\n",
            "Epoch:  2 , val_f1_score:  0.5263158082962036 , val_precision:  0.8333333134651184 ,val_recall:  0.38461539149284363 , val_auc:  0.8108974695205688\n",
            "Epoch:  3 , train_loss:  0.45067692601518516 , train_acc:  0.8066666666666666 ,val_loss:  0.6408294477009189 , val_acc:  0.7567567567567568\n",
            "Epoch:  3 , train_f1_score:  0.6666666865348816 , train_precision:  0.7837837934494019 ,train_recall:  0.5799999833106995 , train_auc:  0.8489999771118164\n",
            "Epoch:  3 , val_f1_score:  0.5263158082962036 , val_precision:  0.8333333134651184 ,val_recall:  0.38461539149284363 , val_auc:  0.8237179517745972\n",
            "Epoch:  4 , train_loss:  0.43919461658357856 , train_acc:  0.8133333333333334 ,val_loss:  0.6486048723847883 , val_acc:  0.7567567567567568\n",
            "Epoch:  4 , train_f1_score:  0.6818181872367859 , train_precision:  0.7894737124443054 ,train_recall:  0.6000000238418579 , train_auc:  0.8603999614715576\n",
            "Epoch:  4 , val_f1_score:  0.5263158082962036 , val_precision:  0.8333333134651184 ,val_recall:  0.38461539149284363 , val_auc:  0.8237179517745972\n",
            "Epoch:  5 , train_loss:  0.4164276694376167 , train_acc:  0.8066666666666666 ,val_loss:  0.6655465214941148 , val_acc:  0.7297297297297297\n",
            "Epoch:  5 , train_f1_score:  0.6741573214530945 , train_precision:  0.7692307829856873 ,train_recall:  0.6000000238418579 , train_auc:  0.8808000087738037\n",
            "Epoch:  5 , val_f1_score:  0.4444444477558136 , val_precision:  0.800000011920929 ,val_recall:  0.3076923191547394 , val_auc:  0.8269230723381042\n",
            "Epoch:  6 , train_loss:  0.3867430528840244 , train_acc:  0.84 ,val_loss:  0.6964920847424986 , val_acc:  0.7297297297297297\n",
            "Epoch:  6 , train_f1_score:  0.739130437374115 , train_precision:  0.8095238208770752 ,train_recall:  0.6800000071525574 , train_auc:  0.8976000547409058\n",
            "Epoch:  6 , val_f1_score:  0.4444444477558136 , val_precision:  0.800000011920929 ,val_recall:  0.3076923191547394 , val_auc:  0.817307710647583\n",
            "Epoch:  7 , train_loss:  0.3602220500251375 , train_acc:  0.84 ,val_loss:  0.6997045674361289 , val_acc:  0.7297297297297297\n",
            "Epoch:  7 , train_f1_score:  0.7333333492279053 , train_precision:  0.824999988079071 ,train_recall:  0.6600000262260437 , train_auc:  0.9084000587463379\n",
            "Epoch:  7 , val_f1_score:  0.4444444477558136 , val_precision:  0.800000011920929 ,val_recall:  0.3076923191547394 , val_auc:  0.807692289352417\n",
            "Early stopping in epoch:  7 best_accuracy:  0.7567567567567568 current_acc:  0.7297297297297297\n",
            "F1 Score Mean:  0.5857773261410849 , F1 Score Std:  0.12857474122464774 , Precision Mean:  0.8039332057748522 ,Precision Std:  0.13107747108226195\n",
            "Recall Mean:  0.4924009425299508 Recall Std:  0.17951603857808424 AUC Mean:  0.7809591546433653 AUC Std:  0.053605526818856816\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "run_test('mixed')"
      ],
      "metadata": {
        "id": "aQWzH4zDRfpc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "run_test('ct_mult_seg')"
      ],
      "metadata": {
        "id": "gcXcTW2SRlyJ"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "YP_6Tux9195A"
      ],
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.16 (main, Mar  8 2023, 04:29:24) \n[Clang 14.0.6 ]"
    },
    "vscode": {
      "interpreter": {
        "hash": "d4a0e24facaea50a256170e3b525c70e6fdbdb3d6190c196f8fcb54cf7441dd7"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}