PadChest_multi_N.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Padchest Unimodal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import ast\n",
    "from utils import *\n",
    "from sklearn import preprocessing, metrics\n",
    "import json\n",
    "from keras.backend import squeeze, expand_dims\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from keras.models import load_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pipeline Parameters\n",
    "createEmbeddings = False\n",
    "createFolds = True\n",
    "onlyPhysicianLabels = True\n",
    "projection = 'PA'\n",
    "workers = 3\n",
    "n_epochs = 80\n",
    "image_loc = '/gpfs/ysm/home/kl533/krauthammer_partition/padchest_2019_small/images-224/'\n",
    "fitVerbose = 0\n",
    "with open('n_splits.json') as json_file:\n",
    "    json_data = json.load(json_file)\n",
    "n_splits = json_data['n_splits']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#word2vec parameters\n",
    "w2v_params = {'vocabulary_size':10000, 'window_size':5,'filters':'','vector_dim':224}\n",
    "#parameters\n",
    "max_text_len = 200#this is the same size as the sequence length\n",
    "vocabulary_size = 10000\n",
    "embedding_output_dim = 224\n",
    "shape=(224,224,1)\n",
    "channels = shape[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#build our clean csv\n",
    "#%run -i 'python3 extract_cohort.py' '/gpfs/ysm/home/kl533/krauthammer_partition/padchest_2019_small/PADCHEST_chest_x_ray_images_labels_160K_01.02.19.csv' '/gpfs/ysm/home/kl533/krauthammer_partition/padchest_2019_small/PADCHEST_CLEAN.csv' '/gpfs/ysm/home/kl533/krauthammer_partition/padchest_2019_small/images-224/'\n",
    "#load already setup csv, with clean labels\n",
    "df = pd.read_csv('/gpfs/ysm/pi/krauthammer/kl533/padchest_2019_small/PADCHEST_CLEAN.csv')\n",
    "\n",
    "#lets only pull the labels done by physicians:\n",
    "if onlyPhysicianLabels:\n",
    "    pattern = 'Physician'\n",
    "    df = df[df['MethodLabel'] == pattern]\n",
    "\n",
    "df['Report'] = df['Report'].apply(lambda x: x if type(x) == str else '')#fix any reports that are na"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "l = df.groupby('StudyID').first()['Clean_Labels'].apply(lambda x: ast.literal_eval(x)).to_list()\n",
    "#list(set([item for sublist in l for item in sublist]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_counts = {}\n",
    "for ll in l:\n",
    "    for label in ll:\n",
    "        if label in label_counts:\n",
    "            label_counts[label] += 1\n",
    "        else:\n",
    "            label_counts[label] = 1\n",
    "label_counts['other'] = 0\n",
    "to_del = []\n",
    "for k,v in label_counts.items():\n",
    "    if v < 400:\n",
    "        label_counts['other'] += v\n",
    "        to_del.append(k)\n",
    "for key in to_del: del label_counts[key]\n",
    "#print(label_counts)\n",
    "#print(len(label_counts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'normal': 3056,\n",
       " 'cardiomegaly': 753,\n",
       " 'COPD signs': 1072,\n",
       " 'infiltrates': 964,\n",
       " 'aortic elongation': 621,\n",
       " 'thoracic cage deformation': 1028,\n",
       " 'atelectasis': 447,\n",
       " 'pneumonia': 490,\n",
       " 'air trapping': 509,\n",
       " 'vertebral degenerative changes': 743,\n",
       " 'other': 6364}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df['Final_Labels'] = df['Clean_Labels'].apply(lambda x: [t if t in label_counts.keys() else 'other' for t in ast.literal_eval(x)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_class = len(label_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "if createEmbeddings:\n",
    "    print('Section 0.5: create embeddings')\n",
    "    from w2v_keras import w2v_keras\n",
    "    w2v = w2v_keras(**w2v_params)\n",
    "    w2v.fit(df['Report'], epochs=20)\n",
    "    w2v.save_embeddings('w2v_embeddings_spanish.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build our tokenizer to pass into generators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "filtering sentences\n",
      "filtering sents and removing stopwords\n"
     ]
    }
   ],
   "source": [
    "tokenizer = Tokenizer(num_words = 10000, lower=True, filters='')#make this a param\n",
    "tokenizer.fit_on_texts(filter_sentences_spanish(df['Report'].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultiLabelBinarizer(classes=None, sparse_output=False)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlb = preprocessing.MultiLabelBinarizer()\n",
    "mlb.fit(df['Final_Labels'].to_list())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Embeddings from JSON file and map tokens to embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load embedding weights from file\n",
    "location = 'w2v_embeddings_spanish.json'\n",
    "import json\n",
    "with open(location) as f:\n",
    "    embs = json.load(f)\n",
    "    emb = json.loads(embs)\n",
    "embedding_weights = mapTokensToEmbedding(emb, tokenizer.word_index, \n",
    "                                         vocabulary_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras_applications.resnet_v2 import ResNet50V2\n",
    "from keras_applications.densenet import DenseNet121\n",
    "from keras.layers import GlobalAveragePooling2D, Dense\n",
    "from keras import Sequential\n",
    "def DenseNetModelFactory():\n",
    "    model = DenseNet121(input_shape=shape, include_top=False, weights=None,backend=keras.backend,\n",
    "        layers=keras.layers,\n",
    "        models=keras.models,\n",
    "        utils=keras.utils)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Model\n",
    "from keras.layers import Dense, Reshape, concatenate, Lambda, Average, Maximum, Add, Multiply, Concatenate, BatchNormalization, Activation, GlobalAveragePooling2D, GlobalAveragePooling1D, Flatten\n",
    "from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.engine.input_layer import Input\n",
    "from keras.optimizers import Adam\n",
    "from keras.activations import relu\n",
    "\n",
    "\n",
    "def Generate_Model(model_type = 'LateFusion'):\n",
    "    #set up our text input\n",
    "    if model_type == 'LateFusion':\n",
    "        input_text = Input(shape=(max_text_len,), name='text_input')\n",
    "        embedding = Embedding(input_length=max_text_len, input_dim=vocabulary_size, output_dim=embedding_output_dim, \n",
    "                               name='embedding_layer', trainable=False, weights=[embedding_weights])\n",
    "        text_embedding = embedding(input_text)\n",
    "        #set up our image branch\n",
    "        image_input = Input(shape, name='image_input')\n",
    "        image_branch = DenseNetModelFactory()(image_input)\n",
    "        image_branch = Flatten()(image_branch)\n",
    "        image_branch = Dense(1024, activation='relu', name='dense_layer1_image_branch')(image_branch)\n",
    "        image_branch = BatchNormalization()(image_branch)\n",
    "        image_branch = Dropout(0.5)(image_branch)        \n",
    "        image_branch = Dense(512, activation='relu', name='dense_layer2_image_branch')(image_branch)\n",
    "        image_branch = BatchNormalization()(image_branch)\n",
    "        image_branch = Dropout(0.5)(image_branch)\n",
    "        image_branch_out = Dense(num_class, activation='sigmoid', name='image_branch_output')(image_branch)\n",
    "        image_branch_model = Model(inputs=[image_input], outputs=[image_branch_out])\n",
    "        #set up text branch\n",
    "        text_branch = get_text_branch()(text_embedding)\n",
    "        text_branch = Dense(1024, activation='relu', name='dense_layer1_text_branch')(text_branch)\n",
    "        text_branch = BatchNormalization()(text_branch)\n",
    "        text_branch = Dropout(0.5)(text_branch)        \n",
    "        text_branch = Dense(512, activation='relu', name='dense_layer2_text_branch')(text_branch)\n",
    "        text_branch = BatchNormalization()(text_branch)\n",
    "        text_branch = Dropout(0.5)(text_branch)\n",
    "        text_branch_out = Dense(num_class, activation='sigmoid', name='text_branch_output')(text_branch)\n",
    "        text_branch_model = Model(inputs=[input_text], outputs=[text_branch_out])\n",
    "        outputs = [model.outputs[0] for model in [text_branch_model, image_branch_model]]\n",
    "        output = Average()(outputs)\n",
    "        classificationModel = Model(inputs=[text_branch_model.input, image_branch_model.input], outputs=[output])\n",
    "    if model_type == \"EarlyFusion\":\n",
    "        input_text = Input(shape=(max_text_len,), name='text_input')\n",
    "        embedding = Embedding(input_length=max_text_len, input_dim=vocabulary_size, output_dim=embedding_output_dim, \n",
    "                               name='embedding_layer', trainable=False, weights=[embedding_weights])\n",
    "        embedded_text = embedding(input_text)\n",
    "        #expand dims if channel = 1 as we need (height, width, channels) \n",
    "        #and a matrix with one channel only has 2 dims\n",
    "        if channels == 1:\n",
    "            expandedText = Lambda(lambda x: expand_dims(x, axis=-1))\n",
    "            embedded_text = expandedText(embedded_text)\n",
    "        else:\n",
    "            embedded_text = Lambda(embedding_3Ch)(embedded_text)\n",
    "        #set up our image input\n",
    "        image_input = Input(shape, name='image_input')\n",
    "        '''merge our data for early fusion. Essentially we are just concatenating our data together'''\n",
    "        merged = concatenate([embedded_text, image_input], axis=1, name='merged')\n",
    "        x = DenseNetModelFactory()(merged)\n",
    "        x = Flatten()(x)\n",
    "        x = Dense(1024, activation='relu', name='dense_layer1_earlyFusion')(x)\n",
    "        x = BatchNormalization()(x)\n",
    "        x = Dropout(0.5)(x)        \n",
    "        x = Dense(512, activation='relu', name='dense_layer2_earlyFusion')(x)\n",
    "        x = BatchNormalization()(x)\n",
    "        x = Dropout(0.5)(x)\n",
    "        output = Dense(num_class, activation='sigmoid', name='output')(x)\n",
    "        classificationModel = Model(inputs=[input_text, image_input], outputs=[output])\n",
    "    if model_type == \"EarlyFusion_Wide\":\n",
    "        input_text = Input(shape=(max_text_len,), name='text_input')\n",
    "        embedding = Embedding(input_length=max_text_len, input_dim=vocabulary_size, output_dim=embedding_output_dim, \n",
    "                               name='embedding_layer', trainable=False, weights=[embedding_weights])\n",
    "        embedded_text = embedding(input_text)\n",
    "        #set up our image input\n",
    "        image_input = Input(shape, name='image_input')\n",
    "        image_input_reshaped = Reshape((shape[0]*channels,shape[0]))(image_input)#224 is because we have a 224x224 image with 2 channels, 224x1 channels = 224\n",
    "        '''merge our data for early fusion. Essentially we are just concatenating our data together'''\n",
    "        merged = concatenate([embedded_text, image_input_reshaped], axis=1, name='merged')\n",
    "        x = get_text_branch()(merged)\n",
    "        x = Dense(1024, activation='relu', name='dense_layer1')(x)\n",
    "        x = BatchNormalization()(x)\n",
    "        x = Dropout(0.5)(x)        \n",
    "        x = Dense(512, activation='relu', name='dense_layer2')(x)\n",
    "        x = BatchNormalization()(x)\n",
    "        x = Dropout(0.5)(x)\n",
    "        output = Dense(num_class, activation='sigmoid', name='output')(x)\n",
    "        classificationModel = Model(inputs=[input_text, image_input], outputs=[output])        \n",
    "    if model_type == 'ModelFusion':\n",
    "        input_text = Input(shape=(max_text_len,), name='text_input')\n",
    "        embedding = Embedding(input_length=max_text_len, input_dim=vocabulary_size, output_dim=embedding_output_dim, \n",
    "                               name='embedding_layer', trainable=False, weights=[embedding_weights])\n",
    "        text_embedding = embedding(input_text)\n",
    "        image_input = Input(shape, name='image_input')\n",
    "        txt = get_text_branch()            \n",
    "        #set up our image branch\n",
    "        image_branch = DenseNetModelFactory()(image_input)\n",
    "        image_branch = Flatten()(image_branch)\n",
    "        #set up text branch\n",
    "        text_branch = (txt)(text_embedding)\n",
    "        #add some dense layers as test\n",
    "        x = concatenate([image_branch, text_branch])\n",
    "        x = Dense(1024, activation='relu', name='dense_layer1_modelFusion')(x)\n",
    "        x = BatchNormalization()(x)\n",
    "        x = Dropout(0.5)(x)        \n",
    "        x = Dense(512, activation='relu', name='dense_layer2_modelFusion')(x)\n",
    "        x = BatchNormalization()(x)\n",
    "        x = Dropout(0.5)(x)\n",
    "        #add some dense layers as test\n",
    "        output = Dense(num_class, activation='sigmoid', name='output')(x)\n",
    "        classificationModel = Model(inputs=[input_text, image_input], outputs=[output])\n",
    "    if model_type == \"text_1d\":#text_1d only\n",
    "        input_text = Input(shape=(max_text_len,), name='text_input')\n",
    "        embedding = Embedding(input_length=max_text_len, input_dim=vocabulary_size, output_dim=embedding_output_dim, \n",
    "                               name='embedding_layer', trainable=False, weights=[embedding_weights])\n",
    "        embedded_text = embedding(input_text)\n",
    "        x = get_text_branch()(embedded_text)\n",
    "        x = Dense(1024, activation='relu', name='dense_layer1_text_branch')(x)\n",
    "        x = BatchNormalization()(x)\n",
    "        x = Dropout(0.5)(x)        \n",
    "        x = Dense(512, activation='relu', name='dense_layer2_text_branch')(x)\n",
    "        x = BatchNormalization()(x)\n",
    "        x = Dropout(0.5)(x)\n",
    "        output = Dense(num_class, activation='sigmoid', name='text_output')(x)\n",
    "        classificationModel = Model(inputs=[input_text], outputs=[output])\n",
    "    if model_type == \"image\":#image only\n",
    "        image_input = Input(shape, name='image_input')\n",
    "        x = DenseNetModelFactory()(image_input)\n",
    "        x = Flatten()(x)\n",
    "        x = Dense(1024, activation='relu', name='dense_layer1_image_branch')(x)\n",
    "        x = BatchNormalization()(x)\n",
    "        x = Dropout(0.5)(x)        \n",
    "        x = Dense(512, activation='relu', name='dense_layer2_image_branch')(x)\n",
    "        x = BatchNormalization()(x)\n",
    "        x = Dropout(0.5)(x)\n",
    "        output = Dense(num_class, activation='sigmoid', name='image_output')(x)\n",
    "        classificationModel = Model(inputs=[image_input], outputs=[output])\n",
    "    return classificationModel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set up environment for Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#split into train, valid, test of 80-10-10\n",
    "if createFolds:\n",
    "    #lets split the arrays\n",
    "    splitDfs = np.array_split(df, n_splits)\n",
    "    splits = {}\n",
    "    for i, split in enumerate(splitDfs):\n",
    "        train, valid, test = np.split(split[split['Projection'] == projection].sample(frac=1.), [int(.8*len(split[split['Projection'] == projection])), int(.9*len(split[split['Projection'] == projection]))])    \n",
    "        splits[i] = {}\n",
    "        splits[i]['train'] = train.index.tolist()\n",
    "        splits[i]['valid'] = valid.index.tolist()\n",
    "        splits[i]['test'] = test.index.tolist()\n",
    "    with open('padchestFolds_{}.json'.format(projection), 'w') as fp:\n",
    "        json.dump(splits, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load folds\n",
    "with open('padchestFolds_{}.json'.format(projection), 'r') as fp:\n",
    "    data = json.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "chunkList = []\n",
    "for ix,chunk in data.items():\n",
    "    chunkList.append(chunk['train'])\n",
    "    chunkList.append(chunk['valid'])\n",
    "    chunkList.append(chunk['test'])\n",
    "for i in range(len(chunkList)):\n",
    "    for j in range(i+1, len(chunkList)):\n",
    "        assert len(set(chunkList[i]).intersection(chunkList[j])) == 0, \"You have an element from one chunk in another! Each chunk should have unique indicies\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "from sklearn.metrics import hamming_loss\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import tensorflow as tf\n",
    "def full_multi_label_metric(y_true, y_pred):\n",
    "    comp = K.equal(y_true, K.round(y_pred))\n",
    "    return K.cast(K.all(comp, axis=-1), K.floatx())\n",
    "def Hamming_loss(y_true, y_pred):\n",
    "    return K.cast(hamming_loss(K.eval(y_true), K.round(K.eval(y_pred))), K.floatx())\n",
    "def auc( y_true, y_pred ) :\n",
    "    score = tf.py_func( lambda y_true, y_pred : roc_auc_score( y_true, y_pred, average='weighted', sample_weight=None).astype('float32'),\n",
    "                        [y_true, y_pred],\n",
    "                        'float32',\n",
    "                        stateful=False,\n",
    "                        name='sklearnAUC' )\n",
    "    return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#call back AUC score\n",
    "#callback code: https://github.com/keras-team/keras/issues/10472#issuecomment-472543538\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from keras.callbacks import Callback\n",
    "class AUC(Callback):\n",
    "    \n",
    "    def __init__(self, val_data, batch_size = 8):\n",
    "        super().__init__()\n",
    "        self.validation_data = val_data\n",
    "        self.batch_size = batch_size\n",
    "    \n",
    "    def on_train_begin(self, logs={}):\n",
    "        #print(self.validation_data)\n",
    "        self.val_f1s = []\n",
    "        self.val_recalls = []\n",
    "        self.val_precisions = []\n",
    "        self._val_auc = []\n",
    "        \n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        batches = len(self.validation_data)\n",
    "        total = batches * self.batch_size\n",
    "        y_shape = len(self.validation_data.mlb.classes_)\n",
    "        val_pred = np.zeros((total, y_shape))\n",
    "        val_true = np.zeros((total, y_shape))\n",
    "        \n",
    "        for batch in range(batches):\n",
    "            if len(self.validation_data.modality) == 1:\n",
    "                xVal, yVal = self.validation_data.__getitem__(batch)\n",
    "                val_pred[batch * self.batch_size : (batch+1) * self.batch_size] = np.asarray(self.model.predict(xVal))\n",
    "            elif len(self.validation_data.modality) == 2:\n",
    "                [xValTxt, xValImg], yVal = self.validation_data.__getitem__(batch)\n",
    "                val_pred[batch * self.batch_size : (batch+1) * self.batch_size] = np.asarray(self.model.predict([xValTxt, xValImg]))\n",
    "            val_true[batch * self.batch_size : (batch+1) * self.batch_size] = yVal\n",
    "            \n",
    "        #val_pred = np.squeeze(val_pred)\n",
    "        _val_auc = roc_auc_score(val_true, val_pred, average='weighted')#something has to be fixed\n",
    "        print('Validation AUC: {}'.format(_val_auc))\n",
    "        self._val_auc.append(_val_auc)\n",
    "        \n",
    "        return"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup our Experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.optimizers import Adam\n",
    "import tensorflow as tf\n",
    "def optimizerFactory():\n",
    "    return Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, amsgrad=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "# learning rate schedule\n",
    "def step_decay(epoch):\n",
    "    initial_lrate = 0.0001\n",
    "    drop = 0.5\n",
    "    epochs_drop = 10.0\n",
    "    lrate = initial_lrate * math.pow(drop,  \n",
    "                math.floor((1+epoch)/epochs_drop))\n",
    "    print('Lowering LR to {}'.format(lrate))\n",
    "    return lrate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.callbacks import ModelCheckpoint, LearningRateScheduler\n",
    "def getCheckpointCallback(name):\n",
    "    return ModelCheckpoint(name, monitor='full_multi_label_metric', verbose=1, save_best_only=True, save_weights_only=False, mode='max', period=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5, min_lr=1e-6, verbose=1)\n",
    "#early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=15, verbose=0, mode='auto', baseline=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#scheduled learning rate\n",
    "lrate = LearningRateScheduler(step_decay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0301 03:54:51.764790 47597341553536 deprecation_wrapper.py:119] From /gpfs/ysm/project/kl533/conda_envs/dlnn/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:95: The name tf.reset_default_graph is deprecated. Please use tf.compat.v1.reset_default_graph instead.\n",
      "\n",
      "W0301 03:54:51.766256 47597341553536 deprecation_wrapper.py:119] From /gpfs/ysm/project/kl533/conda_envs/dlnn/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:98: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
      "\n",
      "W0301 03:54:51.793039 47597341553536 deprecation_wrapper.py:119] From /gpfs/ysm/project/kl533/conda_envs/dlnn/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:102: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "W0301 03:54:51.794197 47597341553536 deprecation_wrapper.py:119] From /gpfs/ysm/project/kl533/conda_envs/dlnn/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "W0301 03:54:51.801361 47597341553536 deprecation_wrapper.py:119] From /gpfs/ysm/project/kl533/conda_envs/dlnn/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================Starting Split Number 0====================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0301 03:54:53.167625 47597341553536 deprecation_wrapper.py:119] From /gpfs/ysm/project/kl533/conda_envs/dlnn/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:1834: The name tf.nn.fused_batch_norm is deprecated. Please use tf.compat.v1.nn.fused_batch_norm instead.\n",
      "\n",
      "W0301 03:54:53.225205 47597341553536 deprecation_wrapper.py:119] From /gpfs/ysm/project/kl533/conda_envs/dlnn/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3976: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
      "\n",
      "W0301 03:54:54.223280 47597341553536 deprecation_wrapper.py:119] From /gpfs/ysm/project/kl533/conda_envs/dlnn/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3980: The name tf.nn.avg_pool is deprecated. Please use tf.nn.avg_pool2d instead.\n",
      "\n",
      "W0301 03:55:09.474051 47597341553536 deprecation.py:506] From /gpfs/ysm/project/kl533/conda_envs/dlnn/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "W0301 03:55:09.592993 47597341553536 deprecation_wrapper.py:119] From /gpfs/ysm/project/kl533/conda_envs/dlnn/lib/python3.6/site-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n",
      "W0301 03:55:09.600471 47597341553536 deprecation.py:323] From /gpfs/ysm/project/kl533/conda_envs/dlnn/lib/python3.6/site-packages/tensorflow/python/ops/nn_impl.py:180: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoding Labels\n",
      "Encoding Labels\n",
      "Encoding Labels\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00001: full_multi_label_metric improved from -inf to 0.00397, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.6122184192018193\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00002: full_multi_label_metric improved from 0.00397 to 0.03419, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.6793105141616443\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00003: full_multi_label_metric improved from 0.03419 to 0.10791, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.6909555535977735\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00004: full_multi_label_metric improved from 0.10791 to 0.16361, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7247060295294036\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00005: full_multi_label_metric improved from 0.16361 to 0.18391, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7231978892292426\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00006: full_multi_label_metric improved from 0.18391 to 0.19658, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7387454038740757\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00007: full_multi_label_metric improved from 0.19658 to 0.21368, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7369716786611752\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00008: full_multi_label_metric improved from 0.21368 to 0.21459, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7380220513195725\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00009: full_multi_label_metric improved from 0.21459 to 0.22512, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7424034373374824\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00010: full_multi_label_metric improved from 0.22512 to 0.24313, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7603527704343253\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00011: full_multi_label_metric improved from 0.24313 to 0.25748, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7701235755104489\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00012: full_multi_label_metric improved from 0.25748 to 0.26190, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7574687650295854\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00013: full_multi_label_metric improved from 0.26190 to 0.27396, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7668407960730818\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00014: full_multi_label_metric improved from 0.27396 to 0.29090, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7580443376311701\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00015: full_multi_label_metric improved from 0.29090 to 0.29686, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.758910984423534\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00016: full_multi_label_metric improved from 0.29686 to 0.32799, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7430886949366626\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00017: full_multi_label_metric improved from 0.32799 to 0.34493, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7512319030154543\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00018: full_multi_label_metric improved from 0.34493 to 0.36615, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7573759883792488\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00019: full_multi_label_metric improved from 0.36615 to 0.39209, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7450298744742739\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00020: full_multi_label_metric improved from 0.39209 to 0.43681, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7326567834063815\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00021: full_multi_label_metric improved from 0.43681 to 0.46749, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.733829731519555\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00022: full_multi_label_metric improved from 0.46749 to 0.50183, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7492275484655663\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00023: full_multi_label_metric improved from 0.50183 to 0.51145, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7442803156377147\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00024: full_multi_label_metric improved from 0.51145 to 0.54197, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7498139617964324\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00025: full_multi_label_metric improved from 0.54197 to 0.56090, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7396958223463392\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00026: full_multi_label_metric improved from 0.56090 to 0.58364, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7462695601831291\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00027: full_multi_label_metric improved from 0.58364 to 0.62607, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7332082526251475\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00028: full_multi_label_metric improved from 0.62607 to 0.63858, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7366059866310622\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00029: full_multi_label_metric improved from 0.63858 to 0.66606, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7343936050905214\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00030: full_multi_label_metric improved from 0.66606 to 0.71184, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.73841679037707\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00031: full_multi_label_metric improved from 0.71184 to 0.74786, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7374621876390645\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00032: full_multi_label_metric improved from 0.74786 to 0.76221, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7360516675146173\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00033: full_multi_label_metric improved from 0.76221 to 0.77320, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7413177159541783\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00034: full_multi_label_metric improved from 0.77320 to 0.80067, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7330641058644294\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00035: full_multi_label_metric improved from 0.80067 to 0.81502, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7335248650957533\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00036: full_multi_label_metric improved from 0.81502 to 0.82372, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7329740876674585\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00037: full_multi_label_metric improved from 0.82372 to 0.84158, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7324797603936637\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00038: full_multi_label_metric improved from 0.84158 to 0.85424, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7322611685241318\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00039: full_multi_label_metric improved from 0.85424 to 0.85745, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.72499096015341\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00040: full_multi_label_metric improved from 0.85745 to 0.88141, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7374120860286926\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00041: full_multi_label_metric improved from 0.88141 to 0.89530, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.733875779010415\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00042: full_multi_label_metric improved from 0.89530 to 0.90354, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7359914604062908\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00043: full_multi_label_metric improved from 0.90354 to 0.90522, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7326516301011654\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00044: full_multi_label_metric improved from 0.90522 to 0.91071, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7331489627193455\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00045: full_multi_label_metric improved from 0.91071 to 0.92231, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7332497681679369\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00046: full_multi_label_metric did not improve from 0.92231\n",
      "Validation AUC: 0.7344904469997959\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00047: full_multi_label_metric improved from 0.92231 to 0.92872, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7319961985753533\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00048: full_multi_label_metric did not improve from 0.92872\n",
      "Validation AUC: 0.7291745769653081\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00049: full_multi_label_metric improved from 0.92872 to 0.93193, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.73114125719751\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00050: full_multi_label_metric improved from 0.93193 to 0.94093, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7330806420323518\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00051: full_multi_label_metric improved from 0.94093 to 0.94673, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7316969261079723\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00052: full_multi_label_metric did not improve from 0.94673\n",
      "Validation AUC: 0.7332557115330477\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00053: full_multi_label_metric improved from 0.94673 to 0.95131, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.733083184298705\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00054: full_multi_label_metric improved from 0.95131 to 0.95452, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7289199615233499\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00055: full_multi_label_metric did not improve from 0.95452\n",
      "Validation AUC: 0.7334864269741054\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00056: full_multi_label_metric did not improve from 0.95452\n",
      "Validation AUC: 0.7334077544314656\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00057: full_multi_label_metric improved from 0.95452 to 0.95620, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7300624405887343\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00058: full_multi_label_metric improved from 0.95620 to 0.95635, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7331866187971527\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00059: full_multi_label_metric did not improve from 0.95635\n",
      "Validation AUC: 0.7316962247658771\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00060: full_multi_label_metric improved from 0.95635 to 0.95833, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.734321101142253\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00061: full_multi_label_metric improved from 0.95833 to 0.95894, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7318307612164888\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00062: full_multi_label_metric improved from 0.95894 to 0.96169, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7300256331912579\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00063: full_multi_label_metric did not improve from 0.96169\n",
      "Validation AUC: 0.7323206565688155\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00064: full_multi_label_metric improved from 0.96169 to 0.96261, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7338189556171494\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00065: full_multi_label_metric improved from 0.96261 to 0.96398, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7309333393045695\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00066: full_multi_label_metric improved from 0.96398 to 0.96581, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7308827588406875\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00067: full_multi_label_metric did not improve from 0.96581\n",
      "Validation AUC: 0.7304907893864678\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00068: full_multi_label_metric did not improve from 0.96581\n",
      "Validation AUC: 0.7325922948624082\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00069: full_multi_label_metric improved from 0.96581 to 0.96719, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7328805906074581\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00070: full_multi_label_metric did not improve from 0.96719\n",
      "Validation AUC: 0.732729099154162\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00071: full_multi_label_metric did not improve from 0.96719\n",
      "Validation AUC: 0.7332229300586413\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00072: full_multi_label_metric improved from 0.96719 to 0.96841, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7329671263515385\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00073: full_multi_label_metric improved from 0.96841 to 0.96856, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7323537409041313\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00074: full_multi_label_metric did not improve from 0.96856\n",
      "Validation AUC: 0.7330923656304666\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00075: full_multi_label_metric did not improve from 0.96856\n",
      "Validation AUC: 0.7323310853229564\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00076: full_multi_label_metric did not improve from 0.96856\n",
      "Validation AUC: 0.7328905462417051\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00077: full_multi_label_metric improved from 0.96856 to 0.96902, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7332971855338818\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00078: full_multi_label_metric did not improve from 0.96902\n",
      "Validation AUC: 0.7323982762676315\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00079: full_multi_label_metric improved from 0.96902 to 0.97176, saving model to imageOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7329445152833114\n",
      "Lowering LR to 3.90625e-07\n",
      "\n",
      "Epoch 00080: full_multi_label_metric did not improve from 0.97176\n",
      "Validation AUC: 0.7332720265195986\n",
      "Weighted AUC score for split 0: 0.7373529337519468\n"
     ]
    }
   ],
   "source": [
    "params = {'dim': (224,224),\n",
    "              'batch_size': 8,\n",
    "              'n_channels': 1,\n",
    "              'loc': image_loc,\n",
    "              'shuffle': False,\n",
    "              'modality': ['image'],\n",
    "              'tokenizer':tokenizer,\n",
    "               'mlb': mlb}\n",
    "weightedAUCList = []\n",
    "AUCList = []\n",
    "for ix, split in data.items():\n",
    "    K.clear_session()\n",
    "    print('====================Starting Split Number {}===================='.format(ix))\n",
    "    model = Generate_Model(model_type='image')#create model\n",
    "    model.compile(optimizer=optimizerFactory(), loss='binary_crossentropy', metrics=[full_multi_label_metric])\n",
    "    \n",
    "    train_generator = padchestDataGenerator_DF(df.iloc[split['train']].reset_index(drop=True), **params)\n",
    "    valid_generator = padchestDataGenerator_DF(df.iloc[split['valid']].reset_index(drop=True), **params)\n",
    "    test_generator = padchestDataGenerator_DF(df.iloc[split['test']].reset_index(drop=True), **params)\n",
    "    \n",
    "    #checkpoint\n",
    "    filename = 'imageOnly_{}_split_{}.hdf5'.format(projection,ix)\n",
    "    checkpoint = getCheckpointCallback(filename)\n",
    "    model.fit_generator(train_generator, steps_per_epoch=len(train_generator), validation_data=valid_generator, \n",
    "                        validation_steps=len(valid_generator), workers=workers, epochs=n_epochs, verbose=fitVerbose, class_weight=train_generator.labels_weights, \n",
    "                        callbacks=[checkpoint,lrate, AUC(valid_generator)])\n",
    "    y_hat = model.predict_generator(test_generator, steps=len(test_generator), workers=workers)\n",
    "    auc = roc_auc_score(test_generator.mlb.transform(df.iloc[split['test']].iloc[:len(y_hat)]['Final_Labels']), y_hat, average=None)\n",
    "    AUCList.append(auc)\n",
    "    weighted_auc = roc_auc_score(test_generator.mlb.transform(df.iloc[split['test']].iloc[:len(y_hat)]['Final_Labels']), y_hat, average='weighted')\n",
    "    weightedAUCList.append(weighted_auc)\n",
    "    print('Weighted AUC score for split {}: {}'.format(ix,weighted_auc))\n",
    "AUC_Scores = pd.DataFrame({'Label': test_generator.mlb.classes_, 'Score': np.mean(AUCList, axis=0)}, columns=['Label', 'Score'])\n",
    "AUC_Scores = AUC_Scores.append({'Label': 'weighted_auc', 'Score':np.mean(weightedAUCList, axis=0)}, ignore_index=True)#to csv    \n",
    "AUC_Scores.to_csv('AUC_scores_imageOnly_{}'.format(projection))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"for key in sorted(train_generator.labels_count):\\n    print('{}:{}'.format(key,train_generator.labels_count[key]))\\nprint(len(train_generator.labels_count))\\nprint('====================================')\\nfor key in sorted(valid_generator.labels_count):\\n    print('{}:{}'.format(key,valid_generator.labels_count[key]))\\nprint(len(valid_generator.labels_count))\\nprint('====================================')\\nfor key in sorted(test_generator.labels_count):\\n    print('{}:{}'.format(key,test_generator.labels_count[key]))\\nprint(len(test_generator.labels_count))\""
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''for key in sorted(train_generator.labels_count):\n",
    "    print('{}:{}'.format(key,train_generator.labels_count[key]))\n",
    "print(len(train_generator.labels_count))\n",
    "print('====================================')\n",
    "for key in sorted(valid_generator.labels_count):\n",
    "    print('{}:{}'.format(key,valid_generator.labels_count[key]))\n",
    "print(len(valid_generator.labels_count))\n",
    "print('====================================')\n",
    "for key in sorted(test_generator.labels_count):\n",
    "    print('{}:{}'.format(key,test_generator.labels_count[key]))\n",
    "print(len(test_generator.labels_count))'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================Starting Split Number 0====================\n",
      "Encoding Labels\n",
      "You have passed in a tokenizer, we will just convert to sequences\n",
      "filtering sentences\n",
      "filtering sents and removing stopwords\n",
      "Encoding Labels\n",
      "You have passed in a tokenizer, we will just convert to sequences\n",
      "filtering sentences\n",
      "filtering sents and removing stopwords\n",
      "Encoding Labels\n",
      "You have passed in a tokenizer, we will just convert to sequences\n",
      "filtering sentences\n",
      "filtering sents and removing stopwords\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00001: full_multi_label_metric improved from -inf to 0.00977, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7759700067964073\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00002: full_multi_label_metric improved from 0.00977 to 0.08471, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.7988968798830886\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00003: full_multi_label_metric improved from 0.08471 to 0.24100, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.8318469922640525\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00004: full_multi_label_metric improved from 0.24100 to 0.32647, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.837703470517877\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00005: full_multi_label_metric improved from 0.32647 to 0.35226, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.8684439349031363\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00006: full_multi_label_metric improved from 0.35226 to 0.37439, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.8822740439378701\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00007: full_multi_label_metric improved from 0.37439 to 0.38233, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.8952081049173524\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00008: full_multi_label_metric improved from 0.38233 to 0.39927, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9027218664719805\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00009: full_multi_label_metric improved from 0.39927 to 0.41239, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9169977011054691\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00010: full_multi_label_metric improved from 0.41239 to 0.43132, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9209968536437728\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00011: full_multi_label_metric did not improve from 0.43132\n",
      "Validation AUC: 0.9230599541491626\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00012: full_multi_label_metric improved from 0.43132 to 0.44704, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9294407140105334\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00013: full_multi_label_metric improved from 0.44704 to 0.44872, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9333954086302519\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00014: full_multi_label_metric improved from 0.44872 to 0.46551, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9354338591595599\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00015: full_multi_label_metric improved from 0.46551 to 0.47176, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9378878437719257\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00016: full_multi_label_metric improved from 0.47176 to 0.47283, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9391430308969632\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00017: full_multi_label_metric improved from 0.47283 to 0.48214, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9409526749510293\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00018: full_multi_label_metric improved from 0.48214 to 0.49512, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9426768645505812\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00019: full_multi_label_metric improved from 0.49512 to 0.50397, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9440309699964633\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00020: full_multi_label_metric improved from 0.50397 to 0.50580, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9454560767752062\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00021: full_multi_label_metric improved from 0.50580 to 0.50809, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9468126515830715\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00022: full_multi_label_metric improved from 0.50809 to 0.51297, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9477385892609825\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00023: full_multi_label_metric improved from 0.51297 to 0.51984, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9482080626733622\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00024: full_multi_label_metric did not improve from 0.51984\n",
      "Validation AUC: 0.9483205080516154\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00025: full_multi_label_metric improved from 0.51984 to 0.52457, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9480695021115269\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00026: full_multi_label_metric improved from 0.52457 to 0.53266, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9495882174216452\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00027: full_multi_label_metric did not improve from 0.53266\n",
      "Validation AUC: 0.9502735773400399\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00028: full_multi_label_metric did not improve from 0.53266\n",
      "Validation AUC: 0.9503341005755542\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00029: full_multi_label_metric improved from 0.53266 to 0.53342, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9513047083772109\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00030: full_multi_label_metric did not improve from 0.53342\n",
      "Validation AUC: 0.9502737510708756\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00031: full_multi_label_metric improved from 0.53342 to 0.53678, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9520281639935825\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00032: full_multi_label_metric improved from 0.53678 to 0.54106, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9513486013147707\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00033: full_multi_label_metric improved from 0.54106 to 0.54121, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9525713785012859\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00034: full_multi_label_metric did not improve from 0.54121\n",
      "Validation AUC: 0.9523437229259257\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00035: full_multi_label_metric improved from 0.54121 to 0.54426, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.953622862880507\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00036: full_multi_label_metric improved from 0.54426 to 0.55098, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9536176847061654\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00037: full_multi_label_metric did not improve from 0.55098\n",
      "Validation AUC: 0.9534530089165714\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00038: full_multi_label_metric did not improve from 0.55098\n",
      "Validation AUC: 0.9544538438098819\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00039: full_multi_label_metric did not improve from 0.55098\n",
      "Validation AUC: 0.9551097590167048\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00040: full_multi_label_metric did not improve from 0.55098\n",
      "Validation AUC: 0.9550640498065318\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00041: full_multi_label_metric improved from 0.55098 to 0.55388, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9550308719249955\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00042: full_multi_label_metric did not improve from 0.55388\n",
      "Validation AUC: 0.9551579643933047\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00043: full_multi_label_metric did not improve from 0.55388\n",
      "Validation AUC: 0.9552108223739773\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00044: full_multi_label_metric did not improve from 0.55388\n",
      "Validation AUC: 0.9558445339014345\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00045: full_multi_label_metric did not improve from 0.55388\n",
      "Validation AUC: 0.9556537135233927\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00046: full_multi_label_metric did not improve from 0.55388\n",
      "Validation AUC: 0.9557530873918015\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00047: full_multi_label_metric did not improve from 0.55388\n",
      "Validation AUC: 0.9555014573296066\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00048: full_multi_label_metric did not improve from 0.55388\n",
      "Validation AUC: 0.9555908366530405\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00049: full_multi_label_metric improved from 0.55388 to 0.55693, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9554080583878821\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00050: full_multi_label_metric did not improve from 0.55693\n",
      "Validation AUC: 0.9558080249568028\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00051: full_multi_label_metric did not improve from 0.55693\n",
      "Validation AUC: 0.956426480984227\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00052: full_multi_label_metric improved from 0.55693 to 0.55907, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.956184292776006\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00053: full_multi_label_metric did not improve from 0.55907\n",
      "Validation AUC: 0.9557435684314097\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00054: full_multi_label_metric improved from 0.55907 to 0.55983, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.956087693326188\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00055: full_multi_label_metric did not improve from 0.55983\n",
      "Validation AUC: 0.9561170293247723\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00056: full_multi_label_metric improved from 0.55983 to 0.56013, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9563078095887498\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00057: full_multi_label_metric improved from 0.56013 to 0.56090, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9560815640623037\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00058: full_multi_label_metric did not improve from 0.56090\n",
      "Validation AUC: 0.956383212515499\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00059: full_multi_label_metric did not improve from 0.56090\n",
      "Validation AUC: 0.957153463934099\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00060: full_multi_label_metric did not improve from 0.56090\n",
      "Validation AUC: 0.9565462846564072\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00061: full_multi_label_metric did not improve from 0.56090\n",
      "Validation AUC: 0.9567644606054907\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00062: full_multi_label_metric improved from 0.56090 to 0.56319, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9567777292173683\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00063: full_multi_label_metric did not improve from 0.56319\n",
      "Validation AUC: 0.9562340506352227\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00064: full_multi_label_metric did not improve from 0.56319\n",
      "Validation AUC: 0.9570052211705404\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00065: full_multi_label_metric improved from 0.56319 to 0.56502, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9571275144646552\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00066: full_multi_label_metric did not improve from 0.56502\n",
      "Validation AUC: 0.9568010929791062\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00067: full_multi_label_metric improved from 0.56502 to 0.56929, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.956894660966658\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00068: full_multi_label_metric did not improve from 0.56929\n",
      "Validation AUC: 0.9572847102825207\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00069: full_multi_label_metric did not improve from 0.56929\n",
      "Validation AUC: 0.9574134015798336\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00070: full_multi_label_metric improved from 0.56929 to 0.57158, saving model to textOnly_PA_split_0.hdf5\n",
      "Validation AUC: 0.9572754182017281\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00071: full_multi_label_metric did not improve from 0.57158\n",
      "Validation AUC: 0.9571155861559573\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00072: full_multi_label_metric did not improve from 0.57158\n",
      "Validation AUC: 0.9570227862690759\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00073: full_multi_label_metric did not improve from 0.57158\n",
      "Validation AUC: 0.9565801232333477\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00074: full_multi_label_metric did not improve from 0.57158\n",
      "Validation AUC: 0.9570624581937536\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00075: full_multi_label_metric did not improve from 0.57158\n",
      "Validation AUC: 0.9573122503317187\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00076: full_multi_label_metric did not improve from 0.57158\n",
      "Validation AUC: 0.9569455821759585\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00077: full_multi_label_metric did not improve from 0.57158\n",
      "Validation AUC: 0.9572791547892588\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00078: full_multi_label_metric did not improve from 0.57158\n",
      "Validation AUC: 0.9571367935240103\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00079: full_multi_label_metric did not improve from 0.57158\n",
      "Validation AUC: 0.9568206374819362\n",
      "Lowering LR to 3.90625e-07\n",
      "\n",
      "Epoch 00080: full_multi_label_metric did not improve from 0.57158\n",
      "Validation AUC: 0.9567694028735335\n",
      "Weighted AUC score for split 0: 0.9620166410158828\n"
     ]
    }
   ],
   "source": [
    "params = {'dim': (224,224),\n",
    "              'batch_size': 8,\n",
    "              'n_channels': 1,\n",
    "              'loc': image_loc,\n",
    "              'shuffle': False,\n",
    "              'modality': ['text'],\n",
    "              'tokenizer':tokenizer,\n",
    "               'mlb': mlb}\n",
    "weightedAUCList = []\n",
    "AUCList = []\n",
    "for ix, split in data.items():\n",
    "    K.clear_session()\n",
    "    print('====================Starting Split Number {}===================='.format(ix))\n",
    "    model = Generate_Model(model_type='text_1d')#create model\n",
    "    model.compile(optimizer=optimizerFactory(), loss='binary_crossentropy', metrics=[full_multi_label_metric])\n",
    "    \n",
    "    train_generator = padchestDataGenerator_DF(df.iloc[split['train']].reset_index(drop=True), **params)\n",
    "    valid_generator = padchestDataGenerator_DF(df.iloc[split['valid']].reset_index(drop=True), **params)\n",
    "    test_generator = padchestDataGenerator_DF(df.iloc[split['test']].reset_index(drop=True), **params)\n",
    "    #checkpoint\n",
    "    filename = 'textOnly_{}_split_{}.hdf5'.format(projection,ix)\n",
    "    checkpoint = getCheckpointCallback(filename)\n",
    "    model.fit_generator(train_generator, steps_per_epoch=len(train_generator), validation_data=valid_generator, \n",
    "                        validation_steps=len(valid_generator), workers=workers, epochs=n_epochs, verbose=fitVerbose, class_weight=train_generator.labels_weights, \n",
    "                        callbacks=[checkpoint,lrate, AUC(valid_generator)])\n",
    "    y_hat = model.predict_generator(test_generator, steps=len(test_generator), workers=workers)\n",
    "    auc = roc_auc_score(test_generator.mlb.transform(df.iloc[split['test']].iloc[:len(y_hat)]['Final_Labels']), y_hat, average=None)\n",
    "    AUCList.append(auc)\n",
    "    weighted_auc = roc_auc_score(test_generator.mlb.transform(df.iloc[split['test']].iloc[:len(y_hat)]['Final_Labels']), y_hat, average='weighted')\n",
    "    weightedAUCList.append(weighted_auc)\n",
    "    print('Weighted AUC score for split {}: {}'.format(ix,weighted_auc))\n",
    "AUC_Scores = pd.DataFrame({'Label': test_generator.mlb.classes_, 'Score': np.mean(AUCList, axis=0)}, columns=['Label', 'Score'])\n",
    "AUC_Scores = AUC_Scores.append({'Label': 'weighted_auc', 'Score':np.mean(weightedAUCList, axis=0)}, ignore_index=True)#to csv    \n",
    "AUC_Scores.to_csv('AUC_scores_textOnly_{}'.format(projection))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train multimodal Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {'dim': (224,224),\n",
    "              'batch_size': 8,\n",
    "              'n_channels': 1,\n",
    "              'loc': '/gpfs/ysm/home/kl533/krauthammer_partition/padchest_2019_small/images-224/',\n",
    "              'shuffle': False,\n",
    "              'modality': ['image', 'text'],\n",
    "              'tokenizer':tokenizer,\n",
    "               'mlb': mlb}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Fusion Randomly Initialized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================Starting Split Number 0====================\n",
      "Encoding Labels\n",
      "You have passed in a tokenizer, we will just convert to sequences\n",
      "filtering sentences\n",
      "filtering sents and removing stopwords\n",
      "Encoding Labels\n",
      "You have passed in a tokenizer, we will just convert to sequences\n",
      "filtering sentences\n",
      "filtering sents and removing stopwords\n",
      "Encoding Labels\n",
      "You have passed in a tokenizer, we will just convert to sequences\n",
      "filtering sentences\n",
      "filtering sents and removing stopwords\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00001: full_multi_label_metric improved from -inf to 0.00733, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.7752638695777186\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00002: full_multi_label_metric improved from 0.00733 to 0.07845, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.8124004521313167\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00003: full_multi_label_metric improved from 0.07845 to 0.23230, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.8658039340723366\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00004: full_multi_label_metric improved from 0.23230 to 0.33578, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.8931046397383334\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00005: full_multi_label_metric improved from 0.33578 to 0.38446, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9097772640288394\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00006: full_multi_label_metric improved from 0.38446 to 0.40842, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9126348659383275\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00007: full_multi_label_metric improved from 0.40842 to 0.42933, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9287788422399975\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00008: full_multi_label_metric improved from 0.42933 to 0.44918, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9323852110618994\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00009: full_multi_label_metric improved from 0.44918 to 0.46642, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9411277686135627\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00010: full_multi_label_metric improved from 0.46642 to 0.48611, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.947164276057501\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00011: full_multi_label_metric improved from 0.48611 to 0.49847, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9493125206151509\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00012: full_multi_label_metric improved from 0.49847 to 0.51038, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9508850276509219\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00013: full_multi_label_metric improved from 0.51038 to 0.51358, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9532374687140485\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00014: full_multi_label_metric improved from 0.51358 to 0.52198, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9537078012882917\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00015: full_multi_label_metric improved from 0.52198 to 0.53388, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9546990280348397\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00016: full_multi_label_metric improved from 0.53388 to 0.54533, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.957399260718976\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00017: full_multi_label_metric improved from 0.54533 to 0.55143, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9551998304117088\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00018: full_multi_label_metric improved from 0.55143 to 0.56487, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9584045983021113\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00019: full_multi_label_metric improved from 0.56487 to 0.57005, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9590390863344243\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00020: full_multi_label_metric improved from 0.57005 to 0.59127, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9593773916869408\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00021: full_multi_label_metric improved from 0.59127 to 0.60195, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9582664693681763\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00022: full_multi_label_metric improved from 0.60195 to 0.61462, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9589726155958606\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00023: full_multi_label_metric improved from 0.61462 to 0.62531, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9590301654304765\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00024: full_multi_label_metric improved from 0.62531 to 0.64347, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9587305946810111\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00025: full_multi_label_metric improved from 0.64347 to 0.65339, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9585871128193447\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00026: full_multi_label_metric improved from 0.65339 to 0.67750, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9571758110309585\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00027: full_multi_label_metric improved from 0.67750 to 0.68788, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9573757949645136\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00028: full_multi_label_metric improved from 0.68788 to 0.71047, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9553383770063296\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00029: full_multi_label_metric improved from 0.71047 to 0.72436, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9566609228870677\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00030: full_multi_label_metric improved from 0.72436 to 0.75855, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9569887598109658\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00031: full_multi_label_metric improved from 0.75855 to 0.77991, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9555275945965374\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00032: full_multi_label_metric improved from 0.77991 to 0.79396, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9560947019268432\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00033: full_multi_label_metric improved from 0.79396 to 0.80388, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9559266131618805\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00034: full_multi_label_metric improved from 0.80388 to 0.82234, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9544641959178978\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00035: full_multi_label_metric improved from 0.82234 to 0.83578, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.955459328950431\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00036: full_multi_label_metric improved from 0.83578 to 0.84341, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9545383334209022\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00037: full_multi_label_metric improved from 0.84341 to 0.85226, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9544847437644983\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00038: full_multi_label_metric improved from 0.85226 to 0.86355, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9544268270272764\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00039: full_multi_label_metric improved from 0.86355 to 0.86844, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9540595569502668\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00040: full_multi_label_metric improved from 0.86844 to 0.88187, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9544857174523825\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00041: full_multi_label_metric improved from 0.88187 to 0.89499, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9542249524264222\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00042: full_multi_label_metric improved from 0.89499 to 0.89881, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9534266186778726\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00043: full_multi_label_metric improved from 0.89881 to 0.90400, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.953611606817576\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00044: full_multi_label_metric improved from 0.90400 to 0.90659, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9524592265761875\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00045: full_multi_label_metric improved from 0.90659 to 0.91117, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9526587851997019\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00046: full_multi_label_metric improved from 0.91117 to 0.91361, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9532514836425512\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00047: full_multi_label_metric improved from 0.91361 to 0.91682, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9535153513589706\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00048: full_multi_label_metric improved from 0.91682 to 0.92338, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9534399050394563\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00049: full_multi_label_metric improved from 0.92338 to 0.92506, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.952178499644982\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00050: full_multi_label_metric improved from 0.92506 to 0.93315, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9511556960184437\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00051: full_multi_label_metric improved from 0.93315 to 0.93636, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9519178112963335\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00052: full_multi_label_metric did not improve from 0.93636\n",
      "Validation AUC: 0.951651746441227\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00053: full_multi_label_metric improved from 0.93636 to 0.93895, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9514802845247904\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00054: full_multi_label_metric improved from 0.93895 to 0.94490, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.951051601941762\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00055: full_multi_label_metric improved from 0.94490 to 0.94505, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9506964462170963\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00056: full_multi_label_metric did not improve from 0.94505\n",
      "Validation AUC: 0.9520491134603937\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00057: full_multi_label_metric did not improve from 0.94505\n",
      "Validation AUC: 0.9518210814674567\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00058: full_multi_label_metric improved from 0.94505 to 0.94521, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9512988454130868\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00059: full_multi_label_metric did not improve from 0.94521\n",
      "Validation AUC: 0.952333811542491\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00060: full_multi_label_metric improved from 0.94521 to 0.94918, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9513791493978256\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00061: full_multi_label_metric did not improve from 0.94918\n",
      "Validation AUC: 0.9510559239274989\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00062: full_multi_label_metric did not improve from 0.94918\n",
      "Validation AUC: 0.9506611827318642\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00063: full_multi_label_metric improved from 0.94918 to 0.95513, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9509063634294264\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00064: full_multi_label_metric did not improve from 0.95513\n",
      "Validation AUC: 0.95187745691757\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00065: full_multi_label_metric did not improve from 0.95513\n",
      "Validation AUC: 0.9514947821796164\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00066: full_multi_label_metric did not improve from 0.95513\n",
      "Validation AUC: 0.9520989797165819\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00067: full_multi_label_metric did not improve from 0.95513\n",
      "Validation AUC: 0.9518794655925465\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00068: full_multi_label_metric improved from 0.95513 to 0.95650, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9515430699185295\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00069: full_multi_label_metric did not improve from 0.95650\n",
      "Validation AUC: 0.951266439755596\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00070: full_multi_label_metric did not improve from 0.95650\n",
      "Validation AUC: 0.9515130167387638\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00071: full_multi_label_metric did not improve from 0.95650\n",
      "Validation AUC: 0.9515776450274286\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00072: full_multi_label_metric did not improve from 0.95650\n",
      "Validation AUC: 0.9509514709078953\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00073: full_multi_label_metric did not improve from 0.95650\n",
      "Validation AUC: 0.9506975813996583\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00074: full_multi_label_metric improved from 0.95650 to 0.95864, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9511751741378917\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00075: full_multi_label_metric did not improve from 0.95864\n",
      "Validation AUC: 0.9507564451582134\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00076: full_multi_label_metric improved from 0.95864 to 0.95971, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9509792891391897\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00077: full_multi_label_metric improved from 0.95971 to 0.96154, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9512502927741495\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00078: full_multi_label_metric improved from 0.96154 to 0.96368, saving model to modelFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9509487620063292\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00079: full_multi_label_metric did not improve from 0.96368\n",
      "Validation AUC: 0.9513209652178006\n",
      "Lowering LR to 3.90625e-07\n",
      "\n",
      "Epoch 00080: full_multi_label_metric did not improve from 0.96368\n",
      "Validation AUC: 0.9508720981157633\n",
      "Weighted AUC score for split 0: 0.9600444319338092\n"
     ]
    }
   ],
   "source": [
    "weightedAUCList = []\n",
    "AUCList = []\n",
    "for ix, split in data.items():\n",
    "    K.clear_session()\n",
    "    print('====================Starting Split Number {}===================='.format(ix))\n",
    "    model = Generate_Model(model_type='ModelFusion')#create model\n",
    "    model.compile(optimizer=optimizerFactory(), loss='binary_crossentropy', metrics=[full_multi_label_metric])\n",
    "    \n",
    "    train_generator = padchestDataGenerator_DF(df.iloc[split['train']].reset_index(drop=True), **params)\n",
    "    valid_generator = padchestDataGenerator_DF(df.iloc[split['valid']].reset_index(drop=True), **params)\n",
    "    test_generator = padchestDataGenerator_DF(df.iloc[split['test']].reset_index(drop=True), **params)\n",
    "    #checkpoint\n",
    "    filename = 'modelFusion_{}_split_{}.hdf5'.format(projection,ix)#'imageOnly_lat.hdf5'\n",
    "    checkpoint = getCheckpointCallback(filename)\n",
    "    model.fit_generator(train_generator, steps_per_epoch=len(train_generator), validation_data=valid_generator, \n",
    "                        validation_steps=len(valid_generator), workers=workers, epochs=n_epochs, verbose=fitVerbose, class_weight=train_generator.labels_weights, \n",
    "                        callbacks=[checkpoint,lrate, AUC(valid_generator)])\n",
    "    y_hat = model.predict_generator(test_generator, steps=len(test_generator), workers=workers)\n",
    "    auc = roc_auc_score(test_generator.mlb.transform(df.iloc[split['test']].iloc[:len(y_hat)]['Final_Labels']), y_hat, average=None)\n",
    "    AUCList.append(auc)\n",
    "    weighted_auc = roc_auc_score(test_generator.mlb.transform(df.iloc[split['test']].iloc[:len(y_hat)]['Final_Labels']), y_hat, average='weighted')\n",
    "    weightedAUCList.append(weighted_auc)\n",
    "    print('Weighted AUC score for split {}: {}'.format(ix,weighted_auc))\n",
    "AUC_Scores = pd.DataFrame({'Label': test_generator.mlb.classes_, 'Score': np.mean(AUCList, axis=0)}, columns=['Label', 'Score'])\n",
    "AUC_Scores = AUC_Scores.append({'Label': 'weighted_auc', 'Score':np.mean(weightedAUCList, axis=0)}, ignore_index=True)#to csv    \n",
    "AUC_Scores.to_csv('AUC_scores_ModelFusion_{}'.format(projection))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Fusion Pretrained"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================Starting Split Number 0====================\n",
      "Encoding Labels\n",
      "You have passed in a tokenizer, we will just convert to sequences\n",
      "filtering sentences\n",
      "filtering sents and removing stopwords\n",
      "Encoding Labels\n",
      "You have passed in a tokenizer, we will just convert to sequences\n",
      "filtering sentences\n",
      "filtering sents and removing stopwords\n",
      "Encoding Labels\n",
      "You have passed in a tokenizer, we will just convert to sequences\n",
      "filtering sentences\n",
      "filtering sents and removing stopwords\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00001: full_multi_label_metric improved from -inf to 0.01389, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.890046140056836\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00002: full_multi_label_metric improved from 0.01389 to 0.10821, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9269861768123374\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00003: full_multi_label_metric improved from 0.10821 to 0.29594, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9395924202255602\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00004: full_multi_label_metric improved from 0.29594 to 0.41621, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.94198265802782\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00005: full_multi_label_metric improved from 0.41621 to 0.45940, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9494999408032851\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00006: full_multi_label_metric improved from 0.45940 to 0.48840, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9527076759556561\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00007: full_multi_label_metric improved from 0.48840 to 0.49969, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9596340711456431\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00008: full_multi_label_metric improved from 0.49969 to 0.52259, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.962717193275056\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00009: full_multi_label_metric improved from 0.52259 to 0.53495, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9624883043900896\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00010: full_multi_label_metric improved from 0.53495 to 0.56410, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.966504712591027\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00011: full_multi_label_metric improved from 0.56410 to 0.57448, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9686455580603698\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00012: full_multi_label_metric improved from 0.57448 to 0.59539, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9672705279481519\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00013: full_multi_label_metric did not improve from 0.59539\n",
      "Validation AUC: 0.9681308092747174\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00014: full_multi_label_metric improved from 0.59539 to 0.60745, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9694643981042786\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00015: full_multi_label_metric improved from 0.60745 to 0.61615, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9696251541918202\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00016: full_multi_label_metric improved from 0.61615 to 0.63431, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9682604717197789\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00017: full_multi_label_metric improved from 0.63431 to 0.64301, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.970398616963042\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00018: full_multi_label_metric improved from 0.64301 to 0.66880, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.969332828523358\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00019: full_multi_label_metric improved from 0.66880 to 0.68483, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9693215694529828\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00020: full_multi_label_metric improved from 0.68483 to 0.70864, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9712508104124856\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00021: full_multi_label_metric improved from 0.70864 to 0.73184, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9687470432665991\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00022: full_multi_label_metric improved from 0.73184 to 0.75565, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9678139612570357\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00023: full_multi_label_metric improved from 0.75565 to 0.77549, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9693468301748567\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00024: full_multi_label_metric improved from 0.77549 to 0.78861, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9681480507716869\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00025: full_multi_label_metric improved from 0.78861 to 0.80998, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.969268692701639\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00026: full_multi_label_metric improved from 0.80998 to 0.82692, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9676558244398997\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00027: full_multi_label_metric improved from 0.82692 to 0.84234, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9684398435057995\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00028: full_multi_label_metric improved from 0.84234 to 0.84875, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.967905817284226\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00029: full_multi_label_metric improved from 0.84875 to 0.86325, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9680138225863887\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00030: full_multi_label_metric improved from 0.86325 to 0.87882, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9673005359861109\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00031: full_multi_label_metric improved from 0.87882 to 0.89591, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9663725260872864\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00032: full_multi_label_metric improved from 0.89591 to 0.90171, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9669353937953755\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00033: full_multi_label_metric improved from 0.90171 to 0.90980, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9662310341538725\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00034: full_multi_label_metric improved from 0.90980 to 0.91194, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9650177548147569\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00035: full_multi_label_metric improved from 0.91194 to 0.92552, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9646767479519638\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00036: full_multi_label_metric did not improve from 0.92552\n",
      "Validation AUC: 0.9656151615933112\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00037: full_multi_label_metric improved from 0.92552 to 0.92613, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9655193897934113\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00038: full_multi_label_metric improved from 0.92613 to 0.92872, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9653917422998914\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00039: full_multi_label_metric did not improve from 0.92872\n",
      "Validation AUC: 0.9654319917937768\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00040: full_multi_label_metric improved from 0.92872 to 0.94017, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9645910977919216\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00041: full_multi_label_metric improved from 0.94017 to 0.94292, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9649666445671746\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00042: full_multi_label_metric improved from 0.94292 to 0.94521, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9649614351650394\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00043: full_multi_label_metric improved from 0.94521 to 0.95040, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9651593664366543\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00044: full_multi_label_metric improved from 0.95040 to 0.95284, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9646113225904218\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00045: full_multi_label_metric improved from 0.95284 to 0.95467, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.964423280708825\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00046: full_multi_label_metric did not improve from 0.95467\n",
      "Validation AUC: 0.9643102512778229\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00047: full_multi_label_metric did not improve from 0.95467\n",
      "Validation AUC: 0.9637767637733612\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00048: full_multi_label_metric improved from 0.95467 to 0.95498, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9640523886029688\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00049: full_multi_label_metric improved from 0.95498 to 0.95574, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9631990825142539\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00050: full_multi_label_metric improved from 0.95574 to 0.96062, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9643765354854473\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00051: full_multi_label_metric improved from 0.96062 to 0.96123, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9644357972528634\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00052: full_multi_label_metric improved from 0.96123 to 0.96459, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9628844276792979\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00053: full_multi_label_metric improved from 0.96459 to 0.96535, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9632552874518436\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00054: full_multi_label_metric did not improve from 0.96535\n",
      "Validation AUC: 0.9635614023066351\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00055: full_multi_label_metric did not improve from 0.96535\n",
      "Validation AUC: 0.9634792189526616\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00056: full_multi_label_metric improved from 0.96535 to 0.96795, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9629319915297568\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00057: full_multi_label_metric did not improve from 0.96795\n",
      "Validation AUC: 0.963939991774139\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00058: full_multi_label_metric did not improve from 0.96795\n",
      "Validation AUC: 0.963455585063285\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00059: full_multi_label_metric did not improve from 0.96795\n",
      "Validation AUC: 0.9632403142304632\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00060: full_multi_label_metric improved from 0.96795 to 0.97024, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9635123542585533\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00061: full_multi_label_metric did not improve from 0.97024\n",
      "Validation AUC: 0.9629026113014395\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00062: full_multi_label_metric did not improve from 0.97024\n",
      "Validation AUC: 0.9628553270123805\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00063: full_multi_label_metric improved from 0.97024 to 0.97131, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9628658634989947\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00064: full_multi_label_metric did not improve from 0.97131\n",
      "Validation AUC: 0.963282663752774\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00065: full_multi_label_metric did not improve from 0.97131\n",
      "Validation AUC: 0.9628498064789853\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00066: full_multi_label_metric improved from 0.97131 to 0.97207, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9632133103014489\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00067: full_multi_label_metric did not improve from 0.97207\n",
      "Validation AUC: 0.9628823351535687\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00068: full_multi_label_metric did not improve from 0.97207\n",
      "Validation AUC: 0.9629945604903882\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00069: full_multi_label_metric did not improve from 0.97207\n",
      "Validation AUC: 0.9629317708363415\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00070: full_multi_label_metric did not improve from 0.97207\n",
      "Validation AUC: 0.9627276621987169\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00071: full_multi_label_metric improved from 0.97207 to 0.97527, saving model to modelFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9633358719739161\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00072: full_multi_label_metric did not improve from 0.97527\n",
      "Validation AUC: 0.9633695746426773\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00073: full_multi_label_metric did not improve from 0.97527\n",
      "Validation AUC: 0.9627627722362934\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00074: full_multi_label_metric did not improve from 0.97527\n",
      "Validation AUC: 0.9632882463016634\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00075: full_multi_label_metric did not improve from 0.97527\n",
      "Validation AUC: 0.9630660791481843\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00076: full_multi_label_metric did not improve from 0.97527\n",
      "Validation AUC: 0.9632178452592369\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00077: full_multi_label_metric did not improve from 0.97527\n",
      "Validation AUC: 0.9633502619742859\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00078: full_multi_label_metric did not improve from 0.97527\n",
      "Validation AUC: 0.9630102686260985\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00079: full_multi_label_metric did not improve from 0.97527\n",
      "Validation AUC: 0.9626340476466874\n",
      "Lowering LR to 3.90625e-07\n",
      "\n",
      "Epoch 00080: full_multi_label_metric did not improve from 0.97527\n",
      "Validation AUC: 0.9628600732333337\n",
      "Weighted AUC score for split 0: 0.9677683062930118\n"
     ]
    }
   ],
   "source": [
    "weightedAUCList = []\n",
    "AUCList = []\n",
    "for ix, split in data.items():\n",
    "    K.clear_session()\n",
    "    print('====================Starting Split Number {}===================='.format(ix))\n",
    "    model = Generate_Model(model_type='ModelFusion')#create model\n",
    "    model.compile(optimizer=optimizerFactory(), loss='binary_crossentropy', metrics=[full_multi_label_metric])\n",
    "    \n",
    "    train_generator = padchestDataGenerator_DF(df.iloc[split['train']].reset_index(drop=True), **params)\n",
    "    valid_generator = padchestDataGenerator_DF(df.iloc[split['valid']].reset_index(drop=True), **params)\n",
    "    test_generator = padchestDataGenerator_DF(df.iloc[split['test']].reset_index(drop=True), **params)\n",
    "    #checkpoint\n",
    "    filename = 'modelFusion_PT_{}_split_{}.hdf5'.format(projection,ix)\n",
    "    checkpoint = getCheckpointCallback(filename)\n",
    "    \n",
    "    #Load weights\n",
    "    model.layers[2].set_weights(load_model('imageOnly_{}_split_{}.hdf5'.format(projection,ix),compile=False).layers[1].get_weights())#image\n",
    "    model.layers[5].set_weights(load_model('textOnly_{}_split_{}.hdf5'.format(projection,ix),compile=False).layers[2].get_weights())#text\n",
    "    \n",
    "    #Fit Model\n",
    "    model.fit_generator(train_generator, steps_per_epoch=len(train_generator), validation_data=valid_generator, \n",
    "                        validation_steps=len(valid_generator), workers=workers, epochs=n_epochs, verbose=fitVerbose, class_weight=train_generator.labels_weights, \n",
    "                        callbacks=[checkpoint,lrate, AUC(valid_generator)])\n",
    "    y_hat = model.predict_generator(test_generator, steps=len(test_generator), workers=workers)\n",
    "    auc = roc_auc_score(test_generator.mlb.transform(df.iloc[split['test']].iloc[:len(y_hat)]['Final_Labels']), y_hat, average=None)\n",
    "    AUCList.append(auc)\n",
    "    weighted_auc = roc_auc_score(test_generator.mlb.transform(df.iloc[split['test']].iloc[:len(y_hat)]['Final_Labels']), y_hat, average='weighted')\n",
    "    weightedAUCList.append(weighted_auc)\n",
    "    print('Weighted AUC score for split {}: {}'.format(ix,weighted_auc))\n",
    "AUC_Scores = pd.DataFrame({'Label': test_generator.mlb.classes_, 'Score': np.mean(AUCList, axis=0)}, columns=['Label', 'Score'])\n",
    "AUC_Scores = AUC_Scores.append({'Label': 'weighted_auc', 'Score':np.mean(weightedAUCList, axis=0)}, ignore_index=True)#to csv    \n",
    "AUC_Scores.to_csv('AUC_scores_ModelFusion_PT_{}'.format(projection))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Late Fusion Randomly Initialized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================Starting Split Number 0====================\n",
      "Encoding Labels\n",
      "You have passed in a tokenizer, we will just convert to sequences\n",
      "filtering sentences\n",
      "filtering sents and removing stopwords\n",
      "Encoding Labels\n",
      "You have passed in a tokenizer, we will just convert to sequences\n",
      "filtering sentences\n",
      "filtering sents and removing stopwords\n",
      "Encoding Labels\n",
      "You have passed in a tokenizer, we will just convert to sequences\n",
      "filtering sentences\n",
      "filtering sents and removing stopwords\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00001: full_multi_label_metric improved from -inf to 0.04197, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.7785030824222612\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00002: full_multi_label_metric improved from 0.04197 to 0.18147, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.8096275297050022\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00003: full_multi_label_metric improved from 0.18147 to 0.29197, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.8512563564724381\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00004: full_multi_label_metric improved from 0.29197 to 0.34966, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.8791874469868397\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00005: full_multi_label_metric improved from 0.34966 to 0.38538, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.8977179983988232\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00006: full_multi_label_metric improved from 0.38538 to 0.39789, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9077679488781341\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00007: full_multi_label_metric improved from 0.39789 to 0.42247, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9185581042322755\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00008: full_multi_label_metric improved from 0.42247 to 0.43101, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9228397754675746\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00009: full_multi_label_metric improved from 0.43101 to 0.44109, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9312705256923707\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00010: full_multi_label_metric improved from 0.44109 to 0.45742, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9350712042046013\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00011: full_multi_label_metric improved from 0.45742 to 0.46016, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9354748627786265\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00012: full_multi_label_metric improved from 0.46016 to 0.46688, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9344239589557792\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00013: full_multi_label_metric improved from 0.46688 to 0.46749, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9372665697298247\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00014: full_multi_label_metric improved from 0.46749 to 0.47405, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9392218488063191\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00015: full_multi_label_metric improved from 0.47405 to 0.47497, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9399767366999667\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00016: full_multi_label_metric improved from 0.47497 to 0.48947, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9382392890298934\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00017: full_multi_label_metric did not improve from 0.48947\n",
      "Validation AUC: 0.9424542907464163\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00018: full_multi_label_metric improved from 0.48947 to 0.49206, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9386180375156868\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00019: full_multi_label_metric improved from 0.49206 to 0.49893, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9442522497509469\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00020: full_multi_label_metric improved from 0.49893 to 0.51526, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9419778267372013\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00021: full_multi_label_metric improved from 0.51526 to 0.53236, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9423475823869516\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00022: full_multi_label_metric improved from 0.53236 to 0.53648, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9448027489759648\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00023: full_multi_label_metric improved from 0.53648 to 0.53724, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9444036682962154\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00024: full_multi_label_metric improved from 0.53724 to 0.55662, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9421356002473209\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00025: full_multi_label_metric did not improve from 0.55662\n",
      "Validation AUC: 0.9438663587392868\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00026: full_multi_label_metric improved from 0.55662 to 0.56578, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9431261238100863\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00027: full_multi_label_metric improved from 0.56578 to 0.57005, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9435380451881725\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00028: full_multi_label_metric improved from 0.57005 to 0.57112, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9408281795573532\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00029: full_multi_label_metric improved from 0.57112 to 0.58379, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9454769367295683\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00030: full_multi_label_metric improved from 0.58379 to 0.59753, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9427806142279908\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00031: full_multi_label_metric improved from 0.59753 to 0.60501, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9409176187088704\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00032: full_multi_label_metric improved from 0.60501 to 0.60791, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9440432870573587\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00033: full_multi_label_metric improved from 0.60791 to 0.61203, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9416719442941905\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00034: full_multi_label_metric improved from 0.61203 to 0.61355, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9411408401095154\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00035: full_multi_label_metric improved from 0.61355 to 0.62485, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9420298054791059\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00036: full_multi_label_metric improved from 0.62485 to 0.63202, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9406516637963862\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00037: full_multi_label_metric improved from 0.63202 to 0.63294, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9400326575013842\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00038: full_multi_label_metric improved from 0.63294 to 0.63980, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.939620191798821\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00039: full_multi_label_metric improved from 0.63980 to 0.64103, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9397022014260858\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00040: full_multi_label_metric improved from 0.64103 to 0.65385, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9399331478717692\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00041: full_multi_label_metric improved from 0.65385 to 0.65629, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9393462651885758\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00042: full_multi_label_metric improved from 0.65629 to 0.65934, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9391368646353966\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00043: full_multi_label_metric improved from 0.65934 to 0.66270, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9379190704119621\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00044: full_multi_label_metric improved from 0.66270 to 0.66300, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9387847752724835\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00045: full_multi_label_metric improved from 0.66300 to 0.66422, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9393678847288429\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00046: full_multi_label_metric improved from 0.66422 to 0.66835, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.938525841309972\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00047: full_multi_label_metric improved from 0.66835 to 0.67903, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9387523517524338\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00048: full_multi_label_metric did not improve from 0.67903\n",
      "Validation AUC: 0.9383988895430202\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00049: full_multi_label_metric improved from 0.67903 to 0.68407, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9374854211899725\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00050: full_multi_label_metric did not improve from 0.68407\n",
      "Validation AUC: 0.9383355941263651\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00051: full_multi_label_metric did not improve from 0.68407\n",
      "Validation AUC: 0.9389025102356311\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00052: full_multi_label_metric did not improve from 0.68407\n",
      "Validation AUC: 0.9386297547929908\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00053: full_multi_label_metric improved from 0.68407 to 0.68712, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9379672440738498\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00054: full_multi_label_metric did not improve from 0.68712\n",
      "Validation AUC: 0.9385917524890393\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00055: full_multi_label_metric improved from 0.68712 to 0.69017, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9375096961596813\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00056: full_multi_label_metric improved from 0.69017 to 0.70131, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9372805774224351\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00057: full_multi_label_metric did not improve from 0.70131\n",
      "Validation AUC: 0.9378076282092361\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00058: full_multi_label_metric did not improve from 0.70131\n",
      "Validation AUC: 0.9376411749125518\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00059: full_multi_label_metric did not improve from 0.70131\n",
      "Validation AUC: 0.9378474921697656\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00060: full_multi_label_metric did not improve from 0.70131\n",
      "Validation AUC: 0.9373127271141581\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00061: full_multi_label_metric did not improve from 0.70131\n",
      "Validation AUC: 0.9370475479381687\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00062: full_multi_label_metric did not improve from 0.70131\n",
      "Validation AUC: 0.9364174111998267\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00063: full_multi_label_metric did not improve from 0.70131\n",
      "Validation AUC: 0.937314817917337\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00064: full_multi_label_metric did not improve from 0.70131\n",
      "Validation AUC: 0.9367831548082297\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00065: full_multi_label_metric did not improve from 0.70131\n",
      "Validation AUC: 0.9373092894078197\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00066: full_multi_label_metric improved from 0.70131 to 0.70620, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9367231824520358\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00067: full_multi_label_metric did not improve from 0.70620\n",
      "Validation AUC: 0.9364190103818132\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00068: full_multi_label_metric improved from 0.70620 to 0.70971, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9367900075311457\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00069: full_multi_label_metric did not improve from 0.70971\n",
      "Validation AUC: 0.9364050582974898\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00070: full_multi_label_metric did not improve from 0.70971\n",
      "Validation AUC: 0.9364265290307882\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00071: full_multi_label_metric did not improve from 0.70971\n",
      "Validation AUC: 0.9366205792739911\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00072: full_multi_label_metric did not improve from 0.70971\n",
      "Validation AUC: 0.936469381399515\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00073: full_multi_label_metric improved from 0.70971 to 0.71001, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.936130455497281\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00074: full_multi_label_metric improved from 0.71001 to 0.71337, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9368693565536083\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00075: full_multi_label_metric did not improve from 0.71337\n",
      "Validation AUC: 0.9369859625787473\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00076: full_multi_label_metric did not improve from 0.71337\n",
      "Validation AUC: 0.9367081405360129\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00077: full_multi_label_metric did not improve from 0.71337\n",
      "Validation AUC: 0.9372490669971892\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00078: full_multi_label_metric improved from 0.71337 to 0.71383, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9355418285007532\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00079: full_multi_label_metric did not improve from 0.71383\n",
      "Validation AUC: 0.9361794249681225\n",
      "Lowering LR to 3.90625e-07\n",
      "\n",
      "Epoch 00080: full_multi_label_metric improved from 0.71383 to 0.71764, saving model to lateFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9370879594477937\n",
      "Weighted AUC score for split 0: 0.9474778730668943\n"
     ]
    }
   ],
   "source": [
    "weightedAUCList = []\n",
    "AUCList = []\n",
    "for ix, split in data.items():\n",
    "    K.clear_session()\n",
    "    print('====================Starting Split Number {}===================='.format(ix))\n",
    "    model = Generate_Model(model_type='LateFusion')#create model\n",
    "    model.compile(optimizer=optimizerFactory(), loss='binary_crossentropy', metrics=[full_multi_label_metric])\n",
    "    \n",
    "    train_generator = padchestDataGenerator_DF(df.iloc[split['train']].reset_index(drop=True), **params)\n",
    "    valid_generator = padchestDataGenerator_DF(df.iloc[split['valid']].reset_index(drop=True), **params)\n",
    "    test_generator = padchestDataGenerator_DF(df.iloc[split['test']].reset_index(drop=True), **params)\n",
    "    #checkpoint\n",
    "    filename = 'lateFusion_{}_split_{}.hdf5'.format(projection,ix)#'imageOnly_lat.hdf5'\n",
    "    checkpoint = getCheckpointCallback(filename)\n",
    "    model.fit_generator(train_generator, steps_per_epoch=len(train_generator), validation_data=valid_generator, \n",
    "                        validation_steps=len(valid_generator), workers=workers, epochs=n_epochs, verbose=fitVerbose, class_weight=train_generator.labels_weights, \n",
    "                        callbacks=[checkpoint,lrate, AUC(valid_generator)])\n",
    "    y_hat = model.predict_generator(test_generator, steps=len(test_generator), workers=workers)\n",
    "    auc = roc_auc_score(test_generator.mlb.transform(df.iloc[split['test']].iloc[:len(y_hat)]['Final_Labels']), y_hat, average=None)\n",
    "    AUCList.append(auc)\n",
    "    weighted_auc = roc_auc_score(test_generator.mlb.transform(df.iloc[split['test']].iloc[:len(y_hat)]['Final_Labels']), y_hat, average='weighted')\n",
    "    weightedAUCList.append(weighted_auc)\n",
    "    print('Weighted AUC score for split {}: {}'.format(ix,weighted_auc))\n",
    "AUC_Scores = pd.DataFrame({'Label': test_generator.mlb.classes_, 'Score': np.mean(AUCList, axis=0)}, columns=['Label', 'Score'])\n",
    "AUC_Scores = AUC_Scores.append({'Label': 'weighted_auc', 'Score':np.mean(weightedAUCList, axis=0)}, ignore_index=True)#to csv    \n",
    "AUC_Scores.to_csv('AUC_scores_lateFusion_{}'.format(projection))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Late Fusion Pretrained"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================Starting Split Number 0====================\n",
      "Encoding Labels\n",
      "You have passed in a tokenizer, we will just convert to sequences\n",
      "filtering sentences\n",
      "filtering sents and removing stopwords\n",
      "Encoding Labels\n",
      "You have passed in a tokenizer, we will just convert to sequences\n",
      "filtering sentences\n",
      "filtering sents and removing stopwords\n",
      "Encoding Labels\n",
      "You have passed in a tokenizer, we will just convert to sequences\n",
      "filtering sentences\n",
      "filtering sents and removing stopwords\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00001: full_multi_label_metric improved from -inf to 0.04686, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9166832140042581\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00002: full_multi_label_metric improved from 0.04686 to 0.23764, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9238844831840631\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00003: full_multi_label_metric improved from 0.23764 to 0.40293, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9346871405774114\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00004: full_multi_label_metric improved from 0.40293 to 0.47726, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9393480604657742\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00005: full_multi_label_metric improved from 0.47726 to 0.49863, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9381598431050165\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00006: full_multi_label_metric improved from 0.49863 to 0.51343, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9450315009890401\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00007: full_multi_label_metric improved from 0.51343 to 0.51709, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9458745160379856\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00008: full_multi_label_metric improved from 0.51709 to 0.52778, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9513546176777157\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00009: full_multi_label_metric improved from 0.52778 to 0.52915, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9527299906797054\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00010: full_multi_label_metric improved from 0.52915 to 0.55846, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.951932195151472\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00011: full_multi_label_metric improved from 0.55846 to 0.56792, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9540229105685315\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00012: full_multi_label_metric improved from 0.56792 to 0.57585, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9495849825784225\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00013: full_multi_label_metric improved from 0.57585 to 0.58501, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9530639813802498\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00014: full_multi_label_metric improved from 0.58501 to 0.59661, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9451960512360695\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00015: full_multi_label_metric improved from 0.59661 to 0.60272, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9523682276891373\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00016: full_multi_label_metric improved from 0.60272 to 0.62821, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9484590808716356\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00017: full_multi_label_metric improved from 0.62821 to 0.63172, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9536283821301713\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00018: full_multi_label_metric improved from 0.63172 to 0.63477, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9525674479908256\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00019: full_multi_label_metric improved from 0.63477 to 0.64576, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.950003450401372\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00020: full_multi_label_metric improved from 0.64576 to 0.66468, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9508255193818035\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00021: full_multi_label_metric improved from 0.66468 to 0.67338, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9519879678580385\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00022: full_multi_label_metric improved from 0.67338 to 0.68361, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9507153001656143\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00023: full_multi_label_metric improved from 0.68361 to 0.69750, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9484471587233103\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00024: full_multi_label_metric improved from 0.69750 to 0.70284, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.951021819242806\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00025: full_multi_label_metric improved from 0.70284 to 0.70681, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9513416576984881\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00026: full_multi_label_metric improved from 0.70681 to 0.71352, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9517907038591239\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00027: full_multi_label_metric improved from 0.71352 to 0.71886, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9508475243845294\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00028: full_multi_label_metric improved from 0.71886 to 0.72589, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9481028411967369\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00029: full_multi_label_metric improved from 0.72589 to 0.73245, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9504463425837638\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00030: full_multi_label_metric improved from 0.73245 to 0.74191, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9512732549823547\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00031: full_multi_label_metric improved from 0.74191 to 0.74481, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9522573311849042\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00032: full_multi_label_metric improved from 0.74481 to 0.75229, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9524877602517821\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00033: full_multi_label_metric improved from 0.75229 to 0.75382, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9519871099799615\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00034: full_multi_label_metric did not improve from 0.75382\n",
      "Validation AUC: 0.9513128808493758\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00035: full_multi_label_metric improved from 0.75382 to 0.76465, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9511710770916347\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00036: full_multi_label_metric improved from 0.76465 to 0.76816, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9522691473674976\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00037: full_multi_label_metric did not improve from 0.76816\n",
      "Validation AUC: 0.9512886119197552\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00038: full_multi_label_metric improved from 0.76816 to 0.77060, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9509997498050539\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00039: full_multi_label_metric did not improve from 0.77060\n",
      "Validation AUC: 0.9515414203684475\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00040: full_multi_label_metric improved from 0.77060 to 0.77885, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9520520314851483\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00041: full_multi_label_metric improved from 0.77885 to 0.78770, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9521116978563544\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00042: full_multi_label_metric did not improve from 0.78770\n",
      "Validation AUC: 0.9511954675051835\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00043: full_multi_label_metric did not improve from 0.78770\n",
      "Validation AUC: 0.9507712113734265\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00044: full_multi_label_metric did not improve from 0.78770\n",
      "Validation AUC: 0.9509252910220761\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00045: full_multi_label_metric improved from 0.78770 to 0.79502, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9503863185403902\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00046: full_multi_label_metric improved from 0.79502 to 0.79548, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9503384760322756\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00047: full_multi_label_metric did not improve from 0.79548\n",
      "Validation AUC: 0.950896407503356\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00048: full_multi_label_metric did not improve from 0.79548\n",
      "Validation AUC: 0.9504421931817906\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00049: full_multi_label_metric improved from 0.79548 to 0.80113, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.950470713652794\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00050: full_multi_label_metric did not improve from 0.80113\n",
      "Validation AUC: 0.9508917842283965\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00051: full_multi_label_metric improved from 0.80113 to 0.80433, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.950310431936607\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00052: full_multi_label_metric did not improve from 0.80433\n",
      "Validation AUC: 0.951025801550019\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00053: full_multi_label_metric did not improve from 0.80433\n",
      "Validation AUC: 0.9509897792398164\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00054: full_multi_label_metric did not improve from 0.80433\n",
      "Validation AUC: 0.949868866148853\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00055: full_multi_label_metric improved from 0.80433 to 0.80586, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.950872481799867\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00056: full_multi_label_metric did not improve from 0.80586\n",
      "Validation AUC: 0.9517935137528043\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00057: full_multi_label_metric improved from 0.80586 to 0.80723, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9508210236254877\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00058: full_multi_label_metric improved from 0.80723 to 0.80876, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9510635149312489\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00059: full_multi_label_metric did not improve from 0.80876\n",
      "Validation AUC: 0.9506215983446342\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00060: full_multi_label_metric did not improve from 0.80876\n",
      "Validation AUC: 0.9503778875737797\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00061: full_multi_label_metric improved from 0.80876 to 0.81242, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9513129529506807\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00062: full_multi_label_metric improved from 0.81242 to 0.81502, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9517768775753536\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00063: full_multi_label_metric did not improve from 0.81502\n",
      "Validation AUC: 0.9513519919296004\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00064: full_multi_label_metric did not improve from 0.81502\n",
      "Validation AUC: 0.9512645133697619\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00065: full_multi_label_metric improved from 0.81502 to 0.82021, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9510753035754084\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00066: full_multi_label_metric did not improve from 0.82021\n",
      "Validation AUC: 0.9508995896872785\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00067: full_multi_label_metric did not improve from 0.82021\n",
      "Validation AUC: 0.951058813650438\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00068: full_multi_label_metric did not improve from 0.82021\n",
      "Validation AUC: 0.9514551058630245\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00069: full_multi_label_metric did not improve from 0.82021\n",
      "Validation AUC: 0.9501688306271154\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00070: full_multi_label_metric did not improve from 0.82021\n",
      "Validation AUC: 0.9513198823756783\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00071: full_multi_label_metric did not improve from 0.82021\n",
      "Validation AUC: 0.9509585363905037\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00072: full_multi_label_metric did not improve from 0.82021\n",
      "Validation AUC: 0.9512078387170735\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00073: full_multi_label_metric did not improve from 0.82021\n",
      "Validation AUC: 0.9505086485510388\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00074: full_multi_label_metric did not improve from 0.82021\n",
      "Validation AUC: 0.9508107994914281\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00075: full_multi_label_metric did not improve from 0.82021\n",
      "Validation AUC: 0.950432951633634\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00076: full_multi_label_metric did not improve from 0.82021\n",
      "Validation AUC: 0.950626354356086\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00077: full_multi_label_metric improved from 0.82021 to 0.82250, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9508490500821252\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00078: full_multi_label_metric did not improve from 0.82250\n",
      "Validation AUC: 0.9509508337158399\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00079: full_multi_label_metric did not improve from 0.82250\n",
      "Validation AUC: 0.9515369264588421\n",
      "Lowering LR to 3.90625e-07\n",
      "\n",
      "Epoch 00080: full_multi_label_metric improved from 0.82250 to 0.82540, saving model to lateFusion_PT_PA_split_0.hdf5\n",
      "Validation AUC: 0.9499828652918534\n",
      "Weighted AUC score for split 0: 0.9575019213661663\n"
     ]
    }
   ],
   "source": [
    "weightedAUCList = []\n",
    "AUCList = []\n",
    "for ix, split in data.items():\n",
    "    K.clear_session()\n",
    "    print('====================Starting Split Number {}===================='.format(ix))\n",
    "    model = Generate_Model(model_type='LateFusion')#create model\n",
    "    model.compile(optimizer=optimizerFactory(), loss='binary_crossentropy', metrics=[full_multi_label_metric])\n",
    "    \n",
    "    train_generator = padchestDataGenerator_DF(df.iloc[split['train']].reset_index(drop=True), **params)\n",
    "    valid_generator = padchestDataGenerator_DF(df.iloc[split['valid']].reset_index(drop=True), **params)\n",
    "    test_generator = padchestDataGenerator_DF(df.iloc[split['test']].reset_index(drop=True), **params)\n",
    "    #checkpoint\n",
    "    filename = 'lateFusion_PT_{}_split_{}.hdf5'.format(projection,ix)\n",
    "    checkpoint = getCheckpointCallback(filename)\n",
    "    \n",
    "    #Load weights: This will only load the feature extractors, not the 2 dense layers at the end.\n",
    "    model.layers[3].set_weights(load_model('imageOnly_{}_split_{}.hdf5'.format(projection,ix),compile=False).layers[1].get_weights())#image\n",
    "    model.layers[4].set_weights(load_model('textOnly_{}_split_{}.hdf5'.format(projection,ix),compile=False).layers[2].get_weights())#text\n",
    "    \n",
    "    #Fit Model\n",
    "    model.fit_generator(train_generator, steps_per_epoch=len(train_generator), validation_data=valid_generator, \n",
    "                        validation_steps=len(valid_generator), workers=workers, epochs=n_epochs, verbose=fitVerbose, class_weight=train_generator.labels_weights, \n",
    "                        callbacks=[checkpoint,lrate, AUC(valid_generator)])\n",
    "    y_hat = model.predict_generator(test_generator, steps=len(test_generator), workers=workers)\n",
    "    auc = roc_auc_score(test_generator.mlb.transform(df.iloc[split['test']].iloc[:len(y_hat)]['Final_Labels']), y_hat, average=None)\n",
    "    AUCList.append(auc)\n",
    "    weighted_auc = roc_auc_score(test_generator.mlb.transform(df.iloc[split['test']].iloc[:len(y_hat)]['Final_Labels']), y_hat, average='weighted')\n",
    "    weightedAUCList.append(weighted_auc)\n",
    "    print('Weighted AUC score for split {}: {}'.format(ix,weighted_auc))\n",
    "AUC_Scores = pd.DataFrame({'Label': test_generator.mlb.classes_, 'Score': np.mean(AUCList, axis=0)}, columns=['Label', 'Score'])\n",
    "AUC_Scores = AUC_Scores.append({'Label': 'weighted_auc', 'Score':np.mean(weightedAUCList, axis=0)}, ignore_index=True)#to csv    \n",
    "AUC_Scores.to_csv('AUC_scores_lateFusion_PT_{}'.format(projection))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Early Fusion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================Starting Split Number 0====================\n",
      "Encoding Labels\n",
      "You have passed in a tokenizer, we will just convert to sequences\n",
      "filtering sentences\n",
      "filtering sents and removing stopwords\n",
      "Encoding Labels\n",
      "You have passed in a tokenizer, we will just convert to sequences\n",
      "filtering sentences\n",
      "filtering sents and removing stopwords\n",
      "Encoding Labels\n",
      "You have passed in a tokenizer, we will just convert to sequences\n",
      "filtering sentences\n",
      "filtering sents and removing stopwords\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00001: full_multi_label_metric improved from -inf to 0.01160, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.7700144411153766\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00002: full_multi_label_metric improved from 0.01160 to 0.08501, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.8554921446278595\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00003: full_multi_label_metric improved from 0.08501 to 0.23855, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.905421231713029\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00004: full_multi_label_metric improved from 0.23855 to 0.36142, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.93181345262068\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00005: full_multi_label_metric improved from 0.36142 to 0.42430, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9428364833808887\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00006: full_multi_label_metric improved from 0.42430 to 0.46993, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9521276186594168\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00007: full_multi_label_metric improved from 0.46993 to 0.50137, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9537488131323505\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00008: full_multi_label_metric improved from 0.50137 to 0.52534, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9577085995383613\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00009: full_multi_label_metric improved from 0.52534 to 0.54396, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.963179840127951\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00010: full_multi_label_metric improved from 0.54396 to 0.60089, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9705749742080905\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00011: full_multi_label_metric improved from 0.60089 to 0.62653, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9713430644165402\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00012: full_multi_label_metric improved from 0.62653 to 0.65308, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9748633337983155\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00013: full_multi_label_metric improved from 0.65308 to 0.68529, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9751006492642835\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00014: full_multi_label_metric improved from 0.68529 to 0.71047, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9730185787399943\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00015: full_multi_label_metric improved from 0.71047 to 0.74451, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9762504953603022\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00016: full_multi_label_metric improved from 0.74451 to 0.76542, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9755843278271037\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00017: full_multi_label_metric improved from 0.76542 to 0.78602, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.97435268509802\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00018: full_multi_label_metric improved from 0.78602 to 0.80357, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.976735713634096\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00019: full_multi_label_metric improved from 0.80357 to 0.82830, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9779548880757946\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00020: full_multi_label_metric improved from 0.82830 to 0.87134, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9782452684285591\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00021: full_multi_label_metric improved from 0.87134 to 0.89194, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9793689285107192\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00022: full_multi_label_metric improved from 0.89194 to 0.90186, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9778777309142866\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00023: full_multi_label_metric improved from 0.90186 to 0.91636, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9777935946009169\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00024: full_multi_label_metric improved from 0.91636 to 0.92201, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9769345223642967\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00025: full_multi_label_metric improved from 0.92201 to 0.92689, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9771553275447505\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00026: full_multi_label_metric improved from 0.92689 to 0.92903, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9762344665078729\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00027: full_multi_label_metric improved from 0.92903 to 0.93636, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9761147084592846\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00028: full_multi_label_metric improved from 0.93636 to 0.93926, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9767582576433148\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00029: full_multi_label_metric improved from 0.93926 to 0.94368, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9757495419840523\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00030: full_multi_label_metric improved from 0.94368 to 0.94994, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9781695705475626\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00031: full_multi_label_metric improved from 0.94994 to 0.96184, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9786553625240668\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00032: full_multi_label_metric improved from 0.96184 to 0.96413, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9771317090882877\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00033: full_multi_label_metric improved from 0.96413 to 0.96719, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9774765927811523\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00034: full_multi_label_metric improved from 0.96719 to 0.96795, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9766961994271326\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00035: full_multi_label_metric improved from 0.96795 to 0.96886, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9771746492102114\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00036: full_multi_label_metric did not improve from 0.96886\n",
      "Validation AUC: 0.9776772602080046\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00037: full_multi_label_metric improved from 0.96886 to 0.96993, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9766177048379687\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00038: full_multi_label_metric did not improve from 0.96993\n",
      "Validation AUC: 0.9776010856133756\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00039: full_multi_label_metric improved from 0.96993 to 0.97207, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9767681484404224\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00040: full_multi_label_metric improved from 0.97207 to 0.97329, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9771317201293303\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00041: full_multi_label_metric improved from 0.97329 to 0.97604, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9771602349652864\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00042: full_multi_label_metric did not improve from 0.97604\n",
      "Validation AUC: 0.9775631875926895\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00043: full_multi_label_metric improved from 0.97604 to 0.98016, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.977225470858876\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00044: full_multi_label_metric did not improve from 0.98016\n",
      "Validation AUC: 0.9766582088583133\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00045: full_multi_label_metric did not improve from 0.98016\n",
      "Validation AUC: 0.9771326739645185\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00046: full_multi_label_metric did not improve from 0.98016\n",
      "Validation AUC: 0.9777486837652273\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00047: full_multi_label_metric did not improve from 0.98016\n",
      "Validation AUC: 0.9760495382405703\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00048: full_multi_label_metric did not improve from 0.98016\n",
      "Validation AUC: 0.9768399020925005\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00049: full_multi_label_metric improved from 0.98016 to 0.98184, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9773710506132678\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00050: full_multi_label_metric did not improve from 0.98184\n",
      "Validation AUC: 0.9773946737645206\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00051: full_multi_label_metric improved from 0.98184 to 0.98291, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.97735207436105\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00052: full_multi_label_metric did not improve from 0.98291\n",
      "Validation AUC: 0.9776492841246147\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00053: full_multi_label_metric improved from 0.98291 to 0.98397, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9767788903194059\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00054: full_multi_label_metric improved from 0.98397 to 0.98428, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9768451033086367\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00055: full_multi_label_metric did not improve from 0.98428\n",
      "Validation AUC: 0.9775464727537275\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00056: full_multi_label_metric did not improve from 0.98428\n",
      "Validation AUC: 0.9768606980822171\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00057: full_multi_label_metric improved from 0.98428 to 0.98581, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9771789358331601\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00058: full_multi_label_metric did not improve from 0.98581\n",
      "Validation AUC: 0.9770409137809261\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00059: full_multi_label_metric did not improve from 0.98581\n",
      "Validation AUC: 0.976830414383745\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00060: full_multi_label_metric did not improve from 0.98581\n",
      "Validation AUC: 0.9772048746802285\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00061: full_multi_label_metric did not improve from 0.98581\n",
      "Validation AUC: 0.9769473162158334\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00062: full_multi_label_metric did not improve from 0.98581\n",
      "Validation AUC: 0.9773792203404932\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00063: full_multi_label_metric did not improve from 0.98581\n",
      "Validation AUC: 0.977128285641003\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00064: full_multi_label_metric did not improve from 0.98581\n",
      "Validation AUC: 0.9771697981887684\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00065: full_multi_label_metric improved from 0.98581 to 0.98626, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9771479629633076\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00066: full_multi_label_metric did not improve from 0.98626\n",
      "Validation AUC: 0.9774780494052094\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00067: full_multi_label_metric improved from 0.98626 to 0.98672, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9772820553089048\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00068: full_multi_label_metric did not improve from 0.98672\n",
      "Validation AUC: 0.9770458197861098\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00069: full_multi_label_metric did not improve from 0.98672\n",
      "Validation AUC: 0.977446696548387\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00070: full_multi_label_metric did not improve from 0.98672\n",
      "Validation AUC: 0.976967218477535\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00071: full_multi_label_metric did not improve from 0.98672\n",
      "Validation AUC: 0.976927625218803\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00072: full_multi_label_metric improved from 0.98672 to 0.98764, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9771973772524412\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00073: full_multi_label_metric improved from 0.98764 to 0.98779, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9767806000030952\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00074: full_multi_label_metric did not improve from 0.98779\n",
      "Validation AUC: 0.9767580751712085\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00075: full_multi_label_metric improved from 0.98779 to 0.98855, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9767327983163892\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00076: full_multi_label_metric did not improve from 0.98855\n",
      "Validation AUC: 0.9768064490003526\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00077: full_multi_label_metric did not improve from 0.98855\n",
      "Validation AUC: 0.9768660045282727\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00078: full_multi_label_metric did not improve from 0.98855\n",
      "Validation AUC: 0.976623811968997\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00079: full_multi_label_metric did not improve from 0.98855\n",
      "Validation AUC: 0.9769527875690811\n",
      "Lowering LR to 3.90625e-07\n",
      "\n",
      "Epoch 00080: full_multi_label_metric improved from 0.98855 to 0.98901, saving model to earlyFusion_PA_split_0.hdf5\n",
      "Validation AUC: 0.9767447859212972\n",
      "Weighted AUC score for split 0: 0.9801232505126506\n"
     ]
    }
   ],
   "source": [
    "weightedAUCList = []\n",
    "AUCList = []\n",
    "for ix, split in data.items():\n",
    "    K.clear_session()\n",
    "    print('====================Starting Split Number {}===================='.format(ix))\n",
    "    model = Generate_Model(model_type='EarlyFusion')\n",
    "    model.compile(optimizer=optimizerFactory(), loss='binary_crossentropy', metrics=[full_multi_label_metric])\n",
    "    \n",
    "    train_generator = padchestDataGenerator_DF(df.iloc[split['train']].reset_index(drop=True), **params)\n",
    "    valid_generator = padchestDataGenerator_DF(df.iloc[split['valid']].reset_index(drop=True), **params)\n",
    "    test_generator = padchestDataGenerator_DF(df.iloc[split['test']].reset_index(drop=True), **params)\n",
    "    #checkpoint\n",
    "    filename = 'earlyFusion_{}_split_{}.hdf5'.format(projection,ix)#'imageOnly_lat.hdf5'\n",
    "    checkpoint = getCheckpointCallback(filename)\n",
    "    model.fit_generator(train_generator, steps_per_epoch=len(train_generator), validation_data=valid_generator, \n",
    "                        validation_steps=len(valid_generator), workers=workers, epochs=n_epochs, verbose=fitVerbose, class_weight=train_generator.labels_weights, \n",
    "                        callbacks=[checkpoint,lrate, AUC(valid_generator)])\n",
    "    y_hat = model.predict_generator(test_generator, steps=len(test_generator), workers=workers)\n",
    "    auc = roc_auc_score(test_generator.mlb.transform(df.iloc[split['test']].iloc[:len(y_hat)]['Final_Labels']), y_hat, average=None)\n",
    "    AUCList.append(auc)\n",
    "    weighted_auc = roc_auc_score(test_generator.mlb.transform(df.iloc[split['test']].iloc[:len(y_hat)]['Final_Labels']), y_hat, average='weighted')\n",
    "    weightedAUCList.append(weighted_auc)\n",
    "    print('Weighted AUC score for split {}: {}'.format(ix,weighted_auc))\n",
    "AUC_Scores = pd.DataFrame({'Label': test_generator.mlb.classes_, 'Score': np.mean(AUCList, axis=0)}, columns=['Label', 'Score'])\n",
    "AUC_Scores = AUC_Scores.append({'Label': 'weighted_auc', 'Score':np.mean(weightedAUCList, axis=0)}, ignore_index=True)#to csv    \n",
    "AUC_Scores.to_csv('AUC_scores_earlyFusion_{}'.format(projection))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Early Fusion (Wide)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================Starting Split Number 0====================\n",
      "Encoding Labels\n",
      "You have passed in a tokenizer, we will just convert to sequences\n",
      "filtering sentences\n",
      "filtering sents and removing stopwords\n",
      "Encoding Labels\n",
      "You have passed in a tokenizer, we will just convert to sequences\n",
      "filtering sentences\n",
      "filtering sents and removing stopwords\n",
      "Encoding Labels\n",
      "You have passed in a tokenizer, we will just convert to sequences\n",
      "filtering sentences\n",
      "filtering sents and removing stopwords\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00001: full_multi_label_metric improved from -inf to 0.00687, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.7514203389089252\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00002: full_multi_label_metric improved from 0.00687 to 0.05708, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.8008940211557797\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00003: full_multi_label_metric improved from 0.05708 to 0.20009, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.8239444236945098\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00004: full_multi_label_metric improved from 0.20009 to 0.28495, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.8519790574268262\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00005: full_multi_label_metric improved from 0.28495 to 0.34096, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.8781743995202557\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00006: full_multi_label_metric improved from 0.34096 to 0.37088, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.8968222078892747\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00007: full_multi_label_metric improved from 0.37088 to 0.39438, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9124130465607484\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00008: full_multi_label_metric improved from 0.39438 to 0.41606, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9209643373536426\n",
      "Lowering LR to 0.0001\n",
      "\n",
      "Epoch 00009: full_multi_label_metric improved from 0.41606 to 0.43254, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9262958034001398\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00010: full_multi_label_metric improved from 0.43254 to 0.44551, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9303786658232712\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00011: full_multi_label_metric improved from 0.44551 to 0.45742, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9333271968723051\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00012: full_multi_label_metric improved from 0.45742 to 0.46352, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9354967161640115\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00013: full_multi_label_metric improved from 0.46352 to 0.47176, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9371774359377484\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00014: full_multi_label_metric improved from 0.47176 to 0.48062, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9380836527242198\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00015: full_multi_label_metric improved from 0.48062 to 0.48901, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9402895350838214\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00016: full_multi_label_metric improved from 0.48901 to 0.50351, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9416950038573718\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00017: full_multi_label_metric improved from 0.50351 to 0.50626, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9452992182260238\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00018: full_multi_label_metric improved from 0.50626 to 0.51618, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9472601632968789\n",
      "Lowering LR to 5e-05\n",
      "\n",
      "Epoch 00019: full_multi_label_metric improved from 0.51618 to 0.51740, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9492400682356358\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00020: full_multi_label_metric improved from 0.51740 to 0.53068, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9473101178196777\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00021: full_multi_label_metric improved from 0.53068 to 0.53556, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9495242877670911\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00022: full_multi_label_metric improved from 0.53556 to 0.54182, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9497384139232857\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00023: full_multi_label_metric did not improve from 0.54182\n",
      "Validation AUC: 0.949907269897422\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00024: full_multi_label_metric improved from 0.54182 to 0.55021, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9503339168791874\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00025: full_multi_label_metric did not improve from 0.55021\n",
      "Validation AUC: 0.9510738951800112\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00026: full_multi_label_metric improved from 0.55021 to 0.55067, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.950667332427988\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00027: full_multi_label_metric improved from 0.55067 to 0.55479, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9524519247869434\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00028: full_multi_label_metric improved from 0.55479 to 0.55540, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9518043856288957\n",
      "Lowering LR to 2.5e-05\n",
      "\n",
      "Epoch 00029: full_multi_label_metric improved from 0.55540 to 0.56044, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9526200783743092\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00030: full_multi_label_metric improved from 0.56044 to 0.57799, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9531962834211023\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00031: full_multi_label_metric did not improve from 0.57799\n",
      "Validation AUC: 0.9536152479923378\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00032: full_multi_label_metric improved from 0.57799 to 0.57814, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9535804394183809\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00033: full_multi_label_metric did not improve from 0.57814\n",
      "Validation AUC: 0.9540464074860293\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00034: full_multi_label_metric improved from 0.57814 to 0.59066, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9532243632535671\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00035: full_multi_label_metric did not improve from 0.59066\n",
      "Validation AUC: 0.9534868409371016\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00036: full_multi_label_metric improved from 0.59066 to 0.59081, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.954574639507821\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00037: full_multi_label_metric did not improve from 0.59081\n",
      "Validation AUC: 0.9544983466339937\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00038: full_multi_label_metric did not improve from 0.59081\n",
      "Validation AUC: 0.954655596692579\n",
      "Lowering LR to 1.25e-05\n",
      "\n",
      "Epoch 00039: full_multi_label_metric improved from 0.59081 to 0.59173, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9545155631549969\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00040: full_multi_label_metric improved from 0.59173 to 0.59402, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9549193789214264\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00041: full_multi_label_metric improved from 0.59402 to 0.60211, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9556956841213972\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00042: full_multi_label_metric did not improve from 0.60211\n",
      "Validation AUC: 0.9551970623739632\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00043: full_multi_label_metric improved from 0.60211 to 0.60317, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9555296205198873\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00044: full_multi_label_metric improved from 0.60317 to 0.60394, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9557804401882676\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00045: full_multi_label_metric did not improve from 0.60394\n",
      "Validation AUC: 0.9560141592456659\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00046: full_multi_label_metric did not improve from 0.60394\n",
      "Validation AUC: 0.9559207602451252\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00047: full_multi_label_metric improved from 0.60394 to 0.60485, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.956077948276763\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00048: full_multi_label_metric improved from 0.60485 to 0.60577, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9556091312188274\n",
      "Lowering LR to 6.25e-06\n",
      "\n",
      "Epoch 00049: full_multi_label_metric did not improve from 0.60577\n",
      "Validation AUC: 0.95595048409412\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00050: full_multi_label_metric did not improve from 0.60577\n",
      "Validation AUC: 0.9562276450615312\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00051: full_multi_label_metric improved from 0.60577 to 0.61142, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9564973282151344\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00052: full_multi_label_metric did not improve from 0.61142\n",
      "Validation AUC: 0.9561170868238188\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00053: full_multi_label_metric improved from 0.61142 to 0.61447, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9560696168035471\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00054: full_multi_label_metric did not improve from 0.61447\n",
      "Validation AUC: 0.9560925680305812\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00055: full_multi_label_metric did not improve from 0.61447\n",
      "Validation AUC: 0.9568216350971923\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00056: full_multi_label_metric improved from 0.61447 to 0.61645, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9568927980742961\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00057: full_multi_label_metric did not improve from 0.61645\n",
      "Validation AUC: 0.9568591352960506\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00058: full_multi_label_metric did not improve from 0.61645\n",
      "Validation AUC: 0.9569453369183997\n",
      "Lowering LR to 3.125e-06\n",
      "\n",
      "Epoch 00059: full_multi_label_metric did not improve from 0.61645\n",
      "Validation AUC: 0.9569564519345269\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00060: full_multi_label_metric did not improve from 0.61645\n",
      "Validation AUC: 0.9566600943025301\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00061: full_multi_label_metric did not improve from 0.61645\n",
      "Validation AUC: 0.9574976216182457\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00062: full_multi_label_metric did not improve from 0.61645\n",
      "Validation AUC: 0.9567745682156173\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00063: full_multi_label_metric did not improve from 0.61645\n",
      "Validation AUC: 0.9570459188445913\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00064: full_multi_label_metric improved from 0.61645 to 0.61935, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9570925075580348\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00065: full_multi_label_metric did not improve from 0.61935\n",
      "Validation AUC: 0.9564510846294373\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00066: full_multi_label_metric did not improve from 0.61935\n",
      "Validation AUC: 0.9567731965280686\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00067: full_multi_label_metric did not improve from 0.61935\n",
      "Validation AUC: 0.9572839271792388\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00068: full_multi_label_metric did not improve from 0.61935\n",
      "Validation AUC: 0.9573977112296034\n",
      "Lowering LR to 1.5625e-06\n",
      "\n",
      "Epoch 00069: full_multi_label_metric did not improve from 0.61935\n",
      "Validation AUC: 0.9568664246441969\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00070: full_multi_label_metric did not improve from 0.61935\n",
      "Validation AUC: 0.9570873715077898\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00071: full_multi_label_metric improved from 0.61935 to 0.62408, saving model to earlyFusion_wide_PA_split_0.hdf5\n",
      "Validation AUC: 0.9567925775897961\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00072: full_multi_label_metric did not improve from 0.62408\n",
      "Validation AUC: 0.9570992011212368\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00073: full_multi_label_metric did not improve from 0.62408\n",
      "Validation AUC: 0.9567354735854623\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00074: full_multi_label_metric did not improve from 0.62408\n",
      "Validation AUC: 0.9571117132787228\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00075: full_multi_label_metric did not improve from 0.62408\n",
      "Validation AUC: 0.9579262906270871\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00076: full_multi_label_metric did not improve from 0.62408\n",
      "Validation AUC: 0.9568885555831061\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00077: full_multi_label_metric did not improve from 0.62408\n",
      "Validation AUC: 0.9576146980870233\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00078: full_multi_label_metric did not improve from 0.62408\n",
      "Validation AUC: 0.9566212202581499\n",
      "Lowering LR to 7.8125e-07\n",
      "\n",
      "Epoch 00079: full_multi_label_metric did not improve from 0.62408\n",
      "Validation AUC: 0.9565212255286701\n",
      "Lowering LR to 3.90625e-07\n",
      "\n",
      "Epoch 00080: full_multi_label_metric did not improve from 0.62408\n",
      "Validation AUC: 0.9569119192389276\n",
      "Weighted AUC score for split 0: 0.9640188128180177\n"
     ]
    }
   ],
   "source": [
    "weightedAUCList = []\n",
    "AUCList = []\n",
    "for ix, split in data.items():\n",
    "    K.clear_session()\n",
    "    print('====================Starting Split Number {}===================='.format(ix))\n",
    "    model = Generate_Model(model_type='EarlyFusion_Wide')\n",
    "    model.compile(optimizer=optimizerFactory(), loss='binary_crossentropy', metrics=[full_multi_label_metric])\n",
    "    \n",
    "    train_generator = padchestDataGenerator_DF(df.iloc[split['train']].reset_index(drop=True), **params)\n",
    "    valid_generator = padchestDataGenerator_DF(df.iloc[split['valid']].reset_index(drop=True), **params)\n",
    "    test_generator = padchestDataGenerator_DF(df.iloc[split['test']].reset_index(drop=True), **params)\n",
    "    #checkpoint\n",
    "    filename = 'earlyFusion_wide_{}_split_{}.hdf5'.format(projection,ix)#'imageOnly_lat.hdf5'\n",
    "    checkpoint = getCheckpointCallback(filename)\n",
    "    model.fit_generator(train_generator, steps_per_epoch=len(train_generator), validation_data=valid_generator, \n",
    "                        validation_steps=len(valid_generator), workers=workers, epochs=n_epochs, verbose=fitVerbose, class_weight=train_generator.labels_weights, \n",
    "                        callbacks=[checkpoint,lrate, AUC(valid_generator)])\n",
    "    y_hat = model.predict_generator(test_generator, steps=len(test_generator), workers=workers)\n",
    "    auc = roc_auc_score(test_generator.mlb.transform(df.iloc[split['test']].iloc[:len(y_hat)]['Final_Labels']), y_hat, average=None)\n",
    "    AUCList.append(auc)\n",
    "    weighted_auc = roc_auc_score(test_generator.mlb.transform(df.iloc[split['test']].iloc[:len(y_hat)]['Final_Labels']), y_hat, average='weighted')\n",
    "    weightedAUCList.append(weighted_auc)\n",
    "    print('Weighted AUC score for split {}: {}'.format(ix,weighted_auc))\n",
    "AUC_Scores = pd.DataFrame({'Label': test_generator.mlb.classes_, 'Score': np.mean(AUCList, axis=0)}, columns=['Label', 'Score'])\n",
    "AUC_Scores = AUC_Scores.append({'Label': 'weighted_auc', 'Score':np.mean(weightedAUCList, axis=0)}, ignore_index=True)#to csv    \n",
    "AUC_Scores.to_csv('AUC_scores_earlyFusion_wide_{}'.format(projection))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dlnn",
   "language": "python",
   "name": "dlnn"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}