nbs/05_metadata.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting fastai2\r\n",
      "  Downloading fastai2-0.0.16-py3-none-any.whl (184 kB)\r\n",
      "\u001b[K     |████████████████████████████████| 184 kB 2.7 MB/s \r\n",
      "\u001b[?25hCollecting fastcore\r\n",
      "  Downloading fastcore-0.1.16-py3-none-any.whl (27 kB)\r\n",
      "Requirement already satisfied: pillow in /opt/conda/lib/python3.6/site-packages (from fastai2) (5.4.1)\r\n",
      "Requirement already satisfied: fastprogress>=0.1.22 in /opt/conda/lib/python3.6/site-packages (from fastai2) (0.2.2)\r\n",
      "Requirement already satisfied: spacy in /opt/conda/lib/python3.6/site-packages (from fastai2) (2.2.3)\r\n",
      "Requirement already satisfied: scikit-learn in /opt/conda/lib/python3.6/site-packages (from fastai2) (0.22.2.post1)\r\n",
      "Requirement already satisfied: matplotlib in /opt/conda/lib/python3.6/site-packages (from fastai2) (3.2.1)\r\n",
      "Requirement already satisfied: torch>=1.3.0 in /opt/conda/lib/python3.6/site-packages (from fastai2) (1.4.0)\r\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.6/site-packages (from fastai2) (2.22.0)\r\n",
      "Requirement already satisfied: pyyaml in /opt/conda/lib/python3.6/site-packages (from fastai2) (5.3.1)\r\n",
      "Requirement already satisfied: scipy in /opt/conda/lib/python3.6/site-packages (from fastai2) (1.4.1)\r\n",
      "Requirement already satisfied: pandas in /opt/conda/lib/python3.6/site-packages (from fastai2) (0.25.3)\r\n",
      "Requirement already satisfied: torchvision>=0.5 in /opt/conda/lib/python3.6/site-packages (from fastai2) (0.5.0)\r\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.6/site-packages (from fastcore->fastai2) (1.18.2)\r\n",
      "Requirement already satisfied: dataclasses>='0.7'; python_version < \"3.7\" in /opt/conda/lib/python3.6/site-packages (from fastcore->fastai2) (0.7)\r\n",
      "Requirement already satisfied: thinc<7.4.0,>=7.3.0 in /opt/conda/lib/python3.6/site-packages (from spacy->fastai2) (7.3.1)\r\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.4.0 in /opt/conda/lib/python3.6/site-packages (from spacy->fastai2) (0.6.0)\r\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /opt/conda/lib/python3.6/site-packages (from spacy->fastai2) (3.0.2)\r\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /opt/conda/lib/python3.6/site-packages (from spacy->fastai2) (1.0.2)\r\n",
      "Requirement already satisfied: srsly<1.1.0,>=0.1.0 in /opt/conda/lib/python3.6/site-packages (from spacy->fastai2) (1.0.2)\r\n",
      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.6/site-packages (from spacy->fastai2) (46.1.3.post20200330)\r\n",
      "Requirement already satisfied: plac<1.2.0,>=0.9.6 in /opt/conda/lib/python3.6/site-packages (from spacy->fastai2) (0.9.6)\r\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /opt/conda/lib/python3.6/site-packages (from spacy->fastai2) (2.0.3)\r\n",
      "Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in /opt/conda/lib/python3.6/site-packages (from spacy->fastai2) (1.0.0)\r\n",
      "Requirement already satisfied: blis<0.5.0,>=0.4.0 in /opt/conda/lib/python3.6/site-packages (from spacy->fastai2) (0.4.1)\r\n",
      "Requirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.6/site-packages (from scikit-learn->fastai2) (0.14.1)\r\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /opt/conda/lib/python3.6/site-packages (from matplotlib->fastai2) (2.8.1)\r\n",
      "Requirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.6/site-packages (from matplotlib->fastai2) (0.10.0)\r\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /opt/conda/lib/python3.6/site-packages (from matplotlib->fastai2) (2.4.6)\r\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.6/site-packages (from matplotlib->fastai2) (1.1.0)\r\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.6/site-packages (from requests->fastai2) (1.24.3)\r\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /opt/conda/lib/python3.6/site-packages (from requests->fastai2) (3.0.4)\r\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /opt/conda/lib/python3.6/site-packages (from requests->fastai2) (2.8)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.6/site-packages (from requests->fastai2) (2019.11.28)\r\n",
      "Requirement already satisfied: pytz>=2017.2 in /opt/conda/lib/python3.6/site-packages (from pandas->fastai2) (2019.3)\r\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.6/site-packages (from torchvision>=0.5->fastai2) (1.14.0)\r\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.10.0 in /opt/conda/lib/python3.6/site-packages (from thinc<7.4.0,>=7.3.0->spacy->fastai2) (4.42.0)\r\n",
      "Requirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in /opt/conda/lib/python3.6/site-packages (from catalogue<1.1.0,>=0.0.7->spacy->fastai2) (1.5.0)\r\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.6/site-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy->fastai2) (2.2.0)\r\n",
      "Installing collected packages: fastcore, fastai2\r\n",
      "Successfully installed fastai2-0.0.16 fastcore-0.1.16\r\n"
     ]
    }
   ],
   "source": [
    "!pip install fastai2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "from fastai2.vision.all import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "base = Path('../input/rsna-hemorrhage-jpg')\n",
    "trn_path = base/'train_jpg/train_jpg'\n",
    "csv_path = base/'meta/meta'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "194082"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn = get_image_files(trn_path)\n",
    "len(fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>any</th>\n",
       "      <th>epidural</th>\n",
       "      <th>intraparenchymal</th>\n",
       "      <th>intraventricular</th>\n",
       "      <th>subarachnoid</th>\n",
       "      <th>subdural</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_000039fa0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_00005679d</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_00008ce3c</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_0000950d7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_0000aee4b</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             ID  any  epidural  intraparenchymal  intraventricular  \\\n",
       "0  ID_000039fa0    0         0                 0                 0   \n",
       "1  ID_00005679d    0         0                 0                 0   \n",
       "2  ID_00008ce3c    0         0                 0                 0   \n",
       "3  ID_0000950d7    0         0                 0                 0   \n",
       "4  ID_0000aee4b    0         0                 0                 0   \n",
       "\n",
       "   subarachnoid  subdural  \n",
       "0             0         0  \n",
       "1             0         0  \n",
       "2             0         0  \n",
       "3             0         0  \n",
       "4             0         0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = pd.read_feather(csv_path/'labels.fth')\n",
    "labels.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((674258, 7),\n",
       " Path('../input/rsna-hemorrhage-jpg/train_jpg/train_jpg/ID_2369d2208.jpg'))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels.shape, fn[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ID_2369d2208.jpg'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn[0].name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extracting just the file names from the list of paths :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(194082, 'ID_2369d2208.jpg')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "names = fn.attrgot('name')\n",
    "len(names),names[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>any</th>\n",
       "      <th>epidural</th>\n",
       "      <th>intraparenchymal</th>\n",
       "      <th>intraventricular</th>\n",
       "      <th>subarachnoid</th>\n",
       "      <th>subdural</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_000039fa0.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_00005679d.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_00008ce3c.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_0000950d7.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_0000aee4b.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 ID  any  epidural  intraparenchymal  intraventricular  \\\n",
       "0  ID_000039fa0.jpg    0         0                 0                 0   \n",
       "1  ID_00005679d.jpg    0         0                 0                 0   \n",
       "2  ID_00008ce3c.jpg    0         0                 0                 0   \n",
       "3  ID_0000950d7.jpg    0         0                 0                 0   \n",
       "4  ID_0000aee4b.jpg    0         0                 0                 0   \n",
       "\n",
       "   subarachnoid  subdural  \n",
       "0             0         0  \n",
       "1             0         0  \n",
       "2             0         0  \n",
       "3             0         0  \n",
       "4             0         0  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels['ID'] = labels['ID'].astype(str)+'.jpg'\n",
    "labels.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(194082, 7)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = labels[labels['ID'].isin(names)]\n",
    "\n",
    "labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BitsAllocated</th>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BitsStored</th>\n",
       "      <td>16</td>\n",
       "      <td>12</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Columns</th>\n",
       "      <td>512</td>\n",
       "      <td>512</td>\n",
       "      <td>512</td>\n",
       "      <td>512</td>\n",
       "      <td>512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HighBit</th>\n",
       "      <td>15</td>\n",
       "      <td>11</td>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ImageOrientationPatient</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ImageOrientationPatient1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ImageOrientationPatient2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ImageOrientationPatient3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ImageOrientationPatient4</th>\n",
       "      <td>0.927184</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.927184</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ImageOrientationPatient5</th>\n",
       "      <td>-0.374607</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.374607</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ImagePositionPatient</th>\n",
       "      <td>-125</td>\n",
       "      <td>-125</td>\n",
       "      <td>-126.437</td>\n",
       "      <td>-125</td>\n",
       "      <td>-125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ImagePositionPatient1</th>\n",
       "      <td>-119.998</td>\n",
       "      <td>4</td>\n",
       "      <td>-126.437</td>\n",
       "      <td>-109.698</td>\n",
       "      <td>-16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ImagePositionPatient2</th>\n",
       "      <td>44.7323</td>\n",
       "      <td>130.2</td>\n",
       "      <td>87.5</td>\n",
       "      <td>152.779</td>\n",
       "      <td>178.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Modality</th>\n",
       "      <td>CT</td>\n",
       "      <td>CT</td>\n",
       "      <td>CT</td>\n",
       "      <td>CT</td>\n",
       "      <td>CT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MultiImageOrientationPatient</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MultiImagePositionPatient</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MultiPixelSpacing</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MultiWindowCenter</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MultiWindowWidth</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PatientID</th>\n",
       "      <td>ID_d557ddd2</td>\n",
       "      <td>ID_16b2ad86</td>\n",
       "      <td>ID_ff137633</td>\n",
       "      <td>ID_42940b2c</td>\n",
       "      <td>ID_75df27b6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PhotometricInterpretation</th>\n",
       "      <td>MONOCHROME2</td>\n",
       "      <td>MONOCHROME2</td>\n",
       "      <td>MONOCHROME2</td>\n",
       "      <td>MONOCHROME2</td>\n",
       "      <td>MONOCHROME2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PixelRepresentation</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PixelSpacing</th>\n",
       "      <td>0.488281</td>\n",
       "      <td>0.488281</td>\n",
       "      <td>0.494863</td>\n",
       "      <td>0.488281</td>\n",
       "      <td>0.488281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PixelSpacing1</th>\n",
       "      <td>0.488281</td>\n",
       "      <td>0.488281</td>\n",
       "      <td>0.494863</td>\n",
       "      <td>0.488281</td>\n",
       "      <td>0.488281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RescaleIntercept</th>\n",
       "      <td>-1024</td>\n",
       "      <td>-1024</td>\n",
       "      <td>-1024</td>\n",
       "      <td>-1024</td>\n",
       "      <td>-1024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RescaleSlope</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rows</th>\n",
       "      <td>512</td>\n",
       "      <td>512</td>\n",
       "      <td>512</td>\n",
       "      <td>512</td>\n",
       "      <td>512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SOPInstanceUID</th>\n",
       "      <td>ID_352e89f1c</td>\n",
       "      <td>ID_3cf4fb50f</td>\n",
       "      <td>ID_2a8702d25</td>\n",
       "      <td>ID_66891ac22</td>\n",
       "      <td>ID_54f412d54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SamplesPerPixel</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SeriesInstanceUID</th>\n",
       "      <td>ID_be6165332c</td>\n",
       "      <td>ID_2c1454e208</td>\n",
       "      <td>ID_7098f7c836</td>\n",
       "      <td>ID_e14dd0090b</td>\n",
       "      <td>ID_f3e012b95a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>StudyID</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>StudyInstanceUID</th>\n",
       "      <td>ID_05074a0d95</td>\n",
       "      <td>ID_c3a404ea2e</td>\n",
       "      <td>ID_d17053848c</td>\n",
       "      <td>ID_17e33f43d0</td>\n",
       "      <td>ID_5a466c0489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WindowCenter</th>\n",
       "      <td>30</td>\n",
       "      <td>36</td>\n",
       "      <td>35</td>\n",
       "      <td>30</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WindowCenter1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>36</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WindowWidth</th>\n",
       "      <td>80</td>\n",
       "      <td>80</td>\n",
       "      <td>135</td>\n",
       "      <td>80</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WindowWidth1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>80</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fname</th>\n",
       "      <td>/home/jhoward/data/rsna/stage_1_train_images/ID_352e89f1c.dcm</td>\n",
       "      <td>/home/jhoward/data/rsna/stage_1_train_images/ID_3cf4fb50f.dcm</td>\n",
       "      <td>/home/jhoward/data/rsna/stage_1_train_images/ID_2a8702d25.dcm</td>\n",
       "      <td>/home/jhoward/data/rsna/stage_1_train_images/ID_66891ac22.dcm</td>\n",
       "      <td>/home/jhoward/data/rsna/stage_1_train_images/ID_54f412d54.dcm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>img_max</th>\n",
       "      <td>2787</td>\n",
       "      <td>2412</td>\n",
       "      <td>2806</td>\n",
       "      <td>2795</td>\n",
       "      <td>2491</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>img_mean</th>\n",
       "      <td>35.1129</td>\n",
       "      <td>234.55</td>\n",
       "      <td>482.249</td>\n",
       "      <td>106.238</td>\n",
       "      <td>536.799</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>img_min</th>\n",
       "      <td>-2000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-2000</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>img_pct_window</th>\n",
       "      <td>0.164139</td>\n",
       "      <td>0.0760155</td>\n",
       "      <td>0.241489</td>\n",
       "      <td>0.242466</td>\n",
       "      <td>0.345436</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>img_std</th>\n",
       "      <td>1166.72</td>\n",
       "      <td>392.132</td>\n",
       "      <td>571.236</td>\n",
       "      <td>1253.46</td>\n",
       "      <td>568.654</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                          0  \\\n",
       "BitsAllocated                                                                            16   \n",
       "BitsStored                                                                               16   \n",
       "Columns                                                                                 512   \n",
       "HighBit                                                                                  15   \n",
       "ImageOrientationPatient                                                                   1   \n",
       "ImageOrientationPatient1                                                                  0   \n",
       "ImageOrientationPatient2                                                                  0   \n",
       "ImageOrientationPatient3                                                                  0   \n",
       "ImageOrientationPatient4                                                           0.927184   \n",
       "ImageOrientationPatient5                                                          -0.374607   \n",
       "ImagePositionPatient                                                                   -125   \n",
       "ImagePositionPatient1                                                              -119.998   \n",
       "ImagePositionPatient2                                                               44.7323   \n",
       "Modality                                                                                 CT   \n",
       "MultiImageOrientationPatient                                                              1   \n",
       "MultiImagePositionPatient                                                                 1   \n",
       "MultiPixelSpacing                                                                         1   \n",
       "MultiWindowCenter                                                                       NaN   \n",
       "MultiWindowWidth                                                                        NaN   \n",
       "PatientID                                                                       ID_d557ddd2   \n",
       "PhotometricInterpretation                                                       MONOCHROME2   \n",
       "PixelRepresentation                                                                       1   \n",
       "PixelSpacing                                                                       0.488281   \n",
       "PixelSpacing1                                                                      0.488281   \n",
       "RescaleIntercept                                                                      -1024   \n",
       "RescaleSlope                                                                              1   \n",
       "Rows                                                                                    512   \n",
       "SOPInstanceUID                                                                 ID_352e89f1c   \n",
       "SamplesPerPixel                                                                           1   \n",
       "SeriesInstanceUID                                                             ID_be6165332c   \n",
       "StudyID                                                                                       \n",
       "StudyInstanceUID                                                              ID_05074a0d95   \n",
       "WindowCenter                                                                             30   \n",
       "WindowCenter1                                                                           NaN   \n",
       "WindowWidth                                                                              80   \n",
       "WindowWidth1                                                                            NaN   \n",
       "fname                         /home/jhoward/data/rsna/stage_1_train_images/ID_352e89f1c.dcm   \n",
       "img_max                                                                                2787   \n",
       "img_mean                                                                            35.1129   \n",
       "img_min                                                                               -2000   \n",
       "img_pct_window                                                                     0.164139   \n",
       "img_std                                                                             1166.72   \n",
       "\n",
       "                                                                                          1  \\\n",
       "BitsAllocated                                                                            16   \n",
       "BitsStored                                                                               12   \n",
       "Columns                                                                                 512   \n",
       "HighBit                                                                                  11   \n",
       "ImageOrientationPatient                                                                   1   \n",
       "ImageOrientationPatient1                                                                  0   \n",
       "ImageOrientationPatient2                                                                  0   \n",
       "ImageOrientationPatient3                                                                  0   \n",
       "ImageOrientationPatient4                                                                  1   \n",
       "ImageOrientationPatient5                                                                  0   \n",
       "ImagePositionPatient                                                                   -125   \n",
       "ImagePositionPatient1                                                                     4   \n",
       "ImagePositionPatient2                                                                 130.2   \n",
       "Modality                                                                                 CT   \n",
       "MultiImageOrientationPatient                                                              1   \n",
       "MultiImagePositionPatient                                                                 1   \n",
       "MultiPixelSpacing                                                                         1   \n",
       "MultiWindowCenter                                                                         1   \n",
       "MultiWindowWidth                                                                          1   \n",
       "PatientID                                                                       ID_16b2ad86   \n",
       "PhotometricInterpretation                                                       MONOCHROME2   \n",
       "PixelRepresentation                                                                       0   \n",
       "PixelSpacing                                                                       0.488281   \n",
       "PixelSpacing1                                                                      0.488281   \n",
       "RescaleIntercept                                                                      -1024   \n",
       "RescaleSlope                                                                              1   \n",
       "Rows                                                                                    512   \n",
       "SOPInstanceUID                                                                 ID_3cf4fb50f   \n",
       "SamplesPerPixel                                                                           1   \n",
       "SeriesInstanceUID                                                             ID_2c1454e208   \n",
       "StudyID                                                                                       \n",
       "StudyInstanceUID                                                              ID_c3a404ea2e   \n",
       "WindowCenter                                                                             36   \n",
       "WindowCenter1                                                                            36   \n",
       "WindowWidth                                                                              80   \n",
       "WindowWidth1                                                                             80   \n",
       "fname                         /home/jhoward/data/rsna/stage_1_train_images/ID_3cf4fb50f.dcm   \n",
       "img_max                                                                                2412   \n",
       "img_mean                                                                             234.55   \n",
       "img_min                                                                                   0   \n",
       "img_pct_window                                                                    0.0760155   \n",
       "img_std                                                                             392.132   \n",
       "\n",
       "                                                                                          2  \\\n",
       "BitsAllocated                                                                            16   \n",
       "BitsStored                                                                               16   \n",
       "Columns                                                                                 512   \n",
       "HighBit                                                                                  15   \n",
       "ImageOrientationPatient                                                                   1   \n",
       "ImageOrientationPatient1                                                                  0   \n",
       "ImageOrientationPatient2                                                                  0   \n",
       "ImageOrientationPatient3                                                                  0   \n",
       "ImageOrientationPatient4                                                                  1   \n",
       "ImageOrientationPatient5                                                                  0   \n",
       "ImagePositionPatient                                                               -126.437   \n",
       "ImagePositionPatient1                                                              -126.437   \n",
       "ImagePositionPatient2                                                                  87.5   \n",
       "Modality                                                                                 CT   \n",
       "MultiImageOrientationPatient                                                              1   \n",
       "MultiImagePositionPatient                                                                 1   \n",
       "MultiPixelSpacing                                                                         1   \n",
       "MultiWindowCenter                                                                       NaN   \n",
       "MultiWindowWidth                                                                        NaN   \n",
       "PatientID                                                                       ID_ff137633   \n",
       "PhotometricInterpretation                                                       MONOCHROME2   \n",
       "PixelRepresentation                                                                       1   \n",
       "PixelSpacing                                                                       0.494863   \n",
       "PixelSpacing1                                                                      0.494863   \n",
       "RescaleIntercept                                                                      -1024   \n",
       "RescaleSlope                                                                              1   \n",
       "Rows                                                                                    512   \n",
       "SOPInstanceUID                                                                 ID_2a8702d25   \n",
       "SamplesPerPixel                                                                           1   \n",
       "SeriesInstanceUID                                                             ID_7098f7c836   \n",
       "StudyID                                                                                       \n",
       "StudyInstanceUID                                                              ID_d17053848c   \n",
       "WindowCenter                                                                             35   \n",
       "WindowCenter1                                                                           NaN   \n",
       "WindowWidth                                                                             135   \n",
       "WindowWidth1                                                                            NaN   \n",
       "fname                         /home/jhoward/data/rsna/stage_1_train_images/ID_2a8702d25.dcm   \n",
       "img_max                                                                                2806   \n",
       "img_mean                                                                            482.249   \n",
       "img_min                                                                                   0   \n",
       "img_pct_window                                                                     0.241489   \n",
       "img_std                                                                             571.236   \n",
       "\n",
       "                                                                                          3  \\\n",
       "BitsAllocated                                                                            16   \n",
       "BitsStored                                                                               16   \n",
       "Columns                                                                                 512   \n",
       "HighBit                                                                                  15   \n",
       "ImageOrientationPatient                                                                   1   \n",
       "ImageOrientationPatient1                                                                  0   \n",
       "ImageOrientationPatient2                                                                  0   \n",
       "ImageOrientationPatient3                                                                  0   \n",
       "ImageOrientationPatient4                                                           0.927184   \n",
       "ImageOrientationPatient5                                                          -0.374607   \n",
       "ImagePositionPatient                                                                   -125   \n",
       "ImagePositionPatient1                                                              -109.698   \n",
       "ImagePositionPatient2                                                               152.779   \n",
       "Modality                                                                                 CT   \n",
       "MultiImageOrientationPatient                                                              1   \n",
       "MultiImagePositionPatient                                                                 1   \n",
       "MultiPixelSpacing                                                                         1   \n",
       "MultiWindowCenter                                                                       NaN   \n",
       "MultiWindowWidth                                                                        NaN   \n",
       "PatientID                                                                       ID_42940b2c   \n",
       "PhotometricInterpretation                                                       MONOCHROME2   \n",
       "PixelRepresentation                                                                       1   \n",
       "PixelSpacing                                                                       0.488281   \n",
       "PixelSpacing1                                                                      0.488281   \n",
       "RescaleIntercept                                                                      -1024   \n",
       "RescaleSlope                                                                              1   \n",
       "Rows                                                                                    512   \n",
       "SOPInstanceUID                                                                 ID_66891ac22   \n",
       "SamplesPerPixel                                                                           1   \n",
       "SeriesInstanceUID                                                             ID_e14dd0090b   \n",
       "StudyID                                                                                       \n",
       "StudyInstanceUID                                                              ID_17e33f43d0   \n",
       "WindowCenter                                                                             30   \n",
       "WindowCenter1                                                                           NaN   \n",
       "WindowWidth                                                                              80   \n",
       "WindowWidth1                                                                            NaN   \n",
       "fname                         /home/jhoward/data/rsna/stage_1_train_images/ID_66891ac22.dcm   \n",
       "img_max                                                                                2795   \n",
       "img_mean                                                                            106.238   \n",
       "img_min                                                                               -2000   \n",
       "img_pct_window                                                                     0.242466   \n",
       "img_std                                                                             1253.46   \n",
       "\n",
       "                                                                                          4  \n",
       "BitsAllocated                                                                            16  \n",
       "BitsStored                                                                               12  \n",
       "Columns                                                                                 512  \n",
       "HighBit                                                                                  11  \n",
       "ImageOrientationPatient                                                                   1  \n",
       "ImageOrientationPatient1                                                                  0  \n",
       "ImageOrientationPatient2                                                                  0  \n",
       "ImageOrientationPatient3                                                                  0  \n",
       "ImageOrientationPatient4                                                                  1  \n",
       "ImageOrientationPatient5                                                                  0  \n",
       "ImagePositionPatient                                                                   -125  \n",
       "ImagePositionPatient1                                                                   -16  \n",
       "ImagePositionPatient2                                                                 178.9  \n",
       "Modality                                                                                 CT  \n",
       "MultiImageOrientationPatient                                                              1  \n",
       "MultiImagePositionPatient                                                                 1  \n",
       "MultiPixelSpacing                                                                         1  \n",
       "MultiWindowCenter                                                                         1  \n",
       "MultiWindowWidth                                                                          1  \n",
       "PatientID                                                                       ID_75df27b6  \n",
       "PhotometricInterpretation                                                       MONOCHROME2  \n",
       "PixelRepresentation                                                                       0  \n",
       "PixelSpacing                                                                       0.488281  \n",
       "PixelSpacing1                                                                      0.488281  \n",
       "RescaleIntercept                                                                      -1024  \n",
       "RescaleSlope                                                                              1  \n",
       "Rows                                                                                    512  \n",
       "SOPInstanceUID                                                                 ID_54f412d54  \n",
       "SamplesPerPixel                                                                           1  \n",
       "SeriesInstanceUID                                                             ID_f3e012b95a  \n",
       "StudyID                                                                                      \n",
       "StudyInstanceUID                                                              ID_5a466c0489  \n",
       "WindowCenter                                                                             36  \n",
       "WindowCenter1                                                                            36  \n",
       "WindowWidth                                                                              80  \n",
       "WindowWidth1                                                                             80  \n",
       "fname                         /home/jhoward/data/rsna/stage_1_train_images/ID_54f412d54.dcm  \n",
       "img_max                                                                                2491  \n",
       "img_mean                                                                            536.799  \n",
       "img_min                                                                                   9  \n",
       "img_pct_window                                                                     0.345436  \n",
       "img_std                                                                             568.654  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dftrain = pd.read_feather(csv_path/'df_trn.fth')\n",
    "dftrain.head().T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The way names of files are stored in the labels and train data is different. One of them is saved with the extension \".jpg\" hence, we add the extension to the other df in order to make the column identical to uniquely identify the data samples.\n",
    "Also, we need to keep data samples which are common to both the dataframes so that we have labels for all the data samples. Hence, we merge the dfs on the ID column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(674258, 42)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(194082, 42)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(dftrain.shape)\n",
    "dftrain['SOPInstanceUID'] = dftrain['SOPInstanceUID'].astype(str)+'.jpg'\n",
    "dftrain = dftrain[dftrain['SOPInstanceUID'].isin(names)]\n",
    "dftrain.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BitsAllocated                        0\n",
       "BitsStored                           0\n",
       "Columns                              0\n",
       "HighBit                              0\n",
       "ImageOrientationPatient              0\n",
       "ImageOrientationPatient1             0\n",
       "ImageOrientationPatient2             0\n",
       "ImageOrientationPatient3             0\n",
       "ImageOrientationPatient4             0\n",
       "ImageOrientationPatient5             0\n",
       "ImagePositionPatient                 0\n",
       "ImagePositionPatient1                0\n",
       "ImagePositionPatient2                0\n",
       "Modality                             0\n",
       "MultiImageOrientationPatient         0\n",
       "MultiImagePositionPatient            0\n",
       "MultiPixelSpacing                    0\n",
       "MultiWindowCenter               103205\n",
       "MultiWindowWidth                103205\n",
       "PatientID                            0\n",
       "PhotometricInterpretation            0\n",
       "PixelRepresentation                  0\n",
       "PixelSpacing                         0\n",
       "PixelSpacing1                        0\n",
       "RescaleIntercept                     0\n",
       "RescaleSlope                         0\n",
       "Rows                                 0\n",
       "SOPInstanceUID                       0\n",
       "SamplesPerPixel                      0\n",
       "SeriesInstanceUID                    0\n",
       "StudyID                              0\n",
       "StudyInstanceUID                     0\n",
       "WindowCenter                         0\n",
       "WindowCenter1                   103205\n",
       "WindowWidth                          0\n",
       "WindowWidth1                    103205\n",
       "fname                                0\n",
       "img_max                              0\n",
       "img_mean                             0\n",
       "img_min                              0\n",
       "img_pct_window                       0\n",
       "img_std                              0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dftrain.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CT    194082\n",
       "Name: Modality, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dftrain['Modality'].value_counts()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "dftrain = dftrain.drop(['MultiWindowCenter','MultiWindowWidth','WindowCenter1','WindowWidth1','Modality','StudyID','PatientID','SOPInstanceUID','SeriesInstanceUID','PhotometricInterpretation','StudyInstanceUID','fname'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BitsAllocated                   0\n",
       "BitsStored                      0\n",
       "Columns                         0\n",
       "HighBit                         0\n",
       "ImageOrientationPatient         0\n",
       "ImageOrientationPatient1        0\n",
       "ImageOrientationPatient2        0\n",
       "ImageOrientationPatient3        0\n",
       "ImageOrientationPatient4        0\n",
       "ImageOrientationPatient5        0\n",
       "ImagePositionPatient            0\n",
       "ImagePositionPatient1           0\n",
       "ImagePositionPatient2           0\n",
       "MultiImageOrientationPatient    0\n",
       "MultiImagePositionPatient       0\n",
       "MultiPixelSpacing               0\n",
       "PixelRepresentation             0\n",
       "PixelSpacing                    0\n",
       "PixelSpacing1                   0\n",
       "RescaleIntercept                0\n",
       "RescaleSlope                    0\n",
       "Rows                            0\n",
       "SamplesPerPixel                 0\n",
       "WindowCenter                    0\n",
       "WindowWidth                     0\n",
       "img_max                         0\n",
       "img_mean                        0\n",
       "img_min                         0\n",
       "img_pct_window                  0\n",
       "img_std                         0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dftrain.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BitsAllocated                     int64\n",
       "BitsStored                        int64\n",
       "Columns                           int64\n",
       "HighBit                           int64\n",
       "ImageOrientationPatient         float64\n",
       "ImageOrientationPatient1        float64\n",
       "ImageOrientationPatient2        float64\n",
       "ImageOrientationPatient3        float64\n",
       "ImageOrientationPatient4        float64\n",
       "ImageOrientationPatient5        float64\n",
       "ImagePositionPatient            float64\n",
       "ImagePositionPatient1           float64\n",
       "ImagePositionPatient2           float64\n",
       "MultiImageOrientationPatient      int64\n",
       "MultiImagePositionPatient         int64\n",
       "MultiPixelSpacing                 int64\n",
       "PixelRepresentation               int64\n",
       "PixelSpacing                    float64\n",
       "PixelSpacing1                   float64\n",
       "RescaleIntercept                float64\n",
       "RescaleSlope                    float64\n",
       "Rows                              int64\n",
       "SamplesPerPixel                   int64\n",
       "WindowCenter                    float64\n",
       "WindowWidth                     float64\n",
       "img_max                           int64\n",
       "img_mean                        float64\n",
       "img_min                           int64\n",
       "img_pct_window                  float64\n",
       "img_std                         float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dftrain.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(194082, 30)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dftrain.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "155265"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "split = int(dftrain.shape[0]*0.8)\n",
    "split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_valid=dftrain[:split],dftrain[split:]\n",
    "Y_train,Y_valid=labels[:split],labels[split:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_train = Y_train['any']\n",
    "Y_valid = Y_valid['any']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def randomfor(n_estimators = 200, max_features='auto', min_samples_leaf = 1):\n",
    "\n",
    "    rf = RandomForestClassifier(n_estimators = n_estimators,max_features =max_features, min_samples_leaf = min_samples_leaf)\n",
    "    rf.fit(X_train, Y_train)\n",
    "    score = rf.score(X_valid, Y_valid)\n",
    "    print(score * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50.23056908055749\n"
     ]
    }
   ],
   "source": [
    "randomfor()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The medadata by itself is not sufficient to achieve good results even after using a robust model like random forest. Hence, we won't focus on the medadata results and instead focus on the CNN to classify images directly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}